From bc187ad7517d71474df23317b323a2d9a11f958c Mon Sep 17 00:00:00 2001 From: Cookie Ramen Date: Thu, 9 Feb 2023 05:24:35 +0900 Subject: [PATCH 001/562] =?UTF-8?q?=E7=8B=AC=E8=87=AA=E3=83=9C=E3=82=A4?= =?UTF-8?q?=E3=82=B9=E8=BF=BD=E5=8A=A0=EF=BC=88=20=E9=9F=B3=E6=BA=90?= =?UTF-8?q?=E3=81=AF=C2=A9=202023=20Misskey.io=20=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/frontend/src/scripts/sound.ts | 61 ++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/packages/frontend/src/scripts/sound.ts b/packages/frontend/src/scripts/sound.ts index 9d1f60323561..3468a0e4b5b0 100644 --- a/packages/frontend/src/scripts/sound.ts +++ b/packages/frontend/src/scripts/sound.ts @@ -32,6 +32,67 @@ export const soundsTypes = [ 'noizenecio/kick_gaba5', 'noizenecio/kick_gaba6', 'noizenecio/kick_gaba7', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeA_Antenna', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeA_Channel', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeA_Chat', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeA_Note1', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeA_Note2', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeA_Notification', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeA_Send1', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeA_Send2', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeB_Antenna', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeB_Channel', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeB_Chat', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeB_Note1', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeB_Note2', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeB_Notification', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeB_Send', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeC_Antenna', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeC_Channel', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeC_Chat', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeC_Note', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeC_Notification', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeC_Send', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeD_Antenna', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeD_Channel', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeD_Chat', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeD_Note', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeD_Notification', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeD_Send', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeE_Antenna', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeE_Channel', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeE_Chat', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeE_Note', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeE_Notification', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeE_Send', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeF_Antenna', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeF_Channel', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeF_Chat', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeF_Note', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeF_Notification', + 'Copyright_Misskey.io/HazumiAi/VoiceTypeF_Send', + 'Copyright_Misskey.io/ThinaticSystem/mata_hazukashiikoto_itteru', + 'Copyright_Misskey.io/ThinaticSystem/akemashite_omedetou_gozaimasu', + 'Copyright_Misskey.io/ThinaticSystem/bibi', + 'Copyright_Misskey.io/ThinaticSystem/doya1', + 'Copyright_Misskey.io/ThinaticSystem/doya2', + 'Copyright_Misskey.io/ThinaticSystem/doya3', + 'Copyright_Misskey.io/ThinaticSystem/gege_ltu_win3.1', + 'Copyright_Misskey.io/ThinaticSystem/hekuchi', + 'Copyright_Misskey.io/ThinaticSystem/moresou', + 'Copyright_Misskey.io/ThinaticSystem/muzumuzu_suru', + 'Copyright_Misskey.io/ThinaticSystem/nsho', + 'Copyright_Misskey.io/ThinaticSystem/pepo', + 'Copyright_Misskey.io/ThinaticSystem/picco_n', + 'Copyright_Misskey.io/ThinaticSystem/tenor_sax', + 'Copyright_Misskey.io/ThinaticSystem/topo', + 'Copyright_Misskey.io/ThinaticSystem/tsukapekepinpa', + 'Copyright_Misskey.io/ThinaticSystem/vun_clean', + 'Copyright_Misskey.io/ThinaticSystem/vun_dirty', + 'Copyright_Misskey.io/ThinaticSystem/wa', + 'Copyright_Misskey.io/ThinaticSystem/yonderuzo1', + 'Copyright_Misskey.io/ThinaticSystem/yonderuzo2', + 'Copyright_Misskey.io/ThinaticSystem/yonderuzo3', ] as const; export function getAudio(file: string, useCache = true): HTMLAudioElement { From b888c66ca29871be337d4302957ec3f8db0836a1 Mon Sep 17 00:00:00 2001 From: Ry0taK <49341894+Ry0taK@users.noreply.github.com> Date: Sat, 11 Feb 2023 21:20:45 +0900 Subject: [PATCH 002/562] =?UTF-8?q?Content-Security-Policy-Report-Only?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .config/docker_example.yml | 3 ++ .config/example.yml | 3 ++ gulpfile.js | 10 ++++- packages/backend/src/config.ts | 2 + .../src/server/web/ClientServerService.ts | 8 ++++ packages/backend/src/server/web/boot.js | 5 ++- packages/backend/src/server/web/flush.js | 44 +++++++++++++++++++ .../backend/src/server/web/views/base.pug | 7 +-- .../backend/src/server/web/views/bios.pug | 3 +- packages/backend/src/server/web/views/cli.pug | 3 +- .../backend/src/server/web/views/flush.pug | 44 +------------------ 11 files changed, 76 insertions(+), 56 deletions(-) create mode 100644 packages/backend/src/server/web/flush.js diff --git a/.config/docker_example.yml b/.config/docker_example.yml index f8124bc9dfc6..e51d7e8f5f30 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -144,3 +144,6 @@ signToActivityPubGet: true # Upload or download file size limits (bytes) #maxFileSize: 262144000 + +# Value of Content-Security-Policy header +#contentSecurityPolicy: "script-src 'self' 'unsafe-eval' https://challenges.cloudflare.com https://hcaptcha.com https://*.hcaptcha.com https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/ https://www.recaptcha.net/recaptcha/; base-uri 'self'; object-src 'self';" diff --git a/.config/example.yml b/.config/example.yml index a19b5d04e826..d9f62382192e 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -145,3 +145,6 @@ signToActivityPubGet: true # Upload or download file size limits (bytes) #maxFileSize: 262144000 + +# Value of Content-Security-Policy header +#contentSecurityPolicy: "script-src 'self' 'unsafe-eval' https://challenges.cloudflare.com https://hcaptcha.com https://*.hcaptcha.com https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/ https://www.recaptcha.net/recaptcha/; base-uri 'self'; object-src 'self';" diff --git a/gulpfile.js b/gulpfile.js index a04ab4c1ad4e..d08b04eae79a 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -36,12 +36,18 @@ gulp.task('copy:frontend:locales', cb => { }); gulp.task('build:backend:script', () => { - return gulp.src(['./packages/backend/src/server/web/boot.js', './packages/backend/src/server/web/bios.js', './packages/backend/src/server/web/cli.js']) + const clientManifestExists = fs.existsSync('./built/_vite_/manifest.json'); + const clientEntry = clientManifestExists ? + JSON.parse(fs.readFileSync('./built/_vite_/manifest.json', 'utf-8'))['src/init.ts'].file + : 'src/init.ts' + + return gulp.src(['./packages/backend/src/server/web/boot.js', './packages/backend/src/server/web/bios.js', './packages/backend/src/server/web/cli.js', './packages/backend/src/server/web/flush.js']) .pipe(replace('LANGS', JSON.stringify(Object.keys(locales)))) + .pipe(replace('CLIENT_ENTRY', JSON.stringify(clientEntry))) .pipe(terser({ toplevel: true })) - .pipe(gulp.dest('./packages/backend/built/server/web/')); + .pipe(gulp.dest('./built/_frontend_dist_/')); }); gulp.task('build:backend:style', () => { diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index aa98ef1d22d2..4309f67cb3b9 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -48,6 +48,8 @@ export type Source = { allowedPrivateNetworks?: string[]; + contentSecurityPolicy?: string; + maxFileSize?: number; accesslog?: string; diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index c69ee33ea369..7f1a43792f07 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -172,6 +172,14 @@ export class ClientServerService { fastify.addHook('onRequest', (request, reply, done) => { // クリックジャッキング防止のためiFrameの中に入れられないようにする reply.header('X-Frame-Options', 'DENY'); + + // XSSが存在した場合に影響を軽減する + // (script-srcにunsafe-inline等を追加すると意味が無くなるので注意) + const csp = this.config.contentSecurityPolicy + ?? 'script-src \'self\' \'unsafe-eval\' ' + + 'https://challenges.cloudflare.com https://hcaptcha.com https://*.hcaptcha.com https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/ https://www.recaptcha.net/recaptcha/; ' + + 'base-uri \'self\'; object-src \'self\';'; + reply.header('Content-Security-Policy', csp); done(); }); diff --git a/packages/backend/src/server/web/boot.js b/packages/backend/src/server/web/boot.js index c6cb25e43ae3..4b9565f4c7d9 100644 --- a/packages/backend/src/server/web/boot.js +++ b/packages/backend/src/server/web/boot.js @@ -154,7 +154,7 @@

An error has occurred!

-

Don't worry, it's (probably) not your fault.

@@ -181,6 +181,9 @@
`; + document.getElementById("reload").addEventListener('click', () => { + location.reload(); + }); errorsElement = document.getElementById('errors'); } const detailsElement = document.createElement('details'); diff --git a/packages/backend/src/server/web/flush.js b/packages/backend/src/server/web/flush.js new file mode 100644 index 000000000000..bbc9356d00f2 --- /dev/null +++ b/packages/backend/src/server/web/flush.js @@ -0,0 +1,44 @@ +'use strict'; + +const msg = document.getElementById('msg'); +const successText = `\nSuccess Flush! Back to Misskey\n成功しました。Misskeyを開き直してください。`; + +message('Start flushing.'); + +(async function () { + try { + localStorage.clear(); + message('localStorage cleared.'); + + const idbPromises = ['MisskeyClient', 'keyval-store'].map((name, i, arr) => new Promise((res, rej) => { + const delidb = indexedDB.deleteDatabase(name); + delidb.onsuccess = () => res(message(`indexedDB "${name}" cleared. (${i + 1}/${arr.length})`)); + delidb.onerror = e => rej(e) + })); + + await Promise.all(idbPromises); + + if (navigator.serviceWorker.controller) { + navigator.serviceWorker.controller.postMessage('clear'); + await navigator.serviceWorker.getRegistrations() + .then(registrations => { + return Promise.all(registrations.map(registration => registration.unregister())); + }) + .catch(e => { throw new Error(e) }); + } + + message(successText); + } catch (e) { + message(`\n${e}\n\nFlush Failed. Please retry.\n失敗しました。もう一度試してみてください。`); + message(`\nIf you retry more than 3 times, clear the browser cache or contact to instance admin.\n3回以上試しても失敗する場合、ブラウザのキャッシュを消去し、それでもだめならインスタンス管理者に連絡してみてください。\n`) + + console.error(e); + setTimeout(() => { + location = '/'; + }, 10000) + } +})(); + +function message(text) { + msg.insertAdjacentHTML('beforeend', `

[${(new Date()).toString()}] ${text.replace(/\n/g, '
')}

`) +} diff --git a/packages/backend/src/server/web/views/base.pug b/packages/backend/src/server/web/views/base.pug index 8d6897c46d35..5b3217aa95be 100644 --- a/packages/backend/src/server/web/views/base.pug +++ b/packages/backend/src/server/web/views/base.pug @@ -63,12 +63,7 @@ html style include ../style.css - script. - var VERSION = "#{version}"; - var CLIENT_ENTRY = "#{clientEntry.file}"; - - script - include ../boot.js + script(src=`/assets/boot.js?v=${version}`) body noscript: p diff --git a/packages/backend/src/server/web/views/bios.pug b/packages/backend/src/server/web/views/bios.pug index 39a151a29b7b..54706b9c90bb 100644 --- a/packages/backend/src/server/web/views/bios.pug +++ b/packages/backend/src/server/web/views/bios.pug @@ -8,8 +8,7 @@ html title Misskey Repair Tool style include ../bios.css - script - include ../bios.js + script(src=`/assets/bios.js?v=${version}`) body header diff --git a/packages/backend/src/server/web/views/cli.pug b/packages/backend/src/server/web/views/cli.pug index d2cf7c4335df..51d45cefd388 100644 --- a/packages/backend/src/server/web/views/cli.pug +++ b/packages/backend/src/server/web/views/cli.pug @@ -8,8 +8,7 @@ html title Misskey Cli style include ../cli.css - script - include ../cli.js + script(src=`/assets/cli.js?v=${version}`) body header diff --git a/packages/backend/src/server/web/views/flush.pug b/packages/backend/src/server/web/views/flush.pug index a73a45212f12..2ce426ae1d7d 100644 --- a/packages/backend/src/server/web/views/flush.pug +++ b/packages/backend/src/server/web/views/flush.pug @@ -2,46 +2,4 @@ doctype html html #msg - script. - const msg = document.getElementById('msg'); - const successText = `\nSuccess Flush! Back to Misskey\n成功しました。Misskeyを開き直してください。`; - - message('Start flushing.'); - - (async function() { - try { - localStorage.clear(); - message('localStorage cleared.'); - - const idbPromises = ['MisskeyClient', 'keyval-store'].map((name, i, arr) => new Promise((res, rej) => { - const delidb = indexedDB.deleteDatabase(name); - delidb.onsuccess = () => res(message(`indexedDB "${name}" cleared. (${i + 1}/${arr.length})`)); - delidb.onerror = e => rej(e) - })); - - await Promise.all(idbPromises); - - if (navigator.serviceWorker.controller) { - navigator.serviceWorker.controller.postMessage('clear'); - await navigator.serviceWorker.getRegistrations() - .then(registrations => { - return Promise.all(registrations.map(registration => registration.unregister())); - }) - .catch(e => { throw new Error(e) }); - } - - message(successText); - } catch (e) { - message(`\n${e}\n\nFlush Failed. Please retry.\n失敗しました。もう一度試してみてください。`); - message(`\nIf you retry more than 3 times, clear the browser cache or contact to instance admin.\n3回以上試しても失敗する場合、ブラウザのキャッシュを消去し、それでもだめならインスタンス管理者に連絡してみてください。\n`) - - console.error(e); - setTimeout(() => { - location = '/'; - }, 10000) - } - })(); - - function message(text) { - msg.insertAdjacentHTML('beforeend', `

[${(new Date()).toString()}] ${text.replace(/\n/g,'
')}

`) - } + script(src=`/assets/flush.js?v=${version}`) From ffd4d44ecb2387e215411f278a2c6bc07499995d Mon Sep 17 00:00:00 2001 From: Ry0taK <49341894+Ry0taK@users.noreply.github.com> Date: Sat, 11 Feb 2023 12:37:39 +0000 Subject: [PATCH 003/562] =?UTF-8?q?Content-Security-Policy-Report-Only?= =?UTF-8?q?=E3=82=92=E4=BD=BF=E7=94=A8=E3=81=99=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/server/web/ClientServerService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index 7f1a43792f07..a83137f74cd2 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -178,8 +178,8 @@ export class ClientServerService { const csp = this.config.contentSecurityPolicy ?? 'script-src \'self\' \'unsafe-eval\' ' + 'https://challenges.cloudflare.com https://hcaptcha.com https://*.hcaptcha.com https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/ https://www.recaptcha.net/recaptcha/; ' + - 'base-uri \'self\'; object-src \'self\';'; - reply.header('Content-Security-Policy', csp); + 'base-uri \'self\'; object-src \'self\'; report-uri /csp-error'; + reply.header('Content-Security-Policy-Report-Only', csp); done(); }); From f9d254b3e18ed97ddd0b6e8c23e94ca159a01829 Mon Sep 17 00:00:00 2001 From: Ry0taK <49341894+Ry0taK@users.noreply.github.com> Date: Sat, 11 Feb 2023 12:50:03 +0000 Subject: [PATCH 004/562] =?UTF-8?q?unsafe-eval=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/server/web/ClientServerService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index a83137f74cd2..5bc5e1889f07 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -176,7 +176,7 @@ export class ClientServerService { // XSSが存在した場合に影響を軽減する // (script-srcにunsafe-inline等を追加すると意味が無くなるので注意) const csp = this.config.contentSecurityPolicy - ?? 'script-src \'self\' \'unsafe-eval\' ' + + ?? 'script-src \'self\' ' + 'https://challenges.cloudflare.com https://hcaptcha.com https://*.hcaptcha.com https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/ https://www.recaptcha.net/recaptcha/; ' + 'base-uri \'self\'; object-src \'self\'; report-uri /csp-error'; reply.header('Content-Security-Policy-Report-Only', csp); From 716619541518e8f251f47b7bdfb655e6874e88c1 Mon Sep 17 00:00:00 2001 From: Ry0taK <49341894+Ry0taK@users.noreply.github.com> Date: Tue, 7 Mar 2023 08:31:16 +0000 Subject: [PATCH 005/562] Add x-worker-host header --- packages/backend/src/server/ServerService.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index e61383468c07..1c5ce7f2dbe4 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -1,4 +1,5 @@ import cluster from 'node:cluster'; +import os from 'node:os'; import * as fs from 'node:fs'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import Fastify, { FastifyInstance } from 'fastify'; @@ -71,6 +72,12 @@ export class ServerService implements OnApplicationShutdown { }); } + const hostname = os.hostname(); + fastify.addHook('onRequest', (request, reply, done) => { + reply.header('x-worker-host', hostname); + done(); + }); + fastify.register(this.apiServerService.createServer, { prefix: '/api' }); fastify.register(this.fileServerService.createServer); fastify.register(this.activityPubServerService.createServer); From 8249819a6e2cc9b863218d58b6503fa85bb9ab66 Mon Sep 17 00:00:00 2001 From: RyotaK <49341894+Ry0taK@users.noreply.github.com> Date: Sat, 11 Mar 2023 19:38:15 +0900 Subject: [PATCH 006/562] =?UTF-8?q?createdAt=E3=81=A7=E3=81=AF=E3=81=AA?= =?UTF-8?q?=E3=81=8Fid=E3=81=A7=E7=B5=9E=E3=82=8A=E8=BE=BC=E3=82=80?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=20(#71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/misc/id/aid.ts | 4 ++-- packages/backend/src/server/api/endpoints/notes/timeline.ts | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/misc/id/aid.ts b/packages/backend/src/misc/id/aid.ts index 19c8546f95c6..158e5699c18c 100644 --- a/packages/backend/src/misc/id/aid.ts +++ b/packages/backend/src/misc/id/aid.ts @@ -6,7 +6,7 @@ import * as crypto from 'node:crypto'; const TIME2000 = 946684800000; let counter = crypto.randomBytes(2).readUInt16LE(0); -function getTime(time: number): string { +export function getTimeId(time: number): string { time = time - TIME2000; if (time < 0) time = 0; @@ -21,5 +21,5 @@ export function genAid(date: Date): string { const t = date.getTime(); if (isNaN(t)) throw 'Failed to create AID: Invalid Date'; counter++; - return getTime(t) + getNoise(); + return getTimeId(t) + getNoise(); } diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index e6de087c4aac..c7f42c91a625 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -6,6 +6,7 @@ import { QueryService } from '@/core/QueryService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; +import { getTimeId } from '@/misc/id/aid.js'; export const meta = { tags: ['notes'], @@ -63,10 +64,11 @@ export default class extends Endpoint { .where('following.followerId = :followerId', { followerId: me.id }) .getMany(); + const minId = getTimeId(new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)).getTime()) + '00'; // 30日前まで //#region Construct query const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) - .andWhere('note.createdAt > :minDate', { minDate: new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)) }) // 30日前まで + .andWhere('note.id > :minId', { minId }) .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('user.avatar', 'avatar') .leftJoinAndSelect('user.banner', 'banner') From 8a02b806c97de16319453518fa62ed81b1e9a152 Mon Sep 17 00:00:00 2001 From: KOBA789 Date: Sat, 18 Mar 2023 11:10:06 +0900 Subject: [PATCH 007/562] Use limit() instead of take() because take() produces needless DISTINCT (#72) --- packages/backend/src/server/api/endpoints/notes/timeline.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index c7f42c91a625..07179553baa5 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -131,7 +131,7 @@ export default class extends Endpoint { } //#endregion - const timeline = await query.take(ps.limit).getMany(); + const timeline = await query.limit(ps.limit).getMany(); process.nextTick(() => { this.activeUsersChart.read(me); From c0d0c9ada2c7316b724d8b8c3a15a5544a1ada15 Mon Sep 17 00:00:00 2001 From: Ry0taK <49341894+Ry0taK@users.noreply.github.com> Date: Sat, 18 Mar 2023 02:15:36 +0000 Subject: [PATCH 008/562] WIP: Send nonce in CSP --- packages/backend/src/@types/fastify.d.ts | 9 +++++++++ .../backend/src/server/web/ClientServerService.ts | 11 ++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 packages/backend/src/@types/fastify.d.ts diff --git a/packages/backend/src/@types/fastify.d.ts b/packages/backend/src/@types/fastify.d.ts new file mode 100644 index 000000000000..83712cc6b6b1 --- /dev/null +++ b/packages/backend/src/@types/fastify.d.ts @@ -0,0 +1,9 @@ +import FastifyReply from "fastify"; + +declare module 'fastify' { + interface FastifyReply { + cspNonce: { + script: string + } + } +} diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index 94679c14edb1..e347017960ed 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -1,5 +1,6 @@ import { dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { randomBytes } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; import { createBullBoard } from '@bull-board/api'; import { BullAdapter } from '@bull-board/api/bullAdapter.js'; @@ -174,12 +175,16 @@ export class ClientServerService { reply.header('X-Frame-Options', 'DENY'); // XSSが存在した場合に影響を軽減する - // (script-srcにunsafe-inline等を追加すると意味が無くなるので注意) + // (インラインスクリプトはreply.cspNonce内の値をnonce属性に設定することで使える) + const scriptNonce = randomBytes(16).toString('hex'); + reply.cspNonce = { + script: scriptNonce, + }; const csp = this.config.contentSecurityPolicy ?? 'script-src \'self\' ' + - 'https://challenges.cloudflare.com https://hcaptcha.com https://*.hcaptcha.com https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/ https://www.recaptcha.net/recaptcha/; ' + + 'https://challenges.cloudflare.com https://hcaptcha.com https://*.hcaptcha.com https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/ https://www.recaptcha.net/recaptcha/ {scriptNonce}; ' + 'base-uri \'self\'; object-src \'self\'; report-uri /csp-error'; - reply.header('Content-Security-Policy-Report-Only', csp); + reply.header('Content-Security-Policy-Report-Only', csp.replace('{scriptNonce}', `'nonce-${scriptNonce}'`)); done(); }); From c3659a4ca298d852ad638861e7337889d4eeecdf Mon Sep 17 00:00:00 2001 From: Ry0taK <49341894+Ry0taK@users.noreply.github.com> Date: Sat, 18 Mar 2023 02:42:05 +0000 Subject: [PATCH 009/562] Add worker-src --- packages/backend/src/server/web/ClientServerService.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index e347017960ed..761cf4ba703b 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -183,6 +183,7 @@ export class ClientServerService { const csp = this.config.contentSecurityPolicy ?? 'script-src \'self\' ' + 'https://challenges.cloudflare.com https://hcaptcha.com https://*.hcaptcha.com https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/ https://www.recaptcha.net/recaptcha/ {scriptNonce}; ' + + 'worker-src blob: \'self\'; ' + 'base-uri \'self\'; object-src \'self\'; report-uri /csp-error'; reply.header('Content-Security-Policy-Report-Only', csp.replace('{scriptNonce}', `'nonce-${scriptNonce}'`)); done(); From 17bada4ada9cf498aa9c163c4d6182e6c258e4dd Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 8 Apr 2023 14:36:39 +0000 Subject: [PATCH 010/562] =?UTF-8?q?fix(server):=20=E3=83=AA=E3=82=A2?= =?UTF-8?q?=E3=82=AF=E3=82=B7=E3=83=A7=E3=83=B3=E3=81=A7=E3=81=8D=E3=81=AA?= =?UTF-8?q?=E3=81=84=E5=95=8F=E9=A1=8C=E3=82=92=E3=81=A8=E3=82=8A=E3=81=82?= =?UTF-8?q?=E3=81=88=E3=81=9A=E4=BF=AE=E6=AD=A3=20Fix=20#10502?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/CustomEmojiService.ts | 5 ++++- packages/backend/src/misc/cache.ts | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index 604a94707f8d..1c877477fcbd 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -44,7 +44,10 @@ export class CustomEmojiService { memoryCacheLifetime: 1000 * 60 * 3, // 3m fetcher: () => this.emojisRepository.find({ where: { host: IsNull() } }).then(emojis => new Map(emojis.map(emoji => [emoji.name, emoji]))), toRedisConverter: (value) => JSON.stringify(value.values()), - fromRedisConverter: (value) => new Map(JSON.parse(value).map((x: Emoji) => [x.name, x])), // TODO: Date型の変換 + fromRedisConverter: (value) => { + if (!Array.isArray(value)) return undefined; + return new Map(JSON.parse(value).map((x: Emoji) => [x.name, x])); + }, // TODO: Date型の変換 }); } diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index d35414acf7de..a4abd4f8781e 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -8,7 +8,7 @@ export class RedisKVCache { private memoryCache: MemoryKVCache; private fetcher: (key: string) => Promise; private toRedisConverter: (value: T) => string; - private fromRedisConverter: (value: string) => T; + private fromRedisConverter: (value: string) => T | undefined; // undefined means no cache constructor(redisClient: RedisKVCache['redisClient'], name: RedisKVCache['name'], opts: { lifetime: RedisKVCache['lifetime']; @@ -92,7 +92,7 @@ export class RedisSingleCache { private memoryCache: MemorySingleCache; private fetcher: () => Promise; private toRedisConverter: (value: T) => string; - private fromRedisConverter: (value: string) => T; + private fromRedisConverter: (value: string) => T | undefined; // undefined means no cache constructor(redisClient: RedisSingleCache['redisClient'], name: RedisSingleCache['name'], opts: { lifetime: RedisSingleCache['lifetime']; From 94f5b78b6b7eaa4f33db196c57e4f031463cb075 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Sun, 9 Apr 2023 02:59:50 +0900 Subject: [PATCH 011/562] =?UTF-8?q?13.11.0=20=E3=83=9E=E3=83=BC=E3=82=B8?= =?UTF-8?q?=E6=99=82=E3=81=AE=E3=83=9F=E3=82=B9=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/misc/id/aid.ts | 4 ++-- packages/backend/src/server/api/endpoints/notes/timeline.ts | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/misc/id/aid.ts b/packages/backend/src/misc/id/aid.ts index b69cc0c4f171..93a9929aa74c 100644 --- a/packages/backend/src/misc/id/aid.ts +++ b/packages/backend/src/misc/id/aid.ts @@ -6,7 +6,7 @@ import * as crypto from 'node:crypto'; const TIME2000 = 946684800000; let counter = crypto.randomBytes(2).readUInt16LE(0); -export function getTimeId(time: number): string { +function getTime(time: number): string { time = time - TIME2000; if (time < 0) time = 0; @@ -21,7 +21,7 @@ export function genAid(date: Date): string { const t = date.getTime(); if (isNaN(t)) throw 'Failed to create AID: Invalid Date'; counter++; - return getTimeId(t) + getNoise(); + return getTime(t) + getNoise(); } export function parseAid(id: string): { date: Date; } { diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 7eea1d850f02..dc894ea46354 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -65,7 +65,6 @@ export default class extends Endpoint { .where('following.followerId = :followerId', { followerId: me.id }) .getMany(); - const minId = getTimeId(new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)).getTime()) + '00'; // 30日前まで //#region Construct query const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) From d2b4e9aba63dc5b2f195a489bf69129924d159f3 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 9 Apr 2023 17:09:27 +0900 Subject: [PATCH 012/562] =?UTF-8?q?fix(backend):=20=E3=82=A4=E3=83=99?= =?UTF-8?q?=E3=83=B3=E3=83=88=E7=94=A8redis=E5=88=86=E9=9B=A2=E3=81=8C?= =?UTF-8?q?=E4=B8=8A=E6=89=8B=E3=81=8F=E5=8B=95=E3=81=8B=E3=81=AA=E3=81=84?= =?UTF-8?q?=E5=95=8F=E9=A1=8C=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/GlobalModule.ts | 30 +++++++++++++++---- packages/backend/src/core/AntennaService.ts | 8 ++--- packages/backend/src/core/CacheService.ts | 8 ++--- .../backend/src/core/GlobalEventService.ts | 6 ++-- packages/backend/src/core/MetaService.ts | 8 ++--- packages/backend/src/core/RoleService.ts | 8 ++--- packages/backend/src/core/WebhookService.ts | 8 ++--- packages/backend/src/di-symbols.ts | 3 +- .../server/api/StreamingApiServerService.ts | 8 ++--- 9 files changed, 53 insertions(+), 34 deletions(-) diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index cb713b25ad35..174d0d8bebcf 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -37,8 +37,24 @@ const $redis: Provider = { inject: [DI.config], }; -const $redisForPubsub: Provider = { - provide: DI.redisForPubsub, +const $redisForPub: Provider = { + provide: DI.redisForPub, + useFactory: (config) => { + const redis = new Redis({ + port: config.redisForPubsub.port, + host: config.redisForPubsub.host, + family: config.redisForPubsub.family == null ? 0 : config.redisForPubsub.family, + password: config.redisForPubsub.pass, + keyPrefix: `${config.redisForPubsub.prefix}:`, + db: config.redisForPubsub.db ?? 0, + }); + return redis; + }, + inject: [DI.config], +}; + +const $redisForSub: Provider = { + provide: DI.redisForSub, useFactory: (config) => { const redis = new Redis({ port: config.redisForPubsub.port, @@ -57,14 +73,15 @@ const $redisForPubsub: Provider = { @Global() @Module({ imports: [RepositoryModule], - providers: [$config, $db, $redis, $redisForPubsub], - exports: [$config, $db, $redis, $redisForPubsub, RepositoryModule], + providers: [$config, $db, $redis, $redisForPub, $redisForSub], + exports: [$config, $db, $redis, $redisForPub, $redisForSub, RepositoryModule], }) export class GlobalModule implements OnApplicationShutdown { constructor( @Inject(DI.db) private db: DataSource, @Inject(DI.redis) private redisClient: Redis.Redis, - @Inject(DI.redisForPubsub) private redisForPubsub: Redis.Redis, + @Inject(DI.redisForPub) private redisForPub: Redis.Redis, + @Inject(DI.redisForSub) private redisForSub: Redis.Redis, ) {} async onApplicationShutdown(signal: string): Promise { @@ -79,7 +96,8 @@ export class GlobalModule implements OnApplicationShutdown { await Promise.all([ this.db.destroy(), this.redisClient.disconnect(), - this.redisForPubsub.disconnect(), + this.redisForPub.disconnect(), + this.redisForSub.disconnect(), ]); } } diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 35266ac16d5c..47ebd4c74825 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -27,8 +27,8 @@ export class AntennaService implements OnApplicationShutdown { @Inject(DI.redis) private redisClient: Redis.Redis, - @Inject(DI.redisForPubsub) - private redisForPubsub: Redis.Redis, + @Inject(DI.redisForSub) + private redisForSub: Redis.Redis, @Inject(DI.mutingsRepository) private mutingsRepository: MutingsRepository, @@ -52,12 +52,12 @@ export class AntennaService implements OnApplicationShutdown { this.antennasFetched = false; this.antennas = []; - this.redisForPubsub.on('message', this.onRedisMessage); + this.redisForSub.on('message', this.onRedisMessage); } @bindThis public onApplicationShutdown(signal?: string | undefined) { - this.redisForPubsub.off('message', this.onRedisMessage); + this.redisForSub.off('message', this.onRedisMessage); } @bindThis diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index d74f3e8788f2..561face5c358 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -27,8 +27,8 @@ export class CacheService implements OnApplicationShutdown { @Inject(DI.redis) private redisClient: Redis.Redis, - @Inject(DI.redisForPubsub) - private redisForPubsub: Redis.Redis, + @Inject(DI.redisForSub) + private redisForSub: Redis.Redis, @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -116,7 +116,7 @@ export class CacheService implements OnApplicationShutdown { fromRedisConverter: (value) => new Set(JSON.parse(value)), }); - this.redisForPubsub.on('message', this.onMessage); + this.redisForSub.on('message', this.onMessage); } @bindThis @@ -167,6 +167,6 @@ export class CacheService implements OnApplicationShutdown { @bindThis public onApplicationShutdown(signal?: string | undefined) { - this.redisForPubsub.off('message', this.onMessage); + this.redisForSub.off('message', this.onMessage); } } diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index 25c064a2b400..9f4de5f9853a 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -26,8 +26,8 @@ export class GlobalEventService { @Inject(DI.config) private config: Config, - @Inject(DI.redis) - private redisClient: Redis.Redis, + @Inject(DI.redisForPub) + private redisForPub: Redis.Redis, ) { } @@ -37,7 +37,7 @@ export class GlobalEventService { { type: type, body: null } : { type: type, body: value }; - this.redisClient.publish(this.config.host, JSON.stringify({ + this.redisForPub.publish(this.config.host, JSON.stringify({ channel: channel, message: message, })); diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 2b6160c82e3c..1322927c2c64 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -14,8 +14,8 @@ export class MetaService implements OnApplicationShutdown { private intervalId: NodeJS.Timer; constructor( - @Inject(DI.redisForPubsub) - private redisForPubsub: Redis.Redis, + @Inject(DI.redisForSub) + private redisForSub: Redis.Redis, @Inject(DI.db) private db: DataSource, @@ -33,7 +33,7 @@ export class MetaService implements OnApplicationShutdown { }, 1000 * 60 * 5); } - this.redisForPubsub.on('message', this.onMessage); + this.redisForSub.on('message', this.onMessage); } @bindThis @@ -122,6 +122,6 @@ export class MetaService implements OnApplicationShutdown { @bindThis public onApplicationShutdown(signal?: string | undefined) { clearInterval(this.intervalId); - this.redisForPubsub.off('message', this.onMessage); + this.redisForSub.off('message', this.onMessage); } } diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index c8ebe1adb756..77645e3f0642 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -64,8 +64,8 @@ export class RoleService implements OnApplicationShutdown { public static NotAssignedError = class extends Error {}; constructor( - @Inject(DI.redisForPubsub) - private redisForPubsub: Redis.Redis, + @Inject(DI.redisForSub) + private redisForSub: Redis.Redis, @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -87,7 +87,7 @@ export class RoleService implements OnApplicationShutdown { this.rolesCache = new MemorySingleCache(1000 * 60 * 60 * 1); this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 60 * 1); - this.redisForPubsub.on('message', this.onMessage); + this.redisForSub.on('message', this.onMessage); } @bindThis @@ -400,6 +400,6 @@ export class RoleService implements OnApplicationShutdown { @bindThis public onApplicationShutdown(signal?: string | undefined) { - this.redisForPubsub.off('message', this.onMessage); + this.redisForSub.off('message', this.onMessage); } } diff --git a/packages/backend/src/core/WebhookService.ts b/packages/backend/src/core/WebhookService.ts index 85594f85578b..926115613b85 100644 --- a/packages/backend/src/core/WebhookService.ts +++ b/packages/backend/src/core/WebhookService.ts @@ -13,14 +13,14 @@ export class WebhookService implements OnApplicationShutdown { private webhooks: Webhook[] = []; constructor( - @Inject(DI.redisForPubsub) - private redisForPubsub: Redis.Redis, + @Inject(DI.redisForSub) + private redisForSub: Redis.Redis, @Inject(DI.webhooksRepository) private webhooksRepository: WebhooksRepository, ) { //this.onMessage = this.onMessage.bind(this); - this.redisForPubsub.on('message', this.onMessage); + this.redisForSub.on('message', this.onMessage); } @bindThis @@ -82,6 +82,6 @@ export class WebhookService implements OnApplicationShutdown { @bindThis public onApplicationShutdown(signal?: string | undefined) { - this.redisForPubsub.off('message', this.onMessage); + this.redisForSub.off('message', this.onMessage); } } diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index 482e8f83e151..d4b1fb31b107 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -2,7 +2,8 @@ export const DI = { config: Symbol('config'), db: Symbol('db'), redis: Symbol('redis'), - redisForPubsub: Symbol('redisForPubsub'), + redisForPub: Symbol('redisForPub'), + redisForSub: Symbol('redisForSub'), //#region Repositories usersRepository: Symbol('usersRepository'), diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index e0e5b71a824c..769a4490d618 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -22,8 +22,8 @@ export class StreamingApiServerService { @Inject(DI.config) private config: Config, - @Inject(DI.redisForPubsub) - private redisForPubsub: Redis.Redis, + @Inject(DI.redisForSub) + private redisForSub: Redis.Redis, @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -81,7 +81,7 @@ export class StreamingApiServerService { ev.emit(parsed.channel, parsed.message); } - this.redisForPubsub.on('message', onRedisMessage); + this.redisForSub.on('message', onRedisMessage); const main = new MainStreamConnection( this.channelsService, @@ -111,7 +111,7 @@ export class StreamingApiServerService { connection.once('close', () => { ev.removeAllListeners(); main.dispose(); - this.redisForPubsub.off('message', onRedisMessage); + this.redisForSub.off('message', onRedisMessage); if (intervalId) clearInterval(intervalId); }); From 3639390789b44521f869064bbff9ba61e1ff6d6c Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 17:53:34 +0900 Subject: [PATCH 013/562] Add ghcr push (experimental) --- .github/workflows/build.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/build.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 000000000000..34fba1e86fe5 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,33 @@ +name: Build +on: + push: + # タグ付きpushの時にする + tags: + + # TODO: 後で消す + branches: + - migrate/github-actions + + # 手動ビルドもできるように(いる？) + workflow_dispatch: + +jobs: + build: + name: build-images + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - name: "Login to GitHub Container Registry" + uses: docker/login-aciton@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: "Build and Push" + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: | + ghcr.io/misskeyio/misskey:latest + ghcr.io/misskeyio/misskey:${{ github.ref_name }} From b6af3ab89172371d01330980e9467e970039102b Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 17:56:32 +0900 Subject: [PATCH 014/562] Fix typo :p --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 34fba1e86fe5..28b13f2bf83c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: "Login to GitHub Container Registry" - uses: docker/login-aciton@v2 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} From eb71aa8f8cc7989cdb3787cc66b834a484668b62 Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 18:00:01 +0900 Subject: [PATCH 015/562] Format image tags :thinking_face: --- .github/workflows/build.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 28b13f2bf83c..fcd3d2b3ac18 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -23,6 +23,9 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Prepare image tags + run: | + echo "FORMATTED_BRANCH_NAME=$(echo ${{ github.ref_name }} | sed -e 's/\//-/g' )" >> $GITHUB_ENV - name: "Build and Push" uses: docker/build-push-action@v4 with: @@ -30,4 +33,4 @@ jobs: push: true tags: | ghcr.io/misskeyio/misskey:latest - ghcr.io/misskeyio/misskey:${{ github.ref_name }} + ghcr.io/misskeyio/misskey:${FORMATTED_BRANCH_NAME} From 69711f89e694fdd682fc6145dff7c7b65b4417c9 Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 18:01:49 +0900 Subject: [PATCH 016/562] Format image tags (2) :thinking_face: --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fcd3d2b3ac18..de73dfa3ee46 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -33,4 +33,4 @@ jobs: push: true tags: | ghcr.io/misskeyio/misskey:latest - ghcr.io/misskeyio/misskey:${FORMATTED_BRANCH_NAME} + ghcr.io/misskeyio/misskey:${{ env.FORMATTED_BRANCH_NAME }} From 2331483cd8753adf5524d7622391dfb6158d4a60 Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 18:28:49 +0900 Subject: [PATCH 017/562] Remove build.yaml --- .github/workflows/build.yaml | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 .github/workflows/build.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index de73dfa3ee46..000000000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: Build -on: - push: - # タグ付きpushの時にする - tags: - - # TODO: 後で消す - branches: - - migrate/github-actions - - # 手動ビルドもできるように(いる？) - workflow_dispatch: - -jobs: - build: - name: build-images - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - name: "Login to GitHub Container Registry" - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Prepare image tags - run: | - echo "FORMATTED_BRANCH_NAME=$(echo ${{ github.ref_name }} | sed -e 's/\//-/g' )" >> $GITHUB_ENV - - name: "Build and Push" - uses: docker/build-push-action@v4 - with: - context: . - push: true - tags: | - ghcr.io/misskeyio/misskey:latest - ghcr.io/misskeyio/misskey:${{ env.FORMATTED_BRANCH_NAME }} From e2d0fbb85977976ca5f8b85d90174e0b195f11bd Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 18:29:06 +0900 Subject: [PATCH 018/562] Rewrite github actions (from original build workflow) --- .github/workflows/docker-io.yaml | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/docker-io.yaml diff --git a/.github/workflows/docker-io.yaml b/.github/workflows/docker-io.yaml new file mode 100644 index 000000000000..fd18d1f12a2f --- /dev/null +++ b/.github/workflows/docker-io.yaml @@ -0,0 +1,53 @@ +name: Publish Docker image (io) + +on: + push: + tags: + + # TODO: 後で消す + branches: + - migrate/github-actions + + workflow_dispatch: + +jobs: + push_to_registry: + name: Push Docker image to GitHub Container Registry + runs-on: ubuntu-22.04 + if: github.repository == 'MisskeyIO/misskey' + steps: + - name: Check out the repo + uses: actions/checkout@v3 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2.5.0 + with: + platforms: linux/amd64,linux/arm64 + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/misskeyio/misskey + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Prepare image tags + run: | + echo "FORMATTED_BRANCH_NAME=$(echo ${{ github.ref_name }} | sed -e 's/\//-/g' )" >> $GITHUB_ENV + - name: Build and Push to Docker Hub + uses: docker/build-push-action@v4 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + push: true + platforms: ${{ steps.buildx.outputs.platforms }} + provenance: false + labels: ${{ env.FORMATTED_BRANCH_NAME }} + cache-from: type=gha + cache-to: type=gha,mode=max + tags: | + ghcr.io/misskeyio/misskey:latest + ghcr.io/misskeyio/misskey:${{ env.FORMATTED_BRANCH_NAME }} From 242b8910f727f015f2c198be3775538c14634025 Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 18:35:53 +0900 Subject: [PATCH 019/562] Fix step name (Docker Hub -> GitHub Container Registry) --- .github/workflows/docker-io.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-io.yaml b/.github/workflows/docker-io.yaml index fd18d1f12a2f..612e02ac915b 100644 --- a/.github/workflows/docker-io.yaml +++ b/.github/workflows/docker-io.yaml @@ -28,7 +28,7 @@ jobs: uses: docker/metadata-action@v4 with: images: ghcr.io/misskeyio/misskey - - name: Log in to Docker Hub + - name: Log in to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io @@ -37,7 +37,7 @@ jobs: - name: Prepare image tags run: | echo "FORMATTED_BRANCH_NAME=$(echo ${{ github.ref_name }} | sed -e 's/\//-/g' )" >> $GITHUB_ENV - - name: Build and Push to Docker Hub + - name: Build and Push to GitHub Container Registry uses: docker/build-push-action@v4 with: builder: ${{ steps.buildx.outputs.name }} From 08d2ca9b022de5706b0c79cc12795d6e354483d5 Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 18:38:55 +0900 Subject: [PATCH 020/562] Disable arm build --- .github/workflows/docker-io.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-io.yaml b/.github/workflows/docker-io.yaml index 612e02ac915b..a43e439333f5 100644 --- a/.github/workflows/docker-io.yaml +++ b/.github/workflows/docker-io.yaml @@ -22,7 +22,7 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2.5.0 with: - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 - name: Docker meta id: meta uses: docker/metadata-action@v4 From defe9d78c93068bd746e4e0afa9039cf2f346280 Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 19:00:02 +0900 Subject: [PATCH 021/562] Test cached build --- .github/workflows/docker-io.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-io.yaml b/.github/workflows/docker-io.yaml index a43e439333f5..3e4733135192 100644 --- a/.github/workflows/docker-io.yaml +++ b/.github/workflows/docker-io.yaml @@ -4,7 +4,7 @@ on: push: tags: - # TODO: 後で消す + # TODO: 後で消すヨ branches: - migrate/github-actions From 5db1c74e9db87ade273e91062609c29147e48a3c Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 19:08:14 +0900 Subject: [PATCH 022/562] Remove test trigger --- .github/workflows/docker-io.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/docker-io.yaml b/.github/workflows/docker-io.yaml index 3e4733135192..e524ffbe43c7 100644 --- a/.github/workflows/docker-io.yaml +++ b/.github/workflows/docker-io.yaml @@ -4,10 +4,6 @@ on: push: tags: - # TODO: 後で消すヨ - branches: - - migrate/github-actions - workflow_dispatch: jobs: From c22f5fb6c85c726293202473aee1c2d517d3376a Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 19:10:46 +0900 Subject: [PATCH 023/562] Fix tags resolution (maybe) --- .github/workflows/docker-io.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-io.yaml b/.github/workflows/docker-io.yaml index e524ffbe43c7..65f1c778c0c7 100644 --- a/.github/workflows/docker-io.yaml +++ b/.github/workflows/docker-io.yaml @@ -3,6 +3,7 @@ name: Publish Docker image (io) on: push: tags: + - * workflow_dispatch: From 78617695a6cf99ad0de5729c4f6d69febbfd473b Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 19:12:38 +0900 Subject: [PATCH 024/562] Fix tags resolution (2) (maybe) --- .github/workflows/docker-io.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-io.yaml b/.github/workflows/docker-io.yaml index 65f1c778c0c7..d803cd47f120 100644 --- a/.github/workflows/docker-io.yaml +++ b/.github/workflows/docker-io.yaml @@ -3,7 +3,7 @@ name: Publish Docker image (io) on: push: tags: - - * + - "**" workflow_dispatch: From 74fc946be416888d54940b86366d43eb3c9c1163 Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 19:15:38 +0900 Subject: [PATCH 025/562] Rename .yaml -> .yml --- .github/workflows/{docker-io.yaml => docker-io.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{docker-io.yaml => docker-io.yml} (100%) diff --git a/.github/workflows/docker-io.yaml b/.github/workflows/docker-io.yml similarity index 100% rename from .github/workflows/docker-io.yaml rename to .github/workflows/docker-io.yml From 385be641d866587bfcc38ed188e5e4b80cc31c14 Mon Sep 17 00:00:00 2001 From: Laica Lunasys Date: Sun, 9 Apr 2023 19:49:38 +0900 Subject: [PATCH 026/562] Fix setup-buildx-action version --- .github/workflows/docker-io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-io.yml b/.github/workflows/docker-io.yml index d803cd47f120..1e8eff26d436 100644 --- a/.github/workflows/docker-io.yml +++ b/.github/workflows/docker-io.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v3 - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v2.5.0 + uses: docker/setup-buildx-action@v2 with: platforms: linux/amd64 - name: Docker meta From 89f4e8f14d4856ff2aca855efc13ecc44432cacb Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Sun, 9 Apr 2023 19:49:37 +0900 Subject: [PATCH 027/562] =?UTF-8?q?x-worker-host=20=E3=82=92=E5=BE=A9?= =?UTF-8?q?=E6=B4=BB=E3=81=95=E3=81=9B=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/server/ServerService.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index 68850d5b46e2..a320a55cc55d 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -85,6 +85,12 @@ export class ServerService implements OnApplicationShutdown { serve: false, }); + const hostname = os.hostname(); + fastify.addHook('onRequest', (request, reply, done) => { + reply.header('x-worker-host', hostname); + done(); + }); + fastify.register(this.apiServerService.createServer, { prefix: '/api' }); fastify.register(this.openApiServerService.createServer); fastify.register(this.fileServerService.createServer); From d429f810a9ea3fda9efed2ff51483d25a288ecc9 Mon Sep 17 00:00:00 2001 From: Ebise Lutica <7106976+EbiseLutica@users.noreply.github.com> Date: Thu, 13 Apr 2023 00:31:22 +0900 Subject: [PATCH 028/562] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41353c346b23..df2265727d41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ ## 13.11.2 +### Note +- 13.11.0または13.11.1から13.11.2以降にアップデートする場合、Redisのカスタム絵文字のキャッシュを削除する必要があります(https://github.com/misskey-dev/misskey/issues/10502#issuecomment-1502790755 参照) + ### General - チャンネルの検索用ページの追加 From da277eb3e5f0b9191d531fd30bb9f46e22c17d37 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Thu, 13 Apr 2023 14:38:08 +0900 Subject: [PATCH 029/562] =?UTF-8?q?=E4=B8=80=E9=83=A8=E3=81=AE=20Workflow?= =?UTF-8?q?=20=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/reviewer-lottery.yml | 10 ---- .github/workflows/reviewer_lottery.yml | 13 ----- .github/workflows/storybook.yml | 74 -------------------------- 3 files changed, 97 deletions(-) delete mode 100644 .github/reviewer-lottery.yml delete mode 100644 .github/workflows/reviewer_lottery.yml delete mode 100644 .github/workflows/storybook.yml diff --git a/.github/reviewer-lottery.yml b/.github/reviewer-lottery.yml deleted file mode 100644 index fd2fb1913ffa..000000000000 --- a/.github/reviewer-lottery.yml +++ /dev/null @@ -1,10 +0,0 @@ -groups: - - name: devs - reviewers: 2 - internal_reviewers: 1 - usernames: - - syuilo - - acid-chicken - - EbiseLutica - - rinsuki - - tamaina diff --git a/.github/workflows/reviewer_lottery.yml b/.github/workflows/reviewer_lottery.yml deleted file mode 100644 index 33228d7465a6..000000000000 --- a/.github/workflows/reviewer_lottery.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: "Reviewer lottery" -on: - pull_request_target: - types: [opened, ready_for_review, reopened] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: uesteibar/reviewer-lottery@v2 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/storybook.yml b/.github/workflows/storybook.yml deleted file mode 100644 index 6792674d9fe0..000000000000 --- a/.github/workflows/storybook.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Storybook - -on: - push: - branches-ignore: - - l10n_develop - -jobs: - build: - runs-on: ubuntu-latest - - env: - NODE_OPTIONS: "--max_old_space_size=7168" - - steps: - - uses: actions/checkout@v3.3.0 - with: - fetch-depth: 0 - submodules: true - - name: Install pnpm - uses: pnpm/action-setup@v2 - with: - version: 7 - run_install: false - - name: Use Node.js 18.x - uses: actions/setup-node@v3.6.0 - with: - node-version: 18.x - cache: 'pnpm' - - run: corepack enable - - run: pnpm i --frozen-lockfile - - name: Check pnpm-lock.yaml - run: git diff --exit-code pnpm-lock.yaml - - name: Build misskey-js - run: pnpm --filter misskey-js build - - name: Build storybook - run: pnpm --filter frontend build-storybook - - name: Publish to Chromatic - if: github.ref == 'refs/heads/master' - run: pnpm --filter frontend chromatic --exit-once-uploaded -d storybook-static - env: - CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} - - name: Publish to Chromatic - if: github.ref != 'refs/heads/master' - id: chromatic - run: | - DIFF="${{ github.event.before }} HEAD" - if [ "$DIFF" = "0000000000000000000000000000000000000000 HEAD" ]; then - DIFF="HEAD" - fi - CHROMATIC_PARAMETER="$(node packages/frontend/.storybook/changes.js $(git diff-tree --no-commit-id --name-only -r $(echo "$DIFF") | xargs))" - if [ "$CHROMATIC_PARAMETER" = " --skip" ]; then - echo "skip=true" >> $GITHUB_OUTPUT - fi - pnpm --filter frontend chromatic --exit-once-uploaded -d storybook-static $(echo "$CHROMATIC_PARAMETER") - env: - CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} - - name: Notify that Chromatic will skip testing - uses: actions/github-script@v6.4.0 - if: github.ref != 'refs/heads/master' && github.ref != 'refs/heads/develop' && steps.chromatic.outputs.skip == 'true' - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - github.rest.repos.createCommitComment({ - owner: context.repo.owner, - repo: context.repo.repo, - commit_sha: context.sha, - body: 'Chromatic will skip testing but you may still have to [review the changes on Chromatic](https://www.chromatic.com/pullrequests?appId=6428f7d7b962f0b79f97d6e4).' - }) - - name: Upload Artifacts - uses: actions/upload-artifact@v3 - with: - name: storybook - path: packages/frontend/storybook-static From d4903a476ae1ffdbd579c90d2072bb94fdefcf08 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Tue, 13 Jun 2023 17:05:18 +0900 Subject: [PATCH 030/562] fix --- gulpfile.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gulpfile.js b/gulpfile.js index 55bc8d368fe5..97e5c30b9525 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -38,8 +38,8 @@ gulp.task('copy:frontend:locales', cb => { gulp.task('build:backend:script', () => { const clientManifestExists = fs.existsSync('./built/_vite_/manifest.json'); const clientEntry = clientManifestExists ? - JSON.parse(fs.readFileSync('./built/_vite_/manifest.json', 'utf-8'))['src/init.ts'].file - : 'src/init.ts' + JSON.parse(fs.readFileSync('./built/_vite_/manifest.json', 'utf-8'))['src/_boot_.ts'].file + : 'src/_boot_.ts' return gulp.src(['./packages/backend/src/server/web/boot.js', './packages/backend/src/server/web/bios.js', './packages/backend/src/server/web/cli.js', './packages/backend/src/server/web/flush.js']) .pipe(replace('LANGS', JSON.stringify(Object.keys(locales)))) From 9b3a92e99920000cb724216977387a57cbfe41a9 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Thu, 15 Jun 2023 02:57:13 +0900 Subject: [PATCH 031/562] Revert "enhance(backend): migrate bull to bullmq (#10910)" This reverts commit fd7b77c542b51313d8b8ea60124725fe65a295d5. --- packages/backend/package.json | 2 +- packages/backend/src/boot/common.ts | 1 - .../backend/src/core/NoteCreateService.ts | 2 +- packages/backend/src/core/QueueModule.ts | 53 ++- packages/backend/src/core/QueueService.ts | 65 +--- .../core/activitypub/models/ApNoteService.ts | 2 +- .../backend/src/daemons/QueueStatsService.ts | 16 +- packages/backend/src/misc/prelude/time.ts | 15 +- .../src/queue/QueueProcessorService.ts | 350 +++++++----------- packages/backend/src/queue/const.ts | 26 -- packages/backend/src/queue/get-job-info.ts | 15 + .../AggregateRetentionProcessorService.ts | 6 +- .../CheckExpiredMutingsProcessorService.ts | 5 +- .../processors/CleanChartsProcessorService.ts | 5 +- .../queue/processors/CleanProcessorService.ts | 5 +- .../CleanRemoteFilesProcessorService.ts | 11 +- .../DeleteAccountProcessorService.ts | 4 +- .../DeleteDriveFilesProcessorService.ts | 12 +- .../processors/DeleteFileProcessorService.ts | 4 +- .../processors/DeliverProcessorService.ts | 8 +- .../EndedPollNotificationProcessorService.ts | 7 +- .../ExportAntennasProcessorService.ts | 6 +- .../ExportBlockingProcessorService.ts | 13 +- .../ExportCustomEmojisProcessorService.ts | 39 +- .../ExportFavoritesProcessorService.ts | 11 +- .../ExportFollowingProcessorService.ts | 9 +- .../ExportMutingProcessorService.ts | 13 +- .../processors/ExportNotesProcessorService.ts | 11 +- .../ExportUserListsProcessorService.ts | 9 +- .../ImportAntennasProcessorService.ts | 6 +- .../ImportBlockingProcessorService.ts | 11 +- .../ImportCustomEmojisProcessorService.ts | 6 +- .../ImportFollowingProcessorService.ts | 11 +- .../ImportMutingProcessorService.ts | 9 +- .../ImportUserListsProcessorService.ts | 7 +- .../queue/processors/InboxProcessorService.ts | 36 +- .../RelationshipProcessorService.ts | 2 +- .../ResyncChartsProcessorService.ts | 5 +- .../processors/TickChartsProcessorService.ts | 5 +- .../WebhookDeliverProcessorService.ts | 4 +- packages/frontend/src/scripts/time.ts | 15 +- pnpm-lock.yaml | 173 +++++---- 42 files changed, 491 insertions(+), 524 deletions(-) delete mode 100644 packages/backend/src/queue/const.ts create mode 100644 packages/backend/src/queue/get-job-info.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 56ecbc2eaf00..85677f8a9e25 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -78,7 +78,7 @@ "autwh": "0.1.0", "bcryptjs": "2.4.3", "blurhash": "2.0.5", - "bullmq": "3.15.0", + "bull": "4.10.4", "cacheable-lookup": "6.1.0", "cbor": "9.0.0", "chalk": "5.2.0", diff --git a/packages/backend/src/boot/common.ts b/packages/backend/src/boot/common.ts index 3995545d7f74..b6ab939644ae 100644 --- a/packages/backend/src/boot/common.ts +++ b/packages/backend/src/boot/common.ts @@ -34,7 +34,6 @@ export async function jobQueue() { }); jobQueue.enableShutdownHooks(); - jobQueue.get(QueueProcessorService).start(); jobQueue.get(ChartManagementService).start(); return jobQueue; diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 1c8491bf573b..2c7c78f950c7 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -510,7 +510,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (data.poll && data.poll.expiresAt) { const delay = data.poll.expiresAt.getTime() - Date.now(); - this.queueService.endedPollNotificationQueue.add(note.id, { + this.queueService.endedPollNotificationQueue.add({ noteId: note.id, }, { delay, diff --git a/packages/backend/src/core/QueueModule.ts b/packages/backend/src/core/QueueModule.ts index 3384ca457785..bacc59aae211 100644 --- a/packages/backend/src/core/QueueModule.ts +++ b/packages/backend/src/core/QueueModule.ts @@ -1,11 +1,42 @@ import { setTimeout } from 'node:timers/promises'; import { Inject, Module, OnApplicationShutdown } from '@nestjs/common'; -import * as Bull from 'bullmq'; +import Bull from 'bull'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; -import { QUEUE, baseQueueOptions } from '@/queue/const.js'; import type { Provider } from '@nestjs/common'; -import type { DeliverJobData, InboxJobData, EndedPollNotificationJobData, WebhookDeliverJobData, RelationshipJobData } from '../queue/types.js'; +import type { DeliverJobData, InboxJobData, DbJobData, ObjectStorageJobData, EndedPollNotificationJobData, WebhookDeliverJobData, RelationshipJobData, DbJobMap } from '../queue/types.js'; + +function q(config: Config, name: string, limitPerSec = -1) { + return new Bull(name, { + redis: { + port: config.redisForJobQueue.port, + host: config.redisForJobQueue.host, + family: config.redisForJobQueue.family == null ? 0 : config.redisForJobQueue.family, + password: config.redisForJobQueue.pass, + db: config.redisForJobQueue.db ?? 0, + }, + prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue` : 'queue', + limiter: limitPerSec > 0 ? { + max: limitPerSec, + duration: 1000, + } : undefined, + settings: { + backoffStrategies: { + apBackoff, + }, + }, + }); +} + +// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 +function apBackoff(attemptsMade: number, err: Error) { + const baseDelay = 60 * 1000; // 1min + const maxBackoff = 8 * 60 * 60 * 1000; // 8hours + let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay; + backoff = Math.min(backoff, maxBackoff); + backoff += Math.round(backoff * Math.random() * 0.2); + return backoff; +} export type SystemQueue = Bull.Queue>; export type EndedPollNotificationQueue = Bull.Queue; @@ -18,49 +49,49 @@ export type WebhookDeliverQueue = Bull.Queue; const $system: Provider = { provide: 'queue:system', - useFactory: (config: Config) => new Bull.Queue(QUEUE.SYSTEM, baseQueueOptions(config, QUEUE.SYSTEM)), + useFactory: (config: Config) => q(config, 'system'), inject: [DI.config], }; const $endedPollNotification: Provider = { provide: 'queue:endedPollNotification', - useFactory: (config: Config) => new Bull.Queue(QUEUE.ENDED_POLL_NOTIFICATION, baseQueueOptions(config, QUEUE.ENDED_POLL_NOTIFICATION)), + useFactory: (config: Config) => q(config, 'endedPollNotification'), inject: [DI.config], }; const $deliver: Provider = { provide: 'queue:deliver', - useFactory: (config: Config) => new Bull.Queue(QUEUE.DELIVER, baseQueueOptions(config, QUEUE.DELIVER)), + useFactory: (config: Config) => q(config, 'deliver', config.deliverJobPerSec ?? 128), inject: [DI.config], }; const $inbox: Provider = { provide: 'queue:inbox', - useFactory: (config: Config) => new Bull.Queue(QUEUE.INBOX, baseQueueOptions(config, QUEUE.INBOX)), + useFactory: (config: Config) => q(config, 'inbox', config.inboxJobPerSec ?? 16), inject: [DI.config], }; const $db: Provider = { provide: 'queue:db', - useFactory: (config: Config) => new Bull.Queue(QUEUE.DB, baseQueueOptions(config, QUEUE.DB)), + useFactory: (config: Config) => q(config, 'db'), inject: [DI.config], }; const $relationship: Provider = { provide: 'queue:relationship', - useFactory: (config: Config) => new Bull.Queue(QUEUE.RELATIONSHIP, baseQueueOptions(config, QUEUE.RELATIONSHIP)), + useFactory: (config: Config) => q(config, 'relationship', config.relashionshipJobPerSec ?? 64), inject: [DI.config], }; const $objectStorage: Provider = { provide: 'queue:objectStorage', - useFactory: (config: Config) => new Bull.Queue(QUEUE.OBJECT_STORAGE, baseQueueOptions(config, QUEUE.OBJECT_STORAGE)), + useFactory: (config: Config) => q(config, 'objectStorage'), inject: [DI.config], }; const $webhookDeliver: Provider = { provide: 'queue:webhookDeliver', - useFactory: (config: Config) => new Bull.Queue(QUEUE.WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.WEBHOOK_DELIVER)), + useFactory: (config: Config) => q(config, 'webhookDeliver', 64), inject: [DI.config], }; diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 2ae8a2b7548f..b4ffffecc0a6 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -1,5 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { v4 as uuid } from 'uuid'; +import Bull from 'bull'; import type { IActivity } from '@/core/activitypub/type.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Webhook, webhookEventTypes } from '@/models/entities/Webhook.js'; @@ -10,7 +11,6 @@ import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; import type { DbJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; import type httpSignature from '@peertube/http-signature'; -import type * as Bull from 'bullmq'; @Injectable() export class QueueService { @@ -26,43 +26,7 @@ export class QueueService { @Inject('queue:relationship') public relationshipQueue: RelationshipQueue, @Inject('queue:objectStorage') public objectStorageQueue: ObjectStorageQueue, @Inject('queue:webhookDeliver') public webhookDeliverQueue: WebhookDeliverQueue, - ) { - this.systemQueue.add('tickCharts', { - }, { - repeat: { pattern: '55 * * * *' }, - removeOnComplete: true, - }); - - this.systemQueue.add('resyncCharts', { - }, { - repeat: { pattern: '0 0 * * *' }, - removeOnComplete: true, - }); - - this.systemQueue.add('cleanCharts', { - }, { - repeat: { pattern: '0 0 * * *' }, - removeOnComplete: true, - }); - - this.systemQueue.add('aggregateRetention', { - }, { - repeat: { pattern: '0 0 * * *' }, - removeOnComplete: true, - }); - - this.systemQueue.add('clean', { - }, { - repeat: { pattern: '0 0 * * *' }, - removeOnComplete: true, - }); - - this.systemQueue.add('checkExpiredMutings', { - }, { - repeat: { pattern: '*/5 * * * *' }, - removeOnComplete: true, - }); - } + ) {} @bindThis public deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean) { @@ -78,10 +42,11 @@ export class QueueService { isSharedInbox, }; - return this.deliverQueue.add(to, data, { + return this.deliverQueue.add(data, { attempts: this.config.deliverJobMaxAttempts ?? 12, + timeout: 1 * 60 * 1000, // 1min backoff: { - type: 'custom', + type: 'apBackoff', }, removeOnComplete: true, removeOnFail: true, @@ -95,10 +60,11 @@ export class QueueService { signature, }; - return this.inboxQueue.add('', data, { + return this.inboxQueue.add(data, { attempts: this.config.inboxJobMaxAttempts ?? 8, + timeout: 5 * 60 * 1000, // 5min backoff: { - type: 'custom', + type: 'apBackoff', }, removeOnComplete: true, removeOnFail: true, @@ -246,7 +212,7 @@ export class QueueService { private generateToDbJobData>(name: T, data: D): { name: string, data: D, - opts: Bull.JobsOptions, + opts: Bull.JobOptions, } { return { name, @@ -333,10 +299,10 @@ export class QueueService { } @bindThis - private generateRelationshipJobData(name: 'follow' | 'unfollow' | 'block' | 'unblock', data: RelationshipJobData, opts: Bull.JobsOptions = {}): { + private generateRelationshipJobData(name: 'follow' | 'unfollow' | 'block' | 'unblock', data: RelationshipJobData, opts: Bull.JobOptions = {}): { name: string, data: RelationshipJobData, - opts: Bull.JobsOptions, + opts: Bull.JobOptions, } { return { name, @@ -385,10 +351,11 @@ export class QueueService { eventId: uuid(), }; - return this.webhookDeliverQueue.add(webhook.id, data, { + return this.webhookDeliverQueue.add(data, { attempts: 4, + timeout: 1 * 60 * 1000, // 1min backoff: { - type: 'custom', + type: 'apBackoff', }, removeOnComplete: true, removeOnFail: true, @@ -400,11 +367,11 @@ export class QueueService { this.deliverQueue.once('cleaned', (jobs, status) => { //deliverLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); }); - this.deliverQueue.clean(0, Infinity, 'delayed'); + this.deliverQueue.clean(0, 'delayed'); this.inboxQueue.once('cleaned', (jobs, status) => { //inboxLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); }); - this.inboxQueue.clean(0, Infinity, 'delayed'); + this.inboxQueue.clean(0, 'delayed'); } } diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 76757f530a6d..7f09ae7d3372 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -18,7 +18,6 @@ import { PollService } from '@/core/PollService.js'; import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; -import { checkHttps } from '@/misc/check-https.js'; import { getOneApId, getApId, getOneApHrefNullable, validPost, isEmoji, getApType } from '../type.js'; // eslint-disable-next-line @typescript-eslint/consistent-type-imports import { ApLoggerService } from '../ApLoggerService.js'; @@ -33,6 +32,7 @@ import { ApQuestionService } from './ApQuestionService.js'; import { ApImageService } from './ApImageService.js'; import type { Resolver } from '../ApResolverService.js'; import type { IObject, IPost } from '../type.js'; +import { checkHttps } from '@/misc/check-https.js'; @Injectable() export class ApNoteService { diff --git a/packages/backend/src/daemons/QueueStatsService.ts b/packages/backend/src/daemons/QueueStatsService.ts index 53a0d14cd78d..f6f26db1fb90 100644 --- a/packages/backend/src/daemons/QueueStatsService.ts +++ b/packages/backend/src/daemons/QueueStatsService.ts @@ -1,11 +1,7 @@ -import { Inject, Injectable } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import Xev from 'xev'; -import * as Bull from 'bullmq'; import { QueueService } from '@/core/QueueService.js'; import { bindThis } from '@/decorators.js'; -import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; -import { QUEUE, baseQueueOptions } from '@/queue/const.js'; import type { OnApplicationShutdown } from '@nestjs/common'; const ev = new Xev(); @@ -17,9 +13,6 @@ export class QueueStatsService implements OnApplicationShutdown { private intervalId: NodeJS.Timer; constructor( - @Inject(DI.config) - private config: Config, - private queueService: QueueService, ) { } @@ -38,14 +31,11 @@ export class QueueStatsService implements OnApplicationShutdown { let activeDeliverJobs = 0; let activeInboxJobs = 0; - const deliverQueueEvents = new Bull.QueueEvents(QUEUE.DELIVER, baseQueueOptions(this.config, QUEUE.DELIVER)); - const inboxQueueEvents = new Bull.QueueEvents(QUEUE.INBOX, baseQueueOptions(this.config, QUEUE.INBOX)); - - deliverQueueEvents.on('active', () => { + this.queueService.deliverQueue.on('global:active', () => { activeDeliverJobs++; }); - inboxQueueEvents.on('active', () => { + this.queueService.inboxQueue.on('global:active', () => { activeInboxJobs++; }); diff --git a/packages/backend/src/misc/prelude/time.ts b/packages/backend/src/misc/prelude/time.ts index b21978b18689..0da1f791395d 100644 --- a/packages/backend/src/misc/prelude/time.ts +++ b/packages/backend/src/misc/prelude/time.ts @@ -5,14 +5,13 @@ const dateTimeIntervals = { }; export function dateUTC(time: number[]): Date { - const d = - time.length === 2 ? Date.UTC(time[0], time[1]) - : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) - : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) - : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) - : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) - : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) - : null; + const d = time.length === 2 ? Date.UTC(time[0], time[1]) + : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) + : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) + : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) + : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) + : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) + : null; if (!d) throw new Error('wrong number of arguments'); diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 42f9c1af7d7a..6ef760afb4c9 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -1,9 +1,10 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; -import * as Bull from 'bullmq'; import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; import type Logger from '@/logger.js'; +import { QueueService } from '@/core/QueueService.js'; import { bindThis } from '@/decorators.js'; +import { getJobInfo } from './get-job-info.js'; import { WebhookDeliverProcessorService } from './processors/WebhookDeliverProcessorService.js'; import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js'; import { DeliverProcessorService } from './processors/DeliverProcessorService.js'; @@ -34,51 +35,17 @@ import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMu import { CleanProcessorService } from './processors/CleanProcessorService.js'; import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; import { QueueLoggerService } from './QueueLoggerService.js'; -import { QUEUE, baseQueueOptions } from './const.js'; - -// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 -function httpRelatedBackoff(attemptsMade: number) { - const baseDelay = 60 * 1000; // 1min - const maxBackoff = 8 * 60 * 60 * 1000; // 8hours - let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay; - backoff = Math.min(backoff, maxBackoff); - backoff += Math.round(backoff * Math.random() * 0.2); - return backoff; -} - -function getJobInfo(job: Bull.Job | undefined, increment = false): string { - if (job == null) return '-'; - - const age = Date.now() - job.timestamp; - - const formated = age > 60000 ? `${Math.floor(age / 1000 / 60)}m` - : age > 10000 ? `${Math.floor(age / 1000)}s` - : `${age}ms`; - - // onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする - const currentAttempts = job.attemptsMade + (increment ? 1 : 0); - const maxAttempts = job.opts ? job.opts.attempts : 0; - - return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`; -} @Injectable() export class QueueProcessorService implements OnApplicationShutdown { private logger: Logger; - private systemQueueWorker: Bull.Worker; - private dbQueueWorker: Bull.Worker; - private deliverQueueWorker: Bull.Worker; - private inboxQueueWorker: Bull.Worker; - private webhookDeliverQueueWorker: Bull.Worker; - private relationshipQueueWorker: Bull.Worker; - private objectStorageQueueWorker: Bull.Worker; - private endedPollNotificationQueueWorker: Bull.Worker; constructor( @Inject(DI.config) private config: Config, private queueLoggerService: QueueLoggerService, + private queueService: QueueService, private webhookDeliverProcessorService: WebhookDeliverProcessorService, private endedPollNotificationProcessorService: EndedPollNotificationProcessorService, private deliverProcessorService: DeliverProcessorService, @@ -127,227 +94,160 @@ export class QueueProcessorService implements OnApplicationShutdown { } } - //#region system - this.systemQueueWorker = new Bull.Worker(QUEUE.SYSTEM, (job) => { - switch (job.name) { - case 'tickCharts': return this.tickChartsProcessorService.process(); - case 'resyncCharts': return this.resyncChartsProcessorService.process(); - case 'cleanCharts': return this.cleanChartsProcessorService.process(); - case 'aggregateRetention': return this.aggregateRetentionProcessorService.process(); - case 'checkExpiredMutings': return this.checkExpiredMutingsProcessorService.process(); - case 'clean': return this.cleanProcessorService.process(); - default: throw new Error(`unrecognized job type ${job.name} for system`); - } - }, { - ...baseQueueOptions(this.config, QUEUE.SYSTEM), - autorun: false, - }); - const systemLogger = this.logger.createSubLogger('system'); + const deliverLogger = this.logger.createSubLogger('deliver'); + const webhookLogger = this.logger.createSubLogger('webhook'); + const inboxLogger = this.logger.createSubLogger('inbox'); + const dbLogger = this.logger.createSubLogger('db'); + const relationshipLogger = this.logger.createSubLogger('relationship'); + const objectStorageLogger = this.logger.createSubLogger('objectStorage'); - this.systemQueueWorker + this.queueService.systemQueue + .on('waiting', (jobId) => systemLogger.debug(`waiting id=${jobId}`)) .on('active', (job) => systemLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) - .on('error', (err: Error) => systemLogger.error(`error ${err}`, { e: renderError(err) })) - .on('stalled', (jobId) => systemLogger.warn(`stalled id=${jobId}`)); - //#endregion - - //#region db - this.dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => { - switch (job.name) { - case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job); - case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job); - case 'exportNotes': return this.exportNotesProcessorService.process(job); - case 'exportFavorites': return this.exportFavoritesProcessorService.process(job); - case 'exportFollowing': return this.exportFollowingProcessorService.process(job); - case 'exportMuting': return this.exportMutingProcessorService.process(job); - case 'exportBlocking': return this.exportBlockingProcessorService.process(job); - case 'exportUserLists': return this.exportUserListsProcessorService.process(job); - case 'exportAntennas': return this.exportAntennasProcessorService.process(job); - case 'importFollowing': return this.importFollowingProcessorService.process(job); - case 'importFollowingToDb': return this.importFollowingProcessorService.processDb(job); - case 'importMuting': return this.importMutingProcessorService.process(job); - case 'importBlocking': return this.importBlockingProcessorService.process(job); - case 'importBlockingToDb': return this.importBlockingProcessorService.processDb(job); - case 'importUserLists': return this.importUserListsProcessorService.process(job); - case 'importCustomEmojis': return this.importCustomEmojisProcessorService.process(job); - case 'importAntennas': return this.importAntennasProcessorService.process(job); - case 'deleteAccount': return this.deleteAccountProcessorService.process(job); - default: throw new Error(`unrecognized job type ${job.name} for db`); - } - }, { - ...baseQueueOptions(this.config, QUEUE.DB), - autorun: false, - }); + .on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) + .on('error', (job: any, err: Error) => systemLogger.error(`error ${err}`, { job, e: renderError(err) })) + .on('stalled', (job) => systemLogger.warn(`stalled id=${job.id}`)); - const dbLogger = this.logger.createSubLogger('db'); - - this.dbQueueWorker - .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) - .on('error', (err: Error) => dbLogger.error(`error ${err}`, { e: renderError(err) })) - .on('stalled', (jobId) => dbLogger.warn(`stalled id=${jobId}`)); - //#endregion - - //#region deliver - this.deliverQueueWorker = new Bull.Worker(QUEUE.DELIVER, (job) => this.deliverProcessorService.process(job), { - ...baseQueueOptions(this.config, QUEUE.DELIVER), - autorun: false, - concurrency: this.config.deliverJobConcurrency ?? 128, - limiter: { - max: this.config.deliverJobPerSec ?? 128, - duration: 1000, - }, - settings: { - backoffStrategy: httpRelatedBackoff, - }, - }); - - const deliverLogger = this.logger.createSubLogger('deliver'); - - this.deliverQueueWorker + this.queueService.deliverQueue + .on('waiting', (jobId) => deliverLogger.debug(`waiting id=${jobId}`)) .on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) - .on('error', (err: Error) => deliverLogger.error(`error ${err}`, { e: renderError(err) })) - .on('stalled', (jobId) => deliverLogger.warn(`stalled id=${jobId}`)); - //#endregion - - //#region inbox - this.inboxQueueWorker = new Bull.Worker(QUEUE.INBOX, (job) => this.inboxProcessorService.process(job), { - ...baseQueueOptions(this.config, QUEUE.INBOX), - autorun: false, - concurrency: this.config.inboxJobConcurrency ?? 16, - limiter: { - max: this.config.inboxJobPerSec ?? 16, - duration: 1000, - }, - settings: { - backoffStrategy: httpRelatedBackoff, - }, - }); + .on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`)) + .on('error', (job: any, err: Error) => deliverLogger.error(`error ${err}`, { job, e: renderError(err) })) + .on('stalled', (job) => deliverLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`)); - const inboxLogger = this.logger.createSubLogger('inbox'); - - this.inboxQueueWorker + this.queueService.inboxQueue + .on('waiting', (jobId) => inboxLogger.debug(`waiting id=${jobId}`)) .on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`)) .on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) - .on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) })) - .on('error', (err: Error) => inboxLogger.error(`error ${err}`, { e: renderError(err) })) - .on('stalled', (jobId) => inboxLogger.warn(`stalled id=${jobId}`)); - //#endregion + .on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`, { job, e: renderError(err) })) + .on('error', (job: any, err: Error) => inboxLogger.error(`error ${err}`, { job, e: renderError(err) })) + .on('stalled', (job) => inboxLogger.warn(`stalled ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`)); - //#region webhook deliver - this.webhookDeliverQueueWorker = new Bull.Worker(QUEUE.WEBHOOK_DELIVER, (job) => this.webhookDeliverProcessorService.process(job), { - ...baseQueueOptions(this.config, QUEUE.WEBHOOK_DELIVER), - autorun: false, - concurrency: 64, - limiter: { - max: 64, - duration: 1000, - }, - settings: { - backoffStrategy: httpRelatedBackoff, - }, - }); + this.queueService.dbQueue + .on('waiting', (jobId) => dbLogger.debug(`waiting id=${jobId}`)) + .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) + .on('error', (job: any, err: Error) => dbLogger.error(`error ${err}`, { job, e: renderError(err) })) + .on('stalled', (job) => dbLogger.warn(`stalled id=${job.id}`)); - const webhookLogger = this.logger.createSubLogger('webhook'); + this.queueService.relationshipQueue + .on('waiting', (jobId) => relationshipLogger.debug(`waiting id=${jobId}`)) + .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) + .on('error', (job: any, err: Error) => relationshipLogger.error(`error ${err}`, { job, e: renderError(err) })) + .on('stalled', (job) => relationshipLogger.warn(`stalled id=${job.id}`)); - this.webhookDeliverQueueWorker + this.queueService.objectStorageQueue + .on('waiting', (jobId) => objectStorageLogger.debug(`waiting id=${jobId}`)) + .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) + .on('error', (job: any, err: Error) => objectStorageLogger.error(`error ${err}`, { job, e: renderError(err) })) + .on('stalled', (job) => objectStorageLogger.warn(`stalled id=${job.id}`)); + + this.queueService.webhookDeliverQueue + .on('waiting', (jobId) => webhookLogger.debug(`waiting id=${jobId}`)) .on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) - .on('error', (err: Error) => webhookLogger.error(`error ${err}`, { e: renderError(err) })) - .on('stalled', (jobId) => webhookLogger.warn(`stalled id=${jobId}`)); - //#endregion + .on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`)) + .on('error', (job: any, err: Error) => webhookLogger.error(`error ${err}`, { job, e: renderError(err) })) + .on('stalled', (job) => webhookLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`)); - //#region relationship - this.relationshipQueueWorker = new Bull.Worker(QUEUE.RELATIONSHIP, (job) => { - switch (job.name) { - case 'follow': return this.relationshipProcessorService.processFollow(job); - case 'unfollow': return this.relationshipProcessorService.processUnfollow(job); - case 'block': return this.relationshipProcessorService.processBlock(job); - case 'unblock': return this.relationshipProcessorService.processUnblock(job); - default: throw new Error(`unrecognized job type ${job.name} for relationship`); - } + this.queueService.systemQueue.add('tickCharts', { }, { - ...baseQueueOptions(this.config, QUEUE.RELATIONSHIP), - autorun: false, - concurrency: this.config.relashionshipJobConcurrency ?? 16, - limiter: { - max: this.config.relashionshipJobPerSec ?? 64, - duration: 1000, - }, + repeat: { cron: '55 * * * *' }, + removeOnComplete: true, }); - const relationshipLogger = this.logger.createSubLogger('relationship'); - - this.relationshipQueueWorker - .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) - .on('error', (err: Error) => relationshipLogger.error(`error ${err}`, { e: renderError(err) })) - .on('stalled', (jobId) => relationshipLogger.warn(`stalled id=${jobId}`)); - //#endregion + this.queueService.systemQueue.add('resyncCharts', { + }, { + repeat: { cron: '0 0 * * *' }, + removeOnComplete: true, + }); - //#region object storage - this.objectStorageQueueWorker = new Bull.Worker(QUEUE.OBJECT_STORAGE, (job) => { - switch (job.name) { - case 'deleteFile': return this.deleteFileProcessorService.process(job); - case 'cleanRemoteFiles': return this.cleanRemoteFilesProcessorService.process(job); - default: throw new Error(`unrecognized job type ${job.name} for objectStorage`); - } + this.queueService.systemQueue.add('cleanCharts', { }, { - ...baseQueueOptions(this.config, QUEUE.OBJECT_STORAGE), - autorun: false, - concurrency: 16, + repeat: { cron: '0 0 * * *' }, + removeOnComplete: true, }); - const objectStorageLogger = this.logger.createSubLogger('objectStorage'); + this.queueService.systemQueue.add('aggregateRetention', { + }, { + repeat: { cron: '0 0 * * *' }, + removeOnComplete: true, + }); - this.objectStorageQueueWorker - .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) - .on('error', (err: Error) => objectStorageLogger.error(`error ${err}`, { e: renderError(err) })) - .on('stalled', (jobId) => objectStorageLogger.warn(`stalled id=${jobId}`)); - //#endregion + this.queueService.systemQueue.add('clean', { + }, { + repeat: { cron: '0 0 * * *' }, + removeOnComplete: true, + }); - //#region ended poll notification - this.endedPollNotificationQueueWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => this.endedPollNotificationProcessorService.process(job), { - ...baseQueueOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION), - autorun: false, + this.queueService.systemQueue.add('checkExpiredMutings', { + }, { + repeat: { cron: '*/5 * * * *' }, + removeOnComplete: true, }); - //#endregion - } - @bindThis - public async start(): Promise { - await Promise.all([ - this.systemQueueWorker.run(), - this.dbQueueWorker.run(), - this.deliverQueueWorker.run(), - this.inboxQueueWorker.run(), - this.webhookDeliverQueueWorker.run(), - this.relationshipQueueWorker.run(), - this.objectStorageQueueWorker.run(), - this.endedPollNotificationQueueWorker.run(), - ]); + this.queueService.deliverQueue.process(this.config.deliverJobConcurrency ?? 128, (job) => this.deliverProcessorService.process(job)); + this.queueService.inboxQueue.process(this.config.inboxJobConcurrency ?? 16, (job) => this.inboxProcessorService.process(job)); + this.queueService.endedPollNotificationQueue.process((job, done) => this.endedPollNotificationProcessorService.process(job, done)); + this.queueService.webhookDeliverQueue.process(64, (job) => this.webhookDeliverProcessorService.process(job)); + + this.queueService.dbQueue.process('deleteDriveFiles', (job, done) => this.deleteDriveFilesProcessorService.process(job, done)); + this.queueService.dbQueue.process('exportCustomEmojis', (job, done) => this.exportCustomEmojisProcessorService.process(job, done)); + this.queueService.dbQueue.process('exportNotes', (job, done) => this.exportNotesProcessorService.process(job, done)); + this.queueService.dbQueue.process('exportFavorites', (job, done) => this.exportFavoritesProcessorService.process(job, done)); + this.queueService.dbQueue.process('exportFollowing', (job, done) => this.exportFollowingProcessorService.process(job, done)); + this.queueService.dbQueue.process('exportMuting', (job, done) => this.exportMutingProcessorService.process(job, done)); + this.queueService.dbQueue.process('exportBlocking', (job, done) => this.exportBlockingProcessorService.process(job, done)); + this.queueService.dbQueue.process('exportUserLists', (job, done) => this.exportUserListsProcessorService.process(job, done)); + this.queueService.dbQueue.process('exportAntennas', (job, done) => this.exportAntennasProcessorService.process(job, done)); + this.queueService.dbQueue.process('importFollowing', (job, done) => this.importFollowingProcessorService.process(job, done)); + this.queueService.dbQueue.process('importFollowingToDb', (job) => this.importFollowingProcessorService.processDb(job)); + this.queueService.dbQueue.process('importMuting', (job, done) => this.importMutingProcessorService.process(job, done)); + this.queueService.dbQueue.process('importBlocking', (job, done) => this.importBlockingProcessorService.process(job, done)); + this.queueService.dbQueue.process('importBlockingToDb', (job) => this.importBlockingProcessorService.processDb(job)); + this.queueService.dbQueue.process('importUserLists', (job, done) => this.importUserListsProcessorService.process(job, done)); + this.queueService.dbQueue.process('importCustomEmojis', (job, done) => this.importCustomEmojisProcessorService.process(job, done)); + this.queueService.dbQueue.process('importAntennas', (job, done) => this.importAntennasProcessorService.process(job, done)); + this.queueService.dbQueue.process('deleteAccount', (job) => this.deleteAccountProcessorService.process(job)); + + this.queueService.objectStorageQueue.process('deleteFile', 16, (job) => this.deleteFileProcessorService.process(job)); + this.queueService.objectStorageQueue.process('cleanRemoteFiles', 16, (job, done) => this.cleanRemoteFilesProcessorService.process(job, done)); + + { + const maxJobs = this.config.relashionshipJobConcurrency ?? 16; + this.queueService.relationshipQueue.process('follow', maxJobs, (job) => this.relationshipProcessorService.processFollow(job)); + this.queueService.relationshipQueue.process('unfollow', maxJobs, (job) => this.relationshipProcessorService.processUnfollow(job)); + this.queueService.relationshipQueue.process('block', maxJobs, (job) => this.relationshipProcessorService.processBlock(job)); + this.queueService.relationshipQueue.process('unblock', maxJobs, (job) => this.relationshipProcessorService.processUnblock(job)); + } + + this.queueService.systemQueue.process('tickCharts', (job, done) => this.tickChartsProcessorService.process(job, done)); + this.queueService.systemQueue.process('resyncCharts', (job, done) => this.resyncChartsProcessorService.process(job, done)); + this.queueService.systemQueue.process('cleanCharts', (job, done) => this.cleanChartsProcessorService.process(job, done)); + this.queueService.systemQueue.process('aggregateRetention', (job, done) => this.aggregateRetentionProcessorService.process(job, done)); + this.queueService.systemQueue.process('checkExpiredMutings', (job, done) => this.checkExpiredMutingsProcessorService.process(job, done)); + this.queueService.systemQueue.process('clean', (job, done) => this.cleanProcessorService.process(job, done)); } @bindThis public async stop(): Promise { await Promise.all([ - this.systemQueueWorker.close(), - this.dbQueueWorker.close(), - this.deliverQueueWorker.close(), - this.inboxQueueWorker.close(), - this.webhookDeliverQueueWorker.close(), - this.relationshipQueueWorker.close(), - this.objectStorageQueueWorker.close(), - this.endedPollNotificationQueueWorker.close(), + this.queueService.systemQueue.close(false), + this.queueService.endedPollNotificationQueue.close(false), + this.queueService.deliverQueue.close(false), + this.queueService.inboxQueue.close(false), + this.queueService.dbQueue.close(false), + this.queueService.relationshipQueue.close(false), + this.queueService.objectStorageQueue.close(false), + this.queueService.webhookDeliverQueue.close(false), ]); } diff --git a/packages/backend/src/queue/const.ts b/packages/backend/src/queue/const.ts deleted file mode 100644 index d240fe70e0c5..000000000000 --- a/packages/backend/src/queue/const.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Config } from '@/config.js'; -import type * as Bull from 'bullmq'; - -export const QUEUE = { - DELIVER: 'deliver', - INBOX: 'inbox', - SYSTEM: 'system', - ENDED_POLL_NOTIFICATION: 'endedPollNotification', - DB: 'db', - RELATIONSHIP: 'relationship', - OBJECT_STORAGE: 'objectStorage', - WEBHOOK_DELIVER: 'webhookDeliver', -}; - -export function baseQueueOptions(config: Config, queueName: typeof QUEUE[keyof typeof QUEUE]): Bull.QueueOptions { - return { - connection: { - port: config.redisForJobQueue.port, - host: config.redisForJobQueue.host, - family: config.redisForJobQueue.family == null ? 0 : config.redisForJobQueue.family, - password: config.redisForJobQueue.pass, - db: config.redisForJobQueue.db ?? 0, - }, - prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue:${queueName}` : `queue:${queueName}`, - }; -} diff --git a/packages/backend/src/queue/get-job-info.ts b/packages/backend/src/queue/get-job-info.ts new file mode 100644 index 000000000000..d33e349c363c --- /dev/null +++ b/packages/backend/src/queue/get-job-info.ts @@ -0,0 +1,15 @@ +import Bull from 'bull'; + +export function getJobInfo(job: Bull.Job, increment = false) { + const age = Date.now() - job.timestamp; + + const formated = age > 60000 ? `${Math.floor(age / 1000 / 60)}m` + : age > 10000 ? `${Math.floor(age / 1000)}s` + : `${age}ms`; + + // onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする + const currentAttempts = job.attemptsMade + (increment ? 1 : 0); + const maxAttempts = job.opts ? job.opts.attempts : 0; + + return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`; +} diff --git a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts index 600ce0828f91..e2720b4fe01e 100644 --- a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts +++ b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts @@ -9,7 +9,7 @@ import { deepClone } from '@/misc/clone.js'; import { IdService } from '@/core/IdService.js'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; @Injectable() export class AggregateRetentionProcessorService { @@ -32,7 +32,7 @@ export class AggregateRetentionProcessorService { } @bindThis - public async process(): Promise { + public async process(job: Bull.Job>, done: () => void): Promise { this.logger.info('Aggregating retention...'); const now = new Date(); @@ -62,6 +62,7 @@ export class AggregateRetentionProcessorService { } catch (err) { if (isDuplicateKeyValueError(err)) { this.logger.succ('Skip because it has already been processed by another worker.'); + done(); return; } throw err; @@ -87,5 +88,6 @@ export class AggregateRetentionProcessorService { } this.logger.succ('Retention aggregated.'); + done(); } } diff --git a/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts b/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts index c4ee212bab73..2476d71a5ebc 100644 --- a/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts +++ b/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts @@ -7,7 +7,7 @@ import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { UserMutingService } from '@/core/UserMutingService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; @Injectable() export class CheckExpiredMutingsProcessorService { @@ -27,7 +27,7 @@ export class CheckExpiredMutingsProcessorService { } @bindThis - public async process(): Promise { + public async process(job: Bull.Job>, done: () => void): Promise { this.logger.info('Checking expired mutings...'); const expired = await this.mutingsRepository.createQueryBuilder('muting') @@ -41,5 +41,6 @@ export class CheckExpiredMutingsProcessorService { } this.logger.succ('All expired mutings checked.'); + done(); } } diff --git a/packages/backend/src/queue/processors/CleanChartsProcessorService.ts b/packages/backend/src/queue/processors/CleanChartsProcessorService.ts index 22d7c1b4fb2d..b45816704207 100644 --- a/packages/backend/src/queue/processors/CleanChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanChartsProcessorService.ts @@ -16,7 +16,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; @Injectable() export class CleanChartsProcessorService { @@ -45,7 +45,7 @@ export class CleanChartsProcessorService { } @bindThis - public async process(): Promise { + public async process(job: Bull.Job>, done: () => void): Promise { this.logger.info('Clean charts...'); await Promise.all([ @@ -64,5 +64,6 @@ export class CleanChartsProcessorService { ]); this.logger.succ('All charts successfully cleaned.'); + done(); } } diff --git a/packages/backend/src/queue/processors/CleanProcessorService.ts b/packages/backend/src/queue/processors/CleanProcessorService.ts index cefa6da5e946..1936e8df23ee 100644 --- a/packages/backend/src/queue/processors/CleanProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanProcessorService.ts @@ -7,7 +7,7 @@ import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; @Injectable() export class CleanProcessorService { @@ -36,7 +36,7 @@ export class CleanProcessorService { } @bindThis - public async process(): Promise { + public async process(job: Bull.Job>, done: () => void): Promise { this.logger.info('Cleaning...'); this.userIpsRepository.delete({ @@ -72,5 +72,6 @@ export class CleanProcessorService { } this.logger.succ('Cleaned.'); + done(); } } diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index c54bf59ae4c2..5a33c271889c 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -5,9 +5,9 @@ import type { DriveFilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; -import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; +import { bindThis } from '@/decorators.js'; @Injectable() export class CleanRemoteFilesProcessorService { @@ -27,7 +27,7 @@ export class CleanRemoteFilesProcessorService { } @bindThis - public async process(job: Bull.Job>): Promise { + public async process(job: Bull.Job>, done: () => void): Promise { this.logger.info('Deleting cached remote files...'); let deletedCount = 0; @@ -47,7 +47,7 @@ export class CleanRemoteFilesProcessorService { }); if (files.length === 0) { - job.updateProgress(100); + job.progress(100); break; } @@ -62,9 +62,10 @@ export class CleanRemoteFilesProcessorService { isLink: false, }); - job.updateProgress(deletedCount / total); + job.progress(deletedCount / total); } this.logger.succ('All cached remote files has been deleted.'); + done(); } } diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index 39dd801af017..e36a78de6aa2 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -8,10 +8,10 @@ import { DriveService } from '@/core/DriveService.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Note } from '@/models/entities/Note.js'; import { EmailService } from '@/core/EmailService.js'; -import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbUserDeleteJobData } from '../types.js'; +import { bindThis } from '@/decorators.js'; @Injectable() export class DeleteAccountProcessorService { diff --git a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts index 6772c5dc7662..604497cf5425 100644 --- a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts @@ -5,10 +5,10 @@ import type { UsersRepository, DriveFilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; -import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbJobDataWithUser } from '../types.js'; +import { bindThis } from '@/decorators.js'; @Injectable() export class DeleteDriveFilesProcessorService { @@ -31,11 +31,12 @@ export class DeleteDriveFilesProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info(`Deleting drive files of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } @@ -55,7 +56,7 @@ export class DeleteDriveFilesProcessorService { }); if (files.length === 0) { - job.updateProgress(100); + job.progress(100); break; } @@ -70,9 +71,10 @@ export class DeleteDriveFilesProcessorService { userId: user.id, }); - job.updateProgress(deletedCount / total); + job.progress(deletedCount / total); } this.logger.succ(`All drive files (${deletedCount}) of ${user.id} has been deleted.`); + done(); } } diff --git a/packages/backend/src/queue/processors/DeleteFileProcessorService.ts b/packages/backend/src/queue/processors/DeleteFileProcessorService.ts index edf87bd92174..2fb2f56f8def 100644 --- a/packages/backend/src/queue/processors/DeleteFileProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteFileProcessorService.ts @@ -3,10 +3,10 @@ import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; -import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { ObjectStorageFileJobData } from '../types.js'; +import { bindThis } from '@/decorators.js'; @Injectable() export class DeleteFileProcessorService { diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 406e9df850d8..380671302d69 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -1,5 +1,4 @@ import { Inject, Injectable } from '@nestjs/common'; -import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository, InstancesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; @@ -17,6 +16,7 @@ import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; +import type Bull from 'bull'; import type { DeliverJobData } from '../types.js'; @Injectable() @@ -121,9 +121,11 @@ export class DeliverProcessorService { isSuspended: true, }); }); - throw new Bull.UnrecoverableError(`${host} is gone`); + return `${host} is gone`; } - throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); + // HTTPステータスコード4xxはクライアントエラーであり、それはつまり + // 何回再送しても成功することはないということなのでエラーにはしないでおく + return `${res.statusCode} ${res.statusMessage}`; } // 5xx etc. diff --git a/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts b/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts index 21501592f2cd..501ed4090a73 100644 --- a/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts +++ b/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts @@ -6,7 +6,7 @@ import type Logger from '@/logger.js'; import { NotificationService } from '@/core/NotificationService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { EndedPollNotificationJobData } from '../types.js'; @Injectable() @@ -30,9 +30,10 @@ export class EndedPollNotificationProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { const note = await this.notesRepository.findOneBy({ id: job.data.noteId }); if (note == null || !note.hasPoll) { + done(); return; } @@ -50,5 +51,7 @@ export class EndedPollNotificationProcessorService { noteId: note.id, }); } + + done(); } } diff --git a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts index ac52325c8d99..894903e79b10 100644 --- a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts @@ -12,7 +12,7 @@ import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { DBExportAntennasData } from '../types.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; @Injectable() export class ExportAntennasProcessorService { @@ -39,9 +39,10 @@ export class ExportAntennasProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } const [path, cleanup] = await createTemp(); @@ -95,6 +96,7 @@ export class ExportAntennasProcessorService { this.logger.succ('Exported to: ' + driveFile.id); } finally { cleanup(); + done(); } } } diff --git a/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts b/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts index eb758e162dfc..c7b54070d6a7 100644 --- a/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts @@ -9,10 +9,10 @@ import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbJobDataWithUser } from '../types.js'; +import { bindThis } from '@/decorators.js'; @Injectable() export class ExportBlockingProcessorService { @@ -36,11 +36,12 @@ export class ExportBlockingProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info(`Exporting blocking of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } @@ -68,7 +69,7 @@ export class ExportBlockingProcessorService { }); if (blockings.length === 0) { - job.updateProgress(100); + job.progress(100); break; } @@ -98,7 +99,7 @@ export class ExportBlockingProcessorService { blockerId: user.id, }); - job.updateProgress(exportedCount / total); + job.progress(exportedCount / total); } stream.end(); @@ -111,5 +112,7 @@ export class ExportBlockingProcessorService { } finally { cleanup(); } + + done(); } } diff --git a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts index 3203d9f3e5c9..b50f373ef820 100644 --- a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts @@ -13,7 +13,7 @@ import { createTemp, createTempDir } from '@/misc/create-temp.js'; import { DownloadService } from '@/core/DownloadService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; @Injectable() export class ExportCustomEmojisProcessorService { @@ -37,11 +37,12 @@ export class ExportCustomEmojisProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info('Exporting custom emojis ...'); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } @@ -116,26 +117,24 @@ export class ExportCustomEmojisProcessorService { metaStream.end(); // Create archive - await new Promise(async (resolve) => { - const [archivePath, archiveCleanup] = await createTemp(); - const archiveStream = fs.createWriteStream(archivePath); - const archive = archiver('zip', { - zlib: { level: 0 }, - }); - archiveStream.on('close', async () => { - this.logger.succ(`Exported to: ${archivePath}`); + const [archivePath, archiveCleanup] = await createTemp(); + const archiveStream = fs.createWriteStream(archivePath); + const archive = archiver('zip', { + zlib: { level: 0 }, + }); + archiveStream.on('close', async () => { + this.logger.succ(`Exported to: ${archivePath}`); - const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; - const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); + const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; + const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); - this.logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); - archiveCleanup(); - resolve(); - }); - archive.pipe(archiveStream); - archive.directory(path, false); - archive.finalize(); + this.logger.succ(`Exported to: ${driveFile.id}`); + cleanup(); + archiveCleanup(); + done(); }); + archive.pipe(archiveStream); + archive.directory(path, false); + archive.finalize(); } } diff --git a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts index 76c38a6b8648..f2f2383a8837 100644 --- a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts @@ -12,7 +12,7 @@ import type { Poll } from '@/models/entities/Poll.js'; import type { Note } from '@/models/entities/Note.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbJobDataWithUser } from '../types.js'; @Injectable() @@ -42,11 +42,12 @@ export class ExportFavoritesProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info(`Exporting favorites of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } @@ -90,7 +91,7 @@ export class ExportFavoritesProcessorService { }) as (NoteFavorite & { note: Note & { user: User } })[]; if (favorites.length === 0) { - job.updateProgress(100); + job.progress(100); break; } @@ -111,7 +112,7 @@ export class ExportFavoritesProcessorService { userId: user.id, }); - job.updateProgress(exportedFavoritesCount / total); + job.progress(exportedFavoritesCount / total); } await write(']'); @@ -126,6 +127,8 @@ export class ExportFavoritesProcessorService { } finally { cleanup(); } + + done(); } } diff --git a/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts b/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts index 8726cb140278..fa9c1ac1ea94 100644 --- a/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts @@ -10,10 +10,10 @@ import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import type { Following } from '@/models/entities/Following.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbExportFollowingData } from '../types.js'; +import { bindThis } from '@/decorators.js'; @Injectable() export class ExportFollowingProcessorService { @@ -40,11 +40,12 @@ export class ExportFollowingProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info(`Exporting following of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } @@ -115,5 +116,7 @@ export class ExportFollowingProcessorService { } finally { cleanup(); } + + done(); } } diff --git a/packages/backend/src/queue/processors/ExportMutingProcessorService.ts b/packages/backend/src/queue/processors/ExportMutingProcessorService.ts index 0f11a9e8435f..b14bf5f5b18a 100644 --- a/packages/backend/src/queue/processors/ExportMutingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportMutingProcessorService.ts @@ -9,10 +9,10 @@ import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbJobDataWithUser } from '../types.js'; +import { bindThis } from '@/decorators.js'; @Injectable() export class ExportMutingProcessorService { @@ -39,11 +39,12 @@ export class ExportMutingProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info(`Exporting muting of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } @@ -72,7 +73,7 @@ export class ExportMutingProcessorService { }); if (mutes.length === 0) { - job.updateProgress(100); + job.progress(100); break; } @@ -102,7 +103,7 @@ export class ExportMutingProcessorService { muterId: user.id, }); - job.updateProgress(exportedCount / total); + job.progress(exportedCount / total); } stream.end(); @@ -115,5 +116,7 @@ export class ExportMutingProcessorService { } finally { cleanup(); } + + done(); } } diff --git a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts index 24fb33188373..e4f12ad10105 100644 --- a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts @@ -12,7 +12,7 @@ import type { Poll } from '@/models/entities/Poll.js'; import type { Note } from '@/models/entities/Note.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbJobDataWithUser } from '../types.js'; @Injectable() @@ -39,11 +39,12 @@ export class ExportNotesProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info(`Exporting notes of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } @@ -86,7 +87,7 @@ export class ExportNotesProcessorService { }) as Note[]; if (notes.length === 0) { - job.updateProgress(100); + job.progress(100); break; } @@ -107,7 +108,7 @@ export class ExportNotesProcessorService { userId: user.id, }); - job.updateProgress(exportedNotesCount / total); + job.progress(exportedNotesCount / total); } await write(']'); @@ -122,6 +123,8 @@ export class ExportNotesProcessorService { } finally { cleanup(); } + + done(); } } diff --git a/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts b/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts index ec633580535a..54bde44044de 100644 --- a/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts @@ -9,10 +9,10 @@ import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbJobDataWithUser } from '../types.js'; +import { bindThis } from '@/decorators.js'; @Injectable() export class ExportUserListsProcessorService { @@ -39,11 +39,12 @@ export class ExportUserListsProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info(`Exporting user lists of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } @@ -91,5 +92,7 @@ export class ExportUserListsProcessorService { } finally { cleanup(); } + + done(); } } diff --git a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts index 575cad69d5c4..d06131b8c81b 100644 --- a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts @@ -8,7 +8,7 @@ import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import { DBAntennaImportJobData } from '../types.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; const validate = new Ajv().compile({ type: 'object', @@ -59,7 +59,7 @@ export class ImportAntennasProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { const now = new Date(); try { for (const antenna of job.data.antenna) { @@ -89,6 +89,8 @@ export class ImportAntennasProcessorService { } } catch (err: any) { this.logger.error(err); + } finally { + done(); } } } diff --git a/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts b/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts index 2f1a9e5b03a1..c34ab9e36ed3 100644 --- a/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts @@ -7,11 +7,11 @@ import * as Acct from '@/misc/acct.js'; import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { bindThis } from '@/decorators.js'; -import { QueueService } from '@/core/QueueService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; +import { bindThis } from '@/decorators.js'; +import { QueueService } from '@/core/QueueService.js'; @Injectable() export class ImportBlockingProcessorService { @@ -34,11 +34,12 @@ export class ImportBlockingProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info(`Importing blocking of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } @@ -46,6 +47,7 @@ export class ImportBlockingProcessorService { id: job.data.fileId, }); if (file == null) { + done(); return; } @@ -54,6 +56,7 @@ export class ImportBlockingProcessorService { this.queueService.createImportBlockingToDbJob({ id: user.id }, targets); this.logger.succ('Import jobs created'); + done(); } @bindThis diff --git a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts index d86256787157..600468a286ad 100644 --- a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts @@ -12,7 +12,7 @@ import { DriveService } from '@/core/DriveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbUserImportJobData } from '../types.js'; // TODO: 名前衝突時の動作を選べるようにする @@ -45,13 +45,14 @@ export class ImportCustomEmojisProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info('Importing custom emojis ...'); const file = await this.driveFilesRepository.findOneBy({ id: job.data.fileId, }); if (file == null) { + done(); return; } @@ -115,6 +116,7 @@ export class ImportCustomEmojisProcessorService { cleanup(); this.logger.succ('Imported'); + done(); }); unzipStream.pipe(extractor); this.logger.succ(`Unzipping to ${outputPath}`); diff --git a/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts b/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts index 15bee9672eac..2f2e9ada99b3 100644 --- a/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts @@ -7,11 +7,11 @@ import * as Acct from '@/misc/acct.js'; import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { bindThis } from '@/decorators.js'; -import { QueueService } from '@/core/QueueService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; +import { bindThis } from '@/decorators.js'; +import { QueueService } from '@/core/QueueService.js'; @Injectable() export class ImportFollowingProcessorService { @@ -34,11 +34,12 @@ export class ImportFollowingProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info(`Importing following of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } @@ -46,6 +47,7 @@ export class ImportFollowingProcessorService { id: job.data.fileId, }); if (file == null) { + done(); return; } @@ -54,6 +56,7 @@ export class ImportFollowingProcessorService { this.queueService.createImportFollowingToDbJob({ id: user.id }, targets); this.logger.succ('Import jobs created'); + done(); } @bindThis diff --git a/packages/backend/src/queue/processors/ImportMutingProcessorService.ts b/packages/backend/src/queue/processors/ImportMutingProcessorService.ts index 723935cd3113..d9ca09617633 100644 --- a/packages/backend/src/queue/processors/ImportMutingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportMutingProcessorService.ts @@ -9,10 +9,10 @@ import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UserMutingService } from '@/core/UserMutingService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbUserImportJobData } from '../types.js'; +import { bindThis } from '@/decorators.js'; @Injectable() export class ImportMutingProcessorService { @@ -38,11 +38,12 @@ export class ImportMutingProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info(`Importing muting of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } @@ -50,6 +51,7 @@ export class ImportMutingProcessorService { id: job.data.fileId, }); if (file == null) { + done(); return; } @@ -96,5 +98,6 @@ export class ImportMutingProcessorService { } this.logger.succ('Imported'); + done(); } } diff --git a/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts b/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts index 824ee8157ab1..c423863410fe 100644 --- a/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts @@ -12,7 +12,7 @@ import { IdService } from '@/core/IdService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import type { DbUserImportJobData } from '../types.js'; @Injectable() @@ -46,11 +46,12 @@ export class ImportUserListsProcessorService { } @bindThis - public async process(job: Bull.Job): Promise { + public async process(job: Bull.Job, done: () => void): Promise { this.logger.info(`Importing user lists of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + done(); return; } @@ -58,6 +59,7 @@ export class ImportUserListsProcessorService { id: job.data.fileId, }); if (file == null) { + done(); return; } @@ -107,5 +109,6 @@ export class ImportUserListsProcessorService { } this.logger.succ('Imported'); + done(); } } diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index ce1d7aaa1bfc..535a806d7f9c 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -1,8 +1,8 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import httpSignature from '@peertube/http-signature'; -import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; +import type { InstancesRepository, DriveFilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { MetaService } from '@/core/MetaService.js'; @@ -23,8 +23,10 @@ import { LdSignatureService } from '@/core/activitypub/LdSignatureService.js'; import { ApInboxService } from '@/core/activitypub/ApInboxService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; +import type Bull from 'bull'; import type { InboxJobData } from '../types.js'; +// ユーザーのinboxにアクティビティが届いた時の処理 @Injectable() export class InboxProcessorService { private logger: Logger; @@ -33,6 +35,12 @@ export class InboxProcessorService { @Inject(DI.config) private config: Config, + @Inject(DI.instancesRepository) + private instancesRepository: InstancesRepository, + + @Inject(DI.driveFilesRepository) + private driveFilesRepository: DriveFilesRepository, + private utilityService: UtilityService, private metaService: MetaService, private apInboxService: ApInboxService, @@ -85,10 +93,10 @@ export class InboxProcessorService { try { authUser = await this.apDbResolverService.getAuthUserFromApId(getApId(activity.actor)); } catch (err) { - // 対象が4xxならスキップ + // 対象が4xxならスキップ if (err instanceof StatusError) { if (err.isClientError) { - throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`); + return `skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`; } throw new Error(`Error in actor ${activity.actor} - ${err.statusCode ?? err}`); } @@ -97,12 +105,12 @@ export class InboxProcessorService { // それでもわからなければ終了 if (authUser == null) { - throw new Bull.UnrecoverableError('skip: failed to resolve user'); + return 'skip: failed to resolve user'; } // publicKey がなくても終了 if (authUser.key == null) { - throw new Bull.UnrecoverableError('skip: failed to resolve user publicKey'); + return 'skip: failed to resolve user publicKey'; } // HTTP-Signatureの検証 @@ -110,10 +118,10 @@ export class InboxProcessorService { // また、signatureのsignerは、activity.actorと一致する必要がある if (!httpSignatureValidated || authUser.user.uri !== activity.actor) { - // 一致しなくても、でもLD-Signatureがありそうならそっちも見る + // 一致しなくても、でもLD-Signatureがありそうならそっちも見る if (activity.signature) { if (activity.signature.type !== 'RsaSignature2017') { - throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${activity.signature.type}`); + return `skip: unsupported LD-signature type ${activity.signature.type}`; } // activity.signature.creator: https://example.oom/users/user#main-key @@ -126,32 +134,32 @@ export class InboxProcessorService { // keyIdからLD-Signatureのユーザーを取得 authUser = await this.apDbResolverService.getAuthUserFromKeyId(activity.signature.creator); if (authUser == null) { - throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした'); + return 'skip: LD-Signatureのユーザーが取得できませんでした'; } if (authUser.key == null) { - throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'); + return 'skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'; } // LD-Signature検証 const ldSignature = this.ldSignatureService.use(); const verified = await ldSignature.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); if (!verified) { - throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました'); + return 'skip: LD-Signatureの検証に失敗しました'; } // もう一度actorチェック if (authUser.user.uri !== activity.actor) { - throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`); + return `skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`; } // ブロックしてたら中断 const ldHost = this.utilityService.extractDbHost(authUser.user.uri); if (this.utilityService.isBlockedHost(meta.blockedHosts, ldHost)) { - throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`); + return `Blocked request: ${ldHost}`; } } else { - throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`); + return `skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`; } } @@ -160,7 +168,7 @@ export class InboxProcessorService { const signerHost = this.utilityService.extractDbHost(authUser.user.uri!); const activityIdHost = this.utilityService.extractDbHost(activity.id); if (signerHost !== activityIdHost) { - throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`); + return `skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`; } } diff --git a/packages/backend/src/queue/processors/RelationshipProcessorService.ts b/packages/backend/src/queue/processors/RelationshipProcessorService.ts index 722260d94821..ff454df45550 100644 --- a/packages/backend/src/queue/processors/RelationshipProcessorService.ts +++ b/packages/backend/src/queue/processors/RelationshipProcessorService.ts @@ -1,5 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; diff --git a/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts b/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts index eab8e1e68d70..e5840f3da8c8 100644 --- a/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts @@ -15,7 +15,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; @Injectable() export class ResyncChartsProcessorService { @@ -43,7 +43,7 @@ export class ResyncChartsProcessorService { } @bindThis - public async process(): Promise { + public async process(job: Bull.Job>, done: () => void): Promise { this.logger.info('Resync charts...'); // TODO: ユーザーごとのチャートも更新する @@ -55,5 +55,6 @@ export class ResyncChartsProcessorService { ]); this.logger.succ('All charts successfully resynced.'); + done(); } } diff --git a/packages/backend/src/queue/processors/TickChartsProcessorService.ts b/packages/backend/src/queue/processors/TickChartsProcessorService.ts index f1696bf56775..7ff84c15a5c0 100644 --- a/packages/backend/src/queue/processors/TickChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/TickChartsProcessorService.ts @@ -16,7 +16,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type * as Bull from 'bullmq'; +import type Bull from 'bull'; @Injectable() export class TickChartsProcessorService { @@ -45,7 +45,7 @@ export class TickChartsProcessorService { } @bindThis - public async process(): Promise { + public async process(job: Bull.Job>, done: () => void): Promise { this.logger.info('Tick charts...'); await Promise.all([ @@ -64,5 +64,6 @@ export class TickChartsProcessorService { ]); this.logger.succ('All charts successfully ticked.'); + done(); } } diff --git a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts index 8b40c167496d..a96301f5bece 100644 --- a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts @@ -1,5 +1,4 @@ import { Inject, Injectable } from '@nestjs/common'; -import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { WebhooksRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; @@ -8,6 +7,7 @@ import { HttpRequestService } from '@/core/HttpRequestService.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; +import type Bull from 'bull'; import type { WebhookDeliverJobData } from '../types.js'; @Injectable() @@ -66,7 +66,7 @@ export class WebhookDeliverProcessorService { if (res instanceof StatusError) { // 4xx if (res.isClientError) { - throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); + return `${res.statusCode} ${res.statusMessage}`; } // 5xx etc. diff --git a/packages/frontend/src/scripts/time.ts b/packages/frontend/src/scripts/time.ts index b21978b18689..0da1f791395d 100644 --- a/packages/frontend/src/scripts/time.ts +++ b/packages/frontend/src/scripts/time.ts @@ -5,14 +5,13 @@ const dateTimeIntervals = { }; export function dateUTC(time: number[]): Date { - const d = - time.length === 2 ? Date.UTC(time[0], time[1]) - : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) - : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) - : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) - : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) - : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) - : null; + const d = time.length === 2 ? Date.UTC(time[0], time[1]) + : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) + : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) + : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) + : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) + : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) + : null; if (!d) throw new Error('wrong number of arguments'); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f48fac3f479c..063ec0ed10ae 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -149,9 +149,9 @@ importers: blurhash: specifier: 2.0.5 version: 2.0.5 - bullmq: - specifier: 3.15.0 - version: 3.15.0 + bull: + specifier: 4.10.4 + version: 4.10.4 cacheable-lookup: specifier: 6.1.0 version: 6.1.0 @@ -2113,7 +2113,7 @@ packages: '@babel/traverse': 7.22.4 '@babel/types': 7.22.4 convert-source-map: 1.9.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.0 @@ -2136,7 +2136,7 @@ packages: '@babel/traverse': 7.22.4 '@babel/types': 7.22.4 convert-source-map: 1.9.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.0 @@ -4322,7 +4322,7 @@ packages: '@babel/helper-split-export-declaration': 7.18.6 '@babel/parser': 7.22.4 '@babel/types': 7.22.4 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -4751,7 +4751,7 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dependencies: ajv: 6.12.6 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 espree: 9.5.2 globals: 13.19.0 ignore: 5.2.4 @@ -4916,7 +4916,7 @@ packages: engines: {node: '>=10.10.0'} dependencies: '@humanwhocodes/object-schema': 1.2.1 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -5342,48 +5342,48 @@ packages: os-filter-obj: 2.0.0 dev: false - /@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2: - resolution: {integrity: sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==} + /@msgpackr-extract/msgpackr-extract-darwin-arm64@2.2.0: + resolution: {integrity: sha512-Z9LFPzfoJi4mflGWV+rv7o7ZbMU5oAU9VmzCgL240KnqDW65Y2HFCT3MW06/ITJSnbVLacmcEJA8phywK7JinQ==} cpu: [arm64] os: [darwin] requiresBuild: true dev: false optional: true - /@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.2: - resolution: {integrity: sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==} + /@msgpackr-extract/msgpackr-extract-darwin-x64@2.2.0: + resolution: {integrity: sha512-vq0tT8sjZsy4JdSqmadWVw6f66UXqUCabLmUVHZwUFzMgtgoIIQjT4VVRHKvlof3P/dMCkbMJ5hB1oJ9OWHaaw==} cpu: [x64] os: [darwin] requiresBuild: true dev: false optional: true - /@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.2: - resolution: {integrity: sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==} + /@msgpackr-extract/msgpackr-extract-linux-arm64@2.2.0: + resolution: {integrity: sha512-hlxxLdRmPyq16QCutUtP8Tm6RDWcyaLsRssaHROatgnkOxdleMTgetf9JsdncL8vLh7FVy/RN9i3XR5dnb9cRA==} cpu: [arm64] os: [linux] requiresBuild: true dev: false optional: true - /@msgpackr-extract/msgpackr-extract-linux-arm@3.0.2: - resolution: {integrity: sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==} + /@msgpackr-extract/msgpackr-extract-linux-arm@2.2.0: + resolution: {integrity: sha512-SaJ3Qq4lX9Syd2xEo9u3qPxi/OB+5JO/ngJKK97XDpa1C587H9EWYO6KD8995DAjSinWvdHKRrCOXVUC5fvGOg==} cpu: [arm] os: [linux] requiresBuild: true dev: false optional: true - /@msgpackr-extract/msgpackr-extract-linux-x64@3.0.2: - resolution: {integrity: sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==} + /@msgpackr-extract/msgpackr-extract-linux-x64@2.2.0: + resolution: {integrity: sha512-94y5PJrSOqUNcFKmOl7z319FelCLAE0rz/jPCWS+UtdMZvpa4jrQd+cJPQCLp2Fes1yAW/YUQj/Di6YVT3c3Iw==} cpu: [x64] os: [linux] requiresBuild: true dev: false optional: true - /@msgpackr-extract/msgpackr-extract-win32-x64@3.0.2: - resolution: {integrity: sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==} + /@msgpackr-extract/msgpackr-extract-win32-x64@2.2.0: + resolution: {integrity: sha512-XrC0JzsqQSvOyM3t04FMLO6z5gCuhPE6k4FXuLK5xf52ZbdvcFe1yBmo7meCew9B8G2f0T9iu9t3kfTYRYROgA==} cpu: [x64] os: [win32] requiresBuild: true @@ -8156,7 +8156,7 @@ packages: '@typescript-eslint/scope-manager': 5.59.8 '@typescript-eslint/type-utils': 5.59.8(eslint@8.41.0)(typescript@5.1.3) '@typescript-eslint/utils': 5.59.8(eslint@8.41.0)(typescript@5.1.3) - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 eslint: 8.41.0 grapheme-splitter: 1.0.4 ignore: 5.2.4 @@ -8201,7 +8201,7 @@ packages: '@typescript-eslint/scope-manager': 5.59.8 '@typescript-eslint/types': 5.59.8 '@typescript-eslint/typescript-estree': 5.59.8(typescript@5.1.3) - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 eslint: 8.41.0 typescript: 5.1.3 transitivePeerDependencies: @@ -8256,7 +8256,7 @@ packages: dependencies: '@typescript-eslint/typescript-estree': 5.59.8(typescript@5.1.3) '@typescript-eslint/utils': 5.59.8(eslint@8.41.0)(typescript@5.1.3) - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 eslint: 8.41.0 tsutils: 3.21.0(typescript@5.1.3) typescript: 5.1.3 @@ -8306,7 +8306,7 @@ packages: dependencies: '@typescript-eslint/types': 5.59.8 '@typescript-eslint/visitor-keys': 5.59.8 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 globby: 11.1.0 is-glob: 4.0.3 semver: 7.5.1 @@ -8827,7 +8827,7 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 transitivePeerDependencies: - supports-color @@ -8835,7 +8835,7 @@ packages: resolution: {integrity: sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA==} engines: {node: '>= 8.0.0'} dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 depd: 1.1.2 humanize-ms: 1.2.1 transitivePeerDependencies: @@ -9315,7 +9315,7 @@ packages: resolution: {integrity: sha512-bbCQdg7bpEv6kGH41RO/3B2/GMMmJSo2iBK+X8AWN9mujtfUipMDfIjsgHCfpnKqoGEQrrmCDKSa5OQ19+fDmg==} dependencies: archy: 1.0.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 fastq: 1.15.0 transitivePeerDependencies: - supports-color @@ -9338,7 +9338,7 @@ packages: /axios@0.24.0: resolution: {integrity: sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==} dependencies: - follow-redirects: 1.15.2(debug@4.3.4) + follow-redirects: 1.15.2 transitivePeerDependencies: - debug dev: false @@ -9849,17 +9849,18 @@ packages: dependencies: node-gyp-build: 4.6.0 - /bullmq@3.15.0: - resolution: {integrity: sha512-U0LSRjuoyIBpnE62T4maCWMYEt3qdBCa1lnlPxYKQmRF/Y+FQ9W6iW5JvNNN+NA5Jet7k0uX71a93EX1zGnrhw==} + /bull@4.10.4: + resolution: {integrity: sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA==} + engines: {node: '>=12'} dependencies: - cron-parser: 4.8.1 - glob: 8.1.0 + cron-parser: 4.7.1 + debuglog: 1.0.1 + get-port: 5.1.1 ioredis: 5.3.2 lodash: 4.17.21 - msgpackr: 1.9.2 + msgpackr: 1.8.1 semver: 7.5.1 - tslib: 2.5.2 - uuid: 9.0.0 + uuid: 8.3.2 transitivePeerDependencies: - supports-color dev: false @@ -10749,11 +10750,11 @@ packages: readable-stream: 3.6.0 dev: false - /cron-parser@4.8.1: - resolution: {integrity: sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==} + /cron-parser@4.7.1: + resolution: {integrity: sha512-WguFaoQ0hQ61SgsCZLHUcNbAvlK0lypKXu62ARguefYmjzaOXIVRNrAmyXzabTwUn4sQvQLkk6bjH+ipGfw8bA==} engines: {node: '>=12.0.0'} dependencies: - luxon: 3.3.0 + luxon: 3.2.1 dev: false /cropperjs@2.0.0-beta.2: @@ -11006,6 +11007,16 @@ packages: dependencies: ms: 2.0.0 + /debug@3.2.7: + resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.3 + /debug@3.2.7(supports-color@8.1.1): resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} peerDependencies: @@ -11016,6 +11027,18 @@ packages: dependencies: ms: 2.1.3 supports-color: 8.1.1 + dev: true + + /debug@4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.2 /debug@4.3.4(supports-color@8.1.1): resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} @@ -11029,6 +11052,10 @@ packages: ms: 2.1.2 supports-color: 8.1.1 + /debuglog@1.0.1: + resolution: {integrity: sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==} + dev: false + /decamelize-keys@1.1.1: resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} engines: {node: '>=0.10.0'} @@ -11722,7 +11749,7 @@ packages: /eslint-import-resolver-node@0.3.7: resolution: {integrity: sha512-gozW2blMLJCeFpBwugLTGyvVjNoeo1knonXAcatC6bjPBZitotxdWf7Gimr25N4c0AAOo4eOUfaG82IJPDpqCA==} dependencies: - debug: 3.2.7(supports-color@8.1.1) + debug: 3.2.7 is-core-module: 2.11.0 resolve: 1.22.1 transitivePeerDependencies: @@ -11780,7 +11807,7 @@ packages: optional: true dependencies: '@typescript-eslint/parser': 5.59.8(eslint@8.41.0)(typescript@5.1.3) - debug: 3.2.7(supports-color@8.1.1) + debug: 3.2.7 eslint: 8.41.0 eslint-import-resolver-node: 0.3.7 transitivePeerDependencies: @@ -11834,7 +11861,7 @@ packages: array-includes: 3.1.6 array.prototype.flat: 1.3.1 array.prototype.flatmap: 1.3.1 - debug: 3.2.7(supports-color@8.1.1) + debug: 3.2.7 doctrine: 2.1.0 eslint: 8.41.0 eslint-import-resolver-node: 0.3.7 @@ -11960,7 +11987,7 @@ packages: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.0 @@ -12724,6 +12751,16 @@ packages: readable-stream: 2.3.7 dev: false + /follow-redirects@1.15.2: + resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + dev: false + /follow-redirects@1.15.2(debug@4.3.4): resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} engines: {node: '>=4.0'} @@ -12734,6 +12771,7 @@ packages: optional: true dependencies: debug: 4.3.4(supports-color@8.1.1) + dev: true /for-each@0.3.3: resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} @@ -13003,7 +13041,6 @@ packages: /get-port@5.1.1: resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==} engines: {node: '>=8'} - dev: true /get-stream@3.0.0: resolution: {integrity: sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==} @@ -13610,7 +13647,7 @@ packages: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 transitivePeerDependencies: - supports-color dev: false @@ -13660,7 +13697,7 @@ packages: engines: {node: '>= 4.5.0'} dependencies: agent-base: 4.3.0 - debug: 3.2.7(supports-color@8.1.1) + debug: 3.2.7 transitivePeerDependencies: - supports-color dev: false @@ -13681,7 +13718,7 @@ packages: engines: {node: '>= 6'} dependencies: agent-base: 6.0.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 transitivePeerDependencies: - supports-color @@ -13846,7 +13883,7 @@ packages: dependencies: '@ioredis/commands': 1.2.0 cluster-key-slot: 1.1.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 denque: 2.1.0 lodash.defaults: 4.2.0 lodash.isarguments: 3.1.0 @@ -14428,7 +14465,7 @@ packages: resolution: {integrity: sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==} engines: {node: '>=10'} dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 istanbul-lib-coverage: 3.2.0 source-map: 0.6.1 transitivePeerDependencies: @@ -15711,8 +15748,8 @@ packages: engines: {node: '>=16.14'} dev: true - /luxon@3.3.0: - resolution: {integrity: sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==} + /luxon@3.2.1: + resolution: {integrity: sha512-QrwPArQCNLAKGO/C+ZIilgIuDnEnKx5QYODdDtbFaxzsbZcc/a7WFq7MhsVYgRlwawLtvOUESTlfJ+hc/USqPg==} engines: {node: '>=12'} dev: false @@ -16190,26 +16227,25 @@ packages: engines: {node: '>=12.13'} dev: false - /msgpackr-extract@3.0.2: - resolution: {integrity: sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==} - hasBin: true + /msgpackr-extract@2.2.0: + resolution: {integrity: sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==} requiresBuild: true dependencies: - node-gyp-build-optional-packages: 5.0.7 + node-gyp-build-optional-packages: 5.0.3 optionalDependencies: - '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.2 - '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.2 - '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.2 - '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.2 - '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.2 - '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 2.2.0 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 2.2.0 + '@msgpackr-extract/msgpackr-extract-linux-arm': 2.2.0 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 2.2.0 + '@msgpackr-extract/msgpackr-extract-linux-x64': 2.2.0 + '@msgpackr-extract/msgpackr-extract-win32-x64': 2.2.0 dev: false optional: true - /msgpackr@1.9.2: - resolution: {integrity: sha512-xtDgI3Xv0AAiZWLRGDchyzBwU6aq0rwJ+W+5Y4CZhEWtkl/hJtFFLc+3JtGTw7nz1yquxs7nL8q/yA2aqpflIQ==} + /msgpackr@1.8.1: + resolution: {integrity: sha512-05fT4J8ZqjYlR4QcRDIhLCYKUOHXk7C/xa62GzMKj74l3up9k2QZ3LgFc6qWdsPHl91QA2WLWqWc8b8t7GLNNw==} optionalDependencies: - msgpackr-extract: 3.0.2 + msgpackr-extract: 2.2.0 dev: false /msw-storybook-addon@1.8.0(msw@1.2.1): @@ -16362,7 +16398,7 @@ packages: resolution: {integrity: sha512-6R9fqJ5Zcmf+uYaFgdIHmLwNldn5HbK8L5ybn7Uz+ylX/rnOsSp1AHcvQSrCaFN+qNM1wpymHqD7mVasEOlHGQ==} engines: {node: '>= 4.4.x'} dependencies: - debug: 3.2.7(supports-color@8.1.1) + debug: 3.2.7 iconv-lite: 0.4.24 sax: 1.2.4 transitivePeerDependencies: @@ -16462,9 +16498,8 @@ packages: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 - /node-gyp-build-optional-packages@5.0.7: - resolution: {integrity: sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==} - hasBin: true + /node-gyp-build-optional-packages@5.0.3: + resolution: {integrity: sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==} dev: false optional: true @@ -19467,7 +19502,7 @@ packages: engines: {node: '>= 10'} dependencies: agent-base: 6.0.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 socks: 2.7.1 transitivePeerDependencies: - supports-color @@ -20586,7 +20621,7 @@ packages: chalk: 4.1.2 cli-highlight: 2.1.11 date-fns: 2.30.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 dotenv: 16.0.3 glob: 8.1.0 ioredis: 5.3.2 From e88617e3c3418316152e2f8a62b951d71991deca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Thu, 15 Jun 2023 19:29:27 +0900 Subject: [PATCH 032/562] fix(backend): restore start method in QueueProcessorService (#87) --- packages/backend/src/boot/common.ts | 1 + packages/backend/src/queue/QueueProcessorService.ts | 3 +++ 2 files changed, 4 insertions(+) diff --git a/packages/backend/src/boot/common.ts b/packages/backend/src/boot/common.ts index b6ab939644ae..3995545d7f74 100644 --- a/packages/backend/src/boot/common.ts +++ b/packages/backend/src/boot/common.ts @@ -34,6 +34,7 @@ export async function jobQueue() { }); jobQueue.enableShutdownHooks(); + jobQueue.get(QueueProcessorService).start(); jobQueue.get(ChartManagementService).start(); return jobQueue; diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 6ef760afb4c9..566d855308dc 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -77,7 +77,10 @@ export class QueueProcessorService implements OnApplicationShutdown { private cleanProcessorService: CleanProcessorService, ) { this.logger = this.queueLoggerService.logger; + } + @bindThis + public start(): void { function renderError(e: Error): any { if (e) { // 何故かeがundefinedで来ることがある return { From 91af2d7b068999829ba692eab9803d8417f6d918 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Fri, 16 Jun 2023 14:33:13 +0900 Subject: [PATCH 033/562] Fix offscreencanvas undefined (#88) * Suppress ReferenceError on some environments (i.e. older iOS) * fix * fix * lint --------- Co-authored-by: yuriha-chan --- packages/frontend/src/workers/test-webgl2.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/frontend/src/workers/test-webgl2.ts b/packages/frontend/src/workers/test-webgl2.ts index 4769524d9c81..5027fa0784de 100644 --- a/packages/frontend/src/workers/test-webgl2.ts +++ b/packages/frontend/src/workers/test-webgl2.ts @@ -1,7 +1,13 @@ -const canvas = new OffscreenCanvas(1, 1); -const gl = canvas.getContext('webgl2'); -if (gl) { - postMessage({ result: true }); -} else { +try { + // throw ReferenceError in Safari <= 16.3 + const canvas = new OffscreenCanvas(1, 1); + const gl = canvas.getContext('webgl2'); + if (gl) { + postMessage({ result: true }); + } else { + postMessage({ result: false }); + } +} catch (err) { + // assert(e instanceof ReferenceError) postMessage({ result: false }); } From c492d450e9242c082704414d0dd9fee16acb838a Mon Sep 17 00:00:00 2001 From: nenohi Date: Fri, 30 Jun 2023 06:35:20 +0900 Subject: [PATCH 034/562] =?UTF-8?q?admin/role/user=E3=81=A7userid=E3=81=8C?= =?UTF-8?q?=E6=8C=87=E5=AE=9A=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=20(#90)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * admin/role/userでuseridが指定できるように * Update packages/backend/src/server/api/endpoints/admin/roles/users.ts --------- Co-authored-by: riku6460 <17585784+riku6460@users.noreply.github.com> --- .../backend/src/server/api/endpoints/admin/roles/users.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/backend/src/server/api/endpoints/admin/roles/users.ts b/packages/backend/src/server/api/endpoints/admin/roles/users.ts index 35edca546055..328c0bb91bce 100644 --- a/packages/backend/src/server/api/endpoints/admin/roles/users.ts +++ b/packages/backend/src/server/api/endpoints/admin/roles/users.ts @@ -28,6 +28,7 @@ export const paramDef = { roleId: { type: 'string', format: 'misskey:id' }, sinceId: { type: 'string', format: 'misskey:id' }, untilId: { type: 'string', format: 'misskey:id' }, + userId: { type: 'string', format: 'misskey:id' }, limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, }, required: ['roleId'], @@ -63,6 +64,10 @@ export default class extends Endpoint { })) .innerJoinAndSelect('assign.user', 'user'); + if (ps.userId != null) { + query.andWhere('assign.userId = :userId', { userId: ps.userId }); + } + const assigns = await query .take(ps.limit) .getMany(); From b4f720cab697c3792cfd63fc2ed2570d509a2c25 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Sun, 2 Jul 2023 09:54:26 +0900 Subject: [PATCH 035/562] =?UTF-8?q?ServerStatsService=20=E3=82=92=E7=84=A1?= =?UTF-8?q?=E5=8A=B9=E5=8C=96=20(#91)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/boot/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/boot/common.ts b/packages/backend/src/boot/common.ts index 3995545d7f74..a3bbd6bfa7ea 100644 --- a/packages/backend/src/boot/common.ts +++ b/packages/backend/src/boot/common.ts @@ -22,7 +22,7 @@ export async function server() { app.get(ChartManagementService).start(); app.get(JanitorService).start(); app.get(QueueStatsService).start(); - app.get(ServerStatsService).start(); + // app.get(ServerStatsService).start(); } return app; From 65a75ecd9af579ef23c84f2da8d44ba4b2dfd3d2 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Sun, 2 Jul 2023 10:20:40 +0900 Subject: [PATCH 036/562] perf(backend): allow get for some endpoints (#92) Co-authored-by: syuilo --- .../backend/src/server/api/endpoints/get-online-users-count.ts | 2 ++ packages/backend/src/server/api/endpoints/hashtags/trend.ts | 2 ++ packages/frontend/src/pages/admin/overview.stats.vue | 2 +- packages/frontend/src/widgets/WidgetOnlineUsers.vue | 2 +- packages/frontend/src/widgets/WidgetTrends.vue | 2 +- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/get-online-users-count.ts b/packages/backend/src/server/api/endpoints/get-online-users-count.ts index dea0f4799c2b..810bde03e8dc 100644 --- a/packages/backend/src/server/api/endpoints/get-online-users-count.ts +++ b/packages/backend/src/server/api/endpoints/get-online-users-count.ts @@ -9,6 +9,8 @@ export const meta = { tags: ['meta'], requireCredential: false, + allowGet: true, + cacheSec: 60 * 1, } as const; export const paramDef = { diff --git a/packages/backend/src/server/api/endpoints/hashtags/trend.ts b/packages/backend/src/server/api/endpoints/hashtags/trend.ts index cf45cc6c2483..ce1cd9f01f74 100644 --- a/packages/backend/src/server/api/endpoints/hashtags/trend.ts +++ b/packages/backend/src/server/api/endpoints/hashtags/trend.ts @@ -26,6 +26,8 @@ export const meta = { tags: ['hashtags'], requireCredential: false, + allowGet: true, + cacheSec: 60 * 1, res: { type: 'array', diff --git a/packages/frontend/src/pages/admin/overview.stats.vue b/packages/frontend/src/pages/admin/overview.stats.vue index 142e70c69819..f746ad14b97a 100644 --- a/packages/frontend/src/pages/admin/overview.stats.vue +++ b/packages/frontend/src/pages/admin/overview.stats.vue @@ -73,7 +73,7 @@ let fetching = $ref(true); onMounted(async () => { const [_stats, _onlineUsersCount] = await Promise.all([ os.api('stats', {}), - os.api('get-online-users-count').then(res => res.count), + os.apiGet('get-online-users-count').then(res => res.count), ]); stats = _stats; onlineUsersCount = _onlineUsersCount; diff --git a/packages/frontend/src/widgets/WidgetOnlineUsers.vue b/packages/frontend/src/widgets/WidgetOnlineUsers.vue index c920c3ca5372..7e01eba4879b 100644 --- a/packages/frontend/src/widgets/WidgetOnlineUsers.vue +++ b/packages/frontend/src/widgets/WidgetOnlineUsers.vue @@ -40,7 +40,7 @@ const { widgetProps, configure } = useWidgetPropsManager(name, const onlineUsersCount = ref(0); const tick = () => { - os.api('get-online-users-count').then(res => { + os.apiGet('get-online-users-count').then(res => { onlineUsersCount.value = res.count; }); }; diff --git a/packages/frontend/src/widgets/WidgetTrends.vue b/packages/frontend/src/widgets/WidgetTrends.vue index 36f908d5ea28..5fce4aedca1d 100644 --- a/packages/frontend/src/widgets/WidgetTrends.vue +++ b/packages/frontend/src/widgets/WidgetTrends.vue @@ -53,7 +53,7 @@ const stats = ref([]); const fetching = ref(true); const fetch = () => { - os.api('hashtags/trend').then(res => { + os.apiGet('hashtags/trend').then(res => { stats.value = res; fetching.value = false; }); From 59b1064fad5fbbdc7eb2b1fe81383ba92c374aab Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Sun, 2 Jul 2023 12:17:55 +0900 Subject: [PATCH 037/562] =?UTF-8?q?JSON.parse=20=E3=81=AE=E5=9B=9E?= =?UTF-8?q?=E6=95=B0=E3=82=92=E5=89=8A=E6=B8=9B=E3=81=99=E3=82=8B=20(#93)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/server/api/StreamingApiServerService.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index d1394d6d761e..8c3025e62555 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -93,6 +93,13 @@ export class StreamingApiServerService { }); }); + const globalEv = new EventEmitter(); + + this.redisForSub.on('message', (_: string, data: string) => { + const parsed = JSON.parse(data); + globalEv.emit('message', parsed); + }); + this.#wss.on('connection', async (connection: WebSocket.WebSocket, request: http.IncomingMessage, ctx: { stream: MainStreamConnection, user: LocalUser | null; @@ -102,12 +109,11 @@ export class StreamingApiServerService { const ev = new EventEmitter(); - async function onRedisMessage(_: string, data: string): Promise { - const parsed = JSON.parse(data); - ev.emit(parsed.channel, parsed.message); + function onRedisMessage(data: any): void { + ev.emit(data.channel, data.message); } - this.redisForSub.on('message', onRedisMessage); + globalEv.on('message', onRedisMessage); await stream.listen(ev, connection); @@ -127,7 +133,7 @@ export class StreamingApiServerService { connection.once('close', () => { ev.removeAllListeners(); stream.dispose(); - this.redisForSub.off('message', onRedisMessage); + globalEv.off('message', onRedisMessage); this.#connections.delete(connection); if (userUpdateIntervalId) clearInterval(userUpdateIntervalId); }); From 680f3679241d2aa9fc7c96c1c8539315416a4efd Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Sun, 2 Jul 2023 20:32:40 +0900 Subject: [PATCH 038/562] =?UTF-8?q?af3258dc79488b73435819e7799eb1515f15a6a?= =?UTF-8?q?a=20=E3=81=AE=E4=B8=80=E9=83=A8=E3=82=92=E5=8F=96=E3=82=8A?= =?UTF-8?q?=E8=BE=BC=E3=82=80=20(#94)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/assets/avatar.png | Bin 0 -> 13477 bytes .../backend/src/daemons/ServerStatsService.ts | 6 ++++-- packages/backend/src/server/ServerService.ts | 3 +++ .../src/server/api/endpoints/server-info.ts | 2 ++ .../src/widgets/server-metric/index.vue | 2 +- 5 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 packages/backend/assets/avatar.png diff --git a/packages/backend/assets/avatar.png b/packages/backend/assets/avatar.png new file mode 100644 index 0000000000000000000000000000000000000000..1b95a0c560c32cf2a0060d9e911b1cc3171dfaba GIT binary patch literal 13477 zcmajGbzGdyvMr2TaM$1j9bj;Gcb7oW0S0#q?iSqLB@o=*CAbH7C%6asc;CJ6-sgP# zoO}Nm>1S1~s$SJy^Lu7`!j%=JP>=|bARr)6WTeGa-(UUydLTf(Kg;XgWkNu}G=bH% zowXI@c}?tW07j;E#vp*Zjr}_g0l_ciZf|5_1#%`c2AP9x1t`wjx+uuNrUDdN911K7 z_F^Clu(YQmNX=7G-Ne(%gvXRZNDztNo%fx<2IOo+=5Awc>%{9WK=C&(@B90|E;Cb* z{cYlGB|xFApiCxa=LjO>0B`_UCefcDCnbW_ELP1GupP z>>SORS$TMPm|1|#Kp@k*1(TD9t+SCkldTiwUljk~5C=J#ID+k+!FIM}e{mWa+qpOk zP*A+%Wd9@wxr6`BZtL{V>AwrX{1?K^3SeRWujbBRv;W2XFXTVW?Jew_?VK#^?Ef`^ z{}$q3@P9bI%lALKxEtC3PsR!g|95j6oBuWqCua%QcOm{M>VM1W{{(PS_pk>stAd>D zTpUe660RUyXUe~o_)8aFWw1NQT3Z}!1G05`S66_770CVnLv{W)RLsuW&Qa}sS%Kc& z{&xCHJ6;(ZBXf`z*woqL-)#Q|9U7A__hpgJoDFFO}62PXpy4=)SL-^_nozwZZABWI)km;GO> znvDP5LqUO88tmk3=jieGrGG4SHITzUM*kREga2ORWMqF=n%BtWFOdW&>>ce)T}(ix ze+Pf3`v>Y|XXfl?-O0@Y^iKG{ zPmlSZ$ua-6@%~SC{LKH6KmXqb|7EAX`~BnEdk1)LCCvZoCGUfObt91Ndkb=W?=^V6 z(8&-GkOAQL%KS$~ARuVtT;kf~pheI_yQnP1ccAO5s}V3^$uYj6*Vjm2z*@$Mvsfl% zdLVcZyJ28_?$K06xADSx!n0t|{t%FlU|imi|1HD`fGFNS2ISj#bv6H$1#89q!}KQI zXV~;w*M9XpGjfdYH}kI=RVn6i6$z~pF%zTj%Lf9uZ0nw5y7QASR^Q|t^daRsF2eFH zJ+=_H%I4cGucGbS;v)cvIf!i>Ex8w2Wka&hmX5Xk8#Gp1h7c7$dM9?7qhE}vi$Fnp zd(FTjoFXsMYwBosS5B%Lxpu59vYiJ9;0P_G7cvWD9U9t~CsT=}=U3xbWS$WXQ%xN) zlQb&2t7ZxPxMPQK#OWR2Ohwwu+lrO|>mied+e6;n63cp9PbW$+Q{MwGFshgJSN10G z^T;Gg?^NjFXPj{t2qq$XAB}0_Hv1Yxk{C<%NL=cSW{x<&CN+x);T05i#zare^@Cl= z-jgeh#p)(B0mn|zB|Hng`FOWO)bRVK2BN*RjuQmL2duvykPzva_z)0e5HjK->h768 zyg!fGXt}I&?XMp^aQ_ng_3&#CVuD-)2R&FkWJa7^L;QO`RZ!4XuUOF+b*gkp4sx81 zI9b{wIZ{Tw+pBId`l2#b%#tG6QW4oXY*<|m6$i@hRRstWA|kK8hfLopo+p<5hkdSn z!6EoLKw97H;uotggAapT0H?}^-#)$=5N^)7e5+f2&}&BcpCGxb_nI8cBI~;MA*Sju zc460c{&-M+Ua-27j4_Hj@qk1>&<991vcXe+lB#o2&;E9y#Ld}l$w%yTKV8guMwbtu zn5pCn?L%2uyIseUN?>gptLORbjBf}B0jKR6emXzsK7|b7iciw?+>4y7aNpi3SBK4d z^<}7e%+Q2z81mPHX2S&M5-L%+4hNCR7fIM0ldz@Rf(7pS=@bz-z9EQ=K38xx0rIa5 z9mU<)tZdn`kjyoBab$~ZmU{__rPfHzSTzahpZoT4@eyfF#6^mH>6SnSIf7tRWQ#}h z(p4u#H zI^Qo(|1n_^F#zYoMQV9ScW6cziw#E3rArJx={44!+}Cqw`{k+)7|N7no&H3zpcB`; z4-@ap!%c1A2C;tuiE=bIldbMYF+Vi&{a{_)5XsGo&d(pbsuZDx9T<`&!q91nf z=c?V`a<6{sKYc1~+4#EPL#iH1x+g-G2~R*>OkiN%N>?zT6}j@sL@h*8dMVLz1n zMSM)?hzNHaCoN<}N$vXPx_)OVYgKV_32aX1nP6uU^pz-Np3&X(Nju_XbAW)%SUH#6 z&J2OH$woAb@c0Vq0p^2Je1%4TF>$;fMY?j8^J6u!Y_XacMHm1V_evCQ>5{y4y&sA_ zptkUJ>!X@^vIF*8mYGUg7M6n&mP~T-4FO}YHXLk2W?dVZ3MTEGFCA(Xa=}s5pP%^B zH@82TQnoG`Xwk~PYGizhQHj1*BFciBVD!jM;e;WkE07ct@73!zQqx1Qi(Bxuw=?_s zJAz2ALxl|b+yQ#WK)SPp(q>*P%Hhv+63=G^y(a6fymiz?a)5a0j6pBcw_tvoOo6LA zrt{-8E_U#ScdNHyh-+UV&@2bhD?-qA1C%+)kdfEsR2pY)EvP_>TGw9l$*t{B=Z#2A zNksxybYP8oq)>;Jg$y0>9$dt#3czOvHGtOSS_Y z)Q26+^Si?MWN@O#z0Fzh`D(@Q0OrjaT8oL=@M~W&DCWsrGH^g3sH2S=^xv{m3o+^uz0yiYD;`uIQ!D7Xh|{* zC@ps4H~{hXV1Qiq33*OOenQ5HJP{oCqhzA2NaWD*qnu*@!;3U;q6uC`sxF2Q9T#sT zfjSF(uAUumAUL}H4nfr-L^zSTPnOA^>Fyv3!zMa2&Nh4za;}&3B&e=Ed9hSTs`fB9 zcZ+_AeYe1HX!LD?tTu?*kJmiog$#pZdIbV_lGMdUVT{e=D{b<|(ttHX45 zf?{NfD9H*U;z!2DME*+mpGQRg?2{+yp@IW*TxwwaaAGK3$kMkrmCnZS0c_B?9= zjIq1F)Ds&6sX&d~4WP=lTSnkl+|05E*%;RS!Q>(bq)G=t^kFuapz*Y?Hg1$e#=Cm` zgdZlH9SGgN3-8FQ{ep^pVqrtZD>2vPR!G`y08K_>vnB@6pVzX^s}mthYl0ii)yCI?w}k2qRY-v)M4rbB788EG0dsoXK33JLeU#V$>1!7Yf_PN;u;l zTF>HaRK=VjOmI6ri_4_b0zllPg8~CK_M21igWeA|vkUUoVR~ed;KurPxE^q9O)X43 z0m2vnd(H_ab143mk@SWEPw!{iky)<)H-MaBtADbus5*j5Ar!7LZZSq)DT-+PIL#g3 z$;?tko^pU?bYJRdc`M9z0V?iL_A)B5&X=YzCI1Zx;wbxJ9Ns2^RtH<)D!{DgD^C(R z&G_`Lc)*`i4N)1jEooygVMiK+b`WUAXL^>mA;HW=wB^g~) z9DOsTh-7^gLm zBCZe-d2(4%#Uprrypr#xYcZ)q%BConvsehvA*OH=sHwpwAFV-STd2c_sY9HzrJR&Vwx|^2sE)K| z`i0PVj4a=0fZj9gV{M`&t#j0TTsDCYnU6Va@c06qHPEjk!cUADdB+J#ZVXFdYKLT2 z-FDV+W&eVSOCa6MgQ63G%`+)+G!&LSwmEn>)dGc1I8+5;-15BL5P5ytZ(Npag_Y9K z$qb+EH7|$kV2U_RH3%;ax4Q*6qDe);2iP}Yo=T?irX~-GYsCk)Eoh|dL{;Gjxso$r zZBY^E;@Fnms~VyA_XEnQ!!`{*<4TD*;HfLb{vJa;WmER5$XikN<3mDxlj-iya_ zsBvR>Dn1rv9xoQatWW|y0Y?L6SO7^=%lS4e(oOZW-Wr9D7(n9$%e@S)zUXEWd(HmP z)el3&PG|Z1EAR?iS(YlO5-eS(k57c&IMWwdh^Nt0)1d%_IZy8Juk%HVLjrK7}ASCf`x4?&kB`B{g*`i$7kkX9$VgC$g z725IuKW__O$*ST|4P3!HTpBD9r1E=$$I<{+DDn@P7&~Xw3#NF)I2esW#iA)ui3%6| z@hLo{zai9E*Qpq1^^97L;gLJ4VjaLO$YkehGYW0dvXwFQR#*UIh7Nie^ClU4kHZ}? zKT6GhPF9ctzeze(hjil(?j(Kvff=r@lU~<(eU^A=fO;@cp`Tf-c!*APk~0juXrH^B zCsDT>DiXjI-Lhb|b6%E5JQJlLAr|?SWvs4W1mbbZJnF#HK_r`K>U|l-0!hO=A>qrc zJV-^fgpHC5$}&f~C_t|5;mUYQWXUH(Mv^Jqp{ff>aA!GNIC^$9yuN#+Ai1AKB%}Jj za{BDo6yxuD9#tdN+w)B^r<%dq)6$L8^s74pOMj~{sjSmNh4-PK_xV2C2qXIvS7Tx?j1_S$ zZPnHBFeB;K<;Ts_^tLLQ1`*#Oci@>z3`{BL2y{URF-~9O&cVyID`IMJUAN;<(bMS{ z(=x^IppqAl%?&@C(SjZFm3Kg+Z3hBs}FBFoh*rX~jd2s>Qr* z>1_a?sY)NtF)^!H=BOflv04Kzeg50WF z215-nAmKnp0m0tcf3WjME2H$U32|}OFY%qkLqQ(5@TkO|%P-wm$N+>m2)?Conzokn zvDTQw**aL1Oj={Z>>Dat+wyF=cm*?Qx>VLp2jswp!`KY8xuL5{nIU?;@JHNDgSNIk zoW;)D1+}Wy$GOq$U$3Wma`)a>X=wb8j$PlsJ7bGt+%Wl9jf#J9A0M^tIx{zL+=u^C z-SxXi?%wxk>Fv7alh3t-FBLYik96OO;X!U3J}~FvxL^XyYB`wp09RZ`&}pPY{~`3$ zyzBB;B!a-GcWkWTAmmJ@PTE0$a}nMP5+)(Qb&3S_>r#Wz{ZrlBvpiq<^^woRaDton z&4};Xi!eYGgjxB!dABL5bJL>H_vKsHMFQ8Waa{-J>)2htulqyJ^JN0*ufm|}z2a|| zu`pQ*6UGUO-kA=HFgo-H7woizDt0-jz}VJ5OK$q-GT+IQoew2LRCs`!pC7Y;Ah9y4 zaogzYnXU%`yN;c~wtH8&TNW$4iRa<_Q`_g$rZ+=f=kcgiOg^M-evR9Gr>>_RuGjXZ zD|u|i2DT=Hx7nC%uf0(`A;=0v+Y63z1P`DfJMRFpf3oQ}!8AV$p!zp)&og3)Gc$*o zGkfO8lF{7~)L9PwDmePIxWBFWze#?e_aFmrF&nc@M4R^BuzH z%zA%X7qec7CZ``h+j&naluW2u)<}e+dR#v~r_jjM+{qm|efFJXvP+I}%*EW#3^3hf zg85xG2~aK2q}v9JQZkem{;=jiVC@+wEPlSbJLvdbuYXSrgt{m&-w{$>X~f<=mY?LtXrBra29=+!Iq%p}v3Bk<&LhpWZNO;gM41M)^u!_)@ zp+uH$szBo&8YC+y>jafAFL zk+@bQOi(Yer!9N=W9#=Q;@7#mTgNs_UCC11i~g+>xm5S1oy&yKO_OQjGTq!L+>zys zf`c;#XhqeEdT_CRRLc$+H9=};XPHT5J+Xq^n@zLm?w7Nhu%?uLtm3CSU?)}3^G^~q z{nqpEK+l`qI=q+9l^v0O$>@GZlXWjgq0HWA-@o}@F8zs>A%GP}XdR=iCVh6YKNoFm zBqQsS!rKsL9$(0hUv7psmXqp{jDzz}bgY_$zu+Iwk2Fq|By0H9vHD^2)$VOw19TH^ z=Z#**uLz7*aEFjdof{g;>Ga(3ecpv{`n6@f^Qf0F#2R}wlbD*^!}P|_eK{ak7qB*< z@N|(YXh(*G|G^`mqXsO|JlpF(96s~jJ~FDOqSP{ zkk88j^J7srB;7PD18YmLbisX~f#*h`@BNmXL9_MFz0mdahWp_ftGJRJh-!AemTh59 ziw}6I`Rc$LR`k@r8oc_^;J42csh*O%YY*@xWDhz?U0-jr_FPRTxWkg`x>9$>VRLhG z{r<+zx6OBFwd`DI);3Hs`)K3AK2j7-gUCW@t)jVl{+O#}7XM_Eki zj^(bWWsje>0DFV2L%~S&$kjYVdK$%sIv!L^=^wxOtJa*)ZtjI#ugsM`emvE|PV>-Q zb7{|Xrh6j^TWas@vE_Gp-B)Y!eq2%#YVa1Dw&ZNoX|@)lu(P=IaB3=qF-0DvFjIds z-)FAgWXcwdzhlJ~E*MmqADH1N%kh&|(EkbIs5!M)kvaO7t*3Qdk!C(l1pDVZVu}NA zEnOU}7AV{Sz)41Q(DlKt|1Cj)v+LPaRe#``SJ5Wc^A==)UZADi%sC`A0b(L5qyM?~ zou*OG!Ff8Ugr1wmMye{!5=(4)8k4k?amaacUZYS}(a{DM(RE>w(M^!~yjtaxoenGL z5AhFPBY66NbV)WWI_Q8ZL{q&LJ zfP&ra%*xy(D+_OzZ2jZS%TMu_*bfQ^_H$Q2-w8IFg*pUe#Cqe9Y}s3R>|*x1SgB6U z9p(yU1u7VqSoj`k18yLVG{Q$Hq5ga?Lsosp|0NK?TwlYeQd%@ylVr{kqp_(FOIH(UMH)NV{#xLVu zLkt>e@Q_U%KQfW;hnDGcFNFV-RSQ%*%jd&zH0nh72J$}3_}f!UvIf7J6W7Y0bu_h& z^ko4R1qr3jd0?s?>IR$+)kun2q}d(|j9^S=kL`MEDaU9hyH@^<{K5Vsw#r*mY)8K7 zT*DuHJI-&T=90T# z!RRb{tK(24_KE0gNjf+uIOD@*8nVlsULm;Vk)tNvJNL$ht<)vv1sbaTs3d6l3CP=h zyrm)@OF#!t0Q%8Ro)5aNGf<)GV(&P-IE2;2wfqA+a=JQ4)(|W5^GGhs%9IS-g zb#c@X<>1B^E_Uj(H0I4PtYWS^aQJcVjZ5n>jvPZOXJ=uv`e44gY)fIB)Yi{Rn#nAzlgaZ1tMaJR4sD%ILzScJe4)mrwq{L* zWi?AD&*XWKRB!n8drfCN{W&XlbaJJJx;0QISM#0F>iQJmG?^-}n?smg61Y&DETuCg z%4~_41&2gLZJvZ)ks`la_E3vd=Qve^k<{q|N0S32C&X{T6o&(z+%_o-`Iin3OjVw@ zmHf$whdy3F2&V}L0Upmnb!g|g!iQ6sQ-@H zktlvZ5-Z-hmd$~f<`0H`>N#0={^R2NdtYtiJ}J`o>9WN4UMc5Qd9snCco|nAOe$?5 zWVPl)?wO9GT~*80r?;ymE}xBtDzA%eJi)7nbFb6OH_X4US;>^5bZ4 zhFhC-zBiYpB#ApxLhniFgOYfwh|;G;+_Lo|wx1HrXFy=NXjF`yR}b^suj`G|@C7q#8g%88DZGs#yhWO$rDsD3q`eY1yoZhKYi{l^?Fvx$BqI3P_!_0WI%#G^E(<#uj#-DTx`$8kuD+2^=R&~?#UY2)M&;|JRroK((qkH7oaO4GE}*IShL z(8(G%CciG+eD;UB9>!iY!`l+M=Fm#Q=m=hU*bsB#YGxn^WYSzueg*PM#gXNi_kv)6 zdu1{Zd!hmPiyW^yqsdk6XVIgwq}`E~9*-M5Z=F?Np6B36+fICiT$rcY{8TFpC;6|Xw30CV@C>f6nz|0X&wY+UeK%Wk zL{H6O$-H>?_kZ7=_#SJ$J%^rqjr{4vI)Z-aZxWO}WP}w-<{=muszH42s~G)?!ji8s z`qnzQ3iZ=(`b_G)JsWCZ8hZblA-clR8_$B$trq(U6Rk`fW6^8pBI#VxK_w z0p*WVRmYY>)5b-}C$t!!-`fY{nLWf)U%dN>Ue_bv#+BMP?Q+&~{wV4~Uwl$}?fFCc zn$lgEIB>&f)-LTMhO~SGTJ42!Nb}T<_4{q~d6`U*)^=N7%5ub1etj-07-(sJNk_;2HrCVOeU|2@?-wvV-0hu#@1DXKJX?^iK%zb{Kpop)p(}x?!*^-X?*+Z zqYrMY&If4TDV7hYG3YM>F{>~0=3T!Y5wxR|;|7vf7~MFPtCCsE^k&jwkLt+><&1bg z7pI8e7kq!HoXk0|jfo29n^^^9WU*H0$%~}mEM6t}86&grGsY|qay02VZK1pw%ujj$ z=^HwK{2hAk^WHLI3OW_YYKwB|YJ)l37+oEaGPJKTN!d8Fh$iCaLv4MIoDMhw3ZqUB zk!wd~bXq?qqCJ=0UOr7(-8VE0$Ow-;C(q*f?Cd|(e!4C9z1a@xH7VJ;(c-2QaH$# zd)MUc_2FHUi#0R@&&{bF72ZYBR7{#TM%bubVWcqSDXN>PA!y050d;seQK<of-7V{%>NJ2;artH>>r;J9(K(6+-h@j< zcS&k&0`D+4T8j`!9Tga0#$IOl$%+A^%*dltsazwc2U4w3y~{}|P(T4wd*E5y2X-(3UxjpcA>VZ6^A|m64HEV&AWacZRZy7CT$SN8t%-y6~J8AmMB2ydMRI%0cgY2_*ielU; zYZ%pa=y3fX%9H&<5?hIW&<(pzrXZTbzLpd&v(=AFe!zJg>42nI}#u zPpwj8jF;j@`Zd$S%diox5c57xM&XI3AYlO(GT^3b<51`hCEEDeXyqz6Lyt*zis}_$ zT7R7-<0!Bo&%ay_jpBP0oUtF)T8w$?BlVBhUD0AMsEw}+CdU83Cp~I{cLO8xlpLti zs+%7_z}=`$KeNIlXRYZ!Fmk5-#w1-4E)NzHh z&{Dps)22C}k0hn+Ba3ywGI%W0Fn-M}cCe}*$w0u8ulQ6Kk!y0;xjiE>j!}233twR8 zMh;!1#>VY+{=^^N^Yzh*uxfW<+&&&x{CxZwN=6*{w9&Lj0XZT3P^6K;jiBg*Ha6** zeZNvqi)hdXe0ta!3=Ly~eDn3;P$3@lYr%HKZ|nEF5wVc`NPklgq*wyaA6RvRr6 zWe=WmZsKJ9em8VFBqUi>T6p`CBq-G6_z81~l#dqxG)0#O;p_U(y z)Z3=Y;jVmO69>Fdy_c=4wiY=3g{8L%#y-p>Y%K7`hAYYccx z(d1qthmx_@M+)n8g4B9?#;<}E3SFQ?;tmN$xZ%cjaEC21^29a14t&@l-uGYSn{4KT z`VQD0uAU4$92$l4M1)VfG)LPWbL=~LU^1X<%ouC~st@$jEE!i*i|mU=Wz13K2R`x! zdc~&dvnWTI-Dil?NDf;J4&QRx)klgy2&s(es ziNBdnh*A zR??Z=7{`KS9i8VFhTk{|b5WB_>Gi}a!M>|Jug%RsImgOgI5A}G%jh-JAl5%f>(+Wh z_9xu?8tbWtwX!W6jyoAAq2}K?w`s~NZn+88HnLVa$YmN}HDUC~4oW)1xV{7L&oC$(q8Mr@8!-;+0~>A%hh2$+`zPU~3i@mmGZG}_Yo z4)$zhP{E(+;01nb)i%tTIaT$^YtyMY)U^lgThGVEbh5pulP3f=fURx0Riet(b_`ff?!F^^#|@ zDEkyqfpbo?kfFd_BxtBWeL! z)T_0cO)FxC{fI}KF&s@nknNZ&yru1dKcJf}@mGoDsQd?`E^mNA^4%W$N- zvdiC`OV}AvsMCX-3VH}-FTx~Iq39fcu=QQ!&!xr%!v($0}^cz|JoT$dIAzbSzo z4~QcK7STbC(uD`w3h7z3qf-GH<|#>MuBKR1gGHSNH%vVScxo)0g-RosGJp~FlxOV6 zg_aR?%y{kBX8wLMPbM77$Jtwv-vFWo`Jb3`-#i08%BXEK_&kz#`(O$ke;VA~CHB}9 z=24O2Y|k9GlaVFf+LAsjRhx&TrNjDyULx>2H>4!Q$cU|bl3ay5IdODE(rT3wMg+f4 z4;V_G6U+NR%dYyR;OFuB7$yeban0{BR$sD`X;y0){l`ieO-|ef%~y@$UV45PTQ78< z%t<>1LFc2=@P}9t#mSkne0;EFXuD&qs-}-&!68~2VIadteWowpjj~&(m8<#gYU;7= zQR#-v0dASjO6Qi@x5I!xBCfh_CKhp_j?`?FrWhG6~&0$azBw3&|#5PWYBqrkrj;lVrQG17557ISn%hz>c!?18DCIA{1ApYp9-7 zEY&9d(g}^-jG)#Cvpd(TR;Y9f*E<+E>iBhUTGzdbeTYxKCY%tiMo1^Fm90XmxaGc9 z+=XsP{6`yGKt=`ADpEGwJE6ljuSJY-3c1Nd5bQ@-PTQ~dsy-jRWd>3=vH2qAxg{NC zbvWNveYRG3g<<2N3k_R%3b`p>tn_VJo3~!fjq7}#kw0+l&{K|Uzn2|rBQ8uNK+aeJ ziEtKAWr~T*i7&#LTNYQAPKQY{y7TEGKOHMgc}g|6D1a-6#yGzapqt07S1rmw4E7*p z1dIYm8aOA_rM)ot68?Zcm3ud6O_fl6umFfBkk6756vJs6vw zY&0#l2+C)^zo?+od|Z4_kM)e8=)x7jl;Sy)I7)4lD z#WW2epD0fXn9aG29Q`mt?AOj!KXPr>XZj?)6!%fS?Ay9%3WOV`@v(*3t`(?))j#V2 z=ZCl}D=LbFJ7b|Qx`J}zXL>tso=Td1{NhE7mO~2F_>7ZmI6^G8FDzsCWp!zC2U!9M zwXx4Wm;#g$t=l4I)vMi@i#84Ved5k|{WS`bWVw-)xWFXJJTPP+`ZR5r&5UuW&}d|a zf)A;P{n6g7fog5{75aaMgS?|n%rfXjNcGG( zFD1rnMDvA~yW4w|$s{q=RBH*x4Abql>a{7%E5$dRdI6(+{g48IrUkvHL4M}N_(93=);I-ud z>a1sOEOLZ6ut1R?9!50XeDRvRHkDo)FX?uCY=r*mfRjCwM@(K=h(rXve{uXlxI!_a zi5Bt{Oc83EDw%v$4W+U*kt#_q&{D_zGO*+u&F*NbvStv zvf2yHMpLviP-(1pYRGBbqRgr{|}U&QN#cM literal 0 HcmV?d00001 diff --git a/packages/backend/src/daemons/ServerStatsService.ts b/packages/backend/src/daemons/ServerStatsService.ts index 6cd71c0e2ae3..e23309a0e6e6 100644 --- a/packages/backend/src/daemons/ServerStatsService.ts +++ b/packages/backend/src/daemons/ServerStatsService.ts @@ -14,7 +14,7 @@ const round = (num: number) => Math.round(num * 10) / 10; @Injectable() export class ServerStatsService implements OnApplicationShutdown { - private intervalId: NodeJS.Timer; + private intervalId: NodeJS.Timer | null = null; constructor( ) { @@ -64,7 +64,9 @@ export class ServerStatsService implements OnApplicationShutdown { @bindThis public dispose(): void { - clearInterval(this.intervalId); + if (this.intervalId) { + clearInterval(this.intervalId); + } } @bindThis diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index cd72e9ce7d96..af376dbe342e 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -168,11 +168,14 @@ export class ServerService implements OnApplicationShutdown { }); fastify.get<{ Params: { x: string } }>('/identicon/:x', async (request, reply) => { + /* const [temp, cleanup] = await createTemp(); await genIdenticon(request.params.x, fs.createWriteStream(temp)); reply.header('Content-Type', 'image/png'); reply.header('Cache-Control', 'public, max-age=86400'); return fs.createReadStream(temp).on('close', () => cleanup()); + */ + return reply.redirect('/static-assets/avatar.png'); }); fastify.get<{ Params: { code: string } }>('/verify-email/:code', async (request, reply) => { diff --git a/packages/backend/src/server/api/endpoints/server-info.ts b/packages/backend/src/server/api/endpoints/server-info.ts index 1620e8ae52b5..09c8be909430 100644 --- a/packages/backend/src/server/api/endpoints/server-info.ts +++ b/packages/backend/src/server/api/endpoints/server-info.ts @@ -5,6 +5,8 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; export const meta = { requireCredential: false, + allowGet: true, + cacheSec: 60 * 1, tags: ['meta'], } as const; diff --git a/packages/frontend/src/widgets/server-metric/index.vue b/packages/frontend/src/widgets/server-metric/index.vue index e019ff540b8e..1a78f4bcd6eb 100644 --- a/packages/frontend/src/widgets/server-metric/index.vue +++ b/packages/frontend/src/widgets/server-metric/index.vue @@ -62,7 +62,7 @@ const { widgetProps, configure, save } = useWidgetPropsManager(name, const meta = ref(null); -os.api('server-info', {}).then(res => { +os.apiGet('server-info', {}).then(res => { meta.value = res; }); From 3e93450fd4b8f8a08110ccb2cda17297ae1c7c7f Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Wed, 5 Jul 2023 23:59:32 +0900 Subject: [PATCH 039/562] perf(backend): Use addBulk to add deliver queues (#96) * perf(backend): Use addBulk to add deliver queues (#11114) * BullMQ -> Bull --------- Co-authored-by: tamaina --- packages/backend/src/core/QueueService.ts | 37 ++++++++++++++++++- .../activitypub/ApDeliverManagerService.ts | 19 ++++++---- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index b4ffffecc0a6..1c32208c053f 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -9,7 +9,7 @@ import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; -import type { DbJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; +import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; import type httpSignature from '@peertube/http-signature'; @Injectable() @@ -33,7 +33,7 @@ export class QueueService { if (content == null) return null; if (to == null) return null; - const data = { + const data: DeliverJobData = { user: { id: user.id, }, @@ -53,6 +53,39 @@ export class QueueService { }); } + /** + * ApDeliverManager-DeliverManager.execute()からinboxesを突っ込んでaddBulkしたい + * @param user `{ id: string; }` この関数ではThinUserに変換しないので前もって変換してください + * @param content IActivity | null + * @param inboxes `Map` / key: to (inbox url), value: isSharedInbox (whether it is sharedInbox) + * @returns void + */ + @bindThis + public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map) { + const opts = { + attempts: this.config.deliverJobMaxAttempts ?? 12, + timeout: 1 * 60 * 1000, // 1min + backoff: { + type: 'apBackoff', + }, + removeOnComplete: true, + removeOnFail: true, + }; + + await this.deliverQueue.addBulk(Array.from(inboxes.entries()).map(d => ({ + name: d[0], + data: { + user, + content, + to: d[0], + isSharedInbox: d[1], + } as DeliverJobData, + opts, + }))); + + return; + } + @bindThis public inbox(activity: IActivity, signature: httpSignature.IParsedSignature) { const data = { diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index 62a2a33a1996..66e77611871c 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -7,6 +7,7 @@ import type { LocalUser, RemoteUser, User } from '@/models/entities/User.js'; import { QueueService } from '@/core/QueueService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; +import { ThinUser } from '@/queue/types.js'; interface IRecipe { type: string; @@ -94,7 +95,7 @@ export class ApDeliverManagerService { } class DeliverManager { - private actor: { id: User['id']; host: null; }; + private actor: ThinUser; private activity: any; private recipes: IRecipe[] = []; @@ -111,7 +112,13 @@ class DeliverManager { actor: { id: User['id']; host: null; }, activity: any, ) { - this.actor = actor; + // 型で弾いてはいるが一応ローカルユーザーかチェック + if (actor.host != null) throw new Error('actor.host must be null'); + + // パフォーマンス向上のためキューに突っ込むのはidのみに絞る + this.actor = { + id: actor.id, + }; this.activity = activity; } @@ -155,9 +162,8 @@ class DeliverManager { */ @bindThis public async execute() { - if (!this.userEntityService.isLocalUser(this.actor)) return; - // The value flags whether it is shared or not. + // key: inbox URL, value: whether it is sharedInbox const inboxes = new Map(); /* @@ -201,9 +207,6 @@ class DeliverManager { .forEach(recipe => inboxes.set(recipe.to.inbox!, false)); // deliver - for (const inbox of inboxes) { - // inbox[0]: inbox, inbox[1]: whether it is sharedInbox - this.queueService.deliver(this.actor, this.activity, inbox[0], inbox[1]); - } + this.queueService.deliverMany(this.actor, this.activity, inboxes); } } From f7ad3e847ab4215bb26b2ca30746b2d49ce9dd18 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Thu, 6 Jul 2023 03:37:35 +0900 Subject: [PATCH 040/562] perf(backend): use mutex for nsfw model loading (#97) Co-authored-by: popkirby --- packages/backend/package.json | 1 + packages/backend/src/core/AiService.ts | 18 ++- pnpm-lock.yaml | 158 +++++++++++++------------ 3 files changed, 99 insertions(+), 78 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 85677f8a9e25..39282efa0847 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -75,6 +75,7 @@ "accepts": "1.3.8", "ajv": "8.12.0", "archiver": "5.3.1", + "async-mutex": "^0.4.0", "autwh": "0.1.0", "bcryptjs": "2.4.3", "blurhash": "2.0.5", diff --git a/packages/backend/src/core/AiService.ts b/packages/backend/src/core/AiService.ts index 059e335effa7..c0596446dd60 100644 --- a/packages/backend/src/core/AiService.ts +++ b/packages/backend/src/core/AiService.ts @@ -4,6 +4,7 @@ import { dirname } from 'node:path'; import { Inject, Injectable } from '@nestjs/common'; import * as nsfw from 'nsfwjs'; import si from 'systeminformation'; +import { Mutex } from 'async-mutex'; import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; @@ -17,6 +18,7 @@ let isSupportedCpu: undefined | boolean = undefined; @Injectable() export class AiService { private model: nsfw.NSFWJS; + private modelLoadMutex: Mutex = new Mutex(); constructor( @Inject(DI.config) @@ -31,16 +33,22 @@ export class AiService { const cpuFlags = await this.getCpuFlags(); isSupportedCpu = REQUIRED_CPU_FLAGS.every(required => cpuFlags.includes(required)); } - + if (!isSupportedCpu) { console.error('This CPU cannot use TensorFlow.'); return null; } - + const tf = await import('@tensorflow/tfjs-node'); - - if (this.model == null) this.model = await nsfw.load(`file://${_dirname}/../../nsfw-model/`, { size: 299 }); - + + if (this.model == null) { + await this.modelLoadMutex.runExclusive(async () => { + if (this.model == null) { + this.model = await nsfw.load(`file://${_dirname}/../../nsfw-model/`, { size: 299 }); + } + }); + } + const buffer = await fs.promises.readFile(path); const image = await tf.node.decodeImage(buffer, 3) as any; try { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 063ec0ed10ae..0ed97ba293e4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -140,6 +140,9 @@ importers: archiver: specifier: 5.3.1 version: 5.3.1 + async-mutex: + specifier: ^0.4.0 + version: 0.4.0 autwh: specifier: 0.1.0 version: 0.1.0 @@ -1195,13 +1198,13 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/chunked-blob-reader@3.310.0: resolution: {integrity: sha512-CrJS3exo4mWaLnWxfCH+w88Ou0IcAZSIkk4QbmxiHl/5Dq705OLoxf4385MVyExpqpeVJYOYQ2WaD8i/pQZ2fg==} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/client-s3@3.321.1: @@ -1386,7 +1389,7 @@ packages: '@aws-sdk/util-user-agent-node': 3.310.0 '@aws-sdk/util-utf8': 3.310.0 fast-xml-parser: 4.1.2 - tslib: 2.5.2 + tslib: 2.5.3 transitivePeerDependencies: - aws-crt dev: false @@ -1398,7 +1401,7 @@ packages: '@aws-sdk/types': 3.310.0 '@aws-sdk/util-config-provider': 3.310.0 '@aws-sdk/util-middleware': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/credential-provider-env@3.310.0: @@ -1407,7 +1410,7 @@ packages: dependencies: '@aws-sdk/property-provider': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/credential-provider-imds@3.310.0: @@ -1418,7 +1421,7 @@ packages: '@aws-sdk/property-provider': 3.310.0 '@aws-sdk/types': 3.310.0 '@aws-sdk/url-parser': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/credential-provider-ini@3.321.1: @@ -1433,7 +1436,7 @@ packages: '@aws-sdk/property-provider': 3.310.0 '@aws-sdk/shared-ini-file-loader': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 transitivePeerDependencies: - aws-crt dev: false @@ -1451,7 +1454,7 @@ packages: '@aws-sdk/property-provider': 3.310.0 '@aws-sdk/shared-ini-file-loader': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 transitivePeerDependencies: - aws-crt dev: false @@ -1463,7 +1466,7 @@ packages: '@aws-sdk/property-provider': 3.310.0 '@aws-sdk/shared-ini-file-loader': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/credential-provider-sso@3.321.1: @@ -1475,7 +1478,7 @@ packages: '@aws-sdk/shared-ini-file-loader': 3.310.0 '@aws-sdk/token-providers': 3.321.1 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 transitivePeerDependencies: - aws-crt dev: false @@ -1486,7 +1489,7 @@ packages: dependencies: '@aws-sdk/property-provider': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/eventstream-codec@3.310.0: @@ -1504,7 +1507,7 @@ packages: dependencies: '@aws-sdk/eventstream-serde-universal': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/eventstream-serde-config-resolver@3.310.0: @@ -1512,7 +1515,7 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/eventstream-serde-node@3.310.0: @@ -1521,7 +1524,7 @@ packages: dependencies: '@aws-sdk/eventstream-serde-universal': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/eventstream-serde-universal@3.310.0: @@ -1530,7 +1533,7 @@ packages: dependencies: '@aws-sdk/eventstream-codec': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/fetch-http-handler@3.310.0: @@ -1540,7 +1543,7 @@ packages: '@aws-sdk/querystring-builder': 3.310.0 '@aws-sdk/types': 3.310.0 '@aws-sdk/util-base64': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/hash-blob-browser@3.310.0: @@ -1548,7 +1551,7 @@ packages: dependencies: '@aws-sdk/chunked-blob-reader': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/hash-node@3.310.0: @@ -1558,7 +1561,7 @@ packages: '@aws-sdk/types': 3.310.0 '@aws-sdk/util-buffer-from': 3.310.0 '@aws-sdk/util-utf8': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/hash-stream-node@3.310.0: @@ -1567,21 +1570,21 @@ packages: dependencies: '@aws-sdk/types': 3.310.0 '@aws-sdk/util-utf8': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/invalid-dependency@3.310.0: resolution: {integrity: sha512-1s5RG5rSPXoa/aZ/Kqr5U/7lqpx+Ry81GprQ2bxWqJvWQIJ0IRUwo5pk8XFxbKVr/2a+4lZT/c3OGoBOM1yRRA==} dependencies: '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/is-array-buffer@3.310.0: resolution: {integrity: sha512-urnbcCR+h9NWUnmOtet/s4ghvzsidFmspfhYaHAmSRdy9yDjdjBJMFjjsn85A1ODUktztm+cVncXjQ38WCMjMQ==} engines: {node: '>=14.0.0'} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/lib-storage@3.321.1(@aws-sdk/abort-controller@3.310.0)(@aws-sdk/client-s3@3.321.1): @@ -1606,7 +1609,7 @@ packages: dependencies: '@aws-sdk/types': 3.310.0 '@aws-sdk/util-utf8': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-bucket-endpoint@3.310.0: @@ -1617,7 +1620,7 @@ packages: '@aws-sdk/types': 3.310.0 '@aws-sdk/util-arn-parser': 3.310.0 '@aws-sdk/util-config-provider': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-content-length@3.310.0: @@ -1626,7 +1629,7 @@ packages: dependencies: '@aws-sdk/protocol-http': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-endpoint@3.310.0: @@ -1637,7 +1640,7 @@ packages: '@aws-sdk/types': 3.310.0 '@aws-sdk/url-parser': 3.310.0 '@aws-sdk/util-middleware': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-expect-continue@3.310.0: @@ -1646,7 +1649,7 @@ packages: dependencies: '@aws-sdk/protocol-http': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-flexible-checksums@3.310.0: @@ -1659,7 +1662,7 @@ packages: '@aws-sdk/protocol-http': 3.310.0 '@aws-sdk/types': 3.310.0 '@aws-sdk/util-utf8': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-host-header@3.310.0: @@ -1668,7 +1671,7 @@ packages: dependencies: '@aws-sdk/protocol-http': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-location-constraint@3.310.0: @@ -1676,7 +1679,7 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-logger@3.310.0: @@ -1684,7 +1687,7 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-recursion-detection@3.310.0: @@ -1693,7 +1696,7 @@ packages: dependencies: '@aws-sdk/protocol-http': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-retry@3.310.0: @@ -1705,7 +1708,7 @@ packages: '@aws-sdk/types': 3.310.0 '@aws-sdk/util-middleware': 3.310.0 '@aws-sdk/util-retry': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 uuid: 8.3.2 dev: false @@ -1716,7 +1719,7 @@ packages: '@aws-sdk/protocol-http': 3.310.0 '@aws-sdk/types': 3.310.0 '@aws-sdk/util-arn-parser': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-sdk-sts@3.310.0: @@ -1725,7 +1728,7 @@ packages: dependencies: '@aws-sdk/middleware-signing': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-serde@3.310.0: @@ -1733,7 +1736,7 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-signing@3.310.0: @@ -1745,7 +1748,7 @@ packages: '@aws-sdk/signature-v4': 3.310.0 '@aws-sdk/types': 3.310.0 '@aws-sdk/util-middleware': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-ssec@3.310.0: @@ -1753,14 +1756,14 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-stack@3.310.0: resolution: {integrity: sha512-010O1PD+UAcZVKRvqEusE1KJqN96wwrf6QsqbRM0ywsKQ21NDweaHvEDlds2VHpgmofxkRLRu/IDrlPkKRQrRg==} engines: {node: '>=14.0.0'} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/middleware-user-agent@3.319.0: @@ -1770,7 +1773,7 @@ packages: '@aws-sdk/protocol-http': 3.310.0 '@aws-sdk/types': 3.310.0 '@aws-sdk/util-endpoints': 3.319.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/node-config-provider@3.310.0: @@ -1780,7 +1783,7 @@ packages: '@aws-sdk/property-provider': 3.310.0 '@aws-sdk/shared-ini-file-loader': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/node-http-handler@3.321.1: @@ -1799,7 +1802,7 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/protocol-http@3.310.0: @@ -1807,7 +1810,7 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/querystring-builder@3.310.0: @@ -1816,7 +1819,7 @@ packages: dependencies: '@aws-sdk/types': 3.310.0 '@aws-sdk/util-uri-escape': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/querystring-parser@3.310.0: @@ -1824,7 +1827,7 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/service-error-classification@3.310.0: @@ -1837,7 +1840,7 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/signature-v4-multi-region@3.310.0: @@ -1852,7 +1855,7 @@ packages: '@aws-sdk/protocol-http': 3.310.0 '@aws-sdk/signature-v4': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/signature-v4@3.310.0: @@ -1865,7 +1868,7 @@ packages: '@aws-sdk/util-middleware': 3.310.0 '@aws-sdk/util-uri-escape': 3.310.0 '@aws-sdk/util-utf8': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/smithy-client@3.316.0: @@ -1874,7 +1877,7 @@ packages: dependencies: '@aws-sdk/middleware-stack': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/token-providers@3.321.1: @@ -1894,7 +1897,7 @@ packages: resolution: {integrity: sha512-j8eamQJ7YcIhw7fneUfs8LYl3t01k4uHi4ZDmNRgtbmbmTTG3FZc2MotStZnp3nZB6vLiPF1o5aoJxWVvkzS6A==} engines: {node: '>=14.0.0'} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/url-parser@3.310.0: @@ -1902,14 +1905,14 @@ packages: dependencies: '@aws-sdk/querystring-parser': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-arn-parser@3.310.0: resolution: {integrity: sha512-jL8509owp/xB9+Or0pvn3Fe+b94qfklc2yPowZZIFAkFcCSIdkIglz18cPDWnYAcy9JGewpMS1COXKIUhZkJsA==} engines: {node: '>=14.0.0'} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-base64@3.310.0: @@ -1917,20 +1920,20 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/util-buffer-from': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-body-length-browser@3.310.0: resolution: {integrity: sha512-sxsC3lPBGfpHtNTUoGXMQXLwjmR0zVpx0rSvzTPAuoVILVsp5AU/w5FphNPxD5OVIjNbZv9KsKTuvNTiZjDp9g==} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-body-length-node@3.310.0: resolution: {integrity: sha512-2tqGXdyKhyA6w4zz7UPoS8Ip+7sayOg9BwHNidiGm2ikbDxm1YrCfYXvCBdwaJxa4hJfRVz+aL9e+d3GqPI9pQ==} engines: {node: '>=14.0.0'} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-buffer-from@3.310.0: @@ -1938,14 +1941,14 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/is-array-buffer': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-config-provider@3.310.0: resolution: {integrity: sha512-xIBaYo8dwiojCw8vnUcIL4Z5tyfb1v3yjqyJKJWV/dqKUFOOS0U591plmXbM+M/QkXyML3ypon1f8+BoaDExrg==} engines: {node: '>=14.0.0'} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-defaults-mode-browser@3.316.0: @@ -1955,7 +1958,7 @@ packages: '@aws-sdk/property-provider': 3.310.0 '@aws-sdk/types': 3.310.0 bowser: 2.11.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-defaults-mode-node@3.316.0: @@ -1967,7 +1970,7 @@ packages: '@aws-sdk/node-config-provider': 3.310.0 '@aws-sdk/property-provider': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-endpoints@3.319.0: @@ -1975,28 +1978,28 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-hex-encoding@3.310.0: resolution: {integrity: sha512-sVN7mcCCDSJ67pI1ZMtk84SKGqyix6/0A1Ab163YKn+lFBQRMKexleZzpYzNGxYzmQS6VanP/cfU7NiLQOaSfA==} engines: {node: '>=14.0.0'} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-locate-window@3.208.0: resolution: {integrity: sha512-iua1A2+P7JJEDHVgvXrRJSvsnzG7stYSGQnBVphIUlemwl6nN5D+QrgbjECtrbxRz8asYFHSzhdhECqN+tFiBg==} engines: {node: '>=14.0.0'} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-middleware@3.310.0: resolution: {integrity: sha512-FTSUKL/eRb9X6uEZClrTe27QFXUNNp7fxYrPndZwk1hlaOP5ix+MIHBcI7pIiiY/JPfOUmPyZOu+HetlFXjWog==} engines: {node: '>=14.0.0'} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-retry@3.310.0: @@ -2004,7 +2007,7 @@ packages: engines: {node: '>= 14.0.0'} dependencies: '@aws-sdk/service-error-classification': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-stream-browser@3.310.0: @@ -2015,7 +2018,7 @@ packages: '@aws-sdk/util-base64': 3.310.0 '@aws-sdk/util-hex-encoding': 3.310.0 '@aws-sdk/util-utf8': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-stream-node@3.321.1: @@ -2025,14 +2028,14 @@ packages: '@aws-sdk/node-http-handler': 3.321.1 '@aws-sdk/types': 3.310.0 '@aws-sdk/util-buffer-from': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-uri-escape@3.310.0: resolution: {integrity: sha512-drzt+aB2qo2LgtDoiy/3sVG8w63cgLkqFIa2NFlGpUgHFWTXkqtbgf4L5QdjRGKWhmZsnqkbtL7vkSWEcYDJ4Q==} engines: {node: '>=14.0.0'} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-user-agent-browser@3.310.0: @@ -2040,7 +2043,7 @@ packages: dependencies: '@aws-sdk/types': 3.310.0 bowser: 2.11.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-user-agent-node@3.310.0: @@ -2054,13 +2057,13 @@ packages: dependencies: '@aws-sdk/node-config-provider': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-utf8-browser@3.259.0: resolution: {integrity: sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-utf8@3.310.0: @@ -2068,7 +2071,7 @@ packages: engines: {node: '>=14.0.0'} dependencies: '@aws-sdk/util-buffer-from': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/util-waiter@3.310.0: @@ -2077,14 +2080,14 @@ packages: dependencies: '@aws-sdk/abort-controller': 3.310.0 '@aws-sdk/types': 3.310.0 - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@aws-sdk/xml-builder@3.310.0: resolution: {integrity: sha512-TqELu4mOuSIKQCqj63fGVs86Yh+vBx5nHRpWKNUNhB2nPTpfbziTs5c1X358be3peVWA4wPxW7Nt53KIg1tnNw==} engines: {node: '>=14.0.0'} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: false /@babel/code-frame@7.21.4: @@ -9214,7 +9217,7 @@ packages: resolution: {integrity: sha512-O0yuUDnZeQDL+ncNGlJ78BiO4jnYI3bvMsD5prT0/nsgijG/LpNBIr63gTjVTNsiGkgQhiyCShTgxt8oXOrklA==} engines: {node: '>=4'} dependencies: - tslib: 2.5.2 + tslib: 2.5.3 dev: true /ast-types@0.15.2: @@ -9253,6 +9256,12 @@ packages: resolution: {integrity: sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==} dev: true + /async-mutex@0.4.0: + resolution: {integrity: sha512-eJFZ1YhRR8UN8eBLoNzcDPcy/jqjsg6I1AP+KvWQX80BqOSW1oJPJXDylPUEeMr2ZQvHgnQ//Lp6f3RQ1zI7HA==} + dependencies: + tslib: 2.5.3 + dev: false + /async-settle@1.0.0: resolution: {integrity: sha512-VPXfB4Vk49z1LHHodrEQ6Xf7W4gg1w0dAPROHngx7qgDjqmIQ+fXmwgGXTW/ITLai0YLSvWepJOP9EVpMnEAcw==} engines: {node: '>= 0.10'} @@ -20449,6 +20458,9 @@ packages: /tslib@2.5.2: resolution: {integrity: sha512-5svOrSA2w3iGFDs1HibEVBGbDrAY82bFQ3HZ3ixB+88nsbsWQoKqDRb5UBYAUPEzbBn6dAp5gRNXglySbx1MlA==} + /tslib@2.5.3: + resolution: {integrity: sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==} + /tsutils@3.21.0(typescript@5.0.4): resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} engines: {node: '>= 6'} From 53931ec59acf592b0e600d5cafab1174b5780e29 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Thu, 6 Jul 2023 07:51:01 +0900 Subject: [PATCH 041/562] =?UTF-8?q?=E5=A4=96=E9=83=A8=E3=82=B5=E3=83=BC?= =?UTF-8?q?=E3=83=90=E3=83=BC=E3=81=B8=E3=81=AE=E9=85=8D=E9=80=81=E3=81=8C?= =?UTF-8?q?=E8=A1=8C=E3=82=8F=E3=82=8C=E3=81=AA=E3=81=84=E5=95=8F=E9=A1=8C?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3=20(=E3=81=9F=E3=81=B6=E3=82=93)=20(?= =?UTF-8?q?#98)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/QueueService.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 1c32208c053f..cd2019548199 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -73,7 +73,6 @@ export class QueueService { }; await this.deliverQueue.addBulk(Array.from(inboxes.entries()).map(d => ({ - name: d[0], data: { user, content, From b77e7361fcf97fa17b15ec023869a3f3d28398b4 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Thu, 6 Jul 2023 19:13:39 +0900 Subject: [PATCH 042/562] =?UTF-8?q?deliverMany=20=E3=81=AB=20content=20?= =?UTF-8?q?=E3=81=AE=20null=20=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0=20(#99)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/QueueService.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index cd2019548199..13d9d7123338 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -62,6 +62,8 @@ export class QueueService { */ @bindThis public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map) { + if (content == null) return null; + const opts = { attempts: this.config.deliverJobMaxAttempts ?? 12, timeout: 1 * 60 * 1000, // 1min From 6611b74a0c6b1f60c2eac35a276af87ca4ee96c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Sun, 9 Jul 2023 02:12:56 +0900 Subject: [PATCH 043/562] =?UTF-8?q?feat:=20Meilisearch=E3=81=ABIndex?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=83=8E=E3=83=BC=E3=83=88=E3=81=AE=E7=AF=84?= =?UTF-8?q?=E5=9B=B2=E3=82=92=E8=A8=AD=E5=AE=9A=E3=81=A7=E3=81=8D=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=20(#101)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .config/example.yml | 1 + packages/backend/src/config.ts | 1 + packages/backend/src/core/SearchService.ts | 22 +++++++++++++++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.config/example.yml b/.config/example.yml index 13bf57a5aadd..1549368dfc48 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -104,6 +104,7 @@ redis: # apiKey: '' # ssl: true # index: '' +# scope: local # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 69bf812bc340..7cae0d9b7e58 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -63,6 +63,7 @@ export type Source = { apiKey: string; ssl?: boolean; index: string; + scope?: 'local' | 'global' | string[]; }; proxy?: string; diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index 9502afcc9b52..6a75e7253019 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -52,6 +52,7 @@ function compileQuery(q: Q): string { @Injectable() export class SearchService { + private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local'; private meilisearchNoteIndex: Index | null = null; constructor( @@ -92,6 +93,10 @@ export class SearchService { }, }); } + + if (config.meilisearch?.scope) { + this.meilisearchIndexScope = config.meilisearch.scope; + } } @bindThis @@ -100,7 +105,22 @@ export class SearchService { if (!['home', 'public'].includes(note.visibility)) return; if (this.meilisearch) { - this.meilisearchNoteIndex!.addDocuments([{ + switch (this.meilisearchIndexScope) { + case 'global': + break; + + case 'local': + if (note.userHost == null) break; + return; + + default: { + if (note.userHost == null) break; + if (this.meilisearchIndexScope.includes(note.userHost)) break; + return; + } + } + + await this.meilisearchNoteIndex?.addDocuments([{ id: note.id, createdAt: note.createdAt.getTime(), userId: note.userId, From 760bcf26a072201ee3516e62332d4f053a737f73 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Sun, 2 Jul 2023 09:54:26 +0900 Subject: [PATCH 044/562] =?UTF-8?q?Revert=20"ServerStatsService=20?= =?UTF-8?q?=E3=82=92=E7=84=A1=E5=8A=B9=E5=8C=96=20(#91)"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit b4f720cab697c3792cfd63fc2ed2570d509a2c25. --- packages/backend/src/boot/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/boot/common.ts b/packages/backend/src/boot/common.ts index a3bbd6bfa7ea..3995545d7f74 100644 --- a/packages/backend/src/boot/common.ts +++ b/packages/backend/src/boot/common.ts @@ -22,7 +22,7 @@ export async function server() { app.get(ChartManagementService).start(); app.get(JanitorService).start(); app.get(QueueStatsService).start(); - // app.get(ServerStatsService).start(); + app.get(ServerStatsService).start(); } return app; From eca41ca472f902df5211afdc6c62205f4b49bcf3 Mon Sep 17 00:00:00 2001 From: nenohi Date: Fri, 30 Jun 2023 06:35:20 +0900 Subject: [PATCH 045/562] =?UTF-8?q?Revert=20"admin/role/user=E3=81=A7useri?= =?UTF-8?q?d=E3=81=8C=E6=8C=87=E5=AE=9A=E3=81=A7=E3=81=8D=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=20(#90)"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit c492d450e9242c082704414d0dd9fee16acb838a. --- .../backend/src/server/api/endpoints/admin/roles/users.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/admin/roles/users.ts b/packages/backend/src/server/api/endpoints/admin/roles/users.ts index 328c0bb91bce..35edca546055 100644 --- a/packages/backend/src/server/api/endpoints/admin/roles/users.ts +++ b/packages/backend/src/server/api/endpoints/admin/roles/users.ts @@ -28,7 +28,6 @@ export const paramDef = { roleId: { type: 'string', format: 'misskey:id' }, sinceId: { type: 'string', format: 'misskey:id' }, untilId: { type: 'string', format: 'misskey:id' }, - userId: { type: 'string', format: 'misskey:id' }, limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, }, required: ['roleId'], @@ -64,10 +63,6 @@ export default class extends Endpoint { })) .innerJoinAndSelect('assign.user', 'user'); - if (ps.userId != null) { - query.andWhere('assign.userId = :userId', { userId: ps.userId }); - } - const assigns = await query .take(ps.limit) .getMany(); From 757eae25bf5cee6a8e78f2d596ff75947c7f8cc4 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Thu, 15 Jun 2023 11:40:50 +0900 Subject: [PATCH 046/562] Revert "Merge pull request #86 from MisskeyIO/revert-bullmq" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit e8cf53a1ea961364f3dcc22b35cd37edf2c7e213. Revert "外部サーバーへの配送が行われない問題を修正 (たぶん) (#98)" This reverts commit 53931ec59acf592b0e600d5cafab1174b5780e29. Revert "fix(backend): restore start method in QueueProcessorService (#87)" This reverts commit e88617e3c3418316152e2f8a62b951d71991deca. --- packages/backend/package.json | 2 +- .../backend/src/core/NoteCreateService.ts | 2 +- packages/backend/src/core/QueueModule.ts | 53 +-- packages/backend/src/core/QueueService.ts | 66 +++- .../core/activitypub/models/ApNoteService.ts | 2 +- .../backend/src/daemons/QueueStatsService.ts | 16 +- packages/backend/src/misc/prelude/time.ts | 15 +- .../src/queue/QueueProcessorService.ts | 353 +++++++++++------- packages/backend/src/queue/const.ts | 26 ++ packages/backend/src/queue/get-job-info.ts | 15 - .../AggregateRetentionProcessorService.ts | 6 +- .../CheckExpiredMutingsProcessorService.ts | 5 +- .../processors/CleanChartsProcessorService.ts | 5 +- .../queue/processors/CleanProcessorService.ts | 5 +- .../CleanRemoteFilesProcessorService.ts | 11 +- .../DeleteAccountProcessorService.ts | 4 +- .../DeleteDriveFilesProcessorService.ts | 12 +- .../processors/DeleteFileProcessorService.ts | 4 +- .../processors/DeliverProcessorService.ts | 8 +- .../EndedPollNotificationProcessorService.ts | 7 +- .../ExportAntennasProcessorService.ts | 6 +- .../ExportBlockingProcessorService.ts | 13 +- .../ExportCustomEmojisProcessorService.ts | 39 +- .../ExportFavoritesProcessorService.ts | 11 +- .../ExportFollowingProcessorService.ts | 9 +- .../ExportMutingProcessorService.ts | 13 +- .../processors/ExportNotesProcessorService.ts | 11 +- .../ExportUserListsProcessorService.ts | 9 +- .../ImportAntennasProcessorService.ts | 6 +- .../ImportBlockingProcessorService.ts | 11 +- .../ImportCustomEmojisProcessorService.ts | 6 +- .../ImportFollowingProcessorService.ts | 11 +- .../ImportMutingProcessorService.ts | 9 +- .../ImportUserListsProcessorService.ts | 7 +- .../queue/processors/InboxProcessorService.ts | 36 +- .../RelationshipProcessorService.ts | 2 +- .../ResyncChartsProcessorService.ts | 5 +- .../processors/TickChartsProcessorService.ts | 5 +- .../WebhookDeliverProcessorService.ts | 4 +- packages/frontend/src/scripts/time.ts | 15 +- pnpm-lock.yaml | 173 ++++----- 41 files changed, 524 insertions(+), 494 deletions(-) create mode 100644 packages/backend/src/queue/const.ts delete mode 100644 packages/backend/src/queue/get-job-info.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 39282efa0847..f88101a32de4 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -79,7 +79,7 @@ "autwh": "0.1.0", "bcryptjs": "2.4.3", "blurhash": "2.0.5", - "bull": "4.10.4", + "bullmq": "3.15.0", "cacheable-lookup": "6.1.0", "cbor": "9.0.0", "chalk": "5.2.0", diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 2c7c78f950c7..1c8491bf573b 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -510,7 +510,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (data.poll && data.poll.expiresAt) { const delay = data.poll.expiresAt.getTime() - Date.now(); - this.queueService.endedPollNotificationQueue.add({ + this.queueService.endedPollNotificationQueue.add(note.id, { noteId: note.id, }, { delay, diff --git a/packages/backend/src/core/QueueModule.ts b/packages/backend/src/core/QueueModule.ts index bacc59aae211..3384ca457785 100644 --- a/packages/backend/src/core/QueueModule.ts +++ b/packages/backend/src/core/QueueModule.ts @@ -1,42 +1,11 @@ import { setTimeout } from 'node:timers/promises'; import { Inject, Module, OnApplicationShutdown } from '@nestjs/common'; -import Bull from 'bull'; +import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; +import { QUEUE, baseQueueOptions } from '@/queue/const.js'; import type { Provider } from '@nestjs/common'; -import type { DeliverJobData, InboxJobData, DbJobData, ObjectStorageJobData, EndedPollNotificationJobData, WebhookDeliverJobData, RelationshipJobData, DbJobMap } from '../queue/types.js'; - -function q(config: Config, name: string, limitPerSec = -1) { - return new Bull(name, { - redis: { - port: config.redisForJobQueue.port, - host: config.redisForJobQueue.host, - family: config.redisForJobQueue.family == null ? 0 : config.redisForJobQueue.family, - password: config.redisForJobQueue.pass, - db: config.redisForJobQueue.db ?? 0, - }, - prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue` : 'queue', - limiter: limitPerSec > 0 ? { - max: limitPerSec, - duration: 1000, - } : undefined, - settings: { - backoffStrategies: { - apBackoff, - }, - }, - }); -} - -// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 -function apBackoff(attemptsMade: number, err: Error) { - const baseDelay = 60 * 1000; // 1min - const maxBackoff = 8 * 60 * 60 * 1000; // 8hours - let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay; - backoff = Math.min(backoff, maxBackoff); - backoff += Math.round(backoff * Math.random() * 0.2); - return backoff; -} +import type { DeliverJobData, InboxJobData, EndedPollNotificationJobData, WebhookDeliverJobData, RelationshipJobData } from '../queue/types.js'; export type SystemQueue = Bull.Queue>; export type EndedPollNotificationQueue = Bull.Queue; @@ -49,49 +18,49 @@ export type WebhookDeliverQueue = Bull.Queue; const $system: Provider = { provide: 'queue:system', - useFactory: (config: Config) => q(config, 'system'), + useFactory: (config: Config) => new Bull.Queue(QUEUE.SYSTEM, baseQueueOptions(config, QUEUE.SYSTEM)), inject: [DI.config], }; const $endedPollNotification: Provider = { provide: 'queue:endedPollNotification', - useFactory: (config: Config) => q(config, 'endedPollNotification'), + useFactory: (config: Config) => new Bull.Queue(QUEUE.ENDED_POLL_NOTIFICATION, baseQueueOptions(config, QUEUE.ENDED_POLL_NOTIFICATION)), inject: [DI.config], }; const $deliver: Provider = { provide: 'queue:deliver', - useFactory: (config: Config) => q(config, 'deliver', config.deliverJobPerSec ?? 128), + useFactory: (config: Config) => new Bull.Queue(QUEUE.DELIVER, baseQueueOptions(config, QUEUE.DELIVER)), inject: [DI.config], }; const $inbox: Provider = { provide: 'queue:inbox', - useFactory: (config: Config) => q(config, 'inbox', config.inboxJobPerSec ?? 16), + useFactory: (config: Config) => new Bull.Queue(QUEUE.INBOX, baseQueueOptions(config, QUEUE.INBOX)), inject: [DI.config], }; const $db: Provider = { provide: 'queue:db', - useFactory: (config: Config) => q(config, 'db'), + useFactory: (config: Config) => new Bull.Queue(QUEUE.DB, baseQueueOptions(config, QUEUE.DB)), inject: [DI.config], }; const $relationship: Provider = { provide: 'queue:relationship', - useFactory: (config: Config) => q(config, 'relationship', config.relashionshipJobPerSec ?? 64), + useFactory: (config: Config) => new Bull.Queue(QUEUE.RELATIONSHIP, baseQueueOptions(config, QUEUE.RELATIONSHIP)), inject: [DI.config], }; const $objectStorage: Provider = { provide: 'queue:objectStorage', - useFactory: (config: Config) => q(config, 'objectStorage'), + useFactory: (config: Config) => new Bull.Queue(QUEUE.OBJECT_STORAGE, baseQueueOptions(config, QUEUE.OBJECT_STORAGE)), inject: [DI.config], }; const $webhookDeliver: Provider = { provide: 'queue:webhookDeliver', - useFactory: (config: Config) => q(config, 'webhookDeliver', 64), + useFactory: (config: Config) => new Bull.Queue(QUEUE.WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.WEBHOOK_DELIVER)), inject: [DI.config], }; diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 13d9d7123338..c2c88270fd30 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -1,6 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; import { v4 as uuid } from 'uuid'; -import Bull from 'bull'; import type { IActivity } from '@/core/activitypub/type.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Webhook, webhookEventTypes } from '@/models/entities/Webhook.js'; @@ -11,6 +10,7 @@ import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; import type httpSignature from '@peertube/http-signature'; +import type * as Bull from 'bullmq'; @Injectable() export class QueueService { @@ -26,7 +26,43 @@ export class QueueService { @Inject('queue:relationship') public relationshipQueue: RelationshipQueue, @Inject('queue:objectStorage') public objectStorageQueue: ObjectStorageQueue, @Inject('queue:webhookDeliver') public webhookDeliverQueue: WebhookDeliverQueue, - ) {} + ) { + this.systemQueue.add('tickCharts', { + }, { + repeat: { pattern: '55 * * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('resyncCharts', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('cleanCharts', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('aggregateRetention', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('clean', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('checkExpiredMutings', { + }, { + repeat: { pattern: '*/5 * * * *' }, + removeOnComplete: true, + }); + } @bindThis public deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean) { @@ -42,11 +78,10 @@ export class QueueService { isSharedInbox, }; - return this.deliverQueue.add(data, { + return this.deliverQueue.add(to, data, { attempts: this.config.deliverJobMaxAttempts ?? 12, - timeout: 1 * 60 * 1000, // 1min backoff: { - type: 'apBackoff', + type: 'custom', }, removeOnComplete: true, removeOnFail: true, @@ -75,6 +110,7 @@ export class QueueService { }; await this.deliverQueue.addBulk(Array.from(inboxes.entries()).map(d => ({ + name: d[0], data: { user, content, @@ -94,11 +130,10 @@ export class QueueService { signature, }; - return this.inboxQueue.add(data, { + return this.inboxQueue.add('', data, { attempts: this.config.inboxJobMaxAttempts ?? 8, - timeout: 5 * 60 * 1000, // 5min backoff: { - type: 'apBackoff', + type: 'custom', }, removeOnComplete: true, removeOnFail: true, @@ -246,7 +281,7 @@ export class QueueService { private generateToDbJobData>(name: T, data: D): { name: string, data: D, - opts: Bull.JobOptions, + opts: Bull.JobsOptions, } { return { name, @@ -333,10 +368,10 @@ export class QueueService { } @bindThis - private generateRelationshipJobData(name: 'follow' | 'unfollow' | 'block' | 'unblock', data: RelationshipJobData, opts: Bull.JobOptions = {}): { + private generateRelationshipJobData(name: 'follow' | 'unfollow' | 'block' | 'unblock', data: RelationshipJobData, opts: Bull.JobsOptions = {}): { name: string, data: RelationshipJobData, - opts: Bull.JobOptions, + opts: Bull.JobsOptions, } { return { name, @@ -385,11 +420,10 @@ export class QueueService { eventId: uuid(), }; - return this.webhookDeliverQueue.add(data, { + return this.webhookDeliverQueue.add(webhook.id, data, { attempts: 4, - timeout: 1 * 60 * 1000, // 1min backoff: { - type: 'apBackoff', + type: 'custom', }, removeOnComplete: true, removeOnFail: true, @@ -401,11 +435,11 @@ export class QueueService { this.deliverQueue.once('cleaned', (jobs, status) => { //deliverLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); }); - this.deliverQueue.clean(0, 'delayed'); + this.deliverQueue.clean(0, Infinity, 'delayed'); this.inboxQueue.once('cleaned', (jobs, status) => { //inboxLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); }); - this.inboxQueue.clean(0, 'delayed'); + this.inboxQueue.clean(0, Infinity, 'delayed'); } } diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 7f09ae7d3372..76757f530a6d 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -18,6 +18,7 @@ import { PollService } from '@/core/PollService.js'; import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; +import { checkHttps } from '@/misc/check-https.js'; import { getOneApId, getApId, getOneApHrefNullable, validPost, isEmoji, getApType } from '../type.js'; // eslint-disable-next-line @typescript-eslint/consistent-type-imports import { ApLoggerService } from '../ApLoggerService.js'; @@ -32,7 +33,6 @@ import { ApQuestionService } from './ApQuestionService.js'; import { ApImageService } from './ApImageService.js'; import type { Resolver } from '../ApResolverService.js'; import type { IObject, IPost } from '../type.js'; -import { checkHttps } from '@/misc/check-https.js'; @Injectable() export class ApNoteService { diff --git a/packages/backend/src/daemons/QueueStatsService.ts b/packages/backend/src/daemons/QueueStatsService.ts index f6f26db1fb90..53a0d14cd78d 100644 --- a/packages/backend/src/daemons/QueueStatsService.ts +++ b/packages/backend/src/daemons/QueueStatsService.ts @@ -1,7 +1,11 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import Xev from 'xev'; +import * as Bull from 'bullmq'; import { QueueService } from '@/core/QueueService.js'; import { bindThis } from '@/decorators.js'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; +import { QUEUE, baseQueueOptions } from '@/queue/const.js'; import type { OnApplicationShutdown } from '@nestjs/common'; const ev = new Xev(); @@ -13,6 +17,9 @@ export class QueueStatsService implements OnApplicationShutdown { private intervalId: NodeJS.Timer; constructor( + @Inject(DI.config) + private config: Config, + private queueService: QueueService, ) { } @@ -31,11 +38,14 @@ export class QueueStatsService implements OnApplicationShutdown { let activeDeliverJobs = 0; let activeInboxJobs = 0; - this.queueService.deliverQueue.on('global:active', () => { + const deliverQueueEvents = new Bull.QueueEvents(QUEUE.DELIVER, baseQueueOptions(this.config, QUEUE.DELIVER)); + const inboxQueueEvents = new Bull.QueueEvents(QUEUE.INBOX, baseQueueOptions(this.config, QUEUE.INBOX)); + + deliverQueueEvents.on('active', () => { activeDeliverJobs++; }); - this.queueService.inboxQueue.on('global:active', () => { + inboxQueueEvents.on('active', () => { activeInboxJobs++; }); diff --git a/packages/backend/src/misc/prelude/time.ts b/packages/backend/src/misc/prelude/time.ts index 0da1f791395d..b21978b18689 100644 --- a/packages/backend/src/misc/prelude/time.ts +++ b/packages/backend/src/misc/prelude/time.ts @@ -5,13 +5,14 @@ const dateTimeIntervals = { }; export function dateUTC(time: number[]): Date { - const d = time.length === 2 ? Date.UTC(time[0], time[1]) - : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) - : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) - : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) - : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) - : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) - : null; + const d = + time.length === 2 ? Date.UTC(time[0], time[1]) + : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) + : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) + : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) + : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) + : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) + : null; if (!d) throw new Error('wrong number of arguments'); diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 566d855308dc..42f9c1af7d7a 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -1,10 +1,9 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; +import * as Bull from 'bullmq'; import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; import type Logger from '@/logger.js'; -import { QueueService } from '@/core/QueueService.js'; import { bindThis } from '@/decorators.js'; -import { getJobInfo } from './get-job-info.js'; import { WebhookDeliverProcessorService } from './processors/WebhookDeliverProcessorService.js'; import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js'; import { DeliverProcessorService } from './processors/DeliverProcessorService.js'; @@ -35,17 +34,51 @@ import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMu import { CleanProcessorService } from './processors/CleanProcessorService.js'; import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; import { QueueLoggerService } from './QueueLoggerService.js'; +import { QUEUE, baseQueueOptions } from './const.js'; + +// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 +function httpRelatedBackoff(attemptsMade: number) { + const baseDelay = 60 * 1000; // 1min + const maxBackoff = 8 * 60 * 60 * 1000; // 8hours + let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay; + backoff = Math.min(backoff, maxBackoff); + backoff += Math.round(backoff * Math.random() * 0.2); + return backoff; +} + +function getJobInfo(job: Bull.Job | undefined, increment = false): string { + if (job == null) return '-'; + + const age = Date.now() - job.timestamp; + + const formated = age > 60000 ? `${Math.floor(age / 1000 / 60)}m` + : age > 10000 ? `${Math.floor(age / 1000)}s` + : `${age}ms`; + + // onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする + const currentAttempts = job.attemptsMade + (increment ? 1 : 0); + const maxAttempts = job.opts ? job.opts.attempts : 0; + + return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`; +} @Injectable() export class QueueProcessorService implements OnApplicationShutdown { private logger: Logger; + private systemQueueWorker: Bull.Worker; + private dbQueueWorker: Bull.Worker; + private deliverQueueWorker: Bull.Worker; + private inboxQueueWorker: Bull.Worker; + private webhookDeliverQueueWorker: Bull.Worker; + private relationshipQueueWorker: Bull.Worker; + private objectStorageQueueWorker: Bull.Worker; + private endedPollNotificationQueueWorker: Bull.Worker; constructor( @Inject(DI.config) private config: Config, private queueLoggerService: QueueLoggerService, - private queueService: QueueService, private webhookDeliverProcessorService: WebhookDeliverProcessorService, private endedPollNotificationProcessorService: EndedPollNotificationProcessorService, private deliverProcessorService: DeliverProcessorService, @@ -77,10 +110,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private cleanProcessorService: CleanProcessorService, ) { this.logger = this.queueLoggerService.logger; - } - @bindThis - public start(): void { function renderError(e: Error): any { if (e) { // 何故かeがundefinedで来ることがある return { @@ -97,160 +127,227 @@ export class QueueProcessorService implements OnApplicationShutdown { } } + //#region system + this.systemQueueWorker = new Bull.Worker(QUEUE.SYSTEM, (job) => { + switch (job.name) { + case 'tickCharts': return this.tickChartsProcessorService.process(); + case 'resyncCharts': return this.resyncChartsProcessorService.process(); + case 'cleanCharts': return this.cleanChartsProcessorService.process(); + case 'aggregateRetention': return this.aggregateRetentionProcessorService.process(); + case 'checkExpiredMutings': return this.checkExpiredMutingsProcessorService.process(); + case 'clean': return this.cleanProcessorService.process(); + default: throw new Error(`unrecognized job type ${job.name} for system`); + } + }, { + ...baseQueueOptions(this.config, QUEUE.SYSTEM), + autorun: false, + }); + const systemLogger = this.logger.createSubLogger('system'); - const deliverLogger = this.logger.createSubLogger('deliver'); - const webhookLogger = this.logger.createSubLogger('webhook'); - const inboxLogger = this.logger.createSubLogger('inbox'); - const dbLogger = this.logger.createSubLogger('db'); - const relationshipLogger = this.logger.createSubLogger('relationship'); - const objectStorageLogger = this.logger.createSubLogger('objectStorage'); - this.queueService.systemQueue - .on('waiting', (jobId) => systemLogger.debug(`waiting id=${jobId}`)) + this.systemQueueWorker .on('active', (job) => systemLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => systemLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => systemLogger.warn(`stalled id=${job.id}`)); + .on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => systemLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => systemLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.deliverQueue - .on('waiting', (jobId) => deliverLogger.debug(`waiting id=${jobId}`)) + //#region db + this.dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => { + switch (job.name) { + case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job); + case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job); + case 'exportNotes': return this.exportNotesProcessorService.process(job); + case 'exportFavorites': return this.exportFavoritesProcessorService.process(job); + case 'exportFollowing': return this.exportFollowingProcessorService.process(job); + case 'exportMuting': return this.exportMutingProcessorService.process(job); + case 'exportBlocking': return this.exportBlockingProcessorService.process(job); + case 'exportUserLists': return this.exportUserListsProcessorService.process(job); + case 'exportAntennas': return this.exportAntennasProcessorService.process(job); + case 'importFollowing': return this.importFollowingProcessorService.process(job); + case 'importFollowingToDb': return this.importFollowingProcessorService.processDb(job); + case 'importMuting': return this.importMutingProcessorService.process(job); + case 'importBlocking': return this.importBlockingProcessorService.process(job); + case 'importBlockingToDb': return this.importBlockingProcessorService.processDb(job); + case 'importUserLists': return this.importUserListsProcessorService.process(job); + case 'importCustomEmojis': return this.importCustomEmojisProcessorService.process(job); + case 'importAntennas': return this.importAntennasProcessorService.process(job); + case 'deleteAccount': return this.deleteAccountProcessorService.process(job); + default: throw new Error(`unrecognized job type ${job.name} for db`); + } + }, { + ...baseQueueOptions(this.config, QUEUE.DB), + autorun: false, + }); + + const dbLogger = this.logger.createSubLogger('db'); + + this.dbQueueWorker + .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => dbLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => dbLogger.warn(`stalled id=${jobId}`)); + //#endregion + + //#region deliver + this.deliverQueueWorker = new Bull.Worker(QUEUE.DELIVER, (job) => this.deliverProcessorService.process(job), { + ...baseQueueOptions(this.config, QUEUE.DELIVER), + autorun: false, + concurrency: this.config.deliverJobConcurrency ?? 128, + limiter: { + max: this.config.deliverJobPerSec ?? 128, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); + + const deliverLogger = this.logger.createSubLogger('deliver'); + + this.deliverQueueWorker .on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`)) - .on('error', (job: any, err: Error) => deliverLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => deliverLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`)); + .on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) + .on('error', (err: Error) => deliverLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => deliverLogger.warn(`stalled id=${jobId}`)); + //#endregion + + //#region inbox + this.inboxQueueWorker = new Bull.Worker(QUEUE.INBOX, (job) => this.inboxProcessorService.process(job), { + ...baseQueueOptions(this.config, QUEUE.INBOX), + autorun: false, + concurrency: this.config.inboxJobConcurrency ?? 16, + limiter: { + max: this.config.inboxJobPerSec ?? 16, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); - this.queueService.inboxQueue - .on('waiting', (jobId) => inboxLogger.debug(`waiting id=${jobId}`)) + const inboxLogger = this.logger.createSubLogger('inbox'); + + this.inboxQueueWorker .on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`)) .on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) - .on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => inboxLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => inboxLogger.warn(`stalled ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`)); - - this.queueService.dbQueue - .on('waiting', (jobId) => dbLogger.debug(`waiting id=${jobId}`)) - .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => dbLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => dbLogger.warn(`stalled id=${job.id}`)); + .on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => inboxLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => inboxLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.relationshipQueue - .on('waiting', (jobId) => relationshipLogger.debug(`waiting id=${jobId}`)) - .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => relationshipLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => relationshipLogger.warn(`stalled id=${job.id}`)); + //#region webhook deliver + this.webhookDeliverQueueWorker = new Bull.Worker(QUEUE.WEBHOOK_DELIVER, (job) => this.webhookDeliverProcessorService.process(job), { + ...baseQueueOptions(this.config, QUEUE.WEBHOOK_DELIVER), + autorun: false, + concurrency: 64, + limiter: { + max: 64, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); - this.queueService.objectStorageQueue - .on('waiting', (jobId) => objectStorageLogger.debug(`waiting id=${jobId}`)) - .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => objectStorageLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => objectStorageLogger.warn(`stalled id=${job.id}`)); + const webhookLogger = this.logger.createSubLogger('webhook'); - this.queueService.webhookDeliverQueue - .on('waiting', (jobId) => webhookLogger.debug(`waiting id=${jobId}`)) + this.webhookDeliverQueueWorker .on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`)) - .on('error', (job: any, err: Error) => webhookLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => webhookLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`)); + .on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) + .on('error', (err: Error) => webhookLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => webhookLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.systemQueue.add('tickCharts', { + //#region relationship + this.relationshipQueueWorker = new Bull.Worker(QUEUE.RELATIONSHIP, (job) => { + switch (job.name) { + case 'follow': return this.relationshipProcessorService.processFollow(job); + case 'unfollow': return this.relationshipProcessorService.processUnfollow(job); + case 'block': return this.relationshipProcessorService.processBlock(job); + case 'unblock': return this.relationshipProcessorService.processUnblock(job); + default: throw new Error(`unrecognized job type ${job.name} for relationship`); + } }, { - repeat: { cron: '55 * * * *' }, - removeOnComplete: true, + ...baseQueueOptions(this.config, QUEUE.RELATIONSHIP), + autorun: false, + concurrency: this.config.relashionshipJobConcurrency ?? 16, + limiter: { + max: this.config.relashionshipJobPerSec ?? 64, + duration: 1000, + }, }); - this.queueService.systemQueue.add('resyncCharts', { - }, { - repeat: { cron: '0 0 * * *' }, - removeOnComplete: true, - }); + const relationshipLogger = this.logger.createSubLogger('relationship'); + + this.relationshipQueueWorker + .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => relationshipLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => relationshipLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.systemQueue.add('cleanCharts', { + //#region object storage + this.objectStorageQueueWorker = new Bull.Worker(QUEUE.OBJECT_STORAGE, (job) => { + switch (job.name) { + case 'deleteFile': return this.deleteFileProcessorService.process(job); + case 'cleanRemoteFiles': return this.cleanRemoteFilesProcessorService.process(job); + default: throw new Error(`unrecognized job type ${job.name} for objectStorage`); + } }, { - repeat: { cron: '0 0 * * *' }, - removeOnComplete: true, + ...baseQueueOptions(this.config, QUEUE.OBJECT_STORAGE), + autorun: false, + concurrency: 16, }); - this.queueService.systemQueue.add('aggregateRetention', { - }, { - repeat: { cron: '0 0 * * *' }, - removeOnComplete: true, - }); + const objectStorageLogger = this.logger.createSubLogger('objectStorage'); - this.queueService.systemQueue.add('clean', { - }, { - repeat: { cron: '0 0 * * *' }, - removeOnComplete: true, - }); + this.objectStorageQueueWorker + .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => objectStorageLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => objectStorageLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.systemQueue.add('checkExpiredMutings', { - }, { - repeat: { cron: '*/5 * * * *' }, - removeOnComplete: true, + //#region ended poll notification + this.endedPollNotificationQueueWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => this.endedPollNotificationProcessorService.process(job), { + ...baseQueueOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION), + autorun: false, }); + //#endregion + } - this.queueService.deliverQueue.process(this.config.deliverJobConcurrency ?? 128, (job) => this.deliverProcessorService.process(job)); - this.queueService.inboxQueue.process(this.config.inboxJobConcurrency ?? 16, (job) => this.inboxProcessorService.process(job)); - this.queueService.endedPollNotificationQueue.process((job, done) => this.endedPollNotificationProcessorService.process(job, done)); - this.queueService.webhookDeliverQueue.process(64, (job) => this.webhookDeliverProcessorService.process(job)); - - this.queueService.dbQueue.process('deleteDriveFiles', (job, done) => this.deleteDriveFilesProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportCustomEmojis', (job, done) => this.exportCustomEmojisProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportNotes', (job, done) => this.exportNotesProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportFavorites', (job, done) => this.exportFavoritesProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportFollowing', (job, done) => this.exportFollowingProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportMuting', (job, done) => this.exportMutingProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportBlocking', (job, done) => this.exportBlockingProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportUserLists', (job, done) => this.exportUserListsProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportAntennas', (job, done) => this.exportAntennasProcessorService.process(job, done)); - this.queueService.dbQueue.process('importFollowing', (job, done) => this.importFollowingProcessorService.process(job, done)); - this.queueService.dbQueue.process('importFollowingToDb', (job) => this.importFollowingProcessorService.processDb(job)); - this.queueService.dbQueue.process('importMuting', (job, done) => this.importMutingProcessorService.process(job, done)); - this.queueService.dbQueue.process('importBlocking', (job, done) => this.importBlockingProcessorService.process(job, done)); - this.queueService.dbQueue.process('importBlockingToDb', (job) => this.importBlockingProcessorService.processDb(job)); - this.queueService.dbQueue.process('importUserLists', (job, done) => this.importUserListsProcessorService.process(job, done)); - this.queueService.dbQueue.process('importCustomEmojis', (job, done) => this.importCustomEmojisProcessorService.process(job, done)); - this.queueService.dbQueue.process('importAntennas', (job, done) => this.importAntennasProcessorService.process(job, done)); - this.queueService.dbQueue.process('deleteAccount', (job) => this.deleteAccountProcessorService.process(job)); - - this.queueService.objectStorageQueue.process('deleteFile', 16, (job) => this.deleteFileProcessorService.process(job)); - this.queueService.objectStorageQueue.process('cleanRemoteFiles', 16, (job, done) => this.cleanRemoteFilesProcessorService.process(job, done)); - - { - const maxJobs = this.config.relashionshipJobConcurrency ?? 16; - this.queueService.relationshipQueue.process('follow', maxJobs, (job) => this.relationshipProcessorService.processFollow(job)); - this.queueService.relationshipQueue.process('unfollow', maxJobs, (job) => this.relationshipProcessorService.processUnfollow(job)); - this.queueService.relationshipQueue.process('block', maxJobs, (job) => this.relationshipProcessorService.processBlock(job)); - this.queueService.relationshipQueue.process('unblock', maxJobs, (job) => this.relationshipProcessorService.processUnblock(job)); - } - - this.queueService.systemQueue.process('tickCharts', (job, done) => this.tickChartsProcessorService.process(job, done)); - this.queueService.systemQueue.process('resyncCharts', (job, done) => this.resyncChartsProcessorService.process(job, done)); - this.queueService.systemQueue.process('cleanCharts', (job, done) => this.cleanChartsProcessorService.process(job, done)); - this.queueService.systemQueue.process('aggregateRetention', (job, done) => this.aggregateRetentionProcessorService.process(job, done)); - this.queueService.systemQueue.process('checkExpiredMutings', (job, done) => this.checkExpiredMutingsProcessorService.process(job, done)); - this.queueService.systemQueue.process('clean', (job, done) => this.cleanProcessorService.process(job, done)); + @bindThis + public async start(): Promise { + await Promise.all([ + this.systemQueueWorker.run(), + this.dbQueueWorker.run(), + this.deliverQueueWorker.run(), + this.inboxQueueWorker.run(), + this.webhookDeliverQueueWorker.run(), + this.relationshipQueueWorker.run(), + this.objectStorageQueueWorker.run(), + this.endedPollNotificationQueueWorker.run(), + ]); } @bindThis public async stop(): Promise { await Promise.all([ - this.queueService.systemQueue.close(false), - this.queueService.endedPollNotificationQueue.close(false), - this.queueService.deliverQueue.close(false), - this.queueService.inboxQueue.close(false), - this.queueService.dbQueue.close(false), - this.queueService.relationshipQueue.close(false), - this.queueService.objectStorageQueue.close(false), - this.queueService.webhookDeliverQueue.close(false), + this.systemQueueWorker.close(), + this.dbQueueWorker.close(), + this.deliverQueueWorker.close(), + this.inboxQueueWorker.close(), + this.webhookDeliverQueueWorker.close(), + this.relationshipQueueWorker.close(), + this.objectStorageQueueWorker.close(), + this.endedPollNotificationQueueWorker.close(), ]); } diff --git a/packages/backend/src/queue/const.ts b/packages/backend/src/queue/const.ts new file mode 100644 index 000000000000..d240fe70e0c5 --- /dev/null +++ b/packages/backend/src/queue/const.ts @@ -0,0 +1,26 @@ +import { Config } from '@/config.js'; +import type * as Bull from 'bullmq'; + +export const QUEUE = { + DELIVER: 'deliver', + INBOX: 'inbox', + SYSTEM: 'system', + ENDED_POLL_NOTIFICATION: 'endedPollNotification', + DB: 'db', + RELATIONSHIP: 'relationship', + OBJECT_STORAGE: 'objectStorage', + WEBHOOK_DELIVER: 'webhookDeliver', +}; + +export function baseQueueOptions(config: Config, queueName: typeof QUEUE[keyof typeof QUEUE]): Bull.QueueOptions { + return { + connection: { + port: config.redisForJobQueue.port, + host: config.redisForJobQueue.host, + family: config.redisForJobQueue.family == null ? 0 : config.redisForJobQueue.family, + password: config.redisForJobQueue.pass, + db: config.redisForJobQueue.db ?? 0, + }, + prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue:${queueName}` : `queue:${queueName}`, + }; +} diff --git a/packages/backend/src/queue/get-job-info.ts b/packages/backend/src/queue/get-job-info.ts deleted file mode 100644 index d33e349c363c..000000000000 --- a/packages/backend/src/queue/get-job-info.ts +++ /dev/null @@ -1,15 +0,0 @@ -import Bull from 'bull'; - -export function getJobInfo(job: Bull.Job, increment = false) { - const age = Date.now() - job.timestamp; - - const formated = age > 60000 ? `${Math.floor(age / 1000 / 60)}m` - : age > 10000 ? `${Math.floor(age / 1000)}s` - : `${age}ms`; - - // onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする - const currentAttempts = job.attemptsMade + (increment ? 1 : 0); - const maxAttempts = job.opts ? job.opts.attempts : 0; - - return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`; -} diff --git a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts index e2720b4fe01e..600ce0828f91 100644 --- a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts +++ b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts @@ -9,7 +9,7 @@ import { deepClone } from '@/misc/clone.js'; import { IdService } from '@/core/IdService.js'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class AggregateRetentionProcessorService { @@ -32,7 +32,7 @@ export class AggregateRetentionProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Aggregating retention...'); const now = new Date(); @@ -62,7 +62,6 @@ export class AggregateRetentionProcessorService { } catch (err) { if (isDuplicateKeyValueError(err)) { this.logger.succ('Skip because it has already been processed by another worker.'); - done(); return; } throw err; @@ -88,6 +87,5 @@ export class AggregateRetentionProcessorService { } this.logger.succ('Retention aggregated.'); - done(); } } diff --git a/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts b/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts index 2476d71a5ebc..c4ee212bab73 100644 --- a/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts +++ b/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts @@ -7,7 +7,7 @@ import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { UserMutingService } from '@/core/UserMutingService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class CheckExpiredMutingsProcessorService { @@ -27,7 +27,7 @@ export class CheckExpiredMutingsProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Checking expired mutings...'); const expired = await this.mutingsRepository.createQueryBuilder('muting') @@ -41,6 +41,5 @@ export class CheckExpiredMutingsProcessorService { } this.logger.succ('All expired mutings checked.'); - done(); } } diff --git a/packages/backend/src/queue/processors/CleanChartsProcessorService.ts b/packages/backend/src/queue/processors/CleanChartsProcessorService.ts index b45816704207..22d7c1b4fb2d 100644 --- a/packages/backend/src/queue/processors/CleanChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanChartsProcessorService.ts @@ -16,7 +16,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class CleanChartsProcessorService { @@ -45,7 +45,7 @@ export class CleanChartsProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Clean charts...'); await Promise.all([ @@ -64,6 +64,5 @@ export class CleanChartsProcessorService { ]); this.logger.succ('All charts successfully cleaned.'); - done(); } } diff --git a/packages/backend/src/queue/processors/CleanProcessorService.ts b/packages/backend/src/queue/processors/CleanProcessorService.ts index 1936e8df23ee..cefa6da5e946 100644 --- a/packages/backend/src/queue/processors/CleanProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanProcessorService.ts @@ -7,7 +7,7 @@ import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class CleanProcessorService { @@ -36,7 +36,7 @@ export class CleanProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Cleaning...'); this.userIpsRepository.delete({ @@ -72,6 +72,5 @@ export class CleanProcessorService { } this.logger.succ('Cleaned.'); - done(); } } diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 5a33c271889c..c54bf59ae4c2 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -5,9 +5,9 @@ import type { DriveFilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; @Injectable() export class CleanRemoteFilesProcessorService { @@ -27,7 +27,7 @@ export class CleanRemoteFilesProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(job: Bull.Job>): Promise { this.logger.info('Deleting cached remote files...'); let deletedCount = 0; @@ -47,7 +47,7 @@ export class CleanRemoteFilesProcessorService { }); if (files.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -62,10 +62,9 @@ export class CleanRemoteFilesProcessorService { isLink: false, }); - job.progress(deletedCount / total); + job.updateProgress(deletedCount / total); } this.logger.succ('All cached remote files has been deleted.'); - done(); } } diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index e36a78de6aa2..39dd801af017 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -8,10 +8,10 @@ import { DriveService } from '@/core/DriveService.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Note } from '@/models/entities/Note.js'; import { EmailService } from '@/core/EmailService.js'; +import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbUserDeleteJobData } from '../types.js'; -import { bindThis } from '@/decorators.js'; @Injectable() export class DeleteAccountProcessorService { diff --git a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts index 604497cf5425..6772c5dc7662 100644 --- a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts @@ -5,10 +5,10 @@ import type { UsersRepository, DriveFilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; +import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; -import { bindThis } from '@/decorators.js'; @Injectable() export class DeleteDriveFilesProcessorService { @@ -31,12 +31,11 @@ export class DeleteDriveFilesProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Deleting drive files of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -56,7 +55,7 @@ export class DeleteDriveFilesProcessorService { }); if (files.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -71,10 +70,9 @@ export class DeleteDriveFilesProcessorService { userId: user.id, }); - job.progress(deletedCount / total); + job.updateProgress(deletedCount / total); } this.logger.succ(`All drive files (${deletedCount}) of ${user.id} has been deleted.`); - done(); } } diff --git a/packages/backend/src/queue/processors/DeleteFileProcessorService.ts b/packages/backend/src/queue/processors/DeleteFileProcessorService.ts index 2fb2f56f8def..edf87bd92174 100644 --- a/packages/backend/src/queue/processors/DeleteFileProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteFileProcessorService.ts @@ -3,10 +3,10 @@ import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; +import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { ObjectStorageFileJobData } from '../types.js'; -import { bindThis } from '@/decorators.js'; @Injectable() export class DeleteFileProcessorService { diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 380671302d69..406e9df850d8 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -1,4 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; +import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository, InstancesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; @@ -16,7 +17,6 @@ import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; import type { DeliverJobData } from '../types.js'; @Injectable() @@ -121,11 +121,9 @@ export class DeliverProcessorService { isSuspended: true, }); }); - return `${host} is gone`; + throw new Bull.UnrecoverableError(`${host} is gone`); } - // HTTPステータスコード4xxはクライアントエラーであり、それはつまり - // 何回再送しても成功することはないということなのでエラーにはしないでおく - return `${res.statusCode} ${res.statusMessage}`; + throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); } // 5xx etc. diff --git a/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts b/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts index 501ed4090a73..21501592f2cd 100644 --- a/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts +++ b/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts @@ -6,7 +6,7 @@ import type Logger from '@/logger.js'; import { NotificationService } from '@/core/NotificationService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { EndedPollNotificationJobData } from '../types.js'; @Injectable() @@ -30,10 +30,9 @@ export class EndedPollNotificationProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { const note = await this.notesRepository.findOneBy({ id: job.data.noteId }); if (note == null || !note.hasPoll) { - done(); return; } @@ -51,7 +50,5 @@ export class EndedPollNotificationProcessorService { noteId: note.id, }); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts index 894903e79b10..ac52325c8d99 100644 --- a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts @@ -12,7 +12,7 @@ import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { DBExportAntennasData } from '../types.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class ExportAntennasProcessorService { @@ -39,10 +39,9 @@ export class ExportAntennasProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } const [path, cleanup] = await createTemp(); @@ -96,7 +95,6 @@ export class ExportAntennasProcessorService { this.logger.succ('Exported to: ' + driveFile.id); } finally { cleanup(); - done(); } } } diff --git a/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts b/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts index c7b54070d6a7..eb758e162dfc 100644 --- a/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts @@ -9,10 +9,10 @@ import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; -import { bindThis } from '@/decorators.js'; @Injectable() export class ExportBlockingProcessorService { @@ -36,12 +36,11 @@ export class ExportBlockingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting blocking of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -69,7 +68,7 @@ export class ExportBlockingProcessorService { }); if (blockings.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -99,7 +98,7 @@ export class ExportBlockingProcessorService { blockerId: user.id, }); - job.progress(exportedCount / total); + job.updateProgress(exportedCount / total); } stream.end(); @@ -112,7 +111,5 @@ export class ExportBlockingProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts index b50f373ef820..3203d9f3e5c9 100644 --- a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts @@ -13,7 +13,7 @@ import { createTemp, createTempDir } from '@/misc/create-temp.js'; import { DownloadService } from '@/core/DownloadService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class ExportCustomEmojisProcessorService { @@ -37,12 +37,11 @@ export class ExportCustomEmojisProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info('Exporting custom emojis ...'); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -117,24 +116,26 @@ export class ExportCustomEmojisProcessorService { metaStream.end(); // Create archive - const [archivePath, archiveCleanup] = await createTemp(); - const archiveStream = fs.createWriteStream(archivePath); - const archive = archiver('zip', { - zlib: { level: 0 }, - }); - archiveStream.on('close', async () => { - this.logger.succ(`Exported to: ${archivePath}`); + await new Promise(async (resolve) => { + const [archivePath, archiveCleanup] = await createTemp(); + const archiveStream = fs.createWriteStream(archivePath); + const archive = archiver('zip', { + zlib: { level: 0 }, + }); + archiveStream.on('close', async () => { + this.logger.succ(`Exported to: ${archivePath}`); - const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; - const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); + const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; + const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); - this.logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); - archiveCleanup(); - done(); + this.logger.succ(`Exported to: ${driveFile.id}`); + cleanup(); + archiveCleanup(); + resolve(); + }); + archive.pipe(archiveStream); + archive.directory(path, false); + archive.finalize(); }); - archive.pipe(archiveStream); - archive.directory(path, false); - archive.finalize(); } } diff --git a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts index f2f2383a8837..76c38a6b8648 100644 --- a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts @@ -12,7 +12,7 @@ import type { Poll } from '@/models/entities/Poll.js'; import type { Note } from '@/models/entities/Note.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; @Injectable() @@ -42,12 +42,11 @@ export class ExportFavoritesProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting favorites of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -91,7 +90,7 @@ export class ExportFavoritesProcessorService { }) as (NoteFavorite & { note: Note & { user: User } })[]; if (favorites.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -112,7 +111,7 @@ export class ExportFavoritesProcessorService { userId: user.id, }); - job.progress(exportedFavoritesCount / total); + job.updateProgress(exportedFavoritesCount / total); } await write(']'); @@ -127,8 +126,6 @@ export class ExportFavoritesProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts b/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts index fa9c1ac1ea94..8726cb140278 100644 --- a/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts @@ -10,10 +10,10 @@ import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import type { Following } from '@/models/entities/Following.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbExportFollowingData } from '../types.js'; -import { bindThis } from '@/decorators.js'; @Injectable() export class ExportFollowingProcessorService { @@ -40,12 +40,11 @@ export class ExportFollowingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting following of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -116,7 +115,5 @@ export class ExportFollowingProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportMutingProcessorService.ts b/packages/backend/src/queue/processors/ExportMutingProcessorService.ts index b14bf5f5b18a..0f11a9e8435f 100644 --- a/packages/backend/src/queue/processors/ExportMutingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportMutingProcessorService.ts @@ -9,10 +9,10 @@ import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; -import { bindThis } from '@/decorators.js'; @Injectable() export class ExportMutingProcessorService { @@ -39,12 +39,11 @@ export class ExportMutingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting muting of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -73,7 +72,7 @@ export class ExportMutingProcessorService { }); if (mutes.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -103,7 +102,7 @@ export class ExportMutingProcessorService { muterId: user.id, }); - job.progress(exportedCount / total); + job.updateProgress(exportedCount / total); } stream.end(); @@ -116,7 +115,5 @@ export class ExportMutingProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts index e4f12ad10105..24fb33188373 100644 --- a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts @@ -12,7 +12,7 @@ import type { Poll } from '@/models/entities/Poll.js'; import type { Note } from '@/models/entities/Note.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; @Injectable() @@ -39,12 +39,11 @@ export class ExportNotesProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting notes of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -87,7 +86,7 @@ export class ExportNotesProcessorService { }) as Note[]; if (notes.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -108,7 +107,7 @@ export class ExportNotesProcessorService { userId: user.id, }); - job.progress(exportedNotesCount / total); + job.updateProgress(exportedNotesCount / total); } await write(']'); @@ -123,8 +122,6 @@ export class ExportNotesProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts b/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts index 54bde44044de..ec633580535a 100644 --- a/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts @@ -9,10 +9,10 @@ import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; -import { bindThis } from '@/decorators.js'; @Injectable() export class ExportUserListsProcessorService { @@ -39,12 +39,11 @@ export class ExportUserListsProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting user lists of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -92,7 +91,5 @@ export class ExportUserListsProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts index d06131b8c81b..575cad69d5c4 100644 --- a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts @@ -8,7 +8,7 @@ import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import { DBAntennaImportJobData } from '../types.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; const validate = new Ajv().compile({ type: 'object', @@ -59,7 +59,7 @@ export class ImportAntennasProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { const now = new Date(); try { for (const antenna of job.data.antenna) { @@ -89,8 +89,6 @@ export class ImportAntennasProcessorService { } } catch (err: any) { this.logger.error(err); - } finally { - done(); } } } diff --git a/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts b/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts index c34ab9e36ed3..2f1a9e5b03a1 100644 --- a/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts @@ -7,11 +7,11 @@ import * as Acct from '@/misc/acct.js'; import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; import { bindThis } from '@/decorators.js'; import { QueueService } from '@/core/QueueService.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; @Injectable() export class ImportBlockingProcessorService { @@ -34,12 +34,11 @@ export class ImportBlockingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Importing blocking of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -47,7 +46,6 @@ export class ImportBlockingProcessorService { id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -56,7 +54,6 @@ export class ImportBlockingProcessorService { this.queueService.createImportBlockingToDbJob({ id: user.id }, targets); this.logger.succ('Import jobs created'); - done(); } @bindThis diff --git a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts index 600468a286ad..d86256787157 100644 --- a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts @@ -12,7 +12,7 @@ import { DriveService } from '@/core/DriveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbUserImportJobData } from '../types.js'; // TODO: 名前衝突時の動作を選べるようにする @@ -45,14 +45,13 @@ export class ImportCustomEmojisProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info('Importing custom emojis ...'); const file = await this.driveFilesRepository.findOneBy({ id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -116,7 +115,6 @@ export class ImportCustomEmojisProcessorService { cleanup(); this.logger.succ('Imported'); - done(); }); unzipStream.pipe(extractor); this.logger.succ(`Unzipping to ${outputPath}`); diff --git a/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts b/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts index 2f2e9ada99b3..15bee9672eac 100644 --- a/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts @@ -7,11 +7,11 @@ import * as Acct from '@/misc/acct.js'; import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; import { bindThis } from '@/decorators.js'; import { QueueService } from '@/core/QueueService.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; @Injectable() export class ImportFollowingProcessorService { @@ -34,12 +34,11 @@ export class ImportFollowingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Importing following of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -47,7 +46,6 @@ export class ImportFollowingProcessorService { id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -56,7 +54,6 @@ export class ImportFollowingProcessorService { this.queueService.createImportFollowingToDbJob({ id: user.id }, targets); this.logger.succ('Import jobs created'); - done(); } @bindThis diff --git a/packages/backend/src/queue/processors/ImportMutingProcessorService.ts b/packages/backend/src/queue/processors/ImportMutingProcessorService.ts index d9ca09617633..723935cd3113 100644 --- a/packages/backend/src/queue/processors/ImportMutingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportMutingProcessorService.ts @@ -9,10 +9,10 @@ import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UserMutingService } from '@/core/UserMutingService.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbUserImportJobData } from '../types.js'; -import { bindThis } from '@/decorators.js'; @Injectable() export class ImportMutingProcessorService { @@ -38,12 +38,11 @@ export class ImportMutingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Importing muting of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -51,7 +50,6 @@ export class ImportMutingProcessorService { id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -98,6 +96,5 @@ export class ImportMutingProcessorService { } this.logger.succ('Imported'); - done(); } } diff --git a/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts b/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts index c423863410fe..824ee8157ab1 100644 --- a/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts @@ -12,7 +12,7 @@ import { IdService } from '@/core/IdService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbUserImportJobData } from '../types.js'; @Injectable() @@ -46,12 +46,11 @@ export class ImportUserListsProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Importing user lists of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -59,7 +58,6 @@ export class ImportUserListsProcessorService { id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -109,6 +107,5 @@ export class ImportUserListsProcessorService { } this.logger.succ('Imported'); - done(); } } diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 535a806d7f9c..ce1d7aaa1bfc 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -1,8 +1,8 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import httpSignature from '@peertube/http-signature'; +import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; -import type { InstancesRepository, DriveFilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { MetaService } from '@/core/MetaService.js'; @@ -23,10 +23,8 @@ import { LdSignatureService } from '@/core/activitypub/LdSignatureService.js'; import { ApInboxService } from '@/core/activitypub/ApInboxService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; import type { InboxJobData } from '../types.js'; -// ユーザーのinboxにアクティビティが届いた時の処理 @Injectable() export class InboxProcessorService { private logger: Logger; @@ -35,12 +33,6 @@ export class InboxProcessorService { @Inject(DI.config) private config: Config, - @Inject(DI.instancesRepository) - private instancesRepository: InstancesRepository, - - @Inject(DI.driveFilesRepository) - private driveFilesRepository: DriveFilesRepository, - private utilityService: UtilityService, private metaService: MetaService, private apInboxService: ApInboxService, @@ -93,10 +85,10 @@ export class InboxProcessorService { try { authUser = await this.apDbResolverService.getAuthUserFromApId(getApId(activity.actor)); } catch (err) { - // 対象が4xxならスキップ + // 対象が4xxならスキップ if (err instanceof StatusError) { if (err.isClientError) { - return `skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`; + throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`); } throw new Error(`Error in actor ${activity.actor} - ${err.statusCode ?? err}`); } @@ -105,12 +97,12 @@ export class InboxProcessorService { // それでもわからなければ終了 if (authUser == null) { - return 'skip: failed to resolve user'; + throw new Bull.UnrecoverableError('skip: failed to resolve user'); } // publicKey がなくても終了 if (authUser.key == null) { - return 'skip: failed to resolve user publicKey'; + throw new Bull.UnrecoverableError('skip: failed to resolve user publicKey'); } // HTTP-Signatureの検証 @@ -118,10 +110,10 @@ export class InboxProcessorService { // また、signatureのsignerは、activity.actorと一致する必要がある if (!httpSignatureValidated || authUser.user.uri !== activity.actor) { - // 一致しなくても、でもLD-Signatureがありそうならそっちも見る + // 一致しなくても、でもLD-Signatureがありそうならそっちも見る if (activity.signature) { if (activity.signature.type !== 'RsaSignature2017') { - return `skip: unsupported LD-signature type ${activity.signature.type}`; + throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${activity.signature.type}`); } // activity.signature.creator: https://example.oom/users/user#main-key @@ -134,32 +126,32 @@ export class InboxProcessorService { // keyIdからLD-Signatureのユーザーを取得 authUser = await this.apDbResolverService.getAuthUserFromKeyId(activity.signature.creator); if (authUser == null) { - return 'skip: LD-Signatureのユーザーが取得できませんでした'; + throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした'); } if (authUser.key == null) { - return 'skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'; + throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'); } // LD-Signature検証 const ldSignature = this.ldSignatureService.use(); const verified = await ldSignature.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); if (!verified) { - return 'skip: LD-Signatureの検証に失敗しました'; + throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました'); } // もう一度actorチェック if (authUser.user.uri !== activity.actor) { - return `skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`; + throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`); } // ブロックしてたら中断 const ldHost = this.utilityService.extractDbHost(authUser.user.uri); if (this.utilityService.isBlockedHost(meta.blockedHosts, ldHost)) { - return `Blocked request: ${ldHost}`; + throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`); } } else { - return `skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`; + throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`); } } @@ -168,7 +160,7 @@ export class InboxProcessorService { const signerHost = this.utilityService.extractDbHost(authUser.user.uri!); const activityIdHost = this.utilityService.extractDbHost(activity.id); if (signerHost !== activityIdHost) { - return `skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`; + throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`); } } diff --git a/packages/backend/src/queue/processors/RelationshipProcessorService.ts b/packages/backend/src/queue/processors/RelationshipProcessorService.ts index ff454df45550..722260d94821 100644 --- a/packages/backend/src/queue/processors/RelationshipProcessorService.ts +++ b/packages/backend/src/queue/processors/RelationshipProcessorService.ts @@ -1,5 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; diff --git a/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts b/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts index e5840f3da8c8..eab8e1e68d70 100644 --- a/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts @@ -15,7 +15,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class ResyncChartsProcessorService { @@ -43,7 +43,7 @@ export class ResyncChartsProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Resync charts...'); // TODO: ユーザーごとのチャートも更新する @@ -55,6 +55,5 @@ export class ResyncChartsProcessorService { ]); this.logger.succ('All charts successfully resynced.'); - done(); } } diff --git a/packages/backend/src/queue/processors/TickChartsProcessorService.ts b/packages/backend/src/queue/processors/TickChartsProcessorService.ts index 7ff84c15a5c0..f1696bf56775 100644 --- a/packages/backend/src/queue/processors/TickChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/TickChartsProcessorService.ts @@ -16,7 +16,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class TickChartsProcessorService { @@ -45,7 +45,7 @@ export class TickChartsProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Tick charts...'); await Promise.all([ @@ -64,6 +64,5 @@ export class TickChartsProcessorService { ]); this.logger.succ('All charts successfully ticked.'); - done(); } } diff --git a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts index a96301f5bece..8b40c167496d 100644 --- a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts @@ -1,4 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; +import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { WebhooksRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; @@ -7,7 +8,6 @@ import { HttpRequestService } from '@/core/HttpRequestService.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; import type { WebhookDeliverJobData } from '../types.js'; @Injectable() @@ -66,7 +66,7 @@ export class WebhookDeliverProcessorService { if (res instanceof StatusError) { // 4xx if (res.isClientError) { - return `${res.statusCode} ${res.statusMessage}`; + throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); } // 5xx etc. diff --git a/packages/frontend/src/scripts/time.ts b/packages/frontend/src/scripts/time.ts index 0da1f791395d..b21978b18689 100644 --- a/packages/frontend/src/scripts/time.ts +++ b/packages/frontend/src/scripts/time.ts @@ -5,13 +5,14 @@ const dateTimeIntervals = { }; export function dateUTC(time: number[]): Date { - const d = time.length === 2 ? Date.UTC(time[0], time[1]) - : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) - : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) - : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) - : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) - : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) - : null; + const d = + time.length === 2 ? Date.UTC(time[0], time[1]) + : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) + : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) + : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) + : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) + : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) + : null; if (!d) throw new Error('wrong number of arguments'); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0ed97ba293e4..9151b5cd930e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -152,9 +152,9 @@ importers: blurhash: specifier: 2.0.5 version: 2.0.5 - bull: - specifier: 4.10.4 - version: 4.10.4 + bullmq: + specifier: 3.15.0 + version: 3.15.0 cacheable-lookup: specifier: 6.1.0 version: 6.1.0 @@ -2116,7 +2116,7 @@ packages: '@babel/traverse': 7.22.4 '@babel/types': 7.22.4 convert-source-map: 1.9.0 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.0 @@ -2139,7 +2139,7 @@ packages: '@babel/traverse': 7.22.4 '@babel/types': 7.22.4 convert-source-map: 1.9.0 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.0 @@ -4325,7 +4325,7 @@ packages: '@babel/helper-split-export-declaration': 7.18.6 '@babel/parser': 7.22.4 '@babel/types': 7.22.4 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -4754,7 +4754,7 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dependencies: ajv: 6.12.6 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) espree: 9.5.2 globals: 13.19.0 ignore: 5.2.4 @@ -4919,7 +4919,7 @@ packages: engines: {node: '>=10.10.0'} dependencies: '@humanwhocodes/object-schema': 1.2.1 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -5345,48 +5345,48 @@ packages: os-filter-obj: 2.0.0 dev: false - /@msgpackr-extract/msgpackr-extract-darwin-arm64@2.2.0: - resolution: {integrity: sha512-Z9LFPzfoJi4mflGWV+rv7o7ZbMU5oAU9VmzCgL240KnqDW65Y2HFCT3MW06/ITJSnbVLacmcEJA8phywK7JinQ==} + /@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2: + resolution: {integrity: sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==} cpu: [arm64] os: [darwin] requiresBuild: true dev: false optional: true - /@msgpackr-extract/msgpackr-extract-darwin-x64@2.2.0: - resolution: {integrity: sha512-vq0tT8sjZsy4JdSqmadWVw6f66UXqUCabLmUVHZwUFzMgtgoIIQjT4VVRHKvlof3P/dMCkbMJ5hB1oJ9OWHaaw==} + /@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.2: + resolution: {integrity: sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==} cpu: [x64] os: [darwin] requiresBuild: true dev: false optional: true - /@msgpackr-extract/msgpackr-extract-linux-arm64@2.2.0: - resolution: {integrity: sha512-hlxxLdRmPyq16QCutUtP8Tm6RDWcyaLsRssaHROatgnkOxdleMTgetf9JsdncL8vLh7FVy/RN9i3XR5dnb9cRA==} + /@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.2: + resolution: {integrity: sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==} cpu: [arm64] os: [linux] requiresBuild: true dev: false optional: true - /@msgpackr-extract/msgpackr-extract-linux-arm@2.2.0: - resolution: {integrity: sha512-SaJ3Qq4lX9Syd2xEo9u3qPxi/OB+5JO/ngJKK97XDpa1C587H9EWYO6KD8995DAjSinWvdHKRrCOXVUC5fvGOg==} + /@msgpackr-extract/msgpackr-extract-linux-arm@3.0.2: + resolution: {integrity: sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==} cpu: [arm] os: [linux] requiresBuild: true dev: false optional: true - /@msgpackr-extract/msgpackr-extract-linux-x64@2.2.0: - resolution: {integrity: sha512-94y5PJrSOqUNcFKmOl7z319FelCLAE0rz/jPCWS+UtdMZvpa4jrQd+cJPQCLp2Fes1yAW/YUQj/Di6YVT3c3Iw==} + /@msgpackr-extract/msgpackr-extract-linux-x64@3.0.2: + resolution: {integrity: sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==} cpu: [x64] os: [linux] requiresBuild: true dev: false optional: true - /@msgpackr-extract/msgpackr-extract-win32-x64@2.2.0: - resolution: {integrity: sha512-XrC0JzsqQSvOyM3t04FMLO6z5gCuhPE6k4FXuLK5xf52ZbdvcFe1yBmo7meCew9B8G2f0T9iu9t3kfTYRYROgA==} + /@msgpackr-extract/msgpackr-extract-win32-x64@3.0.2: + resolution: {integrity: sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==} cpu: [x64] os: [win32] requiresBuild: true @@ -8159,7 +8159,7 @@ packages: '@typescript-eslint/scope-manager': 5.59.8 '@typescript-eslint/type-utils': 5.59.8(eslint@8.41.0)(typescript@5.1.3) '@typescript-eslint/utils': 5.59.8(eslint@8.41.0)(typescript@5.1.3) - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) eslint: 8.41.0 grapheme-splitter: 1.0.4 ignore: 5.2.4 @@ -8204,7 +8204,7 @@ packages: '@typescript-eslint/scope-manager': 5.59.8 '@typescript-eslint/types': 5.59.8 '@typescript-eslint/typescript-estree': 5.59.8(typescript@5.1.3) - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) eslint: 8.41.0 typescript: 5.1.3 transitivePeerDependencies: @@ -8259,7 +8259,7 @@ packages: dependencies: '@typescript-eslint/typescript-estree': 5.59.8(typescript@5.1.3) '@typescript-eslint/utils': 5.59.8(eslint@8.41.0)(typescript@5.1.3) - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) eslint: 8.41.0 tsutils: 3.21.0(typescript@5.1.3) typescript: 5.1.3 @@ -8309,7 +8309,7 @@ packages: dependencies: '@typescript-eslint/types': 5.59.8 '@typescript-eslint/visitor-keys': 5.59.8 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) globby: 11.1.0 is-glob: 4.0.3 semver: 7.5.1 @@ -8830,7 +8830,7 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} dependencies: - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -8838,7 +8838,7 @@ packages: resolution: {integrity: sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA==} engines: {node: '>= 8.0.0'} dependencies: - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) depd: 1.1.2 humanize-ms: 1.2.1 transitivePeerDependencies: @@ -9324,7 +9324,7 @@ packages: resolution: {integrity: sha512-bbCQdg7bpEv6kGH41RO/3B2/GMMmJSo2iBK+X8AWN9mujtfUipMDfIjsgHCfpnKqoGEQrrmCDKSa5OQ19+fDmg==} dependencies: archy: 1.0.0 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) fastq: 1.15.0 transitivePeerDependencies: - supports-color @@ -9347,7 +9347,7 @@ packages: /axios@0.24.0: resolution: {integrity: sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==} dependencies: - follow-redirects: 1.15.2 + follow-redirects: 1.15.2(debug@4.3.4) transitivePeerDependencies: - debug dev: false @@ -9858,18 +9858,17 @@ packages: dependencies: node-gyp-build: 4.6.0 - /bull@4.10.4: - resolution: {integrity: sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA==} - engines: {node: '>=12'} + /bullmq@3.15.0: + resolution: {integrity: sha512-U0LSRjuoyIBpnE62T4maCWMYEt3qdBCa1lnlPxYKQmRF/Y+FQ9W6iW5JvNNN+NA5Jet7k0uX71a93EX1zGnrhw==} dependencies: - cron-parser: 4.7.1 - debuglog: 1.0.1 - get-port: 5.1.1 + cron-parser: 4.8.1 + glob: 8.1.0 ioredis: 5.3.2 lodash: 4.17.21 - msgpackr: 1.8.1 + msgpackr: 1.9.2 semver: 7.5.1 - uuid: 8.3.2 + tslib: 2.5.2 + uuid: 9.0.0 transitivePeerDependencies: - supports-color dev: false @@ -10759,11 +10758,11 @@ packages: readable-stream: 3.6.0 dev: false - /cron-parser@4.7.1: - resolution: {integrity: sha512-WguFaoQ0hQ61SgsCZLHUcNbAvlK0lypKXu62ARguefYmjzaOXIVRNrAmyXzabTwUn4sQvQLkk6bjH+ipGfw8bA==} + /cron-parser@4.8.1: + resolution: {integrity: sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==} engines: {node: '>=12.0.0'} dependencies: - luxon: 3.2.1 + luxon: 3.3.0 dev: false /cropperjs@2.0.0-beta.2: @@ -11016,16 +11015,6 @@ packages: dependencies: ms: 2.0.0 - /debug@3.2.7: - resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - dependencies: - ms: 2.1.3 - /debug@3.2.7(supports-color@8.1.1): resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} peerDependencies: @@ -11036,18 +11025,6 @@ packages: dependencies: ms: 2.1.3 supports-color: 8.1.1 - dev: true - - /debug@4.3.4: - resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - dependencies: - ms: 2.1.2 /debug@4.3.4(supports-color@8.1.1): resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} @@ -11061,10 +11038,6 @@ packages: ms: 2.1.2 supports-color: 8.1.1 - /debuglog@1.0.1: - resolution: {integrity: sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==} - dev: false - /decamelize-keys@1.1.1: resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} engines: {node: '>=0.10.0'} @@ -11758,7 +11731,7 @@ packages: /eslint-import-resolver-node@0.3.7: resolution: {integrity: sha512-gozW2blMLJCeFpBwugLTGyvVjNoeo1knonXAcatC6bjPBZitotxdWf7Gimr25N4c0AAOo4eOUfaG82IJPDpqCA==} dependencies: - debug: 3.2.7 + debug: 3.2.7(supports-color@8.1.1) is-core-module: 2.11.0 resolve: 1.22.1 transitivePeerDependencies: @@ -11816,7 +11789,7 @@ packages: optional: true dependencies: '@typescript-eslint/parser': 5.59.8(eslint@8.41.0)(typescript@5.1.3) - debug: 3.2.7 + debug: 3.2.7(supports-color@8.1.1) eslint: 8.41.0 eslint-import-resolver-node: 0.3.7 transitivePeerDependencies: @@ -11870,7 +11843,7 @@ packages: array-includes: 3.1.6 array.prototype.flat: 1.3.1 array.prototype.flatmap: 1.3.1 - debug: 3.2.7 + debug: 3.2.7(supports-color@8.1.1) doctrine: 2.1.0 eslint: 8.41.0 eslint-import-resolver-node: 0.3.7 @@ -11996,7 +11969,7 @@ packages: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.0 @@ -12760,16 +12733,6 @@ packages: readable-stream: 2.3.7 dev: false - /follow-redirects@1.15.2: - resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - dev: false - /follow-redirects@1.15.2(debug@4.3.4): resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} engines: {node: '>=4.0'} @@ -12780,7 +12743,6 @@ packages: optional: true dependencies: debug: 4.3.4(supports-color@8.1.1) - dev: true /for-each@0.3.3: resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} @@ -13050,6 +13012,7 @@ packages: /get-port@5.1.1: resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==} engines: {node: '>=8'} + dev: true /get-stream@3.0.0: resolution: {integrity: sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==} @@ -13656,7 +13619,7 @@ packages: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) transitivePeerDependencies: - supports-color dev: false @@ -13706,7 +13669,7 @@ packages: engines: {node: '>= 4.5.0'} dependencies: agent-base: 4.3.0 - debug: 3.2.7 + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color dev: false @@ -13727,7 +13690,7 @@ packages: engines: {node: '>= 6'} dependencies: agent-base: 6.0.2 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -13892,7 +13855,7 @@ packages: dependencies: '@ioredis/commands': 1.2.0 cluster-key-slot: 1.1.2 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) denque: 2.1.0 lodash.defaults: 4.2.0 lodash.isarguments: 3.1.0 @@ -14474,7 +14437,7 @@ packages: resolution: {integrity: sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==} engines: {node: '>=10'} dependencies: - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) istanbul-lib-coverage: 3.2.0 source-map: 0.6.1 transitivePeerDependencies: @@ -15757,8 +15720,8 @@ packages: engines: {node: '>=16.14'} dev: true - /luxon@3.2.1: - resolution: {integrity: sha512-QrwPArQCNLAKGO/C+ZIilgIuDnEnKx5QYODdDtbFaxzsbZcc/a7WFq7MhsVYgRlwawLtvOUESTlfJ+hc/USqPg==} + /luxon@3.3.0: + resolution: {integrity: sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==} engines: {node: '>=12'} dev: false @@ -16236,25 +16199,26 @@ packages: engines: {node: '>=12.13'} dev: false - /msgpackr-extract@2.2.0: - resolution: {integrity: sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==} + /msgpackr-extract@3.0.2: + resolution: {integrity: sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==} + hasBin: true requiresBuild: true dependencies: - node-gyp-build-optional-packages: 5.0.3 + node-gyp-build-optional-packages: 5.0.7 optionalDependencies: - '@msgpackr-extract/msgpackr-extract-darwin-arm64': 2.2.0 - '@msgpackr-extract/msgpackr-extract-darwin-x64': 2.2.0 - '@msgpackr-extract/msgpackr-extract-linux-arm': 2.2.0 - '@msgpackr-extract/msgpackr-extract-linux-arm64': 2.2.0 - '@msgpackr-extract/msgpackr-extract-linux-x64': 2.2.0 - '@msgpackr-extract/msgpackr-extract-win32-x64': 2.2.0 + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.2 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.2 dev: false optional: true - /msgpackr@1.8.1: - resolution: {integrity: sha512-05fT4J8ZqjYlR4QcRDIhLCYKUOHXk7C/xa62GzMKj74l3up9k2QZ3LgFc6qWdsPHl91QA2WLWqWc8b8t7GLNNw==} + /msgpackr@1.9.2: + resolution: {integrity: sha512-xtDgI3Xv0AAiZWLRGDchyzBwU6aq0rwJ+W+5Y4CZhEWtkl/hJtFFLc+3JtGTw7nz1yquxs7nL8q/yA2aqpflIQ==} optionalDependencies: - msgpackr-extract: 2.2.0 + msgpackr-extract: 3.0.2 dev: false /msw-storybook-addon@1.8.0(msw@1.2.1): @@ -16407,7 +16371,7 @@ packages: resolution: {integrity: sha512-6R9fqJ5Zcmf+uYaFgdIHmLwNldn5HbK8L5ybn7Uz+ylX/rnOsSp1AHcvQSrCaFN+qNM1wpymHqD7mVasEOlHGQ==} engines: {node: '>= 4.4.x'} dependencies: - debug: 3.2.7 + debug: 3.2.7(supports-color@8.1.1) iconv-lite: 0.4.24 sax: 1.2.4 transitivePeerDependencies: @@ -16507,8 +16471,9 @@ packages: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 - /node-gyp-build-optional-packages@5.0.3: - resolution: {integrity: sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==} + /node-gyp-build-optional-packages@5.0.7: + resolution: {integrity: sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==} + hasBin: true dev: false optional: true @@ -19511,7 +19476,7 @@ packages: engines: {node: '>= 10'} dependencies: agent-base: 6.0.2 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) socks: 2.7.1 transitivePeerDependencies: - supports-color @@ -20633,7 +20598,7 @@ packages: chalk: 4.1.2 cli-highlight: 2.1.11 date-fns: 2.30.0 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) dotenv: 16.0.3 glob: 8.1.0 ioredis: 5.3.2 From 6d3f64f60631c79ef3d3bf876f158ba85e367282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Mon, 24 Jul 2023 02:48:01 +0900 Subject: [PATCH 047/562] =?UTF-8?q?feat:=20=E3=83=A1=E3=83=87=E3=82=A3?= =?UTF-8?q?=E3=82=A2=E3=82=BF=E3=82=A4=E3=83=A0=E3=83=A9=E3=82=A4=E3=83=B3?= =?UTF-8?q?=E3=82=92=E8=BC=B8=E5=85=A5=20(#103)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * メディアタイムラインの実装 * refactor(stream): ストリーミングにwithFilesオプションを実装 --------- Co-authored-by: tar_bin --- locales/en-US.yml | 1 + locales/index.d.ts | 1 + locales/ja-JP.yml | 1 + .../src/server/api/stream/channels/global-timeline.ts | 5 +++++ .../src/server/api/stream/channels/home-timeline.ts | 5 +++++ .../src/server/api/stream/channels/hybrid-timeline.ts | 5 +++++ .../src/server/api/stream/channels/local-timeline.ts | 5 +++++ packages/frontend/src/components/MkTimeline.vue | 11 +++++++++++ packages/frontend/src/pages/timeline.vue | 5 +++++ packages/frontend/src/ui/deck/deck-store.ts | 2 +- packages/frontend/src/ui/deck/tl-column.vue | 3 +++ packages/frontend/src/widgets/WidgetTimeline.vue | 4 ++++ 12 files changed, 47 insertions(+), 1 deletion(-) diff --git a/locales/en-US.yml b/locales/en-US.yml index ea82525513ed..94cb9ed5030a 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1882,6 +1882,7 @@ _instanceCharts: _timelines: home: "Home" local: "Local" + media: "Media" social: "Social" global: "Global" _play: diff --git a/locales/index.d.ts b/locales/index.d.ts index db7e3e957560..31b2c2a4aa28 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -2012,6 +2012,7 @@ export interface Locale { "_timelines": { "home": string; "local": string; + "media": string; "social": string; "global": string; }; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index b2fa9c337e20..f36af0c64635 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1928,6 +1928,7 @@ _instanceCharts: _timelines: home: "ホーム" local: "ローカル" + media: "メディア" social: "ソーシャル" global: "グローバル" diff --git a/packages/backend/src/server/api/stream/channels/global-timeline.ts b/packages/backend/src/server/api/stream/channels/global-timeline.ts index d3339072c1ef..77d4f0592a63 100644 --- a/packages/backend/src/server/api/stream/channels/global-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/global-timeline.ts @@ -14,6 +14,7 @@ class GlobalTimelineChannel extends Channel { public static shouldShare = true; public static requireCredential = false; private withReplies: boolean; + private withFiles: boolean; constructor( private metaService: MetaService, @@ -33,6 +34,7 @@ class GlobalTimelineChannel extends Channel { if (!policies.gtlAvailable) return; this.withReplies = params.withReplies as boolean; + this.withFiles = params.withFiles as boolean; // Subscribe events this.subscriber.on('notesStream', this.onNote); @@ -43,6 +45,9 @@ class GlobalTimelineChannel extends Channel { if (note.visibility !== 'public') return; if (note.channelId != null) return; + // ファイルを含まない投稿は除外 + if (this.withFiles && (note.files === undefined || note.files.length === 0)) return; + // リプライなら再pack if (note.replyId != null) { note.reply = await this.noteEntityService.pack(note.replyId, this.user, { diff --git a/packages/backend/src/server/api/stream/channels/home-timeline.ts b/packages/backend/src/server/api/stream/channels/home-timeline.ts index fe0cc37b6b9e..d740b2efb731 100644 --- a/packages/backend/src/server/api/stream/channels/home-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/home-timeline.ts @@ -12,6 +12,7 @@ class HomeTimelineChannel extends Channel { public static shouldShare = true; public static requireCredential = true; private withReplies: boolean; + private withFiles: boolean; constructor( private noteEntityService: NoteEntityService, @@ -26,6 +27,7 @@ class HomeTimelineChannel extends Channel { @bindThis public async init(params: any) { this.withReplies = params.withReplies as boolean; + this.withFiles = params.withFiles as boolean; this.subscriber.on('notesStream', this.onNote); } @@ -42,6 +44,9 @@ class HomeTimelineChannel extends Channel { // Ignore notes from instances the user has muted if (isInstanceMuted(note, new Set(this.userProfile!.mutedInstances ?? []))) return; + // ファイルを含まない投稿は除外 + if (this.withFiles && (note.files === undefined || note.files.length === 0)) return; + if (['followers', 'specified'].includes(note.visibility)) { note = await this.noteEntityService.pack(note.id, this.user!, { detail: true, diff --git a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts index 5a33e13cf55b..6627621dc822 100644 --- a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts @@ -14,6 +14,7 @@ class HybridTimelineChannel extends Channel { public static shouldShare = true; public static requireCredential = true; private withReplies: boolean; + private withFiles: boolean; constructor( private metaService: MetaService, @@ -33,6 +34,7 @@ class HybridTimelineChannel extends Channel { if (!policies.ltlAvailable) return; this.withReplies = params.withReplies as boolean; + this.withFiles = params.withFiles as boolean; // Subscribe events this.subscriber.on('notesStream', this.onNote); @@ -51,6 +53,9 @@ class HybridTimelineChannel extends Channel { (note.channelId != null && this.followingChannels.has(note.channelId)) )) return; + // ファイルを含まない投稿は除外 + if (this.withFiles && (note.files === undefined || note.files.length === 0)) return; + if (['followers', 'specified'].includes(note.visibility)) { note = await this.noteEntityService.pack(note.id, this.user!, { detail: true, diff --git a/packages/backend/src/server/api/stream/channels/local-timeline.ts b/packages/backend/src/server/api/stream/channels/local-timeline.ts index 9ca4db8ced92..30be8778ede4 100644 --- a/packages/backend/src/server/api/stream/channels/local-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/local-timeline.ts @@ -13,6 +13,7 @@ class LocalTimelineChannel extends Channel { public static shouldShare = true; public static requireCredential = false; private withReplies: boolean; + private withFiles: boolean; constructor( private metaService: MetaService, @@ -32,6 +33,7 @@ class LocalTimelineChannel extends Channel { if (!policies.ltlAvailable) return; this.withReplies = params.withReplies as boolean; + this.withFiles = params.withFiles as boolean; // Subscribe events this.subscriber.on('notesStream', this.onNote); @@ -43,6 +45,9 @@ class LocalTimelineChannel extends Channel { if (note.visibility !== 'public') return; if (note.channelId != null && !this.followingChannels.has(note.channelId)) return; + // ファイルを含まない投稿は除外 + if (this.withFiles && (note.files === undefined || note.files.length === 0)) return; + // リプライなら再pack if (note.replyId != null) { note.reply = await this.noteEntityService.pack(note.replyId, this.user, { diff --git a/packages/frontend/src/components/MkTimeline.vue b/packages/frontend/src/components/MkTimeline.vue index 2595ebc45d1b..6164b03adc0d 100644 --- a/packages/frontend/src/components/MkTimeline.vue +++ b/packages/frontend/src/components/MkTimeline.vue @@ -82,6 +82,17 @@ if (props.src === 'antenna') { withReplies: defaultStore.state.showTimelineReplies, }); connection.on('note', prepend); +} else if (props.src === 'media') { + endpoint = 'notes/hybrid-timeline'; + query = { + withFiles: true, + withReplies: defaultStore.state.showTimelineReplies, + }; + connection = stream.useChannel('hybridTimeline', { + withFiles: true, + withReplies: defaultStore.state.showTimelineReplies, + }); + connection.on('note', prepend); } else if (props.src === 'social') { endpoint = 'notes/hybrid-timeline'; query = { diff --git a/packages/frontend/src/pages/timeline.vue b/packages/frontend/src/pages/timeline.vue index a441c6f72866..d085443290ae 100644 --- a/packages/frontend/src/pages/timeline.vue +++ b/packages/frontend/src/pages/timeline.vue @@ -128,6 +128,11 @@ const headerTabs = $computed(() => [{ title: i18n.ts._timelines.local, icon: 'ti ti-planet', iconOnly: true, +}, { + key: 'media', + title: i18n.ts._timelines.media, + icon: 'ti ti-photo', + iconOnly: true, }, { key: 'social', title: i18n.ts._timelines.social, diff --git a/packages/frontend/src/ui/deck/deck-store.ts b/packages/frontend/src/ui/deck/deck-store.ts index 46012078587b..0e5534b10261 100644 --- a/packages/frontend/src/ui/deck/deck-store.ts +++ b/packages/frontend/src/ui/deck/deck-store.ts @@ -24,7 +24,7 @@ export type Column = { channelId?: string; roleId?: string; includingTypes?: typeof notificationTypes[number][]; - tl?: 'home' | 'local' | 'social' | 'global'; + tl?: 'home' | 'local' | 'media' | 'social' | 'global'; }; export const deckStore = markRaw(new Storage('deck', { diff --git a/packages/frontend/src/ui/deck/tl-column.vue b/packages/frontend/src/ui/deck/tl-column.vue index 4844ad11ff18..5e16956e3832 100644 --- a/packages/frontend/src/ui/deck/tl-column.vue +++ b/packages/frontend/src/ui/deck/tl-column.vue @@ -4,6 +4,7 @@ + {{ column.name }} @@ -56,6 +57,8 @@ async function setType() { value: 'home' as const, text: i18n.ts._timelines.home, }, { value: 'local' as const, text: i18n.ts._timelines.local, + }, { + value: 'media' as const, text: i18n.ts._timelines.media, }, { value: 'social' as const, text: i18n.ts._timelines.social, }, { diff --git a/packages/frontend/src/widgets/WidgetTimeline.vue b/packages/frontend/src/widgets/WidgetTimeline.vue index 3d497c2e2377..dea0a816a51f 100644 --- a/packages/frontend/src/widgets/WidgetTimeline.vue +++ b/packages/frontend/src/widgets/WidgetTimeline.vue @@ -117,6 +117,10 @@ const choose = async (ev) => { text: i18n.ts._timelines.local, icon: 'ti ti-planet', action: () => { setSrc('local'); }, + }, { + text: i18n.ts._timelines.media, + icon: 'ti ti-photo', + action: () => { setSrc('media'); }, }, { text: i18n.ts._timelines.social, icon: 'ti ti-rocket', From 7b1efd6b973d3117d41b2e730936c1d7b3cca757 Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Mon, 24 Jul 2023 13:08:39 +0900 Subject: [PATCH 048/562] =?UTF-8?q?feat:=20=E5=80=8B=E4=BA=BA=E5=AE=9B?= =?UTF-8?q?=E3=81=A6=E3=81=8A=E7=9F=A5=E3=82=89=E3=81=9B=E6=A9=9F=E8=83=BD?= =?UTF-8?q?=20(#107)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 個人宛てお知らせ機能 * Remove unused import * Update packages/backend/src/server/api/endpoints/admin/announcements/create.ts Co-authored-by: riku6460 <17585784+riku6460@users.noreply.github.com> * Update packages/frontend/src/pages/announcements.vue Co-authored-by: riku6460 <17585784+riku6460@users.noreply.github.com> * Restore breakline * 一般向けAPIにはuserオブジェクトを提供しない * fix * Fix * Update packages/misskey-js/src/entities.ts Co-authored-by: riku6460 <17585784+riku6460@users.noreply.github.com> * Fix * Update misskey-js.api.md * Fix lint * 他のテーブルに合わせて character varying(32) にした * count クエリを1つにまとめた * user を pack するようにした * いろいろ修正 * 個人宛てのお知らせの表示を改善 * Update misskey-js.api.md * Merge migration scripts * Fix * Update packages/backend/migration/1688647797135-userannouncement.js Co-authored-by: riku6460 <17585784+riku6460@users.noreply.github.com> * Update packages/backend/src/models/entities/Announcement.ts Co-authored-by: まっちゃとーにゅ <17376330+u1-liquid@users.noreply.github.com> * Fix * Update migration script --------- Co-authored-by: CyberRex Co-authored-by: まっちゃとーにゅ <17376330+u1-liquid@users.noreply.github.com> --- locales/index.d.ts | 3 + locales/ja-JP.yml | 3 + .../1688647797135-userannouncement.js | 15 ++++ .../src/core/entities/UserEntityService.ts | 10 ++- .../src/models/entities/Announcement.ts | 13 +++ .../endpoints/admin/announcements/create.ts | 12 +++ .../api/endpoints/admin/announcements/list.ts | 41 ++++++++- .../endpoints/admin/announcements/update.ts | 15 +++- .../src/server/api/endpoints/announcements.ts | 23 ++++- .../api/endpoints/i/read-announcement.ts | 14 ++- packages/frontend/src/boot/main-boot.ts | 7 +- .../components/MkNewUserAnnouncementModal.vue | 60 +++++++++++++ .../components/MkUserAnnouncementModal.vue | 90 +++++++++++++++++++ .../src/pages/admin/announcements.vue | 58 +++++++++++- packages/frontend/src/pages/announcements.vue | 21 ++++- packages/misskey-js/etc/misskey-js.api.md | 3 + packages/misskey-js/src/api.types.ts | 2 +- packages/misskey-js/src/entities.ts | 2 + 18 files changed, 374 insertions(+), 18 deletions(-) create mode 100644 packages/backend/migration/1688647797135-userannouncement.js create mode 100644 packages/frontend/src/components/MkNewUserAnnouncementModal.vue create mode 100644 packages/frontend/src/components/MkUserAnnouncementModal.vue diff --git a/locales/index.d.ts b/locales/index.d.ts index 31b2c2a4aa28..9b0823ad6b5c 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -1074,6 +1074,9 @@ export interface Locale { "additionalEmojiDictionary": string; "installed": string; "branding": string; + "newUserAnnouncementAvailable": string; + "viewAnnouncement": string; + "dialogCloseDuration": string; "enableServerMachineStats": string; "enableIdenticonGeneration": string; "turnOffToImprovePerformance": string; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index f36af0c64635..27a05cefb8cf 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1071,6 +1071,9 @@ goToMisskey: "Misskeyへ" additionalEmojiDictionary: "絵文字の追加辞書" installed: "インストール済み" branding: "ブランディング" +newUserAnnouncementAvailable: "新着のあなた宛てのお知らせがあります" +viewAnnouncement: "お知らせを見る" +dialogCloseDuration: "ダイアログを閉じるまでの待機時間" enableServerMachineStats: "サーバーのマシン情報を公開する" enableIdenticonGeneration: "ユーザーごとのIdenticon生成を有効にする" turnOffToImprovePerformance: "オフにするとパフォーマンスが向上します。" diff --git a/packages/backend/migration/1688647797135-userannouncement.js b/packages/backend/migration/1688647797135-userannouncement.js new file mode 100644 index 000000000000..422f3dd27ed6 --- /dev/null +++ b/packages/backend/migration/1688647797135-userannouncement.js @@ -0,0 +1,15 @@ +export class Userannouncement1688647797135 { + name = 'Userannouncement1688647797135' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "announcement" ADD COLUMN "userId" character varying(32)`); + await queryRunner.query(`ALTER TABLE "announcement" ADD COLUMN "closeDuration" integer NOT NULL DEFAULT 0`); + await queryRunner.query(`CREATE INDEX "IDX_fd25dfe3da37df1715f11ba6ec" ON "announcement" ("userId") `); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_fd25dfe3da37df1715f11ba6ec"`); + await queryRunner.query(`ALTER TABLE "announcement" DROP COLUMN "userId"`); + await queryRunner.query(`ALTER TABLE "announcement" DROP COLUMN "closeDuration"`); + } +} diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 7d248f8524a1..3fd1768426f6 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -1,5 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; -import { In, Not } from 'typeorm'; +import { In, IsNull, Not } from 'typeorm'; import * as Redis from 'ioredis'; import _Ajv from 'ajv'; import { ModuleRef } from '@nestjs/core'; @@ -218,9 +218,11 @@ export class UserEntityService implements OnModuleInit { userId: userId, }); - const count = await this.announcementsRepository.countBy(reads.length > 0 ? { - id: Not(In(reads.map(read => read.announcementId))), - } : {}); + const id = reads.length > 0 ? Not(In(reads.map(read => read.announcementId))) : undefined; + const count = await this.announcementsRepository.countBy([ + { id, userId: IsNull() }, + { id, userId: userId }, + ]); return count > 0; } diff --git a/packages/backend/src/models/entities/Announcement.ts b/packages/backend/src/models/entities/Announcement.ts index beb2f82462a6..3fd3cb0a4603 100644 --- a/packages/backend/src/models/entities/Announcement.ts +++ b/packages/backend/src/models/entities/Announcement.ts @@ -33,6 +33,19 @@ export class Announcement { }) public imageUrl: string | null; + @Index() + @Column('varchar', { + ...id(), + nullable: true, + }) + public userId: string | null; + + @Column('integer', { + nullable: false, + default: 0, + }) + public closeDuration: number; + constructor(data: Partial) { if (data == null) return; diff --git a/packages/backend/src/server/api/endpoints/admin/announcements/create.ts b/packages/backend/src/server/api/endpoints/admin/announcements/create.ts index 751b6be7f4a0..04d7336686b5 100644 --- a/packages/backend/src/server/api/endpoints/admin/announcements/create.ts +++ b/packages/backend/src/server/api/endpoints/admin/announcements/create.ts @@ -42,6 +42,14 @@ export const meta = { type: 'string', optional: false, nullable: true, }, + userId: { + type: 'string', + optional: false, nullable: true, + }, + closeDuration: { + type: 'number', + optional: false, nullable: false, + }, }, }, } as const; @@ -52,6 +60,8 @@ export const paramDef = { title: { type: 'string', minLength: 1 }, text: { type: 'string', minLength: 1 }, imageUrl: { type: 'string', nullable: true, minLength: 1 }, + userId: { type: 'string', nullable: true, format: 'misskey:id' }, + closeDuration: { type: 'number', nullable: false }, }, required: ['title', 'text', 'imageUrl'], } as const; @@ -73,6 +83,8 @@ export default class extends Endpoint { title: ps.title, text: ps.text, imageUrl: ps.imageUrl, + userId: ps.userId ?? null, + closeDuration: ps.closeDuration, }).then(x => this.announcementsRepository.findOneByOrFail(x.identifiers[0])); return Object.assign({}, announcement, { createdAt: announcement.createdAt.toISOString(), updatedAt: null }); diff --git a/packages/backend/src/server/api/endpoints/admin/announcements/list.ts b/packages/backend/src/server/api/endpoints/admin/announcements/list.ts index 11231f6e0471..ffcbc0a695fd 100644 --- a/packages/backend/src/server/api/endpoints/admin/announcements/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/announcements/list.ts @@ -1,8 +1,10 @@ import { Inject, Injectable } from '@nestjs/common'; -import type { AnnouncementsRepository, AnnouncementReadsRepository } from '@/models/index.js'; +import { In } from 'typeorm'; +import type { AnnouncementsRepository, AnnouncementReadsRepository, UsersRepository } from '@/models/index.js'; import type { Announcement } from '@/models/entities/Announcement.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueryService } from '@/core/QueryService.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DI } from '@/di-symbols.js'; export const meta = { @@ -46,10 +48,23 @@ export const meta = { type: 'string', optional: false, nullable: true, }, + userId: { + type: 'string', + optional: false, nullable: true, + }, + user: { + type: 'object', + optional: true, nullable: false, + ref: 'UserLite', + }, reads: { type: 'number', optional: false, nullable: false, }, + closeDuration: { + type: 'number', + optional: false, nullable: false, + }, }, }, }, @@ -61,6 +76,7 @@ export const paramDef = { limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, sinceId: { type: 'string', format: 'misskey:id' }, untilId: { type: 'string', format: 'misskey:id' }, + userId: { type: 'string', format: 'misskey:id' }, }, required: [], } as const; @@ -75,10 +91,21 @@ export default class extends Endpoint { @Inject(DI.announcementReadsRepository) private announcementReadsRepository: AnnouncementReadsRepository, + @Inject(DI.usersRepository) + private usersRepository: UsersRepository, + private queryService: QueryService, + private userEntityService: UserEntityService, ) { super(meta, paramDef, async (ps, me) => { - const query = this.queryService.makePaginationQuery(this.announcementsRepository.createQueryBuilder('announcement'), ps.sinceId, ps.untilId); + const builder = this.announcementsRepository.createQueryBuilder('announcement'); + if (ps.userId) { + builder.where('"userId" = :userId', { userId: ps.userId }); + } else { + builder.where('"userId" IS NULL'); + } + + const query = this.queryService.makePaginationQuery(builder, ps.sinceId, ps.untilId); const announcements = await query.limit(ps.limit).getMany(); @@ -90,6 +117,13 @@ export default class extends Endpoint { })); } + const users = await this.usersRepository.findBy({ + id: In(announcements.map(a => a.userId).filter(id => id != null)), + }); + const packedUsers = await this.userEntityService.packMany(users, me, { + detail: false, + }); + return announcements.map(announcement => ({ id: announcement.id, createdAt: announcement.createdAt.toISOString(), @@ -97,7 +131,10 @@ export default class extends Endpoint { title: announcement.title, text: announcement.text, imageUrl: announcement.imageUrl, + userId: announcement.userId, + user: packedUsers.find(user => user.id === announcement.userId), reads: reads.get(announcement)!, + closeDuration: announcement.closeDuration, })); }); } diff --git a/packages/backend/src/server/api/endpoints/admin/announcements/update.ts b/packages/backend/src/server/api/endpoints/admin/announcements/update.ts index 8cf9341a7162..e9e6b7f9c755 100644 --- a/packages/backend/src/server/api/endpoints/admin/announcements/update.ts +++ b/packages/backend/src/server/api/endpoints/admin/announcements/update.ts @@ -1,6 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { AnnouncementsRepository } from '@/models/index.js'; +import type { AnnouncementReadsRepository, AnnouncementsRepository } from '@/models/index.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../../error.js'; @@ -26,8 +26,10 @@ export const paramDef = { title: { type: 'string', minLength: 1 }, text: { type: 'string', minLength: 1 }, imageUrl: { type: 'string', nullable: true, minLength: 0 }, + userId: { type: 'string', nullable: true, format: 'misskey:id' }, + closeDuration: { type: 'number', nullable: false }, }, - required: ['id', 'title', 'text', 'imageUrl'], + required: ['id', 'title', 'text', 'imageUrl', 'closeDuration'], } as const; // eslint-disable-next-line import/no-default-export @@ -36,18 +38,27 @@ export default class extends Endpoint { constructor( @Inject(DI.announcementsRepository) private announcementsRepository: AnnouncementsRepository, + + @Inject(DI.announcementReadsRepository) + private announcementsReadsRepository: AnnouncementReadsRepository, ) { super(meta, paramDef, async (ps, me) => { const announcement = await this.announcementsRepository.findOneBy({ id: ps.id }); if (announcement == null) throw new ApiError(meta.errors.noSuchAnnouncement); + if (announcement.userId && announcement.userId !== ps.userId) { + await this.announcementsReadsRepository.delete({ id: announcement.id, userId: announcement.userId }); + } + await this.announcementsRepository.update(announcement.id, { updatedAt: new Date(), title: ps.title, text: ps.text, /* eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- 空の文字列の場合、nullを渡すようにするため */ imageUrl: ps.imageUrl || null, + userId: ps.userId ?? null, + closeDuration: ps.closeDuration, }); }); } diff --git a/packages/backend/src/server/api/endpoints/announcements.ts b/packages/backend/src/server/api/endpoints/announcements.ts index 735af51ee2b1..3fbf94eb41eb 100644 --- a/packages/backend/src/server/api/endpoints/announcements.ts +++ b/packages/backend/src/server/api/endpoints/announcements.ts @@ -48,6 +48,14 @@ export const meta = { type: 'boolean', optional: true, nullable: false, }, + isPrivate: { + type: 'boolean', + optional: false, nullable: true, + }, + closeDuration: { + type: 'number', + optional: false, nullable: false, + }, }, }, }, @@ -60,6 +68,7 @@ export const paramDef = { withUnreads: { type: 'boolean', default: false }, sinceId: { type: 'string', format: 'misskey:id' }, untilId: { type: 'string', format: 'misskey:id' }, + privateOnly: { type: 'boolean', default: false }, }, required: [], } as const; @@ -77,8 +86,19 @@ export default class extends Endpoint { private queryService: QueryService, ) { super(meta, paramDef, async (ps, me) => { - const query = this.queryService.makePaginationQuery(this.announcementsRepository.createQueryBuilder('announcement'), ps.sinceId, ps.untilId); + const builder = this.announcementsRepository.createQueryBuilder('announcement'); + if (me) { + if (ps.privateOnly) { + builder.where('"userId" = :userId', { userId: me.id }); + } else { + builder.where('"userId" IS NULL'); + builder.orWhere('"userId" = :userId', { userId: me.id }); + } + } else { + builder.where('"userId" IS NULL'); + } + const query = this.queryService.makePaginationQuery(builder, ps.sinceId, ps.untilId); const announcements = await query.limit(ps.limit).getMany(); if (me) { @@ -95,6 +115,7 @@ export default class extends Endpoint { ...a, createdAt: a.createdAt.toISOString(), updatedAt: a.updatedAt?.toISOString() ?? null, + isPrivate: !!a.userId, })); }); } diff --git a/packages/backend/src/server/api/endpoints/i/read-announcement.ts b/packages/backend/src/server/api/endpoints/i/read-announcement.ts index 352fe54c5d63..5a9005625cb1 100644 --- a/packages/backend/src/server/api/endpoints/i/read-announcement.ts +++ b/packages/backend/src/server/api/endpoints/i/read-announcement.ts @@ -1,4 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; +import { IsNull } from 'typeorm'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { IdService } from '@/core/IdService.js'; import type { AnnouncementReadsRepository, AnnouncementsRepository } from '@/models/index.js'; @@ -47,7 +48,18 @@ export default class extends Endpoint { ) { super(meta, paramDef, async (ps, me) => { // Check if announcement exists - const announcementExist = await this.announcementsRepository.exist({ where: { id: ps.announcementId } }); + const announcementExist = await this.announcementsRepository.exist({ + where: [ + { + id: ps.announcementId, + userId: IsNull(), + }, + { + id: ps.announcementId, + userId: me.id, + } + ] + }); if (!announcementExist) { throw new ApiError(meta.errors.noSuchAnnouncement); diff --git a/packages/frontend/src/boot/main-boot.ts b/packages/frontend/src/boot/main-boot.ts index 0a351cd6e342..f5fc6aca9e00 100644 --- a/packages/frontend/src/boot/main-boot.ts +++ b/packages/frontend/src/boot/main-boot.ts @@ -2,7 +2,7 @@ import { computed, createApp, watch, markRaw, version as vueVersion, defineAsync import { common } from './common'; import { version, ui, lang, updateLocale } from '@/config'; import { i18n, updateI18n } from '@/i18n'; -import { confirm, alert, post, popup, toast } from '@/os'; +import { confirm, alert, post, popup, toast, api } from '@/os'; import { useStream } from '@/stream'; import * as sound from '@/scripts/sound'; import { $i, refreshAccount, login, updateAccount, signout } from '@/account'; @@ -246,6 +246,11 @@ export async function mainBoot() { main.on('myTokenRegenerated', () => { signout(); }); + + const unreadUserAnnouncementsList = await api('announcements', { privateOnly: true, withUnreads: true }); + if (unreadUserAnnouncementsList.length > 0) { + unreadUserAnnouncementsList.forEach((v) => popup(defineAsyncComponent(() => import('@/components/MkUserAnnouncementModal.vue')), { title: v.title, text: v.text, closeDuration: v.closeDuration, announcementId: v.id }, {}, 'closed')); + } } // shortcut diff --git a/packages/frontend/src/components/MkNewUserAnnouncementModal.vue b/packages/frontend/src/components/MkNewUserAnnouncementModal.vue new file mode 100644 index 000000000000..fc9335cd27b9 --- /dev/null +++ b/packages/frontend/src/components/MkNewUserAnnouncementModal.vue @@ -0,0 +1,60 @@ + + + + + diff --git a/packages/frontend/src/components/MkUserAnnouncementModal.vue b/packages/frontend/src/components/MkUserAnnouncementModal.vue new file mode 100644 index 000000000000..6d41ab56d32b --- /dev/null +++ b/packages/frontend/src/components/MkUserAnnouncementModal.vue @@ -0,0 +1,90 @@ + + + + + diff --git a/packages/frontend/src/pages/admin/announcements.vue b/packages/frontend/src/pages/admin/announcements.vue index 3cb32c1d9daa..6ae3b2fcea97 100644 --- a/packages/frontend/src/pages/admin/announcements.vue +++ b/packages/frontend/src/pages/admin/announcements.vue @@ -3,10 +3,26 @@
+ + + + + + + +
+
@{{ user.username }}
+
+ {{ i18n.ts.selectUser }} + {{ i18n.ts.remove }} +
+
+ +
- - + + @@ -14,7 +30,13 @@ + + + +

{{ i18n.t('nUsersRead', { n: announcement.reads }) }}

+ + {{ i18n.ts.specifyUser }}
{{ i18n.ts.save }} {{ i18n.ts.remove }} @@ -27,17 +49,40 @@ + + diff --git a/packages/frontend/src/pages/announcements.vue b/packages/frontend/src/pages/announcements.vue index bbfa877767c1..4312ef7fcf94 100644 --- a/packages/frontend/src/pages/announcements.vue +++ b/packages/frontend/src/pages/announcements.vue @@ -2,7 +2,7 @@ - +
@@ -10,7 +10,7 @@
- {{ i18n.ts.gotIt }} + {{ i18n.ts.gotIt }}
@@ -19,7 +19,6 @@ + + diff --git a/packages/frontend/src/components/MkFolder.vue b/packages/frontend/src/components/MkFolder.vue index 70f0cc5cda62..207661ba8546 100644 --- a/packages/frontend/src/components/MkFolder.vue +++ b/packages/frontend/src/components/MkFolder.vue @@ -108,6 +108,10 @@ onMounted(() => { const myBg = computedStyle.getPropertyValue('--panel'); bgSame = parentBg === myBg; }); + +defineExpose({ + toggle, +}); diff --git a/packages/misskey-js/etc/misskey-js.api.md b/packages/misskey-js/etc/misskey-js.api.md index 84d043d7f228..72bcedf7247a 100644 --- a/packages/misskey-js/etc/misskey-js.api.md +++ b/packages/misskey-js/etc/misskey-js.api.md @@ -430,6 +430,22 @@ export type Endpoints = { req: TODO; res: TODO; }; + 'admin/abuse-report-resolver/create': { + req: TODO; + res: TODO; + }; + 'admin/abuse-report-resolver/list': { + req: TODO; + res: TODO; + }; + 'admin/abuse-report-resolver/update': { + req: TODO; + res: TODO; + }; + 'admin/abuse-report-resolver/delete': { + req: TODO; + res: TODO; + }; 'admin/drive/clean-remote-files': { req: TODO; res: TODO; @@ -2813,7 +2829,7 @@ type UserSorting = '+follower' | '-follower' | '+createdAt' | '-createdAt' | '+u // // src/api.types.ts:16:32 - (ae-forgotten-export) The symbol "TODO" needs to be exported by the entry point index.d.ts // src/api.types.ts:18:25 - (ae-forgotten-export) The symbol "NoParams" needs to be exported by the entry point index.d.ts -// src/api.types.ts:629:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts +// src/api.types.ts:633:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts // src/streaming.types.ts:33:4 - (ae-forgotten-export) The symbol "FIXME" needs to be exported by the entry point index.d.ts // (No @packageDocumentation comment for this package) diff --git a/packages/misskey-js/src/api.types.ts b/packages/misskey-js/src/api.types.ts index 0ff910e9e529..fcfa0ed896a4 100644 --- a/packages/misskey-js/src/api.types.ts +++ b/packages/misskey-js/src/api.types.ts @@ -44,6 +44,10 @@ export type Endpoints = { 'admin/announcements/delete': { req: { id: Announcement['id'] }; res: null; }; 'admin/announcements/list': { req: TODO; res: TODO; }; 'admin/announcements/update': { req: TODO; res: TODO; }; + 'admin/abuse-report-resolver/create': { req: TODO; res: TODO; }; + 'admin/abuse-report-resolver/list': { req: TODO; res: TODO; }; + 'admin/abuse-report-resolver/update': { req: TODO; res: TODO; }; + 'admin/abuse-report-resolver/delete': { req: TODO; res: TODO; }; 'admin/drive/clean-remote-files': { req: TODO; res: TODO; }; 'admin/drive/cleanup': { req: TODO; res: TODO; }; 'admin/drive/files': { req: TODO; res: TODO; }; From 46f8a0435c378270598b703463d1c9b7f9d14ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Fri, 28 Jul 2023 04:21:59 +0900 Subject: [PATCH 060/562] =?UTF-8?q?feat:=20=E3=83=AD=E3=83=BC=E3=83=AB?= =?UTF-8?q?=E3=81=AB=E3=82=88=E3=82=8B=E3=82=B3=E3=83=B3=E3=83=86=E3=83=B3?= =?UTF-8?q?=E3=83=84=E3=81=AE=E6=93=8D=E4=BD=9C=E3=81=AE=E5=88=B6=E9=99=90?= =?UTF-8?q?=20(#120)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- locales/en-US.yml | 3 + locales/index.d.ts | 3 + locales/ja-JP.yml | 3 + packages/backend/src/core/RoleService.ts | 9 +++ .../src/core/entities/UserEntityService.ts | 10 ++-- .../backend/src/models/json-schema/user.ts | 4 ++ .../server/api/endpoints/admin/show-user.ts | 5 +- .../server/api/endpoints/antennas/create.ts | 1 + .../server/api/endpoints/antennas/delete.ts | 1 + .../server/api/endpoints/antennas/update.ts | 1 + .../server/api/endpoints/blocking/create.ts | 1 + .../server/api/endpoints/blocking/delete.ts | 1 + .../server/api/endpoints/channels/create.ts | 1 + .../server/api/endpoints/channels/favorite.ts | 1 + .../server/api/endpoints/channels/follow.ts | 1 + .../api/endpoints/channels/unfavorite.ts | 1 + .../server/api/endpoints/channels/unfollow.ts | 1 + .../server/api/endpoints/channels/update.ts | 1 + .../server/api/endpoints/clips/add-note.ts | 1 + .../src/server/api/endpoints/clips/create.ts | 1 + .../src/server/api/endpoints/clips/delete.ts | 1 + .../server/api/endpoints/clips/remove-note.ts | 1 + .../server/api/endpoints/clips/unfavorite.ts | 1 + .../src/server/api/endpoints/clips/update.ts | 1 + .../api/endpoints/drive/files/create.ts | 1 + .../api/endpoints/drive/files/delete.ts | 1 + .../api/endpoints/drive/files/update.ts | 1 + .../endpoints/drive/files/upload-from-url.ts | 1 + .../api/endpoints/drive/folders/create.ts | 1 + .../api/endpoints/drive/folders/delete.ts | 1 + .../api/endpoints/drive/folders/update.ts | 1 + .../src/server/api/endpoints/flash/create.ts | 1 + .../src/server/api/endpoints/flash/delete.ts | 1 + .../src/server/api/endpoints/flash/like.ts | 1 + .../src/server/api/endpoints/flash/unlike.ts | 1 + .../src/server/api/endpoints/flash/update.ts | 1 + .../api/endpoints/gallery/posts/create.ts | 1 + .../api/endpoints/gallery/posts/delete.ts | 1 + .../api/endpoints/gallery/posts/like.ts | 1 + .../api/endpoints/gallery/posts/unlike.ts | 1 + .../api/endpoints/gallery/posts/update.ts | 1 + .../server/api/endpoints/i/delete-account.ts | 1 + .../server/api/endpoints/i/import-antennas.ts | 2 + .../server/api/endpoints/i/import-blocking.ts | 2 + .../server/api/endpoints/i/import-muting.ts | 2 + .../api/endpoints/i/import-user-lists.ts | 3 + .../src/server/api/endpoints/i/move.ts | 3 + .../backend/src/server/api/endpoints/i/pin.ts | 2 + .../src/server/api/endpoints/i/unpin.ts | 1 + .../server/api/endpoints/i/update-email.ts | 5 +- .../src/server/api/endpoints/i/update.ts | 1 + .../server/api/endpoints/i/webhooks/create.ts | 1 + .../server/api/endpoints/i/webhooks/delete.ts | 1 + .../server/api/endpoints/i/webhooks/update.ts | 1 + .../src/server/api/endpoints/mute/create.ts | 2 + .../src/server/api/endpoints/mute/delete.ts | 1 + .../src/server/api/endpoints/notes/create.ts | 1 + .../src/server/api/endpoints/notes/delete.ts | 1 + .../api/endpoints/notes/favorites/create.ts | 2 + .../api/endpoints/notes/favorites/delete.ts | 1 + .../server/api/endpoints/notes/polls/vote.ts | 1 + .../api/endpoints/notes/reactions/create.ts | 1 + .../api/endpoints/notes/reactions/delete.ts | 1 + .../endpoints/notes/thread-muting/create.ts | 1 + .../endpoints/notes/thread-muting/delete.ts | 1 + .../server/api/endpoints/notes/unrenote.ts | 1 + .../src/server/api/endpoints/pages/create.ts | 1 + .../src/server/api/endpoints/pages/delete.ts | 1 + .../src/server/api/endpoints/pages/like.ts | 1 + .../src/server/api/endpoints/pages/unlike.ts | 1 + .../src/server/api/endpoints/pages/update.ts | 1 + .../api/endpoints/renote-mute/create.ts | 2 + .../api/endpoints/renote-mute/delete.ts | 1 + .../users/lists/create-from-public.ts | 3 + .../api/endpoints/users/lists/create.ts | 1 + .../api/endpoints/users/lists/delete.ts | 1 + .../api/endpoints/users/lists/favorite.ts | 2 + .../server/api/endpoints/users/lists/pull.ts | 1 + .../server/api/endpoints/users/lists/push.ts | 1 + .../api/endpoints/users/lists/unfavorite.ts | 2 + .../api/endpoints/users/lists/update.ts | 1 + .../server/api/endpoints/users/update-memo.ts | 1 + packages/backend/test/e2e/users.ts | 2 + packages/frontend/.storybook/fakes.ts | 1 + .../src/components/MkUserCardMini.vue | 2 +- packages/frontend/src/const.ts | 3 + .../frontend/src/pages/admin/roles.editor.vue | 60 +++++++++++++++++++ packages/frontend/src/pages/admin/roles.vue | 26 +++++++- packages/frontend/src/pages/user-info.vue | 10 +++- packages/misskey-js/etc/misskey-js.api.md | 1 + packages/misskey-js/src/entities.ts | 3 +- 91 files changed, 228 insertions(+), 11 deletions(-) diff --git a/locales/en-US.yml b/locales/en-US.yml index 9cb00c8ee9d1..03b37f135640 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1404,6 +1404,9 @@ _role: gtlAvailable: "Can view the global timeline" ltlAvailable: "Can view the local timeline" canPublicNote: "Can send public notes" + canCreateContent: "Can create contents" + canUpdateContent: "Can edit contents" + canDeleteContent: "Can delete contents" canInvite: "Can create instance invite codes" inviteLimit: "Invite limit" inviteLimitCycle: "Invite limit cooldown" diff --git a/locales/index.d.ts b/locales/index.d.ts index e101195ae3e8..c4ef573a803b 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -1490,6 +1490,9 @@ export interface Locale { "gtlAvailable": string; "ltlAvailable": string; "canPublicNote": string; + "canCreateContent": string; + "canUpdateContent": string; + "canDeleteContent": string; "canInvite": string; "inviteLimit": string; "inviteLimitCycle": string; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index d88a8c025c45..07ba3cecc23f 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1412,6 +1412,9 @@ _role: gtlAvailable: "グローバルタイムラインの閲覧" ltlAvailable: "ローカルタイムラインの閲覧" canPublicNote: "パブリック投稿の許可" + canCreateContent: "コンテンツの作成" + canUpdateContent: "コンテンツの編集" + canDeleteContent: "コンテンツの削除" canInvite: "サーバー招待コードの発行" inviteLimit: "招待コードの作成可能数" inviteLimitCycle: "招待コードの発行間隔" diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index d065b460c6b1..58979f7328e2 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -20,6 +20,9 @@ export type RolePolicies = { gtlAvailable: boolean; ltlAvailable: boolean; canPublicNote: boolean; + canCreateContent: boolean; + canUpdateContent: boolean; + canDeleteContent: boolean; canInvite: boolean; inviteLimit: number; inviteLimitCycle: number; @@ -44,6 +47,9 @@ export const DEFAULT_POLICIES: RolePolicies = { gtlAvailable: true, ltlAvailable: true, canPublicNote: true, + canCreateContent: true, + canUpdateContent: true, + canDeleteContent: true, canInvite: false, inviteLimit: 0, inviteLimitCycle: 60 * 24 * 7, @@ -287,6 +293,9 @@ export class RoleService implements OnApplicationShutdown { gtlAvailable: calc('gtlAvailable', vs => vs.some(v => v === true)), ltlAvailable: calc('ltlAvailable', vs => vs.some(v => v === true)), canPublicNote: calc('canPublicNote', vs => vs.some(v => v === true)), + canCreateContent: calc('canCreateContent', vs => vs.some(v => v === true)), + canUpdateContent: calc('canUpdateContent', vs => vs.some(v => v === true)), + canDeleteContent: calc('canDeleteContent', vs => vs.some(v => v === true)), canInvite: calc('canInvite', vs => vs.some(v => v === true)), inviteLimit: calc('inviteLimit', vs => Math.max(...vs)), inviteLimitCycle: calc('inviteLimitCycle', vs => Math.max(...vs)), diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 3fd1768426f6..5987eeb81f9b 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -351,8 +351,9 @@ export class UserEntityService implements OnModuleInit { (profile.ffVisibility === 'followers') && (relation && relation.isFollowing) ? user.followersCount : null; - const isModerator = isMe && opts.detail ? this.roleService.isModerator(user) : null; - const isAdmin = isMe && opts.detail ? this.roleService.isAdministrator(user) : null; + const isModerator = isMe && opts.detail ? await this.roleService.isModerator(user) : null; + const isAdmin = isMe && opts.detail ? await this.roleService.isAdministrator(user) : null; + const policies = opts.detail ? await this.roleService.getUserPolicies(user.id) : null; const falsy = opts.detail ? false : undefined; @@ -396,7 +397,8 @@ export class UserEntityService implements OnModuleInit { bannerUrl: user.bannerUrl, bannerBlurhash: user.bannerBlurhash, isLocked: user.isLocked, - isSilenced: this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote), + isSilenced: !policies?.canPublicNote, + isLimited: !(policies?.canCreateContent && policies.canUpdateContent && policies.canDeleteContent), isSuspended: user.isSuspended ?? falsy, description: profile!.description, location: profile!.location, @@ -473,7 +475,7 @@ export class UserEntityService implements OnModuleInit { emailNotificationTypes: profile!.emailNotificationTypes, achievements: profile!.achievements, loggedInDays: profile!.loggedInDates.length, - policies: this.roleService.getUserPolicies(user.id), + policies: policies, } : {}), ...(opts.includeSecrets ? { diff --git a/packages/backend/src/models/json-schema/user.ts b/packages/backend/src/models/json-schema/user.ts index f9a20ac3986b..311e2db9c0ce 100644 --- a/packages/backend/src/models/json-schema/user.ts +++ b/packages/backend/src/models/json-schema/user.ts @@ -121,6 +121,10 @@ export const packedUserDetailedNotMeOnlySchema = { type: 'boolean', nullable: false, optional: false, }, + isLimited: { + type: 'boolean', + nullable: false, optional: false, + }, isSuspended: { type: 'boolean', nullable: false, optional: false, diff --git a/packages/backend/src/server/api/endpoints/admin/show-user.ts b/packages/backend/src/server/api/endpoints/admin/show-user.ts index 6f805b6b4e37..81f3583174a3 100644 --- a/packages/backend/src/server/api/endpoints/admin/show-user.ts +++ b/packages/backend/src/server/api/endpoints/admin/show-user.ts @@ -51,8 +51,10 @@ export default class extends Endpoint { throw new Error('user not found'); } + const policies = await this.roleService.getUserPolicies(user.id); const isModerator = await this.roleService.isModerator(user); - const isSilenced = !(await this.roleService.getUserPolicies(user.id)).canPublicNote; + const isLimited = !(policies.canCreateContent && policies.canUpdateContent && policies.canDeleteContent); + const isSilenced = !policies.canPublicNote; const _me = await this.usersRepository.findOneByOrFail({ id: me.id }); if (!await this.roleService.isAdministrator(_me) && await this.roleService.isAdministrator(user)) { @@ -80,6 +82,7 @@ export default class extends Endpoint { mutingNotificationTypes: profile.mutingNotificationTypes, isModerator: isModerator, isSilenced: isSilenced, + isLimited: isLimited, isSuspended: user.isSuspended, lastActiveDate: user.lastActiveDate, moderationNote: profile.moderationNote ?? '', diff --git a/packages/backend/src/server/api/endpoints/antennas/create.ts b/packages/backend/src/server/api/endpoints/antennas/create.ts index 5754a9f12aa5..6908af7e6c8c 100644 --- a/packages/backend/src/server/api/endpoints/antennas/create.ts +++ b/packages/backend/src/server/api/endpoints/antennas/create.ts @@ -12,6 +12,7 @@ export const meta = { tags: ['antennas'], requireCredential: true, + requireRolePolicy: 'canCreateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/antennas/delete.ts b/packages/backend/src/server/api/endpoints/antennas/delete.ts index 5da7a2cb6668..82f6ffbc0fce 100644 --- a/packages/backend/src/server/api/endpoints/antennas/delete.ts +++ b/packages/backend/src/server/api/endpoints/antennas/delete.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['antennas'], requireCredential: true, + requireRolePolicy: 'canDeleteContent', kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/antennas/update.ts b/packages/backend/src/server/api/endpoints/antennas/update.ts index 55218b644b6c..a732cc9622e5 100644 --- a/packages/backend/src/server/api/endpoints/antennas/update.ts +++ b/packages/backend/src/server/api/endpoints/antennas/update.ts @@ -10,6 +10,7 @@ export const meta = { tags: ['antennas'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/blocking/create.ts b/packages/backend/src/server/api/endpoints/blocking/create.ts index 4ad40c8f1c1a..c55c305314b5 100644 --- a/packages/backend/src/server/api/endpoints/blocking/create.ts +++ b/packages/backend/src/server/api/endpoints/blocking/create.ts @@ -17,6 +17,7 @@ export const meta = { }, requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:blocks', diff --git a/packages/backend/src/server/api/endpoints/blocking/delete.ts b/packages/backend/src/server/api/endpoints/blocking/delete.ts index ad3d9f22b312..df29b123e7ef 100644 --- a/packages/backend/src/server/api/endpoints/blocking/delete.ts +++ b/packages/backend/src/server/api/endpoints/blocking/delete.ts @@ -17,6 +17,7 @@ export const meta = { }, requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:blocks', diff --git a/packages/backend/src/server/api/endpoints/channels/create.ts b/packages/backend/src/server/api/endpoints/channels/create.ts index 69e2f2504c67..a7caef07fa1e 100644 --- a/packages/backend/src/server/api/endpoints/channels/create.ts +++ b/packages/backend/src/server/api/endpoints/channels/create.ts @@ -12,6 +12,7 @@ export const meta = { tags: ['channels'], requireCredential: true, + requireRolePolicy: 'canCreateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/channels/favorite.ts b/packages/backend/src/server/api/endpoints/channels/favorite.ts index c8544273a178..0e359ee2c90b 100644 --- a/packages/backend/src/server/api/endpoints/channels/favorite.ts +++ b/packages/backend/src/server/api/endpoints/channels/favorite.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['channels'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/channels/follow.ts b/packages/backend/src/server/api/endpoints/channels/follow.ts index f3ca66cfd2d6..ba19ca38daef 100644 --- a/packages/backend/src/server/api/endpoints/channels/follow.ts +++ b/packages/backend/src/server/api/endpoints/channels/follow.ts @@ -10,6 +10,7 @@ export const meta = { tags: ['channels'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/channels/unfavorite.ts b/packages/backend/src/server/api/endpoints/channels/unfavorite.ts index 67fb1ea03e11..9516043b949b 100644 --- a/packages/backend/src/server/api/endpoints/channels/unfavorite.ts +++ b/packages/backend/src/server/api/endpoints/channels/unfavorite.ts @@ -8,6 +8,7 @@ export const meta = { tags: ['channels'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/channels/unfollow.ts b/packages/backend/src/server/api/endpoints/channels/unfollow.ts index f46ff9f2865c..d86914b4d047 100644 --- a/packages/backend/src/server/api/endpoints/channels/unfollow.ts +++ b/packages/backend/src/server/api/endpoints/channels/unfollow.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['channels'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/channels/update.ts b/packages/backend/src/server/api/endpoints/channels/update.ts index 30d7f8b244e7..965c74c9795a 100644 --- a/packages/backend/src/server/api/endpoints/channels/update.ts +++ b/packages/backend/src/server/api/endpoints/channels/update.ts @@ -10,6 +10,7 @@ export const meta = { tags: ['channels'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:channels', diff --git a/packages/backend/src/server/api/endpoints/clips/add-note.ts b/packages/backend/src/server/api/endpoints/clips/add-note.ts index 2837f2cf818b..9ef2a8266aeb 100644 --- a/packages/backend/src/server/api/endpoints/clips/add-note.ts +++ b/packages/backend/src/server/api/endpoints/clips/add-note.ts @@ -14,6 +14,7 @@ export const meta = { requireCredential: true, prohibitMoved: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/clips/create.ts b/packages/backend/src/server/api/endpoints/clips/create.ts index 5395a5c373a9..d0dc9baf7554 100644 --- a/packages/backend/src/server/api/endpoints/clips/create.ts +++ b/packages/backend/src/server/api/endpoints/clips/create.ts @@ -11,6 +11,7 @@ export const meta = { tags: ['clips'], requireCredential: true, + requireRolePolicy: 'canCreateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/clips/delete.ts b/packages/backend/src/server/api/endpoints/clips/delete.ts index 077a9ec40f57..210bba0d58d6 100644 --- a/packages/backend/src/server/api/endpoints/clips/delete.ts +++ b/packages/backend/src/server/api/endpoints/clips/delete.ts @@ -8,6 +8,7 @@ export const meta = { tags: ['clips'], requireCredential: true, + requireRolePolicy: 'canDeleteContent', kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/clips/remove-note.ts b/packages/backend/src/server/api/endpoints/clips/remove-note.ts index d0ef795819d1..6eeef06ccff1 100644 --- a/packages/backend/src/server/api/endpoints/clips/remove-note.ts +++ b/packages/backend/src/server/api/endpoints/clips/remove-note.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['account', 'notes', 'clips'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/clips/unfavorite.ts b/packages/backend/src/server/api/endpoints/clips/unfavorite.ts index 3da252a2269d..fe9c965a0331 100644 --- a/packages/backend/src/server/api/endpoints/clips/unfavorite.ts +++ b/packages/backend/src/server/api/endpoints/clips/unfavorite.ts @@ -8,6 +8,7 @@ export const meta = { tags: ['clip'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/clips/update.ts b/packages/backend/src/server/api/endpoints/clips/update.ts index 70f195935392..8c8ca598862a 100644 --- a/packages/backend/src/server/api/endpoints/clips/update.ts +++ b/packages/backend/src/server/api/endpoints/clips/update.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['clips'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/drive/files/create.ts b/packages/backend/src/server/api/endpoints/drive/files/create.ts index a1c1f9325ec7..72900358efde 100644 --- a/packages/backend/src/server/api/endpoints/drive/files/create.ts +++ b/packages/backend/src/server/api/endpoints/drive/files/create.ts @@ -14,6 +14,7 @@ export const meta = { tags: ['drive'], requireCredential: true, + requireRolePolicy: 'canCreateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/drive/files/delete.ts b/packages/backend/src/server/api/endpoints/drive/files/delete.ts index 2ced97ee02ed..ca7f22d65da3 100644 --- a/packages/backend/src/server/api/endpoints/drive/files/delete.ts +++ b/packages/backend/src/server/api/endpoints/drive/files/delete.ts @@ -11,6 +11,7 @@ export const meta = { tags: ['drive'], requireCredential: true, + requireRolePolicy: 'canDeleteContent', kind: 'write:drive', diff --git a/packages/backend/src/server/api/endpoints/drive/files/update.ts b/packages/backend/src/server/api/endpoints/drive/files/update.ts index c43f812e2fb2..92a09e6049b9 100644 --- a/packages/backend/src/server/api/endpoints/drive/files/update.ts +++ b/packages/backend/src/server/api/endpoints/drive/files/update.ts @@ -11,6 +11,7 @@ export const meta = { tags: ['drive'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:drive', diff --git a/packages/backend/src/server/api/endpoints/drive/files/upload-from-url.ts b/packages/backend/src/server/api/endpoints/drive/files/upload-from-url.ts index c835587c4aa0..0c3b74325473 100644 --- a/packages/backend/src/server/api/endpoints/drive/files/upload-from-url.ts +++ b/packages/backend/src/server/api/endpoints/drive/files/upload-from-url.ts @@ -18,6 +18,7 @@ export const meta = { description: 'Request the server to download a new drive file from the specified URL.', requireCredential: true, + requireRolePolicy: 'canCreateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/drive/folders/create.ts b/packages/backend/src/server/api/endpoints/drive/folders/create.ts index 39c9c6bc5802..3b1e2265c697 100644 --- a/packages/backend/src/server/api/endpoints/drive/folders/create.ts +++ b/packages/backend/src/server/api/endpoints/drive/folders/create.ts @@ -12,6 +12,7 @@ export const meta = { tags: ['drive'], requireCredential: true, + requireRolePolicy: 'canCreateContent', kind: 'write:drive', diff --git a/packages/backend/src/server/api/endpoints/drive/folders/delete.ts b/packages/backend/src/server/api/endpoints/drive/folders/delete.ts index d921bc1b17bd..6970c0572b46 100644 --- a/packages/backend/src/server/api/endpoints/drive/folders/delete.ts +++ b/packages/backend/src/server/api/endpoints/drive/folders/delete.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['drive'], requireCredential: true, + requireRolePolicy: 'canDeleteContent', kind: 'write:drive', diff --git a/packages/backend/src/server/api/endpoints/drive/folders/update.ts b/packages/backend/src/server/api/endpoints/drive/folders/update.ts index ff0a78b929e1..05cc1ff5a460 100644 --- a/packages/backend/src/server/api/endpoints/drive/folders/update.ts +++ b/packages/backend/src/server/api/endpoints/drive/folders/update.ts @@ -10,6 +10,7 @@ export const meta = { tags: ['drive'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:drive', diff --git a/packages/backend/src/server/api/endpoints/flash/create.ts b/packages/backend/src/server/api/endpoints/flash/create.ts index 3172bdbfdaf5..fc556ad2b74a 100644 --- a/packages/backend/src/server/api/endpoints/flash/create.ts +++ b/packages/backend/src/server/api/endpoints/flash/create.ts @@ -10,6 +10,7 @@ export const meta = { tags: ['flash'], requireCredential: true, + requireRolePolicy: 'canCreateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/flash/delete.ts b/packages/backend/src/server/api/endpoints/flash/delete.ts index e94ede9f6885..11fc894bded5 100644 --- a/packages/backend/src/server/api/endpoints/flash/delete.ts +++ b/packages/backend/src/server/api/endpoints/flash/delete.ts @@ -8,6 +8,7 @@ export const meta = { tags: ['flashs'], requireCredential: true, + requireRolePolicy: 'canDeleteContent', kind: 'write:flash', diff --git a/packages/backend/src/server/api/endpoints/flash/like.ts b/packages/backend/src/server/api/endpoints/flash/like.ts index 57245f9f4149..c66ccbb8d730 100644 --- a/packages/backend/src/server/api/endpoints/flash/like.ts +++ b/packages/backend/src/server/api/endpoints/flash/like.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['flash'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/flash/unlike.ts b/packages/backend/src/server/api/endpoints/flash/unlike.ts index 696512b06c9e..1a54212d78da 100644 --- a/packages/backend/src/server/api/endpoints/flash/unlike.ts +++ b/packages/backend/src/server/api/endpoints/flash/unlike.ts @@ -8,6 +8,7 @@ export const meta = { tags: ['flash'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/flash/update.ts b/packages/backend/src/server/api/endpoints/flash/update.ts index 78dfd4a06a71..b4c846ae7103 100644 --- a/packages/backend/src/server/api/endpoints/flash/update.ts +++ b/packages/backend/src/server/api/endpoints/flash/update.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['flash'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/create.ts b/packages/backend/src/server/api/endpoints/gallery/posts/create.ts index ca6bfa7e0f1b..2cfed8cacc2d 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/create.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/create.ts @@ -12,6 +12,7 @@ export const meta = { tags: ['gallery'], requireCredential: true, + requireRolePolicy: 'canCreateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/delete.ts b/packages/backend/src/server/api/endpoints/gallery/posts/delete.ts index 6cdcc17b3924..8294aa57cf3d 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/delete.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/delete.ts @@ -8,6 +8,7 @@ export const meta = { tags: ['gallery'], requireCredential: true, + requireRolePolicy: 'canDeleteContent', kind: 'write:gallery', diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/like.ts b/packages/backend/src/server/api/endpoints/gallery/posts/like.ts index c0bb55f64087..ac1e42995af3 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/like.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/like.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['gallery'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/unlike.ts b/packages/backend/src/server/api/endpoints/gallery/posts/unlike.ts index 513089217dfd..887dc6412476 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/unlike.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/unlike.ts @@ -8,6 +8,7 @@ export const meta = { tags: ['gallery'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/update.ts b/packages/backend/src/server/api/endpoints/gallery/posts/update.ts index a2a10d840038..5eb609ef76af 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/update.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/update.ts @@ -10,6 +10,7 @@ export const meta = { tags: ['gallery'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/i/delete-account.ts b/packages/backend/src/server/api/endpoints/i/delete-account.ts index 77a03d9811fa..ffa1fa472bd7 100644 --- a/packages/backend/src/server/api/endpoints/i/delete-account.ts +++ b/packages/backend/src/server/api/endpoints/i/delete-account.ts @@ -7,6 +7,7 @@ import { DI } from '@/di-symbols.js'; export const meta = { requireCredential: true, + requireRolePolicy: 'canDeleteContent', secure: true, } as const; diff --git a/packages/backend/src/server/api/endpoints/i/import-antennas.ts b/packages/backend/src/server/api/endpoints/i/import-antennas.ts index 8582e98f76c6..376b0be9a9d7 100644 --- a/packages/backend/src/server/api/endpoints/i/import-antennas.ts +++ b/packages/backend/src/server/api/endpoints/i/import-antennas.ts @@ -11,6 +11,8 @@ import { ApiError } from '../../error.js'; export const meta = { secure: true, requireCredential: true, + requireRolePolicy: 'canCreateContent', + prohibitMoved: true, limit: { diff --git a/packages/backend/src/server/api/endpoints/i/import-blocking.ts b/packages/backend/src/server/api/endpoints/i/import-blocking.ts index 32c16300fb5e..dda915696e87 100644 --- a/packages/backend/src/server/api/endpoints/i/import-blocking.ts +++ b/packages/backend/src/server/api/endpoints/i/import-blocking.ts @@ -10,6 +10,8 @@ import { ApiError } from '../../error.js'; export const meta = { secure: true, requireCredential: true, + requireRolePolicy: 'canUpdateContent', + prohibitMoved: true, limit: { diff --git a/packages/backend/src/server/api/endpoints/i/import-muting.ts b/packages/backend/src/server/api/endpoints/i/import-muting.ts index 34f262756384..982f25841001 100644 --- a/packages/backend/src/server/api/endpoints/i/import-muting.ts +++ b/packages/backend/src/server/api/endpoints/i/import-muting.ts @@ -10,6 +10,8 @@ import { ApiError } from '../../error.js'; export const meta = { secure: true, requireCredential: true, + requireRolePolicy: 'canUpdateContent', + prohibitMoved: true, limit: { diff --git a/packages/backend/src/server/api/endpoints/i/import-user-lists.ts b/packages/backend/src/server/api/endpoints/i/import-user-lists.ts index 1b3cb5359db4..48c9947622ef 100644 --- a/packages/backend/src/server/api/endpoints/i/import-user-lists.ts +++ b/packages/backend/src/server/api/endpoints/i/import-user-lists.ts @@ -10,7 +10,10 @@ import { ApiError } from '../../error.js'; export const meta = { secure: true, requireCredential: true, + requireRolePolicy: 'canCreateContent', + prohibitMoved: true, + limit: { duration: ms('1hour'), max: 1, diff --git a/packages/backend/src/server/api/endpoints/i/move.ts b/packages/backend/src/server/api/endpoints/i/move.ts index 261dd527c09e..b7fc31791fcf 100644 --- a/packages/backend/src/server/api/endpoints/i/move.ts +++ b/packages/backend/src/server/api/endpoints/i/move.ts @@ -23,7 +23,10 @@ export const meta = { secure: true, requireCredential: true, + requireRolePolicy: 'canUpdateContent', + prohibitMoved: true, + limit: { duration: ms('1day'), max: 5, diff --git a/packages/backend/src/server/api/endpoints/i/pin.ts b/packages/backend/src/server/api/endpoints/i/pin.ts index 2293500945d3..21fc9674c5f8 100644 --- a/packages/backend/src/server/api/endpoints/i/pin.ts +++ b/packages/backend/src/server/api/endpoints/i/pin.ts @@ -8,6 +8,8 @@ export const meta = { tags: ['account', 'notes'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', + prohibitMoved: true, kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/i/unpin.ts b/packages/backend/src/server/api/endpoints/i/unpin.ts index db239dc284fc..72af6b7635eb 100644 --- a/packages/backend/src/server/api/endpoints/i/unpin.ts +++ b/packages/backend/src/server/api/endpoints/i/unpin.ts @@ -8,6 +8,7 @@ export const meta = { tags: ['account', 'notes'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/i/update-email.ts b/packages/backend/src/server/api/endpoints/i/update-email.ts index 58e056bd3786..64b026650cdf 100644 --- a/packages/backend/src/server/api/endpoints/i/update-email.ts +++ b/packages/backend/src/server/api/endpoints/i/update-email.ts @@ -12,10 +12,11 @@ import { L_CHARS, secureRndstr } from '@/misc/secure-rndstr.js'; import { ApiError } from '../../error.js'; export const meta = { - requireCredential: true, - secure: true, + requireCredential: true, + requireRolePolicy: 'canUpdateContent', + limit: { duration: ms('1hour'), max: 3, diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index 8f5e6177c28f..4eaad78445c4 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -30,6 +30,7 @@ export const meta = { tags: ['account'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/i/webhooks/create.ts b/packages/backend/src/server/api/endpoints/i/webhooks/create.ts index 51fcce6cf006..642e497666f5 100644 --- a/packages/backend/src/server/api/endpoints/i/webhooks/create.ts +++ b/packages/backend/src/server/api/endpoints/i/webhooks/create.ts @@ -12,6 +12,7 @@ export const meta = { tags: ['webhooks'], requireCredential: true, + requireRolePolicy: 'canCreateContent', kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/i/webhooks/delete.ts b/packages/backend/src/server/api/endpoints/i/webhooks/delete.ts index 7bdad136aaaa..7cf8394e95b6 100644 --- a/packages/backend/src/server/api/endpoints/i/webhooks/delete.ts +++ b/packages/backend/src/server/api/endpoints/i/webhooks/delete.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['webhooks'], requireCredential: true, + requireRolePolicy: 'canDeleteContent', kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/i/webhooks/update.ts b/packages/backend/src/server/api/endpoints/i/webhooks/update.ts index 8ec308eda777..69e2e4220b44 100644 --- a/packages/backend/src/server/api/endpoints/i/webhooks/update.ts +++ b/packages/backend/src/server/api/endpoints/i/webhooks/update.ts @@ -10,6 +10,7 @@ export const meta = { tags: ['webhooks'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/mute/create.ts b/packages/backend/src/server/api/endpoints/mute/create.ts index ef53f9ef4195..db3a64af8e4f 100644 --- a/packages/backend/src/server/api/endpoints/mute/create.ts +++ b/packages/backend/src/server/api/endpoints/mute/create.ts @@ -11,6 +11,8 @@ export const meta = { tags: ['account'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', + prohibitMoved: true, kind: 'write:mutes', diff --git a/packages/backend/src/server/api/endpoints/mute/delete.ts b/packages/backend/src/server/api/endpoints/mute/delete.ts index 90b74590be4e..3bf928a1f560 100644 --- a/packages/backend/src/server/api/endpoints/mute/delete.ts +++ b/packages/backend/src/server/api/endpoints/mute/delete.ts @@ -10,6 +10,7 @@ export const meta = { tags: ['account'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:mutes', diff --git a/packages/backend/src/server/api/endpoints/notes/create.ts b/packages/backend/src/server/api/endpoints/notes/create.ts index 739316997a88..36c9ee4acb1b 100644 --- a/packages/backend/src/server/api/endpoints/notes/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/create.ts @@ -17,6 +17,7 @@ export const meta = { tags: ['notes'], requireCredential: true, + requireRolePolicy: 'canCreateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/notes/delete.ts b/packages/backend/src/server/api/endpoints/notes/delete.ts index 16c4c0138758..b802975017b6 100644 --- a/packages/backend/src/server/api/endpoints/notes/delete.ts +++ b/packages/backend/src/server/api/endpoints/notes/delete.ts @@ -12,6 +12,7 @@ export const meta = { tags: ['notes'], requireCredential: true, + requireRolePolicy: 'canDeleteContent', kind: 'write:notes', diff --git a/packages/backend/src/server/api/endpoints/notes/favorites/create.ts b/packages/backend/src/server/api/endpoints/notes/favorites/create.ts index 9299d66039eb..7db933322f46 100644 --- a/packages/backend/src/server/api/endpoints/notes/favorites/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/favorites/create.ts @@ -12,6 +12,8 @@ export const meta = { tags: ['notes', 'favorites'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', + prohibitMoved: true, kind: 'write:favorites', diff --git a/packages/backend/src/server/api/endpoints/notes/favorites/delete.ts b/packages/backend/src/server/api/endpoints/notes/favorites/delete.ts index bb3a7c501a01..5b1c2edf1c84 100644 --- a/packages/backend/src/server/api/endpoints/notes/favorites/delete.ts +++ b/packages/backend/src/server/api/endpoints/notes/favorites/delete.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['notes', 'favorites'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:favorites', diff --git a/packages/backend/src/server/api/endpoints/notes/polls/vote.ts b/packages/backend/src/server/api/endpoints/notes/polls/vote.ts index 3a33b037f868..c789313abb48 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/vote.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/vote.ts @@ -16,6 +16,7 @@ export const meta = { tags: ['notes'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/notes/reactions/create.ts b/packages/backend/src/server/api/endpoints/notes/reactions/create.ts index 97cb026779c5..4de379d30f6a 100644 --- a/packages/backend/src/server/api/endpoints/notes/reactions/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/reactions/create.ts @@ -8,6 +8,7 @@ export const meta = { tags: ['reactions', 'notes'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/notes/reactions/delete.ts b/packages/backend/src/server/api/endpoints/notes/reactions/delete.ts index 207f0b4cf2b7..1420f567cf28 100644 --- a/packages/backend/src/server/api/endpoints/notes/reactions/delete.ts +++ b/packages/backend/src/server/api/endpoints/notes/reactions/delete.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['reactions', 'notes'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:reactions', diff --git a/packages/backend/src/server/api/endpoints/notes/thread-muting/create.ts b/packages/backend/src/server/api/endpoints/notes/thread-muting/create.ts index abea069da887..17fcdc3be4e2 100644 --- a/packages/backend/src/server/api/endpoints/notes/thread-muting/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/thread-muting/create.ts @@ -12,6 +12,7 @@ export const meta = { tags: ['notes'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/notes/thread-muting/delete.ts b/packages/backend/src/server/api/endpoints/notes/thread-muting/delete.ts index 30016d48bc54..2337289dca03 100644 --- a/packages/backend/src/server/api/endpoints/notes/thread-muting/delete.ts +++ b/packages/backend/src/server/api/endpoints/notes/thread-muting/delete.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['notes'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/notes/unrenote.ts b/packages/backend/src/server/api/endpoints/notes/unrenote.ts index e9581beedc73..19e4bacb5ed1 100644 --- a/packages/backend/src/server/api/endpoints/notes/unrenote.ts +++ b/packages/backend/src/server/api/endpoints/notes/unrenote.ts @@ -11,6 +11,7 @@ export const meta = { tags: ['notes'], requireCredential: true, + requireRolePolicy: 'canDeleteContent', kind: 'write:notes', diff --git a/packages/backend/src/server/api/endpoints/pages/create.ts b/packages/backend/src/server/api/endpoints/pages/create.ts index e08ab399f88f..00a07f8fa83c 100644 --- a/packages/backend/src/server/api/endpoints/pages/create.ts +++ b/packages/backend/src/server/api/endpoints/pages/create.ts @@ -12,6 +12,7 @@ export const meta = { tags: ['pages'], requireCredential: true, + requireRolePolicy: 'canCreateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/pages/delete.ts b/packages/backend/src/server/api/endpoints/pages/delete.ts index e64733131cd5..c25c28806ed3 100644 --- a/packages/backend/src/server/api/endpoints/pages/delete.ts +++ b/packages/backend/src/server/api/endpoints/pages/delete.ts @@ -8,6 +8,7 @@ export const meta = { tags: ['pages'], requireCredential: true, + requireRolePolicy: 'canDeleteContent', kind: 'write:pages', diff --git a/packages/backend/src/server/api/endpoints/pages/like.ts b/packages/backend/src/server/api/endpoints/pages/like.ts index bc664881031e..cce9ce7bd5b3 100644 --- a/packages/backend/src/server/api/endpoints/pages/like.ts +++ b/packages/backend/src/server/api/endpoints/pages/like.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['pages'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/pages/unlike.ts b/packages/backend/src/server/api/endpoints/pages/unlike.ts index f0c019846014..0fc68ace2e74 100644 --- a/packages/backend/src/server/api/endpoints/pages/unlike.ts +++ b/packages/backend/src/server/api/endpoints/pages/unlike.ts @@ -8,6 +8,7 @@ export const meta = { tags: ['pages'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/pages/update.ts b/packages/backend/src/server/api/endpoints/pages/update.ts index 751274067ed2..39757ff74b24 100644 --- a/packages/backend/src/server/api/endpoints/pages/update.ts +++ b/packages/backend/src/server/api/endpoints/pages/update.ts @@ -10,6 +10,7 @@ export const meta = { tags: ['pages'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/renote-mute/create.ts b/packages/backend/src/server/api/endpoints/renote-mute/create.ts index beb5850d7883..1404a64cc4e9 100644 --- a/packages/backend/src/server/api/endpoints/renote-mute/create.ts +++ b/packages/backend/src/server/api/endpoints/renote-mute/create.ts @@ -13,6 +13,8 @@ export const meta = { tags: ['account'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', + prohibitMoved: true, kind: 'write:mutes', diff --git a/packages/backend/src/server/api/endpoints/renote-mute/delete.ts b/packages/backend/src/server/api/endpoints/renote-mute/delete.ts index 70901a1406f9..aac8176365fc 100644 --- a/packages/backend/src/server/api/endpoints/renote-mute/delete.ts +++ b/packages/backend/src/server/api/endpoints/renote-mute/delete.ts @@ -10,6 +10,7 @@ export const meta = { tags: ['account'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:mutes', diff --git a/packages/backend/src/server/api/endpoints/users/lists/create-from-public.ts b/packages/backend/src/server/api/endpoints/users/lists/create-from-public.ts index beb0ba85ff81..70081d51e71b 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/create-from-public.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/create-from-public.ts @@ -12,7 +12,10 @@ import { UserListService } from '@/core/UserListService.js'; export const meta = { requireCredential: true, + requireRolePolicy: 'canCreateContent', + prohibitMoved: true, + res: { type: 'object', optional: false, nullable: false, diff --git a/packages/backend/src/server/api/endpoints/users/lists/create.ts b/packages/backend/src/server/api/endpoints/users/lists/create.ts index 751088952634..0684d34de476 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/create.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/create.ts @@ -12,6 +12,7 @@ export const meta = { tags: ['lists'], requireCredential: true, + requireRolePolicy: 'canCreateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/users/lists/delete.ts b/packages/backend/src/server/api/endpoints/users/lists/delete.ts index 237cb075abde..9ef9941ca676 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/delete.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/delete.ts @@ -8,6 +8,7 @@ export const meta = { tags: ['lists'], requireCredential: true, + requireRolePolicy: 'canDeleteContent', kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/users/lists/favorite.ts b/packages/backend/src/server/api/endpoints/users/lists/favorite.ts index 2c09a47fefee..e75ac34be175 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/favorite.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/favorite.ts @@ -7,6 +7,8 @@ import { DI } from '@/di-symbols.js'; export const meta = { requireCredential: true, + requireRolePolicy: 'canUpdateContent', + errors: { noSuchList: { message: 'No such user list.', diff --git a/packages/backend/src/server/api/endpoints/users/lists/pull.ts b/packages/backend/src/server/api/endpoints/users/lists/pull.ts index d50b70efc2fb..40ffec473235 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/pull.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/pull.ts @@ -11,6 +11,7 @@ export const meta = { tags: ['lists', 'users'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/users/lists/push.ts b/packages/backend/src/server/api/endpoints/users/lists/push.ts index 6e1f6b2c6262..7a52445e80a6 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/push.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/push.ts @@ -11,6 +11,7 @@ export const meta = { tags: ['lists', 'users'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', prohibitMoved: true, diff --git a/packages/backend/src/server/api/endpoints/users/lists/unfavorite.ts b/packages/backend/src/server/api/endpoints/users/lists/unfavorite.ts index a7c3b5894740..643841e1dca6 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/unfavorite.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/unfavorite.ts @@ -6,6 +6,8 @@ import { DI } from '@/di-symbols.js'; export const meta = { requireCredential: true, + requireRolePolicy: 'canUpdateContent', + errors: { noSuchList: { message: 'No such user list.', diff --git a/packages/backend/src/server/api/endpoints/users/lists/update.ts b/packages/backend/src/server/api/endpoints/users/lists/update.ts index b0a95a2f2864..4bcc816cf1bc 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/update.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/update.ts @@ -9,6 +9,7 @@ export const meta = { tags: ['lists'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:account', diff --git a/packages/backend/src/server/api/endpoints/users/update-memo.ts b/packages/backend/src/server/api/endpoints/users/update-memo.ts index ca7756ef750c..53340a3f9337 100644 --- a/packages/backend/src/server/api/endpoints/users/update-memo.ts +++ b/packages/backend/src/server/api/endpoints/users/update-memo.ts @@ -10,6 +10,7 @@ export const meta = { tags: ['account'], requireCredential: true, + requireRolePolicy: 'canUpdateContent', kind: 'write:account', diff --git a/packages/backend/test/e2e/users.ts b/packages/backend/test/e2e/users.ts index 64efaa57ccfc..622d25a80279 100644 --- a/packages/backend/test/e2e/users.ts +++ b/packages/backend/test/e2e/users.ts @@ -91,6 +91,7 @@ describe('ユーザー', () => { bannerBlurhash: user.bannerBlurhash, isLocked: user.isLocked, isSilenced: user.isSilenced, + isLimited: user.isLimited, isSuspended: user.isSuspended, description: user.description, location: user.location, @@ -356,6 +357,7 @@ describe('ユーザー', () => { assert.strictEqual(response.bannerBlurhash, null); assert.strictEqual(response.isLocked, false); assert.strictEqual(response.isSilenced, false); + assert.strictEqual(response.isLimited, false); assert.strictEqual(response.isSuspended, false); assert.strictEqual(response.description, null); assert.strictEqual(response.location, null); diff --git a/packages/frontend/.storybook/fakes.ts b/packages/frontend/.storybook/fakes.ts index a4289cff7da7..7a8ab8044e40 100644 --- a/packages/frontend/.storybook/fakes.ts +++ b/packages/frontend/.storybook/fakes.ts @@ -99,6 +99,7 @@ export function userDetailed(id = 'someuserid', username = 'miskist', host = 'mi isModerator: false, isMuted: false, isSilenced: false, + isLimited: false, isSuspended: false, lang: 'en', location: 'Fediverse', diff --git a/packages/frontend/src/components/MkUserCardMini.vue b/packages/frontend/src/components/MkUserCardMini.vue index 457504e6ca4d..79a96043eaf4 100644 --- a/packages/frontend/src/components/MkUserCardMini.vue +++ b/packages/frontend/src/components/MkUserCardMini.vue @@ -1,5 +1,5 @@