feat: image blur hash (#2010)

* feat: image blur hash Signed-off-by: Innei <i@innei.in> * fix: split util Signed-off-by: Innei <i@innei.in> * feat: refresh image Signed-off-by: Innei <i@innei.in> * fix: add logger Signed-off-by: Innei <i@innei.in> * chore: cleanup Signed-off-by: Innei <i@innei.in> * fix: update Signed-off-by: Innei <i@innei.in> --------- Signed-off-by: Innei <i@innei.in>
mx-space · Aug 16, 2024 · c27ee8c · c27ee8c
1 parent 5d335fc
commit c27ee8c
Show file tree

Hide file tree

Showing 56 changed files with 507 additions and 332 deletions.
diff --git a/apps/core/package.json b/apps/core/package.json
@@ -48,6 +48,7 @@
   },
   "dependencies": {
     "@algolia/client-search": "^4.22.1",
+    "@antfu/install-pkg": "0.3.5",
     "@aws-sdk/client-s3": "3.632.0",
     "@babel/core": "7.25.2",
     "@babel/plugin-transform-modules-commonjs": "7.24.8",
@@ -81,6 +82,7 @@
     "axios": "^1.7.0",
     "axios-retry": "4.5.0",
     "bcryptjs": "^2.4.3",
+    "blurhash": "2.0.5",
     "cache-manager": "5.7.6",
     "cache-manager-ioredis-yet": "2.1.1",
     "class-transformer": "0.5.1",
@@ -91,8 +93,6 @@
     "dayjs": "1.11.12",
     "ejs": "3.1.10",
     "form-data": "4.0.0",
-    "get-image-colors": "4.0.1",
-    "image-size": "1.1.1",
     "inquirer": "^10.1.0",
     "isbot": "5.1.16",
     "js-yaml": "^4.1.0",
@@ -158,6 +158,7 @@
     "cron": "^3.1.6",
     "husky": "9.1.4",
     "ioredis": "5.4.1",
+    "sharp": "0.33.4",
     "socket.io": "^4.7.4",
     "unplugin-swc": "1.5.1",
     "vite": "5.1.7",

diff --git a/apps/core/src/common/adapters/fastify.adapter.ts b/apps/core/src/common/adapters/fastify.adapter.ts
@@ -5,7 +5,7 @@ import FastifyMultipart from '@fastify/multipart'
 import { Logger } from '@nestjs/common'
 import { FastifyAdapter } from '@nestjs/platform-fastify'
 
-import { getIp } from '~/utils'
+import { getIp } from '~/utils/ip.util'
 
 const app: FastifyAdapter = new FastifyAdapter({
   trustProxy: true,

diff --git a/apps/core/src/common/decorators/demo.decorator.ts b/apps/core/src/common/decorators/demo.decorator.ts
@@ -3,7 +3,7 @@ import type { Observable } from 'rxjs'
 
 import { applyDecorators, UseGuards } from '@nestjs/common'
 
-import { banInDemo } from '~/utils'
+import { banInDemo } from '~/utils/biz.util'
 
 class DemoGuard implements CanActivate {
   canActivate(): boolean | Promise<boolean> | Observable<boolean> {

diff --git a/apps/core/src/common/guards/throttler.guard.ts b/apps/core/src/common/guards/throttler.guard.ts
@@ -5,7 +5,7 @@ import { Injectable } from '@nestjs/common'
 import { ThrottlerGuard } from '@nestjs/throttler'
 
 import { getNestExecutionContextRequest } from '~/transformers/get-req.transformer'
-import { getIp } from '~/utils'
+import { getIp } from '~/utils/ip.util'
 
 @Injectable()
 export class ExtendThrottlerGuard extends ThrottlerGuard {

diff --git a/apps/core/src/common/interceptors/analyze.interceptor.ts b/apps/core/src/common/interceptors/analyze.interceptor.ts
@@ -26,9 +26,9 @@ import { OptionModel } from '~/modules/configs/configs.model'
 import { CacheService } from '~/processors/redis/cache.service'
 import { getNestExecutionContextRequest } from '~/transformers/get-req.transformer'
 import { InjectModel } from '~/transformers/model.transformer'
-import { scheduleManager } from '~/utils'
 import { getIp } from '~/utils/ip.util'
 import { getRedisKey } from '~/utils/redis.util'
+import { scheduleManager } from '~/utils/schedule.util'
 
 @Injectable()
 export class AnalyzeInterceptor implements NestInterceptor {

diff --git a/apps/core/src/common/interceptors/cache.interceptor.ts b/apps/core/src/common/interceptors/cache.interceptor.ts
@@ -23,7 +23,7 @@ import * as META from '~/constants/meta.constant'
 import * as SYSTEM from '~/constants/system.constant'
 import { CacheService } from '~/processors/redis/cache.service'
 import { getNestExecutionContextRequest } from '~/transformers/get-req.transformer'
-import { hashString } from '~/utils'
+import { hashString } from '~/utils/tool.util'
 
 /**
  * @class HttpCacheInterceptor

diff --git a/apps/core/src/common/interceptors/idempotence.interceptor.ts b/apps/core/src/common/interceptors/idempotence.interceptor.ts
@@ -20,7 +20,9 @@ import {
 } from '~/constants/meta.constant'
 import { REFLECTOR } from '~/constants/system.constant'
 import { CacheService } from '~/processors/redis/cache.service'
-import { getIp, getRedisKey, hashString } from '~/utils'
+import { getIp } from '~/utils/ip.util'
+import { getRedisKey } from '~/utils/redis.util'
+import { hashString } from '~/utils/tool.util'
 
 const IdempotenceHeaderKey = 'x-idempotence'
 

diff --git a/apps/core/src/global/consola.global.ts b/apps/core/src/global/consola.global.ts
@@ -2,7 +2,6 @@ import { createLogger, Logger } from '@innei/pretty-logger-nestjs'
 
 import { LOG_DIR } from '~/constants/path.constant'
 
-import { redisSubPub } from '../utils/redis-subpub.util'
 import { isTest } from './env.global'
 
 const logger = createLogger({
@@ -19,6 +18,7 @@ if (!isTest) {
     logger.warn('wrap console failed')
   }
   logger.onData((data) => {
+    const { redisSubPub } = require('../utils/redis-subpub.util')
     redisSubPub.publish('log', data)
   })
 }

diff --git a/apps/core/src/global/env.global.ts b/apps/core/src/global/env.global.ts
@@ -1,7 +1,5 @@
 import cluster from 'node:cluster'
 
-import { parseBooleanishValue } from '~/utils/tool.util'
-
 export const isMainCluster =
   process.env.NODE_APP_INSTANCE &&
   Number.parseInt(process.env.NODE_APP_INSTANCE) === 0
@@ -10,5 +8,5 @@ export const isMainProcess = cluster.isPrimary || isMainCluster
 export const isDev = process.env.NODE_ENV == 'development'
 
 export const isTest = !!process.env.TEST
-export const isDebugMode = parseBooleanishValue(process.env.DEBUG_MODE) || false
+export const isDebugMode = process.env.DEBUG_MODE === '1'
 export const cwd = process.cwd()
diff --git a/apps/core/src/modules/ack/ack.module.ts b/apps/core/src/modules/ack/ack.module.ts
@@ -1,5 +1,4 @@
 import { Module } from '@nestjs/common'
-import { DiscoveryModule } from '@nestjs/core'
 
 import { ExtendedValidationPipe } from '~/common/pipes/validation.pipe'
 import { VALIDATION_PIPE_INJECTION } from '~/constants/system.constant'
@@ -8,7 +7,7 @@ import { AckController } from './ack.controller'
 
 @Module({
   controllers: [AckController],
-  imports: [DiscoveryModule],
+
   providers: [
     {
       provide: VALIDATION_PIPE_INJECTION,

diff --git a/apps/core/src/modules/activity/activity.service.ts b/apps/core/src/modules/activity/activity.service.ts
@@ -36,7 +36,7 @@ import { CountingService } from '~/processors/helper/helper.counting.service'
 import { EventManagerService } from '~/processors/helper/helper.event.service'
 import { InjectModel } from '~/transformers/model.transformer'
 import { transformDataToPaginate } from '~/transformers/paginate.transformer'
-import { checkRefModelCollectionType } from '~/utils'
+import { checkRefModelCollectionType } from '~/utils/biz.util'
 
 import { CommentState } from '../comment/comment.model'
 import { CommentService } from '../comment/comment.service'

diff --git a/apps/core/src/modules/ai/ai-summary/ai-summary.service.ts b/apps/core/src/modules/ai/ai-summary/ai-summary.service.ts
@@ -13,7 +13,7 @@ import { DatabaseService } from '~/processors/database/database.service'
 import { CacheService } from '~/processors/redis/cache.service'
 import { InjectModel } from '~/transformers/model.transformer'
 import { transformDataToPaginate } from '~/transformers/paginate.transformer'
-import { md5 } from '~/utils'
+import { md5 } from '~/utils/tool.util'
 
 import { ConfigsService } from '../../configs/configs.service'
 import { DEFAULT_SUMMARY_LANG, LANGUAGE_CODE_TO_NAME } from '../ai.constants'

diff --git a/apps/core/src/modules/authn/authn.service.ts b/apps/core/src/modules/authn/authn.service.ts
@@ -23,7 +23,7 @@ import { RequestContext } from '~/common/contexts/request.context'
 import { RedisKeys } from '~/constants/cache.constant'
 import { CacheService } from '~/processors/redis/cache.service'
 import { InjectModel } from '~/transformers/model.transformer'
-import { getRedisKey } from '~/utils'
+import { getRedisKey } from '~/utils/redis.util'
 
 import { ConfigsService } from '../configs/configs.service'
 import { AuthnModel } from './authn.model'

diff --git a/apps/core/src/modules/backup/backup.controller.ts b/apps/core/src/modules/backup/backup.controller.ts
@@ -22,8 +22,8 @@ import { HTTPDecorators } from '~/common/decorators/http.decorator'
 import { BizException } from '~/common/exceptions/biz.exception'
 import { ErrorCodeEnum } from '~/constants/error-code.constant'
 import { UploadService } from '~/processors/helper/helper.upload.service'
-import { getMediumDateTime } from '~/utils'
 import { isZipMinetype } from '~/utils/mine.util'
+import { getMediumDateTime } from '~/utils/time.util'
 
 import { BackupService } from './backup.service'
 

diff --git a/apps/core/src/modules/backup/backup.service.ts b/apps/core/src/modules/backup/backup.service.ts
@@ -26,8 +26,9 @@ import { BACKUP_DIR, DATA_DIR } from '~/constants/path.constant'
 import { migrateDatabase } from '~/migration/migrate'
 import { EventManagerService } from '~/processors/helper/helper.event.service'
 import { CacheService } from '~/processors/redis/cache.service'
-import { getMediumDateTime, scheduleManager } from '~/utils'
+import { scheduleManager } from '~/utils/schedule.util'
 import { getFolderSize, installPKG } from '~/utils/system.util'
+import { getMediumDateTime } from '~/utils/time.util'
 
 import { ConfigsService } from '../configs/configs.service'
 

diff --git a/apps/core/src/modules/category/category.service.ts b/apps/core/src/modules/category/category.service.ts
@@ -18,7 +18,7 @@ import { BusinessEvents, EventScope } from '~/constants/business-event.constant'
 import { EventBusEvents } from '~/constants/event-bus.constant'
 import { EventManagerService } from '~/processors/helper/helper.event.service'
 import { InjectModel } from '~/transformers/model.transformer'
-import { scheduleManager } from '~/utils'
+import { scheduleManager } from '~/utils/schedule.util'
 
 import { PostService } from '../post/post.service'
 import { SlugTrackerService } from '../slug-tracker/slug-tracker.service'

diff --git a/apps/core/src/modules/comment/comment.controller.ts b/apps/core/src/modules/comment/comment.controller.ts
@@ -31,7 +31,7 @@ import { EventManagerService } from '~/processors/helper/helper.event.service'
 import { MongoIdDto } from '~/shared/dto/id.dto'
 import { PagerDto } from '~/shared/dto/pager.dto'
 import { transformDataToPaginate } from '~/transformers/paginate.transformer'
-import { scheduleManager } from '~/utils'
+import { scheduleManager } from '~/utils/schedule.util'
 
 import { ConfigsService } from '../configs/configs.service'
 import { UserModel } from '../user/user.model'

diff --git a/apps/core/src/modules/comment/comment.interceptor.ts b/apps/core/src/modules/comment/comment.interceptor.ts
@@ -10,7 +10,7 @@ import type {
 import { Injectable } from '@nestjs/common'
 
 import { getNestExecutionContextRequest } from '~/transformers/get-req.transformer'
-import { getAvatar } from '~/utils'
+import { getAvatar } from '~/utils/tool.util'
 
 @Injectable()
 export class CommentFilterEmailInterceptor implements NestInterceptor {

diff --git a/apps/core/src/modules/comment/comment.service.ts b/apps/core/src/modules/comment/comment.service.ts
@@ -29,7 +29,8 @@ import { BarkPushService } from '~/processors/helper/helper.bark.service'
 import { EmailService } from '~/processors/helper/helper.email.service'
 import { EventManagerService } from '~/processors/helper/helper.event.service'
 import { InjectModel } from '~/transformers/model.transformer'
-import { getAvatar, hasChinese, scheduleManager } from '~/utils'
+import { scheduleManager } from '~/utils/schedule.util'
+import { getAvatar, hasChinese } from '~/utils/tool.util'
 
 import { ConfigsService } from '../configs/configs.service'
 import { createMockedContextResponse } from '../serverless/mock-response.util'

diff --git a/apps/core/src/modules/configs/configs.service.ts b/apps/core/src/modules/configs/configs.service.ts
@@ -23,8 +23,8 @@ import { EventManagerService } from '~/processors/helper/helper.event.service'
 import { CacheService } from '~/processors/redis/cache.service'
 import { SubPubBridgeService } from '~/processors/redis/subpub.service'
 import { InjectModel } from '~/transformers/model.transformer'
-import { camelcaseKeys, sleep } from '~/utils'
 import { getRedisKey } from '~/utils/redis.util'
+import { camelcaseKeys } from '~/utils/tool.util'
 
 import { generateDefaultConfig } from './configs.default'
 import { decryptObject, encryptObject } from './configs.encrypt.util'

diff --git a/apps/core/src/modules/dependency/dependency.controller.ts b/apps/core/src/modules/dependency/dependency.controller.ts
@@ -7,7 +7,7 @@ import { ApiController } from '~/common/decorators/api-controller.decorator'
 import { Auth } from '~/common/decorators/auth.decorator'
 import { HTTPDecorators } from '~/common/decorators/http.decorator'
 import { DATA_DIR } from '~/constants/path.constant'
-import { installPKG } from '~/utils'
+import { installPKG } from '~/utils/system.util'
 
 import { ServerlessService } from '../serverless/serverless.service'
 

diff --git a/apps/core/src/modules/feed/feed.controller.ts b/apps/core/src/modules/feed/feed.controller.ts
@@ -7,7 +7,7 @@ import { Controller, Get, Header } from '@nestjs/common'
 
 import { HTTPDecorators } from '~/common/decorators/http.decorator'
 import { CacheKeys } from '~/constants/cache.constant'
-import { escapeXml } from '~/utils'
+import { escapeXml } from '~/utils/tool.util'
 
 import { AggregateService } from '../aggregate/aggregate.service'
 import { ConfigsService } from '../configs/configs.service'

diff --git a/apps/core/src/modules/health/sub-controller/log.controller.ts b/apps/core/src/modules/health/sub-controller/log.controller.ts
@@ -15,8 +15,8 @@ import { Auth } from '~/common/decorators/auth.decorator'
 import { HTTPDecorators } from '~/common/decorators/http.decorator'
 import { LOG_DIR } from '~/constants/path.constant'
 import { AdapterResponse } from '~/types/request'
-import { formatByteSize } from '~/utils'
 import { getTodayLogFilePath } from '~/utils/path.util'
+import { formatByteSize } from '~/utils/system.util'
 
 import { LogQueryDto, LogTypeDto } from '../health.dto'
 

diff --git a/apps/core/src/modules/helper/helper.controller.ts b/apps/core/src/modules/helper/helper.controller.ts
@@ -1,13 +1,27 @@
 import { FastifyReply } from 'fastify'
 
-import { BadRequestException, Get, Param, Query, Res } from '@nestjs/common'
+import {
+  BadRequestException,
+  Get,
+  Param,
+  Post,
+  Query,
+  Res,
+} from '@nestjs/common'
+import { ModuleRef } from '@nestjs/core'
 
 import { ApiController } from '~/common/decorators/api-controller.decorator'
+import { Auth } from '~/common/decorators/auth.decorator'
 import { CollectionRefTypes } from '~/constants/db.constant'
 import { DatabaseService } from '~/processors/database/database.service'
+import { ImageService } from '~/processors/helper/helper.image.service'
 import { UrlBuilderService } from '~/processors/helper/helper.url-builder.service'
 import { MongoIdDto } from '~/shared/dto/id.dto'
+import { AsyncQueue } from '~/utils/queue.util'
 
+import { NoteService } from '../note/note.service'
+import { PageService } from '../page/page.service'
+import { PostService } from '../post/post.service'
 import { HelperService } from './helper.service'
 
 @ApiController('helper')
@@ -17,6 +31,8 @@ export class HelperController {
 
     private readonly urlBulderService: UrlBuilderService,
     private readonly databaseService: DatabaseService,
+
+    private readonly moduleRef: ModuleRef,
   ) {}
 
   @Get('/url-builder/:id')
@@ -46,4 +62,31 @@ export class HelperController {
       res.send({ data: url })
     }
   }
+
+  @Post('/refresh-images')
+  @Auth()
+  async refreshImages() {
+    const postService = this.moduleRef.get(PostService, { strict: false })
+    const noteService = this.moduleRef.get(NoteService, { strict: false })
+    const pageService = this.moduleRef.get(PageService, { strict: false })
+    const imageService = this.moduleRef.get(ImageService, { strict: false })
+    const post = await postService.model.find()
+    const notes = await noteService.model.find()
+    const pages = await pageService.model.find()
+
+    const q = new AsyncQueue(10)
+    q.addMultiple(
+      [...post, ...notes, ...pages].map(
+        (doc) => () =>
+          imageService.saveImageDimensionsFromMarkdownText(
+            doc.text,
+            doc.images,
+            (images) => {
+              doc.images = images
+              return doc.save()
+            },
+          ),
+      ),
+    )
+  }
 }
diff --git a/apps/core/src/modules/link/link.controller.ts b/apps/core/src/modules/link/link.controller.ts
@@ -21,7 +21,7 @@ import {
   BaseCrudFactory,
   BaseCrudModuleType,
 } from '~/transformers/crud-factor.transformer'
-import { scheduleManager } from '~/utils'
+import { scheduleManager } from '~/utils/schedule.util'
 
 import { AuditReasonDto, LinkDto } from './link.dto'
 import { LinkModel, LinkState } from './link.model'

diff --git a/apps/core/src/modules/link/link.service.ts b/apps/core/src/modules/link/link.service.ts
@@ -14,7 +14,7 @@ import { EmailService } from '~/processors/helper/helper.email.service'
 import { EventManagerService } from '~/processors/helper/helper.event.service'
 import { HttpService } from '~/processors/helper/helper.http.service'
 import { InjectModel } from '~/transformers/model.transformer'
-import { scheduleManager } from '~/utils'
+import { scheduleManager } from '~/utils/schedule.util'
 
 import { ConfigsService } from '../configs/configs.service'
 import { UserService } from '../user/user.service'

diff --git a/apps/core/src/modules/note/note.service.ts b/apps/core/src/modules/note/note.service.ts
@@ -15,7 +15,8 @@ import { EventManagerService } from '~/processors/helper/helper.event.service'
 import { ImageService } from '~/processors/helper/helper.image.service'
 import { TextMacroService } from '~/processors/helper/helper.macro.service'
 import { InjectModel } from '~/transformers/model.transformer'
-import { getLessThanNow, scheduleManager } from '~/utils'
+import { scheduleManager } from '~/utils/schedule.util'
+import { getLessThanNow } from '~/utils/time.util'
 
 import { getArticleIdFromRoomName } from '../activity/activity.util'
 import { CommentService } from '../comment/comment.service'

diff --git a/apps/core/src/modules/page/page.service.ts b/apps/core/src/modules/page/page.service.ts
@@ -12,7 +12,7 @@ import { EventManagerService } from '~/processors/helper/helper.event.service'
 import { ImageService } from '~/processors/helper/helper.image.service'
 import { TextMacroService } from '~/processors/helper/helper.macro.service'
 import { InjectModel } from '~/transformers/model.transformer'
-import { scheduleManager } from '~/utils'
+import { scheduleManager } from '~/utils/schedule.util'
 
 import { PageModel } from './page.model'
 

diff --git a/apps/core/src/modules/post/post.service.ts b/apps/core/src/modules/post/post.service.ts
@@ -22,7 +22,8 @@ import { EventManagerService } from '~/processors/helper/helper.event.service'
 import { ImageService } from '~/processors/helper/helper.image.service'
 import { TextMacroService } from '~/processors/helper/helper.macro.service'
 import { InjectModel } from '~/transformers/model.transformer'
-import { getLessThanNow, scheduleManager } from '~/utils'
+import { scheduleManager } from '~/utils/schedule.util'
+import { getLessThanNow } from '~/utils/time.util'
 
 import { getArticleIdFromRoomName } from '../activity/activity.util'
 import { CategoryService } from '../category/category.service'

diff --git a/apps/core/src/modules/recently/recently.service.ts b/apps/core/src/modules/recently/recently.service.ts
@@ -17,7 +17,8 @@ import { DatabaseService } from '~/processors/database/database.service'
 import { EventManagerService } from '~/processors/helper/helper.event.service'
 import { CacheService } from '~/processors/redis/cache.service'
 import { InjectModel } from '~/transformers/model.transformer'
-import { getRedisKey, scheduleManager } from '~/utils'
+import { getRedisKey } from '~/utils/redis.util'
+import { scheduleManager } from '~/utils/schedule.util'
 
 import { CommentState } from '../comment/comment.model'
 import { CommentService } from '../comment/comment.service'

diff --git a/apps/core/src/modules/render/render.controller.ts b/apps/core/src/modules/render/render.controller.ts
@@ -23,7 +23,7 @@ import { HttpCache } from '~/common/decorators/cache.decorator'
 import { HTTPDecorators } from '~/common/decorators/http.decorator'
 import { IsAuthenticated } from '~/common/decorators/role.decorator'
 import { MongoIdDto } from '~/shared/dto/id.dto'
-import { getShortDateTime } from '~/utils'
+import { getShortDateTime } from '~/utils/time.util'
 
 import { ConfigsService } from '../configs/configs.service'
 import { MarkdownPreviewDto } from '../markdown/markdown.dto'