feat: use lru_map for caching

azu · Apr 24, 2021 · 57952f8 · 57952f8
1 parent 85cbf36
commit 57952f8
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 3 deletions.
diff --git a/package.json b/package.json
@@ -34,7 +34,8 @@
     "netlify": "mkdir -p public && cp -r node_modules/kuromoji/dict public/"
   },
   "dependencies": {
-    "kuromoji": "0.1.1"
+    "kuromoji": "0.1.1",
+    "lru_map": "^0.4.1"
   },
   "devDependencies": {
     "@types/mocha": "^8.2.0",

diff --git a/src/kuromojin.ts b/src/kuromojin.ts
@@ -1,9 +1,10 @@
 // LICENSE : MIT
 "use strict";
 import path from "path";
+import { LRUMap } from 'lru_map'
+import Deferred from "./Deferred";
 
 const kuromoji = require("kuromoji");
-import Deferred from "./Deferred";
 
 export type Tokenizer = {
     tokenize: (text: string) => KuromojiToken[];
@@ -66,9 +67,10 @@ const getNodeModuleDirPath = () => {
 
 // cache for tokenizer
 let _tokenizer: null | Tokenizer = null;
-let tokenizeCacheMap = new Map<string, KuromojiToken[]>()
 // lock boolean
 let isLoading = false;
+// cache for tokenize
+const tokenizeCacheMap = new LRUMap<string, KuromojiToken[]>(10000);
 
 export type getTokenizerOption = {
     dicPath: string;

diff --git a/yarn.lock b/yarn.lock
@@ -763,6 +763,11 @@ log-update@^4.0.0:
     slice-ansi "^4.0.0"
     wrap-ansi "^6.2.0"
 
+lru_map@^0.4.1:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.4.1.tgz#f7b4046283c79fb7370c36f8fca6aee4324b0a98"
+  integrity sha512-I+lBvqMMFfqaV8CJCISjI3wbjmwVu/VyOoU7+qtu9d7ioW5klMgsTTiUOUp+DJvfTTzKXoPbyC6YfgkNcyPSOg==
+
 make-error@^1.1.1:
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.5.tgz#efe4e81f6db28cadd605c70f29c831b58ef776c8"