fix: resolve duplicated after sync (#36)

The problem was caused by serialization when auto saving. The serialization would lost all history before, so the recover and sync would create a duplicated file.

BREAKING CHANGE: The format of auto saving file was changed, so the old file can not be loaded

src/components/codemirror/index.tsx

@@ -1,5 +1,5 @@
 import clsx from "clsx"
-import { memo, useCallback, useEffect, useMemo, useRef } from "react"
+import { memo, useEffect, useMemo, useRef } from "react"
 import { EditorView, keymap } from "@codemirror/view"
 import { indentWithTab } from "@codemirror/commands"
 import { basicSetup } from "codemirror"
@@ -15,7 +15,6 @@ import { UndoManager } from "yjs"
 import cache from "@/lib/fs/cache"
 // @ts-ignore
 import { yCollab } from "y-codemirror.next"
-import { fromSource } from "@/lib/fs/model"
 
 type CodemirrorProps = {
   className?: string
@@ -52,12 +51,11 @@ const Codemirror = memo((props: CodemirrorProps) => {
     useCommonConfigurationExtension(cm),
   )
 
-  const buildStaticSourceforCache = useCallback(() => fromSource(props.source), [props.source])
   const cacheUpdateListenerExtension = useMemo(
     () =>
       EditorView.updateListener.of((e) => {
         if (!e.docChanged) return
-        cache.debouncedUpdateCache(props.source.id, buildStaticSourceforCache)
+        cache.debouncedUpdateCache(props.source.id, () => props.source.serialize())
       }),
     [props.source],
   )

src/hooks/useY.ts

@@ -1,7 +1,7 @@
 import { Getter } from "jotai"
 import { atomWithObservable } from "jotai/utils"
 import { useEffect, useState } from "react"
-import { Text, Array, AbstractType, Map, Doc } from "yjs"
+import { Text, Array, AbstractType, Map, Doc, encodeStateAsUpdateV2, applyUpdateV2 } from "yjs"
 
 export function createYjsHook<T, V extends AbstractType<any>>(initialValue: T, observer: V, updater: (y: V) => T): T {
   const [content, setContent] = useState(initialValue)
@@ -66,6 +66,7 @@ export function useYMap<V>(ymap: Map<V>) {
 export class YjsNS {
   doc: Doc
   id: string
+
   public constructor(doc: Doc, id: string) {
     const map = doc.getMap()
     this.id = id
@@ -76,6 +77,7 @@ export class YjsNS {
       map.set(id, this.doc)
     }
     doc.load()
+
   }
 
   public destroy() {
@@ -156,4 +158,11 @@ export class YjsNS {
   public useMap(name: string) {
     return useYMap(this.getMap(name))
   }
+
+  public encode(): Uint8Array{
+    return encodeStateAsUpdateV2(this.doc)
+  }
+  public decode(data: Uint8Array){
+    applyUpdateV2(this.doc, data)
+  }
 }

src/lib/fs/cache.ts

@@ -1,11 +1,11 @@
 import { path, fs } from "@tauri-apps/api"
 import * as log from "tauri-plugin-log-api"
-import { StaticSourceData } from "./model"
 import { debounce } from "lodash"
+import { map, zip } from "lodash/fp"
 
 /**
  * get dir that source data cache in
- * @returns 
+ * @returns
  */
 async function getDataDir() {
   const filesDir = await path.join(await path.appDataDir(), "files")
@@ -17,34 +17,34 @@ async function getDataDir() {
 
 /**
  * get expected filepath in datadir by source id
- * @param id 
- * @returns 
+ * @param id
+ * @returns
  */
 async function getFilePath(id: string): Promise<string> {
   const dir = await getDataDir()
-  const file = await path.join(dir, `${id}.src.json`)
+  const file = await path.join(dir, `${id}.bin`)
   return file
 }
 
 /**
  * create or update cache of source
- * @param src 
+ * @param src
  */
-async function updateCache(id: string, src: ()=>StaticSourceData) {
+async function updateCache(id: string, data: ()=>Uint8Array) {
   const file = await getFilePath(id)
   log.info(`update cache ${id} to path ${file}`)
-  fs.writeTextFile(file, JSON.stringify(src()))
+  fs.writeBinaryFile(file, data())
 }
 
-const debouncedUpdateCache = debounce(updateCache, 1000, {maxWait: 30*1000})
+const debouncedUpdateCache = debounce(updateCache, 3000, { maxWait: 30 * 1000 })
 
 /**
  * delete source cache if exists
- * @param id 
+ * @param id
  */
 async function dropCache(id: string) {
   const file = await getFilePath(id)
-  if ((await fs.exists(file))) {
+  if (await fs.exists(file)) {
     log.info(`drop cache ${id} (${file})`)
     await fs.removeFile(file)
   }
@@ -64,26 +64,25 @@ async function dropAll() {
   await Promise.all(tasks)
 }
 
+async function loadFile(file: string): Promise<Uint8Array> {
+  return await fs.readBinaryFile(file)
+}
 
-/**
- * 
- * @param file 
- * @returns 
- */
-async function loadFile(file: string): Promise<StaticSourceData> {
-  const content = await fs.readTextFile(file)
-  const obj = JSON.parse(content)
-  return obj 
+interface SerializedSource {
+  id: string
+  data: Uint8Array
 }
 
-async function loadAll(): Promise<StaticSourceData[]> {
+async function loadAll(): Promise<SerializedSource[]> {
   const dir = await getDataDir()
   const files = (await fs.readDir(dir, { recursive: false }))
-    .filter((p) => p.children == null && p.name != null && p.name.endsWith(".src.json"))
+    .filter((p) => p.children == null && p.name != null && p.name.endsWith(".bin"))
     .map((e) => e.path!)
 
-  const tasks = files.map(loadFile)
-  return await Promise.all(tasks)
+  const tasks = await Promise.all(map(loadFile, files))
+  const ids = map((name: string) => name.substring(0, name.lastIndexOf(".bin")), files)
+  const construct = (v: [Uint8Array, string]) => ({ data: v[0], id: v[1] }) as SerializedSource
+  return map((v) => construct(v as [Uint8Array, string]), zip(tasks, ids))
 }
 
 export default {

src/lib/fs/model.ts

@@ -21,7 +21,13 @@ export interface StaticSourceData {
   readonly tests: StaticTestData[]
 }
 
-export function fromSource(source: Source): StaticSourceData {
+/**
+ * Get data from Yjs
+ * this would lost all history info
+ * @param source 
+ * @returns 
+ */
+export function intoStaticSource(source: Source): StaticSourceData {
   return {
     id: source.id,
     name: source.name.toString(),
@@ -53,7 +59,7 @@ export function fromSource(source: Source): StaticSourceData {
  * Attation: it would not change the id of source object,
  * if you want to specify its id, use store.create plz
  */
-export function intoSource(
+export function fillSource(
   data: StaticSourceData,
   source: Source,
   defaultTimeLimit: number = 5000,
@@ -86,4 +92,4 @@ export function intoSource(
     },
     zip(data.tests, range(0, data.tests.length)),
   )
-}
+}

src/lib/fs/problem.ts

@@ -6,8 +6,8 @@ import { parse, right } from "../parse"
 import { capitalize } from "lodash"
 import { map, __, flatten, zip, range } from "lodash/fp"
 import { crc16 } from "crc"
-import { StaticSourceData, StaticTestData } from "./model"
 import {v4 as uuid} from 'uuid'
+import { StaticSourceData, StaticTestData } from "./model"
 
 /**
  * get language mode from file extension name

src/pages/Main/event/index.tsx

@@ -1,11 +1,11 @@
-import CacheLoader from "./cache-loader"
 import MenuEventReceiver from "./menu-event"
+import useAutoSaveLoader from "./useAutoSaveLoader"
 
 export default function MainEventRegister() {
+  useAutoSaveLoader()
   return (
     <>
       <MenuEventReceiver />
-      <CacheLoader />
       {/* <StatusRecover /> */}
     </>
   )

src/pages/Main/event/menu-event.tsx

@@ -1,5 +1,4 @@
 import { useMitt } from "@/hooks/useMitt"
-import { fromSource } from "@/lib/fs/model"
 import { openProblem, saveProblem } from "@/lib/fs/problem"
 import { defaultLanguageAtom, defaultMemoryLimitsAtom, defaultTimeLimitsAtom } from "@/store/setting/setup"
 import { activedSourceAtom, createSourceAtom, sourceAtom } from "@/store/source"
@@ -9,6 +8,7 @@ import { useAtomValue, useSetAtom } from "jotai"
 import { LocalSourceMetadata, getSourceMetaAtom, setSourceMetaAtom } from "@/store/source/local"
 import { crc16 } from "crc"
 import { zip } from "lodash/fp"
+import { intoStaticSource } from "@/lib/fs/model"
 
 /**
  * Save source to file
@@ -39,7 +39,7 @@ async function saveFile(source: Source, getSaveTarget?: (id: string) => string |
 
   // if it is null, that's means user cancel operation
   if (file) {
-    const staticSourceData = fromSource(source)
+    const staticSourceData = intoStaticSource(source)
     await saveProblem(staticSourceData, file)
   }
 }
@@ -109,3 +109,4 @@ export default function MenuEventReceiver() {
 
   return null
 }
+

src/pages/Main/event/cache-loader.tsx → src/pages/Main/event/useAutoSaveLoader.tsx

@@ -4,15 +4,14 @@ import cache from "@/lib/fs/cache"
 import useReadAtom from "@/hooks/useReadAtom"
 import { sourceAtom } from "@/store/source"
 import { forEach } from "lodash/fp"
-import { StaticSourceData } from "@/lib/fs/model"
 
 const alreadyLoadedAtom = atom(false)
 /**
  * Load cache and insert to source store
  * It only be execuate on startup
  */
 
-export default function CacheLoader() {
+export default function useAutoSaveLoader() {
   const [alreadyLoaded, setAlreadyLoaded] = useAtom(alreadyLoadedAtom)
   const currentLoaded = useRef(false)
   const readSourceStore = useReadAtom(sourceAtom)
@@ -24,10 +23,9 @@ export default function CacheLoader() {
       const store = readSourceStore()
       cache.loadAll().then((data) => {
         store.doc.transact(() => {
-          forEach((v: StaticSourceData)=>store.createFromStatic(v, v.id), data)
+          forEach((v)=>store.createByDeserialization(v.data, v.id), data)
         })
       })
     }
   }, [alreadyLoaded])
-  return null
 }

src/store/source/index.ts

@@ -6,7 +6,6 @@ import * as log from "tauri-plugin-log-api"
 import { LanguageMode } from "@/lib/ipc"
 import { createYjsHookAtom } from "@/hooks/useY"
 import cache from "@/lib/fs/cache"
-import { fromSource } from "@/lib/fs/model"
 import generateRandomName from "@/lib/names"
 
 export const docAtom = atom(new Doc())
@@ -84,7 +83,7 @@ export const createSourceAtom = atom(
       log.info(`create new source: ${id}`)
       log.info(JSON.stringify(get(sourceIdsAtom)))
     })
-    cache.updateCache(id, () => fromSource(source))
+    cache.updateCache(id, ()=>source.serialize())
     return source
   },
 )
@@ -120,7 +119,7 @@ export const duplicateSourceAtom = atom(null, (get, set, id: string, newName: (o
       )
     })
 
-    cache.updateCache(newSource.id, () => fromSource(newSource))
+    cache.updateCache(newSource.id, ()=>newSource.serialize())
     return newSource
   } else {
     return null

src/store/source/model.ts

@@ -2,10 +2,10 @@ import { YjsNS, createYjsHook } from "@/hooks/useY"
 import { LanguageMode } from "@/lib/ipc"
 import { v4 as uuid } from "uuid"
 import { atom } from "jotai"
-import { map } from "lodash/fp"
+import { map, uniq } from "lodash/fp"
 import { useEffect, useState } from "react"
 import { Doc, Text, Array, Map } from "yjs"
-import { StaticSourceData, intoSource } from "@/lib/fs/model"
+import { StaticSourceData, fillSource } from "@/lib/fs/model"
 
 export const rootDocument = atom(new Doc())
 
@@ -18,7 +18,7 @@ export enum JudgeStatus {
   CE,
   UK,
   UKE,
-  INT
+  INT,
 }
 
 /**
@@ -60,9 +60,9 @@ export class Testcase {
   }
   get status(): JudgeStatus {
     let value = this.map.get("status")
-    if(value != undefined && JudgeStatus[value] != undefined){
+    if (value != undefined && JudgeStatus[value] != undefined) {
       return value
-    }else {
+    } else {
       return JudgeStatus.UK
     }
   }
@@ -150,7 +150,7 @@ export class Source {
   get timelimit(): number {
     return this.store.getNumber("timelimit")
   }
-  useTimelimit():number{
+  useTimelimit(): number {
     return this.store.useNumber("timelimit")
   }
   set memorylimit(bytes: number) {
@@ -202,25 +202,51 @@ export class Source {
   deleteTest(index: number, length: number | undefined = undefined) {
     this.tests.delete(index, length)
   }
+
+  serialize(): Uint8Array {
+    return this.store.encode()
+  }
+  deserialize(data: Uint8Array) {
+    this.store.decode(data)
+  }
 }
 
 export class SourceStore {
   doc: Doc
   constructor(doc: Doc) {
     this.doc = doc
+
+    // make id unique
+    this.list.observe((event) => {
+      if (event.changes.added.size > 0) {
+        const data = this.list.toArray()
+        const uniqueData = uniq(data)
+        if (data.length != uniqueData.length) {
+          this.doc.transact(() => {
+            this.list.delete(0, this.list.length)
+            this.list.insert(0, uniqueData)
+          })
+        }
+      }
+    })
   }
   get list(): Array<string> {
     return this.doc.getArray("source/list")
   }
-  create(id: string = uuid() ): [Source, string] {
+  create(id: string = uuid()): [Source, string] {
     let subDoc = new YjsNS(this.doc, id)
     let store = new Source(subDoc)
     this.list.push([subDoc.id])
     return [store, subDoc.id]
   }
-  createFromStatic(data: StaticSourceData, specifyId: string = uuid()): [Source, string]{
+  createByDeserialization(data: Uint8Array, specifyId: string = uuid()): [Source, string] {
+    const [src, id] = this.create(specifyId)
+    src.deserialize(data)
+    return [src, id]
+  }
+  createFromStatic(data: StaticSourceData, specifyId: string = uuid()): [Source, string] {
     const [src, id] = this.create(specifyId)
-    intoSource(data, src)
+    fillSource(data, src)
     return [src, id]
   }
   get(id: string): Source | undefined {