cache smoldot db

package.json

@@ -45,6 +45,7 @@
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
     "cmdk": "1.0.4",
+    "idb-keyval": "^6.2.1",
     "lucide-react": "^0.456.0",
     "polkadot-api": "^1.7.3",
     "react": "^18.3.1",

pnpm-lock.yaml

@@ -109,6 +109,9 @@ importers:
       cmdk:
         specifier: 1.0.4
         version: 1.0.4(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
+      idb-keyval:
+        specifier: ^6.2.1
+        version: 6.2.1
       lucide-react:
         specifier: ^0.456.0
         version: 0.456.0(react@18.3.1)

src/main.tsx

@@ -5,6 +5,7 @@ import { BrowserRouter } from "react-router-dom"
 import { merge } from "rxjs"
 import App from "./App.tsx"
 import { dynamicBuilder$ } from "@/state/chains/chain.state"
+import { persistSyncState$ } from "./state/chains/chain.state.ts"
 import { TooltipProvider } from "./components/Tooltip.tsx"
 import "./index.css"
 import { explorer$ } from "./pages/Explorer"
@@ -12,7 +13,14 @@ import { transactions$ } from "./pages/Transactions"
 import { ThemeProvider } from "./ThemeProvider.tsx"
 
 createRoot(document.getElementById("root")!).render(
-  <Subscribe source$={merge(dynamicBuilder$, explorer$, transactions$)}>
+  <Subscribe
+    source$={merge(
+      dynamicBuilder$,
+      explorer$,
+      transactions$,
+      persistSyncState$,
+    )}
+  >
     <RemoveSubscribe>
       <StrictMode>
         <ThemeProvider>

src/state/chains/chain.state.ts

@@ -1,12 +1,14 @@
+import { cyclingLocalCache } from "@/utils/cyclingLocalCache"
 import { getDynamicBuilder, getLookupFn } from "@polkadot-api/metadata-builders"
 import { getObservableClient } from "@polkadot-api/observable-client"
-import { decAnyMetadata, HexString } from "@polkadot-api/substrate-bindings"
+import { decAnyMetadata } from "@polkadot-api/substrate-bindings"
 import { createClient as createSubstrateClient } from "@polkadot-api/substrate-client"
 import { toHex } from "@polkadot-api/utils"
 import { sinkSuspense, state, SUSPENSE } from "@react-rxjs/core"
 import { createSignal } from "@react-rxjs/utils"
 import { createClient } from "polkadot-api"
 import {
+  catchError,
   concat,
   EMPTY,
   filter,
@@ -23,12 +25,15 @@ import {
   createSmoldotSource,
   getSmoldotProvider,
   SmoldotSource,
+  RelayChain,
+  relayChains,
 } from "./smoldot"
 import {
   createWebsocketSource,
   getWebsocketProvider,
   WebsocketSource,
 } from "./websocket"
+import { setCachedSmoldotDb } from "./smoldot.cache"
 
 export type ChainSource = WebsocketSource | SmoldotSource
 
@@ -88,34 +93,18 @@ export const unsafeApi$ = chainClient$.pipeState(
   map(({ client }) => client.getUnsafeApi()),
 )
 
-const uncachedRuntimeCtx$ = chainClient$.pipeState(
+export const uncachedRuntimeCtx$ = chainClient$.pipeState(
   switchMap(({ chainHead }) => chainHead.runtime$),
   filter((v) => !!v),
 )
 
-const getMetadataCache = () => {
-  const cached = localStorage.getItem(`metadata-cache`)
-  return new Map<string, { time: number; data: HexString }>(
-    cached ? JSON.parse(cached) : [],
-  )
-}
-const getCachedMetadata = (id: string) =>
-  getMetadataCache().get(id)?.data ?? null
-const setCachedMetadata = (id: string, data: HexString) => {
-  const cached = getMetadataCache()
-  cached.set(id, { time: Date.now(), data })
-  if (cached.size > 3) {
-    const oldest = [...cached.entries()].reduce((a, b) =>
-      a[1].time < b[1].time ? a : b,
-    )[0]
-    cached.delete(oldest)
-  }
-  localStorage.setItem("metadata-cache", JSON.stringify([...cached.entries()]))
-}
+const [getCachedMetadata, setCachedMetadata] =
+  cyclingLocalCache("metadata-cache")
 export const runtimeCtx$ = chainClient$.pipeState(
-  switchMap(({ id }) => {
-    const cached = getCachedMetadata(id)
-
+  switchMap(({ id }) =>
+    getCachedMetadata(id).then((cached) => ({ id, cached })),
+  ),
+  switchMap(({ id, cached }) => {
     const realCtx$ = uncachedRuntimeCtx$.pipe(
       tap((v) => {
         setCachedMetadata(id, toHex(v.metadataRaw))
@@ -144,4 +133,23 @@ export const dynamicBuilder$ = runtimeCtx$.pipeState(
   map((ctx) => ctx.dynamicBuilder),
 )
 
+export const persistSyncState$ = chainClient$.pipe(
+  switchMap(({ id, client }) => {
+    if (!relayChains.includes(id as RelayChain)) return EMPTY
+
+    return uncachedRuntimeCtx$.pipe(
+      switchMap(() =>
+        client._request<string, []>("chainHead_unstable_finalizedDatabase", []),
+      ),
+      catchError(() => EMPTY),
+      filter((v) => !!v),
+      map((db) => ({
+        id,
+        db,
+      })),
+    )
+  }),
+  tap(({ id, db }) => setCachedSmoldotDb(id, db)),
+)
+
 export { networkCategories, type Network } from "./networks"

src/state/chains/smoldot.cache.ts

@@ -0,0 +1,4 @@
+import { cyclingLocalCache } from "@/utils/cyclingLocalCache"
+
+export const [getCachedSmoldotDb, setCachedSmoldotDb] =
+  cyclingLocalCache("smoldot-db")

src/state/chains/smoldot.ts

@@ -4,6 +4,7 @@ import { getSmProvider } from "polkadot-api/sm-provider"
 import { Chain } from "polkadot-api/smoldot"
 import { startFromWorker } from "polkadot-api/smoldot/from-worker"
 import SmWorker from "polkadot-api/smoldot/worker?worker"
+import { getCachedSmoldotDb } from "./smoldot.cache"
 
 const [dotChainSpec, ksmChainSpec, paseoChainSpec, westendChainSpec] = [
   import("polkadot-api/chains/polkadot"),
@@ -11,13 +12,18 @@ const [dotChainSpec, ksmChainSpec, paseoChainSpec, westendChainSpec] = [
   import("polkadot-api/chains/paseo"),
   import("polkadot-api/chains/westend2"),
 ].map((x) => x.then((y) => y.chainSpec))
+
 const smoldot = startFromWorker(new SmWorker(), {
   logCallback: (level, target, message) => {
     console.debug("smoldot[%s(%s)] %s", target, level, message)
   },
 })
-const relayChains: Record<
-  string,
+
+export const relayChains = ["polkadot", "kusama", "paseo", "westend"] as const
+export type RelayChain = (typeof relayChains)[number]
+
+const relaySmoldotChains: Record<
+  RelayChain,
   { chainSpec: Promise<string>; chain: Promise<Chain> | null }
 > = {
   polkadot: { chainSpec: dotChainSpec, chain: null },
@@ -25,21 +31,39 @@ const relayChains: Record<
   paseo: { chainSpec: paseoChainSpec, chain: null },
   westend: { chainSpec: westendChainSpec, chain: null },
 }
-const getRelayChain = async (name: string) => {
-  if (!relayChains[name].chain) {
-    relayChains[name].chain = smoldot.addChain({
-      chainSpec: await relayChains[name].chainSpec,
-    })
-  }
-  return relayChains[name].chain
+const createChain = async (
+  id: string,
+  chainSpec: string | Promise<string>,
+  relayChain?: Promise<Chain>,
+) => {
+  const databaseContent =
+    (id in relaySmoldotChains && (await getCachedSmoldotDb(id))) || undefined
+  const solvedChain = relayChain ? await relayChain : undefined
+  return smoldot.addChain({
+    chainSpec: await chainSpec,
+    databaseContent,
+    potentialRelayChains: solvedChain ? [solvedChain] : undefined,
+  })
 }
+const getRelayChain = async (name: RelayChain) => {
+  if (!relaySmoldotChains[name].chain) {
+    relaySmoldotChains[name].chain = createChain(
+      name,
+      relaySmoldotChains[name].chainSpec,
+    )
+  }
+  return relaySmoldotChains[name].chain
+}
+
+// Pre-initialize all relay chains
+relayChains.forEach(getRelayChain)
 
 export interface SmoldotSource {
   type: "chainSpec"
   id: string
   value: {
     chainSpec: string
-    relayChain?: string
+    relayChain?: RelayChain
   }
 }
 
@@ -47,8 +71,8 @@ export function createSmoldotSource(
   id: string,
   relayChain?: string,
 ): Promise<SmoldotSource> {
-  if (id in relayChains) {
-    return relayChains[id].chainSpec.then((chainSpec) => ({
+  if (id in relaySmoldotChains) {
+    return relaySmoldotChains[id as RelayChain].chainSpec.then((chainSpec) => ({
       id,
       type: "chainSpec",
       value: { chainSpec },
@@ -69,18 +93,12 @@ export function createSmoldotSource(
 
 export function getSmoldotProvider(source: SmoldotSource): JsonRpcProvider {
   const chain = source.value.relayChain
-    ? getRelayChain(source.value.relayChain).then((chain) =>
-        smoldot.addChain({
-          chainSpec: source.value.chainSpec,
-          potentialRelayChains: [chain],
-        }),
+    ? createChain(
+        source.id,
+        source.value.chainSpec,
+        getRelayChain(source.value.relayChain),
       )
-    : smoldot.addChain({
-        chainSpec: source.value.chainSpec,
-      })
+    : createChain(source.id, source.value.chainSpec)
 
-  return withLogsRecorder(
-    (v) => v.includes("initialized") && console.debug(v),
-    getSmProvider(chain),
-  )
+  return withLogsRecorder((v) => console.debug(v), getSmProvider(chain))
 }

src/utils/cyclingLocalCache.ts

@@ -0,0 +1,36 @@
+import { get, set } from "idb-keyval"
+
+export const cyclingLocalCache = (key: string, limit = 10) => {
+  const getCache = async () => {
+    try {
+      const entries = await get(key)
+      return new Map<string, { time: number; data: string }>(entries)
+    } catch (_) {
+      return null
+    }
+  }
+  const getCachedItem = (id: string) =>
+    id === "metadata-cache"
+      ? Promise.resolve(null)
+      : getCache().then((cache) => cache?.get(id)?.data ?? null)
+  const setCachedItem = async (id: string, data: string) => {
+    const cached = await getCache()
+    if (!cached) return
+
+    cached.set(id, { time: Date.now(), data })
+    if (cached.size > limit) {
+      const oldest = [...cached.entries()].reduce((a, b) =>
+        a[1].time < b[1].time ? a : b,
+      )[0]
+      cached.delete(oldest)
+    }
+
+    try {
+      await set(key, [...cached.entries()])
+    } catch (_) {
+      /* empty */
+    }
+  }
+
+  return [getCachedItem, setCachedItem] as const
+}