Merge pull request #65 from polkadot-api/vo/storage-keys

feat: decode storage keys entry and partial keys

src/pages/Storage/Storage.tsx

@@ -5,12 +5,11 @@ import { Chopsticks } from "@/components/Icons"
 import { LoadingMetadata } from "@/components/Loading"
 import { SearchableSelect } from "@/components/Select"
 import { withSubscribe } from "@/components/withSuspense"
-import { useHashState } from "@/lib/externalState"
 import { lookup$ } from "@/state/chains/chain.state"
 import { state, useStateObservable } from "@react-rxjs/core"
-import { FC, useEffect, useState } from "react"
+import { FC, useState } from "react"
 import { map } from "rxjs"
-import { selectedEntry$, setSelectedEntry } from "./storage.state"
+import { partialEntry$, selectedEntry$, selectEntry } from "./storage.state"
 import { StorageDecode } from "./StorageDecode"
 import { StorageQuery } from "./StorageQuery"
 import { StorageSet } from "./StorageSet"
@@ -36,97 +35,32 @@ const metadataStorage$ = state(
 
 export const Storage = withSubscribe(
   () => {
-    const { lookup, entries } = useStateObservable(metadataStorage$)
-    const [pallet, setPallet] = useHashState("pallet", "System")
-    const [entry, setEntry] = useHashState("entry", "Account")
+    const { entries } = useStateObservable(metadataStorage$)
+    const partialEntry = useStateObservable(partialEntry$)
     const selectedEntry = useStateObservable(selectedEntry$)
 
-    const selectedPallet =
-      (pallet && lookup.metadata.pallets.find((p) => p.name === pallet)) || null
-
-    useEffect(
-      () =>
-        setEntry((prev) => {
-          if (!selectedPallet?.storage?.items[0]) return null
-          return selectedPallet.storage.items.some((v) => v.name === prev)
-            ? prev
-            : selectedPallet.storage.items[0].name
-        }),
-      // eslint-disable-next-line react-hooks/exhaustive-deps
-      [selectedPallet?.name],
-    )
-
-    useEffect(() => {
-      const storageEntry =
-        (entry &&
-          selectedPallet?.storage?.items.find((it) => it.name === entry)) ||
-        null
-      const storageEntryType = storageEntry?.type
-      if (!storageEntryType) {
-        return setSelectedEntry(null)
-      }
-
-      if (storageEntryType.tag === "plain") {
-        return setSelectedEntry({
-          value: storageEntryType.value,
-          key: [],
-          pallet: pallet!,
-          entry: entry!,
-          docs: storageEntry.docs,
-        })
-      }
-      if (storageEntryType.value.hashers.length === 1) {
-        return setSelectedEntry({
-          value: storageEntryType.value.value,
-          key: [storageEntryType.value.key],
-          pallet: pallet!,
-          entry: entry!,
-          docs: storageEntry.docs,
-        })
-      }
-
-      const keyDef = lookup(storageEntryType.value.key)
-      const key = (() => {
-        if (keyDef.type === "array") {
-          return new Array(keyDef.len).fill(keyDef.value.id)
-        }
-        if (keyDef.type === "tuple") {
-          return keyDef.value.map((e) => e.id)
-        }
-        throw new Error("Invalid key type " + keyDef.type)
-      })()
-      setSelectedEntry({
-        key,
-        value: storageEntryType.value.value,
-        pallet: pallet!,
-        entry: entry!,
-        docs: storageEntry.docs,
-      })
-      // eslint-disable-next-line react-hooks/exhaustive-deps
-    }, [selectedPallet, entry])
-
     return (
       <div className="p-4 pb-0 flex flex-col gap-2 items-start">
         <div className="flex items-center gap-2">
           <label>
             Pallet
             <SearchableSelect
-              value={pallet}
-              setValue={(v) => setPallet(v)}
+              value={partialEntry.pallet}
+              setValue={(v) => selectEntry({ pallet: v })}
               options={Object.keys(entries).map((e) => ({
                 text: e,
                 value: e,
               }))}
             />
           </label>
-          {selectedPallet && pallet && (
+          {partialEntry.pallet && (
             <label>
               Entry
               <SearchableSelect
-                value={entry}
-                setValue={(v) => setEntry(v)}
+                value={partialEntry.entry}
+                setValue={(v) => selectEntry({ entry: v })}
                 options={
-                  Object.keys(entries[pallet]).map((s) => ({
+                  Object.keys(entries[partialEntry.pallet]).map((s) => ({
                     text: s,
                     value: s,
                   })) ?? []
@@ -170,7 +104,7 @@ const StorageEntry: FC = () => {
           },
           {
             value: "decode",
-            content: "Decode",
+            content: "Decode Value",
           },
           ...(canSetStorage
             ? [

src/pages/Storage/StorageQuery.tsx

@@ -1,18 +1,20 @@
-import {
-  dynamicBuilder$,
-  runtimeCtx$,
-  unsafeApi$,
-} from "@/state/chains/chain.state"
 import { EditCodec } from "@/codec-components/EditCodec"
 import { ActionButton } from "@/components/ActionButton"
 import { BinaryEditButton } from "@/components/BinaryEditButton"
 import { CopyText } from "@/components/Copy"
 import SliderToggle from "@/components/Toggle"
+import {
+  dynamicBuilder$,
+  runtimeCtx$,
+  unsafeApi$,
+} from "@/state/chains/chain.state"
 import {
   CodecComponentType,
   CodecComponentValue,
   NOTIN,
 } from "@polkadot-api/react-builder"
+import { Twox128 } from "@polkadot-api/substrate-bindings"
+import { toHex } from "@polkadot-api/utils"
 import { state, useStateObservable, withDefault } from "@react-rxjs/core"
 import { createSignal } from "@react-rxjs/utils"
 import { Binary } from "polkadot-api"
@@ -23,14 +25,17 @@ import {
   firstValueFrom,
   from,
   map,
+  ObservedValueOf,
   scan,
   startWith,
   switchMap,
+  withLatestFrom,
 } from "rxjs"
 import { twMerge } from "tailwind-merge"
 import {
   addStorageSubscription,
   selectedEntry$,
+  selectEntry,
   stringifyArg,
 } from "./storage.state"
 
@@ -165,28 +170,40 @@ const builderState$ = state(
   ),
   null,
 )
+const keysCodec$ = combineLatest([keys$, builderState$]).pipe(
+  map(([keys, builder]) => keys.map((type) => builder?.buildDefinition(type))),
+)
 const keyInputValue$ = state(
   (idx: number) =>
     keyValues$.pipe(
-      map(
-        (v, i): CodecComponentValue =>
-          i === 0
-            ? {
-                type: CodecComponentType.Initial,
-              }
-            : {
-                type: CodecComponentType.Updated,
-                value:
-                  v[idx] === NOTIN
-                    ? {
-                        empty: true,
-                      }
-                    : {
-                        empty: false,
-                        decoded: v[idx],
-                      },
-              },
-      ),
+      withLatestFrom(keysCodec$),
+      map(([v, codecs], i): CodecComponentValue => {
+        if (i === 0) {
+          try {
+            return {
+              type: CodecComponentType.Initial,
+              value: codecs[idx]?.enc(v[idx]),
+            }
+          } catch {
+            return {
+              type: CodecComponentType.Initial,
+            }
+          }
+        }
+
+        return {
+          type: CodecComponentType.Updated,
+          value:
+            v[idx] === NOTIN
+              ? {
+                  empty: true,
+                }
+              : {
+                  empty: false,
+                  decoded: v[idx],
+                },
+        }
+      }),
     ),
   {
     type: CodecComponentType.Initial,
@@ -310,13 +327,10 @@ export const KeyDisplay: FC = () => {
   const key = useStateObservable(encodedKey$)
   const builder = useStateObservable(builderState$)
   const selectedEntry = useStateObservable(selectedEntry$)
-  const keys = useStateObservable(keys$)
   const keysEnabled = useStateObservable(keysEnabled$)
 
   if (!builder || !selectedEntry) return null
 
-  const codec = builder.buildStorage(selectedEntry.pallet, selectedEntry.entry)
-
   return (
     <div className="flex w-full overflow-hidden border border-card-foreground/60 px-3 p-2 gap-2 items-center bg-card text-card-foreground">
       <div className="shrink-0 text-sm font-bold">Encoded key:</div>
@@ -329,19 +343,117 @@ export const KeyDisplay: FC = () => {
         {key ?? "Fill in all the storage keys to calculate the encoded key"}
       </div>
       <CopyText text={key ?? ""} disabled={key === null} binary />
-      {keys.length === keysEnabled && (
-        <BinaryEditButton
-          initialValue={key ? Binary.fromHex(key).asBytes() : undefined}
-          onValueChange={(value: unknown[]) => {
-            value.forEach((value, idx) => setKeyValue({ idx, value }))
-          }}
-          decode={(v) =>
-            codec.keys.dec(
-              typeof v === "string" ? v : Binary.fromBytes(v).asHex(),
-            )
+      <BinaryEditButton
+        initialValue={key ? Binary.fromHex(key).asBytes() : undefined}
+        onValueChange={(value: NonNullable<DecodedKey>) => {
+          let newKeysEnabled = keysEnabled
+          if (
+            value.pallet.name !== selectedEntry.pallet ||
+            value.item.name !== selectedEntry.entry
+          ) {
+            selectEntry({
+              pallet: value.pallet.name,
+              entry: value.item.name,
+            })
+            newKeysEnabled =
+              value.item.type.tag === "plain"
+                ? 0
+                : value.item.type.value.hashers.length
           }
-        />
-      )}
+
+          if (value.args.length !== newKeysEnabled) {
+            if (value.args.length > newKeysEnabled) {
+              toggleKey(value.args.length - 1)
+            } else {
+              toggleKey(value.args.length)
+            }
+          }
+
+          value.args.forEach((value, idx) =>
+            setKeyValue({
+              idx,
+              value,
+            }),
+          )
+        }}
+        decode={(v) => {
+          const decoded = decodeKey(builder, v)
+          if (!decoded) throw null
+          return decoded
+        }}
+      />
     </div>
   )
 }
+
+const textEncoder = new TextEncoder()
+const hashersToLength: Record<string, number> = {
+  Identity: 0,
+  Twox64Concat: 8,
+  Blake2128Concat: 16,
+  Blake2128: -16,
+  Blake2256: -32,
+  Twox128: -16,
+  Twox256: -32,
+}
+
+const decodeKey = (
+  builder: NonNullable<ObservedValueOf<typeof builderState$>>,
+  key: Uint8Array,
+) => {
+  const twoxHash = (v: string) => toHex(Twox128(textEncoder.encode(v)))
+
+  const keyHex = toHex(key)
+  const pallet = builder.lookup.metadata.pallets.find(
+    (p) => p.storage && keyHex.startsWith(twoxHash(p.storage.prefix)),
+  )
+  if (!pallet) return null
+
+  const keyRemaining = keyHex.replace(twoxHash(pallet.storage!.prefix), "0x")
+  const item = pallet.storage!.items.find((v) =>
+    keyRemaining.startsWith(twoxHash(v.name)),
+  )
+  if (!item) return null
+
+  const codec = builder.buildStorage(pallet.name, item.name)
+  const hasherLengths =
+    item.type.tag === "plain"
+      ? []
+      : item.type.value.hashers.map((x) => hashersToLength[x.tag])
+
+  let argsRemaining = Binary.fromHex(
+    keyRemaining.replace(twoxHash(item.name), "0x"),
+  ).asBytes()
+
+  const args: any[] = []
+  const argsLen = codec.args.inner.length
+  for (let i = 0; i < argsLen && argsRemaining.length; i++) {
+    const hashLength = hasherLengths[i]
+
+    if (argsRemaining.length < Math.abs(hashLength)) return null
+    argsRemaining = argsRemaining.slice(Math.abs(hashLength))
+
+    if (hashLength < 0) {
+      // Signals a non-reversible hasher
+      args.push(null)
+    } else {
+      const argCodec = codec.args.inner[i]
+      try {
+        const value = argCodec.dec(argsRemaining)
+        // This is needed not just for the length, but see case AccountId: Can decode 0x, but then can't re-encode back. <- TODO bug?
+        const reEnc = argCodec.enc(value)
+        argsRemaining = argsRemaining.slice(reEnc.length)
+        args.push(value)
+      } catch {
+        return null
+      }
+    }
+  }
+
+  return {
+    pallet,
+    item,
+    args,
+  }
+}
+type DecodedKey = ReturnType<typeof decodeKey>

src/pages/Storage/storage.state.ts

@@ -1,9 +1,11 @@
 import { bytesToString } from "@/components/BinaryInput"
-import { selectedChainChanged$ } from "@/state/chains/chain.state"
+import { getHashParams } from "@/hashParams"
+import { runtimeCtx$, selectedChainChanged$ } from "@/state/chains/chain.state"
 import { state } from "@react-rxjs/core"
 import {
   createKeyedSignal,
   createSignal,
+  mergeWithKey,
   partitionByKey,
   toKeySet,
 } from "@react-rxjs/utils"
@@ -22,6 +24,7 @@ import {
   startWith,
   switchMap,
   takeUntil,
+  withLatestFrom,
 } from "rxjs"
 import { v4 as uuid } from "uuid"
 
@@ -33,9 +36,113 @@ export type StorageMetadataEntry = {
   docs: string[]
 }
 
-export const [entryChange$, setSelectedEntry] =
-  createSignal<StorageMetadataEntry | null>()
-export const selectedEntry$ = state(entryChange$, null)
+export const [entryChange$, selectEntry] = createSignal<{
+  pallet?: string | null
+  entry?: string | null
+}>()
+
+const palletEntries$ = runtimeCtx$.pipe(
+  map((ctx) =>
+    Object.fromEntries(
+      ctx.lookup.metadata.pallets.map((p) => [p.name, p.storage?.items ?? []]),
+    ),
+  ),
+)
+
+const initialValue$ = palletEntries$.pipe(
+  map(() => {
+    const params = getHashParams()
+    const pallet = params.get("pallet") ?? "System"
+    const entry = params.get("entry") ?? "Account"
+    return { entry, pallet }
+  }),
+)
+
+export const partialEntry$ = state(
+  mergeWithKey({ entryChange$, initialValue$ }).pipe(
+    withLatestFrom(palletEntries$),
+    scan(
+      (acc, [evt, pallets]) => {
+        const newValue =
+          evt.type === "entryChange$"
+            ? { ...acc, ...evt.payload }
+            : {
+                pallet: acc.pallet ?? evt.payload.pallet,
+                entry: acc.entry ?? evt.payload.entry,
+              }
+        const selectedPallet = newValue.pallet ? pallets[newValue.pallet] : null
+        if (!selectedPallet?.find((it) => it.name === newValue.entry)) {
+          newValue.entry = selectedPallet?.[0]?.name ?? null
+        }
+        return newValue
+      },
+      {
+        pallet: null as string | null,
+        entry: null as string | null,
+      },
+    ),
+  ),
+  {
+    pallet: null,
+    entry: null,
+  },
+)
+
+export const selectedEntry$ = state(
+  partialEntry$.pipe(
+    withLatestFrom(palletEntries$, runtimeCtx$),
+    map(([partialEntry, entries, ctx]): StorageMetadataEntry | null => {
+      const entry = partialEntry.pallet
+        ? entries[partialEntry.pallet]?.find(
+            (v) => v.name === partialEntry.entry,
+          )
+        : null
+      if (!entry?.type) return null
+
+      const { type, docs } = entry
+      const pallet = partialEntry.pallet!
+
+      if (type.tag === "plain") {
+        return {
+          value: type.value,
+          key: [],
+          pallet,
+          entry: entry.name,
+          docs,
+        }
+      }
+
+      if (type.value.hashers.length === 1) {
+        return {
+          value: type.value.value,
+          key: [type.value.key],
+          pallet,
+          entry: entry.name,
+          docs,
+        }
+      }
+
+      const keyDef = ctx.lookup(type.value.key)
+      const key = (() => {
+        if (keyDef.type === "array") {
+          return new Array(keyDef.len).fill(keyDef.value.id)
+        }
+        if (keyDef.type === "tuple") {
+          return keyDef.value.map((e) => e.id)
+        }
+        throw new Error("Invalid key type " + keyDef.type)
+      })()
+      return {
+        key,
+        value: type.value.value,
+        pallet,
+        entry: entry.name,
+        docs,
+      }
+    }),
+  ),
+  null,
+)
 
 export const [newStorageSubscription$, addStorageSubscription] = createSignal<{
   name: string