feat: reintroduce dataloader (#2252) (#2283) (#2299)

* feat: resolve txn metadata with dataloader (#2252) (#2283)

This reverts commit 47d4dbb to
reintroduce the dataloader changes.

* refactor: return undefined array for dataloader txnMetadata errors

* refactor: move repository error msgs into a regex object

* refactor: remove type-cast from const object declarations

This is to ensure a non-key can't be accessed on the object when being
used.

* chore: add new 'InvalidDocumentIdForDbError' for repository error parsing

* chore: add new 'CouldNotFindExpectedTransactionMetadataError'

* chore: add span attribute for txnMetadata returned from dataloader

* chore: filter out OnChainTxHash source.id before dataloader

package.json

@@ -59,6 +59,7 @@
     "bitcoinjs-lib": "^6.1.0",
     "cors": "^2.8.5",
     "csv-writer": "^1.6.0",
+    "dataloader": "^2.1.0",
     "dedent": "^0.7.0",
     "dotenv": "^16.0.3",
     "express": "^4.18.2",

src/app/index.ts

@@ -6,6 +6,7 @@ import * as AdminMod from "./admin"
 import * as ColdStorageMod from "./cold-storage"
 import * as LightningMod from "./lightning"
 import * as PricesMod from "./prices"
+import * as TransactionsMod from "./transactions"
 import * as UsersMod from "./users"
 import * as WalletsMod from "./wallets"
 import * as PaymentsMod from "./payments"
@@ -18,6 +19,7 @@ const allFunctions = {
   ColdStorage: { ...ColdStorageMod },
   Lightning: { ...LightningMod },
   Prices: { ...PricesMod },
+  Transactions: { ...TransactionsMod },
   Users: { ...UsersMod },
   Wallets: { ...WalletsMod },
   Payments: { ...PaymentsMod },
@@ -44,6 +46,7 @@ export const {
   ColdStorage,
   Lightning,
   Prices,
+  Transactions,
   Users,
   Wallets,
   Payments,

src/app/transactions/get-transactions-metadata-by-ids.ts

@@ -0,0 +1,6 @@
+import { TransactionsMetadataRepository } from "@services/ledger/services"
+
+export const getTransactionsMetadataByIds = async (
+  ids: LedgerTransactionId[],
+): Promise<(LedgerTransactionMetadata | RepositoryError)[] | ApplicationError> =>
+  TransactionsMetadataRepository().listByIds(ids)

src/app/transactions/index.ts

@@ -0,0 +1 @@
+export * from "./get-transactions-metadata-by-ids"

src/domain/errors.ts

@@ -14,6 +14,9 @@ export class DuplicateError extends RepositoryError {}
 export class BadInputsForFindError extends RepositoryError {}
 export class CouldNotFindError extends RepositoryError {}
 export class CouldNotUpdateError extends RepositoryError {}
+export class InvalidDocumentIdForDbError extends RepositoryError {
+  level = ErrorLevel.Critical
+}
 export class CannotConnectToDbError extends RepositoryError {
   level = ErrorLevel.Critical
 }

src/domain/ledger/errors.ts

@@ -6,9 +6,13 @@ export class LedgerServiceError extends LedgerError {}
 export class FeeDifferenceError extends LedgerError {}
 export class NoTransactionToSettleError extends LedgerServiceError {}
 export class InvalidPaginationArgumentsError extends LedgerServiceError {}
+export class MismatchedResultForTransactionMetadataQuery extends LedgerServiceError {}
 export class UnknownLedgerError extends LedgerServiceError {
   level = ErrorLevel.Critical
 }
 
 export class CouldNotFindTransactionError extends LedgerError {}
 export class CouldNotFindTransactionMetadataError extends CouldNotFindTransactionError {}
+export class CouldNotFindExpectedTransactionMetadataError extends CouldNotFindTransactionError {
+  level = ErrorLevel.Critical
+}

src/domain/ledger/index.types.d.ts

@@ -70,11 +70,6 @@ type LedgerTransaction<S extends WalletCurrency> = {
   readonly feeUsd: number | undefined
 }
 
-type LedgerTransactionWithMetadata<S extends WalletCurrency> = {
-  hasMetadata: true
-} & LedgerTransaction<S> &
-  LedgerTransactionMetadata
-
 type TxArgs = {
   walletId: WalletId
   walletCurrency: WalletCurrency

src/domain/wallets/index.types.d.ts

@@ -38,12 +38,7 @@ type SettlementViaIntraledger = {
 
 type SettlementViaLn = {
   readonly type: "lightning"
-  readonly revealedPreImage: undefined
-}
-
-type SettlementViaLnWithMetadata = {
-  readonly type: "lightning"
-  revealedPreImage: RevealedPreImage | undefined
+  readonly revealedPreImage: undefined // is added by dataloader in resolver
 }
 
 type SettlementViaOnChain = {
@@ -98,43 +93,19 @@ type WalletLnSettledTransaction = BaseWalletTransaction & {
   readonly settlementVia: SettlementViaLn
 }
 
-type WalletLnSettledTransactionWithMetadata = BaseWalletTransaction & {
-  readonly initiationVia: InitiationViaLn
-  readonly settlementVia: SettlementViaLnWithMetadata
-}
-
-type IntraLedgerTransactionWithMetadata = { hasMetadata: true } & IntraLedgerTransaction
-
 type WalletOnChainTransaction =
   | WalletOnChainIntraledgerTransaction
   | WalletOnChainSettledTransaction
   | WalletLegacyOnChainIntraledgerTransaction
   | WalletLegacyOnChainSettledTransaction
 
-type WalletOnChainTransactionWithMetadata = { hasMetadata: true } & (
-  | WalletOnChainIntraledgerTransaction
-  | WalletOnChainSettledTransaction
-  | WalletLegacyOnChainIntraledgerTransaction
-  | WalletLegacyOnChainSettledTransaction
-)
-
 type WalletLnTransaction = WalletLnIntraledgerTransaction | WalletLnSettledTransaction
 
-type WalletLnTransactionWithMetadata = { hasMetadata: true } & (
-  | WalletLnIntraledgerTransaction
-  | WalletLnSettledTransactionWithMetadata
-)
-
 type WalletTransaction =
   | IntraLedgerTransaction
   | WalletOnChainTransaction
   | WalletLnTransaction
 
-type WalletTransactionWithMetadata =
-  | IntraLedgerTransactionWithMetadata
-  | WalletOnChainTransactionWithMetadata
-  | WalletLnTransactionWithMetadata
-
 type AddPendingIncomingArgs = {
   pendingIncoming: IncomingOnChainTransaction[]
   addressesByWalletId: { [key: WalletId]: OnChainAddress[] }
@@ -149,21 +120,10 @@ type ConfirmedTransactionHistory = {
   addPendingIncoming(args: AddPendingIncomingArgs): WalletTransactionHistoryWithPending
 }
 
-type ConfirmedTransactionHistoryWithMetadata = {
-  readonly transactions: WalletTransactionWithMetadata[]
-  addPendingIncoming(
-    args: AddPendingIncomingArgs,
-  ): WalletTransactionHistoryWithPendingWithMetadata
-}
-
 type WalletTransactionHistoryWithPending = {
   readonly transactions: WalletTransaction[]
 }
 
-type WalletTransactionHistoryWithPendingWithMetadata = {
-  readonly transactions: WalletTransactionWithMetadata[]
-}
-
 type NewWalletInfo = {
   readonly accountId: AccountId
   readonly type: WalletType

src/domain/wallets/tx-history.ts

@@ -217,7 +217,7 @@ const translateLedgerTxnToWalletTxn = <S extends WalletCurrency>(
         },
         settlementVia: {
           type: SettlementMethod.Lightning,
-          revealedPreImage: undefined,
+          revealedPreImage: undefined, // is added by dataloader in resolver
         },
       }
       break
@@ -241,32 +241,6 @@ const translateLedgerTxnToWalletTxn = <S extends WalletCurrency>(
   return walletTransaction
 }
 
-const translateLedgerTxnToWalletTxnWithMetadata = <S extends WalletCurrency>(
-  txn: LedgerTransactionWithMetadata<S>,
-): WalletTransactionWithMetadata => {
-  const walletTxn = translateLedgerTxnToWalletTxn(txn)
-
-  let walletTxnWithMetadata: WalletTransactionWithMetadata = {
-    hasMetadata: true,
-    ...walletTxn,
-  }
-  if ("revealedPreImage" in txn) {
-    if (walletTxnWithMetadata.settlementVia.type !== SettlementMethod.Lightning) {
-      // TODO: return invalid-state error here and remove cast to 'WalletLnTransactionWithMetadata' just below
-    }
-
-    walletTxnWithMetadata = {
-      ...walletTxnWithMetadata,
-      settlementVia: {
-        ...walletTxnWithMetadata.settlementVia,
-        revealedPreImage: txn.revealedPreImage,
-      },
-    } as WalletLnTransactionWithMetadata
-  }
-
-  return walletTxnWithMetadata
-}
-
 const fromLedger = (
   ledgerTransactions: LedgerTransaction<WalletCurrency>[],
 ): ConfirmedTransactionHistory => {
@@ -280,30 +254,6 @@ const fromLedger = (
   }
 }
 
-const fromLedgerWithMetadata = <S extends WalletCurrency>(
-  ledgerTransactions: LedgerTransactionWithMetadata<S>[],
-): ConfirmedTransactionHistoryWithMetadata => {
-  const transactions = ledgerTransactions.map(translateLedgerTxnToWalletTxnWithMetadata)
-
-  const addPendingIncoming = (args: AddPendingIncomingArgs) => {
-    const pendingTxnsWithMetadata = filterPendingIncoming(args).map(
-      (txn: WalletTransaction): WalletTransactionWithMetadata => ({
-        ...txn,
-        hasMetadata: true,
-      }),
-    )
-
-    return {
-      transactions: [...pendingTxnsWithMetadata, ...transactions],
-    }
-  }
-
-  return {
-    transactions,
-    addPendingIncoming,
-  }
-}
-
 const shouldDisplayMemo = ({
   memo,
   credit,
@@ -344,7 +294,6 @@ export const translateMemo = ({
 
 export const WalletTransactionHistory = {
   fromLedger,
-  fromLedgerWithMetadata,
 } as const
 
 // TODO: refactor this to use PriceRatio eventually instead after

src/graphql/error-map.ts

@@ -410,6 +410,9 @@ export const mapError = (error: ApplicationError): CustomApolloError => {
     case "IpFetcherServiceError":
     case "CouldNotFindTransactionError":
     case "CouldNotFindTransactionMetadataError":
+    case "CouldNotFindExpectedTransactionMetadataError":
+    case "InvalidDocumentIdForDbError":
+    case "MismatchedResultForTransactionMetadataQuery":
     case "InvalidLedgerTransactionId":
     case "CacheError":
     case "CacheNotAvailableError":

src/graphql/types/abstract/settlement-via.ts

@@ -31,13 +31,13 @@ const SettlementViaLn = GT.Object({
   fields: () => ({
     paymentSecret: {
       type: LnPaymentSecret,
-      resolve: (source: SettlementViaLn) => source.revealedPreImage,
+      resolve: (source) => source.revealedPreImage,
       deprecationReason:
         "Shifting property to 'preImage' to improve granularity of the LnPaymentSecret type",
     },
     preImage: {
       type: LnPaymentPreImage,
-      resolve: (source: SettlementViaLn) => source.revealedPreImage,
+      resolve: (source) => source.revealedPreImage,
     },
   }),
 })

src/graphql/types/object/transaction.ts

@@ -5,6 +5,10 @@ import { connectionDefinitions } from "@graphql/connections"
 
 import { SAT_PRICE_PRECISION_OFFSET } from "@config"
 
+import { TxStatus as DomainTxStatus } from "@domain/wallets"
+
+import { addAttributesToCurrentSpan } from "@services/tracing"
+
 import Memo from "../scalar/memo"
 
 import InitiationVia from "../abstract/initiation-via"
@@ -35,6 +39,41 @@ const Transaction = GT.Object<WalletTransaction>({
     settlementVia: {
       type: GT.NonNull(SettlementVia),
       description: "To which protocol the payment has settled on.",
+      resolve: async (source, _, { loaders }) => {
+        const { settlementVia } = source
+
+        // Filter out source.id as OnChainTxHash
+        if (
+          settlementVia.type === "onchain" &&
+          source.id === settlementVia.transactionHash &&
+          // If not pending, we would like this to error in the next step with invalid source.id
+          source.status === DomainTxStatus.Pending
+        ) {
+          return settlementVia
+        }
+
+        const result = await loaders.txnMetadata.load(source.id)
+        if (result instanceof Error || result === undefined) return settlementVia
+
+        // TODO: remove after debugging why we aren't getting back expected pre-images
+        if (settlementVia.type === "lightning") {
+          addAttributesToCurrentSpan({
+            txnMetadata: JSON.stringify(result),
+            txnDirection: source.settlementAmount <= 0 ? "SEND" : "RECEIVE",
+          })
+        }
+
+        const updatedSettlementVia = { ...settlementVia }
+        for (const key of Object.keys(settlementVia)) {
+          /* eslint @typescript-eslint/ban-ts-comment: "off" */
+          // @ts-ignore-next-line no-implicit-any
+          updatedSettlementVia[key] =
+            // @ts-ignore-next-line no-implicit-any
+            result[key] !== undefined ? result[key] : settlementVia[key]
+        }
+
+        return updatedSettlementVia
+      },
     },
     settlementAmount: {
       type: GT.NonNull(SignedAmount),

src/servers/graphql-server.ts

@@ -1,6 +1,8 @@
 import { createServer } from "http"
 
-import { Accounts } from "@app"
+import DataLoader from "dataloader"
+
+import { Accounts, Transactions } from "@app"
 import { getApolloConfig, getGeetestConfig, getJwksArgs, isDev } from "@config"
 import Geetest from "@services/geetest"
 import { baseLogger } from "@services/logger"
@@ -9,6 +11,7 @@ import {
   SemanticAttributes,
   addAttributesToCurrentSpan,
   addAttributesToCurrentSpanAndPropagate,
+  recordExceptionInCurrentSpan,
 } from "@services/tracing"
 import {
   ApolloServerPluginDrainHttpServer,
@@ -169,8 +172,27 @@ const sessionContext = ({
         addAttributesToCurrentSpan({ [ACCOUNT_USERNAME]: domainAccount?.username })
       }
 
+      const loaders = {
+        txnMetadata: new DataLoader(async (keys) => {
+          const txnMetadata = await Transactions.getTransactionsMetadataByIds(
+            keys as LedgerTransactionId[],
+          )
+          if (txnMetadata instanceof Error) {
+            recordExceptionInCurrentSpan({
+              error: txnMetadata,
+              level: txnMetadata.level,
+            })
+
+            return keys.map(() => undefined)
+          }
+
+          return txnMetadata
+        }),
+      }
+
       return {
         logger,
+        loaders,
         // FIXME: we should not return this for the admin graphql endpoint
         user,
         domainAccount,

src/servers/index.files.d.ts

@@ -1,5 +1,14 @@
+declare class DataLoader<K, V> {
+  load(key: K): Promise<V>
+}
+
+interface Loaders {
+  txnMetadata: DataLoader<string, LedgerTransactionMetadata | undefined | RepositoryError>
+}
+
 type GraphQLContext = {
   logger: Logger
+  loaders: Loaders
   user: User | undefined
   domainAccount: Account | undefined
   geetest: GeetestType
@@ -8,6 +17,7 @@ type GraphQLContext = {
 
 type GraphQLContextAuth = {
   logger: Logger
+  loaders: Loaders
   user: User
   domainAccount: Account
   geetest: GeetestType

src/services/ledger/domain/index.types.d.ts

@@ -153,4 +153,8 @@ interface ITransactionsMetadataRepository {
   findByHash(
     hash: PaymentHash | OnChainTxHash | SwapHash,
   ): Promise<LedgerTransactionMetadata | RepositoryError>
+
+  listByIds(
+    ids: LedgerTransactionId[],
+  ): Promise<(LedgerTransactionMetadata | RepositoryError)[] | RepositoryError>
 }