improve zero output order pairs handling

config.env.yaml

@@ -30,6 +30,7 @@ route: $ROUTE
 sleep: $SLEEP
 poolUpdateInterval: $POOL_UPDATE_INTERVAL
 gasCoveragePercentage: $GAS_COVER
+headroom: $HEADROOM
 txGas: $TX_GAS
 quoteGas: $QUOTE_GAS
 botMinBalance: $BOT_MIN_BALANCE

config.example.yaml

@@ -98,6 +98,10 @@ poolUpdateInterval: 10
 # Gas coverage percentage for each transaction to be considered profitable to be submitted, default is 100
 gasCoveragePercentage: 250
 
+# Percentage (number) that accumulates on top of actual profitablity index (gasCoveragePercentage) that assures trade success during
+# simulation with that higher profit so to give leeway for price fluctuation between simulation time mining time, default: %2.5
+headroom: 2.5
+
 # Option to set a gas limit for all submitting txs optionally with appended percentage sign to apply as percentage to original gas
 txGas: 110%
 

src/cli/commands/sweep.ts

@@ -120,6 +120,7 @@ export async function sweepFunds(opts: SweepOptions) {
         orderbookTradeTypes: {} as any,
         maxConcurrency: 0,
         skipSweep: new Set(),
+        headroom: 0,
     };
 
     // prepare state config fields

src/cli/index.test.ts

@@ -1,7 +1,7 @@
 import { AppOptions } from "../config";
 import { RainSolver } from "../core";
 import { OrderManager } from "../order";
-import { RainSolverCli } from "./index";
+import { DAY, RainSolverCli } from "./index";
 import { RainSolverLogger } from "../logger";
 import { WalletManager, WalletType } from "../wallet";
 import { SharedState, SharedStateConfig } from "../state";
@@ -182,6 +182,7 @@ describe("Test RainSolverCli", () => {
             sync: vi.fn(),
             downscaleProtection: vi.fn(),
             getCurrentMetadata: vi.fn().mockReturnValue({ key: "value" }),
+            metadata: { key: "value" },
         } as any;
 
         mockWalletManager = {
@@ -198,7 +199,9 @@ describe("Test RainSolverCli", () => {
             retryPendingRemoveWorkers: vi.fn(),
             convertHoldingsToGas: vi.fn(),
             getWorkerWalletsBalance: vi.fn(),
+            sweepWallet: vi.fn(),
             workers: {
+                signers: new Map([["signer", { name: "worker1" }]]),
                 lastUsedDerivationIndex: 5,
             },
         } as any;
@@ -655,11 +658,24 @@ describe("Test RainSolverCli", () => {
             );
         });
 
-        it("should call additional operations on round 250", async () => {
+        it("should call downscale operation on round 250", async () => {
             rainSolverCli.roundCount = 250;
             const mockRoundCtx = { test: "context" };
+
+            (mockWalletManager.retryPendingAddWorkers as Mock).mockResolvedValue([]);
+            (mockWalletManager.assessWorkers as Mock).mockResolvedValue([]);
+            (mockOrderManager.downscaleProtection as Mock).mockResolvedValue(undefined);
+
+            await rainSolverCli.runWalletOpsForRound(mockRoundCtx as any);
+
+            expect(mockOrderManager.downscaleProtection).toHaveBeenCalledOnce();
+        });
+
+        it("should sweep every 5 days and convert to gas evey day", async () => {
+            const mockRoundCtx = { test: "context" };
             const mockPendingRemoveReports = [{ name: "pending-remove-1" }];
             const mockConvertHoldingsReport = { name: "convert-holdings" };
+            const mockSweepReport = { name: "sweep" };
 
             (mockWalletManager.retryPendingAddWorkers as Mock).mockResolvedValue([]);
             (mockWalletManager.assessWorkers as Mock).mockResolvedValue([]);
@@ -669,13 +685,15 @@ describe("Test RainSolverCli", () => {
             (mockWalletManager.convertHoldingsToGas as Mock).mockResolvedValue(
                 mockConvertHoldingsReport,
             );
-            (mockOrderManager.downscaleProtection as Mock).mockResolvedValue(undefined);
+            (mockWalletManager.sweepWallet as Mock).mockResolvedValue(mockSweepReport);
+            (rainSolverCli as any).nextSweepTime = Date.now() - 6 * DAY;
+            (rainSolverCli as any).nextGasConversionTime = Date.now() - 2 * DAY;
 
             await rainSolverCli.runWalletOpsForRound(mockRoundCtx as any);
 
             expect(mockWalletManager.retryPendingRemoveWorkers).toHaveBeenCalledTimes(1);
             expect(mockWalletManager.convertHoldingsToGas).toHaveBeenCalledTimes(1);
-            expect(mockOrderManager.downscaleProtection).toHaveBeenCalledOnce();
+            expect(mockWalletManager.sweepWallet).toHaveBeenCalledTimes(1);
 
             expect(mockLogger.exportPreAssembledSpan).toHaveBeenCalledWith(
                 { name: "pending-remove-1" },
@@ -685,6 +703,25 @@ describe("Test RainSolverCli", () => {
                 mockConvertHoldingsReport,
                 mockRoundCtx,
             );
+            expect(mockLogger.exportPreAssembledSpan).toHaveBeenCalledWith(
+                mockSweepReport,
+                mockRoundCtx,
+            );
+
+            // next time should not call these when time has not reached
+            await rainSolverCli.runWalletOpsForRound(mockRoundCtx as any);
+            expect(mockWalletManager.retryPendingRemoveWorkers).toHaveBeenCalledTimes(1);
+            expect(mockWalletManager.convertHoldingsToGas).toHaveBeenCalledTimes(1);
+            expect(mockWalletManager.sweepWallet).toHaveBeenCalledTimes(1);
+
+            // set timers to simulate some days passed by
+            (rainSolverCli as any).nextSweepTime = Date.now() - 6 * DAY;
+            (rainSolverCli as any).nextGasConversionTime = Date.now() - 2 * DAY;
+
+            await rainSolverCli.runWalletOpsForRound(mockRoundCtx as any);
+            expect(mockWalletManager.retryPendingRemoveWorkers).toHaveBeenCalledTimes(2);
+            expect(mockWalletManager.convertHoldingsToGas).toHaveBeenCalledTimes(2);
+            expect(mockWalletManager.sweepWallet).toHaveBeenCalledTimes(2);
         });
     });
 
@@ -724,7 +761,6 @@ describe("Test RainSolverCli", () => {
             expect(mockRoundSpan.setAttribute).toHaveBeenCalledWith("txUrls.failed", [
                 "https://etherscan.io/tx/0x789",
             ]);
-            expect(mockOrderManager.getCurrentMetadata).toHaveBeenCalledTimes(1);
             expect(mockRoundSpan.setAttribute).toHaveBeenCalledWith("ordersMetadata.key", "value");
         });
 

src/cli/index.ts

@@ -63,7 +63,9 @@ export class RainSolverCli {
     private nextGasReset = Date.now() + DAY;
     private nextDatafetcherReset: number;
     /** Wallet sweeper timer (once every 5 days) */
-    private lastSweepTime = Date.now() + 5 * DAY;
+    private nextSweepTime = Date.now() + 5 * DAY;
+    /** Convert bounties to gas timer (once every day) */
+    private nextGasConversionTime = Date.now() + DAY;
 
     private constructor(
         state: SharedState,
@@ -294,7 +296,7 @@ export class RainSolverCli {
      */
     async processOrdersForRound(roundSpan: Span, roundCtx: Context) {
         // process round and export the reports
-        const { results, checkpointReports } = await this.rainSolver.processNextRound({
+        const { results, totalLength } = await this.rainSolver.processNextRound({
             span: roundSpan,
             context: roundCtx,
         });
@@ -312,15 +314,11 @@ export class RainSolverCli {
         }
         roundSpan.setAttribute("txUrls.success", successTxs);
         roundSpan.setAttribute("txUrls.failed", failedTxs);
-        roundSpan.setAttribute(
-            "ordersMetadata.roundProcessedOrderPairsCount",
-            checkpointReports.length,
-        );
-        const ordersMetadata = this.orderManager.getCurrentMetadata();
-        for (const key in ordersMetadata) {
+        roundSpan.setAttribute("ordersMetadata.roundProcessedOrderPairsCount", totalLength);
+        for (const key in this.orderManager.metadata) {
             roundSpan.setAttribute(
                 `ordersMetadata.${key}`,
-                ordersMetadata[key as keyof typeof ordersMetadata],
+                this.orderManager.metadata[key as keyof typeof this.orderManager.metadata],
             );
         }
     }
@@ -345,25 +343,31 @@ export class RainSolverCli {
             this.logger.exportPreAssembledSpan(report.addWorkerReport, roundCtx);
         });
 
-        // retry wallet removals and sweep funds once every 250 rounds
+        // rescale once every 250 rounds
         if (this.roundCount % 250 === 0) {
+            // re-evaluate owner limits
+            await this.orderManager.downscaleProtection();
+        }
+
+        const now = Date.now();
+
+        if (this.nextGasConversionTime <= now) {
+            this.nextGasConversionTime = now + DAY;
+
             // retry pending remove workers
             const pendingRemoveReports = await this.walletManager.retryPendingRemoveWorkers();
             pendingRemoveReports.forEach((report) => {
                 this.logger.exportPreAssembledSpan(report, roundCtx);
             });
 
             // try to sweep main wallet's tokens back to gas
-            const convertHoldingsToGasReport = await this.walletManager.convertHoldingsToGas();
-            this.logger.exportPreAssembledSpan(convertHoldingsToGasReport, roundCtx);
-
-            // re-evaluate owner limits
-            await this.orderManager.downscaleProtection();
+            await this.walletManager.convertHoldingsToGas().then((convertHoldingsToGasReport) => {
+                this.logger.exportPreAssembledSpan(convertHoldingsToGasReport, roundCtx);
+            });
         }
 
-        const now = Date.now();
-        if (this.lastSweepTime <= now) {
-            this.lastSweepTime = now + 5 * DAY;
+        if (this.nextSweepTime <= now) {
+            this.nextSweepTime = now + 5 * DAY;
             // sweep worker wallet bounties
             for (const [, worker] of this.walletManager.workers.signers) {
                 await this.walletManager.sweepWallet(worker, false).then((report) => {

src/config/yaml.test.ts

@@ -31,6 +31,7 @@ route: multi
 sleep: 20
 poolUpdateInterval: 30
 gasCoveragePercentage: 110
+headroom: 2.5
 txGas: 15000
 quoteGas: 2000000
 botMinBalance: 50.5
@@ -151,6 +152,7 @@ orderbookTradeTypes:
             },
             maxConcurrency: 15,
             skipSweep: new Set([`0x${"8".repeat(40)}`, `0x${"9".repeat(40)}`]),
+            headroom: 102.5,
         };
 
         // AppOptions returned from fromYaml() should match expected
@@ -278,6 +280,7 @@ orderbookTradeTypes:
         assert.deepEqual(result.timeout, 20000);
         assert.equal(result.maxRatio, true);
         assert.equal(result.maxConcurrency, 1);
+        assert.equal(result.headroom, 102.5);
 
         // ownerProfile
         const expectedOwnerProfile = {

src/config/yaml.ts

@@ -108,6 +108,8 @@ export type AppOptions = {
     maxConcurrency: number;
     /** List of tokens to skip when sweeping bounty tokens */
     skipSweep: Set<string>;
+    /** Trade simulation profitablity headroom, default: 2.5 */
+    headroom: number;
 };
 
 /** Provides methods to instantiate and validate AppOptions */
@@ -294,6 +296,13 @@ export namespace AppOptions {
                     input.skipSweep,
                     "invalid skip sweep list, expected an array of token addresses",
                 ),
+                headroom:
+                    Validator.resolveNumericValue(
+                        input.headroom,
+                        FLOAT_PATTERN,
+                        "invalid headroom value, must be a number greater than equal to 0",
+                        "2.5",
+                    ) + 100,
             } as AppOptions);
         } catch (error: any) {
             if (error instanceof AppOptionsError) {

src/core/index.ts

@@ -58,7 +58,7 @@ export class RainSolver {
      * @returns An object containing results and reports of the processed round
      */
     async processNextRound(roundSpanCtx?: SpanWithContext, shuffle = true) {
-        const { settlements, checkpointReports } = await initializeRound.call(
+        const { settlements, checkpointReports, totalLength } = await initializeRound.call(
             this,
             roundSpanCtx,
             shuffle,
@@ -68,6 +68,7 @@ export class RainSolver {
             results,
             reports,
             checkpointReports,
+            totalLength,
         };
     }
 

src/core/modes/simulator.test.ts

@@ -49,6 +49,7 @@ describe("Test TradeSimulatorBase", () => {
             appOptions: {
                 gasLimitMultiplier: 1.5,
                 gasCoveragePercentage: "100",
+                headroom: 3.5,
             },
         } as any as RainSolver;
         mockSigner = { name: "signer" } as RainSolverSigner;
@@ -257,7 +258,10 @@ describe("Test TradeSimulatorBase", () => {
                 Result.err(setTransactionDataError),
             );
             const headroom = BigInt(
-                (Number(mockSolver.appOptions.gasCoveragePercentage) * 102.5).toFixed(),
+                (
+                    Number(mockSolver.appOptions.gasCoveragePercentage) *
+                    mockSolver.appOptions.headroom
+                ).toFixed(),
             );
 
             const result = await mockSimulator.trySimulateTrade();
@@ -326,7 +330,10 @@ describe("Test TradeSimulatorBase", () => {
             };
             (dryrun as Mock).mockResolvedValueOnce(Result.err(dryrunError));
             const headroom = BigInt(
-                (Number(mockSolver.appOptions.gasCoveragePercentage) * 102.5).toFixed(),
+                (
+                    Number(mockSolver.appOptions.gasCoveragePercentage) *
+                    mockSolver.appOptions.headroom
+                ).toFixed(),
             );
 
             const result = await mockSimulator.trySimulateTrade();
@@ -410,7 +417,10 @@ describe("Test TradeSimulatorBase", () => {
                 .mockResolvedValueOnce(Result.ok(dryrunResult))
                 .mockResolvedValueOnce(Result.ok(dryrunResult2));
             const headroom = BigInt(
-                (Number(mockSolver.appOptions.gasCoveragePercentage) * 102.5).toFixed(),
+                (
+                    Number(mockSolver.appOptions.gasCoveragePercentage) *
+                    mockSolver.appOptions.headroom
+                ).toFixed(),
             );
             // last call to setTransactionData fails
             const setTransactionDataError = {
@@ -525,7 +535,10 @@ describe("Test TradeSimulatorBase", () => {
                 .mockResolvedValueOnce(Result.ok(dryrunResult))
                 .mockResolvedValueOnce(Result.ok(dryrunResult2));
             const headroom = BigInt(
-                (Number(mockSolver.appOptions.gasCoveragePercentage) * 102.5).toFixed(),
+                (
+                    Number(mockSolver.appOptions.gasCoveragePercentage) *
+                    mockSolver.appOptions.headroom
+                ).toFixed(),
             );
             const profitEstimate = 1234n;
             (mockSimulator.estimateProfit as Mock).mockReturnValueOnce(profitEstimate);

src/core/modes/simulator.ts

@@ -153,9 +153,12 @@ export abstract class TradeSimulatorBase {
         // delete gas to let signer estimate gas again with updated tx data
         delete prepareParamsResult.value.rawtx.gas;
 
-        // determine the success of the trade with 0.25% headroom
+        // examine the success of the trade with 1.5% headroom
         const headroom = BigInt(
-            (Number(this.tradeArgs.solver.appOptions.gasCoveragePercentage) * 102.5).toFixed(),
+            (
+                Number(this.tradeArgs.solver.appOptions.gasCoveragePercentage) *
+                this.tradeArgs.solver.appOptions.headroom
+            ).toFixed(),
         );
         let minimumExpected = (estimatedGasCost * headroom) / 10000n;
         this.spanAttributes["gasEst.initial.minBountyExpected"] = minimumExpected.toString();