mainnet - incl esbuild (#329)

* Bumping sdk and jit dependencies to 2.104.0-beta.31 and 0.12.18

* Bumping sdk and jit dependencies to 2.104.0-beta.32 and 0.12.19

* Nour/pyth lazer cranker (#321)

* add pyth lazer cranker

* rm unncessary libraries

* remove unnecessary code

* add entrypoint for the cranker

* added improvements

* increase the chunk size

* Bumping sdk and jit dependencies to 2.104.0-beta.33 and 0.12.20

* Bumping sdk and jit dependencies to 2.104.0-beta.34 and 0.12.21

* liquidator: use SOL routes when swapping LSTs (#322)

* Bumping sdk and jit dependencies to 2.104.0-beta.35 and 0.12.22

* fillers: fix multimaker retry logic (#325)

* fillers: fix multimaker retry logic

* add missing tip ix

* extend auction duration for filler examples

* Bumping sdk and jit dependencies to 2.104.0-beta.36 and 0.12.23

* Bumping sdk and jit dependencies to 2.104.0-beta.37 and 0.12.24

* dont throw on timeout

* ignore settled markets (#326)

* Nour/swift filler (#300)

* swift subscriber process

* integrate dlob builder

* everything working but txs too large

* protected maker

* fix fill ix bug

* prettify

* working with filling swift orders

* final ixs working

* revert taker example

* fix weird merge conflict error

* uncomment error code to suppress

* makerBidAskTwapCrank: skip simulation if updating switchboard oracle (#328)

* Bumping sdk and jit dependencies to 2.104.0-beta.38 and 0.12.25

* Bumping sdk and jit dependencies to 2.104.0-beta.39 and 0.12.26

* Bumping sdk and jit dependencies to 2.105.0-beta.0 and 0.12.27

* Chore/esbuild (#316)

* increase swift maker heartbeat window

* esbuild it

* fix

* optimize

---------

Co-authored-by: wphan <[email protected]>

---------

Co-authored-by: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: moosecat <[email protected]>
Co-authored-by: jordy25519 <[email protected]>

.dockerignore

@@ -0,0 +1,5 @@
+*/**/node_modules
+*/**/lib
+*/**/dist
+.git/
+.husky/

Dockerfile

@@ -1,16 +1,21 @@
-FROM public.ecr.aws/bitnami/node:20.18.1
-RUN apt-get install git
-ENV NODE_ENV=production
-RUN npm install -g typescript
-RUN npm install -g ts-node
+FROM node:20.18.1 AS builder
+RUN npm install -g husky
+
+COPY package.json yarn.lock ./
 
 WORKDIR /app
 
 COPY . .
 RUN yarn install
-RUN yarn build
-RUN yarn install --production
+RUN node esbuild.config.js
+
+FROM node:20.18.1-alpine
+# 'bigint-buffer' native lib for performance
+RUN apk add python3 make g++ --virtual .build &&\
+    npm install -C /lib bigint-buffer &&\
+    apk del .build
+COPY --from=builder /app/lib/ ./lib/
 
 EXPOSE 9464
 
-CMD [ "yarn", "start:all" ]
+CMD ["node", "./lib/index.js"]

esbuild.config.js

@@ -0,0 +1,27 @@
+const esbuild = require('esbuild');
+const glob = require('tiny-glob');
+
+const commonConfig = {
+    bundle: true,
+    platform: 'node',
+    target: 'es2020',
+    sourcemap: false,
+    // minify: true, makes messy debug/error output
+    treeShaking: true,
+    legalComments: 'none',
+    mainFields: ['module', 'main'],
+    metafile: true,
+    format: 'cjs',
+    external: [
+        'bigint-buffer'
+    ]
+};
+
+(async () => {
+    let entryPoints = await glob("./src/*.ts", { filesOnly: true });
+    await esbuild.build({
+        ...commonConfig,
+        entryPoints,
+        outdir: 'lib',
+    });
+})().catch(() => process.exit(1));

package.json

@@ -5,8 +5,8 @@
 	"main": "lib/index.js",
 	"license": "Apache-2.0",
 	"dependencies": {
-		"@drift-labs/jit-proxy": "0.12.24",
-		"@drift-labs/sdk": "2.104.0-beta.37",
+		"@drift-labs/jit-proxy": "0.12.27",
+		"@drift-labs/sdk": "2.105.0-beta.0",
 		"@opentelemetry/api": "1.7.0",
 		"@opentelemetry/auto-instrumentations-node": "0.31.2",
 		"@opentelemetry/exporter-prometheus": "0.31.0",
@@ -44,17 +44,19 @@
 		"@typescript-eslint/eslint-plugin": "5.62.0",
 		"@typescript-eslint/parser": "4.33.0",
 		"chai": "4.3.10",
+		"esbuild": "0.24.0",
 		"eslint": "7.32.0",
 		"eslint-config-prettier": "8.10.0",
 		"eslint-plugin-prettier": "3.4.1",
 		"husky": "7.0.4",
 		"mocha": "10.2.0",
 		"prettier": "3.0.1",
+		"tiny-glob": "0.2.9",
 		"ts-node": "10.9.1"
 	},
 	"scripts": {
 		"prepare": "husky install",
-		"build": "yarn clean && tsc",
+		"build": "yarn clean && node esbuild.config.js",
 		"clean": "rm -rf lib",
 		"start": "node lib/index.js",
 		"dev": "NODE_OPTIONS=--max-old-space-size=8192 ts-node src/index.ts",

src/bots/makerBidAskTwapCrank.ts

@@ -365,7 +365,8 @@ export class MakerBidAskTwapCrank implements Bot {
 
 	private async buildTransaction(
 		marketIndex: number,
-		ixs: TransactionInstruction[]
+		ixs: TransactionInstruction[],
+		doSimulation = true
 	): Promise<VersionedTransaction | undefined> {
 		const recentBlockhash =
 			await this.driftClient.connection.getLatestBlockhash('confirmed');
@@ -377,7 +378,7 @@ export class MakerBidAskTwapCrank implements Bot {
 				lookupTableAccounts: this.lookupTableAccounts,
 				cuLimitMultiplier: CU_EST_MULTIPLIER,
 				minCuLimit: TWAP_CRANK_MIN_CU,
-				doSimulation: true,
+				doSimulation,
 				recentBlockhash: recentBlockhash.blockhash,
 			});
 		logger.info(
@@ -476,9 +477,15 @@ export class MakerBidAskTwapCrank implements Bot {
 			let txsToBundle: VersionedTransaction[] = [];
 
 			for (const mi of crankMarkets) {
+				const usingSwitchboardOnDemand = isVariant(
+					this.driftClient.getPerpMarketAccount(mi)!.amm.oracleSource,
+					'switchboardOnDemand'
+				);
 				const ixs = [
 					ComputeBudgetProgram.setComputeUnitLimit({
-						units: 1_400_000, // will be overwritten by simulateAndGetTxWithCUs
+						units: usingSwitchboardOnDemand
+							? 350_000 // switchboard simulation is unreliable, use hardcoded CU limit
+							: 1_400_000, // will be overwritten by simulateAndGetTxWithCUs
 					}),
 				];
 
@@ -547,12 +554,7 @@ export class MakerBidAskTwapCrank implements Bot {
 					const pythIxs = await this.getPythIxsFromTwapCrankInfo(mi);
 					ixs.push(...pythIxs);
 					pythIxsPushed = true;
-				} else if (
-					isVariant(
-						this.driftClient.getPerpMarketAccount(mi)!.amm.oracleSource,
-						'switchboardOnDemand'
-					)
-				) {
+				} else if (usingSwitchboardOnDemand) {
 					const switchboardIx =
 						await this.driftClient.getPostSwitchboardOnDemandUpdateAtomicIx(
 							this.driftClient.getPerpMarketAccount(mi)!.amm.oracle,
@@ -596,7 +598,11 @@ export class MakerBidAskTwapCrank implements Bot {
 					if (isFirstTxInBundle) {
 						ixs.push(this.bundleSender!.getTipIx());
 					}
-					const txToSend = await this.buildTransaction(mi, ixs);
+					const txToSend = await this.buildTransaction(
+						mi,
+						ixs,
+						!usingSwitchboardOnDemand
+					);
 					if (txToSend) {
 						// @ts-ignore;
 						txToSend.sign(jitoSigners);
@@ -605,7 +611,11 @@ export class MakerBidAskTwapCrank implements Bot {
 						logger.error(`[${this.name}] failed to build tx for market: ${mi}`);
 					}
 				} else {
-					const txToSend = await this.buildTransaction(mi, ixs);
+					const txToSend = await this.buildTransaction(
+						mi,
+						ixs,
+						!usingSwitchboardOnDemand
+					);
 					if (txToSend) {
 						await this.sendSingleTx(mi, txToSend);
 					} else {

src/config.ts

@@ -159,6 +159,8 @@ export interface GlobalConfig {
 	/// ws endpoint to use (inferred from endpoint using web3.js rules, only provide if you want to use a different one)
 	wsEndpoint?: string;
 	hermesEndpoint?: string;
+	lazerEndpoint?: string;
+	lazerToken?: string;
 	numNonActiveOraclesToPush?: number;
 
 	// Optional to specify markets loaded by drift client

src/experimental-bots/entrypoint.ts

@@ -39,6 +39,8 @@ import { setGlobalDispatcher, Agent } from 'undici';
 import { PythPriceFeedSubscriber } from '../pythPriceFeedSubscriber';
 import { SwiftMaker } from './swift/makerExample';
 import { SwiftTaker } from './swift/takerExample';
+import * as net from 'net';
+import { PythLazerClient } from '@pythnetwork/pyth-lazer-sdk';
 
 setGlobalDispatcher(
 	new Agent({
@@ -307,6 +309,19 @@ const runBot = async () => {
 			throw new Error('fillerMultithreaded bot config not found');
 		}
 
+		let pythLazerClient: PythLazerClient | undefined;
+		if (config.global.driftEnv! === 'devnet') {
+			if (!config.global.lazerEndpoint || !config.global.lazerToken) {
+				throw new Error(
+					'Must set environment variables LAZER_ENDPOINT and LAZER_TOKEN'
+				);
+			}
+			pythLazerClient = new PythLazerClient(
+				config.global.lazerEndpoint,
+				config.global.lazerToken
+			);
+		}
+
 		// Ensure that there are no duplicate market indexes in the Array<number[]> marketIndexes config
 		const marketIndexes = new Set<number>();
 		for (const marketIndexList of config.botConfigs.fillerMultithreaded
@@ -335,6 +350,7 @@ const runBot = async () => {
 			},
 			bundleSender,
 			pythPriceSubscriber,
+			pythLazerClient,
 			[]
 		);
 		bots.push(fillerMultithreaded);
@@ -423,46 +439,50 @@ const runBot = async () => {
 
 	// start http server listening to /health endpoint using http package
 	const startupTime = Date.now();
-	http
-		.createServer(async (req, res) => {
-			if (req.url === '/health') {
-				if (config.global.testLiveness) {
-					if (Date.now() > startupTime + 60 * 1000) {
-						res.writeHead(500);
-						res.end('Testing liveness test fail');
-						return;
-					}
-				}
-
-				/* @ts-ignore */
-				if (!driftClient.connection._rpcWebSocketConnected) {
-					logger.error(`Connection rpc websocket disconnected`);
+	const createServerCallback = async (req: any, res: any) => {
+		if (req.url === '/health') {
+			if (config.global.testLiveness) {
+				if (Date.now() > startupTime + 60 * 1000) {
 					res.writeHead(500);
-					res.end(`Connection rpc websocket disconnected`);
+					res.end('Testing liveness test fail');
 					return;
 				}
+			}
 
-				// check all bots if they're live
-				for (const bot of bots) {
-					const healthCheck = await promiseTimeout(bot.healthCheck(), 1000);
-					if (!healthCheck) {
-						logger.error(`Health check failed for bot`);
-						res.writeHead(503);
-						res.end(`Bot is not healthy`);
-						return;
-					}
-				}
+			/* @ts-ignore */
+			if (!driftClient.connection._rpcWebSocketConnected) {
+				logger.error(`Connection rpc websocket disconnected`);
+				res.writeHead(500);
+				res.end(`Connection rpc websocket disconnected`);
+				return;
+			}
 
-				// liveness check passed
-				res.writeHead(200);
-				res.end('OK');
-			} else {
-				res.writeHead(404);
-				res.end('Not found');
+			// check all bots if they're live
+			for (const bot of bots) {
+				const healthCheck = await promiseTimeout(bot.healthCheck(), 1000);
+				if (!healthCheck) {
+					logger.error(`Health check failed for bot`);
+					res.writeHead(503);
+					res.end(`Bot is not healthy`);
+					return;
+				}
 			}
-		})
-		.listen(healthCheckPort);
-	logger.info(`Health check server listening on port ${healthCheckPort}`);
+
+			// liveness check passed
+			res.writeHead(200);
+			res.end('OK');
+		} else {
+			res.writeHead(404);
+			res.end('Not found');
+		}
+	};
+
+	let healthCheckPortToUse = Number(healthCheckPort);
+	while (await isPortInUse(healthCheckPortToUse)) {
+		healthCheckPortToUse++;
+	}
+	http.createServer(createServerCallback).listen(healthCheckPortToUse);
+	logger.info(`Server listening on port ${healthCheckPortToUse}`);
 };
 
 recursiveTryCatch(() => runBot());
@@ -476,3 +496,26 @@ async function recursiveTryCatch(f: () => void) {
 		await recursiveTryCatch(f);
 	}
 }
+
+function isPortInUse(port: number, host = '127.0.0.1'): Promise<boolean> {
+	return new Promise((resolve) => {
+		const server = net.createServer();
+
+		server.once('error', (err) => {
+			if (
+				err.name?.includes('EADDRINUSE') ||
+				err.message?.includes('EADDRINUSE')
+			) {
+				resolve(true);
+			} else {
+				resolve(false);
+			}
+		});
+
+		server.once('listening', () => {
+			server.close(() => resolve(false));
+		});
+
+		server.listen(port, host);
+	});
+}