From 8a4e11bb67e0433ac7c8fe80028c4d52ca3ca38e Mon Sep 17 00:00:00 2001 From: Andrew <39019063+manhinhang@users.noreply.github.com> Date: Tue, 16 Apr 2024 02:40:45 +0800 Subject: [PATCH] feat: remove ib-insync (#77) --- .envrc | 2 +- .github/workflows/build-test.yml | 8 +-- Dockerfile | 30 +++------ README.md | 29 ++------- cmd.sh | 11 +++- doc/Debugging.md | 4 +- docker-compose.yaml | 7 +- .../google_cloud_secret_manager/README.md | 35 ---------- examples/ib_insync/README.md | 6 +- src/bootstrap.py | 64 ------------------- src/ib_account.py | 44 ------------- test/test_docker_interactive.py | 4 +- test/test_ib_gateway.py | 27 +------- test/test_ib_gateway_fail.py | 35 ++++++---- 14 files changed, 62 insertions(+), 244 deletions(-) delete mode 100644 examples/google_cloud_secret_manager/README.md delete mode 100644 src/bootstrap.py delete mode 100644 src/ib_account.py diff --git a/.envrc b/.envrc index 8bccf1f..b88be13 100644 --- a/.envrc +++ b/.envrc @@ -1,5 +1,5 @@ export IMAGE_NAME=ib-gateway-docker -export TRADE_MODE=paper +export TRADING_MODE=paper if [ -f ".secrets" ] ; then source ./.secrets fi diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 56bdabe..f03ccc1 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -29,23 +29,23 @@ jobs: env: IB_ACCOUNT: ${{ secrets.IB_ACCOUNT }} IB_PASSWORD: ${{ secrets.IB_PASSWORD }} - TRADE_MODE: paper + TRADING_MODE: paper - name: Run ib_insync example run: | docker run --rm \ -e IB_ACCOUNT=$IB_ACCOUNT \ -e IB_PASSWORD=$IB_PASSWORD \ - -e TRADE_MODE=paper \ + -e TRADING_MODE=paper \ -p 4001:4002 \ -d \ - $IMAGE_NAME tail -f /dev/null; + $IMAGE_NAME; sleep 30; pip install ib_insync pandas; python examples/ib_insync/scripts/connect_gateway.py; docker stop $(docker ps -a -q) env: - TRADE_MODE: paper + TRADING_MODE: paper IB_ACCOUNT: ${{ secrets.IB_ACCOUNT }} IB_PASSWORD: ${{ secrets.IB_PASSWORD }} diff --git a/Dockerfile b/Dockerfile index 1567adb..7791b7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,20 +20,18 @@ RUN apt-get update \ procps \ xterm RUN apt install -y openjdk-17-jre -RUN pip install ib_insync==$IB_INSYNC_VER # set environment variables ENV TWS_INSTALL_LOG=/root/Jts/tws_install.log \ - ibcIni=/root/ibc/config.ini \ - ibcPath=/opt/ibc \ + IBC_INI=/root/ibc/config.ini \ + IBC_PATH=/opt/ibc \ javaPath=/opt/i4j_jres \ - twsPath=/root/Jts \ + TWS_PATH=/root/Jts \ twsSettingsPath=/root/Jts \ - IB_GATEWAY_PING_CLIENT_ID=1 \ - ibAccMaxRetryCount=30 + TWOFA_TIMEOUT_ACTION=restart # make dirs -RUN mkdir -p /tmp && mkdir -p ${ibcPath} && mkdir -p ${twsPath} +RUN mkdir -p /tmp && mkdir -p ${IBC_PATH} && mkdir -p ${TWS_PATH} # download IB TWS RUN wget -q -O /tmp/ibgw.sh https://download2.interactivebrokers.com/installers/ibgateway/stable-standalone/ibgateway-stable-standalone-linux-x64.sh @@ -41,8 +39,8 @@ RUN chmod +x /tmp/ibgw.sh # download IBC RUN wget -q -O /tmp/IBC.zip https://github.com/IbcAlpha/IBC/releases/download/$IBC_VER-Update.1/IBCLinux-$IBC_VER.zip -RUN unzip /tmp/IBC.zip -d ${ibcPath} -RUN chmod +x ${ibcPath}/*.sh ${ibcPath}/*/*.sh +RUN unzip /tmp/IBC.zip -d ${IBC_PATH} +RUN chmod +x ${IBC_PATH}/*.sh ${IBC_PATH}/*/*.sh # install TWS, write output to file so that we can parse the TWS version number later RUN touch $TWS_INSTALL_LOG @@ -54,29 +52,17 @@ RUN /tmp/install_ibgw.exp RUN rm /tmp/ibgw.sh /tmp/IBC.zip # copy IBC/Jts configs -COPY ibc/config.ini ${ibcIni} +COPY ibc/config.ini ${IBC_INI} # copy cmd script WORKDIR /root COPY cmd.sh /root/cmd.sh RUN chmod +x /root/cmd.sh -# python script for /root directory -COPY src/bootstrap.py /root/bootstrap.py -RUN chmod +x /root/bootstrap.py -COPY src/ib_account.py /root/ib_account.py -RUN chmod +x /root/ib_account.py - # set display environment variable (must be set after TWS installation) ENV DISPLAY=:0 -ENV GCP_SECRET=False ENV IBGW_PORT 4002 -ENV IBGW_WATCHDOG_CONNECT_TIMEOUT 30 -ENV IBGW_WATCHDOG_APP_STARTUP_TIME 30 -ENV IBGW_WATCHDOG_APP_TIMEOUT 30 -ENV IBGW_WATCHDOG_RETRY_DELAY 2 -ENV IBGW_WATCHDOG_PROBE_TIMEOUT 4 EXPOSE $IBGW_PORT diff --git a/README.md b/README.md index 5810ff1..19a598b 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,10 @@ It's just pure `IB Gateway` and don't include any VNC service (for security reas This docker image just installed: -- [IB Gateway](https://www.interactivebrokers.com/en/index.php?f=16457) (10.19.2) +- [IB Gateway](https://www.interactivebrokers.com/en/index.php?f=16457) (10.19.2l) - [IBC](https://github.com/IbcAlpha/IBC) (3.18.0) -- [ib_insync](https://github.com/erdewit/ib_insync) (0.9.86) - -- [google-cloud-secret-manager](https://github.com/googleapis/python-secret-manager) (2.11.1) - ## Pull the Docker image from Docker Hub ```bash @@ -29,9 +25,9 @@ docker pull manhinhang/ib-gateway-docker docker run -d \ --env IB_ACCOUNT= \ #YOUR_USER_ID --env IB_PASSWORD= \ #YOUR_PASSWORD ---env TRADE_MODE= \ #paper or live +--env TRADING_MODE= \ #paper or live --p 4002:4002 \ #brige IB gateway port to your local port 4002 -manhinhang/ib-gateway-docker tail -f /dev/null +manhinhang/ib-gateway-docker ``` --- @@ -45,10 +41,9 @@ docker build --no-cache -t ib-gateway-docker . docker run -d \ --env IB_ACCOUNT= \ #YOUR_USER_ID --env IB_PASSWORD= \ #YOUR_PASSWORD ---env TRADE_MODE= \ #paper or live +--env TRADING_MODE= \ #paper or live -p 4002:4002 \ #brige IB gateway port to your local port 4002 -ib-gateway-docker \ -tail -f /dev/null +ib-gateway-docker ``` @@ -57,8 +52,6 @@ tail -f /dev/null | Example | Link | Description | | - | - | - | | ib_insync | [examples/ib_insync](./examples/ib_insync) | This example demonstrated how to connect `IB Gateway` -| google cloud secret manager | [examples/google_cloud_secret_manager](./examples/google_cloud_secret_manager) | retreive your interactive brokers account from google cloud secret manager | - # Tests @@ -79,18 +72,6 @@ After forking `IB Gateway docker` repository, you need config your **interactive | IB_ACCOUNT | your paper account name | | IB_PASSWORD | your paper account password | -# Other environment variable - -| Variable Name | Description | Default value | -| - | - | - | -| IB_GATEWAY_PING_CLIENT_ID | ib gateway client id for pinging client status | 1 | -| IBGW_WATCHDOG_CONNECT_TIMEOUT | Ref to [ib_insync.ibcontroller.Watchdog.connectTimeout](https://ib-insync.readthedocs.io/api.html#ib_insync.ibcontroller.Watchdog.connectTimeout) | 30 | -| IBGW_WATCHDOG_APP_STARTUP_TIME | [ib_insync.ibcontroller.Watchdog.appStartupTime](https://ib-insync.readthedocs.io/api.html#ib_insync.ibcontroller.Watchdog.appStartupTime) | 30 | -| IBGW_WATCHDOG_APP_TIMEOUT | Ref to [ib_insync.ibcontroller.Watchdog.appTimeout](https://ib-insync.readthedocs.io/api.html#ib_insync.ibcontroller.Watchdog.appTimeout) | 30 | -| IBGW_WATCHDOG_RETRY_DELAY | Ref to [ib_insync.ibcontroller.Watchdog.retryDelay](https://ib-insync.readthedocs.io/api.html#ib_insync.ibcontroller.Watchdog.retryDelay) | 2 | -| IBGW_WATCHDOG_PROBE_TIMEOUT | Ref to [ib_insync.ibcontroller.Watchdog.probeTimeout](https://ib-insync.readthedocs.io/api.html#ib_insync.ibcontroller.Watchdog.probeTimeout) | 4 | - - # Disclaimer This project is not affiliated with [Interactive Brokers Group, Inc.'s](https://www.interactivebrokers.com). diff --git a/cmd.sh b/cmd.sh index 926b414..cecab01 100644 --- a/cmd.sh +++ b/cmd.sh @@ -17,9 +17,9 @@ echo "Setup port forwarding..." socat TCP-LISTEN:$IBGW_PORT,fork TCP:localhost:4001,forever & echo "*****************************" -python /root/bootstrap.py +# python /root/bootstrap.py -echo "IB gateway is ready." +# echo "IB gateway is ready." #Define cleanup procedure cleanup() { @@ -31,5 +31,10 @@ cleanup() { #Trap TERM trap 'cleanup' INT TERM +echo "IB gateway starting..." -$@ +${IBC_PATH}/scripts/ibcstart.sh "1019" -g \ + "--tws-path=${TWS_PATH}" \ + "--ibc-path=${IBC_PATH}" "--ibc-ini=${IBC_INI}" \ + "--user=${IB_ACCOUNT}" "--pw=${IB_PASSWORD}" "--mode=${TRADING_MODE}" \ + "--on2fatimeout=${TWOFA_TIMEOUT_ACTION}" diff --git a/doc/Debugging.md b/doc/Debugging.md index b3b156e..89bf92a 100644 --- a/doc/Debugging.md +++ b/doc/Debugging.md @@ -22,9 +22,9 @@ For debugging, Use x11 forwarding to visit IB gateway GUI for the investigation. docker run --platform linux/amd64 -d \ --env IB_ACCOUNT= \ --env IB_PASSWORD= \ - --env TRADE_MODE= \ + --env TRADING_MODE= \ -v ~/.Xauthority:/root/.Xauthority \ -e DISPLAY=$ip:0 \ -p 4002:4002 \ - ib-gateway-docker tail -f /dev/null + ib-gateway-docker ``` diff --git a/docker-compose.yaml b/docker-compose.yaml index 43e62c8..4ac1a70 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,17 +1,16 @@ services: ib-gateway: image: manhinhang/ib-gateway-docker - command: tail -f /dev/null ports: - 4002:4002 environment: - IB_ACCOUNT=$IB_ACCOUNT - IB_PASSWORD=$IB_PASSWORD - - TRADE_MODE=$TRADE_MODE + - TRADING_MODE=$TRADING_MODE secrets: IB_ACCOUNT: environment: "IB_ACCOUNT" IB_PASSWORD: environment: "IB_PWD" - TRADE_MODE: - environment: "TRADE_MODE" + TRADING_MODE: + environment: "TRADING_MODE" diff --git a/examples/google_cloud_secret_manager/README.md b/examples/google_cloud_secret_manager/README.md deleted file mode 100644 index d6220dd..0000000 --- a/examples/google_cloud_secret_manager/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Example for google cloud secret manager - -If you considing choose Google cloud as your IB gateway host, Google recommanded store your sensitive key in secret manager. - -Reference: https://cloud.google.com/functions/docs/env-var#managing_secrets - -> Environment variables can be used for function configuration, but are not recommended as a way to store secrets such as database credentials or API keys. These more sensitive values should be stored outside both your source code and outside environment variables. Some execution environments or the use of some frameworks can result in the contents of environment variables being sent to logs, and storing sensitive credentials in YAML files, deployment scripts or under source control is not recommended. -> -> For storing secrets, we recommend that you review the best practices for secret management. Note that there is no Cloud Functions-specific integration with Cloud KMS. - ---- - -This example just shown you how run docker & retreive secret locally. - -> *The deploy guide for google cloud may provide later.* - -1. Setup your credentials path - - ```bash - export GOOGLE_APPLICATION_CREDENTIALS= #your credentials json path - ``` - -2. Run docker run command - - ```bash - docker run --rm -it \ - --env GCP_SECRET=True \ # Enable Google secret manager feature - --env GCP_SECRET_IB_ACCOUNT= \ # secret key name of your interactive brokers account name - --env GCP_SECRET_IB_PASSWORD= \ # secret key name of your interactive brokers password - --env GCP_SECRET_IB_TRADE_MODE= \ # secret key name of trade mode - --env GCP_PROJECT_ID= \ #your project id - -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/keys/FILE_NAME.json \ - -v $GOOGLE_APPLICATION_CREDENTIALS:/tmp/keys/FILE_NAME.json:ro \ - manhinhang/ib-gateway-docker - ``` \ No newline at end of file diff --git a/examples/ib_insync/README.md b/examples/ib_insync/README.md index 41a290b..c04417d 100644 --- a/examples/ib_insync/README.md +++ b/examples/ib_insync/README.md @@ -10,17 +10,17 @@ Python script ## Docker run command ```bash -export TRADE_MODE=#paper or live +export TRADING_MODE=#paper or live export IB_ACCOUNT=# your interactive brokers account name export IB_PASSWORD=# your interactive brokers account password docker run --rm \ -e IB_ACCOUNT=$IB_ACCOUNT \ -e IB_PASSWORD=$IB_PASSWORD \ --e TRADE_MODE=$TRADE_MODE \ +-e TRADING_MODE=$TRADING_MODE \ -p 4001:4002 \ -d \ -manhinhang/ib-gateway-docker:latest tail -f /dev/null +manhinhang/ib-gateway-docker:latest pip install ib_insync pandas python ib_insync/scripts/connect_gateway.py diff --git a/src/bootstrap.py b/src/bootstrap.py deleted file mode 100644 index a03ce2b..0000000 --- a/src/bootstrap.py +++ /dev/null @@ -1,64 +0,0 @@ -from ib_insync import IBC, IB, Watchdog -import os -import logging -from ib_account import IBAccount -import sys - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="[%(asctime)s]%(levelname)s:%(message)s") - logging.info('start ib gateway...') - logging.info('---ib gateway info---') - twsPath = os.environ['twsPath'] - logging.info(f'twsPath: {twsPath}') - gatewayRootPath = "{}/ibgateway".format(twsPath) - ib_gateway_version = int(os.listdir(gatewayRootPath)[0]) - gatewayPath = "{}/{}".format(gatewayRootPath, ib_gateway_version) - logging.info("ib gateway version:{}".format(ib_gateway_version)) - logging.info("ib gateway path:{}".format(gatewayPath)) - logging.info('-------------------') - account = IBAccount.account() - password = IBAccount.password() - trade_mode = IBAccount.trade_mode() - ibc = IBC(ib_gateway_version, - gateway=True, - tradingMode=trade_mode, - userid=account, - password=password, - twsPath=twsPath) - ib = IB() - def onConnected(): - logging.info('IB gateway connected') - logging.info(ib.accountValues()) - - def onDisconnected(): - logging.info('IB gateway disconnected') - ib.connectedEvent += onConnected - ib.disconnectedEvent += onDisconnected - watchdog = Watchdog(ibc, ib, port=4001, - connectTimeout=int(os.environ['IBGW_WATCHDOG_CONNECT_TIMEOUT']), - appStartupTime=int(os.environ['IBGW_WATCHDOG_APP_STARTUP_TIME']), - appTimeout=int(os.environ['IBGW_WATCHDOG_APP_TIMEOUT']), - retryDelay=int(os.environ['IBGW_WATCHDOG_RETRY_DELAY']), - probeTimeout=int(os.environ['IBGW_WATCHDOG_PROBE_TIMEOUT'])) - def onWatchDogStarting(_): - logging.info('WatchDog Starting...') - def onWatchDogStarted(_): - logging.info('WatchDog Started!') - def onWatchDogStopping(_): - logging.info('WatchDog Stopping...') - def onWatchDogStopped(_): - logging.info('WatchDog Stopped!') - def onWatchDogSoftTimeout(_): - logging.info('WatchDog soft timeout!') - def onWatchDogHardTimeoutEvent(_): - logging.info('WatchDog hard timeout!') - watchdog.startingEvent += onWatchDogStarting - watchdog.startedEvent += onWatchDogStarted - watchdog.stoppingEvent += onWatchDogStopping - watchdog.stoppedEvent += onWatchDogStopped - watchdog.softTimeoutEvent += onWatchDogSoftTimeout - watchdog.hardTimeoutEvent += onWatchDogHardTimeoutEvent - watchdog.start() - ib.run() - logging.info('IB gateway is ready.') - diff --git a/src/ib_account.py b/src/ib_account.py deleted file mode 100644 index c77baeb..0000000 --- a/src/ib_account.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -# from distutils.util import strtobool -# from google.cloud import secretmanager - -class IBAccount(object): - # Create the Secret Manager client. - __client = None - - # @classmethod - # def retrieve_secret(cls, secret_id): - # if not cls.__client: - # cls.__client = secretmanager.SecretManagerServiceClient() - # gcp_project_id = os.environ['GCP_PROJECT_ID'] - # name = cls.__client.secret_version_path(gcp_project_id, secret_id, 'latest') - # response = cls.__client.access_secret_version(name=name) - # payload = response.payload.data.decode('UTF-8') - # return payload - - # @staticmethod - # def isEnabledGCPSecret(): - # try: - # return bool(strtobool(os.environ['GCP_SECRET'])) - # except ValueError: - # return False - - @classmethod - def account(cls): - # if not cls.isEnabledGCPSecret(): - return os.environ['IB_ACCOUNT'] - # return cls.retrieve_secret(os.environ['GCP_SECRET_IB_ACCOUNT']) - - @classmethod - def password(cls): - # if not cls.isEnabledGCPSecret(): - return os.environ['IB_PASSWORD'] - # return cls.retrieve_secret(os.environ['GCP_SECRET_IB_PASSWORD']) - - @classmethod - def trade_mode(cls): - # if not cls.isEnabledGCPSecret(): - return os.environ['TRADE_MODE'] - # return cls.retrieve_secret(os.environ['GCP_SECRET_IB_TRADE_MODE']) - - \ No newline at end of file diff --git a/test/test_docker_interactive.py b/test/test_docker_interactive.py index f604459..a0df7a7 100644 --- a/test/test_docker_interactive.py +++ b/test/test_docker_interactive.py @@ -11,14 +11,14 @@ def ib_docker(): account = os.environ['IB_ACCOUNT'] password = os.environ['IB_PASSWORD'] - trade_mode = os.environ['TRADE_MODE'] + trading_mode = os.environ['TRADING_MODE'] # run a container docker_id = subprocess.check_output( ['docker', 'run', '--env', 'IB_ACCOUNT={}'.format(account), '--env', 'IB_PASSWORD={}'.format(password), - '--env', 'TRADE_MODE={}'.format(trade_mode), + '--env', 'TRADING_MODE={}'.format(trading_mode), '-p', '4002:4002', '-d', IMAGE_NAME, "tail", "-f", "/dev/null"]).decode().strip() diff --git a/test/test_ib_gateway.py b/test/test_ib_gateway.py index 22f7a97..05e1a99 100644 --- a/test/test_ib_gateway.py +++ b/test/test_ib_gateway.py @@ -11,14 +11,14 @@ def host(request): account = os.environ['IB_ACCOUNT'] password = os.environ['IB_PASSWORD'] - trade_mode = os.environ['TRADE_MODE'] + trading_mode = os.environ['TRADING_MODE'] # run a container docker_id = subprocess.check_output( ['docker', 'run', '--env', 'IB_ACCOUNT={}'.format(account), '--env', 'IB_PASSWORD={}'.format(password), - '--env', 'TRADE_MODE={}'.format(trade_mode), + '--env', 'TRADING_MODE={}'.format(trading_mode), '-d', IMAGE_NAME, "tail", "-f", "/dev/null"]).decode().strip() # return a testinfra connection to the container @@ -26,26 +26,3 @@ def host(request): # at the end of the test suite, destroy the container subprocess.check_call(['docker', 'rm', '-f', docker_id]) -def test_ibgateway_version(host): - int(host.run("ls /root/Jts/ibgateway").stdout) - -def test_ib_connect(host): - script = """ -from ib_insync import * -from concurrent.futures import TimeoutError - -ib = IB() -wait = 60 -while not ib.isConnected(): - try: - IB.sleep(1) - ib.connect('localhost', 4002, clientId=999) - except (ConnectionRefusedError, OSError, TimeoutError): - pass - wait -= 1 - if wait <= 0: - break -ib.disconnect() -""" - cmd = host.run("python -c \"{}\"".format(script)) - assert cmd.rc == 0 diff --git a/test/test_ib_gateway_fail.py b/test/test_ib_gateway_fail.py index 6c89536..506a818 100644 --- a/test/test_ib_gateway_fail.py +++ b/test/test_ib_gateway_fail.py @@ -3,6 +3,7 @@ import testinfra import os import time +from ib_insync import IB, util, Forex IMAGE_NAME = os.environ['IMAGE_NAME'] @@ -12,14 +13,14 @@ def host(request): account = 'test' password = 'test' - trade_mode = 'paper' + trading_mode = 'paper' # run a container docker_id = subprocess.check_output( ['docker', 'run', '--env', 'IB_ACCOUNT={}'.format(account), '--env', 'IB_PASSWORD={}'.format(password), - '--env', 'TRADE_MODE={}'.format(trade_mode), + '--env', 'TRADING_MODE={}'.format(trading_mode), '-d', IMAGE_NAME, "tail", "-f", "/dev/null"]).decode().strip() # return a testinfra connection to the container @@ -28,12 +29,24 @@ def host(request): subprocess.check_call(['docker', 'rm', '-f', docker_id]) def test_ib_connect_fail(host): - script = """ -from ib_insync import * -IB.sleep(60) -ib = IB() -ib.connect('localhost', 4001, clientId=1) -ib.disconnect() -""" - cmd = host.run("python -c \"{}\"".format(script)) - assert cmd.rc != 0 + try: + ib = IB() + wait = 60 + while not ib.isConnected(): + try: + IB.sleep(1) + ib.connect('localhost', 4002, clientId=999) + except: + pass + wait -= 1 + if wait <= 0: + break + + contract = Forex('EURUSD') + bars = ib.reqHistoricalData( + contract, endDateTime='', durationStr='30 D', + barSizeSetting='1 hour', whatToShow='MIDPOINT', useRTH=True) + assert False + except: + pass +