make improvements to dashboards and logic of trading script

db.py

@@ -14,10 +14,6 @@ db = PostgresqlDatabase(
     port=getenv('DB_PORT', 5432)
 )
 
-def get_time():
-    now = datetime.now()
-    now = now + timedelta(hours=7)
-    return now
 
 class Ticker(Model):
     id = AutoField()
@@ -32,7 +28,7 @@ class Ticker(Model):
     spread_btc = DoubleField()
     spread_sats = IntegerField()
     spread_perc = DoubleField()
-    date = DateTimeField(default=get_time)
+    date = DateTimeField(default=datetime.utcnow)
 
     class Meta:
         database = db
@@ -41,7 +37,22 @@ class Balance(Model):
     total = DoubleField()
     available = DoubleField()
     currency = CharField()
-    date = DateTimeField(default=get_time)
+    date = DateTimeField(default=datetime.utcnow)
+
+    class Meta:
+        database = db
+
+class Portfolio(Model):
+    usd = DoubleField()
+    btc = DoubleField()
+    date = DateTimeField(default=datetime.utcnow)
+
+    class Meta:
+        database = db
+
+class BitcoinPrice(Model):
+    price_usd = DoubleField()
+    date = DateTimeField(default=datetime.utcnow)
 
     class Meta:
         database = db
@@ -55,7 +66,7 @@ class Order(Model):
     uuid = TextField(null=True)
     active = BooleanField(default=True)
     cancelled = BooleanField(default=False)
-    date = DateTimeField(default=get_time)
+    date = DateTimeField(default=datetime.utcnow)
 
     class Meta:
         database = db
@@ -63,9 +74,9 @@ class Order(Model):
 class Earning(Model):
     trade_pair = CharField()
     quantity = DoubleField()
-    date = DateTimeField(default=get_time)
+    date = DateTimeField(default=datetime.utcnow)
 
     class Meta:
         database = db
 
-db.create_tables([Ticker, Balance, Order, Earning])
+db.create_tables([Ticker, Balance, Order, Earning, BitcoinPrice, Portfolio])

docker-compose.yaml

@@ -20,11 +20,14 @@ services:
       GF_LOG_LEVEL: "debug"
     volumes:
       - grafana:/var/lib/grafana
+      - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
   postgres:
     image: postgres:9.6.15-alpine
     container_name: trader_postgres
     ports:
-      - 5432:5432
+      - 127.0.0.1:5432:5432
     environment:
       POSTGRES_PASSWORD: ${DB_PASS}
       POSTGRES_USER: ${DB_USER}

env-example

@@ -1,8 +1,8 @@
 GRAFANA_PASSWORD=xxxx
 GRAFANA_URL=localhost:3000
-DB_PASS=yyyyy
+DB_PASS=trader
 DB_USER=trader
-DB_NAME=trader_xmr
+DB_NAME=trader
 DB_HOST=127.0.0.1
 TO_USER=zzzzzzzzz
 TO_PASS=vvvvvvvvv

grafana/dashboards.yaml

@@ -1,13 +0,0 @@
-apiVersion: 1
-
-providers:
-- name: 'default'
-  orgId: 1
-  folder: ''
-  type: file
-  disableDeletion: true
-  editable: true
-  updateIntervalSeconds: 60
-  allowUiUpdates: true
-  options:
-    path: /var/lib/grafana/dashboards

grafana/dashboards/trader.json

@@ -21,7 +21,7 @@
   "panels": [
     {
       "content": "\n# Orders\n\nOrders placed on TradeOgre\n\n\n\n",
-      "datasource": null,
+      "datasource": "postgres",
       "gridPos": {
         "h": 3,
         "w": 24,
@@ -72,7 +72,7 @@
       "cacheTimeout": null,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "postgres",
       "decimals": 8,
       "fill": 1,
       "fillGradient": 0,
@@ -360,7 +360,7 @@
         "rgba(237, 129, 40, 0.89)",
         "#d44a3a"
       ],
-      "datasource": null,
+      "datasource": "postgres",
       "format": "none",
       "gauge": {
         "maxValue": 100,
@@ -506,7 +506,7 @@
         "rgba(237, 129, 40, 0.89)",
         "#d44a3a"
       ],
-      "datasource": null,
+      "datasource": "postgres",
       "format": "none",
       "gauge": {
         "maxValue": 100,
@@ -643,7 +643,7 @@
     },
     {
       "content": "\n# Balances\n\nBalances and trade info of your account.\n\n\n\n",
-      "datasource": null,
+      "datasource": "postgres",
       "gridPos": {
         "h": 3,
         "w": 24,
@@ -663,7 +663,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "postgres",
       "decimals": 8,
       "fill": 1,
       "fillGradient": 0,
@@ -884,7 +884,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "postgres",
       "decimals": 8,
       "fill": 1,
       "fillGradient": 0,
@@ -1104,7 +1104,7 @@
     },
     {
       "content": "\n# Markets\n\nGeneral stats and metrics of the $currency markets\n\n\n\n",
-      "datasource": null,
+      "datasource": "postgres",
       "gridPos": {
         "h": 3,
         "w": 24,
@@ -1124,7 +1124,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "postgres",
       "decimals": 8,
       "fill": 1,
       "fillGradient": 0,
@@ -1260,7 +1260,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "postgres",
       "decimals": 0,
       "fill": 1,
       "fillGradient": 0,
@@ -1394,7 +1394,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "postgres",
       "decimals": 8,
       "fill": 1,
       "fillGradient": 0,
@@ -1547,7 +1547,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "postgres",
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -1678,7 +1678,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "postgres",
       "decimals": null,
       "fill": 1,
       "fillGradient": 0,

grafana/grafana.ini

@@ -0,0 +1,29 @@
+[analytics]
+reporting_enabled = false
+check_for_updates = false
+
+[auth]
+disable_login_form = true
+
+[auth.anonymous]
+enabled = true
+org_role = Admin
+
+[dashboards]
+min_refresh_interval = 1m
+default_home_dashboard_path = /var/lib/grafana/dashboards/xmr_trading_desk.json
+
+[paths]
+provisioning = /etc/grafana/provisioning
+
+[server]
+root_url = https://127.0.0.1
+enable_gzip = true
+read_timeout = 2m
+
+[snapshots]
+external_enabled = false
+
+[security]
+admin_user = admin
+admin_password = admin

grafana/prometheus.yaml

@@ -1,9 +0,0 @@
-apiVersion: 1
-
-datasources:
-- name: PostgreSQL
-  type: postgresql
-  url: http://prometheus:9090
-  access: proxy
-  isDefault: true
-  timeInterval: 10s

grafana/provisioning/dashboards/all.yaml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: 'fs'
+    orgId: 1
+    folder: ''
+    type: 'file'
+    updateIntervalSeconds: 30
+    allowUiUpdates: true
+    options:
+      path: '/var/lib/grafana/dashboards'
+      foldersFromFilesStructure: true

grafana/provisioning/datasources/all.yaml

@@ -0,0 +1,17 @@
+apiVersion: 1
+
+datasources:
+  - name: postgres
+    type: postgres
+    url: trader_postgres:5432
+    database: trader
+    user: trader
+    secureJsonData:
+      password: "trader"
+    jsonData:
+      sslmode: "disable"
+      maxOpenConns: 0
+      maxIdleConns: 2
+      connMaxLifetime: 14400
+      postgresVersion: 906
+      timescaledb: false

trade.py

@@ -8,7 +8,7 @@ from datetime import datetime
 from decimal import Decimal
 from random import uniform
 from time import sleep
-from db import Ticker, Balance, Order, Earning
+from db import *
 from tradeogre import TradeOgre
 
 
@@ -22,16 +22,17 @@ class Trader(TradeOgre):
     load_dotenv('.env')
 
     satoshi = .00000001
+    satoshi_multiplier = getenv('SATOSHI_MULTIPLIER', 10)
     base_currency = getenv('BASE_CURRENCY', 'BTC')
-    trade_currency = getenv('TRADE_CURRENCY', 'WOW')
+    trade_currency = getenv('TRADE_CURRENCY', 'XMR')
     trade_pair = f'{base_currency}-{trade_currency}'
-    trade_amount = float(getenv('TRADE_AMOUNT', 200))
+    trade_amount = float(getenv('TRADE_AMOUNT', .5))
-    spread_target = float(getenv('SPREAD_TARGET', 4))
+    spread_target = float(getenv('SPREAD_TARGET', 10))
     amount_multiplier = float(getenv('AMOUNT_MULTIPLIER', 1.2))
     active_order_limit = float(getenv('ACTIVE_ORDER_LIMIT', 10))
 
     def get_market_data(self):
-        logging.info(f'[MARKET] Getting market data for trade pair {self.trade_pair}')
+        logging.info(f'Getting market data for trade pair {self.trade_pair}')
         res = self.get_trade_pair(self.trade_pair)
         spread_btc = Decimal(res['ask']) - Decimal(res['bid'])
         spread_sats = float(spread_btc / Decimal(self.satoshi))
@@ -58,38 +59,40 @@ class Trader(TradeOgre):
             spread_perc=res['spread_perc']
         )
         t.save()
-        logging.info(f'[MARKET] Stored market data as ID {t.id}')
+        logging.info(f'Stored market data as ID {t.id}')
         return t
 
-    def store_balance(self, currency):
+    def store_balance(self, currency, dst):
-        logging.info(f'[BALANCE] Storing balance for currency {currency}')
+        logging.info(f'Storing balance for currency {currency}')
         res = self.get_balance(currency)
         logging.debug(res)
         b = Balance(
             currency=currency,
             total=res['balance'],
-            available=res['available']
+            available=res['available'],
+            date=dst
         )
         b.save()
-        logging.info(f'[BALANCE] Stored market data as ID {b.id}')
+        logging.info(f'Stored market data as ID {b.id}')
         return b
 
     def store_balances(self):
+        now = datetime.utcnow()
         for cur in self.base_currency, self.trade_currency:
-            self.store_balance(cur)
+            self.store_balance(cur, now)
             sleep(3)
 
     def get_active_orders(self):
-        logging.info('[ORDERS] Getting active orders in local database')
+        logging.info('Getting active orders in local database')
-        orders = Order.select().where(Order.active==True, Order.trade_pair==self.trade_pair)
+        orders = Order.select().where(Order.active == True, Order.trade_pair == self.trade_pair)
         logging.debug(f'Found {len(orders)} in database')
         return orders
 
     def reconcile_orders(self):
-        logging.info('[ORDERS] Reconciling orders on TradeOgre with local database')
+        logging.info('Reconciling orders on TradeOgre with local database')
         to_orders = self.get_orders(self.trade_pair)
         for order in to_orders:
-            if not Order.filter(Order.uuid==order['uuid']):
+            if not Order.filter(Order.uuid == order['uuid']):
                 o = Order(
                     trade_pair=order['market'],
                     trade_type='manual',
@@ -100,10 +103,10 @@ class Trader(TradeOgre):
                     date=datetime.utcfromtimestamp(order['date'])
                 )
                 o.save()
-                logging.info(f'[ORDERS] Saved order {order["uuid"]} to the database')
+                logging.info(f'Saved order {order["uuid"]} to the database')
 
     def update_orders(self):
-        logging.info('[ORDERS] Updating orders in local database against TradeOgre')
+        logging.info('Updating orders in local database against TradeOgre')
         for order in self.get_active_orders():
             logging.info(f'Checking order {order.uuid}')
             o = self.get_order(order.uuid)
@@ -116,7 +119,7 @@ class Trader(TradeOgre):
 
     def calculate_earnings(self):
         orders = {'buy': [], 'sell': []}
-        completed_orders = Order.select().where(Order.cancelled==False, Order.active==False)
+        completed_orders = Order.select().where(Order.cancelled == False, Order.active == False)
         for order in completed_orders:
             if order.buy:
                 type = 'buy'
@@ -133,23 +136,42 @@ class Trader(TradeOgre):
             quantity=orders['total_earnings']
         )
         e.save()
+    
+    def update_portfolio(self):
+        logging.info('Updating portfolio')
+        base_balance = Balance.select().where(Balance.currency == self.base_currency).order_by(Balance.date.desc()).first()
+        trade_balance = Balance.select().where(Balance.currency == self.trade_currency).order_by(Balance.date.desc()).first()
+        trade_ticker = Ticker.select().where(Ticker.trade_pair == self.trade_pair).order_by(Ticker.date.desc()).first()
+        btc_price = BitcoinPrice.select().order_by(BitcoinPrice.date.desc()).first()
+        portfolio_btc = (trade_balance.total * trade_ticker.current_price) + base_balance.total
+        portfolio_usd = portfolio_btc * btc_price.price_usd
+        p = Portfolio(
+            usd=float(portfolio_usd),
+            btc=float(portfolio_btc)
+        )
+        p.save()
+    
+    def update_bitcoin_price(self):
+        logging.info('Updating Bitcoin price')
+        btc = BitcoinPrice(price_usd=float(self.get_bitcoin_price()))
+        btc.save()
 
     def start_market_maker(self):
-        logging.info('[MARKET MAKER] Starting market maker')
+        logging.info('Starting market maker')
-        latest = Ticker.select().where(Ticker.trade_pair==self.trade_pair).order_by(Ticker.date.desc()).get()
+        latest = Ticker.select().where(Ticker.trade_pair == self.trade_pair).order_by(Ticker.date.desc()).get()
         trade_type = 'market_maker'
         active_orders = len(self.get_active_orders())
 
-        if active_orders > self.active_order_limit:
+        if active_orders >= self.active_order_limit:
-            logging.info(f'[MARKET MAKER] Too many active orders in place ({active_orders}). Skipping.')
+            logging.info(f'Too many active orders in place ({active_orders}). Skipping.')
             return False
 
         if latest.spread_sats >= self.spread_target:
             bid_amount = uniform(self.trade_amount, self.trade_amount * self.amount_multiplier)
             ask_amount = uniform(self.trade_amount, self.trade_amount * self.amount_multiplier)
-            bid_price = '{:.8f}'.format(latest.bid + self.satoshi)
+            bid_price = '{:.8f}'.format(latest.bid + self.satoshi * self.satoshi_multiplier)
-            ask_price = '{:.8f}'.format(latest.ask - self.satoshi)
+            ask_price = '{:.8f}'.format(latest.ask - self.satoshi * self.satoshi_multiplier)
-            logging.info(f'[MARKET MAKER] Submitting buy order for {bid_amount} at {bid_price} {self.trade_pair}')
+            logging.info(f'Submitting buy order for {bid_amount} at {bid_price} {self.trade_pair}')
             buy = self.submit_order('buy', self.trade_pair, bid_amount, bid_price)
             logging.debug(buy)
             if 'uuid' in buy:
@@ -169,10 +191,10 @@ class Trader(TradeOgre):
                     active=active
                 )
                 _bo.save()
-                logging.info(f'[MARKET MAKER] Stored buy order as ID {_bo.id}')
+                logging.info(f'Stored buy order as ID {_bo.id}')
                 sleep(3)
 
-                logging.info(f'[MARKET MAKER] Submitting sell order for {ask_amount} at {ask_price} {self.trade_pair}')
+                logging.info(f'Submitting sell order for {ask_amount} at {ask_price} {self.trade_pair}')
                 sell = self.submit_order('sell', self.trade_pair, ask_amount, ask_price)
                 logging.debug(sell)
                 if 'uuid' in sell:
@@ -192,13 +214,13 @@ class Trader(TradeOgre):
                         active=active
                     )
                     _so.save()
-                    logging.info(f'[MARKET MAKER] Stored sell order as ID {_so.id}')
+                    logging.info(f'Stored sell order as ID {_so.id}')
                 else:
                     logging.info(sell)
             else:
                 logging.info(buy)
         else:
-            logging.info(f'[MARKET MAKER] Not enough bid-ask spread ({latest.spread_sats} sats). Skipping market maker.')
+            logging.info(f'Not enough bid-ask spread ({latest.spread_sats} sats). Skipping market maker.')
 
 
 if __name__ == '__main__':
@@ -208,12 +230,14 @@ if __name__ == '__main__':
     parser.add_argument('--update-orders', action='store_true', help='Update status of orders')
     parser.add_argument('--market-data', action='store_true', help='Update market data')
     parser.add_argument('--calculate-earnings', action='store_true', help='Calculate earnings from all trades')
+    parser.add_argument('--update-bitcoin-price', action='store_true', help='Update Bitcoin price (USD)')
     parser.add_argument('--run', action='store_true', help='Run continuously')
     args = parser.parse_args()
 
     t = Trader()
     orders_counter = 0
     balances_counter = 0
+    bitcoin_counter = 0
 
     if not args.run and args.update_orders:
         t.reconcile_orders()
@@ -224,12 +248,16 @@ if __name__ == '__main__':
 
     if not args.run and args.balances:
         t.store_balances()
+        t.update_portfolio()
 
     if not args.run and args.market_data:
         t.store_market_data()
 
     if not args.run and args.calculate_earnings:
         t.calculate_earnings()
+    
+    if not args.run and args.update_bitcoin_price:
+        t.update_bitcoin_price()
 
     if args.run:
         while True:
@@ -237,6 +265,15 @@ if __name__ == '__main__':
                 t.store_market_data()
             except Exception as e:
                 logging.info('[ERROR] Unable to store market data!', e)
+            
+            # update bitcoin price every 10 runs
+            if args.update_bitcoin_price:
+                if bitcoin_counter == 10:
+                    try:
+                        t.update_bitcoin_price()
+                        bitcoin_counter = 0
+                    except Exception as e:
+                        logging.info('Unable to update Bitcoin price!', e)
 
             # update orders every 4 runs
             if args.update_orders:
@@ -254,6 +291,7 @@ if __name__ == '__main__':
                 if balances_counter == 2:
                     try:
                         t.store_balances()
+                        t.update_portfolio()
                         logging.info('[BALANCE] Resetting balances counter')
                         balances_counter = 0
                     except Exception as e:
@@ -268,8 +306,9 @@ if __name__ == '__main__':
 
             orders_counter += 1
             balances_counter += 1
+            bitcoin_counter += 1
 
             # sleep
-            sleep_time = int(getenv('SLEEP_TIME', 20))
+            sleep_time = int(getenv('SLEEP_TIME', 1))
             logging.info(f'Sleeping for {sleep_time} seconds')
             sleep(sleep_time)

tradeogre.py

@@ -48,3 +48,18 @@ class TradeOgre(object):
         route = f'/account/orders'
         data = {'market': pair}
         return self.req(route, 'post', data)
+    
+    def get_bitcoin_price(self):
+        url = 'https://api.coingecko.com/api/v3/coins/bitcoin'
+        headers = {'accept': 'application/json'}
+        data = {
+            'localization': False,
+            'tickers': False,
+            'market_data': True,
+            'community_data': False,
+            'developer_data': False,
+            'sparkline': False
+        }
+        r = requests_get(url, headers=headers, data=data)
+        return r.json()['market_data']['current_price']['usd']
+