danny/grafana
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

init: The initial commit for files on the grafana machine

Browse Source
This commit is contained in:
Danny
2026-02-20 09:31:58 -06:00
commit 7336651692
8 changed files with 728 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mon/crypto_daily_live.py
+90
View File
@@ -0,0 +1,90 @@
import sys
import argparse
import datetime
import psycopg2
from pytz import timezone
CUT_HOUR=16
#CUT_MINUTE=0
TZ_CUT = timezone('US/Eastern')
parser = argparse.ArgumentParser(description='Generate daily returns entries in DB')
#parser.add_argument('--sloppy', action='store_true')
args = parser.parse_args()
def get_db_connection():
return psycopg2.connect(host='127.0.0.1', database='crypto_paper', user='joseph')
dbconn = get_db_connection()
# get latest daily pnl creation date and trading date
cur = dbconn.cursor()
cur.execute('SELECT created_at, trading_date FROM pnl_daily ORDER BY id DESC LIMIT 1')
res = cur.fetchone()
checkAfter = None
lastTradingDate = None
if res is not None:
# there are rows
checkAfter = res[0]
lastTradingDate = res[1]
print('checkAfter: %s, lastTradingDate: %s' % (checkAfter, lastTradingDate))
#import sys
#sys.exit(0)
if checkAfter is None:
cur.execute('SELECT * FROM acctval ORDER BY created_at')
else:
cur.execute('SELECT * FROM acctval WHERE created_at > %s ORDER BY created_at', (checkAfter,))
print("checkAfter filtered query is: SELECT * FROM acctval WHERE created_at > '%s' ORDER BY created_at" % (checkAfter,))
valRows = cur.fetchall()
#print(str(valRows))
print('got %s rows' % len(valRows))
thisDate = None
dayStart = None
dayStartTime = None
dayLast = None
for row in valRows:
#print(str(row))
rowDate = TZ_CUT.fromutc(row[1])
#print('rowDate: %s' % rowDate)
if rowDate.date() != thisDate:
print('LIVE new date: %s' % rowDate.date())
if thisDate is not None and dayStart is not None and dayLast is not None:# and thisDate.weekday() < 5:
# we can write out to DB
pnl = dayLast - dayStart
pnl_pct = pnl / dayStart
#markTime = TZ_CUT.localize(datetime.datetime.combine(rowDate.date(), datetime.datetime.min.time().replace(hour=CUT_HOUR, minute=CUT_MINUTE)))
markTime = TZ_CUT.localize(datetime.datetime.combine(thisDate, datetime.datetime.min.time())).date()
if lastTradingDate is None or markTime > lastTradingDate:
print('can mark it for %s, dayLast = %s, dayStart = %s, pnl = %s, pnl_pct = %s, markTime = %s' % (thisDate, dayLast, dayStart, pnl, pnl_pct, markTime))
cur.execute('INSERT INTO pnl_daily (created_at, trading_date, starting_value, pnl, pnl_pct) VALUES (%s, %s, %s, %s, %s)', ('NOW()', markTime, dayStart, pnl, pnl_pct))
thisDate = rowDate.date()
#print('set thisDate to %s' % thisDate)
dayStart = row[2]
dayStartTime = row[1]
dayLast = None
else:
dayLast = row[2]
# try to close "yesterday" even if we don't have records for "today" yet
if len(valRows) > 0:
now = TZ_CUT.fromutc(datetime.datetime.now())
print('now %s' % now)
print('rowDate %s' % rowDate)
if rowDate.date() != now.date() or now.hour >= CUT_HOUR:
if thisDate is not None and dayStart is not None and dayLast is not None:# and thisDate.weekday() < 5:
# we can write out to DB
pnl = dayLast - dayStart
pnl_pct = pnl / dayStart
#markTime = TZ_CUT.localize(datetime.datetime.combine(rowDate.date(), datetime.datetime.min.time().replace(hour=CUT_HOUR, minute=CUT_MINUTE)))
markTime = TZ_CUT.localize(datetime.datetime.combine(thisDate, datetime.datetime.min.time())).date()
if lastTradingDate is None or markTime > lastTradingDate:
print('can mark it for %s, dayLast = %s, dayStart = %s, pnl = %s, pnl_pct = %s, markTime = %s' % (thisDate, dayLast, dayStart, pnl, pnl_pct, markTime))
cur.execute('INSERT INTO pnl_daily (created_at, trading_date, starting_value, pnl, pnl_pct) VALUES (%s, %s, %s, %s, %s)', ('NOW()', markTime, dayStart, pnl, pnl_pct))
dbconn.commit()
cur.close()
dbconn.close()
mon/daily_live.py
+90
View File
@@ -0,0 +1,90 @@
import sys
import argparse
import datetime
import psycopg2
from pytz import timezone
CUT_HOUR=16
#CUT_MINUTE=0
TZ_CUT = timezone('US/Eastern')
parser = argparse.ArgumentParser(description='Generate daily returns entries in DB')
#parser.add_argument('--sloppy', action='store_true')
args = parser.parse_args()
def get_db_connection():
return psycopg2.connect(host='127.0.0.1', database='live_acctval', user='joseph')
dbconn = get_db_connection()
# get latest daily pnl creation date and trading date
cur = dbconn.cursor()
cur.execute('SELECT created_at, trading_date FROM pnl_daily ORDER BY id DESC LIMIT 1')
res = cur.fetchone()
checkAfter = None
lastTradingDate = None
if res is not None:
# there are rows
checkAfter = res[0]
lastTradingDate = res[1]
print('checkAfter: %s, lastTradingDate: %s' % (checkAfter, lastTradingDate))
#import sys
#sys.exit(0)
if checkAfter is None:
cur.execute('SELECT * FROM acctval ORDER BY created_at')
else:
cur.execute('SELECT * FROM acctval WHERE created_at > %s ORDER BY created_at', (checkAfter,))
print("checkAfter filtered query is: SELECT * FROM acctval WHERE created_at > '%s' ORDER BY created_at" % (checkAfter,))
valRows = cur.fetchall()
#print(str(valRows))
print('got %s rows' % len(valRows))
thisDate = None
dayStart = None
dayStartTime = None
dayLast = None
for row in valRows:
#print(str(row))
rowDate = TZ_CUT.fromutc(row[1])
#print('rowDate: %s' % rowDate)
if rowDate.date() != thisDate:
print('LIVE new date: %s' % rowDate.date())
if thisDate is not None and dayStart is not None and dayLast is not None and thisDate.weekday() < 5:
# we can write out to DB
pnl = dayLast - dayStart
pnl_pct = pnl / dayStart
#markTime = TZ_CUT.localize(datetime.datetime.combine(rowDate.date(), datetime.datetime.min.time().replace(hour=CUT_HOUR, minute=CUT_MINUTE)))
markTime = TZ_CUT.localize(datetime.datetime.combine(thisDate, datetime.datetime.min.time())).date()
if lastTradingDate is None or markTime > lastTradingDate:
print('can mark it for %s, dayLast = %s, dayStart = %s, pnl = %s, pnl_pct = %s, markTime = %s' % (thisDate, dayLast, dayStart, pnl, pnl_pct, markTime))
cur.execute('INSERT INTO pnl_daily (created_at, trading_date, starting_value, pnl, pnl_pct) VALUES (%s, %s, %s, %s, %s)', ('NOW()', markTime, dayStart, pnl, pnl_pct))
thisDate = rowDate.date()
#print('set thisDate to %s' % thisDate)
dayStart = row[2]
dayStartTime = row[1]
dayLast = None
else:
dayLast = row[2]
# try to close "yesterday" even if we don't have records for "today" yet
if len(valRows) > 0:
now = TZ_CUT.fromutc(datetime.datetime.now())
print('now %s' % now)
print('rowDate %s' % rowDate)
if rowDate.date() != now.date() or now.hour >= CUT_HOUR:
if thisDate is not None and dayStart is not None and dayLast is not None and thisDate.weekday() < 5:
# we can write out to DB
pnl = dayLast - dayStart
pnl_pct = pnl / dayStart
#markTime = TZ_CUT.localize(datetime.datetime.combine(rowDate.date(), datetime.datetime.min.time().replace(hour=CUT_HOUR, minute=CUT_MINUTE)))
markTime = TZ_CUT.localize(datetime.datetime.combine(thisDate, datetime.datetime.min.time())).date()
if lastTradingDate is None or markTime > lastTradingDate:
print('can mark it for %s, dayLast = %s, dayStart = %s, pnl = %s, pnl_pct = %s, markTime = %s' % (thisDate, dayLast, dayStart, pnl, pnl_pct, markTime))
cur.execute('INSERT INTO pnl_daily (created_at, trading_date, starting_value, pnl, pnl_pct) VALUES (%s, %s, %s, %s, %s)', ('NOW()', markTime, dayStart, pnl, pnl_pct))
dbconn.commit()
cur.close()
dbconn.close()
mon/mon_pnl_live.py
+19
View File
@@ -0,0 +1,19 @@
import time
import psycopg2
SLEEP=5
def get_db_connection_live():
return psycopg2.connect(host='127.0.0.1', database='live_acctval', user='joseph')
dbconn = get_db_connection_live()
cur = dbconn.cursor()
try:
while True:
cur.execute('SELECT pnl FROM pnl_intra ORDER BY id DESC LIMIT 1')
bleh = cur.fetchone()
print(round(bleh[0], 2), flush=True)
time.sleep(SLEEP)
finally:
cur.close()
dbconn.close()
mon/prime_networth.py
+80
View File
@@ -0,0 +1,80 @@
# Copyright 2022-present Coinbase Global, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from urllib.parse import urlparse
import json, hmac, hashlib, time, os, base64, requests, time, datetime
import psycopg2
from pytz import timezone
SLEEP=5
#API_KEY = os.environ.get('ACCESS_KEY')
#SECRET_KEY = os.environ.get('SIGNING_KEY')
#PASSPHRASE = os.environ.get('PASSPHRASE')
#PORTFOLIO_ID = os.environ.get('PORTFOLIO_ID')
PASSPHRASE = '5Rb5KFBCGLQ=aaJvs7jo8bBSLPFS5rWT0w=='
API_KEY = 'ZdlFsitVWo4=fEcyG3K5zeUZb7931FVSiWTO+rOgizJeuzP9uCuy9M0='
SECRET_KEY = '/NqFL5Fk4qM=0sm9mnMqB0cz4+MSKgEQ1Th2lPqMBpYjY5dZTGSlix5jBBodmxfi0jTwDfEcjhoUM0jqrTtgqQCLlSpRetEqOg=='
PORTFOLIO_ID = 'a2005845-db3c-4300-be3f-f5edba80d377'
def get_db_connection():
return psycopg2.connect(host='127.0.0.1', database='crypto_acctval', user='joseph')
def get_it():
uri = f'https://api.prime.coinbase.com/v1/portfolios/{PORTFOLIO_ID}/balances'
url_path = urlparse(uri).path
timestamp = str(int(time.time()))
message = timestamp + 'GET' + url_path
signature_b64 = base64.b64encode(hmac.digest(SECRET_KEY.encode(), message.encode(), hashlib.sha256))
headers = {
'X-CB-ACCESS-SIGNATURE': signature_b64,
'X-CB-ACCESS-TIMESTAMP': timestamp,
'X-CB-ACCESS-KEY': API_KEY,
'X-CB-ACCESS-PASSPHRASE': PASSPHRASE,
'Accept': 'application/json'
}
response = requests.get(uri, headers=headers)
parsed_response = json.loads(response.text)
#balances = [ x for x in parsed_response['balances'] if float(x['amount']) > 0 ]
if 'balances' in parsed_response:
balances = { x['symbol']: x for x in parsed_response['balances'] if float(x['amount']) > 0 }
#nw = sum([ float(x['fiat_amount']) for x in balances ])
nw = sum([ float(x['fiat_amount']) for x in balances.values() ]) + float(balances['usd']['holds'])
return(nw)
if __name__ == '__main__':
TZ_CUT = timezone('US/Eastern')
TZ_UTC = timezone('UTC')
dbconn = get_db_connection()
while True:
nw = get_it()
print(str(nw))
if nw is not None:
cur = dbconn.cursor()
cur.execute("INSERT INTO acctval (created_at, val) VALUES (%s, %s)", ('NOW()', nw))
dbconn.commit()
now_utc = datetime.datetime.now(TZ_CUT).replace(hour=0, minute=0, second=0, microsecond=0).astimezone(TZ_UTC)
cur.execute('SELECT val FROM acctval WHERE created_at > %s ORDER BY created_at LIMIT 1', (now_utc,))
resrow = cur.fetchone()
if resrow is not None:
pnl = nw - float(resrow[0])
print('pnl %s' % pnl)
cur.execute("INSERT INTO pnl_intra (created_at, pnl) VALUES (%s, %s)", ('NOW()', pnl))
dbconn.commit()
cur.close()
time.sleep(SLEEP)
mon/realtime_comms.py
+111
View File
@@ -0,0 +1,111 @@
import sys
import signal
import logging
import datetime
import subprocess
from threading import Timer
from ibapi.client import EClient, ExecutionFilter, Contract
from ibapi.wrapper import EWrapper, CommissionReport, Execution
from ibapi.utils import iswrapper
import psycopg2
from pytz import timezone
#lfmt = '%(asctime)s.%(msecs)03d\t%(levelname)-8s\t%(message)s'
#logging.basicConfig(format=lfmt, encoding='utf-8', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
tgtAccts = [ 'U13232746' ]
EVERY=5
TZ_NY = timezone('US/Eastern')
TZ_UTC = timezone('UTC')
def get_db_connection():
return psycopg2.connect(host='127.0.0.1', database='live_acctval', user='joseph')
class IBapi(EWrapper, EClient):
def __init__(self):
EClient.__init__(self, self)
self.dbconn = get_db_connection()
self.execs = {} # id -> {}
self.totalComms = 0
self.execFilter = ExecutionFilter()
#self.execFilter.time = '20240726 17:42:33' # this needs to be in UTC
#self.getInitialState()
def getInitialState(self):
#now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
#print('now its %s' % now)
cur = self.dbconn.cursor()
cur.execute('SELECT created_at,day_cumulative FROM comms WHERE created_at > %s ORDER BY id DESC LIMIT 1', (datetime.date.today(),))
res = cur.fetchone()
if res is not None:
# there is a row
print('last entry in comms is at %s, cumulative %s' % (res[0], res[1]))
self.execFilter.time = (res[0] + datetime.timedelta(seconds=1)).strftime('%Y%m%d-%H:%M:%S')
self.totalComms = float(res[1])
cur.close()
self.requestExecs() # NOW do this
def shipExec(self, execId):
if self.execs[execId]['comms'] <= 0:
return
self.totalComms += self.execs[execId]['comms']
print('shipped exec %s for trade @ %s, comms %s, total %s' % (execId, self.execs[execId]['date'], self.execs[execId]['comms'], self.totalComms))
#print('type of time is %s' % type(self.execs[execId]['date'])) # 20240726 13:15:56 US/Eastern
timeOfExec = datetime.datetime.strptime(self.execs[execId]['date'], '%Y%m%d %H:%M:%S US/Eastern')
tzLocal = TZ_NY.localize(timeOfExec)
print('datetime+localize = %s' % tzLocal)
cur = self.dbconn.cursor()
cur.execute('INSERT INTO comms (created_at, amount, day_cumulative) VALUES (%s, %s, %s)', (tzLocal, self.execs[execId]['comms'], self.totalComms))
self.dbconn.commit()
cur.close()
#self.execFilter.time = (timeOfExec - datetime.timedelta(seconds=1)).strftime('%Y%m%d %H:%M:%S')
self.execFilter.time = tzLocal.astimezone(TZ_UTC).strftime('%Y%m%d-%H:%M:%S')
print('set execFilter time to %s' % self.execFilter.time)
@iswrapper
def nextValidId(self, orderId:int):
#self.requestExecs()
self.getInitialState()
def requestExecs(self):
print('request executions after time %s' % self.execFilter.time)
now = datetime.datetime.now()
nowNY = now.astimezone(TZ_NY)
#print('in NY rn its %s' % nowNY)
self.reqExecutions(10001, self.execFilter)
self.t = Timer(EVERY, self.requestExecs)
self.t.start()
@iswrapper
def execDetails(self, reqId: int, contract: Contract, execution: Execution):
if execution.execId not in self.execs:
self.execs[execution.execId] = { 'date': execution.time,
'comms': None }
elif self.execs[execution.execId]['date'] is None:
self.execs[execution.execId]['date'] = execution.time
print('ship exec %s from ED' % execution.execId)
self.shipExec(execution.execId)
@iswrapper
def commissionReport(self, commissionReport: CommissionReport):
super().commissionReport(commissionReport)
#print("CommissionReport.", commissionReport)
if commissionReport.execId not in self.execs:
self.execs[commissionReport.execId] = { 'date': None,
'comms': commissionReport.commission }
elif self.execs[commissionReport.execId]['comms'] is None:
self.execs[commissionReport.execId]['comms'] = commissionReport.commission
print('ship exec %s from CR' % commissionReport.execId)
self.shipExec(commissionReport.execId)
app = IBapi()
def signal_handler(sig, frame):
print('You pressed Ctrl+C!')
app.disconnect()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
#app.connect('127.0.0.1', 7497, 123)
app.connect('tws2', 7496, 523)
app.run()
mon/realtime_live.py
+85
View File
@@ -0,0 +1,85 @@
import sys
import signal
import datetime
import subprocess
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.utils import iswrapper
import psycopg2
tgtAccts = [ 'U13232746' ]
tgtTags = [ 'NetLiquidationByCurrency', 'UnrealizedPnL' ]
maxLoss = -4000 # NOTE: WARNING: signed! do not put a positive number here!
if maxLoss is not None and maxLoss > 0:
raise Exception('maxLoss cannot be positive!')
#maxLossScript = '/home/joseph/mon/closeout_mkt.py' # NOTE: this script must be runnable by 'python'
maxLossScript = '/home/joseph/mon/closeout_fake.py' # NOTE: this script must be runnable by 'python'
def get_db_connection():
return psycopg2.connect(host='127.0.0.1', database='live_acctval', user='joseph')
class IBapi(EWrapper, EClient):
def __init__(self):
EClient.__init__(self, self)
self.dbconn = get_db_connection()
self.got = 0
self.maxLossHit = False
@iswrapper
def nextValidId(self, orderId:int):
self.reqAccountSummary(9002, 'All', '$LEDGER')
for i, acct in enumerate(tgtAccts):
self.reqPnL(i, acct, '');
@iswrapper
def accountSummary(self, reqId:int, account:str, tag:str, value:str, currency:str):
print('account summary for %s' % account)
tagged = {}
if account in tgtAccts:
if tag in tgtTags:
#print('reqID %s account %s tag %s value %s currency %s' % (reqId, account, tag, value, currency))
tagged[tag] = value
if len(tagged) > 0:
taggedSorted = sorted(tagged.items())
print('accountSummary %s %s' % (datetime.datetime.now(), taggedSorted))
if taggedSorted[0][0] == 'NetLiquidationByCurrency':
cur = self.dbconn.cursor()
cur.execute("INSERT INTO acctval (created_at, val) VALUES (%s, %s)", ('NOW()', taggedSorted[0][1]))
self.dbconn.commit()
cur.close()
@iswrapper
def accountSummaryEnd(self, reqId:int):
print('account summary done, reqID %s' % reqId)
@iswrapper
def pnl(self, reqId:int, dailyPnL, unrealizedPnL, realizedPnL):
if self.got < 2:
self.got += 1
return
elif dailyPnL == 0 and unrealizedPnL == 0 and realizedPnL == 0:
return
print('%s LIVE pnl! %s %s %s %s' % (datetime.datetime.now(), reqId, dailyPnL, unrealizedPnL, realizedPnL))
if maxLoss is not None and dailyPnL <= maxLoss and (not self.maxLossHit):
# we hit maxLoss
print('*** WE HIT MAXLOSS, RUNNING SCRIPT ***')
subprocess.run('python ' + maxLossScript, shell=True)
self.maxLossHit = True
cur = self.dbconn.cursor()
cur.execute("INSERT INTO pnl_intra (created_at, pnl, pnl_unrealized, pnl_realized) VALUES (%s, %s, %s, %s)", ('NOW()', dailyPnL, unrealizedPnL, realizedPnL))
self.dbconn.commit()
cur.close()
print('PNL script starting at %s (UTC)' % datetime.datetime.now())
app = IBapi()
def signal_handler(sig, frame):
print('You pressed Ctrl+C!')
app.disconnect()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
#app.connect('127.0.0.1', 7497, 123)
app.connect('tws2', 7496, 123)
app.run()