MrKalzu/chatgpt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Tried to make pylint happy.

Browse Source
This commit is contained in:
kalzu 2023-01-01 13:40:47 +02:00
parent ae14376b13
commit cb33802e87
1 changed files with 124 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

196
btc_tracker/kraken_fetch.py
View File

@ -1,14 +1,27 @@
#!/usr/bin/python3 #!/usr/bin/python3
'''
Fetch BTCUSD OHLC data from few market places and serve it forward with simple json api.
import krakenex, math Creates: ./btc_ohlc.db
import json, sqlite3, binascii serves: localhost:5000/[t] and /serverkey
import requests, os, time Authentication via auth header with signatures
import threading, ecdsa '''
from Cryptodome.Cipher import AES
import math
import json
import os
import time
import sqlite3
import binascii
import threading
from hashlib import sha256 from hashlib import sha256
import requests
import ecdsa
import krakenex
from flask import Flask, jsonify, request from flask import Flask, jsonify, request
#from Cryptodome.Cipher import AES
database = "btc_ohlc.db" DATABASE = "btc_ohlc.db"
app = Flask(__name__) app = Flask(__name__)
## Add your public key here ## Add your public key here
@ -30,13 +43,14 @@ database_lock = threading.Lock()
empty_dict = {"exchange": "", "timestamp": 0, "open": 0, "high": 0, "low": 0, "close": 0, "volume_quote": 0, "volume_base": 0, "trades": 0} empty_dict = {"exchange": "", "timestamp": 0, "open": 0, "high": 0, "low": 0, "close": 0, "volume_quote": 0, "volume_base": 0, "trades": 0}
empty_json = json.dumps(empty_dict) empty_json = json.dumps(empty_dict)
def Checkthedatabase(): def check_database():
## Some sanity for the database """
# check if the database file exists Check the database for the 'ohlc' table.
if not os.path.exists(database): If the database file or the table does not exist, create them.
db = sqlite3.connect(database) """
if not os.path.exists(DATABASE):
db.execute("""\ new_db = sqlite3.connect(DATABASE)
new_db.execute("""\
CREATE TABLE ohlc ( CREATE TABLE ohlc (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
exchange TEXT NOT NULL, exchange TEXT NOT NULL,
@ -49,20 +63,20 @@ def Checkthedatabase():
volume_base REAL NOT NULL, volume_base REAL NOT NULL,
trades INTEGER NOT NULL )""") trades INTEGER NOT NULL )""")
db.commit() new_db.commit()
db.close() new_db.close()
db = sqlite3.connect(database) new_db = sqlite3.connect(DATABASE)
# Check if the table exists # Check if the table exists
table_exists = False table_exists = False
cursor = db.execute("PRAGMA table_info(ohlc)") cursor = new_db.execute("PRAGMA table_info(ohlc)")
for row in cursor: for row in cursor:
table_exists = True table_exists = True
# Create the table if it doesn't exist # Create the table if it doesn't exist
if not table_exists: if not table_exists:
db.execute("""\ new_db.execute("""\
CREATE TABLE ohlc ( CREATE TABLE ohlc (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
exchange TEXT NOT NULL, exchange TEXT NOT NULL,
@ -74,15 +88,19 @@ def Checkthedatabase():
volume_quote REAL NOT NULL, volume_quote REAL NOT NULL,
volume_base REAL NOT NULL, volume_base REAL NOT NULL,
trades INTEGER NOT NULL )""") trades INTEGER NOT NULL )""")
db.commit() new_db.commit()
def fetch_kraken(): def fetch_kraken():
### Kraken """
Fetch BTCUSD OHLC data from Kraken in json.
Returns:
str: 5min OHLC data in JSON format.
"""
kraken = krakenex.API() kraken = krakenex.API()
response = kraken.query_public('OHLC', {'pair': 'BTCUSD', 'interval': 240 }) response = kraken.query_public('OHLC', {'pair': 'BTCUSD', 'interval': 240 })
ohlc_data = response['result']['XXBTZUSD'] ohlc_data = response['result']['XXBTZUSD']
candle_stick_data = { candle_stick_data = {
'exchange': 'kraken', 'exchange': 'kraken',
'timestamp': ohlc_data[1][0], 'timestamp': ohlc_data[1][0],
@ -94,20 +112,24 @@ def fetch_kraken():
'volume_base': sum(float(item[6]) for item in ohlc_data), 'volume_base': sum(float(item[6]) for item in ohlc_data),
'trades': sum(item[7] for item in ohlc_data), 'trades': sum(item[7] for item in ohlc_data),
} }
kraken_json = json.dumps(candle_stick_data, indent=2) kraken_json = json.dumps(candle_stick_data, indent=2)
#print("Kraken: OK") #print("Kraken: OK")
#print(kraken_json) #print(kraken_json)
return kraken_json return kraken_json
def fetch_bitstamp(): def fetch_bitstamp():
## Bitstamp """
Fetch Bitstamp data ja serve it as json.
Returns:
str: 5min OHLC data in JSON format.
"""
response = requests.get("https://www.bitstamp.net/api/v2/ohlc/btcusd/?step=300&limit=1") response = requests.get("https://www.bitstamp.net/api/v2/ohlc/btcusd/?step=300&limit=1")
if response.status_code == 200: # check if the request was successful if response.status_code == 200: # check if the request was successful
bitstamp_data = response.json() bitstamp_data = response.json()
ohlc_data = bitstamp_data["data"]["ohlc"] ohlc_data = bitstamp_data["data"]["ohlc"]
candle_stick_data = { candle_stick_data = {
'exchange': 'bitstamp', 'exchange': 'bitstamp',
'timestamp': int(ohlc_data[0]['timestamp']), 'timestamp': int(ohlc_data[0]['timestamp']),
@ -119,7 +141,7 @@ def fetch_bitstamp():
'volume_base': 0, # not provided by Bitstamp API 'volume_base': 0, # not provided by Bitstamp API
'trades': 0, # not provided by Bitstamp API 'trades': 0, # not provided by Bitstamp API
} }
bitstamp_json = json.dumps(candle_stick_data, indent=2) bitstamp_json = json.dumps(candle_stick_data, indent=2)
#print("Bitstamp: OK") #print("Bitstamp: OK")
# print(bitstamp_json) # print(bitstamp_json)
@ -129,9 +151,13 @@ def fetch_bitstamp():
return empty_json return empty_json
def fetch_bitfinex(): def fetch_bitfinex():
## Bitfinex """
Bitfinex
Returns:
str: 5min OHLC data in JSON format.
"""
response = requests.get("https://api-pub.bitfinex.com/v2/candles/trade:5m:tBTCUSD/last") response = requests.get("https://api-pub.bitfinex.com/v2/candles/trade:5m:tBTCUSD/last")
if response.status_code == 200: # check if the request was successful if response.status_code == 200: # check if the request was successful
ohlc_data = response.json() ohlc_data = response.json()
candle_stick_data = { candle_stick_data = {
@ -145,7 +171,7 @@ def fetch_bitfinex():
'volume_base': 0, # not provided by Bitfinex API 'volume_base': 0, # not provided by Bitfinex API
'trades': 0, # not provided by Bitfinex API 'trades': 0, # not provided by Bitfinex API
} }
bitfinex_json = json.dumps(candle_stick_data, indent=2) bitfinex_json = json.dumps(candle_stick_data, indent=2)
#print("Bitfinex: OK") #print("Bitfinex: OK")
#print(bitfinex_json) #print(bitfinex_json)
@ -155,9 +181,13 @@ def fetch_bitfinex():
return empty_json return empty_json
def fetch_gemini(): def fetch_gemini():
## Gemini """
Fetch BTCUSD OHLC data from Gemini
Returns:
str: 5min OHLC data in JSON format.
"""
response = requests.get("https://api.gemini.com/v2/candles/btcusd/5m") response = requests.get("https://api.gemini.com/v2/candles/btcusd/5m")
if response.status_code == 200: # check if the request was successful if response.status_code == 200: # check if the request was successful
gemini_ohlc = response.json() gemini_ohlc = response.json()
candle_stick_data = { candle_stick_data = {
@ -180,8 +210,11 @@ def fetch_gemini():
return empty_json return empty_json
def fetch_bybit(): def fetch_bybit():
## BYBIT """
#curl 'https://api-testnet.bybit.com/v2/public/kline/list?symbol=BTCUSD&interval=5&from=1672225349&limit=3' Fetch BTCUSD OHLC data from Bybit
Returns:
str: 5min OHLC data in JSON format.
"""
base_url = 'https://api.bybit.com/v2/public/kline/list?symbol=BTCUSD&interval=5&from=' base_url = 'https://api.bybit.com/v2/public/kline/list?symbol=BTCUSD&interval=5&from='
current_unixtime = int(time.time()) current_unixtime = int(time.time())
last_minute = math.floor(current_unixtime / 60) last_minute = math.floor(current_unixtime / 60)
@ -209,18 +242,23 @@ def fetch_bybit():
return empty_json return empty_json
def write_dict_to_database(in_dict, connection): def write_dict_to_database(in_dict, connection):
"""
Writes given dict to given database.
Arguments: dict, db.connection()
Uses shared global database_lock.
"""
cursor = connection.cursor() cursor = connection.cursor()
# use placeholders for the values in the insert statement # use placeholders for the values in the insert statement
insert_query = "insert into ohlc (exchange, timestamp, open, high, low, close, volume_quote, volume_base, trades) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" insert_query = "insert into ohlc (exchange, timestamp, open, high, low, close, volume_quote, volume_base, trades) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
values = (in_dict['exchange'], values = (in_dict['exchange'],
in_dict['timestamp'], in_dict['timestamp'],
in_dict['open'], in_dict['open'],
in_dict['high'], in_dict['high'],
in_dict['low'], in_dict['low'],
in_dict['close'], in_dict['close'],
in_dict['volume_quote'], in_dict['volume_quote'],
in_dict['volume_base'], in_dict['volume_base'],
in_dict['trades']) in_dict['trades'])
## apply lock while writing to database ## apply lock while writing to database
with database_lock: with database_lock:
@ -228,19 +266,27 @@ def write_dict_to_database(in_dict, connection):
connection.commit() connection.commit()
def get_the_data(): def get_the_data():
#cursor = db.cursor() """
while True: Creates infinite While True loop to fetch OHLC data and save it to database.
db = sqlite3.connect(database) """
write_dict_to_database(json.loads(fetch_kraken()), db) while True:
write_dict_to_database(json.loads(fetch_bitfinex()), db) ohlc_db = sqlite3.connect(DATABASE)
write_dict_to_database(json.loads(fetch_bitstamp()), db) write_dict_to_database(json.loads(fetch_kraken()), ohlc_db)
write_dict_to_database(json.loads(fetch_gemini()), db) write_dict_to_database(json.loads(fetch_bitfinex()), ohlc_db)
write_dict_to_database(json.loads(fetch_bybit()), db) write_dict_to_database(json.loads(fetch_bitstamp()), ohlc_db)
db.close() write_dict_to_database(json.loads(fetch_gemini()), ohlc_db)
write_dict_to_database(json.loads(fetch_bybit()), ohlc_db)
ohlc_db.close()
print("fetches done at", time.time(), "sleeping now for 290") print("fetches done at", time.time(), "sleeping now for 290")
time.sleep(290) time.sleep(290)
def check_auth(text, signature): def check_auth(text, signature):
"""
Check signatures against known public keys
Arguments: text, signature
Reads: Global public user_publickeys dict.
Returns: True / False
"""
## Make bytes-object from given signature ## Make bytes-object from given signature
sig_bytes = bytes.fromhex(signature) sig_bytes = bytes.fromhex(signature)
## We will iterate over all user keys to determ who is we are talking to and should they have access ## We will iterate over all user keys to determ who is we are talking to and should they have access
@ -248,9 +294,9 @@ def check_auth(text, signature):
## Create bytes-object from the public in 'value' variable ## Create bytes-object from the public in 'value' variable
## and use it to create VerifyingKey (vk) ## and use it to create VerifyingKey (vk)
public_key_bytes = bytes.fromhex(value) public_key_bytes = bytes.fromhex(value)
vk = ecdsa.VerifyingKey.from_string(public_key_bytes, curve=ecdsa.SECP256k1) verifying_key = ecdsa.VerifyingKey.from_string(public_key_bytes, curve=ecdsa.SECP256k1)
try: try:
vk.verify(sig_bytes, bytes(text, 'utf-8')) verifying_key.verify(sig_bytes, bytes(text, 'utf-8'))
print('user is', key) print('user is', key)
return True return True
@ -259,6 +305,10 @@ def check_auth(text, signature):
@app.route('/') @app.route('/')
def get_data(): def get_data():
"""
Serve the data from the database. Limit the responses by given timestamp.
The pretty thing is under consideration...
"""
# Get the time (t) argument from the url" # Get the time (t) argument from the url"
query_timestamp = request.args.get('t') query_timestamp = request.args.get('t')
# Should we make output pretty for curl users? # Should we make output pretty for curl users?
@ -270,39 +320,41 @@ def get_data():
if not check_auth(get_url, signature): if not check_auth(get_url, signature):
return 'Access denied! Check your keys, maybe.', 403 return 'Access denied! Check your keys, maybe.', 403
database_lock.acquire() with database_lock:
db = sqlite3.connect(database) btc_db = sqlite3.connect(DATABASE)
if query_timestamp: if query_timestamp:
rows = db.execute("SELECT exchange, timestamp, open, high, low, close FROM ohlc WHERE timestamp > ? ORDER BY timestamp", (query_timestamp,)).fetchall() rows = btc_db.execute("SELECT exchange, timestamp, open, high, low, close FROM ohlc WHERE timestamp > ? ORDER BY timestamp", (query_timestamp,)).fetchall()
else: else:
rows = db.execute('SELECT exchange, timestamp, open, high, low, close FROM ohlc ORDER BY timestamp').fetchall() rows = btc_db.execute('SELECT exchange, timestamp, open, high, low, close FROM ohlc ORDER BY timestamp').fetchall()
query_timestamp = 0 query_timestamp = 0
database_lock.release()
data = { data = {
"timestamp": time.time(), "timestamp": time.time(),
"rows": rows "rows": rows
} }
# make sha256 checksum and append it to the data object # make sha256 checksum and append it to the data object
data_shasum = sha256(json.dumps(data).encode('utf-8')).hexdigest() data_shasum = sha256(json.dumps(data).encode('utf-8')).hexdigest()
updated_data = {"shasum": data_shasum} updated_data = {"shasum": data_shasum}
updated_data.update(data) updated_data.update(data)
data = updated_data data = updated_data
# sign the response # sign the response
signature = server_private_key.sign(json.dumps(data).encode('utf-8')) signature = server_private_key.sign(json.dumps(data).encode('utf-8'))
signature_hex = binascii.hexlify(signature).decode('utf-8') signature_hex = binascii.hexlify(signature).decode('utf-8')
data['signature'] = signature_hex data['signature'] = signature_hex
if query_pretty: if query_pretty:
response = json.dumps(data, indent=2, separators=(';\n', ' :')) response = json.dumps(data, indent=2, separators=(';\n', ' :'))
else: else:
response = json.dumps(data) response = json.dumps(data)
return response, 200, {'Content-Type': 'application/json'} return response, 200, {'Content-Type': 'application/json'}
@app.route('/serverkey') @app.route('/serverkey')
def give_serverkey(): def give_serverkey():
"""
Serve the public keys of this instace to the world.
"""
## This endpoint also under Authentication? ## This endpoint also under Authentication?
signature = request.headers.get('auth') signature = request.headers.get('auth')
get_url = request.url get_url = request.url
@ -313,12 +365,12 @@ def give_serverkey():
if __name__ == '__main__': if __name__ == '__main__':
# Make sanity checks for the database # Make sanity checks for the database
Checkthedatabase() check_database()
# Start the data fetching backend process # Start the data fetching backend process
fetch_thread = threading.Thread(target=get_the_data) fetch_thread = threading.Thread(target=get_the_data)
fetch_thread.daemon = True fetch_thread.daemon = True
fetch_thread.start() fetch_thread.start()
# Start the Flask app # Start the Flask app
app.run() app.run()