diff --git a/btc_tracker/kraken_fetch.py b/btc_tracker/kraken_fetch.py index 2bf60c5..f53eb1f 100755 --- a/btc_tracker/kraken_fetch.py +++ b/btc_tracker/kraken_fetch.py @@ -1,11 +1,11 @@ #!/usr/bin/python3 -''' +""" Fetch BTCUSD OHLC data from few market places and serve it forward with simple json api. Creates: ./btc_ohlc.db serves: localhost:5000/[t] and /serverkey Authentication via auth header with signatures -''' +""" import math import json @@ -19,38 +19,80 @@ import requests import ecdsa import krakenex from flask import Flask, jsonify, request -#from Cryptodome.Cipher import AES + +# from Cryptodome.Cipher import AES DATABASE = "btc_ohlc.db" +KEYSFILE = "userkeys.json" app = Flask(__name__) -## Add your public key here -user_publickeys = { - "user1": 'f1debc13fb21fe0eee54525aa4f8aae5733b201c755edaa55f8893c90aa375b261a62eaa3110651ac5d7705d402581256a37508b0a1ca28bd919ea44710d9c88' - } - ## Generate the ECDSA keys for this instance print("Generating ECDSA keys for this instance... just wait a bit...") server_private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1) server_public_key = server_private_key.get_verifying_key() # We need the hexadecimal form for sharing over http/json -server_public_key_hex = binascii.hexlify(server_public_key.to_string()).decode('utf-8') - +server_public_key_hex = binascii.hexlify(server_public_key.to_string()).decode("utf-8") database_lock = threading.Lock() # Empty response json -empty_dict = {"exchange": "", "timestamp": 0, "open": 0, "high": 0, "low": 0, "close": 0, "volume_quote": 0, "volume_base": 0, "trades": 0} +empty_dict = { + "exchange": "", + "timestamp": 0, + "open": 0, + "high": 0, + "low": 0, + "close": 0, + "volume_quote": 0, + "volume_base": 0, + "trades": 0, +} empty_json = json.dumps(empty_dict) + +def read_keys(): + with open(KEYSFILE, "r") as cfile: + user_keys = json.load(cfile) + return user_keys["user_publickeys"] + + def check_database(): - """ - Check the database for the 'ohlc' table. - If the database file or the table does not exist, create them. - """ - if not os.path.exists(DATABASE): + """ + Check the database for the 'ohlc' table. + If the database file or the table does not exist, create them. + """ + if not os.path.exists(DATABASE): + new_db = sqlite3.connect(DATABASE) + new_db.execute( + """\ + CREATE TABLE ohlc ( + id INTEGER PRIMARY KEY, + exchange TEXT NOT NULL, + timestamp INTEGER NOT NULL, + open REAL NOT NULL, + high REAL NOT NULL, + low REAL NOT NULL, + close REAL NOT NULL, + volume_quote REAL NOT NULL, + volume_base REAL NOT NULL, + trades INTEGER NOT NULL )""" + ) + + new_db.commit() + new_db.close() + new_db = sqlite3.connect(DATABASE) - new_db.execute("""\ + # Check if the table exists + + table_exists = False + cursor = new_db.execute("PRAGMA table_info(ohlc)") + for row in cursor: + table_exists = True + + # Create the table if it doesn't exist + if not table_exists: + new_db.execute( + """\ CREATE TABLE ohlc ( id INTEGER PRIMARY KEY, exchange TEXT NOT NULL, @@ -61,316 +103,309 @@ def check_database(): close REAL NOT NULL, volume_quote REAL NOT NULL, volume_base REAL NOT NULL, - trades INTEGER NOT NULL )""") + trades INTEGER NOT NULL )""" + ) + new_db.commit() - new_db.commit() - new_db.close() - - new_db = sqlite3.connect(DATABASE) - # Check if the table exists - - table_exists = False - cursor = new_db.execute("PRAGMA table_info(ohlc)") - for row in cursor: - table_exists = True - - # Create the table if it doesn't exist - if not table_exists: - new_db.execute("""\ - CREATE TABLE ohlc ( - id INTEGER PRIMARY KEY, - exchange TEXT NOT NULL, - timestamp INTEGER NOT NULL, - open REAL NOT NULL, - high REAL NOT NULL, - low REAL NOT NULL, - close REAL NOT NULL, - volume_quote REAL NOT NULL, - volume_base REAL NOT NULL, - trades INTEGER NOT NULL )""") - new_db.commit() def fetch_kraken(): - """ - Fetch BTCUSD OHLC data from Kraken in json. - Returns: - str: 5min OHLC data in JSON format. - """ - kraken = krakenex.API() + """ + Fetch BTCUSD OHLC data from Kraken in json. + Returns: + str: 5min OHLC data in JSON format. + """ + kraken = krakenex.API() - response = kraken.query_public('OHLC', {'pair': 'BTCUSD', 'interval': 240 }) - ohlc_data = response['result']['XXBTZUSD'] - - candle_stick_data = { - 'exchange': 'kraken', - 'timestamp': ohlc_data[1][0], - 'open': ohlc_data[0][1], - 'high': max(item[2] for item in ohlc_data), - 'low': min(item[3] for item in ohlc_data), - 'close': ohlc_data[-1][4], - 'volume_quote': sum(float(item[5]) for item in ohlc_data), - 'volume_base': sum(float(item[6]) for item in ohlc_data), - 'trades': sum(item[7] for item in ohlc_data), - } - - kraken_json = json.dumps(candle_stick_data, indent=2) - #print("Kraken: OK") - #print(kraken_json) - return kraken_json - -def fetch_bitstamp(): - """ - Fetch Bitstamp data ja serve it as json. - Returns: - str: 5min OHLC data in JSON format. - """ - response = requests.get("https://www.bitstamp.net/api/v2/ohlc/btcusd/?step=300&limit=1") - - if response.status_code == 200: # check if the request was successful - bitstamp_data = response.json() - ohlc_data = bitstamp_data["data"]["ohlc"] + response = kraken.query_public("OHLC", {"pair": "BTCUSD", "interval": 240}) + ohlc_data = response["result"]["XXBTZUSD"] candle_stick_data = { - 'exchange': 'bitstamp', - 'timestamp': int(ohlc_data[0]['timestamp']), - 'open': float(ohlc_data[0]['open']), - 'high': float(ohlc_data[0]['high']), - 'low': float(ohlc_data[0]['low']), - 'close': float(ohlc_data[0]['close']), - 'volume_quote': float(ohlc_data[0]['volume']), - 'volume_base': 0, # not provided by Bitstamp API - 'trades': 0, # not provided by Bitstamp API + "exchange": "kraken", + "timestamp": ohlc_data[1][0], + "open": ohlc_data[0][1], + "high": max(item[2] for item in ohlc_data), + "low": min(item[3] for item in ohlc_data), + "close": ohlc_data[-1][4], + "volume_quote": sum(float(item[5]) for item in ohlc_data), + "volume_base": sum(float(item[6]) for item in ohlc_data), + "trades": sum(item[7] for item in ohlc_data), } - bitstamp_json = json.dumps(candle_stick_data, indent=2) - #print("Bitstamp: OK") - # print(bitstamp_json) - return bitstamp_json - else: + kraken_json = json.dumps(candle_stick_data, indent=2) + return kraken_json + + +def fetch_bitstamp(): + """ + Fetch Bitstamp data ja serve it as json. + Returns: + str: 5min OHLC data in JSON format. + """ + response = requests.get( + "https://www.bitstamp.net/api/v2/ohlc/btcusd/?step=300&limit=1" + ) + + if response.status_code == 200: # check if the request was successful + bitstamp_data = response.json() + ohlc_data = bitstamp_data["data"]["ohlc"] + + candle_stick_data = { + "exchange": "bitstamp", + "timestamp": int(ohlc_data[0]["timestamp"]), + "open": float(ohlc_data[0]["open"]), + "high": float(ohlc_data[0]["high"]), + "low": float(ohlc_data[0]["low"]), + "close": float(ohlc_data[0]["close"]), + "volume_quote": float(ohlc_data[0]["volume"]), + "volume_base": 0, # not provided by Bitstamp API + "trades": 0, # not provided by Bitstamp API + } + + bitstamp_json = json.dumps(candle_stick_data, indent=2) + return bitstamp_json + # if we get any thing else than http/200 print(f"Error fetching data from Bitstamp API: {response.status_code}") return empty_json + def fetch_bitfinex(): - """ - Bitfinex - Returns: - str: 5min OHLC data in JSON format. - """ - response = requests.get("https://api-pub.bitfinex.com/v2/candles/trade:5m:tBTCUSD/last") + """ + Bitfinex + Returns: + str: 5min OHLC data in JSON format. + """ + response = requests.get( + "https://api-pub.bitfinex.com/v2/candles/trade:5m:tBTCUSD/last" + ) - if response.status_code == 200: # check if the request was successful - ohlc_data = response.json() - candle_stick_data = { - 'exchange': 'bitfinex', - 'timestamp': ohlc_data[0], - 'open': ohlc_data[1], - 'high': ohlc_data[2], - 'low': ohlc_data[3], - 'close': ohlc_data[4], - 'volume_quote': ohlc_data[5], - 'volume_base': 0, # not provided by Bitfinex API - 'trades': 0, # not provided by Bitfinex API - } + if response.status_code == 200: # check if the request was successful + ohlc_data = response.json() + candle_stick_data = { + "exchange": "bitfinex", + "timestamp": ohlc_data[0], + "open": ohlc_data[1], + "high": ohlc_data[2], + "low": ohlc_data[3], + "close": ohlc_data[4], + "volume_quote": ohlc_data[5], + "volume_base": 0, # not provided by Bitfinex API + "trades": 0, # not provided by Bitfinex API + } - bitfinex_json = json.dumps(candle_stick_data, indent=2) - #print("Bitfinex: OK") - #print(bitfinex_json) - return bitfinex_json - else: + bitfinex_json = json.dumps(candle_stick_data, indent=2) + return bitfinex_json + # if we get any thing else than http/200 print(f"Error fetching data from Bitfinex API: {response.status_code}") return empty_json -def fetch_gemini(): - """ - Fetch BTCUSD OHLC data from Gemini - Returns: - str: 5min OHLC data in JSON format. - """ - response = requests.get("https://api.gemini.com/v2/candles/btcusd/5m") - if response.status_code == 200: # check if the request was successful - gemini_ohlc = response.json() - candle_stick_data = { - 'exchange': 'gemini', - 'timestamp': gemini_ohlc[0][0], - 'open': gemini_ohlc[0][1], - 'high': gemini_ohlc[0][2], - 'low': gemini_ohlc[0][3], - 'close': gemini_ohlc[0][4], - 'volume_quote': 0, # not provided by Gemini API - 'volume_base': gemini_ohlc[0][5], - 'trades': 0, # not provided by Gemini API - } - gemini_json = json.dumps(candle_stick_data, indent=2) - #print("Gemini: OK") - #print(gemini_json) - return gemini_json - else: +def fetch_gemini(): + """ + Fetch BTCUSD OHLC data from Gemini + Returns: + str: 5min OHLC data in JSON format. + """ + response = requests.get("https://api.gemini.com/v2/candles/btcusd/5m") + + if response.status_code == 200: # check if the request was successful + gemini_ohlc = response.json() + candle_stick_data = { + "exchange": "gemini", + "timestamp": gemini_ohlc[0][0], + "open": gemini_ohlc[0][1], + "high": gemini_ohlc[0][2], + "low": gemini_ohlc[0][3], + "close": gemini_ohlc[0][4], + "volume_quote": 0, # not provided by Gemini API + "volume_base": gemini_ohlc[0][5], + "trades": 0, # not provided by Gemini API + } + gemini_json = json.dumps(candle_stick_data, indent=2) + return gemini_json + # if we get any thing else than http/200 print(f"Error fetching data from Gemini API: {response.status_code}") return empty_json -def fetch_bybit(): - """ - Fetch BTCUSD OHLC data from Bybit - Returns: - str: 5min OHLC data in JSON format. - """ - base_url = 'https://api.bybit.com/v2/public/kline/list?symbol=BTCUSD&interval=5&from=' - current_unixtime = int(time.time()) - last_minute = math.floor(current_unixtime / 60) - last_minute_unixtime = str(last_minute * 60 - 300) - query_url = ''.join([base_url, last_minute_unixtime]) - response = requests.get(query_url) - if response.status_code == 200: # check if the request was successful - bybit_ohlc = response.json() - candle_stick_data = { - 'exchange': 'bybit', - 'timestamp': bybit_ohlc['result'][0]['open_time'], - 'open': bybit_ohlc['result'][0]['open'], - 'high': bybit_ohlc['result'][0]['high'], - 'low': bybit_ohlc['result'][0]['low'], - 'close': bybit_ohlc['result'][0]['close'], - 'volume_quote': bybit_ohlc['result'][0]['volume'], - 'volume_base': bybit_ohlc['result'][0]['turnover'], - 'trades': 0 - } - bybit_json = json.dumps(candle_stick_data, indent=2) - return bybit_json - else: +def fetch_bybit(): + """ + Fetch BTCUSD OHLC data from Bybit + Returns: + str: 5min OHLC data in JSON format. + """ + base_url = ( + "https://api.bybit.com/v2/public/kline/list?symbol=BTCUSD&interval=5&from=" + ) + current_unixtime = int(time.time()) + last_minute = math.floor(current_unixtime / 60) + last_minute_unixtime = str(last_minute * 60 - 300) + query_url = "".join([base_url, last_minute_unixtime]) + response = requests.get(query_url) + + if response.status_code == 200: # check if the request was successful + bybit_ohlc = response.json() + candle_stick_data = { + "exchange": "bybit", + "timestamp": bybit_ohlc["result"][0]["open_time"], + "open": bybit_ohlc["result"][0]["open"], + "high": bybit_ohlc["result"][0]["high"], + "low": bybit_ohlc["result"][0]["low"], + "close": bybit_ohlc["result"][0]["close"], + "volume_quote": bybit_ohlc["result"][0]["volume"], + "volume_base": bybit_ohlc["result"][0]["turnover"], + "trades": 0, + } + bybit_json = json.dumps(candle_stick_data, indent=2) + return bybit_json + # if we get any thing else than http/200 print(f"Error fetching data from Bybit API: {response.status_code}") return empty_json -def write_dict_to_database(in_dict, connection): - """ - Writes given dict to given database. - Arguments: dict, db.connection() - Uses shared global database_lock. - """ - cursor = connection.cursor() - # use placeholders for the values in the insert statement - insert_query = "insert into ohlc (exchange, timestamp, open, high, low, close, volume_quote, volume_base, trades) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" - values = (in_dict['exchange'], - in_dict['timestamp'], - in_dict['open'], - in_dict['high'], - in_dict['low'], - in_dict['close'], - in_dict['volume_quote'], - in_dict['volume_base'], - in_dict['trades']) - ## apply lock while writing to database - with database_lock: - cursor.execute(insert_query, values) - connection.commit() +def write_dict_to_database(in_dict, connection): + """ + Writes given dict to given database. + Arguments: dict, db.connection() + Uses shared global database_lock. + """ + cursor = connection.cursor() + # use placeholders for the values in the insert statement + insert_query = "insert into ohlc (exchange, timestamp, open, high, low, close, volume_quote, volume_base, trades) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" + + values = ( + in_dict["exchange"], + in_dict["timestamp"], + in_dict["open"], + in_dict["high"], + in_dict["low"], + in_dict["close"], + in_dict["volume_quote"], + in_dict["volume_base"], + in_dict["trades"], + ) + ## apply lock while writing to database + with database_lock: + cursor.execute(insert_query, values) + connection.commit() + def get_the_data(): - """ - Creates infinite While True loop to fetch OHLC data and save it to database. - """ - while True: - ohlc_db = sqlite3.connect(DATABASE) - write_dict_to_database(json.loads(fetch_kraken()), ohlc_db) - write_dict_to_database(json.loads(fetch_bitfinex()), ohlc_db) - write_dict_to_database(json.loads(fetch_bitstamp()), ohlc_db) - write_dict_to_database(json.loads(fetch_gemini()), ohlc_db) - write_dict_to_database(json.loads(fetch_bybit()), ohlc_db) - ohlc_db.close() - print("fetches done at", time.time(), "sleeping now for 290") - time.sleep(290) + """ + Creates infinite While True loop to fetch OHLC data and save it to database. + """ + while True: + ohlc_db = sqlite3.connect(DATABASE) + write_dict_to_database(json.loads(fetch_kraken()), ohlc_db) + write_dict_to_database(json.loads(fetch_bitfinex()), ohlc_db) + write_dict_to_database(json.loads(fetch_bitstamp()), ohlc_db) + write_dict_to_database(json.loads(fetch_gemini()), ohlc_db) + write_dict_to_database(json.loads(fetch_bybit()), ohlc_db) + ohlc_db.close() + print("fetches done at", time.time(), "sleeping now for 290") + time.sleep(290) + def check_auth(text, signature): - """ - Check signatures against known public keys - Arguments: text, signature - Reads: Global public user_publickeys dict. - Returns: True / False - """ - ## Make bytes-object from given signature - sig_bytes = bytes.fromhex(signature) - ## We will iterate over all user keys to determ who is we are talking to and should they have access - for key, value in user_publickeys.items(): - ## Create bytes-object from the public in 'value' variable - ## and use it to create VerifyingKey (vk) - public_key_bytes = bytes.fromhex(value) - verifying_key = ecdsa.VerifyingKey.from_string(public_key_bytes, curve=ecdsa.SECP256k1) - try: - verifying_key.verify(sig_bytes, bytes(text, 'utf-8')) - print('user is', key) + """ + Check signatures against known public keys + Arguments: text, signature + Reads: Global public user_publickeys dict. + Returns: True / False + """ + ## Make bytes-object from given signature + sig_bytes = bytes.fromhex(signature) + ## We will iterate over all user keys to determ who is we are talking to and should they have access + for key, value in user_publickeys.items(): + ## Create bytes-object from the public in 'value' variable + ## and use it to create VerifyingKey (vk) + public_key_bytes = bytes.fromhex(value) + verifying_key = ecdsa.VerifyingKey.from_string( + public_key_bytes, curve=ecdsa.SECP256k1 + ) + try: + verifying_key.verify(sig_bytes, bytes(text, "utf-8")) + print("user is", key) - return True - except ecdsa.BadSignatureError: - return False + return True + except ecdsa.BadSignatureError: + return False -@app.route('/') + +@app.route("/") def get_data(): - """ - Serve the data from the database. Limit the responses by given timestamp. - The pretty thing is under consideration... - """ - # Get the time (t) argument from the url" - query_timestamp = request.args.get('t') - # Should we make output pretty for curl users? - query_pretty = request.args.get('pretty') + """ + Serve the data from the database. Limit the responses by given timestamp. + The pretty thing is under consideration... + """ + # Get the time (t) argument from the url" + query_timestamp = request.args.get("t") + # Should we make output pretty for curl users? + query_pretty = request.args.get("pretty") - # Authentication header, signatured the query with private key of a user - signature = request.headers.get('auth') - get_url = request.url - if not check_auth(get_url, signature): - return 'Access denied! Check your keys, maybe.', 403 + # Authentication header, signatured the query with private key of a user + signature = request.headers.get("auth") + get_url = request.url + if not check_auth(get_url, signature): + return "Access denied! Check your keys, maybe.", 403 - with database_lock: - btc_db = sqlite3.connect(DATABASE) - if query_timestamp: - rows = btc_db.execute("SELECT exchange, timestamp, open, high, low, close FROM ohlc WHERE timestamp > ? ORDER BY timestamp", (query_timestamp,)).fetchall() + with database_lock: + btc_db = sqlite3.connect(DATABASE) + if query_timestamp: + rows = btc_db.execute( + "SELECT exchange, timestamp, open, high, low, close FROM ohlc WHERE timestamp > ? ORDER BY timestamp", + (query_timestamp,), + ).fetchall() + else: + rows = btc_db.execute( + "SELECT exchange, timestamp, open, high, low, close FROM ohlc ORDER BY timestamp" + ).fetchall() + query_timestamp = 0 + + data = {"timestamp": time.time(), "rows": rows} + + # make sha256 checksum and append it to the data object + data_shasum = sha256(json.dumps(data).encode("utf-8")).hexdigest() + updated_data = {"shasum": data_shasum} + updated_data.update(data) + data = updated_data + + # sign the response + signature = server_private_key.sign(json.dumps(data).encode("utf-8")) + signature_hex = binascii.hexlify(signature).decode("utf-8") + data["signature"] = signature_hex + + if query_pretty: + response = json.dumps(data, indent=2, separators=(";\n", " :")) else: - rows = btc_db.execute('SELECT exchange, timestamp, open, high, low, close FROM ohlc ORDER BY timestamp').fetchall() - query_timestamp = 0 + response = json.dumps(data) + return response, 200, {"Content-Type": "application/json"} - data = { - "timestamp": time.time(), - "rows": rows - } - # make sha256 checksum and append it to the data object - data_shasum = sha256(json.dumps(data).encode('utf-8')).hexdigest() - updated_data = {"shasum": data_shasum} - updated_data.update(data) - data = updated_data - - # sign the response - signature = server_private_key.sign(json.dumps(data).encode('utf-8')) - signature_hex = binascii.hexlify(signature).decode('utf-8') - data['signature'] = signature_hex - - if query_pretty: - response = json.dumps(data, indent=2, separators=(';\n', ' :')) - else: - response = json.dumps(data) - return response, 200, {'Content-Type': 'application/json'} - -@app.route('/serverkey') +@app.route("/serverkey") def give_serverkey(): - """ - Serve the public keys of this instace to the world. - """ - ## This endpoint also under Authentication? - signature = request.headers.get('auth') - get_url = request.url - if not check_auth(get_url, signature): - return 'Access denied! Check your keys, maybe.', 403 + """ + Serve the public keys of this instace to the world. + """ + ## This endpoint also under Authentication? + signature = request.headers.get("auth") + get_url = request.url + if not check_auth(get_url, signature): + return "Access denied! Check your keys, maybe.", 403 - return jsonify({'public_key': server_public_key_hex}) + return jsonify({"public_key": server_public_key_hex}) -if __name__ == '__main__': - # Make sanity checks for the database - check_database() - # Start the data fetching backend process - fetch_thread = threading.Thread(target=get_the_data) - fetch_thread.daemon = True - fetch_thread.start() +if __name__ == "__main__": + # Make sanity checks for the database + check_database() - # Start the Flask app - app.run() + # Get the users public keys + user_publickeys = read_keys() + + # Start the data fetching backend process§ + fetch_thread = threading.Thread(target=get_the_data) + fetch_thread.daemon = True + fetch_thread.start() + + # Start the Flask app + app.run() diff --git a/btc_tracker/userkeys.json b/btc_tracker/userkeys.json new file mode 100644 index 0000000..86add59 --- /dev/null +++ b/btc_tracker/userkeys.json @@ -0,0 +1,5 @@ +{ + "user_publickeys": { + "user1": "f1debc13fb21fe0eee54525aa4f8aae5733b201c755edaa55f8893c90aa375b261a62eaa3110651ac5d7705d402581256a37508b0a1ca28bd919ea44710d9c88" + } +}