diff --git a/btc_tracker/kraken_fetch.py b/btc_tracker/kraken_fetch.py index 50a545e..2bf60c5 100755 --- a/btc_tracker/kraken_fetch.py +++ b/btc_tracker/kraken_fetch.py @@ -1,14 +1,27 @@ #!/usr/bin/python3 +''' +Fetch BTCUSD OHLC data from few market places and serve it forward with simple json api. -import krakenex, math -import json, sqlite3, binascii -import requests, os, time -import threading, ecdsa -from Cryptodome.Cipher import AES +Creates: ./btc_ohlc.db +serves: localhost:5000/[t] and /serverkey +Authentication via auth header with signatures +''' + +import math +import json +import os +import time +import sqlite3 +import binascii +import threading from hashlib import sha256 +import requests +import ecdsa +import krakenex from flask import Flask, jsonify, request +#from Cryptodome.Cipher import AES -database = "btc_ohlc.db" +DATABASE = "btc_ohlc.db" app = Flask(__name__) ## Add your public key here @@ -30,13 +43,14 @@ database_lock = threading.Lock() empty_dict = {"exchange": "", "timestamp": 0, "open": 0, "high": 0, "low": 0, "close": 0, "volume_quote": 0, "volume_base": 0, "trades": 0} empty_json = json.dumps(empty_dict) -def Checkthedatabase(): - ## Some sanity for the database - # check if the database file exists - if not os.path.exists(database): - db = sqlite3.connect(database) - - db.execute("""\ +def check_database(): + """ + Check the database for the 'ohlc' table. + If the database file or the table does not exist, create them. + """ + if not os.path.exists(DATABASE): + new_db = sqlite3.connect(DATABASE) + new_db.execute("""\ CREATE TABLE ohlc ( id INTEGER PRIMARY KEY, exchange TEXT NOT NULL, @@ -49,20 +63,20 @@ def Checkthedatabase(): volume_base REAL NOT NULL, trades INTEGER NOT NULL )""") - db.commit() - db.close() - - db = sqlite3.connect(database) - + new_db.commit() + new_db.close() + + new_db = sqlite3.connect(DATABASE) # Check if the table exists + table_exists = False - cursor = db.execute("PRAGMA table_info(ohlc)") + cursor = new_db.execute("PRAGMA table_info(ohlc)") for row in cursor: table_exists = True - + # Create the table if it doesn't exist if not table_exists: - db.execute("""\ + new_db.execute("""\ CREATE TABLE ohlc ( id INTEGER PRIMARY KEY, exchange TEXT NOT NULL, @@ -74,15 +88,19 @@ def Checkthedatabase(): volume_quote REAL NOT NULL, volume_base REAL NOT NULL, trades INTEGER NOT NULL )""") - db.commit() + new_db.commit() def fetch_kraken(): -### Kraken + """ + Fetch BTCUSD OHLC data from Kraken in json. + Returns: + str: 5min OHLC data in JSON format. + """ kraken = krakenex.API() - + response = kraken.query_public('OHLC', {'pair': 'BTCUSD', 'interval': 240 }) ohlc_data = response['result']['XXBTZUSD'] - + candle_stick_data = { 'exchange': 'kraken', 'timestamp': ohlc_data[1][0], @@ -94,20 +112,24 @@ def fetch_kraken(): 'volume_base': sum(float(item[6]) for item in ohlc_data), 'trades': sum(item[7] for item in ohlc_data), } - + kraken_json = json.dumps(candle_stick_data, indent=2) #print("Kraken: OK") #print(kraken_json) return kraken_json def fetch_bitstamp(): -## Bitstamp + """ + Fetch Bitstamp data ja serve it as json. + Returns: + str: 5min OHLC data in JSON format. + """ response = requests.get("https://www.bitstamp.net/api/v2/ohlc/btcusd/?step=300&limit=1") - + if response.status_code == 200: # check if the request was successful bitstamp_data = response.json() ohlc_data = bitstamp_data["data"]["ohlc"] - + candle_stick_data = { 'exchange': 'bitstamp', 'timestamp': int(ohlc_data[0]['timestamp']), @@ -119,7 +141,7 @@ def fetch_bitstamp(): 'volume_base': 0, # not provided by Bitstamp API 'trades': 0, # not provided by Bitstamp API } - + bitstamp_json = json.dumps(candle_stick_data, indent=2) #print("Bitstamp: OK") # print(bitstamp_json) @@ -129,9 +151,13 @@ def fetch_bitstamp(): return empty_json def fetch_bitfinex(): -## Bitfinex + """ + Bitfinex + Returns: + str: 5min OHLC data in JSON format. + """ response = requests.get("https://api-pub.bitfinex.com/v2/candles/trade:5m:tBTCUSD/last") - + if response.status_code == 200: # check if the request was successful ohlc_data = response.json() candle_stick_data = { @@ -145,7 +171,7 @@ def fetch_bitfinex(): 'volume_base': 0, # not provided by Bitfinex API 'trades': 0, # not provided by Bitfinex API } - + bitfinex_json = json.dumps(candle_stick_data, indent=2) #print("Bitfinex: OK") #print(bitfinex_json) @@ -155,9 +181,13 @@ def fetch_bitfinex(): return empty_json def fetch_gemini(): -## Gemini + """ + Fetch BTCUSD OHLC data from Gemini + Returns: + str: 5min OHLC data in JSON format. + """ response = requests.get("https://api.gemini.com/v2/candles/btcusd/5m") - + if response.status_code == 200: # check if the request was successful gemini_ohlc = response.json() candle_stick_data = { @@ -180,8 +210,11 @@ def fetch_gemini(): return empty_json def fetch_bybit(): -## BYBIT - #curl 'https://api-testnet.bybit.com/v2/public/kline/list?symbol=BTCUSD&interval=5&from=1672225349&limit=3' + """ + Fetch BTCUSD OHLC data from Bybit + Returns: + str: 5min OHLC data in JSON format. + """ base_url = 'https://api.bybit.com/v2/public/kline/list?symbol=BTCUSD&interval=5&from=' current_unixtime = int(time.time()) last_minute = math.floor(current_unixtime / 60) @@ -209,18 +242,23 @@ def fetch_bybit(): return empty_json def write_dict_to_database(in_dict, connection): + """ + Writes given dict to given database. + Arguments: dict, db.connection() + Uses shared global database_lock. + """ cursor = connection.cursor() # use placeholders for the values in the insert statement insert_query = "insert into ohlc (exchange, timestamp, open, high, low, close, volume_quote, volume_base, trades) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" - values = (in_dict['exchange'], - in_dict['timestamp'], - in_dict['open'], - in_dict['high'], - in_dict['low'], - in_dict['close'], - in_dict['volume_quote'], - in_dict['volume_base'], + values = (in_dict['exchange'], + in_dict['timestamp'], + in_dict['open'], + in_dict['high'], + in_dict['low'], + in_dict['close'], + in_dict['volume_quote'], + in_dict['volume_base'], in_dict['trades']) ## apply lock while writing to database with database_lock: @@ -228,19 +266,27 @@ def write_dict_to_database(in_dict, connection): connection.commit() def get_the_data(): - #cursor = db.cursor() - while True: - db = sqlite3.connect(database) - write_dict_to_database(json.loads(fetch_kraken()), db) - write_dict_to_database(json.loads(fetch_bitfinex()), db) - write_dict_to_database(json.loads(fetch_bitstamp()), db) - write_dict_to_database(json.loads(fetch_gemini()), db) - write_dict_to_database(json.loads(fetch_bybit()), db) - db.close() + """ + Creates infinite While True loop to fetch OHLC data and save it to database. + """ + while True: + ohlc_db = sqlite3.connect(DATABASE) + write_dict_to_database(json.loads(fetch_kraken()), ohlc_db) + write_dict_to_database(json.loads(fetch_bitfinex()), ohlc_db) + write_dict_to_database(json.loads(fetch_bitstamp()), ohlc_db) + write_dict_to_database(json.loads(fetch_gemini()), ohlc_db) + write_dict_to_database(json.loads(fetch_bybit()), ohlc_db) + ohlc_db.close() print("fetches done at", time.time(), "sleeping now for 290") time.sleep(290) def check_auth(text, signature): + """ + Check signatures against known public keys + Arguments: text, signature + Reads: Global public user_publickeys dict. + Returns: True / False + """ ## Make bytes-object from given signature sig_bytes = bytes.fromhex(signature) ## We will iterate over all user keys to determ who is we are talking to and should they have access @@ -248,9 +294,9 @@ def check_auth(text, signature): ## Create bytes-object from the public in 'value' variable ## and use it to create VerifyingKey (vk) public_key_bytes = bytes.fromhex(value) - vk = ecdsa.VerifyingKey.from_string(public_key_bytes, curve=ecdsa.SECP256k1) + verifying_key = ecdsa.VerifyingKey.from_string(public_key_bytes, curve=ecdsa.SECP256k1) try: - vk.verify(sig_bytes, bytes(text, 'utf-8')) + verifying_key.verify(sig_bytes, bytes(text, 'utf-8')) print('user is', key) return True @@ -259,6 +305,10 @@ def check_auth(text, signature): @app.route('/') def get_data(): + """ + Serve the data from the database. Limit the responses by given timestamp. + The pretty thing is under consideration... + """ # Get the time (t) argument from the url" query_timestamp = request.args.get('t') # Should we make output pretty for curl users? @@ -270,39 +320,41 @@ def get_data(): if not check_auth(get_url, signature): return 'Access denied! Check your keys, maybe.', 403 - database_lock.acquire() - db = sqlite3.connect(database) - if query_timestamp: - rows = db.execute("SELECT exchange, timestamp, open, high, low, close FROM ohlc WHERE timestamp > ? ORDER BY timestamp", (query_timestamp,)).fetchall() - else: - rows = db.execute('SELECT exchange, timestamp, open, high, low, close FROM ohlc ORDER BY timestamp').fetchall() - query_timestamp = 0 - database_lock.release() + with database_lock: + btc_db = sqlite3.connect(DATABASE) + if query_timestamp: + rows = btc_db.execute("SELECT exchange, timestamp, open, high, low, close FROM ohlc WHERE timestamp > ? ORDER BY timestamp", (query_timestamp,)).fetchall() + else: + rows = btc_db.execute('SELECT exchange, timestamp, open, high, low, close FROM ohlc ORDER BY timestamp').fetchall() + query_timestamp = 0 - data = { - "timestamp": time.time(), - "rows": rows - } + data = { + "timestamp": time.time(), + "rows": rows + } # make sha256 checksum and append it to the data object data_shasum = sha256(json.dumps(data).encode('utf-8')).hexdigest() updated_data = {"shasum": data_shasum} updated_data.update(data) data = updated_data - + # sign the response signature = server_private_key.sign(json.dumps(data).encode('utf-8')) signature_hex = binascii.hexlify(signature).decode('utf-8') data['signature'] = signature_hex if query_pretty: - response = json.dumps(data, indent=2, separators=(';\n', ' :')) + response = json.dumps(data, indent=2, separators=(';\n', ' :')) else: response = json.dumps(data) return response, 200, {'Content-Type': 'application/json'} @app.route('/serverkey') def give_serverkey(): + """ + Serve the public keys of this instace to the world. + """ ## This endpoint also under Authentication? signature = request.headers.get('auth') get_url = request.url @@ -313,12 +365,12 @@ def give_serverkey(): if __name__ == '__main__': # Make sanity checks for the database - Checkthedatabase() + check_database() # Start the data fetching backend process fetch_thread = threading.Thread(target=get_the_data) fetch_thread.daemon = True fetch_thread.start() - + # Start the Flask app app.run()