440 lines
14 KiB
Python
Executable File
440 lines
14 KiB
Python
Executable File
#!/usr/bin/python3
|
|
"""
|
|
Fetch BTCUSD OHLC data from few market places and serve it forward with simple json api.
|
|
|
|
Creates: ./btc_ohlc.db
|
|
serves: localhost:5000/[t] and /serverkey
|
|
Authentication via auth header with signatures
|
|
"""
|
|
|
|
import math
|
|
import json
|
|
import os
|
|
import time
|
|
import sys
|
|
import sqlite3
|
|
import binascii
|
|
import threading
|
|
from hashlib import sha256
|
|
import requests
|
|
import ecdsa
|
|
import krakenex
|
|
from flask import Flask, jsonify, request
|
|
|
|
# from Cryptodome.Cipher import AES
|
|
|
|
DATABASE = "btc_ohlc.db"
|
|
KEYSFILE = "userkeys.json"
|
|
app = Flask(__name__)
|
|
|
|
## Generate the ECDSA keys for this instance
|
|
print("Generating ECDSA keys for this instance... just wait a bit...")
|
|
server_private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1)
|
|
server_public_key = server_private_key.get_verifying_key()
|
|
# We need the hexadecimal form for sharing over http/json
|
|
server_public_key_hex = binascii.hexlify(server_public_key.to_string()).decode("utf-8")
|
|
|
|
|
|
# Empty response json
|
|
empty_dict = {
|
|
"exchange": "",
|
|
"timestamp": 0,
|
|
"open": 0,
|
|
"high": 0,
|
|
"low": 0,
|
|
"close": 0,
|
|
"volume_quote": 0,
|
|
"volume_base": 0,
|
|
"trades": 0,
|
|
}
|
|
empty_json = json.dumps(empty_dict)
|
|
|
|
|
|
def read_keys():
|
|
"""
|
|
Reads the declared KEYSFILE and returns "user_publickeys" from it.
|
|
Returns: json object
|
|
"""
|
|
empty_userkeys = {
|
|
"user_publickeys": {
|
|
"user_name": "user_ecdsa_public key"
|
|
}
|
|
}
|
|
# test if the file is there, we could make one if there is non
|
|
try:
|
|
# ascii is subset of UTF-8, this should be quite safe...
|
|
with open(KEYSFILE, "r", encoding='utf-8') as cfile:
|
|
user_keys = json.load(cfile)
|
|
|
|
if 'user_publickeys' not in user_keys:
|
|
print('Missing required property "user_publickeys" in config file')
|
|
sys.exit(1)
|
|
if user_keys == empty_userkeys:
|
|
print('Your userkeys.json file seems to not filled. Please insert your key there.')
|
|
sys.exit(1)
|
|
|
|
return user_keys["user_publickeys"]
|
|
except FileNotFoundError:
|
|
with open(KEYSFILE, "w", encoding='utf-8') as nfile:
|
|
print('You seem be lacking the users key file, created example for you. Exiting')
|
|
json.dump(empty_userkeys, nfile, indent=2)
|
|
sys.exit(1)
|
|
|
|
def check_database():
|
|
"""
|
|
Check the database for the 'ohlc' table.
|
|
If the database file or the table does not exist, create them.
|
|
"""
|
|
if not os.path.exists(DATABASE):
|
|
new_db = sqlite3.connect(DATABASE)
|
|
new_db.execute(
|
|
"""\
|
|
CREATE TABLE ohlc (
|
|
id INTEGER PRIMARY KEY,
|
|
exchange TEXT NOT NULL,
|
|
timestamp INTEGER NOT NULL,
|
|
open REAL NOT NULL,
|
|
high REAL NOT NULL,
|
|
low REAL NOT NULL,
|
|
close REAL NOT NULL,
|
|
volume_quote REAL NOT NULL,
|
|
volume_base REAL NOT NULL,
|
|
trades INTEGER NOT NULL )"""
|
|
)
|
|
|
|
new_db.commit()
|
|
new_db.close()
|
|
|
|
new_db = sqlite3.connect(DATABASE)
|
|
# Check if the table exists
|
|
|
|
table_exists = False
|
|
cursor = new_db.execute("PRAGMA table_info(ohlc)")
|
|
for row in cursor:
|
|
table_exists = True
|
|
|
|
# Create the table if it doesn't exist
|
|
if not table_exists:
|
|
new_db.execute(
|
|
"""\
|
|
CREATE TABLE ohlc (
|
|
id INTEGER PRIMARY KEY,
|
|
exchange TEXT NOT NULL,
|
|
timestamp INTEGER NOT NULL,
|
|
open REAL NOT NULL,
|
|
high REAL NOT NULL,
|
|
low REAL NOT NULL,
|
|
close REAL NOT NULL,
|
|
volume_quote REAL NOT NULL,
|
|
volume_base REAL NOT NULL,
|
|
trades INTEGER NOT NULL )"""
|
|
)
|
|
new_db.commit()
|
|
|
|
|
|
def fetch_kraken():
|
|
"""
|
|
Fetch BTCUSD OHLC data from Kraken in json.
|
|
Returns:
|
|
str: 5min OHLC data in JSON format.
|
|
"""
|
|
kraken = krakenex.API()
|
|
|
|
response = kraken.query_public("OHLC", {"pair": "BTCUSD", "interval": 240})
|
|
ohlc_data = response["result"]["XXBTZUSD"]
|
|
|
|
candle_stick_data = {
|
|
"exchange": "kraken",
|
|
"timestamp": ohlc_data[1][0],
|
|
"open": ohlc_data[0][1],
|
|
"high": max(item[2] for item in ohlc_data),
|
|
"low": min(item[3] for item in ohlc_data),
|
|
"close": ohlc_data[-1][4],
|
|
"volume_quote": sum(float(item[5]) for item in ohlc_data),
|
|
"volume_base": sum(float(item[6]) for item in ohlc_data),
|
|
"trades": sum(item[7] for item in ohlc_data),
|
|
}
|
|
|
|
kraken_json = json.dumps(candle_stick_data, indent=2)
|
|
return kraken_json
|
|
|
|
|
|
def fetch_bitstamp():
|
|
"""
|
|
Fetch Bitstamp data ja serve it as json.
|
|
Returns:
|
|
str: 5min OHLC data in JSON format.
|
|
"""
|
|
response = requests.get(
|
|
"https://www.bitstamp.net/api/v2/ohlc/btcusd/?step=300&limit=1"
|
|
)
|
|
|
|
if response.status_code == 200: # check if the request was successful
|
|
bitstamp_data = response.json()
|
|
ohlc_data = bitstamp_data["data"]["ohlc"]
|
|
|
|
# the bitstamp api gives timestamp with decimals to show less than second accuracy.
|
|
bitstamp_timestamp = int(math.floor(float(ohlc_data[0]["timestamp"])))
|
|
|
|
candle_stick_data = {
|
|
"exchange": "bitstamp",
|
|
"timestamp": bitstamp_timestamp,
|
|
"open": float(ohlc_data[0]["open"]),
|
|
"high": float(ohlc_data[0]["high"]),
|
|
"low": float(ohlc_data[0]["low"]),
|
|
"close": float(ohlc_data[0]["close"]),
|
|
"volume_quote": float(ohlc_data[0]["volume"]),
|
|
"volume_base": 0, # not provided by Bitstamp API
|
|
"trades": 0, # not provided by Bitstamp API
|
|
}
|
|
|
|
bitstamp_json = json.dumps(candle_stick_data, indent=2)
|
|
return bitstamp_json
|
|
# if we get any thing else than http/200
|
|
print(f"Error fetching data from Bitstamp API: {response.status_code}")
|
|
return empty_json
|
|
|
|
|
|
def fetch_bitfinex():
|
|
"""
|
|
Bitfinex
|
|
Returns:
|
|
str: 5min OHLC data in JSON format.
|
|
"""
|
|
response = requests.get(
|
|
"https://api-pub.bitfinex.com/v2/candles/trade:5m:tBTCUSD/last"
|
|
)
|
|
|
|
if response.status_code == 200: # check if the request was successful
|
|
ohlc_data = response.json()
|
|
candle_stick_data = {
|
|
"exchange": "bitfinex",
|
|
"timestamp": ohlc_data[0],
|
|
"open": ohlc_data[1],
|
|
"high": ohlc_data[2],
|
|
"low": ohlc_data[3],
|
|
"close": ohlc_data[4],
|
|
"volume_quote": ohlc_data[5],
|
|
"volume_base": 0, # not provided by Bitfinex API
|
|
"trades": 0, # not provided by Bitfinex API
|
|
}
|
|
|
|
bitfinex_json = json.dumps(candle_stick_data, indent=2)
|
|
return bitfinex_json
|
|
# if we get any thing else than http/200
|
|
print(f"Error fetching data from Bitfinex API: {response.status_code}")
|
|
return empty_json
|
|
|
|
|
|
def fetch_gemini():
|
|
"""
|
|
Fetch BTCUSD OHLC data from Gemini
|
|
Returns:
|
|
str: 5min OHLC data in JSON format.
|
|
"""
|
|
response = requests.get("https://api.gemini.com/v2/candles/btcusd/5m")
|
|
|
|
if response.status_code == 200: # check if the request was successful
|
|
gemini_ohlc = response.json()
|
|
candle_stick_data = {
|
|
"exchange": "gemini",
|
|
"timestamp": gemini_ohlc[0][0],
|
|
"open": gemini_ohlc[0][1],
|
|
"high": gemini_ohlc[0][2],
|
|
"low": gemini_ohlc[0][3],
|
|
"close": gemini_ohlc[0][4],
|
|
"volume_quote": 0, # not provided by Gemini API
|
|
"volume_base": gemini_ohlc[0][5],
|
|
"trades": 0, # not provided by Gemini API
|
|
}
|
|
gemini_json = json.dumps(candle_stick_data, indent=2)
|
|
return gemini_json
|
|
# if we get any thing else than http/200
|
|
print(f"Error fetching data from Gemini API: {response.status_code}")
|
|
return empty_json
|
|
|
|
|
|
def fetch_bybit():
|
|
"""
|
|
Fetch BTCUSD OHLC data from Bybit
|
|
Returns:
|
|
str: 5min OHLC data in JSON format.
|
|
"""
|
|
base_url = (
|
|
"https://api.bybit.com/v2/public/kline/list?symbol=BTCUSD&interval=5&from="
|
|
)
|
|
current_unixtime = int(time.time())
|
|
last_minute = math.floor(current_unixtime / 60)
|
|
last_minute_unixtime = str(last_minute * 60 - 300)
|
|
query_url = "".join([base_url, last_minute_unixtime])
|
|
response = requests.get(query_url)
|
|
|
|
if response.status_code == 200: # check if the request was successful
|
|
bybit_ohlc = response.json()
|
|
candle_stick_data = {
|
|
"exchange": "bybit",
|
|
"timestamp": bybit_ohlc["result"][0]["open_time"],
|
|
"open": bybit_ohlc["result"][0]["open"],
|
|
"high": bybit_ohlc["result"][0]["high"],
|
|
"low": bybit_ohlc["result"][0]["low"],
|
|
"close": bybit_ohlc["result"][0]["close"],
|
|
"volume_quote": bybit_ohlc["result"][0]["volume"],
|
|
"volume_base": bybit_ohlc["result"][0]["turnover"],
|
|
"trades": 0,
|
|
}
|
|
bybit_json = json.dumps(candle_stick_data, indent=2)
|
|
return bybit_json
|
|
# if we get any thing else than http/200
|
|
print(f"Error fetching data from Bybit API: {response.status_code}")
|
|
return empty_json
|
|
|
|
|
|
def write_dict_to_database(in_dict, connection):
|
|
"""
|
|
Writes given dict to given database.
|
|
Arguments: dict, db.connection()
|
|
Uses shared global database_lock.
|
|
"""
|
|
cursor = connection.cursor()
|
|
# use placeholders for the values in the insert statement
|
|
insert_query = "insert into ohlc (exchange, timestamp, open, high, low, close, volume_quote, volume_base, trades) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
|
|
values = (
|
|
in_dict["exchange"],
|
|
in_dict["timestamp"],
|
|
in_dict["open"],
|
|
in_dict["high"],
|
|
in_dict["low"],
|
|
in_dict["close"],
|
|
in_dict["volume_quote"],
|
|
in_dict["volume_base"],
|
|
in_dict["trades"],
|
|
)
|
|
## apply lock while writing to database
|
|
with database_lock:
|
|
cursor.execute(insert_query, values)
|
|
connection.commit()
|
|
|
|
|
|
def get_the_data():
|
|
"""
|
|
Creates infinite While True loop to fetch OHLC data and save it to database.
|
|
"""
|
|
while True:
|
|
ohlc_db = sqlite3.connect(DATABASE)
|
|
write_dict_to_database(json.loads(fetch_kraken()), ohlc_db)
|
|
write_dict_to_database(json.loads(fetch_bitfinex()), ohlc_db)
|
|
write_dict_to_database(json.loads(fetch_bitstamp()), ohlc_db)
|
|
write_dict_to_database(json.loads(fetch_gemini()), ohlc_db)
|
|
write_dict_to_database(json.loads(fetch_bybit()), ohlc_db)
|
|
ohlc_db.close()
|
|
print("fetches done at", time.time(), "sleeping now for 290")
|
|
time.sleep(290)
|
|
|
|
|
|
def check_auth(text, signature):
|
|
"""
|
|
Check signatures against known public keys
|
|
Arguments: text, signature
|
|
Reads: Global public user_publickeys dict.
|
|
Returns: True / False
|
|
"""
|
|
## Make bytes-object from given signature
|
|
sig_bytes = bytes.fromhex(signature)
|
|
## We will iterate over all user keys to determ who is we are talking to and should they have access
|
|
for key, value in user_publickeys.items():
|
|
## Create bytes-object from the public in 'value' variable
|
|
## and use it to create VerifyingKey (vk)
|
|
public_key_bytes = bytes.fromhex(value)
|
|
verifying_key = ecdsa.VerifyingKey.from_string(
|
|
public_key_bytes, curve=ecdsa.SECP256k1
|
|
)
|
|
try:
|
|
verifying_key.verify(sig_bytes, bytes(text, "utf-8"))
|
|
print("user is", key)
|
|
|
|
return True
|
|
except ecdsa.BadSignatureError:
|
|
return False
|
|
|
|
|
|
@app.route("/")
|
|
def get_data():
|
|
"""
|
|
Serve the data from the database. Limit the responses by given timestamp.
|
|
The pretty thing is under consideration...
|
|
"""
|
|
# Get the time (t) argument from the url"
|
|
query_timestamp = request.args.get("t")
|
|
# Should we make output pretty for curl users?
|
|
query_pretty = request.args.get("pretty")
|
|
|
|
# Authentication header, signatured the query with private key of a user
|
|
signature = request.headers.get("auth")
|
|
get_url = request.url
|
|
if not check_auth(get_url, signature):
|
|
return "Access denied! Check your keys, maybe.", 403
|
|
|
|
with database_lock:
|
|
btc_db = sqlite3.connect(DATABASE)
|
|
if query_timestamp:
|
|
rows = btc_db.execute(
|
|
"SELECT exchange, timestamp, open, high, low, close FROM ohlc WHERE timestamp > ? ORDER BY timestamp",
|
|
(query_timestamp,),
|
|
).fetchall()
|
|
else:
|
|
rows = btc_db.execute(
|
|
"SELECT exchange, timestamp, open, high, low, close FROM ohlc ORDER BY timestamp"
|
|
).fetchall()
|
|
query_timestamp = 0
|
|
|
|
data = {"timestamp": time.time(), "rows": rows}
|
|
|
|
# make sha256 checksum and append it to the data object
|
|
data_shasum = sha256(json.dumps(data).encode("utf-8")).hexdigest()
|
|
updated_data = {"shasum": data_shasum}
|
|
updated_data.update(data)
|
|
data = updated_data
|
|
|
|
# sign the response
|
|
signature = server_private_key.sign(json.dumps(data).encode("utf-8"))
|
|
signature_hex = binascii.hexlify(signature).decode("utf-8")
|
|
data["signature"] = signature_hex
|
|
|
|
if query_pretty:
|
|
response = json.dumps(data, indent=2, separators=(";\n", " :"))
|
|
else:
|
|
response = json.dumps(data)
|
|
return response, 200, {"Content-Type": "application/json"}
|
|
|
|
|
|
@app.route("/serverkey")
|
|
def give_serverkey():
|
|
"""
|
|
Serve the public keys of this instace to the world.
|
|
"""
|
|
## This endpoint also under Authentication?
|
|
signature = request.headers.get("auth")
|
|
get_url = request.url
|
|
if not check_auth(get_url, signature):
|
|
return "Access denied! Check your keys, maybe.", 403
|
|
|
|
return jsonify({"public_key": server_public_key_hex})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Make sanity checks for the database
|
|
check_database()
|
|
database_lock = threading.Lock()
|
|
|
|
# Get the users public keys
|
|
user_publickeys = read_keys()
|
|
|
|
# Start the data fetching backend process§
|
|
fetch_thread = threading.Thread(target=get_the_data)
|
|
fetch_thread.daemon = True
|
|
fetch_thread.start()
|
|
|
|
# Start the Flask app
|
|
app.run()
|