peterlu14
472 lines
17 KiB
Python
472 lines
17 KiB
Python
import base64
|
|
import re
|
|
import zlib
|
|
import sys
|
|
import gc
|
|
|
|
from json import loads as json_parse, dumps as _json_stringify
|
|
from typing import Dict, Optional, Any
|
|
from datetime import datetime
|
|
import threading
|
|
|
|
from biopro.recording.file import RecordingMetaFile, RecordingFile, RecordingMini
|
|
|
|
from biopro.util.json import JSON_OBJECT
|
|
from .socket import ServerThread
|
|
|
|
import psycopg2
|
|
# from psycopg2.pool import ThreadedConnectionPool
|
|
from multiprocessing import Process, Queue
|
|
|
|
import biopro.server._identify
|
|
|
|
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
_RUNTIME_COMPILE = False
|
|
|
|
|
|
def json_stringify(o: Any) -> str:
|
|
return _json_stringify(o, separators=(',', ':'))
|
|
|
|
|
|
# noinspection PyUnusedLocal
|
|
class DataBaseProcess(Process):
|
|
"""
|
|
connect to DB and send sql command
|
|
"""
|
|
|
|
def __init__(self, client_id: str, log_verbose, queue_db, queue_ds, queue_error, name = 'Database-Process'):
|
|
super(DataBaseProcess, self).__init__(name = name)
|
|
# super().__init__('DATABASE-' + client_id, MAGENTA)
|
|
|
|
self._db_host = '127.0.0.1'
|
|
self._db_name = 'postgres'
|
|
self._db_user = 'biopro'
|
|
self._db_password = 'BioProControlBox'
|
|
|
|
self._api_host = biopro.server._identify.API_SERVER_HOST
|
|
self._api_port = biopro.server._identify.API_SERVER_PORT
|
|
self._api_token = None
|
|
self._api_header = None
|
|
self.log_verbose = log_verbose
|
|
self._executor = ThreadPoolExecutor(max_workers=4)
|
|
|
|
self._psql_multi_thread = None
|
|
self._psql_conn_pool = None
|
|
self._psql_conn = None
|
|
|
|
self._queue_db = queue_db
|
|
self._queue_ds = queue_ds
|
|
self._queue_error = queue_error
|
|
|
|
self._is_close = False
|
|
|
|
@property
|
|
def db_host(self) -> str:
|
|
return self._db_host
|
|
|
|
@property
|
|
def api_host(self) -> str:
|
|
return self._api_host
|
|
|
|
@property
|
|
def api_port(self) -> int:
|
|
return self._api_port
|
|
|
|
@property
|
|
def api_token(self) -> str:
|
|
return self._api_token
|
|
|
|
def run(self) -> None:
|
|
self.setup()
|
|
self.routine()
|
|
|
|
def setup(self) -> None:
|
|
self.log_verbose('DataBaseProcess setup')
|
|
conn_info = "host={0} user={1} dbname={2} password={3}".format(self._db_host, self._db_user, self._db_name, self._db_password)
|
|
try:
|
|
self._psql_conn = psycopg2.connect(conn_info)
|
|
except:
|
|
self.log_verbose('can not connect to database')
|
|
else:
|
|
self.log_verbose('connect to DB ', self._db_host, self._db_name)
|
|
self._is_close = False
|
|
|
|
def routine(self) -> None:
|
|
while self._is_close is False:
|
|
# request = [ function_name, *args]
|
|
request = None
|
|
try:
|
|
request = self._queue_db.get()
|
|
except KeyboardInterrupt:
|
|
self._is_close = True
|
|
if request is not None:
|
|
func_name = request[0]
|
|
args = request[1:]
|
|
if request[0] != 'data_meta_write':
|
|
try:
|
|
self._executor.submit(self.solve_for, func_name, args)
|
|
except RuntimeError as e:
|
|
self.log_verbose('can not submit database sql:', func_name, e)
|
|
else:
|
|
try:
|
|
self.solve_for(func_name, args)
|
|
except RuntimeError as e:
|
|
self.log_verbose('can not submit database sql:', func_name, e)
|
|
del request
|
|
return None
|
|
|
|
def shutdown(self) -> None:
|
|
if self._psql_conn is not None:
|
|
try:
|
|
self._psql_conn.close()
|
|
except:
|
|
self.log_verbose('can not close database')
|
|
else:
|
|
self.log_verbose('closed DB ', self._db_host, self._db_name)
|
|
self._is_close = True
|
|
|
|
def put_queue(self, request: list) -> None:
|
|
self._queue_db.put(request)
|
|
del request
|
|
return
|
|
|
|
def get_queue(self, device_id) -> None:
|
|
task = None
|
|
try:
|
|
task = self._queue_ds[int(device_id)].get()
|
|
except:
|
|
# Handle empty queue here
|
|
pass
|
|
else:
|
|
# Handle task here and call q.task_done()
|
|
pass
|
|
return task
|
|
|
|
def solve_for(self, name: str, args: list):
|
|
do = f"{name}"
|
|
if callable(getattr(self, do)):
|
|
getattr(self, do)(*args)
|
|
return None
|
|
|
|
def data_meta_write(self, _data: JSON_OBJECT, device_id, _uuid = None, _id = None):
|
|
if _id > 0:
|
|
sql_str = 'UPDATE "public"."recording_data_metas" SET '
|
|
sql_set = []
|
|
for item in _data.keys():
|
|
sql_str = sql_str + str(item) + ' = %s, '
|
|
if isinstance(_data[item], dict):
|
|
sql_set.append(json_stringify(_data[item]))
|
|
else:
|
|
sql_set.append(_data[item])
|
|
|
|
sql_str = sql_str[0:-2] + ' WHERE (id = %s)'
|
|
sql_set.append(_id)
|
|
sql_cursor = self._psql_conn.cursor()
|
|
try:
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
except:
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
self._queue_error.put(device_id)
|
|
finally:
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
return True
|
|
|
|
if _uuid is not None:
|
|
sql_str = 'SELECT id FROM "public"."recording_data_metas" WHERE uuid = %s'
|
|
sql_set = [_uuid, ]
|
|
sql_cursor = self._psql_conn.cursor()
|
|
sql_id = None
|
|
|
|
# try:
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
sql_id = sql_cursor.fetchone()
|
|
# except RuntimeError as e:
|
|
# print(e)
|
|
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
if sql_id is None:
|
|
sql_str = 'INSERT INTO "public"."recording_data_metas" ('
|
|
sql_set = []
|
|
for item in _data.keys():
|
|
sql_str = sql_str + str(item) + ', '
|
|
if isinstance(_data[item], dict):
|
|
sql_set.append(json_stringify(_data[item]))
|
|
else:
|
|
sql_set.append(_data[item])
|
|
|
|
sql_str = sql_str[0:-2] + ') VALUES ('
|
|
for item in _data.keys():
|
|
sql_str = sql_str + '%s, '
|
|
|
|
sql_str = sql_str[0:-2] + ') RETURNING id'
|
|
sql_cursor = self._psql_conn.cursor()
|
|
|
|
try:
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
except:
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
self._queue_error.put(device_id)
|
|
finally:
|
|
self._queue_ds[int(device_id)].put(['update_meta_id', _data['uuid'], int(sql_cursor.fetchone()[0])])
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
else:
|
|
sql_str = 'UPDATE "public"."recording_data_metas" SET '
|
|
sql_set = []
|
|
for item in _data.keys():
|
|
sql_str = sql_str + str(item) + ' = %s, '
|
|
if isinstance(_data[item], dict):
|
|
sql_set.append(json_stringify(_data[item]))
|
|
else:
|
|
sql_set.append(_data[item])
|
|
|
|
sql_str = sql_str[0:-2] + ' WHERE (id = %s)'
|
|
if isinstance(sql_id, tuple):
|
|
sql_id = sql_id[0]
|
|
sql_set.append(sql_id)
|
|
sql_cursor = self._psql_conn.cursor()
|
|
|
|
try:
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
except:
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
self._queue_error.put(device_id)
|
|
finally:
|
|
self._queue_ds[int(device_id)].put(['update_meta_id', _data['uuid'], int(sql_id)])
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
# _conn.close()
|
|
return True
|
|
|
|
# _conn.close()
|
|
return None
|
|
|
|
def data_meta_get(self, _path = None, _id = None):
|
|
if _id is not None:
|
|
sql_str = 'SELECT * FROM "public"."recording_data_metas" WHERE id = %s'
|
|
sql_set = [str(_id), ]
|
|
sql_cursor = self._psql_conn.cursor()
|
|
|
|
ret = None
|
|
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
ret = sql_cursor.fetchone()
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
|
|
return ret
|
|
|
|
elif _path is not None:
|
|
sql_str = 'SELECT * FROM "public"."recording_data_metas" WHERE path = %s'
|
|
sql_set = [str(_path), ]
|
|
sql_cursor = self._psql_conn.cursor()
|
|
|
|
ret = None
|
|
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
ret = sql_cursor.fetchone()
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
|
|
return ret
|
|
|
|
# _conn.close()
|
|
return None
|
|
|
|
def data_meta_id_get_by_path(self, _path = None):
|
|
ret = None
|
|
if _path is not None:
|
|
sql_str = 'SELECT * FROM "public"."recording_data_metas" WHERE path = %s'
|
|
sql_set = [str(_path), ]
|
|
sql_cursor = self._psql_conn.cursor()
|
|
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
ret = 0
|
|
try:
|
|
ret = int(sql_cursor.fetchone()[0])
|
|
except:
|
|
return None
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
return ret
|
|
|
|
# _conn.close()
|
|
return None
|
|
|
|
def data_raw_create(self, _data_dict, _channel_list, device_id):
|
|
|
|
for _channel in _channel_list:
|
|
sql_str = 'INSERT INTO "public"."' + str(_channel) + '_recording_data_raws" ('
|
|
sql_set = []
|
|
for item in _data_dict[_channel].keys():
|
|
sql_str = sql_str + str(item) + ', '
|
|
if isinstance(_data_dict[_channel][item], dict):
|
|
sql_set.append(json_stringify(_data_dict[_channel][item]))
|
|
else:
|
|
sql_set.append(str(_data_dict[_channel][item]))
|
|
|
|
sql_str = sql_str[0:-2] + ') VALUES ('
|
|
for item in _data_dict[_channel].keys():
|
|
sql_str = sql_str + '%s, '
|
|
|
|
sql_str = sql_str[0:-2] + ') RETURNING id'
|
|
sql_cursor = self._psql_conn.cursor()
|
|
|
|
try:
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
except RuntimeError as e:
|
|
print('data_raw_create error', e)
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
self._queue_error.put(device_id)
|
|
finally:
|
|
self._queue_ds[int(device_id)].put(['update_raw_dict_id', _data_dict[_channel]['uuid'], _channel, int(sql_cursor.fetchone()[0])])
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
return True
|
|
|
|
def data_raw_update(self, _id, _channel, _data: JSON_OBJECT):
|
|
sql_str = 'UPDATE "public"."' + str(_channel) + '_recording_data_raws" SET '
|
|
sql_set = []
|
|
for item in _data.keys():
|
|
sql_str = sql_str + str(item) + ' = %s, '
|
|
if isinstance(_data[item], dict):
|
|
sql_set.append(json_stringify(_data[item]))
|
|
else:
|
|
sql_set.append(_data[item])
|
|
|
|
sql_str = sql_str[0:-2] + ' WHERE (id = %s)'
|
|
sql_set.append(_id)
|
|
|
|
sql_cursor = self._psql_conn.cursor()
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
return None
|
|
|
|
def data_raw_recording(self, _id_dict, _channel_list, _data_dict):
|
|
for _channel in _channel_list:
|
|
compressed_value = _data_dict[_channel] + '"***"'
|
|
# compressed_value = bytes(_data_dict[_channel],'utf-8')
|
|
# compressed_value = zlib.compress(compressed_value)
|
|
# compressed_value = base64.b64encode(compressed_value).decode() + '"***"'
|
|
# compressed_value = _data + '"***"'
|
|
sql_str = 'UPDATE "public"."' + str(_channel) + '_recording_data_raws" SET data = concat(data, %s) where id = %s'
|
|
sql_set = [compressed_value, _id_dict[_channel]]
|
|
|
|
sql_cursor = self._psql_conn.cursor()
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
del _data_dict[_channel]
|
|
del compressed_value
|
|
gc.collect()
|
|
# _conn.close()
|
|
return None
|
|
|
|
def data_mini_create(self, _data_dict, _channel_list, _scale, device_id):
|
|
for _channel in _channel_list:
|
|
sql_str = 'INSERT INTO "public"."' + str(_channel) + '_recording_data_minis" ('
|
|
sql_set = []
|
|
for item in _data_dict[_channel].keys():
|
|
sql_str = sql_str + str(item) + ', '
|
|
if isinstance(_data_dict[_channel][item], dict):
|
|
sql_set.append(json_stringify(_data_dict[_channel][item]))
|
|
else:
|
|
sql_set.append(str(_data_dict[_channel][item]))
|
|
|
|
sql_str = sql_str[0:-2] + ') VALUES ('
|
|
for item in _data_dict[_channel].keys():
|
|
sql_str = sql_str + '%s, '
|
|
|
|
sql_str = sql_str[0:-2] + ') RETURNING id'
|
|
sql_cursor = self._psql_conn.cursor()
|
|
|
|
try:
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
except RuntimeError as e:
|
|
print('data_mini_create error', e)
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
self._queue_error.put(device_id)
|
|
finally:
|
|
self._queue_ds[int(device_id)].put(['update_mini_dict_id', _data_dict[_channel]['uuid'], _scale, _channel, int(sql_cursor.fetchone()[0])])
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
# _conn.close()
|
|
return True
|
|
|
|
def data_mini_recording(self, _id_dict, _channel_list, _start_time_dict, _data_mean_dict):
|
|
for _channel in _channel_list:
|
|
data = _data_mean_dict.get(_channel)
|
|
if data is not None:
|
|
if len(data) != 0:
|
|
str_mean = [str(int) for int in _data_mean_dict[_channel]]
|
|
data_mean = _start_time_dict[_channel] + ' ' + ' '.join(str_mean)
|
|
# data_random = _start_time_dict[_channel] + ' ' + ' '.join(_data_random_dict[_channel])
|
|
# data_bar = _start_time_dict[_channel] + ' ' + ' '.join(_data_bar_dict[_channel])
|
|
|
|
compressed_mean = data_mean + '"***"'
|
|
|
|
# compressed_mean = bytes(data_mean,'utf-8')
|
|
# compressed_mean = zlib.compress(compressed_mean)
|
|
# compressed_mean = base64.b64encode(compressed_mean).decode() + '"***"'
|
|
|
|
# compressed_random = bytes(data_bar,'utf-8')
|
|
# compressed_random = zlib.compress(compressed_random)
|
|
# compressed_random = base64.b64encode(compressed_random).decode() + '"***"'
|
|
|
|
# compressed_random = bytes(data_random,'utf-8')
|
|
# compressed_random = zlib.compress(compressed_random)
|
|
# compressed_random = base64.b64encode(compressed_random).decode() + '"***"'
|
|
sql_str = 'UPDATE "public"."' + str(_channel) + '_recording_data_minis" SET data_mean = concat(data_mean, %s) where id = %s'
|
|
sql_set = [compressed_mean, _id_dict[_channel]]
|
|
|
|
sql_cursor = self._psql_conn.cursor()
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
del str_mean, _start_time_dict[_channel], _data_mean_dict[_channel], data_mean, compressed_mean
|
|
gc.collect()
|
|
|
|
return True
|
|
|
|
def data_mini_update(self, _id, _channel, _data: JSON_OBJECT):
|
|
sql_str = 'UPDATE "public"."' + str(_channel) + '_recording_data_minis" SET '
|
|
sql_set = []
|
|
for item in _data.keys():
|
|
sql_str = sql_str + str(item) + ' = %s, '
|
|
if isinstance(_data[item], dict):
|
|
sql_set.append(json_stringify(_data[item]))
|
|
else:
|
|
sql_set.append(_data[item])
|
|
|
|
sql_str = sql_str[0:-2] + ' WHERE (id = %s)'
|
|
sql_set.append(_id)
|
|
|
|
sql_cursor = self._psql_conn.cursor()
|
|
sql_cursor.execute(sql_str, sql_set)
|
|
self._psql_conn.commit()
|
|
sql_cursor.close()
|
|
|
|
return True
|