Compare commits

..

7 Commits

Author SHA1 Message Date
peterlu14 c79f2a0e92 - Add new function data_raw_recording_bytea
- Change data type from string to binary
2024-01-23 16:14:09 +08:00
peterlu14 c92b25e217 - Add new mode 2023-12-19 17:30:31 +08:00
peterlu14 2f4f257974 -update cycle 2023-12-06 15:49:06 +08:00
peterlu14 7adcdf064f -update parameter 2023-12-06 15:12:09 +08:00
peterlu14 94eb567d9b default parameter 2023-12-04 16:37:15 +08:00
peterlu14 536bc23896 -update stop update battery when device running 2023-11-21 16:28:40 +08:00
Roy 7deb709946 [update] print msg when connection fail 2023-07-17 17:37:14 +08:00
14 changed files with 79 additions and 216 deletions
-6
View File
@@ -1595,12 +1595,6 @@ class ControlAPI(metaclass=Router):
def show_device_data(self, device) -> bool:
raise NotImplementedError()
def reset_trigger(self, device) -> bool:
raise NotImplementedError()
def device_alert(self, device) -> bool:
raise NotImplementedError()
# noinspection PyAbstractClass
class ControlClient(SocketClient, ControlAPI, metaclass=SocketClientMacro(ControlAPI)):
"""Connect to controller server through the socket.
-9
View File
@@ -139,12 +139,6 @@ class DataAPI(metaclass=abc.ABCMeta):
:param device: device ID
"""
pass
@abc.abstractmethod
def reset_trigger(self, device):
"""reset trigger
"""
pass
# noinspection PyAbstractClass
@@ -210,9 +204,6 @@ class DataClient(SocketClient, DataAPI, metaclass=SocketClientMacro(DataAPI)):
def show_data(self, device: int):
self.send_command('show_data', device)
def reset_trigger(self, device):
self.send_command('reset_trigger', device)
@staticmethod
def _to_device_id(*device: Union[int, Device]) -> Tuple[int, ...]:
+17 -27
View File
@@ -494,7 +494,8 @@ class CC2650Device(Device):
@property
def battery(self) -> int:
self.update_battery_info()
if self._start_flag == False:
self.update_battery_info()
return self._battery
@property
@@ -2174,7 +2175,7 @@ class CC2650SingleMasterCentralDevice(CC2650MasterDevice, Synchronized):
try:
connect_response = self._cc2650.recv_uart(2)
except RecvTimeout:
self.log_verbose('[CC2650]', 'connect response timeout')
# self.log_verbose('[CC2650]', 'connect response timeout')
if retry < 5:
self.log_verbose('[CC2650]', 'connect retry')
continue
@@ -2182,6 +2183,7 @@ class CC2650SingleMasterCentralDevice(CC2650MasterDevice, Synchronized):
break
if connect_response is None:
self.log_verbose('[CC2650]', 'connect response timeout')
return False
pack_len = connect_response[0]
@@ -2207,7 +2209,8 @@ class CC2650SingleMasterCentralDevice(CC2650MasterDevice, Synchronized):
self.log_verbose('[CC2650]', DEVICE_CONNECTED, address_s)
return ret
self.log_verbose('[CC2650]', 'connect fail')
connect_response_hex = ''.join(format(i, '02X') for i in connect_response)
self.log_verbose('[CC2650]', 'connect fail', '0x'+connect_response_hex)
return False
@synchronized
@@ -2611,34 +2614,21 @@ class CC2650MultiMasterCentralDevice(CC2650MasterDevice, Synchronized):
master = None
# for master in self._foreach_empty_master():
# m = self._cc2650[master]
# d = self._device[master]
for master in self._foreach_empty_master():
m = self._cc2650[master]
d = self._device[master]
# if direct_connect is True and d is None:
# break
if direct_connect is True and d is None:
break
# elif response in m.found() and d is None:
# break
elif response in m.found() and d is None:
break
# else:
# if master is None:
# raise RuntimeError('cannot connect any more device')
# else:
# raise RuntimeError('rescan please')
mac_address = address_str(response.mac_address)
if mac_address == 'A4:DA:32:D4:E6:CD':
master = 4
elif mac_address == 'A4:DA:32:D4:E8:5F':
master = 5
elif mac_address == 'A4:DA:32:D4:E7:E5':
master = 6
elif mac_address == 'A4:DA:32:D4:EF:D8':
master = 7
else:
master = 4
if master is None:
raise RuntimeError('cannot connect any more device')
else:
raise RuntimeError('rescan please')
#
self.log_verbose('use', master)
+2 -21
View File
@@ -968,10 +968,11 @@ class CompletedDevice(Device):
"""
__slots__ = ('_master', '_device_id', '_device', '_library', '_context',
'_parameter', '_configuration', '_lock', '_feature_mask',
'_cache_battery', '_cache_battery_timestamp', '_coeff', '_status', '_occupied_by_project', '_alert', '_alert_time')
'_cache_battery', '_cache_battery_timestamp', '_coeff', '_status', '_occupied_by_project')
def __init__(self, master: MasterDevice, library: DeviceLibrary, device_id: int, device: Device):
"""
:param master: owner master device
:param library: device library
:param device_id: device id
@@ -1008,8 +1009,6 @@ class CompletedDevice(Device):
self._cache_battery: Optional[int] = None
self._cache_battery_timestamp: Optional[float] = None
self._coeff = b''
self._alert = False
self._alert_time = None
# device property for information
@@ -1093,22 +1092,6 @@ class CompletedDevice(Device):
def occupied_by_project(self, new_occupied_by_project):
self._occupied_by_project = new_occupied_by_project
@property
def alert(self) -> str:
return self._alert
@alert.setter
def alert(self, new_alert) -> str:
self._alert = new_alert
@property
def alert_time(self) -> str:
return self._alert_time
@alert_time.setter
def alert_time(self, new_alert_time) -> str:
self._alert_time = new_alert_time
# device parameter getter/setter
def parameters(self) -> List[str]:
@@ -1332,8 +1315,6 @@ class CompletedDevice(Device):
'library_version': str(self._library.version),
'occupied_by_project': self._occupied_by_project,
'configuration': self.configuration.as_json(list_hide=True),
'alert': self.alert,
'alert_time': str(self.alert_time),
}
def __repr__(self):
+22 -43
View File
@@ -22,8 +22,6 @@ import random
from copy import copy
import numpy as np
def json_stringify(o) -> str:
return _json_stringify(o, separators=(',', ':'))
@@ -868,11 +866,13 @@ class RecordingFile:
# def write(self, content: Union[bytes, RecordingData]) -> int:
def write(self, content: str, channels: list) -> int:
if not isinstance(content, str):
raise RuntimeError('wrong data format : ' + repr(self._data_format))
# print('count_size', content, channels)
# if not isinstance(content, str):
# raise RuntimeError('wrong data format : ' + repr(self._data_format))
self._meta_file.update_channels(channels)
sz = sys.getsizeof(content)
# print('sz', sz)
self._size += sz
return sz
@@ -1028,8 +1028,7 @@ class RecordingFileWriter:
'_mini_scale_list', '_time_real_time', '_data_rl', '_data_db',
'_raw_save', '_mini_save', '_data_time_ch', '_data_value_ch_for_rl',
'_data_time_ch_for_rl', '_device_id', '_send_data', '_data_mqtt_ch', '_id_db_save', '_raw_create_not_done',
'_mini_create_not_done', '_trigger_time', '_trigger_time_save', '_threshold', '_threshold_percent', '_first_average')
'_mini_create_not_done')
def __init__(self, meta: RecordingMetaFile, device_id, database = None):
self._meta = meta
@@ -1113,11 +1112,6 @@ class RecordingFileWriter:
self._raw_create_not_done = True
self._mini_create_not_done = True
self._trigger_time = 0
self._trigger_time_save = 0
self._threshold = 0
self._threshold_percent = 0
self._first_average = 0
@property
def meta_file(self) -> RecordingMetaFile:
@@ -1162,15 +1156,15 @@ class RecordingFileWriter:
self._send_data[ch] = False
if self._recording_file_dict[ch]._status:
_data = ' '.join(self._data_db[ch])
self._raw_save['data'][ch] = _data
_data = self._data_db[ch]
self._raw_save['data'][ch] = copy(_data)
self._raw_save['id'][ch] = self._recording_file_dict[ch]._id_db
self._recording_file_dict[ch].write(_data, self._channel_list)
self._recording_file_dict[ch].close(self._time_now)
self._meta._size += self._recording_file_dict[ch]._size
# self._data_db.clear()
if self._database is not None:
self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data'], self._id_db_save])
self._database.put_queue(['data_raw_recording_bytea', self._raw_save['id'], self._channel_list, self._raw_save['data'], self._id_db_save])
# self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data']])
self._recording_file_dict.clear()
for scale in self._mini_scale_list:
@@ -1243,21 +1237,9 @@ class RecordingFileWriter:
ret.append(str(_max))
return ret
def get_data_iter(self, d, mqtt_thread, mqtt_thread_1):
def get_data_iter(self, d, mqtt_thread):
# print('****d size', d.data_size)
for t, c, v in d.entry_iter():
if c == 2:
# print(t,c,v, self._threshold, self._trigger_time)
# print(time(), time() - self._trigger_time)
if v > self._threshold:
self._trigger_time_save = time()
if (time() - self._trigger_time_save) > self._trigger_time and self._trigger_time_save != 0 and self._trigger_time != 0:
self._trigger_time_save = time()
content = {
'header': 'device_alert/0',
'device': self._device_id,
}
mqtt_thread_1.publish('device_alert', json_stringify(content), inter = True)
if c in self._data_db:
### send real-time
if len(self._data_rl[c]) > 0 and self._send_data[c]:
@@ -1344,13 +1326,13 @@ class RecordingFileWriter:
# self._data_mini_ch[c]['10000']['random'].append( str(self._data_mini_ch[c]['1000']['random'][random.randint(-10,-1)]) )
# self._data_mini_ch[c]['1000']['dec'] = int(len(self._data_mini_ch[c]['1000']['mean']) / 10)
# add normal data
self._data_db[c].append(str(int(t)))
self._data_db[c].append(str(v))
self._data_db[c].append(int(t))
self._data_db[c].append(v)
self._time_now = int(t)
return
# @calculate_time(1)
def write(self, data: Union[bytes, RecordingData, List[bytes], List[RecordingData]], mqtt_thread, mqtt_thread_1) -> int:
def write(self, data: Union[bytes, RecordingData, List[bytes], List[RecordingData]], mqtt_thread) -> int:
# check size
ths = self.splitting_threshold_size
tht = self.splitting_threshold_time
@@ -1406,7 +1388,7 @@ class RecordingFileWriter:
'dec': 0,
}
self.get_data_iter(d, mqtt_thread, mqtt_thread_1)
self.get_data_iter(d, mqtt_thread)
if len(self._recording_file_dict) > 0:
for ch in self._data_db.keys():
@@ -1436,19 +1418,10 @@ class RecordingFileWriter:
if len(self._recording_file_dict) > 0:
for ch in self._data_db.keys():
if self._time_now - self._time[ch] > 5000000:
if ch == 2:
if self._first_average == 0:
a = np.array(self._data_db[ch], dtype=np.int64)
b = np.where(a % 2 == 1)[0]
c = a[b]
if len(c) != 0:
self._first_average = sum(c)/ len(c)
# print(c, self._first_average, self._threshold_percent)
self._threshold = self._first_average * (1 - (self._threshold_percent / 100))
if self._recording_file_dict[ch]._status:
_data = ' '.join(self._data_db[ch])
_data = self._data_db[ch]
write_sz = self._recording_file_dict[ch].write(_data, self._channel_list)
self._raw_save['data'][ch] = _data
self._raw_save['data'][ch] = copy(_data)
self._raw_save['id'][ch] = self._recording_file_dict[ch]._id_db
self._raw_save['end_time'][ch] = self._time_now
self._raw_save['size'][ch] = self._recording_file_dict[ch]._size
@@ -1475,9 +1448,15 @@ class RecordingFileWriter:
if data_save is True:
if self._database is not None:
recording_input = ['data_raw_recording_new', copy(self._raw_save['id']), copy(self._channel_list), copy(self._raw_save['data']), copy(self._raw_save['end_time']), copy(self._raw_save['size'])]
recording_input = ['data_raw_recording_bytea', copy(self._raw_save['id']), copy(self._channel_list), copy(self._raw_save['data']), copy(self._raw_save['end_time']), copy(self._raw_save['size'])]
self._database.put_queue(recording_input)
self._meta.update_subfile_time_size(database = self._database)
# if data_save is True:
# if self._database is not None:
# recording_input = ['data_raw_recording_new', copy(self._raw_save['id']), copy(self._channel_list), copy(self._raw_save['data']), copy(self._raw_save['end_time']), copy(self._raw_save['size'])]
# self._database.put_queue(recording_input)
if mini_save is True:
if self._database is not None:
for scale in self._mini_scale_list:
-4
View File
@@ -717,10 +717,6 @@ class DataServer(SocketServer, DataAPI):
def show_data(self, device):
self._configurations[device].put_rec_queue('show_data')
def reset_trigger(self, device):
if self._configurations.get(device, None) != None:
self._configurations[device].put_rec_queue('reset_trigger')
class DataRuntime(metaclass=abc.ABCMeta):
__slots__ = ('_server', '_device', '_meta_file', '_data_format',
+16 -1
View File
@@ -70,7 +70,7 @@ class DataBaseProcess(Process):
self._data_raw_create_sql_str = None
self._data_raw_update_sql_str = None
self._data_raw_recording_sql_str = 'UPDATE "public"."%s_recording_data_raws" SET data = concat(data, %s) where id = %s'
self._new_data_raw_recording_sql_str = 'UPDATE "public"."%s_recording_data_raws" SET data = concat(data, %s), end_time=%s, size=%s where id = %s'
self._new_data_raw_recording_sql_str = 'UPDATE "public"."%s_recording_data_raws" SET bytea_data = bytea_data || %s, end_time=%s, size=%s where id = %s'
self._data_mini_recording_sql_str = 'UPDATE "public"."%s_recording_data_minis" SET data_mean = concat(data_mean, %s) where id = %s'
@property
@@ -325,6 +325,7 @@ class DataBaseProcess(Process):
# @calculate_time()
def data_raw_create(self, _data_dict, _channel_list, device_id):
print('data_raw_create', _data_dict, _channel_list, device_id)
if self._data_raw_create_sql_str == None:
sql_str_list = []
key_list = _data_dict[_channel_list[0]].keys()
@@ -582,3 +583,17 @@ class DataBaseProcess(Process):
self._queue_ds[int(device_id)].put(['project_id', int(sql_cursor.fetchone()[0])])
self._psql_conn.commit()
sql_cursor.close()
def data_raw_recording_bytea(self, _id_dict, _channel_list, _data_dict, _end_time_dict, _size_dict):
# print('data_raw_recording_bytea', _id_dict, _channel_list, _data_dict, _end_time_dict, _size_dict)
try:
para_list = []
for _channel in _channel_list:
bytes_data = b''.join([int.to_bytes(i, 8, 'big', signed=True) for i in _data_dict[_channel]])
para_list.append([_channel, bytes_data, _end_time_dict[_channel], _size_dict[_channel], _id_dict[_channel]])
with self._psql_conn as conn:
with conn.cursor() as sql_cursor:
execute_batch(sql_cursor, self._new_data_raw_recording_sql_str, para_list)
except psycopg2.Error as e:
print('recording error', e)
return None
-21
View File
@@ -1178,8 +1178,6 @@ class ControlServer(SocketServer, ControlServerAPI):
# unset file info, but keep file path cache
self.file_manager.unset(device.device_id)
device.status = 0
device.alert = False
device.alert_time = None
# server provide method implement : websocket broadcast functions
@@ -1400,25 +1398,6 @@ class ControlServer(SocketServer, ControlServerAPI):
if client is not None:
with client:
client.show_data(device)
@logging_info
def reset_trigger(self, device) -> bool:
device = self.device_manager.get_device(device)
client = self.data_server.client()
if client is not None:
with client:
device.alert = False
device.alert_time = None
client.reset_trigger(device.device_id)
self.broadcast_command('refresh')
@logging_info
def device_alert(self, device) -> bool:
device = self.device_manager.get_device(device)
device.alert = True
if device.alert_time == None:
device.alert_time = datetime.now()
self.broadcast_command('refresh')
class _RandomCrashThread(ServerThread):
def __init__(self):
+2 -8
View File
@@ -111,10 +111,6 @@ class RecordingProcess(Process):
self._decoder = self.data_format()
self._start_time = time()
self._writer = RecordingFileWriter(self._meta_file, self._device, self._database)
self._writer._threshold_percent = self._meta_file.configuration.get_parameter('THRESHOLD')
self._writer._trigger_time = self._meta_file.configuration.get_parameter('TRIGGER_TIME')
def ensure_data_format(self) -> DataDecodeFormat:
if isinstance(self._data_format, (str, bytes)):
self._data_format = DataDecodeFormat.parse(self._data_format)
@@ -191,8 +187,6 @@ class RecordingProcess(Process):
return False
elif q == 'show_data':
self._decoder._show_data = not self._decoder._show_data
elif q == 'reset_trigger':
self._writer._trigger_time_save = time()
else:
self.rec_update()
self.sync_data(q)
@@ -251,8 +245,7 @@ class RecordingProcess(Process):
if self._writer is not None and len(ret) > 0:
if len(self._writer.channel_list) == 0:
self._writer.channels_update(ret[0].channels())
self._writer.write(ret, self._mqtt_send_data_ch_level, self._mqtt_thread)
# print('count', self._writer._count)
self._writer.write(ret, self._mqtt_send_data_ch_level)
# print('write time: ', time() - ctime1)
# print(ret)
@@ -592,6 +585,7 @@ class RecordingProcess(Process):
self._is_close = False
if self._mqtt_thread is not None:
self._mqtt_thread.start()
self._writer = RecordingFileWriter(self._meta_file, self._device, self._database)
self.routine()
def routine(self) -> None:
@@ -43,28 +43,6 @@
},
"value": "VALUE"
},
"TRIGGER_TIME": {
"description": "threshold ohm",
"record_meta": true,
"initial": 0,
"domain": [
"TIME_MAX"
],
"value": {
"expression": "VALUE"
}
},
"THRESHOLD": {
"description": "threshold ohm",
"record_meta": true,
"initial": 0,
"domain": [
1000
],
"value": {
"expression": "VALUE"
}
},
"DPV_e_init": {
"description": "DPV initial voltage ",
"record_meta": true,
@@ -931,28 +931,6 @@
"ms"
]
},
"TRIGGER_TIME": {
"description": "threshold ohm",
"record_meta": true,
"initial": 0,
"domain": [
"TIME_MAX"
],
"value": {
"expression": "VALUE"
}
},
"THRESHOLD": {
"description": "threshold ohm",
"record_meta": true,
"initial": 0,
"domain": [
1000
],
"value": {
"expression": "VALUE"
}
},
"TIME_DURATION": {
"description": "Run duration",
"record_meta": true,
@@ -931,28 +931,6 @@
"ms"
]
},
"TRIGGER_TIME": {
"description": "threshold ohm",
"record_meta": true,
"initial": 0,
"domain": [
"TIME_MAX"
],
"value": {
"expression": "VALUE"
}
},
"THRESHOLD": {
"description": "threshold ohm",
"record_meta": true,
"initial": 0,
"domain": [
1000
],
"value": {
"expression": "VALUE"
}
},
"TIME_DURATION": {
"description": "Run duration",
"record_meta": true,
@@ -31,7 +31,7 @@
},
"parameters": {
"USED": {
"initial": [false, false, false, false, false, false, false, false],
"initial": [true, true, true, false, false, false, false, false],
"domain": {
"list": [
0, 2
@@ -40,7 +40,7 @@
"value": "VALUE"
},
"T_EARLY": {
"initial": [0, 0, 0, 0, 0, 0, 0, 0],
"initial": [5000, 5000, 5000, 0, 0, 0, 0, 0],
"domain": {
"list": [
0, 86400000
@@ -49,7 +49,7 @@
"value": "VALUE"
},
"V_EARLY": {
"initial": [false, false, false, false, false, false, false, false],
"initial": [true, false, false, false, false, false, false, false],
"domain": {
"list": [
0, 2
@@ -58,7 +58,7 @@
"value": "VALUE"
},
"CYCLE": {
"initial": [1, 1, 1, 1, 1, 1, 1, 1],
"initial": [10, 10, 10, 1, 1, 1, 1, 1],
"domain": {
"list": [
0, 65535
@@ -67,7 +67,7 @@
"value": "VALUE"
},
"T_MID0": {
"initial": [0, 0, 0, 0, 0, 0, 0, 0],
"initial": [30000, 30000, 30000, 0, 0, 0, 0, 0],
"domain": {
"list": [
0, 86400000
@@ -76,7 +76,7 @@
"value": "VALUE"
},
"T_MID1": {
"initial": [0, 0, 0, 0, 0, 0, 0, 0],
"initial": [30000, 30000, 30000, 0, 0, 0, 0, 0],
"domain": {
"list": [
0, 86400000
@@ -103,7 +103,7 @@
"value": "VALUE"
},
"V_MID0": {
"initial": [false, false, false, false, false, false, false, false],
"initial": [true, true, false, false, false, false, false, false],
"domain": {
"list": [
0, 2
@@ -112,7 +112,7 @@
"value": "VALUE"
},
"V_MID1": {
"initial": [false, false, false, false, false, false, false, false],
"initial": [false, false, true, false, false, false, false, false],
"domain": {
"list": [
0, 2
@@ -337,6 +337,8 @@
"Analog Current Control (ACC)",
"Idle",
"Dev Mode",
"Protocal 1",
"Protocal 2",
"Trigger"
]
},
@@ -399,7 +401,9 @@
"expression": "MODE",
"when": {
"0": "curve_acc",
"3": "trig_timer_mode"
"3": "trig_timer_mode",
"4": "trig_timer_mode",
"5": "trig_timer_mode"
}
},
"_sync(True)",
+7 -1
View File
@@ -35,4 +35,10 @@ sudo su -c "psql -d postgres -c \"ALTER TABLE devices ALTER COLUMN calibration D
sudo su -c "psql -d postgres -c \"ALTER TABLE devices ALTER COLUMN calibration TYPE bytea USING calibration::bytea;\"" postgres
# add column project in recording_data_metas
sudo su -c "psql -d postgres -c \"ALTER TABLE devices ADD COLUMN IF NOT EXISTS calibration_version Int4 DEFAULT -1;\"" postgres
sudo su -c "psql -d postgres -c \"ALTER TABLE devices ADD COLUMN IF NOT EXISTS calibration_version Int4 DEFAULT -1;\"" postgres
# add column bytea_data column in 0-32_recording_data_raws
for i in {0..32}
do
sudo su -c "psql -d postgres -c \"ALTER TABLE \\\"${i}_recording_data_raws\\\" ADD COLUMN IF NOT EXISTS bytea_data BYTEA DEFAULT ''::bytea;\"" postgres
done