[update] fix battery value

python/biopro/device/device_cc2650.py

@@ -536,7 +536,7 @@ class CC2650Device(Device):
             self._master.log_warn('device', self.device_id, 'update_battery_info no response')
 
         else:
-            if data is not None and len(data) == 4 :
+            if data is not None and len(data) > 2 :
                 battery = struct.unpack('<H', data[1:3])[0]
                 if battery is not None:
                     self._battery = battery

python/biopro/devlib/data.py

@@ -882,21 +882,18 @@ class I4V4Z4T4DataDecoder(RecDataDecoder):
         if len(data) < 18:
             return None
 
-        current = struct.unpack('>i', data[1:5])[0]  # unit: 1/1000 nA
-        voltage = struct.unpack('>i', data[5:9])[0]  # unit: mV
-        impedance = struct.unpack('>i', data[9:13])[0]  # unit: kOm
-        time_stamp: float = struct.unpack('<I', data[13:17])[0]  # unit: ms
-        cycle_number = struct.unpack('>H', data[17:19])[0]
-        finish_mode_falg = data[19]
+        mem_cnt = data[1]
+        time_stamp: float = struct.unpack('<I', data[4:8])[0]  # unit: ms 0x18030000
+        current = struct.unpack('<i', data[8:12])[0]  # unit: nA
+        voltage = struct.unpack('<i', data[12:16])[0]  # unit: uV
+        impedance = struct.unpack('<i', data[16:20])[0]  # unit: mOm
+        cycle_number = struct.unpack('<H', data[20:22])[0]
+        finish_mode_falg = data[22]
+        battery = struct.unpack('<i', data[23:27])[0]
+        elite_notify_times = data[27]
 
-        mem_wrong_information = struct.unpack('>i', data[40:40+4])[0]
-
-        # print('decode', list(data[20:]))
-        mem_wrong = data[40]
-        mem_retry_cnt = data[41]
-        mem_green_wrong = data[42]
-        mem_green_retry_cnt = data[43]
-        ram_num = data[44]
+        mem_wrong_information = struct.unpack('<i', data[43:47])[0] # mem_wrong_information = green retry, green wrong, red retry, red wrong
+        ram_num = data[47]
         broken_flag = data[-1]
 
         if (finish_mode_falg & 0b11110000 == 0b10100000):
@@ -917,10 +914,13 @@ class I4V4Z4T4DataDecoder(RecDataDecoder):
                 print("error timeStamp full data:", list(data), datetime.now(), '\n')
                 return None
             else:
-                # print('|', time_stamp, '|', delta, '|', current, '|', voltage, '|', impedance,
+                # print('|', time_stamp, '|', delta, '|', int(time_stamp * 1000 / 2),
+                #       '|', current, '|', voltage, '|', impedance,
                 #       '|', cycle_number, '|', finishMode, '@', str(self.device))
+
                 # print('|', '{:10}'.format(time_stamp),
-                #       '|', '{:10}'.format(delta),
+                #       '|', '{:4}'.format(delta),
+                #       '|', '{:10}'.format(int(time_stamp * 1000 / 2)),
                 #       '|', '{:10}'.format(current),
                 #       '|', '{:10}'.format(voltage),
                 #       '|', '{:10}'.format(impedance),
@@ -928,13 +928,9 @@ class I4V4Z4T4DataDecoder(RecDataDecoder):
                 #       '|', '{:1}'.format(finishMode),
                 #       '@', str(self.device), '|')
 
-                # print('|', '{:5}'.format(mem_wrong),
-                #       '|', '{:5}'.format(mem_retry_cnt),
-                #       '|', '{:5}'.format(mem_green_wrong),
-                #       '|', '{:5}'.format(mem_green_retry_cnt),
-                #       '|', '{:5}'.format(mem_wrong_information),
-                #       '|', '{:5}'.format(ram_num),
-                #       '|', '{:5}'.format(broken_flag),
+                # print('|', '{:5}'.format(mem_wrong_information),
+                #       '|', '{:2}'.format(ram_num),
+                #       '|', '{:2}'.format(broken_flag),
                 #       '@', str(self.device), '|')
                 pass
 
@@ -949,16 +945,20 @@ class I4V4Z4T4DataDecoder(RecDataDecoder):
         ret.append_data(1, voltage)
         ret.append_data(2, impedance)
         ret.append_data(3, cycle_number)
+        ret.append_data(4, battery)
+        ret.append_data(5, elite_notify_times)
+        ret.append_data(6, mem_cnt)
 
         # memoryboard information
-        # ret.append_data(4, mem_wrong)
-        # ret.append_data(5, mem_retry_cnt)
-        # ret.append_data(6, mem_green_wrong)
-        # ret.append_data(7, mem_green_retry_cnt)
-        ret.append_data(4, ram_num)
-        ret.append_data(5, broken_flag)
-        ret.append_data(6, mem_wrong_information)
-        # ret.append_data(4, ram_num)
+        ret.append_data(7, ram_num)
+        ret.append_data(8, broken_flag)
+        try:
+            ret.append_data(9, mem_wrong_information)
+            # print('append_data success, mem_wrong_information:', mem_wrong_information, hex(mem_wrong_information))
+        except:
+            print('append_data fail, mem_wrong_information:', mem_wrong_information, hex(mem_wrong_information))
+
+
     
         if cycle_number != self._cycle_number:
             # notify cycle_number change

python/biopro/impl/ext_mem.py

@@ -24,8 +24,10 @@ def zero_buffer(size: int) -> List[int]:
     return [0] * size
 
 class MultiExtMemSpiInterface(LowLevelHardwareInterface):
-    MEM_INS_MARKED = [MEM_INS_WRITE, 0, 2, 1, 1]
-    MEM_INS_RESET  = [MEM_INS_WRITE, 0, 2, 1, 1, 0, 0xFF]
+    MEM_INS_MARKED1 = [MEM_INS_WRITE, 0, 2, 0x5A, 0xA5]
+    MEM_INS_MARKED2 = [MEM_INS_WRITE, 0, 6, 0x5A, 0xA5]
+    MEM_INS_MARKED3 = [MEM_INS_WRITE, 0, 10, 0x5A, 0xA5]
+    # MEM_INS_RESET  = [MEM_INS_WRITE, 0, 2, 1, 1, 0, 0xFF]
 
     __slots__ = ('_selector', '_wait_for_first_data', '_spi', '_tx_buffer_header', '_tx_buffer_data',
                  'pin_busy', 'pin_mem_req', 'pin_mem_sel', 'pin_ram_sel',
@@ -44,11 +46,11 @@ class MultiExtMemSpiInterface(LowLevelHardwareInterface):
 
         self._elite_data_len = 40
         self._mem_header_len = 3
-        self._mem_tailer_len = 8
+        self._mem_tailer_len = 6
         self._single_data_len = self._elite_data_len + self._mem_header_len + self._mem_tailer_len
 
         # buffer
-        self._tx_buffer_header = [0] * 11
+        self._tx_buffer_header = [0] * 19
         self._tx_buffer_data = [0] * (self._single_data_len * 10 + 3)
 
         # memory control pin
@@ -165,6 +167,52 @@ class MultiExtMemSpiInterface(LowLevelHardwareInterface):
         print("read red data times", self._read_red_times)
         return data
 
+    def _compare_green_data_addr_and_flag(self, data: Union[bytes, List[int]], device: int):
+
+        green_data_section1 = data[3:7]
+        green_data_section2 = data[7:11]
+        green_data_section3 = data[11:15]
+        green_data_section = []
+
+        if green_data_section1 == green_data_section2:
+            green_data_section = green_data_section1
+
+        elif(green_data_section2 == green_data_section3 or green_data_section1 == green_data_section3):
+            print("green data not equal: = ", data[3:15])
+            print("green data print:", data, device, datetime.now())
+            green_data_section = green_data_section3
+
+        else:
+            print("green data not equal: = ", data[3:15])
+            print("green data print:", data, device, datetime.now())
+            green_data_section = green_data_section3 # use last data
+
+
+        return green_data_section
+
+    def _print_ram_all_data(self):
+        addr = 0
+        red_length = int(7000 / 2)
+        tx_temp = [0] * (red_length + 3)
+
+        while True:
+            tx_temp[0] = MEM_INS_READ
+            tx_temp[1] = ((addr >> 8) & 0xFF)
+            tx_temp[2] = (addr & 0xFF)
+
+            ram_data = []
+            ram_data = self._spi.send_byte(tx_temp)
+            ram_data[0:3] = [255, 255, 255]
+            print(list(ram_data), len(ram_data), addr)
+            addr += len(ram_data) - 3
+
+            if (7000 < addr + len(ram_data)):
+                del ram_data
+                break
+        print()
+
+        return
+
     def recv_memory(self, device: int) -> Optional[bytes]:
         self.pin_busy.output(False)
 
@@ -191,7 +239,7 @@ class MultiExtMemSpiInterface(LowLevelHardwareInterface):
                 self._head_wrong_cnt[device] = self._head_wrong_cnt[device] + 1
                 if (self._head_wrong_cnt[device] <= 5): # print 5 times
                     print('data_first[0:3] != [255, 255, 255], device:', device, ',', self._head_wrong_cnt[device], 'times')
-                    print(list(data[0:7]))
+                    print(list(data))
 
             data[0:3] = [255, 255, 255]
             # ----------------------------------------------------------------------------------------------
@@ -210,13 +258,13 @@ class MultiExtMemSpiInterface(LowLevelHardwareInterface):
             #     self._head_wrong_cnt[device] = self._head_wrong_cnt[device] + 1
             #     if (self._head_wrong_cnt[device] < 10):
             #         print('data_first[0:3] != [255, 255, 255], device:', device, ',', self._head_wrong_cnt[device], 'times')
-            #         print(list(data_first[0:7]))
+            #         print(list(data_first))
 
             # if (data_second[0] != 255 or data_second[1] != 255 or data_second[2] != 255):
             #     self._head_wrong_cnt[device] = self._head_wrong_cnt[device] + 1
             #     if (self._head_wrong_cnt[device] < 10):
             #         print('data_second[0:3] != [255, 255, 255], device:', device, ',', self._head_wrong_cnt[device], 'times')
-            #         print(list(data_second[0:7]))
+            #         print(list(data_second))
 
             # if (data_first[3:] == data_second[3:]):
             #     data = data_first
@@ -228,59 +276,47 @@ class MultiExtMemSpiInterface(LowLevelHardwareInterface):
             #     data = self.compare_green_data(data_first, data_second, data_third, len(tx_h))
             # ----------------------------------------------------------------------------------------------
 
-            # debug use
-            # green_data = []
-            # green_data = data
-            # print("_[Debug] @ spi green data = ", green_data)
+            green_data_section = self._compare_green_data_addr_and_flag(data, device)
 
-            length = (data[3] << 8) | data[4]
-            header = data[7]
-            elite_data_len = data[9]
+            length = (green_data_section[0] << 8) | green_data_section[1]
+            header = data[15]
+            elite_data_len = data[17]
+
+            if length <= 12:
+                if length < 12:
+                    print("length < 12")
+                    print("green data print:", data, device, datetime.now())
+
+                if length == 0:
+                    self._print_ram_all_data()
 
-            if length <= 4:
-                # print("length <= 4")
-                # print("green data print:", data, device, datetime.now())
                 return None
 
-            if data[5] != 0 or data[6] != 0:
+            if green_data_section[2] != 0xA5 or green_data_section[3] != 0x5A:
                 # give a default length if the data header did not update
-                length = 7000
-                print("green data: is not [0,0], = ", data[5:7])
+                print("green data: is not [0xA5, 0x5A], = ", data[5:7])
                 print("green data print:", data, device, datetime.now())
                 return None
 
-            if (length >= 7000):
+            if (length >= 4000):
                 flag_print = True
                 print("green data: big length:", length)
 
             if (header != 255):
                 flag_print = True
                 print("green data: header is not 255: ", header)
+                self._print_ram_all_data()
 
             if (elite_data_len != self._elite_data_len):
                 flag_print = True
                 print("green data: length is not", self._elite_data_len, ": ", elite_data_len)
 
             if (flag_print):
-                print("green data print:", data, device, datetime.now())
+                print("green data print:", data, device, datetime.now(), '\n')
 
-            # neulive
-            # address += len(data) - 11
-            # tx_d = self._tx_buffer_data
-            # red_length = len(tx_d)
-
-            # elite read all data of ram
-            # address = 4
-            # red_length = length - 1 
-            # tx_d = [0] * red_length
-
-            address = 4
+            address = 12
             red_length = int(length / 2)
-            tx_d = [0] * red_length
-
-            # elite read len(_tx_buffer_data) byte
-            # address = 4
-            # tx_d = self._tx_buffer_data
+            tx_d = [0] * (red_length + 3)
 
             while True:
                 tx_d[0] = MEM_INS_READ
@@ -319,22 +355,60 @@ class MultiExtMemSpiInterface(LowLevelHardwareInterface):
                     address += len(data) - 3
                     del data
 
-            # print("_[Debug] @ spi recv data rx = ")
+            # last_index = 0
+            # print("_[Debug] @ spi recv data rx, ram_select:", self._pin_ram_sel_value[device], ",", datetime.now())
             # for i in range(0, len(rx), self._single_data_len):
-            #     print(rx[i:i+self._single_data_len])
+            #     last_index = i
+            #     # print(rx[i:i+self._single_data_len])
+            #     if i == 0:
+            #         print(rx[0:8], 'ram:', rx[47])
+
+            # if last_index != 0:
+            #     print(rx[last_index:last_index+8], 'ram:', rx[last_index+47])
             # print()
 
             # /* 
             #  * red data formate:
             #  * ramHdr, ramHdr, ramHdr,                                   (3B)
             #  * 255, #, data_length,                                      (3B)
-            #  * data,                                                     (20B)
+            #  * data,                                                     (40B)
             #  * red_wrong, red_retry_cnt, green_wrong, green_retry_cnt,   (4B)
-            #  * 255, #, data_length,                                      (3B) 
+            #  * check_num,                                                (1B)
             #  *                                                                  */
 
+            # read again if check num is wrong
+            index = 0
+            check_number_print = False
+            for i in range(0, len(rx), self._single_data_len):
+                check_sum = sum(rx[i : i + self._single_data_len - 1]) & 0b11111111
+                # print(check_sum, rx[i + self._single_data_len - 1])
+
+                if (check_sum != rx[i + self._single_data_len - 1]):
+                    print('check_sum wrong, origin value:', 'check_sum =', check_sum, rx[i : i + self._single_data_len])
+                    tx_d = [0] * (self._single_data_len + 3)
+                    address = 12 + self._single_data_len * index
+                    tx_d[0] = MEM_INS_READ
+                    tx_d[1] = ((address >> 8) & 0xFF)
+                    tx_d[2] = (address & 0xFF)
+
+                    data = []
+                    data = self._spi.send_byte(tx_d)
+                    data[0:3] = [255, 255, 255]
+                    rx[i : i + self._single_data_len] = data[3:]
+                    print('check_sum wrong, read again:', 'check_sum =', check_sum, rx[i : i + self._single_data_len])
+                    check_number_print = True
+                index = index + 1
+
+            if check_number_print:
+                print('check_sum wrong:', device, datetime.now())
+                for i in range(0, len(rx), self._single_data_len):
+                    print(rx[i:i+self._single_data_len])
+                print()
+
             # mark read
-            self._spi.send_byte(self.MEM_INS_MARKED)
+            self._spi.send_byte(self.MEM_INS_MARKED1)
+            self._spi.send_byte(self.MEM_INS_MARKED2)
+            self._spi.send_byte(self.MEM_INS_MARKED3)
 
         except BaseException as e:
             print(e)

python/biopro/recording/file.py

@@ -19,6 +19,8 @@ from statistics import mean
 import random
 # from numba import jit
 
+from copy import copy
+
 def json_stringify(o) -> str:
     return _json_stringify(o, separators=(',', ':'))
 
@@ -113,8 +115,8 @@ class RecordingMetaFile(JsonSerialize):
     __slots__ = ('_filename', '_filepath', '_id_db', '_create_time', '_file_version', '_file_uuid', '_data_format', '_last_time',
                  '_device', '_configuration', '_channel_mask', '_database',
                  '_dirty', '_last_modify_time', '_file_size_cache',
-                 '_recording_file_ch', '_recording_sub_file', '_size',
-                 '_parameter', '_parent', '_recording_file_name', '_recording_sub_mini', '_recording_mini_ch',
+                 '_recording_file_ch', '_size',
+                 '_parameter', '_parent', '_recording_file_name', '_recording_mini_ch',
                  '_raw_serial_number', '_mini_serial_number', '_device_id')
 
     def __init__(self, filepath: Union[str, Path], data_format: bytes = None, id_db = 0, database = None):
@@ -165,10 +167,6 @@ class RecordingMetaFile(JsonSerialize):
         self._raw_serial_number = {}
         self._mini_serial_number = {}
 
-        self._recording_sub_file = {}
-        self._recording_sub_mini = {}
-
-
         self._size = 0
 
         self._dirty = True
@@ -306,7 +304,6 @@ class RecordingMetaFile(JsonSerialize):
         if database is not None :
             self._database = database
             self._database.put_queue(['data_meta_write', meta_data, self._device_id, str(self._file_uuid), self._id_db])
-            # database.put_queue(['data_meta_write', self, meta_data, _path, self._id_db])
                 
         with self._filepath.open('wb') as _f:
             f = FileEncoder(_f)
@@ -356,19 +353,15 @@ class RecordingMetaFile(JsonSerialize):
 
     def update_subfile(self, database = None):
         meta_data = {
-            'raw_data': self._recording_sub_file,
-            'mini_data': self._recording_sub_mini,
             'channels': str(self._channel_mask.channels()),
             'size': str(self._size),
             'time_duration': str(self._last_time),
             'uuid': str(self._file_uuid),
         }
-        _path = str(self._filepath).replace('/', '^')
 
         if database is not None :
             self._database = database
             self._database.put_queue(['data_meta_write', meta_data, self._device_id, str(self._file_uuid), self._id_db])
-            # database.put_queue(['data_meta_write', self, meta_data, _path, self._id_db])
         return None
                 
     @property
@@ -517,11 +510,6 @@ class RecordingMetaFile(JsonSerialize):
             'start_time': str(_start_time),
         }
 
-        # if database is not None:
-        #     database.data_raw_create(self, f, raw_data, _channel)
-
-        # self._recording_file_ch[_channel]
-
         return f, raw_data
 
     def new_recording_mini(self, _channel: int, _start_time, scale, database = None) -> 'RecordingMini':
@@ -567,9 +555,6 @@ class RecordingMetaFile(JsonSerialize):
             'start_time': str(_start_time),
         }
 
-        # if database is not None:
-        #     database.data_mini_create(self, f, mini_data, _channel, scale)
-
         return f, mini_data
 
     def clear_recording_file(self):
@@ -628,15 +613,6 @@ class RecordingMetaFile(JsonSerialize):
             'device':        self._device,
         }
 
-        # d = self._device
-
-        # if d is not None:
-        #     ret['device'] = {
-        #         'device_name':    d.device_name,
-        #         'device_address': list(d.mac_address),
-        #         'serial_number':  d.serial_number.as_json(),
-        #     }
-
         return ret
 
 
@@ -792,14 +768,10 @@ class RecordingFile:
                 'end_time': str(_end_time),
                 'size': str(self._size)
             }
-            # if self._meta_file._id_db == 0:
-            #     _path = str(self._meta_file.filepath).replace('/', '^')
-            #     self._database.put_queue(['data_meta_id_get_by_path', _path])
-            #     # self._meta_file._id_db = self._database.put_queue(['data_meta_id_get_by_path', _path])
+
             if self._meta_file._id_db != 0:
                 _data['parent'] = self._meta_file._id_db
             self._database.put_queue(['data_raw_update', self._id_db, self._channel, _data])
-            # self._database.put_queue(['data_raw_update', self._id_db, self._channel, _data])
         self._status = False
         return None
 
@@ -965,10 +937,7 @@ class RecordingMini:
                 'end_time': str(_end_time),
                 'size': str(self._size)
             }
-            # if self._meta_file._id_db == 0:
-            #     _path = str(self._meta_file.filepath).replace('/', '^')
-            #     self._database.put_queue(['data_meta_id_get_by_path', _path])
-            #     # self._meta_file._id_db = self._database.put_queue(['data_meta_id_get_by_path', _path])
+
             if self._meta_file._id_db != 0:
                 _data['parent'] = self._meta_file._id_db
             self._database.put_queue(['data_mini_update', self._id_db, self._channel, _data])
@@ -996,7 +965,7 @@ class RecordingFileWriter:
                  '_data_value_ch', '_close', '_data_mini_ch',
                  '_mini_scale_list', '_time_real_time', '_data_rl', '_data_db',
                  '_raw_save', '_mini_save', '_data_time_ch', '_data_value_ch_for_rl',
-                 '_data_time_ch_for_rl', '_device_id', '_send_data', '_data_mqtt_ch')
+                 '_data_time_ch_for_rl', '_device_id', '_send_data', '_data_mqtt_ch', '_id_db_save')
 
     def __init__(self, meta: RecordingMetaFile, device_id, database = None):
         self._meta = meta
@@ -1017,7 +986,9 @@ class RecordingFileWriter:
 
         self._raw_save = {
             'id': {},
-            'data': {}
+            'data': {},
+            'end_time': {},
+            'size': {}
         }
         self._mini_save = {
             '10': {
@@ -1052,6 +1023,7 @@ class RecordingFileWriter:
         self._data_value_ch_for_rl = {}
         self._data_time_ch_for_rl = {}
         self._data_time_ch = {}
+        self._id_db_save = {}
 
         # mini data
         self._data_mini_ch = {}
@@ -1068,7 +1040,7 @@ class RecordingFileWriter:
 
         # splitting
         self.splitting_threshold_time = 30 * 60 * 1000  # one minute
-        self.splitting_threshold_size = 16 * 16 * 1024  #  16 * 16KB
+        self.splitting_threshold_size = 16 * 1024  #  16 * 16KB
 
         self._writer_batch_size = 8192 
 
@@ -1094,6 +1066,7 @@ class RecordingFileWriter:
 
     def channels_update(self, channels):
         channels.sort()
+        self._database.put_queue(['data_meta_update_raw_empty_channel', self._meta._id_db, channels])
         self._channel_list.extend(channels)
         return self._channel_list
 
@@ -1107,9 +1080,6 @@ class RecordingFileWriter:
         return None
 
     def close(self):
-        # if self._recording_file is not None:
-        #     self._recording_file.close()
-        #     self._recording_file = None
         print('close1')
         self._close = True
         if len(self._recording_file_dict) > 0:
@@ -1123,7 +1093,7 @@ class RecordingFileWriter:
                     self._meta._size += self._recording_file_dict[ch]._size
             # self._data_db.clear()
             if self._database is not None:
-                self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data']])
+                self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data'], self._id_db_save])
                 # self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data']])
             self._recording_file_dict.clear()
         for scale in self._mini_scale_list:
@@ -1348,6 +1318,10 @@ class RecordingFileWriter:
                 
                 self.get_data_iter(d, mqtt_thread)
 
+            for ch in self._recording_file_dict:
+                if self._recording_file_dict[ch]._id_db == 0:
+                    return None
+
             data_save = False
             mini_save = False
             if len(self._recording_file_dict) > 0:
@@ -1355,29 +1329,19 @@ class RecordingFileWriter:
                     if self._time_now - self._time_real_time[ch] > 1000000:
                         self._send_data[ch] = True
                         self._time_real_time[ch] = self._time_now
-                    # # if self._time_now - self._time_real_time[ch] > 500000:
-                    # self._data_rl[ch].append(str(self._time_now))
-                    # mes = ' '.join(self._data_rl[ch])
-                    # # print(mes)
-                    # # if mqtt_thread is not None:
-                    # #     mqtt_thread.publish('data_server/device_data_stream/' + str(self._device_id) + '/' + str(ch), mes)
-                    # # mqtt_queue.put([self._device_id, ch, mes])
-                    # mqtt_thread[ch].on_message(mes)
-                    # self._data_rl[ch].clear()
-                    # # self._time_real_time[ch] = self._time_now
-                    if self._time_now - self._time[ch] > 5000000:
+                    if self._time_now - self._time[ch] > 2000000:
                         if self._recording_file_dict[ch]._status:
                             _data = ' '.join(self._data_db[ch])
                             write_sz = self._recording_file_dict[ch].write(_data, self._channel_list)
-                            sz = sz + write_sz
                             self._raw_save['data'][ch] = _data
                             self._raw_save['id'][ch] = self._recording_file_dict[ch]._id_db
-                            data_save = True
+                            self._raw_save['end_time'][ch] = self._time_now
+                            self._raw_save['size'][ch] = self._recording_file_dict[ch]._size
                             self._data_db[ch].clear()
                             self._time[ch] = self._time_now
-                            self._splitting_size += sz
                             self._meta._last_time = self._time_now
-                            self._recording_file_dict[ch].update_endtime(self._time_now)
+                            self._splitting_size += write_sz
+                            data_save = True
                     if len(self._data_mini_ch[ch]['1000']['mean']) >= 10:
                         mini_save = True
                         for scale in self._mini_scale_list:
@@ -1394,14 +1358,14 @@ class RecordingFileWriter:
 
                 if data_save is True:
                     if self._database is not None:
-                        self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data']])
+                        recording_input = ['data_raw_recording_new', copy(self._raw_save['id']), copy(self._channel_list), copy(self._raw_save['data']), copy(self._raw_save['end_time']), copy(self._raw_save['size'])]
+                        self._database.put_queue(recording_input)
                         self._meta.update_subfile(database = self._database)
                 if mini_save is True:
                     if self._database is not None:
                         for scale in self._mini_scale_list:
                             self._database.put_queue(['data_mini_recording', self._mini_save[str(scale)]['id'], self._channel_list, self._mini_save[str(scale)]['start_time'], self._mini_save[str(scale)]['data_mean']])
                         self._meta.update_subfile(database = self._database)
-            
 
         del data
 
@@ -1409,11 +1373,9 @@ class RecordingFileWriter:
 
 
     def _switch_recording_file(self):
-        # if self._recording_file is not None:
-        #     self._recording_file.close()
-        
         if len(self._recording_file_dict) > 0:
             for ch in self._recording_file_dict.keys():
+                self._id_db_save[ch] = self._recording_file_dict[ch]._id_db
                 self._recording_file_dict[ch].close(self._time_now)
                 self._meta._size += self._recording_file_dict[ch]._size
                 
@@ -1429,7 +1391,6 @@ class RecordingFileWriter:
 
         if self._database is not None:
             self._database.put_queue(['data_raw_create', raw_data_dict, self._channel_list, self._device_id])
-            # self._database.put_queue(['data_raw_create', self._meta, self._recording_file_dict, raw_data_dict, self._channel_list])
 
         self._meta._last_time = self._time_now
         self._meta.update_subfile(database = self._database)
@@ -1456,29 +1417,21 @@ class RecordingFileWriter:
 
         if self._database is not None:
             self._database.put_queue(['data_mini_create', mini_data_dict, self._channel_list, scale, self._device_id])
-            # self._database.put_queue(['data_mini_create', self._meta, self._recording_mini_dict[str(scale)], mini_data_dict, self._channel_list, scale])
             
         self._meta._last_time = self._time_now
         self._meta.update_subfile(database = self._database)
         return None
 
     def update_meta_id(self, _id):
-        # print('update meta')
         self._meta._id_db = _id
         return None
     
     def update_raw_dict_id(self, _channel, _id):
         self._recording_file_dict[int(_channel)]._id_db = _id
-        if int(_channel) not in self._meta._recording_sub_file:
-            self._meta._recording_sub_file[int(_channel)] = []
-        self._meta._recording_sub_file[int(_channel)].append(_id)
+        self._database.put_queue(['data_meta_update_channel_raw_id', self._meta._id_db, _channel, _id])
         return None
 
     def update_mini_dict_id(self, _scale, _channel, _id):
         self._recording_mini_dict[str(_scale)][int(_channel)]._id_db = _id
-        if int(_channel) not in self._meta._recording_sub_mini:
-            self._meta._recording_sub_mini[int(_channel)] = {}
-        if int(_scale) not in self._meta._recording_sub_mini[int(_channel)]:
-            self._meta._recording_sub_mini[int(_channel)][int(_scale)] = []
-        self._meta._recording_sub_mini[int(_channel)][int(_scale)].append(_id)
+        self._database.put_queue(['data_meta_update_channel_mini_id', self._meta._id_db, _channel, _scale, _id])
         return None

python/biopro/server/data.py

@@ -1179,20 +1179,20 @@ class SpiRuntimeThread(ServerThread):
 
         run_time = time() - self._timer
 
+        real_run_time = run_time
+
         if run_time > 0.045:
-            print('time, recv_data_form_spi_routine_time', time(), run_time)
+            # print('time, recv_data_form_spi_routine_time', time(), run_time)
             run_time = 0.045
 
-        if server.sync_started:
-            event.wait(self._interval - run_time)
-            sync.set_pin_mem_sel(True)
-            sleep(0.005)
-            return
-        else:
-            event.wait(self._interval - run_time)
-            sync.set_pin_mem_sel(True)
-            self.close()
-            return
+        if run_time >= 0.030:
+            print('time, recv_data_form_spi_routine_time >= 0.030', 'device:', c, datetime.now(), time(), real_run_time)
+
+        event.wait(self._interval - run_time)
+        sync.set_pin_mem_sel(True)
+        sleep(0.005)
+
+        return
 
 class RecRuntimeThread(ServerThread):
     def __init__(self, server: DataServer):

python/biopro/server/database.py

@@ -15,11 +15,12 @@ from biopro.util.json import JSON_OBJECT
 from .socket import ServerThread
 
 import psycopg2
-# from psycopg2.pool import ThreadedConnectionPool
+from psycopg2.extras import execute_batch
 from multiprocessing import Process, Queue
 
 import biopro.server._identify
 
+from time import time
 
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
@@ -50,7 +51,7 @@ class DataBaseProcess(Process):
         self._api_token = None
         self._api_header = None
         self.log_verbose = log_verbose
-        self._executor = ThreadPoolExecutor(max_workers=4)
+        self._executor = ThreadPoolExecutor(max_workers=8)
 
         self._psql_multi_thread = None
         self._psql_conn_pool = None
@@ -62,6 +63,13 @@ class DataBaseProcess(Process):
 
         self._is_close = False
 
+
+        self._meta_write_sql_str = None
+
+        self._data_raw_create_sql_str = None
+        self._data_raw_update_sql_str = None
+        self._data_raw_recording_sql_str = 'UPDATE "public"."%s_recording_data_raws" SET data = concat(data, %s), end_time=%s, size=%s where id = %s'
+
     @property
     def db_host(self) -> str:
         return self._db_host
@@ -77,6 +85,13 @@ class DataBaseProcess(Process):
     @property
     def api_token(self) -> str:
         return self._api_token
+    
+    def calculate_time(func):
+        def warp(*args, **kwargs):
+            now = time()
+            func(*args, **kwargs)
+            print(func.__name__,'cost:', time() - now)
+        return warp
 
     def run(self) -> None:
         self.setup()
@@ -149,7 +164,8 @@ class DataBaseProcess(Process):
         if callable(getattr(self, do)):
             getattr(self, do)(*args)
         return None
-    
+
+    # @calculate_time
     def data_meta_write(self, _data: JSON_OBJECT, device_id, _uuid = None, _id = None):
         if _id > 0:
             sql_str = 'UPDATE "public"."recording_data_metas" SET '
@@ -307,39 +323,29 @@ class DataBaseProcess(Process):
         # _conn.close()
         return None
 
+    # @calculate_time
     def data_raw_create(self, _data_dict, _channel_list, device_id):
-
-        for _channel in _channel_list:
-            sql_str = 'INSERT INTO "public"."' + str(_channel) + '_recording_data_raws" ('
-            sql_set = []
-            for item in _data_dict[_channel].keys():
-                sql_str = sql_str + str(item) + ', '
-                if isinstance(_data_dict[_channel][item], dict):
-                    sql_set.append(json_stringify(_data_dict[_channel][item]))
-                else:
-                    sql_set.append(str(_data_dict[_channel][item]))
-
-            sql_str = sql_str[0:-2] + ') VALUES ('
-            for item in _data_dict[_channel].keys():
-                sql_str = sql_str + '%s, '
-
-            sql_str = sql_str[0:-2] + ') RETURNING id'
-            sql_cursor = self._psql_conn.cursor()
-
-            try:
-                sql_cursor.execute(sql_str, sql_set)
-            except RuntimeError as e:
-                print('data_raw_create error', e)
-                self._psql_conn.commit()
-                sql_cursor.close()
-                self._queue_error.put(device_id)
-            finally:
-                self._queue_ds[int(device_id)].put(['update_raw_dict_id', _data_dict[_channel]['uuid'], _channel, int(sql_cursor.fetchone()[0])])
-                self._psql_conn.commit()
-                sql_cursor.close()
+        if self._data_raw_create_sql_str == None:
+            sql_str_list = []
+            key_list = _data_dict[_channel_list[0]].keys()
+            key_str = ','.join(key_list)
+            values = ','.join(['%s'] * len(key_list))
+            sql_str_list = ['INSERT INTO "public"."{channel}_recording_data_raws" (', key_str, ') VALUES (', values, ') RETURNING id']
+            self._data_raw_create_sql_str = ''.join(sql_str_list)
+        try:
+            with self._psql_conn as conn:
+                    with conn.cursor() as sql_cursor:
+                        for _channel in _channel_list:
+                            sql_set = list(map(lambda x: str(x), _data_dict[_channel].values()))
+                            sql_cursor.execute(self._data_raw_create_sql_str.format(channel = _channel), sql_set)
+                            self._queue_ds[int(device_id)].put(['update_raw_dict_id', _data_dict[_channel]['uuid'], _channel, int(sql_cursor.fetchone()[0])])
+        except (psycopg2.Error, RuntimeError) as e:
+            print('data_raw_create error', e)
+            self._queue_error.put(device_id)
 
         return True
 
+    # @calculate_time
     def data_raw_update(self, _id, _channel, _data: JSON_OBJECT):
         sql_str = 'UPDATE "public"."' + str(_channel) + '_recording_data_raws" SET '
         sql_set = []
@@ -357,28 +363,33 @@ class DataBaseProcess(Process):
         sql_cursor.execute(sql_str, sql_set)
         self._psql_conn.commit()
         sql_cursor.close()
-        
         return None
 
-    def data_raw_recording(self, _id_dict, _channel_list, _data_dict):
-        for _channel in _channel_list:
-            compressed_value = _data_dict[_channel] + '"***"'
-            # compressed_value = bytes(_data_dict[_channel],'utf-8')
-            # compressed_value = zlib.compress(compressed_value)
-            # compressed_value = base64.b64encode(compressed_value).decode() + '"***"'
-            # compressed_value = _data + '"***"'
-            sql_str = 'UPDATE "public"."' + str(_channel) + '_recording_data_raws" SET data = concat(data, %s) where id = %s'
-            sql_set = [compressed_value, _id_dict[_channel]]
-
-            sql_cursor = self._psql_conn.cursor()
-            sql_cursor.execute(sql_str, sql_set)
-            self._psql_conn.commit()
-            sql_cursor.close()
-
-            del _data_dict[_channel]
-            del compressed_value
-            gc.collect()
-            # _conn.close()
+    # @calculate_time
+    def data_raw_recording(self, _id_dict, _channel_list, _data_dict, _id_db_save):
+        try:
+            with self._psql_conn as conn:
+                with conn.cursor() as sql_cursor:
+                    para_list = []
+                    for _channel in _channel_list:
+                        temp_id = _id_dict[_channel] if _id_dict[_channel] != 0 else _id_db_save[_channel]
+                        para_list.append([_channel, _data_dict[_channel] + '"***"', temp_id])
+                    execute_batch(sql_cursor, self._data_raw_recording_sql_str, para_list)
+        except psycopg2.Error as e:
+            print('recording error', e)
+        return None
+    
+    # @calculate_time
+    def data_raw_recording_new(self, _id_dict, _channel_list, _data_dict, _end_time_dict, _size_dict):
+        try:
+            with self._psql_conn as conn:
+                with conn.cursor() as sql_cursor:
+                    para_list = []
+                    for _channel in _channel_list:
+                        para_list.append([_channel, _data_dict[_channel] + '"***"', _end_time_dict[_channel], _size_dict[_channel], _id_dict[_channel]])
+                    execute_batch(sql_cursor, self._data_raw_recording_sql_str, para_list)
+        except psycopg2.Error as e:
+            print('recording error', e)
         return None
 
     def data_mini_create(self, _data_dict, _channel_list, _scale, device_id):
@@ -415,38 +426,24 @@ class DataBaseProcess(Process):
         return True
 
     def data_mini_recording(self, _id_dict, _channel_list, _start_time_dict, _data_mean_dict):
-        for _channel in _channel_list:
-            data = _data_mean_dict.get(_channel)
-            if data is not None:
-                if len(data) != 0:
-                    str_mean = [str(int) for int in _data_mean_dict[_channel]]
-                    data_mean = _start_time_dict[_channel] + ' ' + ' '.join(str_mean)
-                    # data_random = _start_time_dict[_channel] + ' ' + ' '.join(_data_random_dict[_channel])
-                    # data_bar = _start_time_dict[_channel] + ' ' + ' '.join(_data_bar_dict[_channel])
+        with self._psql_conn as conn:
+            with self._psql_conn.cursor() as sql_cursor:
+                for _channel in _channel_list:
+                    data = _data_mean_dict.get(_channel)
+                    if data is not None:
+                        if len(data) != 0:
+                            str_mean = [str(int) for int in _data_mean_dict[_channel]]
+                            data_mean = _start_time_dict[_channel] + ' ' + ' '.join(str_mean)
 
-                    compressed_mean = data_mean + '"***"'
-
-                    # compressed_mean = bytes(data_mean,'utf-8')
-                    # compressed_mean = zlib.compress(compressed_mean)
-                    # compressed_mean = base64.b64encode(compressed_mean).decode() + '"***"'
-
-                    # compressed_random = bytes(data_bar,'utf-8')
-                    # compressed_random = zlib.compress(compressed_random)
-                    # compressed_random = base64.b64encode(compressed_random).decode() + '"***"'
-
-                    # compressed_random = bytes(data_random,'utf-8')
-                    # compressed_random = zlib.compress(compressed_random)
-                    # compressed_random = base64.b64encode(compressed_random).decode() + '"***"'
                     sql_str = 'UPDATE "public"."' + str(_channel) + '_recording_data_minis" SET data_mean = concat(data_mean, %s) where id = %s'
-                    sql_set = [compressed_mean, _id_dict[_channel]]
-
-                    sql_cursor = self._psql_conn.cursor()
+                    sql_set = [data_mean + '"***"', _id_dict[_channel]]
                     sql_cursor.execute(sql_str, sql_set)
-                    self._psql_conn.commit()
-                    sql_cursor.close()
 
-                    del str_mean, _start_time_dict[_channel], _data_mean_dict[_channel], data_mean, compressed_mean
-                    gc.collect()
+                    del str_mean, data_mean
+        
+        for _channel in _channel_list:
+            del _start_time_dict[_channel], _data_mean_dict[_channel]
+        gc.collect()
         
         return True
     
@@ -469,3 +466,42 @@ class DataBaseProcess(Process):
         sql_cursor.close()
 
         return True
+
+    def data_meta_update_raw_empty_channel(self, meta_id, channel_list):
+        try:
+            sql_str = "UPDATE recording_data_metas SET raw_data = jsonb_set(raw_data, '{%s}', '[]'), mini_data = jsonb_set(mini_data, '{%s}', '{\"10\": [], \"100\": [], \"1000\": []}') WHERE id = %s;"
+            sql_set = []
+            for channel in channel_list:
+                sql_set.append([channel, channel, meta_id])
+            
+            with self._psql_conn as conn:
+                with conn.cursor() as sql_cursor:
+                    execute_batch(sql_cursor, sql_str, sql_set)
+        except psycopg2.Error as e:
+            print('e', e)
+
+        return True
+
+    def data_meta_update_channel_raw_id(self, meta_id, channel, raw_id):
+        try:
+            with self._psql_conn as conn:
+                    with conn.cursor() as sql_cursor:
+                        sql_str = "UPDATE recording_data_metas SET raw_data = jsonb_insert(raw_data, '{%s, -1}', '%s', TRUE) WHERE id = %s;"
+                        sql_set = [channel, raw_id, meta_id]
+                        sql_cursor.execute(sql_str, sql_set)
+        except psycopg2.Error as e:
+            print('e', e)
+
+        return True
+    
+    def data_meta_update_channel_mini_id(self, meta_id, channel, scale, raw_id):
+        try:
+            with self._psql_conn as conn:
+                    with conn.cursor() as sql_cursor:
+                        sql_str = "UPDATE recording_data_metas SET mini_data = jsonb_insert(mini_data, '{%s, %s, -1}', '%s', TRUE) WHERE id = %s;"
+                        sql_set = [channel, scale, raw_id, meta_id]
+                        sql_cursor.execute(sql_str, sql_set)
+        except psycopg2.Error as e:
+            print('e', e)
+
+        return True

python/biopro/server/recording.py

@@ -186,7 +186,7 @@ class RecordingProcess(Process):
         current_time = time()
         if self._timer is not None:
             if current_time - self._timer > 1.5:
-                print('time, sync_data routine_time', current_time, current_time - self._timer)
+                print('time, sync_data routine_time', 'device:', self._device, datetime.now(), current_time, current_time - self._timer)
                 self._isTimeOut = True
         self._timer = current_time
 
@@ -363,14 +363,13 @@ class RecordingProcess(Process):
                     if (device == 7):
                         print('self._last_cnt[7]:', self._last_cnt[7], ', last_data_cnt:', last_data_cnt)
                         
-                    print('skip RAM, but data is right, solve data', list(raw_data[i][0:3]), 'skip Ram', self._skip_ram_cnt, 'times')
+                    print('skip RAM, but data is right, solve data', list(raw_data[i][0:3]),'|', list(raw_data[colum_total-1][0:3]), 'skip Ram', self._skip_ram_cnt, 'times \n')
                     # print('data:')
                     # print('|', list(raw_data[0]))
                     # print('|', list(raw_data[1]))
                     # print('| ... ... ...')
                     # print('| ... ... ...')
                     # print('|', list(raw_data[colum_total-1]))
-                    print()
 
                     if (device == 4):
                         self._last_cnt[4] = last_data_cnt
@@ -500,7 +499,7 @@ class RecordingProcess(Process):
         index = 0
         elite_data_len = 40
         mem_header_len = 3
-        mem_tailer_len = 8
+        mem_tailer_len = 6
         single_data_len = elite_data_len + mem_header_len + mem_tailer_len
         device = self._device
         print_flag = False
@@ -524,23 +523,31 @@ class RecordingProcess(Process):
             if (save == True):
                 section = raw_data[i]
 
-                if (section[0 : mem_header_len] == section[single_data_len - 3: single_data_len]):
-                    section = section + b'\x00'
-                
-                else:
+                check_sum = sum(section[0 : single_data_len - 1]) & 0b11111111
+
+                if (check_sum != section[single_data_len - 1]):
+                    print('check number not match,',
+                        list(section[0 : mem_header_len]), section[-1], '!=', check_sum, 'device:', device, ',', datetime.now())
+                    if (i-1 >= 0):
+                        print('check number not match', list(raw_data[i-1]), 'raw:', i-1)
+
+                    print('check number not match', list(raw_data[i]), 'raw:', i)
+                    
+                    if (i+1 <= index):
+                        print('check number not match', list(raw_data[i+1]), 'raw:', i+1, '\n')
                     section = section + b'\x01'
-                    print('header != tailer,', 
-                        list(section[0 : mem_header_len]), 
-                        list(section[single_data_len - 3: single_data_len]),
-                        'device:', device, ',', datetime.now())
-                    print('header != tailer', list(section), '\n')
-                
-                section = section[3:]
+
+                    print('delete data:', list(raw_data[i][0:3]), 'raw:', i, '\n')
+                    continue
+                else:
+                    section = section + b'\x00'
+
+                # section = section[3:]
                 yield None, section
 
             else:
                 print_flag = True
-                print('delete data:', list(raw_data[i]))
+                print('delete data:', list(raw_data[i][0:3]), 'raw:', i, '\n')
 
         # if print_flag:
         #     print('delete data context:')