- Add new function data_raw_recording_bytea

- Change data type from string to binary

python/biopro/recording/file.py

@@ -866,11 +866,13 @@ class RecordingFile:
 
     # def write(self, content: Union[bytes, RecordingData]) -> int:
     def write(self, content: str, channels: list) -> int:
-        if not isinstance(content, str):
-            raise RuntimeError('wrong data format : ' + repr(self._data_format))
+        # print('count_size', content, channels)
+        # if not isinstance(content, str):
+        #     raise RuntimeError('wrong data format : ' + repr(self._data_format))
 
         self._meta_file.update_channels(channels)
         sz = sys.getsizeof(content)
+        # print('sz', sz)
         self._size += sz
         return sz
 
@@ -1154,15 +1156,15 @@ class RecordingFileWriter:
                     self._send_data[ch] = False
 
                 if self._recording_file_dict[ch]._status:
-                    _data = ' '.join(self._data_db[ch])
-                    self._raw_save['data'][ch] = _data
+                    _data = self._data_db[ch]
+                    self._raw_save['data'][ch] = copy(_data)
                     self._raw_save['id'][ch] = self._recording_file_dict[ch]._id_db
                     self._recording_file_dict[ch].write(_data, self._channel_list)
                     self._recording_file_dict[ch].close(self._time_now)
                     self._meta._size += self._recording_file_dict[ch]._size
             # self._data_db.clear()
             if self._database is not None:
-                self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data'], self._id_db_save])
+                self._database.put_queue(['data_raw_recording_bytea', self._raw_save['id'], self._channel_list, self._raw_save['data'], self._id_db_save])
                 # self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data']])
             self._recording_file_dict.clear()
         for scale in self._mini_scale_list:
@@ -1324,8 +1326,8 @@ class RecordingFileWriter:
                                 #     self._data_mini_ch[c]['10000']['random'].append( str(self._data_mini_ch[c]['1000']['random'][random.randint(-10,-1)]) )
                                 #     self._data_mini_ch[c]['1000']['dec'] = int(len(self._data_mini_ch[c]['1000']['mean']) / 10)
                     # add normal data
-                    self._data_db[c].append(str(int(t)))
-                    self._data_db[c].append(str(v))
+                    self._data_db[c].append(int(t))
+                    self._data_db[c].append(v)
                     self._time_now = int(t)
         return
     
@@ -1417,9 +1419,9 @@ class RecordingFileWriter:
                 for ch in self._data_db.keys():
                     if self._time_now - self._time[ch] > 5000000:
                         if self._recording_file_dict[ch]._status:
-                            _data = ' '.join(self._data_db[ch])
+                            _data = self._data_db[ch]
                             write_sz = self._recording_file_dict[ch].write(_data, self._channel_list)
-                            self._raw_save['data'][ch] = _data
+                            self._raw_save['data'][ch] = copy(_data)
                             self._raw_save['id'][ch] = self._recording_file_dict[ch]._id_db
                             self._raw_save['end_time'][ch] = self._time_now
                             self._raw_save['size'][ch] = self._recording_file_dict[ch]._size
@@ -1446,9 +1448,15 @@ class RecordingFileWriter:
 
                 if data_save is True:
                     if self._database is not None:
-                        recording_input = ['data_raw_recording_new', copy(self._raw_save['id']), copy(self._channel_list), copy(self._raw_save['data']), copy(self._raw_save['end_time']), copy(self._raw_save['size'])]
+                        recording_input = ['data_raw_recording_bytea', copy(self._raw_save['id']), copy(self._channel_list), copy(self._raw_save['data']), copy(self._raw_save['end_time']), copy(self._raw_save['size'])]
                         self._database.put_queue(recording_input)
                         self._meta.update_subfile_time_size(database = self._database)
+
+                # if data_save is True:
+                #     if self._database is not None:
+                #         recording_input = ['data_raw_recording_new', copy(self._raw_save['id']), copy(self._channel_list), copy(self._raw_save['data']), copy(self._raw_save['end_time']), copy(self._raw_save['size'])]
+                #         self._database.put_queue(recording_input)
+                        
                 if mini_save is True:
                     if self._database is not None:
                         for scale in self._mini_scale_list:

python/biopro/server/database.py

@@ -70,7 +70,7 @@ class DataBaseProcess(Process):
         self._data_raw_create_sql_str = None
         self._data_raw_update_sql_str = None
         self._data_raw_recording_sql_str = 'UPDATE "public"."%s_recording_data_raws" SET data = concat(data, %s) where id = %s'
-        self._new_data_raw_recording_sql_str = 'UPDATE "public"."%s_recording_data_raws" SET data = concat(data, %s), end_time=%s, size=%s where id = %s'
+        self._new_data_raw_recording_sql_str = 'UPDATE "public"."%s_recording_data_raws" SET bytea_data = bytea_data || %s, end_time=%s, size=%s where id = %s'
         self._data_mini_recording_sql_str = 'UPDATE "public"."%s_recording_data_minis" SET data_mean = concat(data_mean, %s) where id = %s'
 
     @property
@@ -325,6 +325,7 @@ class DataBaseProcess(Process):
 
     # @calculate_time()
     def data_raw_create(self, _data_dict, _channel_list, device_id):
+        print('data_raw_create', _data_dict, _channel_list, device_id)
         if self._data_raw_create_sql_str == None:
             sql_str_list = []
             key_list = _data_dict[_channel_list[0]].keys()
@@ -582,3 +583,17 @@ class DataBaseProcess(Process):
             self._queue_ds[int(device_id)].put(['project_id', int(sql_cursor.fetchone()[0])])
             self._psql_conn.commit()
             sql_cursor.close()
+    
+    def data_raw_recording_bytea(self, _id_dict, _channel_list, _data_dict, _end_time_dict, _size_dict):
+        # print('data_raw_recording_bytea', _id_dict, _channel_list, _data_dict, _end_time_dict, _size_dict)
+        try:
+            para_list = []
+            for _channel in _channel_list:
+                bytes_data = b''.join([int.to_bytes(i, 8, 'big', signed=True) for i in _data_dict[_channel]])
+                para_list.append([_channel, bytes_data, _end_time_dict[_channel], _size_dict[_channel], _id_dict[_channel]])
+            with self._psql_conn as conn:
+                with conn.cursor() as sql_cursor:
+                    execute_batch(sql_cursor, self._new_data_raw_recording_sql_str, para_list)
+        except psycopg2.Error as e:
+            print('recording error', e)
+        return None

update_db_package.sh

@@ -35,4 +35,10 @@ sudo su -c "psql -d postgres -c \"ALTER TABLE devices ALTER COLUMN calibration D
 sudo su -c "psql -d postgres -c \"ALTER TABLE devices ALTER COLUMN calibration TYPE bytea USING calibration::bytea;\"" postgres
 
 # add column project in recording_data_metas
-sudo su -c "psql -d postgres -c \"ALTER TABLE devices ADD COLUMN IF NOT EXISTS calibration_version Int4 DEFAULT -1;\"" postgres
+sudo su -c "psql -d postgres -c \"ALTER TABLE devices ADD COLUMN IF NOT EXISTS calibration_version Int4 DEFAULT -1;\"" postgres
+
+# add column bytea_data column in 0-32_recording_data_raws
+for i in {0..32}
+do
+  sudo su -c "psql -d postgres -c \"ALTER TABLE \\\"${i}_recording_data_raws\\\" ADD COLUMN IF NOT EXISTS bytea_data BYTEA DEFAULT ''::bytea;\"" postgres
+done