[debug]

src/controllers/file/download_functional.py

@@ -9,7 +9,6 @@ import time
 def log(df, channel):
     # TODO: not sure need use 32 or 64? => 64, since large integral will cause overflow
     df[f'{channel}_log'] = np.log10(np.absolute(df[channel]))
-    # df[f'{channel}_log'] = np.log10(np.absolute(df[channel].astype(np.int64)))
     return df
 def integral(df, x_channel, y_channel, pre_last_x, pre_last_y, integral_result):
     offset = df[x_channel]
@@ -36,6 +35,7 @@ def integral(df, x_channel, y_channel, pre_last_x, pre_last_y, integral_result):
     integral_result = df[f'{x_channel}_{y_channel}_integral'].iat[-1]
     # print('next run info: ', pre_last_x, pre_last_y, integral_result)
     return df, pre_last_x, pre_last_y, integral_result
+
 def differential(df, x_channel, y_channel, pre_last_x, pre_last_y):
     offset = df[x_channel]
     offset = offset.shift(periods=1)
@@ -62,40 +62,42 @@ def download(id, mode, x_channel, y_channel):
     # TODO: need the check port id
     conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="54321")
     print('conn time: ', time.time() - conn_start)
-    # conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="5432")
-    cur = conn.cursor()
+    cursor = conn.cursor()
     # recording_data_metas: meta info to get the raw_data table
-    sql_str = f'select "raw_data" from "recording_data_metas" WHERE id = {id}'
+    sql_str = f'SELECT "raw_data" FROM "recording_data_metas" WHERE id = {id}'
     first_sql = time.time()
-    cur.execute(sql_str)
+    cursor.execute(sql_str)
     print('first sql time: ', time.time() - first_sql)
     raw_data_list = None
     try:
-        raw_data_list = cur.fetchall()[0][0]
-        channels = list(raw_data_list.keys())
+        raw_data_list = cursor.fetchall()[0][0]
+        channels = list(raw_data_list.keys()) # [0,1,2,3]
         df_all = pd.DataFrame()
         # df = pd.DataFrame()
         df_time = []
         pre_last_x = 0
         pre_last_y = 0
         integral_result = 0
-        for raw_data_pages in range(len(raw_data_list[channels[0]])):                
+        print('raw_data_list', raw_data_list)
+        print('channels', channels)
+        for page in range(len(raw_data_list[channels[0]])):
             df = pd.DataFrame()
+            print(raw_data_list[channels[0]][page])
             for channel in channels:
-                sql_str = f'select "data" from "{channel}_recording_data_raws" WHERE id = {raw_data_list[channel][raw_data_pages]}'
+                raw_data_page = raw_data_list[channel][page]
+                sql_str = f'SELECT "data" FROM "{channel}_recording_data_raws" WHERE id = {raw_data_list[channel][page]}'
                 sql_start = time.time()
-                cur.execute(sql_str)
+                cursor.execute(sql_str)
                 sql_total += (time.time() - sql_start)
-                data_value = cur.fetchall()[0][0].replace('"***"', ' ')
-                data_value = data_value.split(' ')[:-1]
-                if df.empty:
-                    df_time = data_value[::2]
-                df[channel] = data_value
-            df = df.iloc[lambda x: x.index % 2 == 1].reset_index(drop=True)
-            df.insert(0, 'Time', df_time)
+                data_value = cursor.fetchall()[0][0].replace('"***"', ' ').split(' ')
+                if ~('Time' in df.columns):
+                    df['Time'] = data_value[:-1:2]
+                df[channel] = data_value[1:-1:2]
+            # df = df.loc[lambda x: x.index % 2 == 1].reset_index(drop=True)
+            # df.insert(0, 'Time', df_time)
             # transfer dataframe from string to number
             cols = df.columns
-            df[cols] = df[cols].apply(pd.to_numeric, errors='coerce')
+            df[cols] = df[cols].apply(pd.to_numeric, errors='coerce') # coerce force non-a-number string be converted to NaN
             if mode == 'Log':
                 df = log(df, x_channel)
             elif mode == 'Integral':
@@ -103,16 +105,10 @@ def download(id, mode, x_channel, y_channel):
             elif mode == 'Differential':
                 df, pre_last_x, pre_last_y = differential(df, x_channel, y_channel, pre_last_x, pre_last_y)
             df_all = pd.concat([df_all, df], ignore_index=True, sort=False)
-            if ~raw_data_pages and (raw_data_pages % pages_chunck_size == 0 or raw_data_pages == len(raw_data_list[channels[0]]) - 1):
-                if not(exists(f"./src/csv/{id}.csv")):
-                    df.to_csv(f"./src/csv/{id}.csv", mode='a', index=False, header=True)
-                else:
-                    df.to_csv(f"./src/csv/{id}.csv", mode='a', index=False, header=False)
-                """ if not(exists(f"./{id}.csv")):
-                    df_all.to_csv(f"./{id}.csv", mode='a', index=False, header=True)
-                else:
-                    df_all.to_csv(f"./{id}.csv", mode='a', index=False, header=False) """
-                df_all = pd.DataFrame()
+            print(df_all)
+            with open(f"../../csv/{id}.csv", mode = 'a') as export_file:
+                df_all.to_csv(export_file, header=(export_file.tell() == 0))
+            df_all = pd.DataFrame()
         print('total sql: ', sql_total)
     except BaseException as e:
         print(e)
@@ -120,9 +116,14 @@ def download(id, mode, x_channel, y_channel):
     return
 
 if __name__ == '__main__':
-    params = {}
-    for input in sys.argv[1:]: #Now we're going to iterate over argv[1:] (argv[0] is the program name)
-        param  = input.split("=") #Get what's left of the '='
+    params = {
+        'id': 0,
+        'mode': 0,
+        'x_channel': 0,
+        'y_channel': 0
+    }
+    for input in sys.argv[1:]: # Now we're going to iterate over argv[1:] (argv[0] is the program name)
+        param = input.split("=") # Get what's left of the '='
         params[param[0]] = param[1]
 
     id = params['id']
@@ -131,9 +132,9 @@ if __name__ == '__main__':
     y_channel = params['y_channel']
     # print('download_functional init at main: ', id, mode, x_channel, y_channel)
     start_time = time.time()
-    download(id, mode, x_channel, y_channel)
-    # download(408, '', 'Time', '0')
-    # download(408, '', 'Time', '0')
+    # download(id, mode, x_channel, y_channel)
+    download(849, '', '', '')
+      # download(408, '', 'Time', '0')
     # download(405, 'Integral', 'Time', '0')
     # download(405, 'Integral', 'Time', '0')
     # download(408, 'Integral', 'Time', '0')

src/controllers/file/recording_data_raw.js

@@ -198,6 +198,6 @@ export const getCSV = async (ctx, next) => {
   ctx.body = createReadStream(path.resolve(__dirname, '../../csv', file_name))
   // need to remove the csv file after download, since using 'append' for set header one time
   const fs = require('fs')
-  fs.unlinkSync(path.resolve(__dirname, '../../csv', file_name))
+  // fs.unlinkSync(path.resolve(__dirname, '../../csv', file_name))
   next()
 }