[update] need to try execute_batch for optimize

src/controllers/file/download_functional.py

@@ -0,0 +1,142 @@
+import psycopg2
+import pandas as pd
+import numpy as np
+from os.path import exists
+import sys
+import os
+import time
+
+def log(df, channel):
+    # TODO: not sure need use 32 or 64? => 64, since large integral will cause overflow
+    df[f'{channel}_log'] = np.log10(np.absolute(df[channel]))
+    # df[f'{channel}_log'] = np.log10(np.absolute(df[channel].astype(np.int64)))
+    return df
+def integral(df, x_channel, y_channel, pre_last_x, pre_last_y, integral_result):
+    offset = df[x_channel]
+    offset = offset.shift(periods=1)
+    offset[0] = pre_last_x
+    x_diff = df[x_channel] - offset
+    # x_diff = df[x_channel].astype(np.int64) - offset.astype(np.int64)
+    # print('x_diff\n', x_diff)
+    offset = df[y_channel]
+    offset = offset.shift(periods=1)
+    offset[0] = pre_last_y
+    y_avg = (offset + df[y_channel]) / 2
+    # y_avg = (offset.astype(np.int64) + df[y_channel].astype(np.int64)) / 2
+    # print('y_avg:', y_avg)
+    # print('y_avg: ', y_avg)
+    local_integral = x_diff * y_avg
+    # TODO: if user don't need, then can remove it
+    df[f'{x_channel}_{y_channel}_integral_delta'] = local_integral
+    local_integral[0] += integral_result
+    df[f'{x_channel}_{y_channel}_integral'] = local_integral.cumsum()
+    # print(df)
+    pre_last_x = df[x_channel].iat[-1]
+    pre_last_y = df[y_channel].iat[-1]
+    integral_result = df[f'{x_channel}_{y_channel}_integral'].iat[-1]
+    # print('next run info: ', pre_last_x, pre_last_y, integral_result)
+    return df, pre_last_x, pre_last_y, integral_result
+def differential(df, x_channel, y_channel, pre_last_x, pre_last_y):
+    offset = df[x_channel]
+    offset = offset.shift(periods=1)
+    offset[0] = pre_last_x
+    x_diff = df[x_channel] - offset
+    # x_diff = df[x_channel].astype(np.int64) - offset.astype(np.int64)
+    offset = df[y_channel]
+    offset = offset.shift(periods=1)
+    offset[0] = pre_last_y
+    y_diff = df[y_channel] - offset
+    # y_diff = df[y_channel].astype(np.int64) - offset.astype(np.int64)
+    local_diff = y_diff / x_diff
+    # TODO: if user don't need, then can remove it
+    df[f'{x_channel}_{y_channel}_diff'] = local_diff
+    pre_last_x = df[x_channel].iat[-1]
+    pre_last_y = df[y_channel].iat[-1]
+    return df, pre_last_x, pre_last_y
+
+def download(id, mode, x_channel, y_channel):
+    # TODO: can change different chunck size
+    pages_chunck_size = 250
+    sql_total = 0
+    conn_start = time.time()
+    # TODO: need the check port id
+    conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="54321")
+    print('conn time: ', time.time() - conn_start)
+    # conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="5432")
+    cur = conn.cursor()
+    # recording_data_metas: meta info to get the raw_data table
+    sql_str = f'select "raw_data" from "recording_data_metas" WHERE id = {id}'
+    first_sql = time.time()
+    cur.execute(sql_str)
+    print('first sql time: ', time.time() - first_sql)
+    raw_data_list = None
+    try:
+        raw_data_list = cur.fetchall()[0][0]
+        channels = list(raw_data_list.keys())
+        df_all = pd.DataFrame()
+        # df = pd.DataFrame()
+        df_time = []
+        pre_last_x = 0
+        pre_last_y = 0
+        integral_result = 0
+        for raw_data_pages in range(len(raw_data_list[channels[0]])):                
+            df = pd.DataFrame()
+            for channel in channels:
+                sql_str = f'select "data" from "{channel}_recording_data_raws" WHERE id = {raw_data_list[channel][raw_data_pages]}'
+                sql_start = time.time()
+                cur.execute(sql_str)
+                sql_total += (time.time() - sql_start)
+                data_value = cur.fetchall()[0][0].replace('"***"', ' ')
+                data_value = data_value.split(' ')[:-1]
+                if df.empty:
+                    df_time = data_value[::2]
+                df[channel] = data_value
+            df = df.iloc[lambda x: x.index % 2 == 1].reset_index(drop=True)
+            df.insert(0, 'Time', df_time)
+            # transfer dataframe from string to number
+            cols = df.columns
+            df[cols] = df[cols].apply(pd.to_numeric, errors='coerce')
+            if mode == 'Log':
+                df = log(df, x_channel)
+            elif mode == 'Integral':
+                df, pre_last_x, pre_last_y, integral_result = integral(df, x_channel, y_channel, pre_last_x, pre_last_y, integral_result)
+            elif mode == 'Differential':
+                df, pre_last_x, pre_last_y = differential(df, x_channel, y_channel, pre_last_x, pre_last_y)
+            df_all = pd.concat([df_all, df], ignore_index=True, sort=False)
+            if ~raw_data_pages and (raw_data_pages % pages_chunck_size == 0 or raw_data_pages == len(raw_data_list[channels[0]]) - 1):
+                if not(exists(f"./src/csv/{id}.csv")):
+                    df.to_csv(f"./src/csv/{id}.csv", mode='a', index=False, header=True)
+                else:
+                    df.to_csv(f"./src/csv/{id}.csv", mode='a', index=False, header=False)
+                """ if not(exists(f"./{id}.csv")):
+                    df_all.to_csv(f"./{id}.csv", mode='a', index=False, header=True)
+                else:
+                    df_all.to_csv(f"./{id}.csv", mode='a', index=False, header=False) """
+                df_all = pd.DataFrame()
+        print('total sql: ', sql_total)
+    except BaseException as e:
+        print(e)
+        exit
+    return
+
+if __name__ == '__main__':
+    params = {}
+    for input in sys.argv[1:]: #Now we're going to iterate over argv[1:] (argv[0] is the program name)
+        param  = input.split("=") #Get what's left of the '='
+        params[param[0]] = param[1]
+
+    id = params['id']
+    mode = params['mode']
+    x_channel = params['x_channel']
+    y_channel = params['y_channel']
+    # print('download_functional init at main: ', id, mode, x_channel, y_channel)
+    start_time = time.time()
+    download(id, mode, x_channel, y_channel)
+    # download(408, '', 'Time', '0')
+    # download(408, '', 'Time', '0')
+    # download(405, 'Integral', 'Time', '0')
+    # download(405, 'Integral', 'Time', '0')
+    # download(408, 'Integral', 'Time', '0')
+    print('total: ', time.time() - start_time)
+    # download(1262, 'differential', '1', '0')
+    # 1262/differential/1/0

src/controllers/file/recording_data_raw.js

@@ -1,7 +1,12 @@
 import RecordingDataRaw from '../../models/file/recording_data_raw'
+import RecordingDataMeta from '../../models/file/recording_data_meta'
 import * as auth from '../auth'
+import childProcess from 'child_process'
+import path from 'path'
+import { createReadStream } from 'fs'
 
 const recordingDataRaw = new RecordingDataRaw()
+const recordingDataMeta = new RecordingDataMeta()
 
 export const create = async (ctx, next) => {
   const index = ctx.params.channel
@@ -162,3 +167,37 @@ export const getAll = async (ctx, next) => {
 
   next()
 }
+
+export const getCSV = async (ctx, next) => {
+  const id = ctx.params.id
+  const mode = ctx.params.mode
+  const x_channel = ctx.params.x_channel
+  const y_channel = ctx.params.y_channel
+  // console.log('getCSV: ', id, mode, x_channel, y_channel)
+  // const result = await recordingDataRaw.getCSV(id, mode, x_channel, y_channel)
+  const process = childProcess.spawnSync('python', [
+    path.join(__dirname, '/download_functional.py'),
+    'id=' + id,
+    'mode=' + mode,
+    'x_channel=' + x_channel,
+    'y_channel=' + y_channel,
+  ], { maxBuffer: 2 * 1024 * 1024 * 1024 })
+  // console.log('process: ', process)
+  // process.stdout.on('data', (data) => {
+  //   conosle.log('data', data)
+  // })
+  const result = await recordingDataMeta.getByID(id)
+  // console.log('result: ', result)
+  // console.log('result: ', result[0]['dataValues']['name'])
+  var file_name = id + '.csv'
+  // ctx.body = fs.stdout.toString()
+  // console.log('process.stdout.toString(): ', process.stdout.toString())
+  // console.log('process.stderr.toString(): ', process.stderr.toString())
+  ctx.type = 'stream'
+  ctx.attachment(result[0]['dataValues']['name']+'.csv')
+  ctx.body = createReadStream(path.resolve(__dirname, '../../csv', file_name))
+  // need to remove the csv file after download, since using 'append' for set header one time
+  const fs = require('fs')
+  fs.unlinkSync(path.resolve(__dirname, '../../csv', file_name))
+  next()
+}

src/controllers/file/test.txt

@@ -0,0 +1,60 @@
+          time       0        1        2  3  time_0_integral
+0            0  -73922  -836265  -848970  0     0.000000e+00
+1        10500  -84368  -834380  -848548  0    -8.310225e+08
+2        21000  -84389  -834380  -848126  0    -1.716997e+09
+3        31500  -84332  -835637  -847704  0    -2.602782e+09
+4        42500  -84166  -835637  -847283  0    -3.529521e+09
+...        ...     ...      ...      ... ..              ...
+1462  15390500  -45761  -449806  -466674  0    -9.716838e+11
+1463  15401000  -45833  -449806  -466445  0    -9.721646e+11
+1464  15411500  -45790  -451063  -466216  0    -9.726457e+11
+1465  15422500  -45638  -449806  -465988  0    -9.731485e+11
+1466  15432500  -45674  -449806  -465759  0    -9.736051e+11
+
+[1467 rows x 6 columns]
+next run info:  15432500 -45674 -973605066500.0
+          time       0        1        2  3  time_0_integral
+0     15443000  -45689  -448549  -465531  0    -9.740847e+11
+1     15453500  -45689  -448549  -465531  0    -9.745645e+11
+2     15464000  -45631  -449178  -465303  0    -9.750439e+11
+3     15474500  -45581  -447921  -465075  0    -9.755227e+11
+4     15485500  -45646  -447921  -464847  0    -9.760245e+11
+...        ...     ...      ...      ... ..              ...
+1443  30720000  -24971  -248122  -259749  0    -1.497382e+12
+1444  30730500  -24920  -247950  -259624  0    -1.497644e+12
+1445  30741000  -24920  -248381  -259624  0    -1.497905e+12
+1446  30751500  -24791  -248036  -259500  0    -1.498166e+12
+1447  30762000  -24856  -247346  -259376  0    -1.498427e+12
+
+[1448 rows x 6 columns]
+next run info:  30762000 -24856 -1498426872750.0
+          time       0        1        2  3  time_0_integral
+0     30772500  -24913  -247669  -259251  0    -1.498688e+12
+1     30783000  -24848  -247669  -259127  0    -1.498949e+12
+2     30794000  -24805  -247669  -259003  0    -1.499222e+12
+3     30804000  -24805  -247454  -259003  0    -1.499471e+12
+4     30814500  -24870  -247238  -258879  0    -1.499731e+12
+...        ...     ...      ...      ... ..              ...
+1456  46091500  -13496  -134943  -147002  0    -1.783935e+12
+1457  46102000  -13510  -134921  -146934  0    -1.784077e+12
+1458  46112500  -13510  -134123  -146934  0    -1.784219e+12
+1459  46123000  -13597  -134339  -146866  0    -1.784361e+12
+1460  46134000  -13525  -134598  -146799  0    -1.784510e+12
+
+[1461 rows x 6 columns]
+next run info:  46134000 -13525 -1784510350500.0
+         time       0        1        2  3  time_0_integral
+0    46144500  -13366  -134210  -146732  0    -1.784652e+12
+1    46154500  -13460  -134210  -146665  0    -1.784786e+12
+2    46165500  -13503  -134016  -146597  0    -1.784934e+12
+3    46176000  -13503  -134102  -146597  0    -1.785076e+12
+4    46186500  -13431  -134167  -146530  0    -1.785217e+12
+..        ...     ...      ...      ... ..              ...
+583  52277500  -10487  -105193  -117883  0    -1.858083e+12
+584  52288000  -10573  -105042  -117830  0    -1.858194e+12
+585  52298500  -10595  -105106  -117778  0    -1.858305e+12
+586  52309000  -10494  -105085  -117725  0    -1.858415e+12
+587  52319500  -10523  -105085  -117672  0    -1.858526e+12
+
+[588 rows x 6 columns]
+next run info:  52319500 -10523 -1858525771000.0

src/routes/main-routes.js

@@ -59,6 +59,7 @@ export default router
   .get('/api/file/raw/get_attr_by_id/:channel/:id/:attr', controllers.recordingDataRaw.getAttrByID)
   .get('/api/file/raw/get_attr_by_ids/:channel/:id/:attr', controllers.recordingDataRaw.getAttrByIDs)
   .get('/api/file/raw/get_attr_by_parent/:channel/:parent/:attr', controllers.recordingDataRaw.getAttrByParent)
+  .get('/api/file/raw/get_csv/:id/:mode/:x_channel/:y_channel', controllers.recordingDataRaw.getCSV)
 
   // mini
   .post('/api/file/mini/create/:channel', controllers.recordingDataMini.create)