[update] down sample

package.json

@@ -7,7 +7,7 @@
   "scripts": {
     "start": "gulp nodemon",
     "dev": "gulp",
-    "build": "babel src -d dist",
+    "build": "babel src -d dist --copy-files",
     "production": "node dist/app.js",
     "test": "jest",
     "link": "eslint .",

src/controllers/analysis/index.js

@@ -0,0 +1,57 @@
+import RecordingDataMeta from '../../models/file/recording_data_meta'
+import childProcess from 'child_process'
+import path from 'path'
+
+const recordingDataMeta = new RecordingDataMeta()
+
+export const spikeDetect = async (ctx, next) => {
+  const request = ctx.request.body
+
+  const id = request.id
+  const channel = request.channel
+  const cutOffFreq = request.cut_off_freq
+  const threshold = request.threshold
+  const waveForm = request.waveForm
+  const retDataType = request.ret_data_type
+  // console.log(id, channel, cutOffFreq, threshold, waveForm)
+
+  const result = await recordingDataMeta.getByID(id)
+  const channels = JSON.parse(result[0].channels)
+  const sampleRate = 8e5 / result[0].parameter_set.ADC_CLOCK / channels.length
+  const timeDuration = result[0].time_duration / 1e6
+  // console.log(sampleRate, timeDuration)
+
+  const process = childProcess.spawnSync('python3', [
+    path.join(__dirname, '/spikeDetect.py'),
+    'meta_id=' + id,
+    'channel=' + channel,
+    'cut_off_freq=' + cutOffFreq,
+    'threshold=' + threshold,
+    'waveform=' + waveForm,
+    'sample_rate=' + sampleRate,
+    'time_duration=' + timeDuration,
+    'ret_data_type=' + retDataType
+  ], { maxBuffer: 2 * 1024 * 1024 * 1024 })
+
+  // console.log('out', process.stdout.toString())
+  console.log('err', process.stderr.toString())
+
+  ctx.body = process.stdout.toString()
+
+  // const process = child_process.execSync('python3 /Users/lubokai/Desktop/bioproapiserver/src/controllers/analysis/test.py 1089 0')
+  // console.log(process)
+  // process.stdout.on('data', (data) => {
+  //   _data += data.toString()
+  // })
+
+  // process.stderr.on('data', (data) => {
+  //   console.error(data.toString())
+  // })
+
+  // process.on('exit', (code) => {
+  //   console.log(_data)
+  //   console.log(`Child exited with code ${code}`)
+  // })
+
+  next()
+}

src/controllers/analysis/spikeDetect.py

@@ -0,0 +1,121 @@
+import sys
+import ast
+import time
+import psycopg2
+import numpy as np
+import pandas as pd
+
+from math import ceil, floor
+from json import loads, dumps
+from scipy.fft import fft, fftfreq, rfft, rfftfreq, irfft, ifft
+
+import lttb
+from lttb.validators import has_two_columns, x_is_regular
+
+def get_downsampled_signal(ori_signal, output_length):
+  signal_length = len(ori_signal)
+  signal = np.array([range(signal_length), ori_signal]).T
+  assert signal.shape == (signal_length, 2)
+
+  # Downsampling
+  downsampled_signal = lttb.downsample(signal, n_out=output_length, validators=[has_two_columns, x_is_regular])
+  assert downsampled_signal.shape == (output_length, 2)
+  downsampled_signal_index = downsampled_signal[:, 0]
+  downsampled_signal_value = downsampled_signal[:, 1]
+  return downsampled_signal_index, downsampled_signal_value
+
+if __name__ == '__main__':
+
+  params = {}
+  for input in sys.argv[1:]: #Now we're going to iterate over argv[1:] (argv[0] is the program name)
+      param  = input.split("=") #Get what's left of the '='
+      params[param[0]] = param[1]
+
+  meta_id = params['meta_id']
+  channel = params['channel']
+  cut_off_freq = int(params['cut_off_freq'])
+  threshold = ast.literal_eval('[' + params['threshold'] + ']')
+  waveform = ast.literal_eval('[' + params['waveform'] + ']')
+  SAMPLE_RATE = int(float(params['sample_rate']))
+  DURATION = ceil(float(params['time_duration']))
+  DELTA_TIME = 1e6 / SAMPLE_RATE
+  N = SAMPLE_RATE * DURATION
+  ret_data_type = params['ret_data_type']
+
+  # print(meta_id, channel, cut_off_freq, threshold, waveForm, SAMPLE_RATE, DURATION, N)
+
+  conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="5432")
+  cur = conn.cursor()
+  sql_str = 'SELECT data FROM "public"."' + str(channel) + '_recording_data_raws" WHERE parent = ' + meta_id + 'ORDER BY id ASC'
+  cur.execute(sql_str)
+  ret = None
+  try:
+      ret = cur.fetchall()
+  except BaseException as e:
+    print(e)
+    exit
+  finally:
+      timeArray = np.array([], dtype=int)
+      dataArray = np.array([])
+
+      for sub_data in ret:
+        for split_data in sub_data[0].split('"***"')[:-1]:
+          value = np.array(split_data.split(' '))
+          timeArray = np.append(timeArray, value[::2].astype(np.int32))
+          dataArray = np.append(dataArray, value[1::2].astype(np.int32))
+      xf = rfftfreq(N, 1 / SAMPLE_RATE)
+      points_per_freq = len(xf) / (SAMPLE_RATE / 2)
+      target_idx = int(points_per_freq * cut_off_freq)
+
+      # print(xf[target_idx])
+      yf = rfft(dataArray)
+      yf[:target_idx] = 0
+      new_sig = np.round(irfft(yf, len(dataArray)),3)
+
+      if threshold[0] == 'below':
+        th = new_sig < threshold[1]
+      elif threshold[0] == 'over':
+        th = new_sig > threshold[1]
+      elif threshold[0] == 'auto':
+        th = new_sig
+            
+      threshold_edges = np.convolve([1, -1], th, mode='same')
+      thresholded_edge_indices = np.where(threshold_edges==1)[0]
+
+      filter_array = []
+      for idx, point in enumerate(thresholded_edge_indices):
+        if idx > 0:
+          if timeArray[thresholded_edge_indices[idx]] - int(waveform[2]) >= timeArray[thresholded_edge_indices[idx-1]]:
+            filter_array.append(True)
+          elif timeArray[thresholded_edge_indices[idx]] - int(waveform[1]) >= timeArray[thresholded_edge_indices[idx-1]]:
+            filter_array.append(True)
+          else:
+            filter_array.append(False)
+        else:
+          if timeArray[thresholded_edge_indices[idx]] - int(waveform[1]) < 0:
+            filter_array.append(False)
+          else:
+            filter_array.append(True)
+      
+      timemark_list = thresholded_edge_indices[filter_array]
+
+      OUTPUT_SIGNAL_SIZE = 2000
+      downsampled_signal_index, downsampled_signal_value = get_downsampled_signal(new_sig, OUTPUT_SIGNAL_SIZE)
+      if ret_data_type == 'all':
+        print(' '.join(str(x) for x in timeArray))
+        print(' '.join(str(x) for x in downsampled_signal_index))
+        print(' '.join(str(x) for x in downsampled_signal_value))
+        print(' '.join(str(x) for x in timemark_list))
+      elif ret_data_type == 'partial':
+        return_sub_signal_lists = {}
+        for timemark in timemark_list:
+          start_index = timemark - floor(int(waveform[1] / DELTA_TIME))
+          end_index = timemark + ceil(int(waveform[2] / DELTA_TIME))
+
+          subSignal = new_sig[start_index: end_index + 2]
+          return_sub_signal_lists[str(timemark)] = ' '.join(str(x) for x in subSignal)
+        print(dumps(return_sub_signal_lists))
+
+  conn.commit()
+  conn.close()
+  exit

src/controllers/controller/task.js

@@ -0,0 +1,69 @@
+import TaskModal from '../../models/controller/task'
+
+const task = new TaskModal()
+
+export const create = async (ctx, next) => {
+  const data = ctx.request.body
+  const result = await task.create(data)
+  ctx.body = result
+
+  next()
+}
+
+export const update = async (ctx, next) => {
+  const data = ctx.request.body
+  const result = await task.update(data)
+  ctx.body = result
+
+  next()
+}
+
+export const updateByID = async (ctx, next) => {
+  const data = ctx.request.body
+  const id = ctx.params.id
+  const result = await task.updateByID(data, id)
+  ctx.body = result
+
+  next()
+}
+
+export const updateByAttr = async (ctx, next) => {
+  const data = ctx.request.body
+  const attr = ctx.params.attr
+  const value = ctx.params.value
+  const result = await task.updateByAttr(data, attr, value)
+  ctx.body = result
+
+  next()
+}
+
+export const clearDeleted = async (ctx, next) => {
+  const result = await task.clearDeleted()
+  ctx.body = result
+
+  next()
+}
+
+export const getByID = async (ctx, next) => {
+  const id = ctx.params.id
+  const result = await task.getByID(id)
+  ctx.body = result
+
+  next()
+}
+
+export const getByAttr = async (ctx, next) => {
+  const attr = ctx.params.attr
+  const value = ctx.params.value
+  const result = await task.getByAttr(attr, value)
+  ctx.body = result
+
+  next()
+}
+
+export const getAll = async (ctx, next) => {
+  const result = await task.getAll()
+  ctx.body = result
+
+  next()
+}

src/controllers/file/collection.js

@@ -20,7 +20,8 @@ export const update = async (ctx, next) => {
 }
 
 export const clearDeleted = async (ctx, next) => {
-  const result = await collection.clearDeleted()
+  const controllerID = ctx.params.controller_id
+  const result = await collection.clearDeleted(controllerID)
   ctx.body = result
 
   next()
@@ -36,7 +37,8 @@ export const getByID = async (ctx, next) => {
 
 export const getByName = async (ctx, next) => {
   const name = ctx.params.name
-  const result = await collection.getByName(name)
+  const controllerID = ctx.params.controller_id
+  const result = await collection.getByName(name, controllerID)
   ctx.body = result
 
   next()
@@ -62,7 +64,8 @@ export const getByParentName = async (ctx, next) => {
 }
 
 export const getAll = async (ctx, next) => {
-  const result = await collection.getAll()
+  const controllerID = ctx.params.controller_id
+  const result = await collection.getAll(controllerID)
   ctx.body = result
 
   next()

src/controllers/file/recording_data_meta.js

@@ -6,6 +6,7 @@ const recordingDataMeta = new RecordingDataMeta()
 export const create = async (ctx, next) => {
   const data = ctx.request.body
   console.log(auth.CheckAuth(ctx))
+  // console.log(data)
   // console.log(ctx.request.header.authorization)
   const result = await recordingDataMeta.create(data)
   // console.log(result)
@@ -47,6 +48,40 @@ export const getByParent = async (ctx, next) => {
   next()
 }
 
+export const getCountByParent = async (ctx, next) => {
+  const type = ctx.params.type
+  const parent = ctx.params.parent
+  const result = await recordingDataMeta.getCountByParent(type, parent)
+  ctx.body = result
+
+  next()
+}
+
+export const getByParentWithLimit = async (ctx, next) => {
+  const type = ctx.params.type
+  const parent = ctx.params.parent
+  const limit = ctx.params.limit
+  const offset = ctx.params.offset
+  const order = ctx.params.order
+  const result = await recordingDataMeta.getByParentWithLimit(type, parent, limit, offset, order)
+  ctx.body = result
+
+  next()
+}
+
+export const getByAttrsParentWithLimit = async (ctx, next) => {
+  const type = ctx.params.type
+  const parent = ctx.params.parent
+  const limit = ctx.params.limit
+  const offset = ctx.params.offset
+  const order = ctx.params.order
+  const attr = ctx.params.attr
+  const result = await recordingDataMeta.getByAttrsParentWithLimit(type, parent, limit, offset, order, attr)
+  ctx.body = result
+
+  next()
+}
+
 export const getByUUID = async (ctx, next) => {
   const uuid = ctx.params.uuid
   const result = await recordingDataMeta.getByUUID(uuid)

src/controllers/file/recording_data_mini.js

@@ -25,6 +25,16 @@ export const update = async (ctx, next) => {
   next()
 }
 
+export const updateByIDs = async (ctx, next) => {
+  const index = ctx.params.channel
+  const data = ctx.request.body
+  const id = ctx.params.id
+  const result = await recordingDataMini.updateByIDs(data, id, index)
+  ctx.body = result
+
+  next()
+}
+
 // export const get = async (ctx, next) => {
 //   const index = ctx.params.channel
 //   const id = ctx.params.id
@@ -51,6 +61,15 @@ export const getByID = async (ctx, next) => {
   next()
 }
 
+export const getByIDs = async (ctx, next) => {
+  const index = ctx.params.channel
+  const id = ctx.params.id
+  const result = await recordingDataMini.getByIDs(id, index)
+  ctx.body = result
+
+  next()
+}
+
 export const getMeanDataByID = async (ctx, next) => {
   const index = ctx.params.channel
   const id = ctx.params.id

src/controllers/file/recording_data_raw.js

@@ -25,6 +25,26 @@ export const update = async (ctx, next) => {
   next()
 }
 
+export const updateSdData = async (ctx, next) => {
+  const index = ctx.params.channel
+  const data = ctx.request.body
+  const id = ctx.params.id
+  const result = await recordingDataRaw.updateSdData(data, id, index)
+  ctx.body = result
+
+  next()
+}
+
+export const updateByIDs = async (ctx, next) => {
+  const index = ctx.params.channel
+  const data = ctx.request.body
+  const id = ctx.params.id
+  const result = await recordingDataRaw.updateByIDs(data, id, index)
+  ctx.body = result
+
+  next()
+}
+
 export const clearDeleted = async (ctx, next) => {
   const index = ctx.params.channel
   const result = await recordingDataRaw.clearDeleted(index)

src/controllers/index.js

@@ -4,9 +4,11 @@ import * as auth from './auth'
 import * as controller from './product/controller'
 import * as device from './product/device'
 import * as collection from './file/collection'
+import * as task from './controller/task'
 import * as recordingDataMeta from './file/recording_data_meta'
 import * as recordingDataRaw from './file/recording_data_raw'
 import * as recordingDataMini from './file/recording_data_mini'
+import * as analysis from './analysis'
 
 export default {
   upload,
@@ -15,7 +17,9 @@ export default {
   controller,
   device,
   collection,
+  task,
   recordingDataMeta,
   recordingDataRaw,
-  recordingDataMini
+  recordingDataMini,
+  analysis
 }

src/controllers/product/controller.js

@@ -12,7 +12,8 @@ export const create = async (ctx, next) => {
 
 export const update = async (ctx, next) => {
   const data = ctx.request.body
-  const result = await controller.update(data)
+  const id = ctx.params.id
+  const result = await controller.update(data, id)
   ctx.body = result
 
   next()

src/controllers/product/device.js

@@ -27,6 +27,15 @@ export const updateByID = async (ctx, next) => {
   next()
 }
 
+export const updateByMac = async (ctx, next) => {
+  const data = ctx.request.body
+  const mac = ctx.params.mac
+  const result = await device.updateByMac(data, mac)
+  ctx.body = result
+
+  next()
+}
+
 export const updateByAttr = async (ctx, next) => {
   const data = ctx.request.body
   const attr = ctx.params.attr

src/db/database.js

@@ -24,6 +24,7 @@ db.sequelize = sequelize
 
 db.tutorials = require('./schema/tutirial.model')(sequelize, Sequelize)
 db.controllers = require('./schema/controller.model')(sequelize, Sequelize)
+db.tasks = require('./schema/task.model')(sequelize, Sequelize)
 db.devices = require('./schema/device.model')(sequelize, Sequelize)
 db.collections = require('./schema/collection.model')(sequelize, Sequelize)
 db.recording_data_metas = require('./schema/recording_data_meta.model')(sequelize, Sequelize)

src/db/schema/collection.model.js

@@ -7,6 +7,10 @@ module.exports = (sequelize, Sequelize) => {
       type: Sequelize.JSONB,
       defaultValue: {}
     },
+    controller_id: {
+      type: Sequelize.INTEGER,
+      defaultValue: -1
+    },
     size: {
       type: Sequelize.INTEGER,
       defaultValue: -1

src/db/schema/controller.model.js

@@ -28,6 +28,10 @@ module.exports = (sequelize, Sequelize) => {
       type: Sequelize.STRING,
       defaultValue: ''
     },
+    mqtt_id: {
+      type: Sequelize.STRING,
+      defaultValue: ''
+    },
     mqtt_address: {
       type: Sequelize.STRING,
       defaultValue: ''
@@ -36,6 +40,10 @@ module.exports = (sequelize, Sequelize) => {
       type: Sequelize.STRING,
       defaultValue: ''
     },
+    root: {
+      type: Sequelize.INTEGER,
+      defaultValue: -1
+    },
     // group/user: {
     //   admin: [id...],
     //   normal: [id...],
@@ -59,33 +67,43 @@ module.exports = (sequelize, Sequelize) => {
     },
     device: {
       type: Sequelize.JSONB,
-      defaultValue: {
-        auto_join: [],
-        scanned: []
-      }
+      defaultValue: {}
+    },
+    device_scanned: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+      defaultValue: []
+    },
+    device_connected: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+      defaultValue: []
     },
     // project/task: {
     //   running: [id...],
     //   library: [id...]
     // }
     project: {
-      type: Sequelize.JSONB,
-      defaultValue: {
-        running: [],
-        library: []
-      }
+      type: Sequelize.STRING,
+      defaultValue: ''
     },
     task: {
-      type: Sequelize.JSONB,
-      defaultValue: {
-        running: [],
-        library: []
-      }
+      type: Sequelize.STRING,
+      defaultValue: ''
     },
+    // task: {
+    //   type: Sequelize.JSONB,
+    //   defaultValue: {
+    //     running: [],
+    //     library: []
+    //   }
+    // },
     software_version: {
       type: Sequelize.STRING,
       defaultValue: ''
     },
+    status: {
+      type: Sequelize.STRING,
+      defaultValue: ''
+    },
     deleted: {
       type: Sequelize.BOOLEAN,
       defaultValue: false

src/db/schema/device.model.js

@@ -55,11 +55,15 @@ module.exports = (sequelize, Sequelize) => {
       defaultValue: -1
     },
     parameter_set: {
-      type: Sequelize.INTEGER,
-      defaultValue: -1
+      type: Sequelize.JSONB,
+      defaultValue: {}
+    },
+    running: {
+      type: Sequelize.BOOLEAN,
+      defaultValue: false
     },
     calibration: {
-      type: Sequelize.STRING,
+      type: Sequelize.TEXT,
       defaultValue: ''
     },
     user_auth: {

src/db/schema/recording_data_raw.model.js

@@ -43,6 +43,10 @@ module.exports = (sequelize, Sequelize, i) => {
       type: Sequelize.TEXT,
       defaultValue: ''
     },
+    sd_data: {
+      type: Sequelize.TEXT,
+      defaultValue: ''
+    },
     compressed: {
       type: Sequelize.BOOLEAN,
       defaultValue: false

src/db/schema/task.model.js

@@ -1,16 +1,33 @@
 module.exports = (sequelize, Sequelize) => {
   const Task = sequelize.define('task', {
     name: {
-      type: Sequelize.STRING
-    },
-    description: {
       type: Sequelize.STRING,
       defaultValue: ''
     },
-    configuration: {
+    describe: {
+      type: Sequelize.STRING,
+      defaultValue: ''
+    },
+    status: {
+      type: Sequelize.STRING,
+      defaultValue: ''
+    },
+    device_config: {
       type: Sequelize.JSONB,
       defaultValue: {}
     },
+    chart_config: {
+      type: Sequelize.JSONB,
+      defaultValue: {}
+    },
+    formula_config: {
+      type: Sequelize.JSONB,
+      defaultValue: {}
+    },
+    controller_id: {
+      type: Sequelize.INTEGER,
+      defaultValue: -1
+    },
     deleted: {
       type: Sequelize.BOOLEAN,
       defaultValue: false

src/models/controller/task.js

@@ -0,0 +1,105 @@
+const db = require('../../db/database')
+
+// const Op = db.Sequelize.Op
+
+class TaskModel {
+  async create (req) {
+    console.log(req)
+    return db.tasks.create(req)
+  }
+
+  async update (req) {
+    const ret = await db.tasks.update(
+      req,
+      {
+        where: {}
+      }
+    )
+    return ret
+  }
+
+  async updateByID (req, id) {
+    const ret = await db.tasks.update(
+      req,
+      {
+        where: {
+          id: id
+        }
+      }
+    )
+    return ret
+  }
+
+  async updateByAttr (req, attr, value) {
+    const whereObj = {}
+    const attrList = attr.split('-')
+    const valueList = value.split('-')
+    attrList.forEach((key, idx) => {
+      whereObj[key] = valueList[idx]
+    })
+
+    const ret = await db.tasks.update(
+      req,
+      {
+        where: whereObj
+      }
+    )
+    return ret
+  }
+
+  async clearDeleted () {
+    const ret = await db.tasks.delete(
+      {
+        where: {
+          deleted: true
+        }
+      }
+    )
+    return ret
+  }
+
+  async getByID (id) {
+    const ret = await db.tasks.findAll(
+      {
+        where: {
+          id: id
+        }
+      }
+    )
+    return ret
+  }
+
+  async getByName (name) {
+    const ret = await db.tasks.findAll(
+      {
+        where: {
+          name: name
+        }
+      }
+    )
+    return ret
+  }
+
+  async getByAttr (attr, value) {
+    const whereObj = {}
+    const attrList = attr.split('-')
+    const valueList = value.split('-')
+    attrList.forEach((key, idx) => {
+      whereObj[key] = valueList[idx]
+    })
+
+    const ret = await db.tasks.findAll(
+      {
+        where: whereObj
+      }
+    )
+    return ret
+  }
+
+  async getAll () {
+    const ret = await db.tasks.findAll()
+    return ret
+  }
+}
+
+export default TaskModel

src/models/file/collection.js

@@ -4,11 +4,7 @@ const Op = db.Sequelize.Op
 
 class CollectionModel {
   async create (req) {
-    return db.collections.create({
-      name: req.name,
-      parent: req.parent,
-      size: req.size
-    })
+    return db.collections.create(req)
   }
 
   async update (req, id) {
@@ -23,14 +19,26 @@ class CollectionModel {
     return ret
   }
 
-  async clearDeleted () {
-    const ret = await db.collections.delete(
-      {
-        where: {
-          deleted: true
+  async clearDeleted (controllerID) {
+    let ret
+    if (controllerID == null) {
+      ret = await db.collections.destroy(
+        {
+          where: {
+            deleted: true
+          }
         }
-      }
-    )
+      )
+    } else {
+      ret = await db.collections.destroy(
+        {
+          where: {
+            controller_id: controllerID,
+            deleted: true
+          }
+        }
+      )
+    }
     return ret
   }
 
@@ -45,14 +53,26 @@ class CollectionModel {
     return ret
   }
 
-  async getByName (name) {
-    const ret = await db.collections.findAll(
-      {
-        where: {
-          name: name
+  async getByName (name, controllerID) {
+    let ret
+    if (controllerID == null) {
+      ret = await db.collections.findAll(
+        {
+          where: {
+            name: name
+          }
         }
-      }
-    )
+      )
+    } else {
+      ret = await db.collections.findAll(
+        {
+          where: {
+            controller_id: controllerID,
+            name: name
+          }
+        }
+      )
+    }
     return ret
   }
 
@@ -100,8 +120,19 @@ class CollectionModel {
     return ret
   }
 
-  async getAll () {
-    const ret = await db.collections.findAll()
+  async getAll (controllerID) {
+    let ret
+    if (controllerID == null) {
+      ret = await db.collections.findAll()
+    } else {
+      ret = await db.collections.findAll(
+        {
+          where: {
+            controller_id: controllerID
+          }
+        }
+      )
+    }
     return ret
   }
 }

src/models/file/recording_data_meta.js

@@ -1,4 +1,10 @@
+import RecordingDataRaw from './recording_data_raw'
+import RecordingDataMini from './recording_data_mini'
 const db = require('../../db/database')
+// const RecordingDataMini = require('./recording_data_mini')
+
+const recordingDataRaw = new RecordingDataRaw()
+const recordingDataMini = new RecordingDataMini()
 
 const Op = db.Sequelize.Op
 
@@ -36,7 +42,40 @@ class RecordingDataMeta {
   }
 
   async clearDeleted () {
-    const ret = await db.recording_data_metas.destroy(
+    let ret = await db.recording_data_metas.findAll(
+      {
+        attributes: [
+          'id',
+          'raw_data',
+          'mini_data'
+        ],
+        where: {
+          deleted: true
+        }
+      }
+    )
+    for (let i = 0; i < ret.length; i++) {
+      const meta = ret[i]
+      for (const channel in meta.raw_data) {
+        if (meta.raw_data[channel] != null) {
+          await recordingDataRaw.updateByIDs({ deleted: true }, meta.raw_data[channel], channel)
+          recordingDataRaw.clearDeleted(channel)
+        }
+        if (meta.mini_data[channel] != null) {
+          if (meta.mini_data[channel]['10'] !== undefined) {
+            await recordingDataMini.updateByIDs({ deleted: true }, meta.mini_data[channel]['10'], channel)
+          }
+          if (meta.mini_data[channel]['100'] !== undefined) {
+            await recordingDataMini.updateByIDs({ deleted: true }, meta.mini_data[channel]['100'], channel)
+          }
+          if (meta.mini_data[channel]['1000'] !== undefined) {
+            await recordingDataMini.updateByIDs({ deleted: true }, meta.mini_data[channel]['1000'], channel)
+          }
+          recordingDataMini.clearDeleted(channel)
+        }
+      }
+    }
+    ret = await db.recording_data_metas.destroy(
       {
         where: {
           deleted: true
@@ -65,6 +104,67 @@ class RecordingDataMeta {
     return ret
   }
 
+  async getCountByParent (type, parent) {
+    const collectionType = type + '::jsonb'
+    const parentOp = {}
+    parentOp[collectionType] = {
+      [Op.contains]: '[' + parent + ']'
+    }
+    const ret = await db.recording_data_metas.count(
+      {
+        where: {
+          parent: parentOp
+        }
+      }
+    )
+    return ret
+  }
+
+  async getByParentWithLimit (type, parent, limit, offset, order) {
+    const collectionType = type + '::jsonb'
+    const parentOp = {}
+    parentOp[collectionType] = {
+      [Op.contains]: '[' + parent + ']'
+    }
+    const ret = await db.recording_data_metas.findAndCountAll(
+      {
+        where: {
+          parent: parentOp
+        },
+        order: [
+          ['id', order]
+        ],
+        limit: limit,
+        offset: offset
+      }
+    )
+    return ret
+  }
+
+  async getByAttrsParentWithLimit (type, parent, limit, offset, order, attr) {
+    const attrList = attr.split('-')
+    const orderList = order.split('-')
+    const collectionType = type + '::jsonb'
+    const parentOp = {}
+    parentOp[collectionType] = {
+      [Op.contains]: '[' + parent + ']'
+    }
+    const ret = await db.recording_data_metas.findAndCountAll(
+      {
+        where: {
+          parent: parentOp
+        },
+        attributes: attrList,
+        order: [
+          [orderList[0], orderList[1]]
+        ],
+        limit: limit,
+        offset: offset
+      }
+    )
+    return ret
+  }
+
   async getByID (id) {
     const ret = await db.recording_data_metas.findAll(
       {

src/models/file/recording_data_mini.js

@@ -19,6 +19,18 @@ class RecordingDataMini {
     return ret
   }
 
+  async updateByIDs (req, id, index) {
+    const ret = await db[index + '_recording_data_minis'].update(
+      req,
+      {
+        where: {
+          id: id
+        }
+      }
+    )
+    return ret
+  }
+
   async clearDeleted (index) {
     const ret = await db[index + '_recording_data_minis'].destroy(
       {
@@ -57,6 +69,19 @@ class RecordingDataMini {
     return ret
   }
 
+  async getByIDs (id, index) {
+    const _id = JSON.parse('[' + id + ']')
+    const ret = await db[index + '_recording_data_minis'].findAllWithStream(
+      {
+        batchSize: 1000,
+        where: {
+          id: _id
+        }
+      }
+    )
+    return ret
+  }
+
   async getMeanDataByID (id, index) {
     const ret = await db[index + '_recording_data_minis'].findAllWithStream(
       {

src/models/file/recording_data_raw.js

@@ -20,6 +20,23 @@ class RecordingDataRaw {
     return ret
   }
 
+  async updateSdData (req, id, index) {
+    const ret = await db.sequelize.query(`UPDATE "public"."${index}_recording_data_raws" SET sd_data = sd_data || '${req.data}' WHERE id = ${String(id)}`)
+    return ret
+  }
+
+  async updateByIDs (req, id, index) {
+    const ret = await db[index + '_recording_data_raws'].update(
+      req,
+      {
+        where: {
+          id: id
+        }
+      }
+    )
+    return ret
+  }
+
   async clearDeleted (index) {
     const ret = await db[index + '_recording_data_raws'].destroy(
       {

src/models/product/controller.js

@@ -4,25 +4,35 @@ const db = require('../../db/database')
 
 class ControllerModel {
   async create (req) {
-    return db.controllers.create({
-      name: req.name,
-      mac_address: req.mac_address,
-      software_version: req.software_version,
-      wifi_name: req.wifi_name
-    })
+    console.log(req)
+    return db.controllers.create(req)
   }
 
-  async update (req) {
-    const ret = await db.controllers.update(
-      req,
-      {
-        where: {}
-      }
-    )
-    return ret
+  async update (req, id) {
+    req.updated_at = db.Sequelize.literal('CURRENT_TIMESTAMP')
+    if (id != null) {
+      const ret = await db.controllers.update(
+        req,
+        {
+          where: {
+            id: id
+          }
+        }
+      )
+      return ret
+    } else {
+      const ret = await db.controllers.update(
+        req,
+        {
+          where: {}
+        }
+      )
+      return ret
+    }
   }
 
   async updateByID (req, id) {
+    req.updated_at = db.Sequelize.literal('CURRENT_TIMESTAMP')
     const ret = await db.controllers.update(
       req,
       {
@@ -41,7 +51,7 @@ class ControllerModel {
     attrList.forEach((key, idx) => {
       whereObj[key] = valueList[idx]
     })
-
+    req.updated_at = db.Sequelize.literal('CURRENT_TIMESTAMP')
     const ret = await db.controllers.update(
       req,
       {

src/models/product/device.js

@@ -12,17 +12,31 @@ class DeviceModel {
     })
   }
 
-  async update (req) {
-    const ret = await db.devices.update(
-      req,
-      {
-        where: {}
-      }
-    )
-    return ret
+  async update (req, id) {
+    req.updated_at = db.Sequelize.literal('CURRENT_TIMESTAMP')
+    if (id != null) {
+      const ret = await db.devices.update(
+        req,
+        {
+          where: {
+            id: id
+          }
+        }
+      )
+      return ret
+    } else {
+      const ret = await db.devices.update(
+        req,
+        {
+          where: {}
+        }
+      )
+      return ret
+    }
   }
 
   async updateByID (req, id) {
+    req.updated_at = db.Sequelize.literal('CURRENT_TIMESTAMP')
     const ret = await db.devices.update(
       req,
       {
@@ -34,6 +48,19 @@ class DeviceModel {
     return ret
   }
 
+  async updateByMac (req, mac) {
+    req.updated_at = db.Sequelize.literal('CURRENT_TIMESTAMP')
+    const ret = await db.devices.update(
+      req,
+      {
+        where: {
+          mac_address: mac
+        }
+      }
+    )
+    return ret
+  }
+
   async updateByAttr (req, attr, value) {
     const whereObj = {}
     const attrList = attr.split('-')

src/routes/main-routes.js

@@ -17,27 +17,39 @@ export default router
   .get('/public/auth/get_cookie', controllers.auth.getCookie)
 
   // collection
-  .post('/api/collection/create', controllers.collection.create)
-  .get('/api/collection/get_by_id/:id', controllers.collection.getByID)
-  .get('/api/collection/get_by_name/:name', controllers.collection.getByName)
-  .get('/api/collection/get_by_parent/:type/:parent', controllers.collection.getByParent)
-  .get('/api/collection/get_by_parent_name/:type/:parent/:name', controllers.collection.getByParentName)
-  .get('/api/collection/get/all', controllers.collection.getAll)
+  .post('/api/file/collection/create', controllers.collection.create)
+  .post('/api/file/collection/update/:id', controllers.collection.update)
+  .get('/api/file/collection/clear_deleted/:controller_id', controllers.collection.clearDeleted)
+  .get('/api/file/collection/get_by_id/:id', controllers.collection.getByID)
+  .get('/api/file/collection/get_by_name/:name/:controller_id', controllers.collection.getByName)
+  .get('/api/file/collection/get_by_parent/:type/:parent', controllers.collection.getByParent)
+  .get('/api/file/collection/get_by_parent_name/:type/:parent/:name', controllers.collection.getByParentName)
+  .get('/api/file/collection/get/all/:controller_id', controllers.collection.getAll)
+
+  .get('/api/file/collection/get_by_name/:name', controllers.collection.getByName)
+  .get('/api/file/collection/get/all', controllers.collection.getAll)
+  .get('/api/file/collection/clear_deleted/', controllers.collection.clearDeleted)
 
   // meta
   .post('/api/file/meta/create', controllers.recordingDataMeta.create)
   .post('/api/file/meta/update/:id', controllers.recordingDataMeta.update)
-  .get('/api/file/meta/clear_deleted/:id', controllers.recordingDataMeta.clearDeleted)
+  .get('/api/file/meta/clear_deleted/', controllers.recordingDataMeta.clearDeleted)
   .get('/api/file/meta/get_by_name/:name', controllers.recordingDataMeta.getByName)
   .get('/api/file/meta/get_by_uuid/:uuid', controllers.recordingDataMeta.getByUUID)
   .get('/api/file/meta/get_by_id/:id', controllers.recordingDataMeta.getByID)
   .get('/api/file/meta/get_by_path/:path', controllers.recordingDataMeta.getByPath)
   .get('/api/file/meta/get_by_parent/:type/:parent', controllers.recordingDataMeta.getByParent)
+  .get('/api/file/meta/get_count_by_parent/:type/:parent', controllers.recordingDataMeta.getCountByParent)
+  .get('/api/file/meta/get_by_parent_with_limit/:type/:parent/:limit/:offset/:order', controllers.recordingDataMeta.getByParentWithLimit)
+  .get('/api/file/meta/get_by_attr_parent_with_limit/:type/:parent/:limit/:offset/:order/:attr', controllers.recordingDataMeta.getByAttrsParentWithLimit)
+
   .get('/api/file/meta/get_or_create_by_path/:path', controllers.recordingDataMeta.getOrCreateByPath)
+  .get('/api/file/meta/get/all', controllers.recordingDataMeta.getAll)
 
   // raw
   .post('/api/file/raw/create/:channel', controllers.recordingDataRaw.create)
   .post('/api/file/raw/update/:channel/:id', controllers.recordingDataRaw.update)
+  .post('/api/file/raw/update_sd_data/:channel/:id', controllers.recordingDataRaw.updateSdData)
   .get('/api/file/raw/clear_deleted/:channel/:id', controllers.recordingDataRaw.clearDeleted)
   .get('/api/file/raw/get_by_id/:channel/:id', controllers.recordingDataRaw.getByID)
   .get('/api/file/raw/get_by_ids/:channel/:id', controllers.recordingDataRaw.getByIDs)
@@ -60,19 +72,40 @@ export default router
 
   // device
   .post('/api/device/create', controllers.device.create)
-  .post('/api/device/update', controllers.device.update)
+  .post('/api/device/update/all', controllers.device.update)
   .post('/api/device/update_by_id/:id', controllers.device.updateByID)
+  .post('/api/device/update_by_mac/:mac', controllers.device.updateByMac)
   .post('/api/device/update_by_attr/:attr/:value', controllers.device.updateByAttr)
   .get('/api/device/clear_deleted/:id', controllers.device.clearDeleted)
   .get('/api/device/get_by_mac/:mac', controllers.device.getByMac)
   .get('/api/device/get_by_attr/:attr/:value', controllers.device.getByAttr)
   .get('/api/device/get/all', controllers.device.getAll)
 
+  .post('/api/device/update_by_id/:id', controllers.device.updateByID)
+  .post('/api/device/update', controllers.device.update)
+
   // controller
   .post('/api/controller/create', controllers.controller.create)
-  .post('/api/controller/update', controllers.controller.update)
-  .post('/api/controller/update_by_id/:id', controllers.controller.updateByID)
+  .post('/api/controller/update/:id', controllers.controller.update)
+  .post('/api/controller/update/all', controllers.controller.update)
   .post('/api/controller/update_by_attr/:attr/:value', controllers.controller.updateByAttr)
   .get('/api/controller/get_by_mac/:mac', controllers.controller.getByMac)
   .get('/api/controller/get_by_attr/:attr/:value', controllers.controller.getByAttr)
   .get('/api/controller/get/all', controllers.controller.getAll)
+
+  .post('/api/controller/update_by_id/:id', controllers.controller.updateByID)
+  .post('/api/controller/update', controllers.controller.update)
+
+  // task
+  .post('/api/task/create', controllers.task.create)
+  .post('/api/task/update/:id', controllers.task.update)
+  .post('/api/task/update_by_attr/:attr/:value', controllers.task.updateByAttr)
+  .post('/api/task/update/all', controllers.task.update)
+  .get('/api/task/get_by_id/:id', controllers.task.getByID)
+  .get('/api/task/get_by_attr/:attr/:value', controllers.task.getByAttr)
+  .get('/api/task/get/all', controllers.task.getAll)
+
+  .post('/api/task/update', controllers.task.update)
+  .post('/api/task/update_by_id/:id', controllers.task.updateByID)
+  // analyze
+  .post('/api/analysis/spike_detect', controllers.analysis.spikeDetect)