Compare commits

..

17 Commits

Author SHA1 Message Date
peterlu14 2fca174483 -add subject subjectData 2023-06-05 14:19:44 +08:00
peterlu14 917380a961 -add configuration table 2023-05-11 15:14:44 +08:00
peterlu14 7d2e8009b6 -fix return result 2023-04-27 11:54:01 +08:00
peterlu14 c277d7c94c -update findWithIdRange1 return result 2023-04-27 11:41:14 +08:00
peterlu14 d93633704e -update add meta findWithIdRange 2023-04-10 09:16:32 +08:00
peterlu14 87bcb29242 [update] add limit in query meta table 2023-01-12 09:49:03 +08:00
peterlu14 442eb1eed5 [update] count result from project table 2023-01-12 09:47:14 +08:00
10 0a0608aa09 [update] add api for downloading analysis file 2023-01-06 13:11:26 +08:00
peterlu14 423a56466a [update] add get by project 2023-01-03 15:44:36 +08:00
peterlu14 eb79905024 [debug] fix recording_data_metas add column project 2022-12-30 15:07:46 +08:00
peterlu14 853cf888de [debug] fix serial_number wrong default 2022-12-30 14:57:58 +08:00
peterlu14 3ee4d5a118 [update] add api for table project_metas 2022-12-28 14:08:14 +08:00
peterlu14 37e13ca93f [update] add api for table project_reports 2022-12-28 13:44:57 +08:00
peterlu14 59ebcb946e [update] add meta with time query & group by time & project 2022-12-28 11:35:55 +08:00
peterlu14 4a6587530c [update] add new collection get api & support all old apis 2022-12-27 11:40:09 +08:00
peterlu14 c264157a15 [update] add new meta get api support all old apis 2022-12-08 16:33:29 +08:00
peterlu14 be1500bbbc [update] add project cycle column 2022-09-28 11:21:14 +08:00
28 changed files with 1349 additions and 123 deletions
+11
View File
@@ -1,6 +1,7 @@
import RecordingDataMeta from '../../models/file/recording_data_meta'
import childProcess from 'child_process'
import path from 'path'
import { createReadStream } from 'fs'
const recordingDataMeta = new RecordingDataMeta()
@@ -55,3 +56,13 @@ export const spikeDetect = async (ctx, next) => {
next()
}
export const getCSV = async (ctx, next) => {
const name = ctx.query.name
ctx.type = 'stream'
// ctx.attachment = name + '.csv'
ctx.set('Content-disposition', `attachment; filename=${name}.csv`)
ctx.body = createReadStream(path.resolve(__dirname, '../../../../data-analysis/csv_file', 'output.csv'))
next()
}
+39
View File
@@ -0,0 +1,39 @@
import ConfigurationModal from '../../models/configuration/configuration'
const configuration = new ConfigurationModal()
export const create = async (ctx, next) => {
const data = ctx.request.body
const result = await configuration.create(data)
ctx.body = result
next()
}
export const update = async (ctx, next) => {
const data = ctx.request.body
const id = ctx.params.id
const result = await configuration.update(data, id)
ctx.body = result
next()
}
export const get = async (ctx, next) => {
const configurationType = ctx.query.type
const id = ctx.query.id
const library = ctx.query.library
const query = { id: id, library: library }
const result = await configuration.get(configurationType, query)
ctx.body = result
next()
}
export const clearDeleted = async (ctx, next) => {
const controllerID = ctx.params.controller_id
const result = await configuration.clearDeleted(controllerID)
ctx.body = result
next()
}
+27
View File
@@ -19,6 +19,33 @@ export const update = async (ctx, next) => {
next()
}
export const get = async (ctx, next) => {
const id = ctx.query.id
const name = ctx.query.name
const type = ctx.query.type
const controllerId = ctx.query.controllerId
// folder
const parent = ctx.query.parent
const limit = ctx.query.limit
const offset = ctx.query.offset
const order = ctx.query.order
const attrs = ctx.query.attrs
const query = {}
let result = ''
if (id) query.id = id
if (name) query.name = name
if (type) query.type = type
if (controllerId) query.controllerId = controllerId
if (id || name) result = await collection.getByQueryWithAttrs(query, attrs)
if (type === 'folder') result = await collection.getByParentWithOffsetLimitOrder(type, parent, limit, offset, order)
ctx.body = result
next()
}
export const clearDeleted = async (ctx, next) => {
const controllerID = ctx.params.controller_id
const result = await collection.clearDeleted(controllerID)
+41 -78
View File
@@ -1,34 +1,27 @@
import psycopg2
import pandas as pd
import numpy as np
from os.path import exists
import sys
import os
import time
def log(df, channel):
# TODO: not sure need use 32 or 64? => 64, since large integral will cause overflow
df[f'{channel}_log'] = np.log10(np.absolute(df[channel]))
# df[f'{channel}_log'] = np.log10(np.absolute(df[channel].astype(np.int64)))
# TODO: not sure need use 32 or 64?
df[f'{channel}_log'] = np.log10(np.absolute(df[channel].astype(np.int32)))
return df
def integral(df, x_channel, y_channel, pre_last_x, pre_last_y, integral_result):
offset = df[x_channel]
offset = offset.shift(periods=1)
offset[0] = pre_last_x
x_diff = df[x_channel] - offset
# x_diff = df[x_channel].astype(np.int64) - offset.astype(np.int64)
x_diff = df[x_channel].astype(np.int32) - offset.astype(np.int32)
# print('x_diff\n', x_diff)
offset = df[y_channel]
offset = offset.shift(periods=1)
offset[0] = pre_last_y
y_avg = (offset + df[y_channel]) / 2
# y_avg = (offset.astype(np.int64) + df[y_channel].astype(np.int64)) / 2
y_avg = (offset.astype(np.int32) + df[y_channel].astype(np.int32)) / 2
# print('y_avg:', y_avg)
# print('y_avg: ', y_avg)
local_integral = x_diff * y_avg
local_integral[0] += integral_result
# TODO: if user don't need, then can remove it
df[f'{x_channel}_{y_channel}_integral_delta'] = local_integral
local_integral[0] += integral_result
df[f'{x_channel}_{y_channel}_integral'] = local_integral.cumsum()
# print(df)
pre_last_x = df[x_channel].iat[-1]
@@ -40,103 +33,73 @@ def differential(df, x_channel, y_channel, pre_last_x, pre_last_y):
offset = df[x_channel]
offset = offset.shift(periods=1)
offset[0] = pre_last_x
x_diff = df[x_channel] - offset
# x_diff = df[x_channel].astype(np.int64) - offset.astype(np.int64)
x_diff = df[x_channel].astype(np.int32) - offset.astype(np.int32)
offset = df[y_channel]
offset = offset.shift(periods=1)
offset[0] = pre_last_y
y_diff = df[y_channel] - offset
# y_diff = df[y_channel].astype(np.int64) - offset.astype(np.int64)
y_diff = df[y_channel].astype(np.int32) - offset.astype(np.int32)
local_diff = y_diff / x_diff
print(local_diff)
# TODO: if user don't need, then can remove it
df[f'{x_channel}_{y_channel}_diff'] = local_diff
pre_last_x = df[x_channel].iat[-1]
pre_last_y = df[y_channel].iat[-1]
return df, pre_last_x, pre_last_y
def download(id, mode, x_channel, y_channel):
# TODO: can change different chunck size
pages_chunck_size = 250
sql_total = 0
conn_start = time.time()
# TODO: need the check port id
conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="54321")
print('conn time: ', time.time() - conn_start)
# conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="5432")
if __name__ == '__main__':
# TODO: function parameters: id, mode, x_channel, y_channel
conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="5432")
cur = conn.cursor()
# recording_data_metas: meta info to get the raw_data table
sql_str = f'select "raw_data" from "recording_data_metas" WHERE id = {id}'
first_sql = time.time()
id = 1262
# id = 407
# id = 1073
# id = 968
mode = 'differential'
# mode = 'integral'
# mode = 'log'
sql_str = f'select "raw_data" from "recording_data_metas" WHERE id = {id}' # recording_data_metas: meta info to get the raw_data table
# sql_str = 'select "raw_data" from "recording_data_metas" WHERE id = 407' # recording_data_metas: meta info to get the raw_data table
# sql_str = 'select * from "0_recording_data_raws" WHERE id = 33384'
cur.execute(sql_str)
print('first sql time: ', time.time() - first_sql)
raw_data_list = None
# x_
try:
raw_data_list = cur.fetchall()[0][0]
# print(ret[0][0]['0'])
channels = list(raw_data_list.keys())
df_all = pd.DataFrame()
# df = pd.DataFrame()
df = pd.DataFrame()
df_time = []
pre_last_x = 0
pre_last_y = 0
integral_result = 0
for raw_data_pages in range(len(raw_data_list[channels[0]])):
df = pd.DataFrame()
# print('len(raw_data_list[channels[0]]): ', len(raw_data_list[channels[0]]))
for raw_data_pages in range(len(raw_data_list[channels[0]])):
for channel in channels:
sql_str = f'select "data" from "{channel}_recording_data_raws" WHERE id = {raw_data_list[channel][raw_data_pages]}'
sql_start = time.time()
cur.execute(sql_str)
sql_total += (time.time() - sql_start)
data_value = cur.fetchall()[0][0].replace('"***"', ' ')
data_value = data_value.split(' ')[:-1]
if df.empty:
df_time = data_value[::2]
df[channel] = data_value
df = df.iloc[lambda x: x.index % 2 == 1].reset_index(drop=True)
df.insert(0, 'Time', df_time)
# transfer dataframe from string to number
cols = df.columns
df[cols] = df[cols].apply(pd.to_numeric, errors='coerce')
if mode == 'Log':
df.insert(0, 'time', df_time)
if mode == 'log':
x_channel = '1'
df = log(df, x_channel)
elif mode == 'Integral':
elif mode == 'integral':
# TODO: x_channel and y_channel will need to be set as parameters
x_channel = 'time'
y_channel = '0'
df, pre_last_x, pre_last_y, integral_result = integral(df, x_channel, y_channel, pre_last_x, pre_last_y, integral_result)
elif mode == 'Differential':
elif mode == 'differential':
x_channel = '1'
y_channel = '0'
df, pre_last_x, pre_last_y = differential(df, x_channel, y_channel, pre_last_x, pre_last_y)
df_all = pd.concat([df_all, df], ignore_index=True, sort=False)
if ~raw_data_pages and (raw_data_pages % pages_chunck_size == 0 or raw_data_pages == len(raw_data_list[channels[0]]) - 1):
if not(exists(f"./src/csv/{id}.csv")):
df.to_csv(f"./src/csv/{id}.csv", mode='a', index=False, header=True)
else:
df.to_csv(f"./src/csv/{id}.csv", mode='a', index=False, header=False)
""" if not(exists(f"./{id}.csv")):
df_all.to_csv(f"./{id}.csv", mode='a', index=False, header=True)
else:
df_all.to_csv(f"./{id}.csv", mode='a', index=False, header=False) """
df_all = pd.DataFrame()
print('total sql: ', sql_total)
# print(df)
df.to_csv(f"csv_test/{id}.csv", mode='a', index=False, header=False)
df = pd.DataFrame()
except BaseException as e:
print(e)
exit
return
if __name__ == '__main__':
params = {}
for input in sys.argv[1:]: #Now we're going to iterate over argv[1:] (argv[0] is the program name)
param = input.split("=") #Get what's left of the '='
params[param[0]] = param[1]
id = params['id']
mode = params['mode']
x_channel = params['x_channel']
y_channel = params['y_channel']
# print('download_functional init at main: ', id, mode, x_channel, y_channel)
start_time = time.time()
download(id, mode, x_channel, y_channel)
# download(408, '', 'Time', '0')
# download(408, '', 'Time', '0')
# download(405, 'Integral', 'Time', '0')
# download(405, 'Integral', 'Time', '0')
# download(408, 'Integral', 'Time', '0')
print('total: ', time.time() - start_time)
# download(1262, 'differential', '1', '0')
# 1262/differential/1/0
exit
@@ -31,6 +31,49 @@ export const clearDeleted = async (ctx, next) => {
next()
}
export const findWithIdRange = async (ctx, next) => {
const id = JSON.parse(ctx.query.id)
const range = JSON.parse(ctx.query.range)
const result = await recordingDataMeta.findWithIdRange1(id, range)
ctx.body = result
next()
}
export const get = async (ctx, next) => {
const id = ctx.query.id
const name = ctx.query.name
const uuid = ctx.query.uuid
const type = ctx.query.type
const time = ctx.query.time
// folder
const parent = ctx.query.parent
const limit = ctx.query.limit
const offset = ctx.query.offset
const order = ctx.query.order
const attrs = ctx.query.attrs
const group = ctx.query.group
// project
const project = ctx.query.project
const query = {}
let result = ''
if (id) query.id = id
if (name) query.name = name
if (uuid) query.uuid = uuid
if (time) query.time = time
if ((id || name || uuid || time) || type === undefined) result = await recordingDataMeta.getByQueryWithAttrs(query, limit, offset, order, attrs, group)
if (type === 'folder' && parent) result = await recordingDataMeta.getByFolderWithParentLimitOffsetOrderAttrs(type, parent, limit, offset, order, attrs)
if (type === 'project' && project) result = await recordingDataMeta.getByProject(project)
ctx.body = result
next()
}
export const getByID = async (ctx, next) => {
const id = ctx.params.id
const result = await recordingDataMeta.getByID(id)
@@ -1,12 +1,7 @@
import RecordingDataRaw from '../../models/file/recording_data_raw'
import RecordingDataMeta from '../../models/file/recording_data_meta'
import * as auth from '../auth'
import childProcess from 'child_process'
import path from 'path'
import { createReadStream } from 'fs'
const recordingDataRaw = new RecordingDataRaw()
const recordingDataMeta = new RecordingDataMeta()
export const create = async (ctx, next) => {
const index = ctx.params.channel
@@ -167,37 +162,3 @@ export const getAll = async (ctx, next) => {
next()
}
export const getCSV = async (ctx, next) => {
const id = ctx.params.id
const mode = ctx.params.mode
const x_channel = ctx.params.x_channel
const y_channel = ctx.params.y_channel
// console.log('getCSV: ', id, mode, x_channel, y_channel)
// const result = await recordingDataRaw.getCSV(id, mode, x_channel, y_channel)
const process = childProcess.spawnSync('python', [
path.join(__dirname, '/download_functional.py'),
'id=' + id,
'mode=' + mode,
'x_channel=' + x_channel,
'y_channel=' + y_channel,
], { maxBuffer: 2 * 1024 * 1024 * 1024 })
// console.log('process: ', process)
// process.stdout.on('data', (data) => {
// conosle.log('data', data)
// })
const result = await recordingDataMeta.getByID(id)
// console.log('result: ', result)
// console.log('result: ', result[0]['dataValues']['name'])
var file_name = id + '.csv'
// ctx.body = fs.stdout.toString()
// console.log('process.stdout.toString(): ', process.stdout.toString())
// console.log('process.stderr.toString(): ', process.stderr.toString())
ctx.type = 'stream'
ctx.attachment(result[0]['dataValues']['name']+'.csv')
ctx.body = createReadStream(path.resolve(__dirname, '../../csv', file_name))
// need to remove the csv file after download, since using 'append' for set header one time
const fs = require('fs')
fs.unlinkSync(path.resolve(__dirname, '../../csv', file_name))
next()
}
+11 -1
View File
@@ -10,6 +10,11 @@ import * as recordingDataRaw from './file/recording_data_raw'
import * as recordingDataMini from './file/recording_data_mini'
import * as analysis from './analysis'
import * as project from './project/project'
import * as projectReport from './project/project_report'
import * as projectMeta from './project/project_meta'
import * as chart from './configuration/chart'
import * as subject from './subject/subject'
import * as subjectData from './subject/subject_data'
export default {
upload,
@@ -23,5 +28,10 @@ export default {
recordingDataRaw,
recordingDataMini,
analysis,
project
project,
projectReport,
projectMeta,
chart,
subject,
subjectData
}
+82
View File
@@ -0,0 +1,82 @@
import ProjectMeta from '../../models/project/project_meta'
const projectMeta = new ProjectMeta()
export const create = async (ctx, next) => {
const data = ctx.request.body
console.log('create', data)
const result = await projectMeta.create(data)
ctx.body = result
next()
}
export const update = async (ctx, next) => {
const data = ctx.request.body
const id = ctx.params.id
const result = await projectMeta.update(data, id)
ctx.body = result
next()
}
export const updateByAttr = async (ctx, next) => {
const attr = ctx.params.attr
const value = ctx.params.value
const result = await projectMeta.updateByAttr(attr, value)
ctx.body = result
next()
}
export const get = async (ctx, next) => {
const id = ctx.query.id
const name = ctx.query.name
const uuid = ctx.query.uuid
const attrs = ctx.query.attrs
const order = ctx.query.order
const query = {}
let result = ''
if (id) query.id = id
if (name) query.name = name
if (uuid) query.uuid = uuid
result = await projectMeta.getByQueryWithAttrs(query, attrs, order)
ctx.body = result
next()
}
export const clearDeleted = async (ctx, next) => {
const result = await projectMeta.clearDeleted()
ctx.body = result
next()
}
export const getByID = async (ctx, next) => {
const index = ctx.params.channel
const id = ctx.params.id
const result = await projectMeta.getByID(id, index)
ctx.body = result
next()
}
export const getByAttr = async (ctx, next) => {
const attr = ctx.params.attr
const value = ctx.params.value
const result = await projectMeta.getByAttr(attr, value)
ctx.body = result
next()
}
export const getAll = async (ctx, next) => {
const result = await projectMeta.getAll()
ctx.body = result
next()
}
+82
View File
@@ -0,0 +1,82 @@
import ProjectReport from '../../models/project/project_report'
const projectReport = new ProjectReport()
export const create = async (ctx, next) => {
const data = ctx.request.body
console.log('create', data)
const result = await projectReport.create(data)
ctx.body = result
next()
}
export const update = async (ctx, next) => {
const data = ctx.request.body
const id = ctx.params.id
const result = await projectReport.update(data, id)
ctx.body = result
next()
}
export const updateByAttr = async (ctx, next) => {
const attr = ctx.params.attr
const value = ctx.params.value
const result = await projectReport.updateByAttr(attr, value)
ctx.body = result
next()
}
export const get = async (ctx, next) => {
const id = ctx.query.id
const name = ctx.query.name
const uuid = ctx.query.uuid
const attrs = ctx.query.attrs
const order = ctx.query.order
const query = {}
let result = ''
if (id) query.id = id
if (name) query.name = name
if (uuid) query.uuid = uuid
result = await projectReport.getByQueryWithAttrs(query, attrs, order)
ctx.body = result
next()
}
export const clearDeleted = async (ctx, next) => {
const result = await projectReport.clearDeleted()
ctx.body = result
next()
}
export const getByID = async (ctx, next) => {
const index = ctx.params.channel
const id = ctx.params.id
const result = await projectReport.getByID(id, index)
ctx.body = result
next()
}
export const getByAttr = async (ctx, next) => {
const attr = ctx.params.attr
const value = ctx.params.value
const result = await projectReport.getByAttr(attr, value)
ctx.body = result
next()
}
export const getAll = async (ctx, next) => {
const result = await projectReport.getAll()
ctx.body = result
next()
}
+40
View File
@@ -0,0 +1,40 @@
import SubjectModel from '../../models/subject/subject'
const subject = new SubjectModel()
export const create = async (ctx, next) => {
const data = ctx.request.body
console.log('create', data)
const result = await subject.create(data)
ctx.body = result
next()
}
export const update = async (ctx, next) => {
const data = ctx.request.body
const id = ctx.params.id
const result = await subject.update(data, id)
ctx.body = result
next()
}
export const get = async (ctx, next) => {
const configurationType = ctx.query.type
const id = ctx.query.id
const library = ctx.query.library
const query = { id: id, library: library }
const result = await subject.get(configurationType, query)
ctx.body = result
next()
}
export const clearDeleted = async (ctx, next) => {
const controllerID = ctx.params.controller_id
const result = await subject.clearDeleted(controllerID)
ctx.body = result
next()
}
+41
View File
@@ -0,0 +1,41 @@
import SubjectDataModel from '../../models/subject/subject_data'
const subjectData = new SubjectDataModel()
export const create = async (ctx, next) => {
const data = ctx.request.body
const result = await subjectData.create(data)
ctx.body = result
next()
}
export const update = async (ctx, next) => {
const data = ctx.request.body
const id = ctx.params.id
const result = await subjectData.update(data, id)
ctx.body = result
next()
}
export const get = async (ctx, next) => {
const subjectId = ctx.query.subject_id
const meta = ctx.query.meta
console.log('subject get', subjectId, meta)
const query = {}
if (subjectId) query.subject_id = subjectId
if (meta) query.meta = meta
const result = await subjectData.get(query)
ctx.body = result
next()
}
export const clearDeleted = async (ctx, next) => {
const controllerID = ctx.params.controller_id
const result = await subjectData.clearDeleted(controllerID)
ctx.body = result
next()
}
+5
View File
@@ -28,7 +28,12 @@ db.tasks = require('./schema/task.model')(sequelize, Sequelize)
db.devices = require('./schema/device.model')(sequelize, Sequelize)
db.collections = require('./schema/collection.model')(sequelize, Sequelize)
db.projects = require('./schema/project.model')(sequelize, Sequelize)
db.project_reports = require('./schema/project_report.model')(sequelize, Sequelize)
db.project_metas = require('./schema/project_meta.model')(sequelize, Sequelize)
db.recording_data_metas = require('./schema/recording_data_meta.model')(sequelize, Sequelize)
db.charts = require('./schema/configuration/chart.model')(sequelize, Sequelize)
db.subjects = require('./schema/subject/subject.model')(sequelize, Sequelize)
db.subject_datas = require('./schema/subject/subject_data.model')(sequelize, Sequelize)
for (let i = 0; i < 32; i++) {
db[i + '_recording_data_raws'] = require('./schema/recording_data_raw.model')(sequelize, Sequelize, i)
}
+47
View File
@@ -0,0 +1,47 @@
module.exports = (sequelize, Sequelize) => {
const Configuration = sequelize.define('configuration', {
name: {
type: Sequelize.STRING
},
library: {
type: Sequelize.STRING
},
layout: {
type: Sequelize.JSONB
},
chart: {
type: Sequelize.JSONB
},
parameter: {
type: Sequelize.JSONB
},
user_auth: {
type: Sequelize.JSONB,
defaultValue: {
owner: [],
maintainer: [],
guest: []
}
},
locked: {
type: Sequelize.BOOLEAN
},
deleted: {
type: Sequelize.BOOLEAN,
defaultValue: false
},
created_at: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP')
},
updated_at: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP')
}
},
{
timestamps: false
})
return Configuration
}
@@ -0,0 +1,48 @@
module.exports = (sequelize, Sequelize) => {
const Chart = sequelize.define('chart', {
name: {
type: Sequelize.STRING
},
description: {
type: Sequelize.STRING
},
library: {
type: Sequelize.STRING
},
num: {
type: Sequelize.INTEGER
},
data: {
type: Sequelize.JSONB
},
user_auth: {
type: Sequelize.JSONB,
defaultValue: {
owner: [],
maintainer: [],
guest: []
}
},
locked: {
type: Sequelize.BOOLEAN,
defaultValue: false
},
deleted: {
type: Sequelize.BOOLEAN,
defaultValue: false
},
created_at: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP')
},
updated_at: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP')
}
},
{
timestamps: false
})
return Chart
}
+8
View File
@@ -10,10 +10,18 @@ module.exports = (sequelize, Sequelize) => {
type: Sequelize.JSONB,
defaultValue: {}
},
cycle: {
type: Sequelize.JSONB,
defaultValue: []
},
device: {
type: Sequelize.JSONB,
defaultValue: {}
},
subject: {
type: Sequelize.JSONB,
defaultValue: []
},
uuid: {
type: Sequelize.STRING,
defaultValue: ''
+35
View File
@@ -0,0 +1,35 @@
module.exports = (sequelize, Sequelize) => {
const ProjectMeta = sequelize.define('project_metas', {
project: {
type: Sequelize.STRING
},
cycle: {
type: Sequelize.JSONB,
defaultValue: []
},
task: {
type: Sequelize.JSONB,
defaultValue: {}
},
serial_number: {
type: Sequelize.INTEGER
},
deleted: {
type: Sequelize.BOOLEAN,
defaultValue: false
},
created_at: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP')
},
updated_at: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP')
}
},
{
timestamps: false
})
return ProjectMeta
}
+52
View File
@@ -0,0 +1,52 @@
module.exports = (sequelize, Sequelize) => {
const ProjectReport = sequelize.define('project_report', {
name: {
type: Sequelize.STRING,
default: ''
},
desc: {
type: Sequelize.STRING,
default: ''
},
task: {
type: Sequelize.JSONB,
defaultValue: {}
},
cycle: {
type: Sequelize.JSONB,
defaultValue: []
},
device: {
type: Sequelize.JSONB,
defaultValue: {}
},
uuid: {
type: Sequelize.STRING(36)
},
user_auth: {
type: Sequelize.JSONB,
defaultValue: {
owner: [],
maintainer: [],
guest: []
}
},
deleted: {
type: Sequelize.BOOLEAN,
defaultValue: false
},
created_at: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP')
},
updated_at: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP')
}
},
{
timestamps: false
})
return ProjectReport
}
@@ -53,6 +53,9 @@ module.exports = (sequelize, Sequelize) => {
type: Sequelize.JSONB,
defaultValue: {}
},
project: {
type: Sequelize.INTEGER
},
parameter_set: {
type: Sequelize.JSONB,
defaultValue: {}
+38
View File
@@ -0,0 +1,38 @@
module.exports = (sequelize, Sequelize) => {
const Subject = sequelize.define('subject', {
name: {
type: Sequelize.STRING
},
description: {
type: Sequelize.STRING
},
cache: {
type: Sequelize.JSONB
},
user_auth: {
type: Sequelize.JSONB,
defaultValue: {
owner: [],
maintainer: [],
guest: []
}
},
deleted: {
type: Sequelize.BOOLEAN,
defaultValue: false
},
created_at: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP')
},
updated_at: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP')
}
},
{
timestamps: false
})
return Subject
}
@@ -0,0 +1,44 @@
module.exports = (sequelize, Sequelize) => {
const SubjectData = sequelize.define('subject_datas', {
subject_id: {
type: Sequelize.INTEGER
},
mode: {
type: Sequelize.JSONB
},
data: {
type: Sequelize.JSONB
},
user_auth: {
type: Sequelize.JSONB,
defaultValue: {
owner: [],
maintainer: [],
guest: []
}
},
meta: {
type: Sequelize.STRING
},
project: {
type: Sequelize.STRING
},
deleted: {
type: Sequelize.BOOLEAN,
defaultValue: false
},
created_at: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP')
},
updated_at: {
type: Sequelize.DATE,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP')
}
},
{
timestamps: false
})
return SubjectData
}
+79
View File
@@ -0,0 +1,79 @@
const db = require('../../db/database')
const Op = db.Sequelize.Op
class ConfigurationModel {
async create (req) {
return db.charts.create(
req
)
}
async update (req, id) {
if (id != null) {
const ret = await db.charts.update(
req,
{
where: {
id: id
}
}
)
return ret
} else {
const ret = await db.charts.update(
req,
{
where: {}
}
)
return ret
}
}
async clearDeleted () {
const ret = await db.charts.destroy(
{
where: {
deleted: true
}
}
)
return ret
}
async get (type, query) {
let where = {}
if (type === 'chart') {
where = {
chart: {
[Op.ne]: null
}
}
}
Object.assign(query, where)
const ret = await db.charts.findAll(
{
where: where
}
)
return ret
}
async getByQueryWithAttrs (query, limit, offset, order, attrs, group) {
const attrList = attrs ? attrs.split('-') : undefined
const ret = await db.charts.findAll(
{
// where: db.Sequelize.where(db.Sequelize.fn('DATE', db.Sequelize.col('created_at')), '2022-12-02'),
where: query,
attributes: attrList,
order: order,
limit: limit,
offset: offset
}
)
return ret
}
}
export default ConfigurationModel
+38
View File
@@ -19,6 +19,44 @@ class CollectionModel {
return ret
}
async getByQueryWithAttrs (query, attrs) {
const attrList = attrs ? attrs.split('-') : undefined
const ret = await db.collections.findAll(
{
where: query,
attributes: attrList
}
)
return ret
}
async getByParentWithOffsetLimitOrder (type, parent, limit, offset, order, attrs) {
const attrList = attrs ? attrs.split('-') : undefined
const orderList = order ? order.split('-') : ['id', 'desc']
const collectionType = type + '::jsonb'
const parentOp = {}
console.log('getByParentWithOffsetLimitOrder', type, parent, limit, offset, order, attrs)
parentOp[collectionType] = {
[Op.contains]: '[' + parent + ']'
}
const ret = await db.collections.findAndCountAll(
{
where: {
parent: parentOp
},
attributes: attrList,
order: [
orderList
],
limit: limit,
offset: offset
}
)
return ret
}
async clearDeleted (controllerID) {
let ret
if (controllerID == null) {
+260 -3
View File
@@ -98,7 +98,7 @@ class RecordingDataMeta {
return ret
}
async getByAttrParent(type, parent, attr) {
async getByAttrParent (type, parent, attr) {
const attrList = attr.split('-')
const collectionType = type + '::jsonb'
const parentOp = {}
@@ -113,13 +113,13 @@ class RecordingDataMeta {
attributes: attrList,
order: [
['id', 'DESC']
],
]
}
)
return ret
}
async getCountByParent(type, parent) {
async getCountByParent (type, parent) {
const collectionType = type + '::jsonb'
const parentOp = {}
parentOp[collectionType] = {
@@ -246,6 +246,263 @@ class RecordingDataMeta {
})
return ret
}
async getByQueryWithAttrs (query, limit, offset, order, attrs, group) {
console.log('getByQueryWithAttrs', query, limit, offset, order, attrs, group)
const result = {}
let attrList = attrs ? attrs.split('-') : undefined
// group by time
if (group === 'time') {
const timeGroupNum = await db.recording_data_metas.findAll(
{
attributes: [[db.Sequelize.fn('COUNT', db.Sequelize.fn('DISTINCT', db.Sequelize.fn('DATE', db.Sequelize.col('created_at')))), 'GroupCount']]
}
)
result.counts = timeGroupNum
attrList = [[db.Sequelize.fn('DATE', db.Sequelize.col('created_at')), 'date'], [db.sequelize.fn('COUNT', 'date'), 'count']]
group = [db.Sequelize.fn('DATE', db.Sequelize.col('created_at')), 'date']
order = [[db.Sequelize.fn('DATE', db.Sequelize.col('created_at')), 'desc']]
}
// group by project
if (group === 'project') {
attrList = [['project', 'project']]
group = ['project', 'project']
order = [['project', 'desc']]
}
// query by time
if (query.time) {
const start = new Date(query.time)
const end = new Date(query.time)
end.setDate(end.getDate() + 1)
delete query.time
query.created_at = {
[Op.gte]: start,
[Op.lte]: end
}
}
const ret = await db.recording_data_metas.findAll(
{
// where: db.Sequelize.where(db.Sequelize.fn('DATE', db.Sequelize.col('created_at')), '2022-12-02'),
where: query,
attributes: attrList,
group: group,
order: order,
limit: limit,
offset: offset
}
)
result.rows = ret
return result
}
async getByFolderWithParentLimitOffsetOrderAttrs (type, parent, limit, offset, order, attrs) {
const attrList = attrs ? attrs.split('-') : undefined
const orderList = order ? order.split('-') : ['id', 'desc']
const collectionType = type + '::jsonb'
const parentOp = {}
console.log('getByFolderWithParentLimitOffsetOrderAttr', type, parent, limit, offset, order, attrs)
parentOp[collectionType] = {
[Op.contains]: '[' + parent + ']'
}
const ret = await db.recording_data_metas.findAndCountAll(
{
where: {
parent: parentOp
},
attributes: attrList,
order: [
[orderList[0], orderList[1]]
],
limit: limit,
offset: offset
}
)
return ret
}
async getByProject (uuid) {
const [results, _] = await db.sequelize.query(`
WITH ProjectData AS (SELECT * FROM project_reports WHERE uuid='${uuid}')
SELECT rdm.* FROM ProjectData pr
LEFT JOIN project_metas pm ON pr.uuid = pm.project
LEFT JOIN recording_data_metas rdm ON pm.id = rdm.project
ORDER BY rdm.id desc
`)
return results
}
async findWithIdRange1 (metaIdArray, range) {
/* get x channel & range, y channel & range */
const channelX = range[0][0]
const rangeX = range[0].slice(1, 3).sort(function (a, b) {
return a - b
})
const channelY = range[1][0]
const rangeY = range[1].slice(1, 3).sort(function (a, b) {
return a - b
})
// console.log(channelX, rangeX, channelY, rangeY)
const pairID = []
/* loop with given idArray */
for (const _id of metaIdArray) {
const meta = await db.recording_data_metas.findOne({
where: {
id: _id
},
attributes: ['id', 'raw_data', 'name']
})
/* if channel has time */
if (channelX === 'Time' || channelY === 'Time') {
if (channelX === 'Time') {
/* if x is time, then get the y channel data */
for (const index in meta.raw_data[channelY]) {
/* if meta id is paired already then break */
if (pairID.findIndex((ele) => ele[0] === meta.id) !== -1) break
const rawID = meta.raw_data[channelY][index]
const rawData = await db[channelY + '_recording_data_raws'].findOne({
where: {
id: rawID
}
})
/* get time & data */
const data = rawData.data.replace(/"\*\*\*"/g, ' ').split(' ')
const time = data.filter((ele, index) => index % 2 === 0)
const value = data.filter((ele, index) => index % 2 === 1)
/* check with value & time range */
for (let index = 0; index < value.length; index++) {
if (value[index] <= rangeY[1] && value[index] >= rangeY[0]) {
if (time[index] <= rangeX[1] && time[index] >= rangeX[0]) {
pairID.push([meta.id, meta.name])
break
}
}
}
}
}
if (channelY === 'Time') {
for (const index in meta.raw_data[channelX]) {
if (pairID.findIndex((ele) => ele[0] === meta.id) !== -1) break
const rawID = meta.raw_data[channelX][index]
const rawData = await db[channelX + '_recording_data_raws'].findOne({
where: {
id: rawID
}
})
/* get time & data */
const data = rawData.data.replace(/"\*\*\*"/g, ' ').split(' ')
const time = data.filter((ele, index) => index % 2 === 0)
const value = data.filter((ele, index) => index % 2 === 1)
/* check with value & time range */
for (let index = 0; index < value.length; index++) {
if (value[index] <= rangeX[1] && value[index] >= rangeX[0]) {
if (time[index] <= rangeY[1] && time[index] >= rangeY[0]) {
pairID.push([meta.id, meta.name])
break
}
}
}
}
}
} else {
/* when two channels are not time */
/* loop with raw data id */
for (const index in meta.raw_data[channelX]) {
/* if meta id is paired already then break */
if (pairID.findIndex((ele) => ele[0] === meta.id) !== -1) break
/* get X channel data */
const rawIDX = meta.raw_data[channelX][index]
const rawDataX = await db[channelX + '_recording_data_raws'].findOne({
where: {
id: rawIDX
}
})
const valueX = rawDataX.data.replace(/"\*\*\*"/g, ' ').split(' ').filter((ele, index) => index % 2 === 1)
/* get Y channel data */
const rawIDY = meta.raw_data[channelY][index]
const rawDataY = await db[channelY + '_recording_data_raws'].findOne({
where: {
id: rawIDY
}
})
const valueY = rawDataY.data.replace(/"\*\*\*"/g, ' ').split(' ').filter((ele, index) => index % 2 === 1)
/* if x value in range then check y value in range */
for (let index = 0; index < valueX.length; index++) {
if (valueX[index] <= rangeX[1] && valueX[index] >= rangeX[0]) {
if (valueY[index] <= rangeY[1] && valueY[index] >= rangeY[0]) {
pairID.push([meta.id, meta.name])
break
}
}
}
}
}
console.log('pair', pairID)
}
return pairID
}
// async findWithIdRange (id, range) {
// const channel = String(range[0][0])
// const dataRange = range[0].slice(1, 3)
// const matchFile = []
// for (const _id of id) {
// console.log('id', _id)
// const meta = await db.recording_data_metas.findOne({
// where: {
// id: _id
// },
// attributes: ['id', 'raw_data', 'name']
// })
// if (await matchRange(channel, meta.raw_data[channel], dataRange) === true) {
// matchFile.push(meta.id)
// }
// }
// console.log('matchFile', matchFile)
// const channel1 = range[1][0]
// const range1 = range[1].slice(1, 3)
// const doubleMatchFile = []
// for (const _id of matchFile) {
// const meta = await db.recording_data_metas.findOne({
// where: {
// id: _id
// },
// attributes: ['id', 'raw_data', 'name']
// })
// console.log('meta', meta.id)
// if (await matchRange(channel1, meta.raw_data[channel1], range1) === true) {
// doubleMatchFile.push([meta.id, meta.name])
// }
// }
// console.log('doubleMatchFile', doubleMatchFile)
// return doubleMatchFile
// }
// }
// async function matchRange (channel, rawIdList, range) {
// console.log('matchRange', channel, rawIdList, range)
// for (const rawId of rawIdList) {
// const raw = await db[channel + '_recording_data_raws'].findOne({
// where: {
// id: rawId
// }
// })
// const value = raw.data.split('"***"')[0].split(' ').filter((ele, index) => index % 2 === 1)
// for (const val of value) {
// console.log('val', val)
// if (val < range[1] && val > range[0]) {
// return true
// }
// }
// }
// return false
}
export default RecordingDataMeta
+46
View File
@@ -0,0 +1,46 @@
const db = require('../../db/database')
class ProjectMeta {
async create (req) {
return db.project_metas.create(req)
}
async update (req, id) {
if (id != null) {
const ret = await db.project_metas.update(
req,
{
where: {
id: id
}
}
)
return ret
} else {
const ret = await db.project_metas.update(
req,
{
where: {}
}
)
return ret
}
}
async getByQueryWithAttrs (query, attrs, order) {
const attrList = attrs ? attrs.split('-') : undefined
const orderList = order ? order.split('-') : ['id', 'desc']
console.log('getByQueryWithAttrs', query, attrs, order)
const ret = await db.project_metas.findAndCountAll(
{
where: query,
attributes: attrList,
// group: group,
order: [orderList]
}
)
return ret
}
}
export default ProjectMeta
+46
View File
@@ -0,0 +1,46 @@
const db = require('../../db/database')
class ProjectReport {
async create (req) {
return db.project_reports.create(req)
}
async update (req, id) {
if (id != null) {
const ret = await db.project_reports.update(
req,
{
where: {
id: id
}
}
)
return ret
} else {
const ret = await db.project_reports.update(
req,
{
where: {}
}
)
return ret
}
}
async getByQueryWithAttrs (query, attrs, order) {
const attrList = attrs ? attrs.split('-') : undefined
const orderList = order ? order.split('-') : ['id', 'desc']
console.log('getByQueryWithAttrs', query, attrs, order)
const ret = await db.project_reports.findAndCountAll(
{
where: query,
attributes: attrList,
// group: group,
order: [orderList]
}
)
return ret
}
}
export default ProjectReport
+79
View File
@@ -0,0 +1,79 @@
const db = require('../../db/database')
const Op = db.Sequelize.Op
class SubjectModel {
async create (req) {
return db.subjects.create(
req
)
}
async update (req, id) {
if (id != null) {
const ret = await db.subjects.update(
req,
{
where: {
id: id
}
}
)
return ret
} else {
const ret = await db.subjects.update(
req,
{
where: {}
}
)
return ret
}
}
async clearDeleted () {
const ret = await db.subjects.destroy(
{
where: {
deleted: true
}
}
)
return ret
}
async get (type, query) {
let where = {}
if (type === 'chart') {
where = {
chart: {
[Op.ne]: null
}
}
}
Object.assign(query, where)
const ret = await db.subjects.findAll(
{
where: where
}
)
return ret
}
async getByQueryWithAttrs (query, limit, offset, order, attrs, group) {
const attrList = attrs ? attrs.split('-') : undefined
const ret = await db.subjects.findAll(
{
// where: db.Sequelize.where(db.Sequelize.fn('DATE', db.Sequelize.col('created_at')), '2022-12-02'),
where: query,
attributes: attrList,
order: order,
limit: limit,
offset: offset
}
)
return ret
}
}
export default SubjectModel
+70
View File
@@ -0,0 +1,70 @@
const db = require('../../db/database')
const Op = db.Sequelize.Op
class SubjectDataModel {
async create (req) {
return db.charts.create(
req
)
}
async update (req, id) {
if (id != null) {
const ret = await db.subject_datas.update(
req,
{
where: {
id: id
}
}
)
return ret
} else {
const ret = await db.subject_datas.update(
req,
{
where: {}
}
)
return ret
}
}
async clearDeleted () {
const ret = await db.subject_datas.destroy(
{
where: {
deleted: true
}
}
)
return ret
}
async get (query) {
const ret = await db.subject_datas.findAll(
{
where: query
}
)
return ret
}
async getByQueryWithAttrs (query, limit, offset, order, attrs, group) {
const attrList = attrs ? attrs.split('-') : undefined
const ret = await db.subject_datas.findAll(
{
// where: db.Sequelize.where(db.Sequelize.fn('DATE', db.Sequelize.col('created_at')), '2022-12-02'),
where: query,
attributes: attrList,
order: order,
limit: limit,
offset: offset
}
)
return ret
}
}
export default SubjectDataModel
+34 -2
View File
@@ -30,6 +30,8 @@ export default router
.get('/api/file/collection/get/all', controllers.collection.getAll)
.get('/api/file/collection/clear_deleted/', controllers.collection.clearDeleted)
.get('/api/file/collection/get', controllers.collection.get)
// meta
.post('/api/file/meta/create', controllers.recordingDataMeta.create)
.post('/api/file/meta/update/:id', controllers.recordingDataMeta.update)
@@ -43,10 +45,12 @@ export default router
.get('/api/file/meta/get_count_by_parent/:type/:parent', controllers.recordingDataMeta.getCountByParent)
.get('/api/file/meta/get_by_parent_with_limit/:type/:parent/:limit/:offset/:order', controllers.recordingDataMeta.getByParentWithLimit)
.get('/api/file/meta/get_by_attr_parent_with_limit/:type/:parent/:limit/:offset/:order/:attr', controllers.recordingDataMeta.getByAttrsParentWithLimit)
.get('/api/file/meta/get_or_create_by_path/:path', controllers.recordingDataMeta.getOrCreateByPath)
.get('/api/file/meta/get/all', controllers.recordingDataMeta.getAll)
.get('/api/file/meta/find', controllers.recordingDataMeta.findWithIdRange)
.get('/api/file/meta/get', controllers.recordingDataMeta.get)
// raw
.post('/api/file/raw/create/:channel', controllers.recordingDataRaw.create)
.post('/api/file/raw/update/:channel/:id', controllers.recordingDataRaw.update)
@@ -59,7 +63,6 @@ export default router
.get('/api/file/raw/get_attr_by_id/:channel/:id/:attr', controllers.recordingDataRaw.getAttrByID)
.get('/api/file/raw/get_attr_by_ids/:channel/:id/:attr', controllers.recordingDataRaw.getAttrByIDs)
.get('/api/file/raw/get_attr_by_parent/:channel/:parent/:attr', controllers.recordingDataRaw.getAttrByParent)
.get('/api/file/raw/get_csv/:id/:mode/:x_channel/:y_channel', controllers.recordingDataRaw.getCSV)
// mini
.post('/api/file/mini/create/:channel', controllers.recordingDataMini.create)
@@ -113,6 +116,7 @@ export default router
.post('/api/task/update_by_id/:id', controllers.task.updateByID)
// analyze
.post('/api/analysis/spike_detect', controllers.analysis.spikeDetect)
.get('/api/analysis/get/csv', controllers.analysis.getCSV)
// project
.post('/api/project/create', controllers.project.create)
@@ -123,3 +127,31 @@ export default router
.get('/api/project/get_by_id/:id', controllers.project.getByID)
.get('/api/project/get_by_attr/:attr/:value', controllers.project.getByAttr)
.get('/api/project/get/all', controllers.project.getAll)
// projectReport
.post('/api/project/report/create', controllers.projectReport.create)
.post('/api/project/report/update/:id', controllers.projectReport.update)
.get('/api/project/report/get', controllers.projectReport.get)
// projectMeta
.post('/api/project/meta/create', controllers.projectMeta.create)
.post('/api/project/meta/update/:id', controllers.projectMeta.update)
.get('/api/project/meta/get', controllers.projectMeta.get)
// configuration
.post('/api/configuration/create', controllers.chart.create)
.post('/api/configuration/update/:id', controllers.chart.update)
.get('/api/configuration/get', controllers.chart.get)
.get('/api/configuration/clear_deleted/', controllers.chart.clearDeleted)
// subject
.post('/api/subject/create', controllers.subject.create)
.post('/api/subject/update/:id', controllers.subject.update)
.get('/api/subject/get', controllers.subject.get)
.get('/api/subject/clear_deleted/', controllers.subject.clearDeleted)
// subject_data
.post('/api/subject_data/create', controllers.subjectData.create)
.post('/api/subject_data/update/:id', controllers.subjectData.update)
.get('/api/subject_data/get', controllers.subjectData.get)
.get('/api/subject_data/clear_deleted/', controllers.subjectData.clearDeleted)