wisetop/controller-data-analysis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add rolling avg function

Browse Source
This commit is contained in:
lai8928
2023-06-01 11:06:14 +08:00
parent fe2e95399d
commit a966dee042
4 changed files with 116 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cynthia reference.xlsx
BIN
View File
Binary file not shown.
ana_func.py
+67
View File
@@ -131,4 +131,71 @@ class dataAnalyticFunc():
temp_value_sum = row[int_value_name]
temp_key_num = 1
return integral_value
def SMA(self, win, n ,value_key='y_data'):
data_df = self._raw_data[value_key]
rev_sum = lambda data_df: data_df[::-1].rolling(window=win).mean()
self._raw_data['rolling_rev_mean'] = rev_sum(self._raw_data[value_key])#[::-1].reset_index(drop=True))
# print(self._raw_data)
raw_data_rev = list(self._raw_data[value_key][::-1].reset_index(drop=True))
rolling_mean_rev = list(self._raw_data['rolling_rev_mean'][::-1].reset_index(drop=True))
win1 = len(raw_data_rev)//10
std_arr = [np.std(raw_data_rev[:win1])] * win1
mean_arr = [np.mean(raw_data_rev[:win1])] * win1
sum_temp = sum(raw_data_rev[:win1])
# start_time1 = time.time()
# std_arr2 = std_arr + [np.std(raw_data_rev[0:win1+ i ]) for i in range(len(raw_data_rev)-win1)]
# mean_arr2 = mean_arr + [np.mean(raw_data_rev[0:win1+ i ]) for i in range(len(raw_data_rev)-win1)]
end_time1 = time.time()
for ind_i in range(len(raw_data_rev)-win1):
key_temp = win1 + ind_i
# sum_temp += raw_data_rev[key_temp]
# mean_arr.append(sum_temp/key_temp)
# std_arr.append(math.sqrt(sum([pow(i- mean_arr[-1],2) for i in raw_data_rev[:key_temp]])/key_temp))
mean_arr.append(np.mean(raw_data_rev[:key_temp]))
std_arr.append(np.std(raw_data_rev[:key_temp]))
if (rolling_mean_rev[key_temp+1]> mean_arr[-1] + n * std_arr[-1]) or (rolling_mean_rev[key_temp+1]< mean_arr[-1] - n * std_arr[-1]):
break
end_time2 = time.time()
# print('old',end_time1-start_time1)
print('time',end_time2-end_time1)
# print(std_arr[:win+10])
plt.plot([df_i + std_j*n for df_i, std_j in zip(mean_arr,std_arr)])
plt.plot([df_i - std_j*n for df_i, std_j in zip(mean_arr,std_arr)])
# plt.plot([df_i + std_j*4 for df_i, std_j in zip(mean_arr2,std_arr2)])
# plt.plot([df_i - std_j*4 for df_i, std_j in zip(mean_arr2,std_arr2)])
plt.plot(raw_data_rev[:len(std_arr)])
plt.plot(rolling_mean_rev)
plt.plot(mean_arr)
plt.plot(std_arr)
# df = list(data_df)[:600:-1]
# std_arr = [np.std(df[0:win+ i ]) for i in range(len(df)-win)]
# mean_arr = [np.mean(df[0:win+ i ]) for i in range(len(df)-win)]
# # print([[0,win+ i ] for i in range(len(df)-win)][:10])
# # print(len(std_arr),len(df))
# # fig, ax1 = plt.subplots()
# # ax2 = ax1.twinx()
# plt.plot(df[win:],color = 'black')
# plt.plot(std_arr, color='b')
# plt.plot([df_i + std_j*2 for df_i, std_j in zip(mean_arr,std_arr)])
# plt.plot(mean_arr,'.')
# # plt.plot(self._raw_data[value_key])
# # plt.plot(df.rolling(win).mean())
# # fig.tight_layout()
plt.show()
# print(df)
# self._raw_data['rolling_mean'] = self._raw_data[value_key].rolling(win, center=True).mean()
# std_arr = self._raw_data[value_key].rolling(win, center=True).std()
# self._raw_data['rolling_std'] = std_arr
# var_arr = self._raw_data[value_key].rolling(win, center=True).var()
# self._raw_data['rolling_var'] = var_arr
# print(np.std(std_arr))
# print('std error: ', self._raw_data[value_key].sem()/self._raw_data[value_key].mean())
# #
return [std_arr[-1],len(self._raw_data)-key_temp]
def cutoff(self):
pass
main.py
+5 -2
View File
@@ -8,7 +8,7 @@ import json,csv
# from deviceManager import DeviceManager
# from requests import Requests
from mqtt import MqttThread
from test import VTmode_run, slop_run ,CVmode_run
from test import VTmode_run, slop_run ,CVmode_run, movstd_run
class Main():
def __init__(self, controller_id = 'dc:a6:32:0f:56:9d', mqtt_ip = '192.168.2.1') -> None:
@@ -45,9 +45,12 @@ class Main():
dict_writer.writerow(csv_data)
elif input_data['mode'] == 3:
raw_data = CVmode_run()
else:
elif input_data['mode'] == 2:
for i in search_id:
raw_data = slop_run(i, input_data['data_channel'], input_data['data']['window'])
else:
for i in search_id:
raw_data = movstd_run(i, input_data['data_channel'])
self._mqttThread.publish(raw_data)
end_time = time.time()
print("執行時間：%f 秒" % (end_time - start_time))
test.py
+44 -41
View File
@@ -1,40 +1,41 @@
import psycopg2
# import psycopg2
import ana_func
import matplotlib.pyplot as plt
import csv
import pandas as pd
import numpy as np
def search_id_by_str(keyword):
conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="5432")
cur = conn.cursor()
sql_str = f"select id from recording_data_metas WHERE name ILIKE '%"+keyword+"%'"
cur.execute(sql_str)
data = cur.fetchall()
data = [i[0] for i in data]
return data
# def search_id_by_str(keyword):
# conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="5432")
# cur = conn.cursor()
# sql_str = f"select id from recording_data_metas WHERE name ILIKE '%"+keyword+"%'"
# cur.execute(sql_str)
# data = cur.fetchall()
# data = [i[0] for i in data]
# return data
def read_data(id,channel_list):
conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="5432")
cur = conn.cursor()
# def read_data(id,channel_list):
# conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="5432")
# cur = conn.cursor()
sql_str = f'select "raw_data" from "recording_data_metas" WHERE id = {id}'
cur.execute(sql_str)
data = cur.fetchall()[0][0]
channel_data = []
for channel in channel_list:
i = str(channel)
channel_data_temp = []
for j in data[i]:
# print(j)
sql_str = f'select "data" from "{i}_recording_data_raws" WHERE id = {j}'
cur.execute(sql_str)
raw_data = cur.fetchall()[0][0]
raw_data = raw_data.replace('"***"',' ').split(" ")
channel_data_temp =channel_data_temp + [int(raw_data[k]) for k in range(len(raw_data)) if k%2]
# print(len(channel_data_temp))
channel_data.append(channel_data_temp)
# sql_str = f'select "raw_data" from "recording_data_metas" WHERE id = {id}'
# cur.execute(sql_str)
# data = cur.fetchall()[0][0]
# channel_data = []
# for channel in channel_list:
# i = str(channel)
# channel_data_temp = []
# for j in data[i]:
# # print(j)
# sql_str = f'select "data" from "{i}_recording_data_raws" WHERE id = {j}'
# cur.execute(sql_str)
# raw_data = cur.fetchall()[0][0]
# raw_data = raw_data.replace('"***"',' ').split(" ")
# channel_data_temp =channel_data_temp + [int(raw_data[k]) for k in range(len(raw_data)) if k%2]
# # print(len(channel_data_temp))
# channel_data.append(channel_data_temp)
return channel_data
# return channel_data
def read_data_csv(file_name=0, start_row=0):
df = pd.read_excel("read_file/2021-8-9-15-42-46-0_CV discussion.xlsx",skiprows = 63, usecols="B,D,F,H")
@@ -99,16 +100,6 @@ def VTmode_run(id, channel_list, perc):
print('trigger point',triggerpoint, slope, center_point)
data_list.update(center_point_dict)
# Plot
# plt.plot(channel_data[0],channel_data[1],'o',markersize=2)
# print(peak_list)
# for i in peak_list[1:]:
# plt.hlines(i,min(channel_data[0]),max(channel_data[0]),color='r')
# for j in triggerpoint:
# plt.plot(j[0],j[1],'o',markersize=5)
# plt.savefig('fig/test'+ str(id) +'.png')
# plt.close()
resultList = [id , *channel_data, peak_list, triggerpoint]
channel_data.append(peak_list)
@@ -116,5 +107,17 @@ def VTmode_run(id, channel_list, perc):
return resultList, data_list
# if __name__ == '__main__':
# VTmode_run()
def movstd_run(channel_data):
# def movstd_run(id, channel_list):
# channel_data = read_data(id, channel_list)
movs_mode = ana_func.dataAnalyticFunc(channel_data[0], channel_data[1])
movs_mode.SMA(20,5)
return np.mean(channel_data[0])
if __name__ == '__main__':
df = pd.read_excel("Cynthia reference.xlsx",sheet_name="ELITE EIS Data set", skiprows=7)
data = list(df["Impedance [ohm]"])[2:][-6000:]
print(movstd_run([data,data]))