add sleep 1 sec

ana_func.py

@@ -0,0 +1,134 @@
+import random, math, time
+import matplotlib.pyplot as plt 
+import numpy as np
+from scipy import signal
+import pandas as pd
+
+class dataAnalyticFunc():
+    def __init__(self, x_data, y_data, ref_data = None) ->None:
+        if len(x_data)!=len(y_data):
+            print('Scale of x_data and y_data are different.')
+            return None
+        self._raw_data = pd.DataFrame({'x_data':x_data,'y_data':y_data})
+        if ref_data:
+            print(ref_data[:10])
+            if len(ref_data) == len(self._raw_data):
+                self._raw_data['ref_data'] = ref_data
+        # print(self._raw_data['y_data'].tolist())
+
+    def find_peak(self, value_name, key_name):
+        signal.find_peaks_cwt(list(self._raw_data[value_name]),np.arange(100,200))
+
+    def trigger_point(self, value_data, key_data, trigger, order = 1, drop = 1):
+        if value_data not in list(self._raw_data) or key_data not in list(self._raw_data):
+            print( 'There is no '+ str(value_data) + 'in datafram. ')
+            return False
+        # print(self._raw_data[value_data].head(10))
+        last_row, last_key = [],False
+        for i,row in self._raw_data.sort_values(key_data)[::order].iterrows():
+            # print(row[key_data])
+            if row[value_data]*drop <= trigger*drop:
+                print (i,type(row))
+                last_row ,last_key = row,True
+                print(last_row[key_data])
+                break
+        # print(last_row==False)
+        if last_key ==False:
+            print('No data meet the criteria.')
+            return False
+        
+        # print(last_key)
+        # return [int(self._raw_data[key_data][last_key]), int(self._raw_data[value_data][last_key])]
+        return [last_row[key_data],last_row[value_data]]
+
+    def histogram_find_peak(self, bins =100): #, plot =True):
+        # if bins == 0:
+        #     bins = 100
+            # bins = round(len(self._raw_data)/10)
+        arr_list = self._raw_data['y_data'].tolist()
+        self._raw_data['hist_list'] = pd.cut(self._raw_data['y_data'], bins)
+ 
+        hist_list = self._raw_data['hist_list'].value_counts(sort=False).tolist()
+        cut_bins = self._raw_data['hist_list'].value_counts(sort=False).keys()
+        # print(cut_bins[0])
+        # hist_list = [len([j for j in self._raw_data_y if j>=hist_range[i] and j<hist_range[i+1]]) for i in range(len(hist_range)-1)]
+        peek_temp_list,_ = signal.find_peaks(list(hist_list))
+        peek_temp_list = list(peek_temp_list)
+        if np.sign(np.diff(hist_list))[0] < 0:
+            peek_temp_list = [0] + peek_temp_list
+        if np.sign(np.diff(hist_list))[-1] > 0:
+            peek_temp_list = peek_temp_list+[len(hist_list)-1]
+        peek_list = [i for i in peek_temp_list if hist_list[i] >= len(self._raw_data)/bins*3]
+        print(peek_list)
+        results_half = signal.peak_widths(hist_list, peek_list, rel_height=0.8)
+        # print(results_half)
+        peak_range = [ pd.Interval(cut_bins[round(results_half[2][i])].left, cut_bins[round(results_half[3][i])].right) for i in range(len(results_half[2]))]
+        # print(peak_range)
+        peak_wid = [ np.mean([ j for j in arr_list if j in i]) for i in peak_range ]
+        data_list = {'ground':round(min(peak_wid)), 'ceiling':round(max(peak_wid)) }
+        # print(peek_list)
+        # print("peak_wid = "+str(peak_wid))
+        print(data_list)
+            
+        return data_list, cut_bins
+
+    def cal_slop_array(self, window = 10):
+        # print(self._raw_data)
+        print('window')
+        if window*2+1 >= len(self._raw_data):
+            print('Scale of data are smaller than window.')
+            return False
+        slop_a = [0]*window
+        for i in range(len(self._raw_data)-window*2):
+            # print(list(self._raw_data['x_data'][i:i+window*2+1]))
+            s,_ = np.polyfit(list(self._raw_data['x_data'][i:i+window*2+1]), list(self._raw_data['y_data'][i:i+window*2+1]), 1)
+            # time.sleep(3)
+            # print(s)
+            slop_a.append(s)
+        slop_a = slop_a+[0]* window
+        self._raw_data['1D'] = slop_a
+
+        print(list(self._raw_data))
+        return slop_a
+
+    def guess_func(self,value_name='x_data', window = 10):
+        win = window
+        print('guess_func')
+        X = np.array([[math.pow(i,j) for j in range(4)] for i in np.linspace(-1*win,win,2*win+1)])
+        J = np.linalg.inv(X.T.dot(X)).dot(X.T)
+        slop_list = []
+        for i in range(len(self._raw_data[value_name][win:-1*win])):
+            slop_list.append(J.dot(np.array(self._raw_data[value_name][i:i+2*win+1])).tolist()[1])
+        slop_list = [slop_list[0]]*win +slop_list+ [slop_list[-1]]*win
+        self._raw_data[value_name+'_guess'] = slop_list
+        # print(value_name+'_guess')
+        # print(self._raw_data[value_name+'_guess'][:10])
+        return slop_list
+
+    def simple_slop(self,value_name='y_data',key_name='x_data'):
+        df = self._raw_data.sort_values(key_name)
+        slop_list = []
+        for i in range(len(df)-1):
+            df[value_name].iloc[i+1]-df[value_name].iloc[i]
+
+    def integral_func(self, int_value_name, int_key_name, sort=True ):
+        print('integral_func')
+        # df = self._raw_data[self._raw_data['y_data_guess']>0].sort_values(int_key_name) if sort else self._raw_data
+        print(self._raw_data[self._raw_data['x_data_guess']>0])
+        df = self._raw_data[self._raw_data['x_data_guess'] > 0]
+        temp_value_sum = 0
+        last_key = df[int_key_name].iloc[0]
+        temp_key_num = 0
+        integral_value = 0
+        for i in range(len(df)):
+            row = df.iloc[i]
+            if last_key == row[int_key_name] :
+                temp_value_sum += row[int_value_name]
+                temp_key_num += 1
+            else:
+                integral_value += (row[int_key_name]-last_key) * temp_value_sum/temp_key_num
+                last_key = row[int_key_name]
+                temp_value_sum = row[int_value_name]
+                temp_key_num = 1
+        return integral_value
+

csv_file/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

db_test.py

@@ -56,7 +56,7 @@ def histogram_slope(x_data, y_data, slope =0, bins =0, plot =True):
 def run():
     conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="5432")
     cur = conn.cursor()
-    id = 14304
+    id = 166
 
     sql_str = f'select "raw_data" from "recording_data_metas" WHERE id = {id}'
     cur.execute(sql_str)
@@ -83,6 +83,9 @@ def run():
     x_data, y_data ,peek_list = histogram_slope(channel_data[0],channel_data[1],bins = 200)
     print([y_data[i] for i in peek_list])
     print(peek_list, channel_data)
+    channel_data.append(1.5e6)
+    channel_data.append(4.5e6)
     return channel_data, peek_list
+
     # print([y_data])
 

fig/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

main.py

@@ -3,23 +3,55 @@
 
 import time
 import sys
-import json
+import json,csv
 
 # from deviceManager import DeviceManager
 # from requests import Requests
 from mqtt import MqttThread
-from db_test import run 
+from test import VTmode_run, slop_run ,CVmode_run
 
 class Main():
-  def __init__(self, controller_id = 'b8:27:eb:18:f8:cc', mqtt_ip = '192.168.2.1') -> None:
+  def __init__(self, controller_id = 'dc:a6:32:0f:56:9d', mqtt_ip = '192.168.2.1') -> None:
       # setup mqtt thread
       self._mqttThread = MqttThread(self, controller_id, mqtt_ip, 1883, 'test')
       self._mqttThread.run()
   
   def get_analysis_data(self, input:str):
-      raw_data, peek_list = run()
-      self._mqttThread.publish(raw_data)
-     
+    start_time = time.time()
+    input_data = json.loads(input)['e']
+    # print(input_data['data_name'])
+    # print([i.split('-') for i in input_data['data_id']])
+    search_id = [int(i.split('-')[1]) for i in input_data['data_id']]
+    print(search_id)
+    with open('csv_file/output.csv', 'w', newline='') as csvfile:
+      writer = csv.writer(csvfile)
+      # print([[i[0],i[1]] for i in input_data.items()])
+      writer.writerows([[i[0],i[1]] for i in input_data.items()])
+      writer.writerow("")
+      if input_data['mode'] == 1:
+        first_flag = True
+        for i in search_id:
+          print(i)
+          raw_data, csv_data = VTmode_run(i, input_data['data_channel'], input_data['data']['persentage'])
+          fieldnames = list(csv_data.keys())
+          print(fieldnames)
+          dict_writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+          if first_flag:
+            # self._mqttThread.publish(raw_data)
+            dict_writer.writeheader()
+            first_flag = False
+          print('csv_data', csv_data)
+          
+          dict_writer.writerow(csv_data)
+      elif input_data['mode'] == 3:
+        raw_data = CVmode_run()
+      else:
+        for i in search_id:
+          raw_data = slop_run(i, input_data['data_channel'], input_data['data']['window'])
+      self._mqttThread.publish(raw_data) 
+      end_time = time.time()
+      print("執行時間：%f 秒" % (end_time - start_time))
+
 
 if __name__ == '__main__':
     # if len(sys.argv) < 3:
@@ -30,6 +62,7 @@ if __name__ == '__main__':
 
     try:
         while True:
+          time.sleep(1)
           pass
     except (KeyboardInterrupt, SystemExit):
         print("Received keyboard interrupt, quitting ...")

mqtt.py

@@ -34,7 +34,7 @@ class MqttThread():
       print('use controller ID', self.__controller_ID)
 
       self._mqtt_client = mqtt.Client(self.__controller_ID + '_' + self._client_id)
-      self._mqtt_client.connect(self._mqtt_url, self._mqtt_port)
+      self._mqtt_client.connect(self._mqtt_url, self._mqtt_port, keepalive=3600)
       self._mqtt_client.on_connect = self.on_connect
       self._mqtt_client.on_disconnect = self.on_disconnect
       self._mqtt_client.on_message = self.on_message
@@ -65,7 +65,7 @@ class MqttThread():
     QoS = 2 if wait_for_ack else 0
     topic = f'{self.__controller_ID}/data_analysis'
     message =  json.dumps(payload)
-    print('publish', topic, message)
+    print('publish', topic)#, message)
     message_info = self._mqtt_client.publish(topic, message, QoS)
 
     if wait_for_ack:

run-data-analysis.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cd /home/pi/data-analysis
+python3 -u main.py

test.py

@@ -0,0 +1,120 @@
+import psycopg2
+import ana_func
+import matplotlib.pyplot as plt
+import csv
+import pandas as pd
+
+def search_id_by_str(keyword):
+    conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="5432")
+    cur = conn.cursor()
+    sql_str = f"select id from recording_data_metas WHERE name ILIKE '%"+keyword+"%'"
+    cur.execute(sql_str)
+    data = cur.fetchall()
+    data = [i[0] for i in data]
+    return data
+
+def read_data(id,channel_list):
+    conn = psycopg2.connect(database="postgres", user="biopro", password="BioProControlBox", host="127.0.0.1", port="5432")
+    cur = conn.cursor()
+
+    sql_str = f'select "raw_data" from "recording_data_metas" WHERE id = {id}'
+    cur.execute(sql_str)
+    data = cur.fetchall()[0][0]
+    channel_data = []
+    for channel in channel_list:
+        i = str(channel)
+        channel_data_temp = []
+        for j in data[i]:
+            # print(j)
+            sql_str = f'select "data" from "{i}_recording_data_raws" WHERE id = {j}'
+            cur.execute(sql_str)
+            raw_data = cur.fetchall()[0][0]
+            raw_data = raw_data.replace('"***"',' ').split(" ")
+            channel_data_temp =channel_data_temp + [int(raw_data[k]) for k in range(len(raw_data)) if k%2]
+            # print(len(channel_data_temp))
+        channel_data.append(channel_data_temp)
+
+    return channel_data
+
+def read_data_csv(file_name=0, start_row=0):
+    df = pd.read_excel("read_file/2021-8-9-15-42-46-0_CV discussion.xlsx",skiprows = 63, usecols="B,D,F,H")
+    data = [list(df['Current[nA]']),list(df['Voltage[uV]']),list(df['Unnamed: 7'])]
+    # print(df)
+    return data
+# read_data_csv()
+
+def CVmode_run():
+    df = read_data_csv() 
+    print(df[0][:10])
+    data = [[df[0][i] for i in range(len(df[0])) if df[2][i] == 5], [df[1][i] for i in range(len(df[1])) if df[2][i] == 5]]
+    print(data[0][:10]) #y:Current[nA]/x:Voltage[uV]
+    CVmode = ana_func.dataAnalyticFunc(data[1], data[0])
+    cal_slop_data = CVmode.guess_func()
+    print(len(cal_slop_data))
+    integel = CVmode.integral_func('x_data','y_data')
+    print(integel)
+
+    # return [data[1],data[0]]
+    return [[i for i in range(len(data[0]))],cal_slop_data]
+    # return [[i for i in range(len(df[0]))],[i/50 for i in df[0]]]
+# CVmode()
+
+def slop_run(id, channel_list, win):
+    # id = 166
+    # win = 20
+    print(win)
+    channel_data = read_data(id, channel_list)
+    slop_mode = ana_func.dataAnalyticFunc(channel_data[0], channel_data[1])
+    cal_slop_data = slop_mode.cal_slop_array(window=win)
+    # return [[i for i in range(len(channel_data[0]))], channel_data[1]]
+    return [[i for i in range(len(channel_data[0]))], cal_slop_data]
+
+
+def VTmode_run(id, channel_list, perc):
+    # id = 166
+    percentage = perc/100
+    data_list = {}
+    print('channel_list', channel_list)
+    print(percentage)
+    channel_data = read_data(id, channel_list)
+    VT_mode = ana_func.dataAnalyticFunc(channel_data[0], channel_data[1])
+    VT_data_list ,_ = VT_mode.histogram_find_peak()
+    # print('peak list =', data_list['ground'])
+    data_list.update(VT_data_list)
+    peak_list = list(data_list.values())
+
+    trigger_line_down = (data_list['ground']-data_list['ceiling'])*percentage + data_list['ceiling']
+    trigger_line_up = (data_list['ground']-data_list['ceiling'])*(1-percentage) + data_list['ceiling']
+
+    trigger = {str(100-perc)+'% point':trigger_line_down, str(perc)+'% point':trigger_line_up}
+    data_list.update(trigger)
+    
+    peak_list.append(trigger_line_down)
+    peak_list.append(trigger_line_up)
+    triggerpoint = [VT_mode.trigger_point('y_data','x_data',trigger_line_down, order=-1,drop=-1), VT_mode.trigger_point('y_data','x_data',trigger_line_up)]
+    # slope_tan = math.atan2(triggerpoint[0][1]-triggerpoint[1][1],triggerpoint[0][0]-triggerpoint[1][0])
+    center_point = [(triggerpoint[0][0]+triggerpoint[1][0])/2,(triggerpoint[0][1]+triggerpoint[1][0])/2]
+    slope = (triggerpoint[0][1]-triggerpoint[1][1])/(triggerpoint[0][0]-triggerpoint[1][0])
+    center_point_dict = {'center point key': center_point[0], 'center point value':center_point[1], 'slope': slope}
+    print('trigger point',triggerpoint, slope, center_point)
+    data_list.update(center_point_dict)
+    
+    # Plot
+    # plt.plot(channel_data[0],channel_data[1],'o',markersize=2)
+    # print(peak_list)
+    # for i in peak_list[1:]:
+    #     plt.hlines(i,min(channel_data[0]),max(channel_data[0]),color='r')
+    # for j in triggerpoint:
+    #     plt.plot(j[0],j[1],'o',markersize=5)
+    # plt.savefig('fig/test'+ str(id) +'.png')
+    # plt.close()
+
+    resultList = [id , *channel_data, peak_list, triggerpoint]
+
+    channel_data.append(peak_list)
+    channel_data.append(triggerpoint)
+
+    return resultList, data_list
+
+# if __name__ == '__main__':
+#     VTmode_run()