[update] 256K -> 64K, 5s -> 2s, save id_db

python/biopro/api/controller.py

@@ -18,9 +18,10 @@ class ControllerAPI():
     @staticmethod
     def getByMac(mac_address):
         try:
-            ret = requests.get(API_URL + 'api/controller/get_by_mac/' + mac_address, headers= AuthAPI.get_key())
-            if ret.status_code == 200:
-                return ret.json()
+            if mac_address != None:
+                ret = requests.get(API_URL + 'api/controller/get_by_mac/' + mac_address, headers= AuthAPI.get_key())
+                if ret.status_code == 200:
+                    return ret.json()
         except (requests.exceptions.ConnectionError, json.decoder.JSONDecodeError) as e:
             print('get controller fail', e)
         return []

python/biopro/controller.py

@@ -1576,6 +1576,11 @@ class ControlAPI(metaclass=Router):
         should handle this exit code and re-run server process.
         """
         raise NotImplementedError()
+    
+    def run_project(self, project) -> bool:
+        print('create_project', project)
+        self._project_manager.create(project)
+        return False
 
 
 # noinspection PyAbstractClass

python/biopro/device/manager.py

@@ -562,6 +562,7 @@ class DeviceManager(MasterDevice, Synchronized):
 
     @logging_verbose
     def get_device(self, device: Union[int, DeviceInfo]) -> Optional[CompletedDevice]:
+        print('get_device', device)
         """get connected device
 
         :param device: device ID or response info
@@ -574,7 +575,7 @@ class DeviceManager(MasterDevice, Synchronized):
 
         elif isinstance(device, str):
             for slave in self._device:
-                if ':'.join('{:02x}'.format(b) for b in slave.mac_address) == device:
+                if ':'.join('{:02x}'.format(b) for b in slave.mac_address) == device.lower():
                     return slave
 
         elif isinstance(device, DeviceInfo):

python/biopro/devlib/library.py

@@ -1082,6 +1082,11 @@ class CompletedDevice(Device):
         :return: parameter V value
         """
         return self._configuration.get_parameter(name, False)
+    
+    def set_multi_parameters(self, parameter):
+        for (name, value) in parameter[0].items():
+            if name != 'target':
+                self.set_parameter(name, value)
 
     def set_parameter(self, name: str, value: Union[int, str]):
         """replace parameter value with *value*

python/biopro/project/action.py

@@ -0,0 +1,65 @@
+
+class Action():
+    def __init__(self, task, action_id, action):
+        self._task = task
+        self._id:str = action_id
+        self._type:str = action['type']
+        self._target:str = action['target']
+        self._condition_list:list[str] = action['condition']
+        self._duration = action.get('duration', None)
+        self._goto = action.get('goto', None)
+        self._cycle = action.get('cycle', None)
+        
+        self._instruction = None
+
+        self.update_instruction()
+    
+    @property
+    def type(self):
+        return self._type
+
+    def update_instruction(self):
+        self._instruction = {
+            "device_instruction": {
+                "header": "call_instruction",
+                "device": self._target,
+                "arguments": {
+                    "instruction": self._type,
+                }
+            },
+            "device_parameter": {
+                "header": "set_multi_parameters",
+                "device": self._target,
+                "arguments": {
+                    "parameter": self._task.get_parameter_set_by_device(self._target),
+                }
+            },
+            "device_recording_file_name": {
+                "header": "update_recording_file_name_info",
+                "device":  self._target,
+                "arguments": {
+                    "content": self._task.file_name,
+                }
+            },
+            "device_parent": {
+                "header": "update_parent_info",
+                "device":  self._target,
+                "arguments": {
+                    "content": self._task.parent
+                }
+            }
+        }
+
+    def get_condition_list(self):
+        return self._condition_list
+
+    def get_instruction(self, value):
+        return self._instruction[value]
+
+    def get_instruction_list(self):
+        instruction_set = {
+            'start': ['device_parent', 'device_recording_file_name', 'device_parameter', 'device_instruction'],
+            'stop': ['device_instruction'],
+        }
+
+        return map(self.get_instruction, instruction_set[self._type])

python/biopro/project/condition.py

@@ -0,0 +1,75 @@
+from datetime import datetime
+from time import time
+
+class Condition():
+    def __init__(self, id, condition):
+        print('condition', id, condition)
+        self._id = id
+        self._type = condition['type']
+        self._comparsion = condition['comparsion']
+        self._value = condition['value']
+    
+    @property
+    def id(self):
+        return self._id
+    @property
+    def type(self):
+        return self._type
+    @property
+    def comparsion(self):
+        return self._comparsion
+    @property
+    def value(self):
+        return self._value
+    
+    def set_comparsion(self, comparsion):
+        self._comparsion = comparsion
+
+    def set_type(self, type):
+        self._type = type    
+    
+    def set_value(self, value):
+        self._value = value
+
+    def compareWith(self, operator:str, x, y) -> bool:
+        cases = {
+            "equal": lambda a, b: a == b,
+            "bigger": lambda a, b: a > b,
+            "smaller": lambda a, b: a < b,
+        }
+
+        # print('x','y',x, y,type(x),type(y))
+        return cases[operator](x, y)
+
+    def match_or_not(self, **kwargs):
+       return self.method_mapping(self.type)()
+       
+    def absolute_time(self, **kwargs):
+        now = time()
+        time_condition = self.datetime_to_timestamp(self.str_to_datetime(self._value))
+
+        return self.compareWith(self.comparsion, int(now), int(time_condition))
+
+    def relative_time(self, **kwargs):
+        now = time()
+    
+    def device(self, **kwargs):
+        print('device')
+    
+    def task_status(self, **kwargs):
+        print('task_status')
+
+    def method_mapping(self, method_name):
+        methods = {
+        "relative_time": self.relative_time,
+        "absolute_time": self.absolute_time,
+        "device": self.device,
+        "task_status": self.task_status,
+        }
+        return methods[method_name]
+
+    def str_to_datetime(self, time_str):
+        return datetime.strptime(time_str,'%Y-%m-%dT%H:%M')
+
+    def datetime_to_timestamp(self, date):
+        return datetime.timestamp(date)

python/biopro/project/project.py

@@ -0,0 +1,78 @@
+from .task import Task
+from .task_manager import TaskManager
+import sys
+import json
+import threading
+import time
+from datetime import datetime
+from collections import deque
+from biopro.device.manager import DeviceManager
+from biopro.text import *
+
+class Project(threading.Thread):
+  def __init__(self, project, device_manager: DeviceManager, mqttThread = None, name="project"):
+    super(Project, self).__init__(name = name)
+    self._project = project
+    self._device_manager = device_manager
+    self._mqtt_thread = mqttThread
+    self._time_interval = 1
+    self._start_time = None
+
+    self._task_manager = TaskManager(self._project['taskList'])
+
+    self._start_time = None
+    
+  @property
+  def desc(self) -> str:
+    return self._project['desc']
+  
+  @property
+  def status(self) -> str:
+    return self._project['status']
+
+  @property
+  def device(self) -> list:
+    return self._project['deviceList']
+
+  @property
+  def task(self) -> list:
+    return self._task_manager.get()
+
+  @property
+  def get_current_task(self) -> str:
+    return self._task_manager.get_current_task()
+
+  def run(self):
+      self._start_time = time.time()
+
+      while True:
+        # current task
+        for task in self._task_manager.get_check_task_list():
+          # [<match_condition>, ...]
+          match_condition_list = task.check_condition('time')
+          # [[<match_action>, ...], ...]
+          match_action_list = task.get_match_action_list(match_condition_list)
+          
+          for action_list in match_action_list:
+            for action in action_list:
+              if action.type == 'start':
+                self._task_manager.set_current_task(task)
+              task.do_action(action)
+              
+          while len(task.instruction_list) > 0:
+            instruction = task.instruction_list.pop(0)
+            if instruction.get('device', None) != None:
+              device = self._device_manager.get_device(self.device[instruction['device']]['connectDevice']['device_address'])
+
+              if device is None:
+                  raise RuntimeError(DEVICE_NOT_FOUND, device)
+                
+              # print(instruction['header'], *instruction['arguments'].values())
+              getattr(device, instruction['header'])(*instruction['arguments'].values()) 
+
+        time.sleep(self._time_interval)
+  def stop(self):
+    return
+
+  def close(self):
+    return

python/biopro/project/project_manager.py

@@ -0,0 +1,33 @@
+from io import StringIO
+from json import loads as json_parse, dumps as _json_stringify
+from typing import Dict, Optional, Any
+import paho.mqtt.client as mqtt
+from .project import Project
+from biopro.device.manager import DeviceManager
+
+def json_stringify(o: Any) -> str:
+    return _json_stringify(o, separators=(',', ':'))
+
+class ProjectManager():
+    def __init__(self, device_manager: DeviceManager, mqtt_thread=None) :
+        self._project_list = []
+        self._mqtt_thread =  mqtt_thread
+        self._device_manager = device_manager
+    
+    def create(self, project):
+      new_project = Project(project, self._device_manager, self._mqtt_thread)
+      self._project_list.append(new_project)
+      return new_project
+    
+    def remove(self, index):
+      self._project_list[index].stop()
+      del self._project_list[index]
+
+    def get(self):
+      return self._project_list
+
+    def run(self, project):
+      project.start()
+
+    def stop(self, project):
+      project.stop()

python/biopro/project/task.py

@@ -0,0 +1,168 @@
+from .condition import Condition
+from .action import Action
+from time import time
+
+class Task:
+  def __init__(self, task):
+    self._task = task
+
+    self._condition_list = []
+    self._action_list = []
+
+    self._start_time = time()
+    self._idle_time = []
+    self._period = None
+    self._next_time = None
+    self._last_time = None
+
+    self._instruction_list = []
+
+    self.setup()
+  
+  def setup(self) -> None:
+    # add condition
+    for key, condition in self._task['condition'].items():
+      new_condition = Condition(key, condition)
+      self._condition_list.append(new_condition)
+    
+    # add action
+    for key, action in self._task['action'].items():
+      new_action = Action(self, key, action)
+      self._action_list.append(new_action)
+
+  @property
+  def id(self) -> str:
+    return self._task['id']
+  
+  @property
+  def file_name(self):
+    return self._task['name']
+
+  @property
+  def parent(self):
+    return self._task.get('parent', {"folder": [2]})
+
+  @property
+  def instruction_list(self):
+    return self._instruction_list
+
+  @property
+  def cycle(self) -> int:
+    return self._task['cycle']
+
+  @property
+  def order(self) -> int:
+    return self._task['order']
+
+  @property
+  def max_cycle(self) -> int:
+    return self._task['max_cycle']
+  
+  @property
+  def devices(self) -> dict:
+    return self._task['devices']
+
+  @property
+  def devices_id(self) -> list:
+    return [device_id for device_id in self.devices]
+  
+  @property
+  def events(self) -> list:
+    return self._task['events']
+
+  @property
+  def triggers(self) -> list:
+    return self._task['triggers']
+  
+  @property
+  def parameter_set(self) -> list:
+    return self._task['parameterSet']
+  
+  @property
+  def instructions(self) -> list:
+    return self._task['instructions']
+
+  @property
+  def conditions(self) -> list:
+    return self._task['conditions']
+  
+  @property
+  def actions(self) -> list:
+    return self._task['actions']
+
+  @property
+  def condition_list(self) -> list:
+    return self._condition_list
+
+  @property
+  def status(self) -> str:
+    return self._task['status']
+  
+  @property
+  def uuid(self) -> str:
+    return self._task['uuid']
+  
+  @property
+  def next(self) -> str:
+    return self._task['next']
+    
+  def get_parameter_set_by_device(self, device):
+     return [parameter_set for parameter_set in self.parameter_set.values() if device == parameter_set['target']]
+
+  def get_parameter(self, name_or_value, device):
+    name_list = []
+    value_list = []
+    for parameter in self.get_parameter_set_by_device(device):
+      for key, value in parameter.items():
+        if key != 'target':
+          name_list.append(key)
+          value_list.append(str(value))
+
+    return '|'.join(name_list) if name_or_value == 'name' else '|'.join(value_list)
+  
+  def run(self):
+    self._status = 'running'
+    self._start_time = time()
+
+  def pause(self):
+    self._status = 'pause'
+
+  def set_cycle(self, cycle):
+    self._cycle = cycle
+
+  def set_status(self, status):
+    self._status = status
+  
+  def start(self):
+    self.set_status('running')
+  
+  def stop(self):
+    if self.status == 'running':
+      self.do_action('stop')
+    self.set_status('idle')
+
+  def done(self):
+    self.set_status('done')
+  
+  def end(self):
+    self.set_status('end')
+
+  def get_match_action_list(self, match_condition_list):
+    return map(lambda condition: [x for x in self._action_list if condition.id in x.get_condition_list()], match_condition_list)
+
+  def do_action(self, action): 
+    # # match_action = [x for x in self._action_list if condition_id in x.get_condition_list()]
+
+    # for action in match_action:
+    for instruction in action.get_instruction_list():
+      self._instruction_list.append(instruction)
+
+  def status_mapping(self, status):
+    statuses = {
+      "start": self.task_start,
+      "done": self.task_done,
+    }
+    return statuses[status]
+
+  def check_condition(self, type):
+    return [condition for condition in self._condition_list if  condition.match_or_not()]

python/biopro/project/task_manager.py

@@ -0,0 +1,56 @@
+from json import loads as json_parse, dumps as _json_stringify
+from typing import Dict, Optional, Any
+from xml.dom.expatbuilder import parseString
+import paho.mqtt.client as mqtt
+from biopro.text import *
+from .task import Task
+
+_RUNTIME_COMPILE = False
+
+def json_stringify(o: Any) -> str:
+    return _json_stringify(o, separators=(',', ':'))
+
+class TaskManager():
+    def __init__(self, task_list):
+      self._current_task = None
+      self._task_list = []
+      self._need_to_check_task_list = []
+
+      for task in task_list:
+        task = Task(task) 
+        self._task_list.append(task)
+      
+      self.set_current_task(self._task_list[0])
+      self._current_task.run()
+    
+    def remove(self, index):
+      pass
+
+    def get_task(self, task_id):
+      return self._task_list[task_id]
+
+    def get_check_task_list(self):
+      return self._need_to_check_task_list
+
+    def get_current_task(self) -> Task:
+      return self._current_task
+
+    def get_next_task(self):
+      return  self._current_task.next
+
+    def set_current_task(self, task):
+      try:
+        if self._current_task != None and self._current_task.id != task.id:
+          self._current_task.stop()
+
+        self._current_task = task
+        self._need_to_check_task_list = []
+        # append current task
+        self._need_to_check_task_list.append(self._current_task)
+
+        # append next task
+        for task_uuid in self._current_task.next:
+          task = next((task for task in self._task_list if task.uuid == task_uuid), None)
+          self._need_to_check_task_list.append(task)
+      except Exception as e:
+        print(e)

python/biopro/recording/file.py

@@ -996,7 +996,7 @@ class RecordingFileWriter:
                  '_data_value_ch', '_close', '_data_mini_ch',
                  '_mini_scale_list', '_time_real_time', '_data_rl', '_data_db',
                  '_raw_save', '_mini_save', '_data_time_ch', '_data_value_ch_for_rl',
-                 '_data_time_ch_for_rl', '_device_id', '_send_data', '_data_mqtt_ch')
+                 '_data_time_ch_for_rl', '_device_id', '_send_data', '_data_mqtt_ch', '_id_db_save')
 
     def __init__(self, meta: RecordingMetaFile, device_id, database = None):
         self._meta = meta
@@ -1052,6 +1052,7 @@ class RecordingFileWriter:
         self._data_value_ch_for_rl = {}
         self._data_time_ch_for_rl = {}
         self._data_time_ch = {}
+        self._id_db_save = {}
 
         # mini data
         self._data_mini_ch = {}
@@ -1068,7 +1069,7 @@ class RecordingFileWriter:
 
         # splitting
         self.splitting_threshold_time = 30 * 60 * 1000  # one minute
-        self.splitting_threshold_size = 16 * 16 * 1024  #  16 * 16KB
+        self.splitting_threshold_size = 4 * 16 * 1024  #  16 * 16KB
 
         self._writer_batch_size = 8192 
 
@@ -1123,7 +1124,7 @@ class RecordingFileWriter:
                     self._meta._size += self._recording_file_dict[ch]._size
             # self._data_db.clear()
             if self._database is not None:
-                self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data']])
+                self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data'], self._id_db_save])
                 # self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data']])
             self._recording_file_dict.clear()
         for scale in self._mini_scale_list:
@@ -1365,7 +1366,7 @@ class RecordingFileWriter:
                     # mqtt_thread[ch].on_message(mes)
                     # self._data_rl[ch].clear()
                     # # self._time_real_time[ch] = self._time_now
-                    if self._time_now - self._time[ch] > 5000000:
+                    if self._time_now - self._time[ch] > 2000000:
                         if self._recording_file_dict[ch]._status:
                             _data = ' '.join(self._data_db[ch])
                             write_sz = self._recording_file_dict[ch].write(_data, self._channel_list)
@@ -1394,7 +1395,9 @@ class RecordingFileWriter:
 
                 if data_save is True:
                     if self._database is not None:
-                        self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data']])
+                        # print('raw_save', self._raw_save['id'])
+                        # print('id_save', self._id_db_save)
+                        self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data'], self._id_db_save])
                         self._meta.update_subfile(database = self._database)
                 if mini_save is True:
                     if self._database is not None:
@@ -1414,6 +1417,7 @@ class RecordingFileWriter:
         
         if len(self._recording_file_dict) > 0:
             for ch in self._recording_file_dict.keys():
+                self._id_db_save[ch] = self._recording_file_dict[ch]._id_db
                 self._recording_file_dict[ch].close(self._time_now)
                 self._meta._size += self._recording_file_dict[ch]._size
                 

python/biopro/server/controller.py

@@ -1335,3 +1335,6 @@ def to_device_info(content: Union[str, ADDRESS, JSON_OBJECT]) -> DeviceResponseI
     return DeviceResponseInfo(None,
                               serial_number=serial,
                               mac_address=address)
+
+def run_project(self, project) -> bool:
+    return False

python/biopro/server/database.py

@@ -15,11 +15,12 @@ from biopro.util.json import JSON_OBJECT
 from .socket import ServerThread
 
 import psycopg2
-# from psycopg2.pool import ThreadedConnectionPool
+from psycopg2.extras import execute_batch
 from multiprocessing import Process, Queue
 
 import biopro.server._identify
 
+from time import time
 
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
@@ -308,22 +309,16 @@ class DataBaseProcess(Process):
         return None
 
     def data_raw_create(self, _data_dict, _channel_list, device_id):
-
         for _channel in _channel_list:
             sql_str = 'INSERT INTO "public"."' + str(_channel) + '_recording_data_raws" ('
             sql_set = []
-            for item in _data_dict[_channel].keys():
-                sql_str = sql_str + str(item) + ', '
-                if isinstance(_data_dict[_channel][item], dict):
-                    sql_set.append(json_stringify(_data_dict[_channel][item]))
-                else:
-                    sql_set.append(str(_data_dict[_channel][item]))
 
-            sql_str = sql_str[0:-2] + ') VALUES ('
-            for item in _data_dict[_channel].keys():
-                sql_str = sql_str + '%s, '
+            key_list = _data_dict[_channel].keys()
+            key_str = ','.join(key_list)
+            sql_set = list(map(lambda x: str(x), _data_dict[_channel].values()))
+            values = ','.join(['%s'] * len(key_list))
 
-            sql_str = sql_str[0:-2] + ') RETURNING id'
+            sql_str = sql_str + key_str + ') VALUES (' + values + ') RETURNING id'
             sql_cursor = self._psql_conn.cursor()
 
             try:
@@ -360,15 +355,32 @@ class DataBaseProcess(Process):
         
         return None
 
-    def data_raw_recording(self, _id_dict, _channel_list, _data_dict):
-        with self._psql_conn as conn:
-            with conn.cursor() as sql_cursor:
-                for _channel in _channel_list:
-                    sql_str = 'UPDATE "public"."' + str(_channel) + '_recording_data_raws" SET data = concat(data, %s) where id = %s'
-                    sql_set = [_data_dict[_channel] + '"***"', _id_dict[_channel]]
-                    sql_cursor.execute(sql_str, sql_set)
-        
-
+    def data_raw_recording(self, _id_dict, _channel_list, _data_dict,  _id_db_save):
+        para_list = []
+        try:
+            ### execute way
+            # with self._psql_conn as conn:
+            #     with conn.cursor() as sql_cursor:
+            #         for _channel in _channel_list:
+            #             temp_id = _id_dict[_channel]
+            #             if temp_id == 0:
+            #                 temp_id = _id_db_save[_channel]
+            #             # print(_channel, temp_id)
+            #             sql_str = 'UPDATE "public"."%s_recording_data_raws" SET data = concat(data, %s) where id = %s'
+            #             sql_set = [_channel, _data_dict[_channel] + '"***"', temp_id]
+            #             sql_cursor.execute(sql_str, sql_set)
+            ### execute_batch
+            with self._psql_conn as conn:
+                with conn.cursor() as sql_cursor:
+                    sql_str = 'UPDATE "public"."%s_recording_data_raws" SET data = concat(data, %s) where id = %s'
+                    for _channel in _channel_list:
+                        temp_id = _id_dict[_channel]
+                        if temp_id == 0:
+                            temp_id = _id_db_save[_channel]
+                        para_list.append((_channel, _data_dict[_channel] + '"***"', temp_id))
+                    execute_batch(sql_cursor,sql_str, para_list)
+        except psycopg2.Error as e:
+            print('recording error', e)
         return None
 
     def data_mini_create(self, _data_dict, _channel_list, _scale, device_id):

python/biopro/server/main.py

@@ -30,6 +30,7 @@ from .api import ApiRequests
 from biopro.api.auth import AuthAPI
 from biopro.api.controller import ControllerAPI
 from biopro.api.device import DeviceAPI
+from biopro.project.project_manager import ProjectManager
 ### do not import Websocket mode
 ###from .websocket import WebsocketThread
 
@@ -154,7 +155,8 @@ class ControlServer(SocketServer, ControlServerAPI, CacheAPI, CacheNotifyHandler
         # _api request build
         self._api = ApiRequests()
 
-        
+        self._project_manager = ProjectManager(self.device_manager, self.mqtt_thread)
+
 
     # init method
 
@@ -193,6 +195,11 @@ class ControlServer(SocketServer, ControlServerAPI, CacheAPI, CacheNotifyHandler
     def device_manager(self) -> DeviceManager:
         return self._device_manager
 
+    @property
+    def project_manager(self) -> ProjectManager:
+        return self._project_manager
+
+
     # server setup
 
     def setup(self) -> None:
@@ -1319,6 +1326,12 @@ class ControlServer(SocketServer, ControlServerAPI, CacheAPI, CacheNotifyHandler
         # p = self.exp_manager.create_protocol(content, user_name)
         return
 
+    @logging_info
+    def run_project(self, project) -> bool:
+        project = self._project_manager.create(project)
+        self._project_manager.run(project)
+        return False
+
     # @logging_info
     # def exp_pro_options(self, protocol: int) -> List[ExpOption]:
     #     return self.exp_manager.protocol_options(protocol)