Update README.md

README.md

@@ -4,7 +4,7 @@ New goal on pi4 without kotlin and django, and with only one dataserver.
 # Installation
 ### Get Raspbrain into SD card
 
-###### Download Raspbian Buster Lite
+###### Download Raspbian Bullseye Lite
 
 >https://www.raspberrypi.org/downloads/raspberry-pi-os/
 
@@ -14,7 +14,7 @@ New goal on pi4 without kotlin and django, and with only one dataserver.
 lsblk
 
 # copy data to sd card
-sudo dd bs=4M if=yyyy-mm-dd-raspbian-buster.img of=/dev/sdX conv=fsync status=progress
+sudo dd bs=4M if=yyyy-mm-dd-raspios-bullseye-armhf-lite.img of=/dev/sdX conv=fsync status=progress
 
 # enable ssh on first boot
 add ssh.txt in boot folder
@@ -77,18 +77,22 @@ add ssh.txt in boot folder
 
 
 ###### Install dependencies
-    sudo apt-get install vim git wiringpi build-essential libi2c-dev i2c-tools python-dev libffi-dev python3-pip postgresql
+    sudo apt-get install vim git wiringpi build-essential libi2c-dev i2c-tools python-dev libffi-dev python3-pip postgresql npm
+
+###### wiringpi conflict
+need to install wiringpi:  https://github.com/WiringPi/WiringPi
+install instructions:  https://github.com/WiringPi/WiringPi/blob/master/INSTALL
 
 ### Setup project
-###### Clone project
+###### Clone data server project
 ```
 cd ~
-git clone https://gitlab.com/bioproscientific/bioprorelife.git
+git clone https://gitlab.com/wisetop/controller/wisetopdataserver.git
 ```
 
 ###### Install python dependencies
 ```
-cd ~/bioprorelife
+ cd ~/wisetopdataserver
 pip3 install -r requirements.txt
 ```
 ##### Packages version
@@ -112,14 +116,14 @@ psycopg2                      2.8.5
 
 ###### Build C extension
 ```
-cd ~/bioprorelife/python/biopro/ext/
+cd ~/wisetopdataserver/python/biopro/ext/
 rm -rf *.so build
 python3 setup.py build_ext --inplace
 ```
 
 ###### Check setup-uart file
 ```
-cd ~/bioprorelife
+cd ~/wisetopdataserver/
 sudo vim ./run-setup-uart.sh
 ```
 ##### Modify the code as following
@@ -149,44 +153,72 @@ sudo raspi-config
 reboot
 ```
 
+###### Set pi password
+```
+$cd ~
+$passwd
+Current password:raspberry
+New password:5k6zj454778225
+Retype new password:5k6zj454778225
+```
+
 ###### Auto run program when device start
 ```
 sudo vim /etc/rc.local
 ```
-add the code following as:
+###### add the code following as:
 ```
-sudo -H -u pi nohup /home/pi/bioprorelife/run-data-server.sh > /home/pi/.data_server.out &
+sudo -H -u pi nohup /home/pi/wisetopdataserver/run-data-server.sh > /home/pi/.data_server.out &
+#sudo -H -u pi nohup /home/pi/wisetopdataserver/run-data-server.sh > /home/pi/.wisetop_data_$now.out &
 ```
+##### /etc/rc.local
+```
+_IP=$(hostname -I) || true
+if [ "$_IP" ]; then
+  printf "My IP address is %s\n" "$_IP"
+fi
 
+now=$(date +'%F_%T')
+
+sudo nohup systemctl restart apache2
+sudo nohup systemctl restart mosquitto
+
+nohup node /home/pi/wisetopapiserver/dist/app.js > /home/pi/.api_server.out &
+
+sleep 5
+
+sudo -H -u pi nohup /home/pi/wisetopdataserver/run-data-server.sh > /home/pi/.data_server.out &
+#sudo -H -u pi nohup /home/pi/wisetopdataserver/run-data-server.sh > /home/pi/.wisetop_data_$now.out &
+
+exit 0
+```
 ### MQTT broker install
 
-1. Install mosquitto
-```
-sudo apt-get install mosquitto mosquitto-clients
-```
+###### 1.Install mosquitto
+$sudo apt-get install mosquitto mosquitto-clients
+
+
+###### 2.Enable Remote Access
+$sudo vim /etc/mosquitto/conf.d/default.conf
 
-2. Enable Remote Access
-```
-sudo vim /etc/mosquitto/conf.d/default.conf
-```
 add the code following as:
 ```
 listener 1883
 protocol mqtt
 listener 8083
 protocol websockets
+allow_anonymous true
 ```
-restart mqtt broker
-```
-sudo systemctl restart mosquitto
-```
+###### 3.restart mqtt broker
+$sudo systemctl restart mosquitto
 
-3. Robust MQTT
+###### 4.Robust MQTT
 create mosquitto_restart.sh
-```
-cd ~
-sudo vim mosquitto_restart.sh
-```
+
+$cd ~
+$sudo vim mosquitto_restart.sh
+
+
 add the code following as:
 ```
 if [ "`ps -aux | grep /usr/sbin/mosquitto | wc -l`" == "1" ]
@@ -200,25 +232,129 @@ fi
 echo "$SERVICE is currently running" >> /home/ubuntu/cron.log
 exit 0
 ```
-add mosquitto_restart.sh to root routing
-```
-sudo chmod +x mosquitto_restart.sh
-sudo  -i
-crontab -e
-```
+###### 5.add mosquitto_restart.sh to root routing
+$sudo chmod +x mosquitto_restart.sh
+$sudo  -i
+$crontab -e
+
 add the code following as:
 ```
 */5 * * * * /home/pi/mosquitto_restart.sh
 ```
-3. Auto run MQTT broker when when pi-start
-```
-sudo vim /etc/rc.local
-```
+###### 6.Auto run MQTT broker when when pi-start
+$sudo vim /etc/rc.local
+
 add the code following as:
 ```
 sudo nohup systemctl restart mosquitto
 ```
 
-### Setup routed wireless access point
+### Setting up a Routed Wireless Access Point
+###### 1.Install AP and Management Software
+$sudo apt install hostapd
+$sudo systemctl unmask hostapd
+$sudo systemctl enable hostapd
+$sudo apt install dnsmasq
+$sudo DEBIAN_FRONTEND=noninteractive apt install -y netfilter-persistent iptables-persistent
+
+###### 2.Set up the Network Router
+$sudo nano /etc/dhcpcd.conf
+```
+interface wlan0
+    static ip_address=192.168.2.1/24
+    nohook wpa_supplicant
+```
+$sudo nano /etc/sysctl.d/routed-ap.conf
+```
+# Enable IPv4 routing
+net.ipv4.ip_forward=1
+```
+
+$sudo iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
+$sudo netfilter-persistent save
+$sudo mv /etc/dnsmasq.conf /etc/dnsmasq.conf.orig
+
+
+$sudo nano /etc/dnsmasq.conf
+
+```
+interface=wlan0 # Listening interface
+dhcp-range=192.168.2.2,192.168.2.255,255.255.255.0,24h
+                # Pool of IP addresses served via DHCP
+domain=wlan     # Local wireless DNS domain
+address=/gw.wlan/192.168.2.1
+                # Alias for this router
+```
+###### 3.Ensure Wireless Operation
+$sudo rfkill unblock wlan
+
+###### 4.Configure the AP Software
+$sudo nano /etc/hostapd/hostapd.conf
+
+```
+country_code=TW
+interface=wlan0
+ssid=WTP_NONE
+hw_mode=g
+channel=7
+macaddr_acl=0
+auth_algs=1
+ignore_broadcast_ssid=0
+wpa=2
+wpa_passphrase=00000000
+wpa_key_mgmt=WPA-PSK
+wpa_pairwise=TKIP
+rsn_pairwise=CCMP
+
+# 802.11n support
+ieee80211n=1
+# make raspberry pi dongle fast
+wme_enabled=1
+# ht_capab: HT capabilities, enable 40MHz
+ht_capab=[HT40+][SHORT-GI-40]
+```
+###### 5.Running the new Wireless AP
+$sudo systemctl reboot
+
+
+### Set web site
+$sudo vim ./sites-available/000-default.conf
+```
+<VirtualHost *:8080> #80------>8080
+        # The ServerName directive sets the request scheme, hostname and port that
+        # the server uses to identify itself. This is used when creating
+        # redirection URLs. In the context of virtual hosts, the ServerName
+        # specifies what hostname must appear in the request's Host: header to
+        # match this virtual host. For the default virtual host (this file) this
+        # value is not decisive as it is used as a last resort host regardless.
+        # However, you must set it for any further virtual host explicitly.
+        #ServerName www.example.com
+
+        ServerAdmin webmaster@localhost
+        DocumentRoot /var/www/html
+
+        # Available loglevels: trace8, ..., trace1, debug, info, notice, warn,
+        # error, crit, alert, emerg.
+        # It is also possible to configure the loglevel for particular
+        # modules, e.g.
+        #LogLevel info ssl:warn
+
+        ErrorLog ${APACHE_LOG_DIR}/error.log
+        CustomLog ${APACHE_LOG_DIR}/access.log combined
+
+        # For most configuration files from conf-available/, which are
+        # enabled or disabled at a global level, it is possible to
+        # include a line for only one particular virtual host. For example the
+        # following line enables the CGI configuration for this host only
+        # after it has been globally disabled with "a2disconf".
+        #Include conf-available/serve-cgi-bin.conf
+</VirtualHost>
+
+<VirtualHost *:80>
+    ServerAdmin bioprovue@localhost
+    DocumentRoot /var/www/bioprovuebuild
+</VirtualHost>
+```
+
 ##### raspberry website document:
 > https://www.raspberrypi.org/documentation/configuration/wireless/access-point-routed.md

python/biopro/server/_identify.py

@@ -9,23 +9,9 @@ _jwt = None
 
 def controller_device_id() -> str:
     ### device ID
-    return 'b8:27:eb:52:ea:97'
-    ret = None
-    try:
-        with open('/etc/BPS/device_id') as f:
-            for line in f:
-                if line.startswith('#'):
-                    continue
-
-                ret = line.strip()
-                break
-    except FileNotFoundError:
-        pass
-
-    if ret is None or len(ret) == 0:
-        return str(uuid.uuid1())
-    else:
-        return ret
+    f = open("/sys/class/net/eth0/address", "r")
+    mac = f.read()[:-1]
+    return mac
 
 def get_api_jwt() -> str:
     global _jwt

python/biopro/server/data.py

@@ -126,9 +126,6 @@ class DataServer(SocketServer, DataAPI):
         self.database_process = DataBaseProcess('data_server', self.log_verbose, self._queue_db, self._queue_ds_dict, self._queue_db_error)
         self.database_process.start()
 
-        self._no_data_total_duration = 0
-        self._recv_memory_cost_time = 0
-
     @logging_info
     def setup(self):
         self._available_channel.clear()
@@ -672,13 +669,7 @@ class DataServer(SocketServer, DataAPI):
                 print("q size= ", self._configurations[device]._queue_rec.qsize())
 
             self._configurations[device].put_rec_queue(data)
-            self._no_data_total_duration = 0
             ret = True
-        else:
-            time_diff = time() - self._recv_memory_cost_time
-            self._no_data_total_duration += time_diff
-            print('data is None', device, datetime.now(), self._no_data_total_duration)
-        self._recv_memory_cost_time = time()
 
         return ret
 
@@ -918,7 +909,7 @@ class DeviceDataRuntime(DataRuntime):
 
     __slots__ = ( '_start_time', '_data_counter', '_last_data_counter', '_data_ch',
                     '_data_ch_time', '_mqtt_send_data_ch_level', '_timer', '_timer_of_not_send',
-                    '_get_rec_data', '_cycle_start_time', '_time_stamp_list', '_prev_data', '_prev_delta_time', '_prev_time_stamp', '_task_fininsh', '_check_spi_routine_timer')
+                    '_get_rec_data', '_cycle_start_time', '_time_stamp_list', '_prev_data', '_prev_delta_time', '_prev_time_stamp', '_task_fininsh')
 
     def __init__(self, server: DataServer, device: int, meta_file: RecordingMetaFile, data_format: AnyStr):
         super().__init__(server, device, meta_file, data_format)
@@ -931,7 +922,6 @@ class DeviceDataRuntime(DataRuntime):
         self._mqtt_send_data_ch_level = {}
 
         self._timer = None #int(time() * 10)
-        self._check_spi_routine_timer = None #int(time() * 10)
 
         self._timer_of_not_send = None
         self._get_rec_data = False
@@ -1157,7 +1147,6 @@ class SpiRuntimeThread(ServerThread):
         self._time_stamp = 0.0
         self._interval = 0.045
         self._timer = time()
-        self._check_spi_routine_timer = time()
         self._read_memory_board_idx = 0
 
     def setup(self) -> None:
@@ -1174,6 +1163,7 @@ class SpiRuntimeThread(ServerThread):
     def routine(self, event = None) -> None: # routine 50 ms
         server = self._server
         self._timer = time()
+
         sync = server.get_spi_obj()
 
         # select channel
@@ -1189,17 +1179,14 @@ class SpiRuntimeThread(ServerThread):
 
         run_time = time() - self._timer
 
+        real_run_time = run_time
+
         if run_time > 0.045:
             # print('time, recv_data_form_spi_routine_time', time(), run_time)
             run_time = 0.045
 
         if run_time >= 0.030:
-            print('time, recv_data_form_spi_routine_time >= 0.030', 'device:', c, datetime.now(), time(), run_time)
-        
-        spi_routine_run_time = time() - self._check_spi_routine_timer
-        if spi_routine_run_time > 1:
-            print('spi_routine_time > 1', 'device:', c, datetime.now(), spi_routine_run_time)
-        self._check_spi_routine_timer = time()
+            print('time, recv_data_form_spi_routine_time >= 0.030', 'device:', c, datetime.now(), time(), real_run_time)
 
         event.wait(self._interval - run_time)
         sync.set_pin_mem_sel(True)

python/biopro/server/database.py

@@ -160,7 +160,7 @@ class DataBaseProcess(Process):
             getattr(self, do)(*args)
         return None
 
-    @calculate_time(1)
+    # @calculate_time(1)
     def data_meta_write(self, _data: JSON_OBJECT, device_id, _uuid = None, _id = None):
         if _id > 0:
             sql_str = 'UPDATE "public"."recording_data_metas" SET '
@@ -376,7 +376,7 @@ class DataBaseProcess(Process):
             print('recording error', e)
         return None
     
-    @calculate_time(1)
+    # @calculate_time(1)
     def data_raw_recording_new(self, _id_dict, _channel_list, _data_dict, _end_time_dict, _size_dict):
         try:
             para_list = []
@@ -446,7 +446,7 @@ class DataBaseProcess(Process):
 
         return True
     
-    @calculate_time(1)
+    # @calculate_time(1)
     def data_mini_recording_new(self, _id_dict, _channel_list, _data_mean_dict):
         try:
             para_list = []
@@ -508,7 +508,7 @@ class DataBaseProcess(Process):
 
         return True
 
-    @calculate_time(1)
+    # @calculate_time()
     def data_meta_update_all_channel_raw_id(self, meta_id:str, raw_id_dict:dict):
         try:
             sql_str = "UPDATE recording_data_metas SET raw_data = jsonb_insert(raw_data, '{%s, -1}', '%s', TRUE) WHERE id = %s;"

python/biopro/server/recording.py

@@ -165,11 +165,11 @@ class RecordingProcess(Process):
         """
         q = None
         try:
-            now = time()
+            # now = time()
             q = self._queue_rec.get()
-            get_queue_time_cost = time() - now
-            if get_queue_time_cost > 1.1:
-                print('get queue cost time', get_queue_time_cost, 'q_size', self._queue_rec.qsize())
+            # get_queue_time_cost = time() - now
+            # if get_queue_time_cost > 2:
+            #     print('get queue cost time', get_queue_time_cost)
         except:
             return False
         finally: