peterlu14
1533 lines
53 KiB
Python
1533 lines
53 KiB
Python
from io import BufferedReader, BufferedWriter
|
|
from time import time
|
|
from typing import BinaryIO, Iterable
|
|
import re
|
|
import zlib
|
|
import base64
|
|
import gc
|
|
|
|
from biopro.util.json import JSON
|
|
from biopro.util.stack import print_exception
|
|
from biopro.util.logger import calculate_time
|
|
from .data import RecordingData
|
|
from .loader import *
|
|
|
|
from json import loads as json_parse, dumps as _json_stringify
|
|
import json
|
|
import sys
|
|
|
|
from statistics import mean
|
|
import random
|
|
# from numba import jit
|
|
|
|
from copy import copy
|
|
|
|
def json_stringify(o) -> str:
|
|
return _json_stringify(o, separators=(',', ':'))
|
|
|
|
# import requests
|
|
|
|
import biopro.server._identify
|
|
|
|
T = TypeVar('T')
|
|
|
|
|
|
class RecordingMetaFile(JsonSerialize):
|
|
"""binary file format
|
|
|
|
**RecordingMetaFile header format**
|
|
|
|
::
|
|
|
|
struct {
|
|
# file format information
|
|
u32_B magic_number
|
|
u32_B version_number
|
|
u64_B create_time
|
|
u8 file_uuid[16]
|
|
u8 data_format[4]
|
|
|
|
# recording device
|
|
struct DeviceResponseInfo {
|
|
u8 device_name_size
|
|
u8 device_name[device_name_size]
|
|
u8 device_address[6] # little endian
|
|
u8 device_serial[6]
|
|
}
|
|
|
|
# recording parameter
|
|
struct DeviceConfiguration
|
|
|
|
# channel information
|
|
u8 channel_size
|
|
u8 channel[channel_size]
|
|
|
|
# recording file list
|
|
struct {
|
|
u8 file_name_size
|
|
u8 file_name[file_name_size]
|
|
} file_list[...] = {0}
|
|
}
|
|
|
|
magic_number
|
|
MAGIC_NUMBER
|
|
|
|
version_number
|
|
|
|
======= ====== ===========================
|
|
version number note
|
|
======= ====== ===========================
|
|
0.1
|
|
0.2 0x0001 data file storage include
|
|
======= ====== ===========================
|
|
|
|
create_time
|
|
unix time stamp
|
|
|
|
file_uuid
|
|
uuid of the moder file.
|
|
|
|
time_stamp
|
|
the time stamp of the first entry.
|
|
If value equals to 0xFFFFFFFF and file_name_size equals to 0, it is the end of the list.
|
|
If value equals to 0xFFFFFFFF but file_name_size not equals to 0, it is un-determined value.
|
|
|
|
**json format**
|
|
|
|
::
|
|
|
|
RecordingMetaFile = {
|
|
"file_size": int
|
|
"time_duration": int
|
|
"channel": List[int]
|
|
"configuration": DeviceConfiguration
|
|
"device": DeviceResponseInfo?
|
|
}
|
|
|
|
"""
|
|
|
|
# noinspection SpellCheckingInspection
|
|
FILE_EXT = '.bprm'
|
|
TAR_FILE_EXT = '.bprm.tar.xz'
|
|
|
|
# noinspection SpellCheckingInspection
|
|
MAGIC_NUMBER = META_MAGIC_NUMBER
|
|
|
|
__slots__ = ('_filename', '_filepath', '_id_db', '_create_time', '_file_version', '_file_uuid', '_data_format', '_last_time',
|
|
'_device', '_configuration', '_channel_mask', '_database',
|
|
'_dirty', '_last_modify_time', '_file_size_cache',
|
|
'_recording_file_ch', '_recording_sub_file', '_size',
|
|
'_parameter', '_parent', '_recording_file_name', '_recording_sub_mini', '_recording_mini_ch',
|
|
'_raw_serial_number', '_mini_serial_number', '_device_id', '_project_id')
|
|
|
|
def __init__(self, filepath: Union[str, Path], data_format: bytes = None, id_db = 0, database = None):
|
|
if isinstance(filepath, str):
|
|
filepath = Path(filepath)
|
|
|
|
name = re.split('[./]+',str(filepath))
|
|
self._filename = name[len(name)-2]
|
|
self._filepath = filepath.absolute().with_suffix(self.FILE_EXT)
|
|
self._id_db = id_db
|
|
|
|
self._database = database
|
|
|
|
# file header
|
|
self._create_time = int(1000 * time())
|
|
self._last_time = int(0)
|
|
self._file_version = RecordingFileVersion.VERSION_00_02
|
|
self._file_uuid = uuid.uuid1()
|
|
|
|
if data_format is None:
|
|
self._data_format = RecordingFileDataFormat.REC_DATA
|
|
|
|
elif data_format not in RecordingFileDataFormat.DATA:
|
|
raise IndexError('unknown data format : ' + repr(data_format))
|
|
|
|
else:
|
|
self._data_format = data_format
|
|
|
|
# recording device
|
|
# self._device = DeviceResponseInfo.UNKNOWN
|
|
self._device = ''
|
|
self._parameter = ''
|
|
self._parent = ''
|
|
self._recording_file_name = ''
|
|
self._device_id = -1
|
|
self._project_id = None
|
|
|
|
# recording parameter
|
|
self._configuration = None
|
|
|
|
# channel mask
|
|
self._channel_mask = ChannelMask()
|
|
|
|
# recording file list
|
|
# self._recording_file = []
|
|
|
|
self._recording_file_ch = []
|
|
self._recording_mini_ch = []
|
|
self._raw_serial_number = {}
|
|
self._mini_serial_number = {}
|
|
|
|
self._recording_sub_file = {}
|
|
self._recording_sub_mini = {}
|
|
|
|
|
|
self._size = 0
|
|
|
|
self._dirty = True
|
|
self._last_modify_time = 0
|
|
|
|
self._file_size_cache = None
|
|
|
|
def __eq__(self, o: object) -> bool:
|
|
if self is o:
|
|
return True
|
|
|
|
if not isinstance(o, RecordingMetaFile):
|
|
return False
|
|
|
|
return self._filepath == o._filepath
|
|
|
|
@classmethod
|
|
def is_recording_meta_file(cls, path: Union[str, Path]) -> bool:
|
|
if isinstance(path, str):
|
|
path = Path(path)
|
|
|
|
if not path.is_file():
|
|
return False
|
|
|
|
with path.open('br') as f:
|
|
try:
|
|
return cls._is_recording_meta_file(f)
|
|
except BaseException as e:
|
|
raise IOError('file ' + str(path)) from e
|
|
return False
|
|
|
|
@classmethod
|
|
def _is_recording_meta_file(cls, io: BinaryIO) -> bool:
|
|
header = io.read(4)
|
|
|
|
return header == RecordingMetaFile.MAGIC_NUMBER
|
|
|
|
@staticmethod
|
|
def load(filepath: Union[str, Path], database = None) -> 'RecordingMetaFile':
|
|
"""load meta file
|
|
|
|
:param filepath: file path
|
|
:return: meta file
|
|
:raises RuntimeError: not a meta file
|
|
:raises FileNotFoundError:
|
|
"""
|
|
if not RecordingMetaFile.is_recording_meta_file(filepath):
|
|
raise RuntimeError('not a recording meta file : ' + str(filepath))
|
|
|
|
meta = RecordingMetaFile(filepath, database = database)
|
|
# meta._filepath = filepath
|
|
meta.reload()
|
|
|
|
return meta
|
|
|
|
def reopen(self, data_format: bytes = None) -> 'RecordingMetaFile':
|
|
"""reopen file.
|
|
|
|
:param data_format: change data format
|
|
:return: new recording meta file
|
|
"""
|
|
if data_format is None:
|
|
data_format = self._data_format
|
|
|
|
ret = RecordingMetaFile(self._filepath, data_format, id_db = self._id_db, database = self._database)
|
|
ret._device = self._device
|
|
ret._configuration = self._configuration
|
|
ret._dirty = True
|
|
|
|
return ret
|
|
|
|
def clone(self, path: Path) -> 'RecordingMetaFile':
|
|
ret = RecordingMetaFile(path, self._data_format, id_db = self._id_db, database = self._database)
|
|
ret._device = self._device
|
|
ret._configuration = self._configuration
|
|
ret._dirty = True
|
|
|
|
return ret
|
|
|
|
def reload(self):
|
|
"""reload meta file
|
|
|
|
:raises RuntimeError: not a meta file
|
|
:raises FileNotFoundError:
|
|
"""
|
|
|
|
loader = RecordingMetaFileLoader.load(self._filepath)
|
|
|
|
if loader.header != self.MAGIC_NUMBER:
|
|
raise IOError('not a BioPro Recording File : ' + str(self._filepath))
|
|
|
|
def _require(it: Optional[T], message: str) -> T:
|
|
if it is None:
|
|
raise RuntimeError('BioPro Recording File lost field : ' + message)
|
|
|
|
return it
|
|
|
|
self._file_version = _require(loader.version, 'file version')
|
|
self._create_time = _require(loader.create_time, 'file create time')
|
|
self._last_time = _require(loader.last_time, 'file last time')
|
|
self._file_uuid = _require(loader.file_uuid, 'file UUID')
|
|
self._data_format = _require(loader.data_format, 'data format')
|
|
|
|
self._device = _require(loader.device_response, 'device information')
|
|
self._configuration = _require(loader.configuration, 'configurations')
|
|
|
|
self._channel_mask = _require(loader.channel_mask, 'channel_mask')
|
|
|
|
|
|
self._dirty = True
|
|
self._last_modify_time = self._filepath.stat().st_mtime
|
|
return None
|
|
|
|
def write(self, database = None):
|
|
# print('******', self._filepath)
|
|
_name = str(self._filepath).split('/')[-1]
|
|
_path = str(self._filepath).replace('/', '^')
|
|
|
|
meta_data = {
|
|
'path': _path,
|
|
'configuration': self._configuration.as_json(),
|
|
'version': str(self._file_version),
|
|
'uuid': str(self._file_uuid),
|
|
'data_format': 'rec',
|
|
'device': self._device,
|
|
'project': self._project_id,
|
|
}
|
|
|
|
if len(self._parent) > 0:
|
|
meta_data['parent'] = self._parent
|
|
if len(self._recording_file_name) > 0:
|
|
meta_data['name'] = self._recording_file_name
|
|
if len(self._parameter) > 0:
|
|
meta_data['parameter_set'] = self._parameter
|
|
|
|
if database is not None :
|
|
self._database = database
|
|
self._database.put_queue(['data_meta_write', meta_data, self._device_id, str(self._file_uuid), self._id_db])
|
|
# database.put_queue(['data_meta_write', self, meta_data, _path, self._id_db])
|
|
|
|
with self._filepath.open('wb') as _f:
|
|
f = FileEncoder(_f)
|
|
|
|
# magic number
|
|
f.write(self.MAGIC_NUMBER)
|
|
|
|
# version number
|
|
f.u32(self._file_version)
|
|
|
|
# create time
|
|
f.u64(self._create_time)
|
|
|
|
# last time
|
|
if self._last_time is None:
|
|
self._last_time = 0
|
|
f.u64(int(self._last_time))
|
|
|
|
# file uuid
|
|
f.write(self._file_uuid.bytes)
|
|
|
|
# data format
|
|
f.write(self._data_format)
|
|
|
|
# recording device
|
|
device = self._device
|
|
if isinstance(device, str):
|
|
device = json.loads(device)
|
|
_device = {}
|
|
_device['device_name'] = device['device_name']
|
|
_device['device_address'] = device['device_address']
|
|
_device = json_stringify(_device)
|
|
f.encode_string(_device)
|
|
|
|
encoder = DeviceConfigurationEncoder()
|
|
encoder.write(self._configuration, f)
|
|
|
|
#channel information
|
|
f.u8(self._channel_mask.size)
|
|
f.u16a(self._channel_mask.channels())
|
|
|
|
f.write(b'\0')
|
|
|
|
self._dirty = False
|
|
self._last_modify_time = self._filepath.stat().st_mtime
|
|
return None
|
|
|
|
def update_subfile(self, database = None):
|
|
meta_data = {
|
|
'raw_data': self._recording_sub_file,
|
|
'mini_data': self._recording_sub_mini,
|
|
'channels': str(self._channel_mask.channels()),
|
|
'size': str(self._size),
|
|
'time_duration': str(self._last_time),
|
|
'uuid': str(self._file_uuid),
|
|
}
|
|
_path = str(self._filepath).replace('/', '^')
|
|
|
|
if database is not None :
|
|
self._database = database
|
|
self._database.put_queue(['data_meta_write', meta_data, self._device_id, str(self._file_uuid), self._id_db])
|
|
# database.put_queue(['data_meta_write', self, meta_data, _path, self._id_db])
|
|
return None
|
|
|
|
def update_subfile_time_size(self, database = None):
|
|
meta_data = {
|
|
'channels': str(self._channel_mask.channels()),
|
|
'size': str(self._size),
|
|
'time_duration': str(self._last_time),
|
|
'uuid': str(self._file_uuid),
|
|
}
|
|
_path = str(self._filepath).replace('/', '^')
|
|
|
|
if database is not None :
|
|
self._database = database
|
|
self._database.put_queue(['data_meta_write', meta_data, self._device_id, str(self._file_uuid), self._id_db])
|
|
# database.put_queue(['data_meta_write', self, meta_data, _path, self._id_db])
|
|
return None
|
|
|
|
def update_subfile_raw(self, database = None):
|
|
meta_data = {
|
|
'raw_data': self._recording_sub_file,
|
|
}
|
|
_path = str(self._filepath).replace('/', '^')
|
|
|
|
if database is not None :
|
|
self._database = database
|
|
self._database.put_queue(['data_meta_write', meta_data, self._device_id, str(self._file_uuid), self._id_db])
|
|
# database.put_queue(['data_meta_write', self, meta_data, _path, self._id_db])
|
|
return None
|
|
|
|
def update_subfile_mini(self, database = None):
|
|
meta_data = {
|
|
'mini_data': self._recording_sub_mini,
|
|
}
|
|
_path = str(self._filepath).replace('/', '^')
|
|
|
|
if database is not None :
|
|
self._database = database
|
|
self._database.put_queue(['data_meta_write', meta_data, self._device_id, str(self._file_uuid), self._id_db])
|
|
# database.put_queue(['data_meta_write', self, meta_data, _path, self._id_db])
|
|
return None
|
|
|
|
@property
|
|
def filename(self) -> str:
|
|
"""
|
|
|
|
:return: filename without file extension
|
|
"""
|
|
return self._filepath.with_suffix('').name
|
|
|
|
@property
|
|
def filepath(self) -> Path:
|
|
return self._filepath
|
|
|
|
@property
|
|
def create_time(self) -> Optional[int]:
|
|
return self._create_time
|
|
|
|
@property
|
|
def last_time(self) -> Optional[int]:
|
|
return self._last_time
|
|
|
|
@property
|
|
def file_version(self) -> int:
|
|
return self._file_version.value
|
|
|
|
@property
|
|
def file_uuid(self) -> uuid.UUID:
|
|
return self._file_uuid
|
|
|
|
@property
|
|
def file_size(self) -> int:
|
|
return self._size
|
|
|
|
@property
|
|
def data_format(self) -> bytes:
|
|
return self._data_format
|
|
|
|
@property
|
|
def device(self) -> str:
|
|
return self._device
|
|
|
|
@device.setter
|
|
def device(self, device):
|
|
self._device = device
|
|
self._dirty = True
|
|
|
|
@property
|
|
def parameter(self) -> str:
|
|
return self._parameter
|
|
|
|
@parameter.setter
|
|
def parameter(self, parameter):
|
|
self._parameter = parameter
|
|
return None
|
|
|
|
@property
|
|
def parent(self) -> str:
|
|
return self._parent
|
|
|
|
@parent.setter
|
|
def parent(self, parent):
|
|
self._parent = parent
|
|
|
|
@property
|
|
def recording_file_name(self) -> str:
|
|
return self._recording_file_name
|
|
|
|
@recording_file_name.setter
|
|
def recording_file_name(self, recording_file_name):
|
|
self._recording_file_name = recording_file_name
|
|
|
|
@property
|
|
def configuration(self) -> DeviceConfiguration:
|
|
return self._configuration
|
|
|
|
@configuration.setter
|
|
def configuration(self, conf: DeviceConfiguration):
|
|
self._configuration = conf
|
|
self._dirty = True
|
|
return None
|
|
|
|
@property
|
|
def project_id(self) -> int:
|
|
return self._project_id
|
|
|
|
@project_id.setter
|
|
def project_id(self, project: int):
|
|
self._project_id = project
|
|
return None
|
|
|
|
@property
|
|
def channels(self) -> List[int]:
|
|
return self._channel_mask.channels()
|
|
|
|
def update_channels(self, channel: List[int]):
|
|
if self._channel_mask.add_channel(channel):
|
|
self._dirty = True
|
|
|
|
@property
|
|
def is_dirty(self) -> bool:
|
|
return self._dirty
|
|
|
|
@property
|
|
def is_modified(self) -> bool:
|
|
return self._last_modify_time == 0 or not self._filepath.exists() or (self._filepath.is_file() and self._filepath.stat().st_mtime > self._last_modify_time)
|
|
|
|
@property
|
|
def last_modified(self) -> int:
|
|
return self._last_modify_time
|
|
|
|
def new_recording_file(self, _channel: int, _start_time, database = None) -> 'RecordingFile':
|
|
|
|
if _channel not in self._raw_serial_number:
|
|
self._raw_serial_number[_channel] = 0
|
|
else:
|
|
self._raw_serial_number[_channel] += 1
|
|
|
|
serial_number = self._raw_serial_number[_channel]
|
|
# if _channel is not None:
|
|
# for rf in self._recording_file_ch:
|
|
# if rf._channel == _channel:
|
|
# serial_number = serial_number + 1
|
|
# else:
|
|
# raise RuntimeError('new file no channel')
|
|
|
|
|
|
n = '{}_{}_{}{}'.format(self.filename, _channel, serial_number, RecordingFile.FILE_EXT)
|
|
|
|
p = self._filepath.with_name(n)
|
|
|
|
|
|
f = RecordingFile(p, database = database)
|
|
|
|
f._meta_file = self
|
|
f._file_uuid = self._file_uuid
|
|
f._serial_number = serial_number
|
|
f._data_format = self._data_format
|
|
f._channel = _channel
|
|
|
|
self._dirty = True
|
|
|
|
_name = str(n)
|
|
_path = str(p).replace('/', '^')
|
|
|
|
raw_data = {
|
|
'name': _name,
|
|
'path': _path,
|
|
'parent': self._id_db,
|
|
'size': '0',
|
|
'uuid': str(self._file_uuid),
|
|
'serial_number': str(serial_number),
|
|
'data_format': 'rec',
|
|
'channel': int(_channel),
|
|
'start_time': str(_start_time),
|
|
}
|
|
|
|
return f, raw_data
|
|
|
|
def new_recording_mini(self, _channel: int, _start_time, scale, database = None) -> 'RecordingMini':
|
|
|
|
if scale not in self._mini_serial_number:
|
|
self._mini_serial_number[scale] = {}
|
|
|
|
if _channel not in self._mini_serial_number[scale]:
|
|
self._mini_serial_number[scale][_channel] = 0
|
|
else:
|
|
self._mini_serial_number[scale][_channel] += 1
|
|
|
|
serial_number = self._mini_serial_number[scale][_channel]
|
|
|
|
|
|
n = '{}_{}_{}{}'.format(self.filename, _channel, serial_number, RecordingMini.FILE_EXT)
|
|
|
|
p = self._filepath.with_name(n)
|
|
|
|
|
|
f = RecordingMini(p, scale = scale, database = database)
|
|
|
|
f._meta_file = self
|
|
f._file_uuid = self._file_uuid
|
|
f._serial_number = serial_number
|
|
f._data_format = self._data_format
|
|
f._channel = _channel
|
|
|
|
self._dirty = True
|
|
|
|
_name = str(n)
|
|
_path = str(p).replace('/', '^')
|
|
|
|
mini_data = {
|
|
'name': _name,
|
|
'path': _path,
|
|
'parent': self._id_db,
|
|
'size': '0',
|
|
'uuid': str(self._file_uuid),
|
|
'serial_number': str(serial_number),
|
|
'scale': scale,
|
|
'channel': int(_channel),
|
|
'start_time': str(_start_time),
|
|
}
|
|
|
|
return f, mini_data
|
|
|
|
def clear_recording_file(self):
|
|
if len(self._channel_mask) > 0:
|
|
self._channel_mask.clear()
|
|
self._dirty = True
|
|
return None
|
|
|
|
def change_filepath(self, target_path: Path) -> Path:
|
|
meta_path = self.filepath
|
|
|
|
new_name = target_path.with_suffix('').name
|
|
|
|
t = target_path.with_name(new_name).with_suffix(RecordingMetaFile.FILE_EXT)
|
|
meta_path.rename(t)
|
|
|
|
return t
|
|
|
|
def remove_file(self):
|
|
meta_path = self.filepath
|
|
meta_path.unlink()
|
|
return None
|
|
|
|
@classmethod
|
|
def remove_meta_file(cls, filepath: Union[str, Path]):
|
|
if isinstance(filepath, str):
|
|
filepath = Path(filepath)
|
|
|
|
if not RecordingMetaFile.is_recording_meta_file(filepath):
|
|
raise RuntimeError('not a recording meta file : ' + str(filepath))
|
|
|
|
try:
|
|
meta = cls.load(filepath)
|
|
meta.remove_file()
|
|
except:
|
|
# force remove
|
|
parent = filepath.parent
|
|
name = filepath.name
|
|
|
|
filepath.unlink()
|
|
|
|
for p in list(parent.iterdir()):
|
|
if p.name.startswith(name) and p.name.endswith(RecordingFile.FILE_EXT):
|
|
p.unlink()
|
|
return None
|
|
|
|
def as_json(self) -> JSON:
|
|
ret = {
|
|
'file_name': str(self._filename),
|
|
'file_path': str(self._filepath),
|
|
'file_size': self.file_size,
|
|
'time_duration': self._last_time,
|
|
'create_time': self._create_time,
|
|
'channel': self.channels,
|
|
'configuration': self._configuration.as_json(),
|
|
'device': self._device,
|
|
}
|
|
|
|
return ret
|
|
|
|
|
|
class RecordingFile:
|
|
"""
|
|
|
|
**RecordingFile header format**
|
|
|
|
::
|
|
|
|
struct {
|
|
u32_B magic_number;
|
|
u32_B version_number
|
|
u64_B create_time;
|
|
|
|
# reference meta
|
|
u8 file_uuid[16];
|
|
u8 serial_number;
|
|
|
|
u8 data_format[4];
|
|
|
|
union {
|
|
# raw data section
|
|
# data_format = b'raw\0'
|
|
struct {
|
|
u8 raw_data[...]
|
|
}
|
|
|
|
# recording data section
|
|
# data_format = b'rec\0'
|
|
struct {
|
|
struct RecordingData[...];
|
|
}
|
|
}
|
|
}
|
|
|
|
create_time
|
|
unix time stamp
|
|
|
|
file_uuid
|
|
meta file uuid.
|
|
|
|
serial_number
|
|
the serial number of the split files. for mother file, the number equal to 0.
|
|
|
|
"""
|
|
|
|
__slots__ = ('_filepath', '_create_time', '_file_version', '_file_uuid', '_serial_number', '_data_format',
|
|
'_meta_file', '_file_object', '_channel', '_status', '_id_db', '_database', '_size')
|
|
|
|
FILE_EXT = '.bprf'
|
|
MAGIC_NUMBER = b'BPRF'
|
|
|
|
def __init__(self, filepath: Path, id_db = 0, database = None):
|
|
# file path
|
|
self._filepath = filepath
|
|
self._id_db = id_db
|
|
|
|
self._database = database
|
|
|
|
self._size = 0
|
|
|
|
# file header
|
|
self._create_time = int(1000 * time())
|
|
self._file_version = RecordingFileVersion.VERSION_00_02
|
|
self._file_uuid = None
|
|
self._serial_number = None
|
|
self._data_format = None
|
|
|
|
self._channel = None
|
|
|
|
# file status (open/close)
|
|
self._status = False
|
|
|
|
# meta file
|
|
self._meta_file = None
|
|
|
|
# cache file object
|
|
self._file_object = None
|
|
|
|
@classmethod
|
|
def is_recording_file(cls, path: Union[str, Path]) -> bool:
|
|
if isinstance(path, str):
|
|
path = Path(path)
|
|
|
|
if not path.is_file():
|
|
return False
|
|
|
|
with path.open('br') as f:
|
|
try:
|
|
return cls._is_recording_file(f)
|
|
except BaseException as e:
|
|
raise IOError('file ' + str(path)) from e
|
|
|
|
@classmethod
|
|
def _is_recording_file(cls, io: BinaryIO) -> bool:
|
|
header = io.read(4)
|
|
|
|
return header == RecordingFile.MAGIC_NUMBER
|
|
|
|
@staticmethod
|
|
def load(filepath: Union[str, Path]):
|
|
if isinstance(filepath, str):
|
|
filepath = Path(filepath)
|
|
|
|
if not RecordingFile.is_recording_file(filepath):
|
|
raise IOError('not a recording file : ' + str(filepath))
|
|
|
|
f = RecordingFile(filepath, database = self._database)
|
|
f._load()
|
|
|
|
return f
|
|
|
|
@property
|
|
def filepath(self) -> Path:
|
|
return self._filepath
|
|
|
|
@property
|
|
def filename(self) -> str:
|
|
"""
|
|
|
|
:return: filename with file extension
|
|
"""
|
|
return self._filepath.name
|
|
|
|
@property
|
|
def create_time(self) -> Optional[int]:
|
|
return self._create_time
|
|
|
|
@property
|
|
def file_version(self) -> Optional[int]:
|
|
return self._file_version
|
|
|
|
@property
|
|
def file_uuid(self) -> uuid.UUID:
|
|
return self._file_uuid
|
|
|
|
@property
|
|
def serial_number(self) -> int:
|
|
return self._serial_number
|
|
|
|
@property
|
|
def data_format(self) -> bytes:
|
|
return self._data_format
|
|
|
|
def open(self):
|
|
self._status = True
|
|
return None
|
|
|
|
def close(self, _end_time):
|
|
if self._database is not None:
|
|
_data = {
|
|
'end_time': str(_end_time),
|
|
'size': str(self._size)
|
|
}
|
|
|
|
if self._meta_file._id_db != 0:
|
|
_data['parent'] = self._meta_file._id_db
|
|
self._database.put_queue(['data_raw_update', self._id_db, self._channel, _data])
|
|
self._status = False
|
|
return None
|
|
|
|
def update_endtime(self, _end_time):
|
|
if self._database is not None:
|
|
_data = {
|
|
'end_time': str(_end_time),
|
|
'size': str(self._size)
|
|
}
|
|
self._database.put_queue(['data_raw_update', self._id_db, self._channel, _data])
|
|
return None
|
|
|
|
def flush(self):
|
|
if self._file_object is not None and self._file_object.writable():
|
|
self._file_object.flush()
|
|
|
|
def _ensure_read(self) -> IO:
|
|
return None
|
|
|
|
def _ensure_write(self) -> BufferedWriter:
|
|
return None
|
|
|
|
def _load(self):
|
|
"""load file header"""
|
|
return None
|
|
|
|
def _write(self, f: FileEncoder):
|
|
return None
|
|
|
|
def read(self, size: Optional[int] = None) -> bytes:
|
|
return None
|
|
|
|
# def write(self, content: Union[bytes, RecordingData]) -> int:
|
|
def write(self, content: str, channels: list) -> int:
|
|
if not isinstance(content, str):
|
|
raise RuntimeError('wrong data format : ' + repr(self._data_format))
|
|
|
|
self._meta_file.update_channels(channels)
|
|
sz = sys.getsizeof(content)
|
|
self._size += sz
|
|
return sz
|
|
|
|
def _write_recording_data(self, data: RecordingData):
|
|
# check channel of data in channel of meta
|
|
|
|
meta = self._meta_file
|
|
|
|
if meta is not None:
|
|
meta.update_channels(data.channels())
|
|
return None
|
|
|
|
def read_block(self) -> Optional[RecordingData]:
|
|
if self._data_format != RecordingFileDataFormat.REC_DATA:
|
|
raise RuntimeError('wrong data format : ' + repr(self._data_format))
|
|
|
|
return RecordingData.read_block(self._ensure_read())
|
|
|
|
def read_block_iter(self) -> Iterable[RecordingData]:
|
|
if self._data_format != RecordingFileDataFormat.REC_DATA:
|
|
raise RuntimeError('wrong data format : ' + repr(self._data_format))
|
|
|
|
auto_close = self._file_object is None
|
|
|
|
file = self._ensure_read()
|
|
|
|
try:
|
|
data = RecordingData.read_block(file)
|
|
|
|
while data is not None:
|
|
yield data
|
|
data = RecordingData.read_block(file)
|
|
|
|
finally:
|
|
if auto_close:
|
|
self.close()
|
|
return None
|
|
|
|
class RecordingMini:
|
|
|
|
__slots__ = ('_filepath', '_create_time', '_file_version', '_file_uuid', '_serial_number', '_data_format',
|
|
'_meta_file', '_file_object', '_channel', '_status', '_id_db', '_database', '_size',
|
|
'_scale')
|
|
|
|
FILE_EXT = '.bpri'
|
|
MAGIC_NUMBER = b'BPRI'
|
|
|
|
def __init__(self, filepath: Path, scale = 100, id_db = 0, database = None):
|
|
# file path
|
|
self._filepath = filepath
|
|
self._id_db = id_db
|
|
|
|
self._database = database
|
|
|
|
self._size = 0
|
|
|
|
# file header
|
|
self._create_time = int(1000 * time())
|
|
self._file_version = RecordingFileVersion.VERSION_00_02
|
|
self._file_uuid = None
|
|
self._serial_number = None
|
|
self._data_format = None
|
|
|
|
self._channel = None
|
|
|
|
# file status (open/close)
|
|
self._status = False
|
|
|
|
# meta file
|
|
self._meta_file = None
|
|
|
|
# cache file object
|
|
self._file_object = None
|
|
|
|
# scale
|
|
self._scale = scale
|
|
|
|
@property
|
|
def filepath(self) -> Path:
|
|
return self._filepath
|
|
|
|
@property
|
|
def size(self) -> int:
|
|
return self._size
|
|
|
|
@property
|
|
def filename(self) -> str:
|
|
"""
|
|
|
|
:return: filename with file extension
|
|
"""
|
|
return self._filepath.name
|
|
|
|
@property
|
|
def create_time(self) -> Optional[int]:
|
|
return self._create_time
|
|
|
|
@property
|
|
def file_version(self) -> Optional[int]:
|
|
return self._file_version
|
|
|
|
@property
|
|
def file_uuid(self) -> uuid.UUID:
|
|
return self._file_uuid
|
|
|
|
@property
|
|
def serial_number(self) -> int:
|
|
return self._serial_number
|
|
|
|
@property
|
|
def data_format(self) -> bytes:
|
|
return self._data_format
|
|
|
|
@property
|
|
def scale(self) -> int:
|
|
return self._scale
|
|
|
|
def open(self):
|
|
self._status = True
|
|
|
|
def close(self, _end_time):
|
|
if self._database is not None:
|
|
_data = {
|
|
'end_time': str(_end_time),
|
|
'size': str(self._size)
|
|
}
|
|
|
|
if self._meta_file._id_db != 0:
|
|
_data['parent'] = self._meta_file._id_db
|
|
self._database.put_queue(['data_mini_update', self._id_db, self._channel, _data])
|
|
# self._database.put_queue(['data_mini_update', self._id_db, self._channel, _data])
|
|
|
|
def _load(self):
|
|
"""load file header"""
|
|
return None
|
|
|
|
def read(self, size: Optional[int] = None) -> bytes:
|
|
return None
|
|
|
|
# def write(self, content: Union[bytes, RecordingData]) -> int:
|
|
def write(self, data_mean: list) -> int:
|
|
|
|
if len(data_mean) > 0:
|
|
self._size += sys.getsizeof(data_mean)
|
|
return 0
|
|
|
|
|
|
class RecordingFileWriter:
|
|
__slots__ = ('_meta', '_recording_file', '_recording_file_dict', '_channel_list', '_database',
|
|
'splitting_threshold_time', 'splitting_threshold_size', '_writer_batch_size',
|
|
'_splitting_size', '_time', '_time_now', '_recording_mini_dict',
|
|
'_data_value_ch', '_close', '_data_mini_ch',
|
|
'_mini_scale_list', '_time_real_time', '_data_rl', '_data_db',
|
|
'_raw_save', '_mini_save', '_data_time_ch', '_data_value_ch_for_rl',
|
|
'_data_time_ch_for_rl', '_device_id', '_send_data', '_data_mqtt_ch', '_id_db_save', '_raw_create_not_done',
|
|
'_mini_create_not_done')
|
|
|
|
def __init__(self, meta: RecordingMetaFile, device_id, database = None):
|
|
self._meta = meta
|
|
# self._recording_file = None
|
|
self._device_id = device_id
|
|
|
|
self._database = database
|
|
|
|
self._mini_scale_list = [10, 100, 1000]
|
|
|
|
self._recording_file_dict = {}
|
|
self._recording_mini_dict = {
|
|
'10': {},
|
|
'100': {},
|
|
'1000': {},
|
|
# '10000': {}
|
|
}
|
|
|
|
self._raw_save = {
|
|
'id': {},
|
|
'data': {},
|
|
'end_time': {},
|
|
'size': {}
|
|
}
|
|
self._mini_save = {
|
|
'10': {
|
|
'id': {},
|
|
'start_time': {},
|
|
'data_mean': {},
|
|
'data_random': {},
|
|
'data_bar': {}
|
|
},
|
|
'100': {
|
|
'id': {},
|
|
'start_time': {},
|
|
'data_mean': {},
|
|
'data_random': {},
|
|
'data_bar': {}
|
|
},
|
|
'1000': {
|
|
'id': {},
|
|
'start_time': {},
|
|
'data_mean': {},
|
|
'data_random': {},
|
|
'data_bar': {}
|
|
}
|
|
}
|
|
|
|
self._channel_list = []
|
|
|
|
# save data chache
|
|
self._data_rl = {}
|
|
self._data_db = {}
|
|
self._data_value_ch = {}
|
|
self._data_value_ch_for_rl = {}
|
|
self._data_time_ch_for_rl = {}
|
|
self._data_time_ch = {}
|
|
self._id_db_save = {}
|
|
|
|
# mini data
|
|
self._data_mini_ch = {}
|
|
|
|
# mqtt real time
|
|
self._data_mqtt_ch = {}
|
|
|
|
self._time = {}
|
|
self._time_now = None
|
|
self._time_real_time = {}
|
|
|
|
self._close = False
|
|
self._send_data = {}
|
|
|
|
# splitting
|
|
self.splitting_threshold_time = 30 * 60 * 1000 # one minute
|
|
self.splitting_threshold_size = 16 * 1024 # 16 * 16KB
|
|
|
|
self._writer_batch_size = 8192
|
|
|
|
self._splitting_size = None
|
|
|
|
self._raw_create_not_done = True
|
|
self._mini_create_not_done = True
|
|
|
|
|
|
@property
|
|
def meta_file(self) -> RecordingMetaFile:
|
|
return self._meta
|
|
|
|
@property
|
|
def channel_list(self):
|
|
return self._channel_list
|
|
|
|
def __enter__(self):
|
|
self.open()
|
|
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
self.close()
|
|
return None
|
|
|
|
def channels_update(self, channels):
|
|
channels.sort()
|
|
self._channel_list.extend(channels)
|
|
return self._channel_list
|
|
|
|
def open(self):
|
|
self._meta.clear_recording_file()
|
|
|
|
if self._meta.is_dirty:
|
|
self._meta.write(database = self._database)
|
|
|
|
self._splitting_size = 0
|
|
return None
|
|
|
|
def close(self, mqtt_thread):
|
|
self._close = True
|
|
if len(self._recording_file_dict) > 0:
|
|
for ch in self._data_db.keys():
|
|
if len(self._data_rl[ch]) > 0:
|
|
self._data_rl[ch].append(str(int(self._time_now)))
|
|
mes = ' '.join(self._data_rl[ch])
|
|
mqtt_thread[ch].on_message(mes)
|
|
self._data_rl[ch].clear()
|
|
self._send_data[ch] = False
|
|
|
|
if self._recording_file_dict[ch]._status:
|
|
_data = ' '.join(self._data_db[ch])
|
|
self._raw_save['data'][ch] = _data
|
|
self._raw_save['id'][ch] = self._recording_file_dict[ch]._id_db
|
|
self._recording_file_dict[ch].write(_data, self._channel_list)
|
|
self._recording_file_dict[ch].close(self._time_now)
|
|
self._meta._size += self._recording_file_dict[ch]._size
|
|
# self._data_db.clear()
|
|
if self._database is not None:
|
|
self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data'], self._id_db_save])
|
|
# self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data']])
|
|
self._recording_file_dict.clear()
|
|
for scale in self._mini_scale_list:
|
|
if len(self._recording_mini_dict[str(scale)]) > 0:
|
|
for ch in self._recording_mini_dict[str(scale)].keys():
|
|
if self._recording_mini_dict[str(scale)][ch]._status:
|
|
self._mini_save[str(scale)]['id'][ch] = self._recording_mini_dict[str(scale)][ch]._id_db
|
|
self._mini_save[str(scale)]['start_time'][ch] = str(self._data_mini_ch[ch][str(scale)]['start_time'])
|
|
self._mini_save[str(scale)]['data_mean'][ch] = self._data_mini_ch[ch][str(scale)]['mean'].copy()
|
|
# self._mini_save[str(scale)]['data_random'][ch] = self._data_mini_ch[ch][str(scale)]['random'].copy()
|
|
# self._mini_save[str(scale)]['data_bar'][ch] = self._data_mini_ch[ch][str(scale)]['bar'].copy()
|
|
# start_time = str(self._data_mini_ch[ch][str(scale)]['start_time'])
|
|
# data_mean = self._data_mini_ch[ch][str(scale)]['mean'].copy()
|
|
# data_random = self._data_mini_ch[ch][str(scale)]['random'].copy()
|
|
# print('channel:', ch, ', data: ', data_mean)
|
|
self._recording_mini_dict[str(scale)][ch].write(self._mini_save[str(scale)]['data_mean'][ch])
|
|
self._recording_mini_dict[str(scale)][ch].close(self._time_now)
|
|
self._data_mini_ch[ch][str(scale)]['mean'].clear()
|
|
# self._data_mini_ch[ch][str(scale)]['random'].clear()
|
|
# del self._data_mini_ch[ch][str(scale)]['mean']
|
|
# del self._data_mini_ch[ch][str(scale)]['random']
|
|
if self._database is not None:
|
|
self._database.put_queue(['data_mini_recording', self._mini_save[str(scale)]['id'], self._channel_list, self._mini_save[str(scale)]['start_time'], self._mini_save[str(scale)]['data_mean']])
|
|
# self._database.put_queue(['data_mini_recording', self._recording_mini_dict[str(scale)], self._mini_save[str(scale)]['id'], self._channel_list, self._mini_save[str(scale)]['start_time'], self._mini_save[str(scale)]['data_mean'], self._mini_save[str(scale)]['data_random']])
|
|
self._data_mini_ch.clear()
|
|
self._recording_mini_dict.clear()
|
|
# del self._data_mini_ch
|
|
# del self._recording_mini_dict
|
|
gc.collect()
|
|
|
|
self._meta._last_time = self._time_now
|
|
self._meta.update_subfile(database = self._database)
|
|
# self._meta.write(database = self._database)
|
|
print('close2')
|
|
return True
|
|
|
|
def flush(self):
|
|
if len(self._recording_file_dict) > 0:
|
|
for ch in self._channel_list:
|
|
if ch in self._recording_file_dict:
|
|
self._recording_file_dict[ch].flush()
|
|
return None
|
|
|
|
def get_bar(self, data_ch: list, time_ch: list):
|
|
ret = []
|
|
_first_time = -1
|
|
_last_time = -1
|
|
_max = max(data_ch)
|
|
_max_index = data_ch.index(_max)
|
|
_min = min(data_ch)
|
|
_min_index = data_ch.index(_min)
|
|
|
|
if time_ch is not None and len(time_ch) >= 2:
|
|
_first_time = time_ch[0]
|
|
_last_time = time_ch[-1]
|
|
|
|
if _max_index < _min_index:
|
|
if _first_time > 0:
|
|
ret.append(str(_first_time))
|
|
ret.append(str(_max))
|
|
if _last_time > 0:
|
|
ret.append(str(_last_time))
|
|
ret.append(str(_min))
|
|
else:
|
|
if _first_time > 0:
|
|
ret.append(str(_first_time))
|
|
ret.append(str(_min))
|
|
if _last_time > 0:
|
|
ret.append(str(_last_time))
|
|
ret.append(str(_max))
|
|
return ret
|
|
|
|
def get_data_iter(self, d, mqtt_thread):
|
|
# print('****d size', d.data_size)
|
|
for t, c, v in d.entry_iter():
|
|
if c in self._data_db:
|
|
### send real-time
|
|
if len(self._data_rl[c]) > 0 and self._send_data[c]:
|
|
self._data_rl[c].append(str(int(t)))
|
|
mes = ' '.join(self._data_rl[c])
|
|
# self._data_mqtt_ch[c] = self._data_rl[c].copy()
|
|
# self._data_mqtt_ch[c].append(str(int(t)))
|
|
mqtt_thread[c].on_message(mes)
|
|
self._data_rl[c].clear()
|
|
self._send_data[c] = False
|
|
### send real-time
|
|
|
|
sample_rate_rl = 1
|
|
# rec data
|
|
if c < 256:
|
|
sample_rate_rl = int(d.sample_rate / ( 500 * len(self.channel_list) )) #* len(self.channel_list)
|
|
# others
|
|
else:
|
|
sample_rate_rl = int(d.sample_rate_channel_upper_256 / ( 1000 * len(self.channel_list)))
|
|
if self._close is False:
|
|
self._data_value_ch[c].append(int(v))
|
|
self._data_value_ch_for_rl[c].append(int(v))
|
|
self._data_time_ch_for_rl[c].append(int(t))
|
|
# self._data_time_ch[c].append(int(t))
|
|
for scale in self._mini_scale_list:
|
|
if self._data_mini_ch[c][str(scale)]['start_time'] is None:
|
|
self._data_mini_ch[c][str(scale)]['start_time'] = str(int(t))
|
|
|
|
if len(self._data_value_ch_for_rl[c]) >= sample_rate_rl:
|
|
if len(self._data_value_ch_for_rl[c]) == 1:
|
|
if len(self._data_rl[c]) == 0:
|
|
self._data_rl[c].append(str(int(t)))
|
|
# self._data_rl[c].append(str(self._data_time_ch_for_rl[c][0]))
|
|
self._data_rl[c].append(str(self._data_value_ch_for_rl[c][0]))
|
|
else:
|
|
if len(self._data_rl[c]) == 0:
|
|
self._data_rl[c].append(str(int(t)))
|
|
# self._data_rl[c].append(str(self._data_time_ch_for_rl[c][0]))
|
|
# self._data_rl[c].append(str(self._data_value_ch_for_rl[c][0]))
|
|
_max = max(self._data_value_ch_for_rl[c])
|
|
_max_index = self._data_value_ch_for_rl[c].index(_max)
|
|
_min = min(self._data_value_ch_for_rl[c])
|
|
_min_index = self._data_value_ch_for_rl[c].index(_min)
|
|
# _mean = mean(self._data_value_ch_for_rl[c])
|
|
|
|
# _first_time = self._data_time_ch_for_rl[c][0]
|
|
# _last_time = self._data_time_ch_for_rl[c][-1]
|
|
|
|
if _max_index < _min_index:
|
|
# self._data_rl[c].append(str(_first_time))
|
|
self._data_rl[c].append(str(_max))
|
|
# self._data_rl[c].append(str(_last_time))
|
|
self._data_rl[c].append(str(_min))
|
|
else:
|
|
# self._data_rl[c].append(str(_first_time))
|
|
self._data_rl[c].append(str(_min))
|
|
# self._data_rl[c].append(str(_last_time))
|
|
self._data_rl[c].append(str(_max))
|
|
|
|
self._data_value_ch_for_rl[c].clear()
|
|
self._data_time_ch_for_rl[c].clear()
|
|
|
|
# mini picture
|
|
if len(self._data_value_ch[c]) >= 10:
|
|
self._data_mini_ch[c]['10']['mean'].append( int(mean(self._data_value_ch[c][0:9])) )
|
|
# self._data_mini_ch[c]['10']['random'].append( str(self._data_value_ch[c][random.randint(0,9)]) )
|
|
# _bar = self.get_bar(self._data_value_ch[c], None)
|
|
# self._data_mini_ch[c]['10']['bar'].extend(_bar)
|
|
self._data_value_ch[c].clear()
|
|
if int(len(self._data_mini_ch[c]['10']['mean']) / 10) - self._data_mini_ch[c]['10']['dec'] > 0:
|
|
self._data_mini_ch[c]['100']['mean'].append( int(mean(self._data_mini_ch[c]['10']['mean'][-10:])) )
|
|
# self._data_mini_ch[c]['100']['random'].append( str(self._data_mini_ch[c]['10']['random'][random.randint(-10,-1)]) )
|
|
# _bar = self.get_bar(self._data_mini_ch[c]['10']['bar'], None)
|
|
# self._data_mini_ch[c]['100']['bar'].extend(_bar)
|
|
self._data_mini_ch[c]['10']['dec'] = int(len(self._data_mini_ch[c]['10']['mean']) / 10)
|
|
if int(len(self._data_mini_ch[c]['100']['mean']) / 10) - self._data_mini_ch[c]['100']['dec'] > 0:
|
|
self._data_mini_ch[c]['1000']['mean'].append( int(mean(self._data_mini_ch[c]['100']['mean'][-10:])) )
|
|
# self._data_mini_ch[c]['1000']['random'].append( str(self._data_mini_ch[c]['100']['random'][random.randint(-10,-1)]) )
|
|
# _bar = self.get_bar(self._data_mini_ch[c]['100']['bar'], None)
|
|
# self._data_mini_ch[c]['1000']['bar'].extend(_bar)
|
|
self._data_mini_ch[c]['100']['dec'] = int(len(self._data_mini_ch[c]['100']['mean']) / 10)
|
|
# if int(len(self._data_mini_ch[c]['1000']['mean']) / 10) - self._data_mini_ch[c]['1000']['dec'] > 0:
|
|
# self._data_mini_ch[c]['10000']['mean'].append( int(mean(self._data_mini_ch[c]['1000']['mean'][-10:])) )
|
|
# self._data_mini_ch[c]['10000']['random'].append( str(self._data_mini_ch[c]['1000']['random'][random.randint(-10,-1)]) )
|
|
# self._data_mini_ch[c]['1000']['dec'] = int(len(self._data_mini_ch[c]['1000']['mean']) / 10)
|
|
# add normal data
|
|
self._data_db[c].append(str(int(t)))
|
|
self._data_db[c].append(str(v))
|
|
self._time_now = int(t)
|
|
return
|
|
|
|
# @calculate_time(1)
|
|
def write(self, data: Union[bytes, RecordingData, List[bytes], List[RecordingData]], mqtt_thread) -> int:
|
|
# check size
|
|
ths = self.splitting_threshold_size
|
|
tht = self.splitting_threshold_time
|
|
|
|
if self._close is False:
|
|
if len(self._recording_file_dict) == 0:
|
|
self._switch_recording_file()
|
|
elif ths is not None and self._splitting_size > ths * len(self._channel_list):
|
|
self._switch_recording_file()
|
|
|
|
for scale in self._mini_scale_list:
|
|
if len(self._recording_mini_dict[str(scale)]) == 0:
|
|
self._switch_recording_mini(scale = scale)
|
|
elif ths is not None and self._recording_mini_dict[str(scale)][list(self._recording_mini_dict[str(scale)].keys())[0]].size > ths:
|
|
self._switch_recording_mini(scale = scale)
|
|
|
|
# write data
|
|
|
|
if isinstance(data, list):
|
|
for d in data:
|
|
if self._close is False:
|
|
if ths is not None and self._splitting_size > ths * len(self._channel_list):
|
|
self._switch_recording_file()
|
|
|
|
for scale in self._mini_scale_list:
|
|
if ths is not None and self._recording_mini_dict[str(scale)][list(self._recording_mini_dict[str(scale)].keys())[0]].size > ths:
|
|
self._switch_recording_mini(scale = scale)
|
|
|
|
sz = 0
|
|
|
|
# init
|
|
if len(self._data_value_ch) == 0:
|
|
for ch in self._channel_list:
|
|
self._data_value_ch[ch] = []
|
|
self._data_value_ch_for_rl[ch] = []
|
|
self._data_time_ch_for_rl[ch] = []
|
|
self._data_time_ch[ch] = []
|
|
self._send_data[ch] = False
|
|
self._data_mqtt_ch[ch] = []
|
|
self._data_rl[ch] = []
|
|
self._data_db[ch] = []
|
|
self._time[ch] = 0
|
|
self._time_real_time[ch] = 0
|
|
if len(self._data_mini_ch) == 0:
|
|
for ch in self._channel_list:
|
|
self._data_mini_ch[ch] = {}
|
|
for scale in self._mini_scale_list:
|
|
self._data_mini_ch[ch][str(scale)] = {
|
|
'start_time': None,
|
|
'mean': [],
|
|
'random': [],
|
|
'bar': [],
|
|
'dec': 0,
|
|
}
|
|
|
|
self.get_data_iter(d, mqtt_thread)
|
|
|
|
if len(self._recording_file_dict) > 0:
|
|
for ch in self._data_db.keys():
|
|
if self._time_now - self._time_real_time[ch] > 1000000:
|
|
self._send_data[ch] = True
|
|
self._time_real_time[ch] = self._time_now
|
|
|
|
for ch in self._recording_file_dict:
|
|
if self._recording_file_dict[ch]._id_db == 0:
|
|
return None
|
|
|
|
for ch in self._data_db.keys():
|
|
for scale in self._mini_scale_list:
|
|
if self._recording_mini_dict[str(scale)][ch]._id_db == 0:
|
|
return None
|
|
|
|
if self._raw_create_not_done :
|
|
self._raw_create_not_done = False
|
|
self._meta.update_subfile_raw(database = self._database)
|
|
|
|
if self._mini_create_not_done :
|
|
self._mini_create_not_done = False
|
|
self._meta.update_subfile_mini(database = self._database)
|
|
|
|
data_save = False
|
|
mini_save = False
|
|
if len(self._recording_file_dict) > 0:
|
|
for ch in self._data_db.keys():
|
|
if self._time_now - self._time[ch] > 5000000:
|
|
if self._recording_file_dict[ch]._status:
|
|
_data = ' '.join(self._data_db[ch])
|
|
write_sz = self._recording_file_dict[ch].write(_data, self._channel_list)
|
|
self._raw_save['data'][ch] = _data
|
|
self._raw_save['id'][ch] = self._recording_file_dict[ch]._id_db
|
|
self._raw_save['end_time'][ch] = self._time_now
|
|
self._raw_save['size'][ch] = self._recording_file_dict[ch]._size
|
|
self._data_db[ch].clear()
|
|
self._time[ch] = self._time_now
|
|
self._meta._last_time = self._time_now
|
|
self._splitting_size += write_sz
|
|
data_save = True
|
|
if len(self._data_mini_ch[ch]['1000']['mean']) >= 10:
|
|
mini_save = True
|
|
for scale in self._mini_scale_list:
|
|
str_mean = [str(int) for int in self._data_mini_ch[ch][str(scale)]['mean']]
|
|
data_mean = str(self._data_mini_ch[ch][str(scale)]['start_time']) + ' ' + ' '.join(str_mean) + '"***"'
|
|
self._mini_save[str(scale)]['id'][ch] = self._recording_mini_dict[str(scale)][ch]._id_db
|
|
self._mini_save[str(scale)]['start_time'][ch] = str(self._data_mini_ch[ch][str(scale)]['start_time'])
|
|
self._mini_save[str(scale)]['data_mean'][ch] = data_mean
|
|
# self._mini_save[str(scale)]['data_random'][ch] = self._data_mini_ch[ch][str(scale)]['random'].copy()
|
|
# self._mini_save[str(scale)]['data_bar'][ch] = self._data_mini_ch[ch][str(scale)]['bar'].copy()
|
|
self._recording_mini_dict[str(scale)][ch].write(self._mini_save[str(scale)]['data_mean'][ch])
|
|
self._data_mini_ch[ch][str(scale)]['mean'].clear()
|
|
# self._data_mini_ch[ch][str(scale)]['random'].clear()
|
|
self._data_mini_ch[ch][str(scale)]['dec'] = 0
|
|
self._data_mini_ch[ch][str(scale)]['start_time'] = None
|
|
|
|
if data_save is True:
|
|
if self._database is not None:
|
|
recording_input = ['data_raw_recording_new', copy(self._raw_save['id']), copy(self._channel_list), copy(self._raw_save['data']), copy(self._raw_save['end_time']), copy(self._raw_save['size'])]
|
|
self._database.put_queue(recording_input)
|
|
self._meta.update_subfile_time_size(database = self._database)
|
|
if mini_save is True:
|
|
if self._database is not None:
|
|
for scale in self._mini_scale_list:
|
|
self._database.put_queue(['data_mini_recording_new', copy(self._mini_save[str(scale)]['id']), self._channel_list, copy(self._mini_save[str(scale)]['data_mean'])])
|
|
self._meta.update_subfile_time_size(database = self._database)
|
|
del data
|
|
|
|
return None
|
|
|
|
|
|
def _switch_recording_file(self):
|
|
self._raw_create_not_done = True
|
|
|
|
if len(self._recording_file_dict) > 0:
|
|
for ch in self._recording_file_dict.keys():
|
|
self._id_db_save[ch] = self._recording_file_dict[ch]._id_db
|
|
self._recording_file_dict[ch].close(self._time_now)
|
|
self._meta._size += self._recording_file_dict[ch]._size
|
|
|
|
start_time = 0
|
|
if self._time_now is not None:
|
|
start_time = self._time_now
|
|
|
|
raw_data_dict = {}
|
|
for ch in self._channel_list:
|
|
self._recording_file_dict[ch], raw_data_tmp = self._meta.new_recording_file(ch, start_time, database = self._database)
|
|
raw_data_dict[ch] = raw_data_tmp
|
|
self._recording_file_dict[ch].open()
|
|
|
|
if self._database is not None:
|
|
self._database.put_queue(['data_raw_create', raw_data_dict, self._channel_list, self._device_id])
|
|
|
|
self._meta._last_time = self._time_now
|
|
# self._meta.update_subfile(database = self._database)
|
|
|
|
self._splitting_size = 0
|
|
|
|
return None
|
|
|
|
def _switch_recording_mini(self, scale):
|
|
self._mini_create_not_done = True
|
|
|
|
if len(self._recording_mini_dict[str(scale)]) > 0:
|
|
for ch in self._recording_mini_dict[str(scale)].keys():
|
|
self._recording_mini_dict[str(scale)][ch].close(self._time_now)
|
|
|
|
start_time = 0
|
|
if self._time_now is not None:
|
|
start_time = self._time_now
|
|
|
|
mini_data_dict = {}
|
|
for ch in self._channel_list:
|
|
self._recording_mini_dict[str(scale)][ch], mini_data_tmp = self._meta.new_recording_mini(ch, start_time, scale = scale, database = self._database)
|
|
mini_data_dict[ch] = mini_data_tmp
|
|
self._recording_mini_dict[str(scale)][ch].open()
|
|
|
|
if self._database is not None:
|
|
self._database.put_queue(['data_mini_create', mini_data_dict, self._channel_list, scale, self._device_id])
|
|
|
|
self._meta._last_time = self._time_now
|
|
# self._meta.update_subfile(database = self._database)
|
|
return None
|
|
|
|
def update_meta_id(self, _id):
|
|
self._meta._id_db = _id
|
|
return None
|
|
|
|
def update_raw_dict_id(self, _channel, _id):
|
|
self._recording_file_dict[int(_channel)]._id_db = _id
|
|
if int(_channel) not in self._meta._recording_sub_file:
|
|
self._meta._recording_sub_file[int(_channel)] = []
|
|
self._meta._recording_sub_file[int(_channel)].append(_id)
|
|
return None
|
|
|
|
def update_mini_dict_id(self, _scale, _channel, _id):
|
|
self._recording_mini_dict[str(_scale)][int(_channel)]._id_db = _id
|
|
if int(_channel) not in self._meta._recording_sub_mini:
|
|
self._meta._recording_sub_mini[int(_channel)] = {}
|
|
if int(_scale) not in self._meta._recording_sub_mini[int(_channel)]:
|
|
self._meta._recording_sub_mini[int(_channel)][int(_scale)] = []
|
|
self._meta._recording_sub_mini[int(_channel)][int(_scale)].append(_id)
|
|
return None |