controller-wisetopdataserver/python/biopro/recording/file.py

from io import BufferedReader, BufferedWriter
from time import time
from typing import BinaryIO, Iterable
import re
import zlib
import base64
import gc

from biopro.util.json import JSON
from biopro.util.stack import print_exception
from biopro.util.logger import calculate_time
from .data import RecordingData
from .loader import *

from json import loads as json_parse, dumps as _json_stringify
import json
import sys

from statistics import mean
import random
# from numba import jit

from copy import copy

def json_stringify(o) -> str:
    return _json_stringify(o, separators=(',', ':'))

# import requests

import biopro.server._identify

T = TypeVar('T')


class RecordingMetaFile(JsonSerialize):
    """binary file format

    **RecordingMetaFile header format**

    ::

        struct {
            # file format information
            u32_B   magic_number
            u32_B   version_number
            u64_B   create_time
            u8      file_uuid[16]
            u8      data_format[4]

            # recording device
            struct DeviceResponseInfo {
                u8      device_name_size
                u8      device_name[device_name_size]
                u8      device_address[6]   # little endian
                u8      device_serial[6]
            }

            # recording parameter
            struct  DeviceConfiguration

            # channel information
            u8      channel_size
            u8      channel[channel_size]

            # recording file list
            struct {
                u8      file_name_size
                u8      file_name[file_name_size]
            } file_list[...] = {0}
        }

    magic_number
        MAGIC_NUMBER

    version_number

        ======= ====== ===========================
        version number note
        ======= ====== ===========================
        0.1
        0.2     0x0001 data file storage include
        ======= ====== ===========================

    create_time
        unix time stamp

    file_uuid
        uuid of the moder file.

    time_stamp
        the time stamp of the first entry.
        If value equals to 0xFFFFFFFF and file_name_size equals to 0, it is the end of the list.
        If value equals to 0xFFFFFFFF but file_name_size not equals to 0, it is un-determined value.

    **json format**

    ::

        RecordingMetaFile = {
            "file_size": int
            "time_duration": int
            "channel": List[int]
            "configuration": DeviceConfiguration
            "device": DeviceResponseInfo?
        }

    """

    # noinspection SpellCheckingInspection
    FILE_EXT = '.bprm'
    TAR_FILE_EXT = '.bprm.tar.xz'

    # noinspection SpellCheckingInspection
    MAGIC_NUMBER = META_MAGIC_NUMBER

    __slots__ = ('_filename', '_filepath', '_id_db', '_create_time', '_file_version', '_file_uuid', '_data_format', '_last_time',
                 '_device', '_configuration', '_channel_mask', '_database',
                 '_dirty', '_last_modify_time', '_file_size_cache',
                 '_recording_file_ch', '_recording_sub_file', '_size',
                 '_parameter', '_parent', '_recording_file_name', '_recording_sub_mini', '_recording_mini_ch',
                 '_raw_serial_number', '_mini_serial_number', '_device_id', '_project_id')

    def __init__(self, filepath: Union[str, Path], data_format: bytes = None, id_db = 0, database = None):
        if isinstance(filepath, str):
            filepath = Path(filepath)
        
        name = re.split('[./]+',str(filepath))
        self._filename = name[len(name)-2]
        self._filepath = filepath.absolute().with_suffix(self.FILE_EXT)
        self._id_db = id_db
        
        self._database = database

        # file header
        self._create_time = int(1000 * time())
        self._last_time = int(0)
        self._file_version = RecordingFileVersion.VERSION_00_02
        self._file_uuid = uuid.uuid1()

        if data_format is None:
            self._data_format = RecordingFileDataFormat.REC_DATA

        elif data_format not in RecordingFileDataFormat.DATA:
            raise IndexError('unknown data format : ' + repr(data_format))

        else:
            self._data_format = data_format

        # recording device
        # self._device = DeviceResponseInfo.UNKNOWN
        self._device = ''
        self._parameter = ''
        self._parent = ''
        self._recording_file_name = ''
        self._device_id = -1
        self._project_id = None

        # recording parameter
        self._configuration = None

        # channel mask
        self._channel_mask = ChannelMask()

        # recording file list
        # self._recording_file = []

        self._recording_file_ch = []
        self._recording_mini_ch = []
        self._raw_serial_number = {}
        self._mini_serial_number = {}

        self._recording_sub_file = {}
        self._recording_sub_mini = {}


        self._size = 0

        self._dirty = True
        self._last_modify_time = 0

        self._file_size_cache = None

    def __eq__(self, o: object) -> bool:
        if self is o:
            return True

        if not isinstance(o, RecordingMetaFile):
            return False

        return self._filepath == o._filepath

    @classmethod
    def is_recording_meta_file(cls, path: Union[str, Path]) -> bool:
        if isinstance(path, str):
            path = Path(path)

        if not path.is_file():
            return False

        with path.open('br') as f:
            try:
                return cls._is_recording_meta_file(f)
            except BaseException as e:
                raise IOError('file ' + str(path)) from e
        return False

    @classmethod
    def _is_recording_meta_file(cls, io: BinaryIO) -> bool:
        header = io.read(4)

        return header == RecordingMetaFile.MAGIC_NUMBER

    @staticmethod
    def load(filepath: Union[str, Path], database = None) -> 'RecordingMetaFile':
        """load meta file

        :param filepath: file path
        :return: meta file
        :raises RuntimeError: not a meta file
        :raises FileNotFoundError:
        """
        if not RecordingMetaFile.is_recording_meta_file(filepath):
            raise RuntimeError('not a recording meta file : ' + str(filepath))

        meta = RecordingMetaFile(filepath, database = database)
        # meta._filepath = filepath
        meta.reload()

        return meta

    def reopen(self, data_format: bytes = None) -> 'RecordingMetaFile':
        """reopen file.

        :param data_format: change data format
        :return: new recording meta file
        """
        if data_format is None:
            data_format = self._data_format

        ret = RecordingMetaFile(self._filepath, data_format, id_db = self._id_db, database = self._database)
        ret._device = self._device
        ret._configuration = self._configuration
        ret._dirty = True

        return ret

    def clone(self, path: Path) -> 'RecordingMetaFile':
        ret = RecordingMetaFile(path, self._data_format, id_db = self._id_db, database = self._database)
        ret._device = self._device
        ret._configuration = self._configuration
        ret._dirty = True

        return ret

    def reload(self):
        """reload meta file

        :raises RuntimeError: not a meta file
        :raises FileNotFoundError:
        """

        loader = RecordingMetaFileLoader.load(self._filepath)

        if loader.header != self.MAGIC_NUMBER:
            raise IOError('not a BioPro Recording File : ' + str(self._filepath))

        def _require(it: Optional[T], message: str) -> T:
            if it is None:
                raise RuntimeError('BioPro Recording File lost field : ' + message)

            return it

        self._file_version = _require(loader.version, 'file version')
        self._create_time = _require(loader.create_time, 'file create time')
        self._last_time = _require(loader.last_time, 'file last time')
        self._file_uuid = _require(loader.file_uuid, 'file UUID')
        self._data_format = _require(loader.data_format, 'data format')

        self._device = _require(loader.device_response, 'device information')
        self._configuration = _require(loader.configuration, 'configurations')

        self._channel_mask = _require(loader.channel_mask, 'channel_mask')


        self._dirty = True
        self._last_modify_time = self._filepath.stat().st_mtime
        return None

    def write(self, database = None):
        # print('******', self._filepath)
        _name = str(self._filepath).split('/')[-1]
        _path = str(self._filepath).replace('/', '^')

        meta_data = {
            'path': _path,
            'configuration': self._configuration.as_json(),
            'version': str(self._file_version),
            'uuid': str(self._file_uuid),
            'data_format': 'rec',
            'device': self._device,
            'project': self._project_id,
        }

        if len(self._parent) > 0:
            meta_data['parent'] = self._parent
        if len(self._recording_file_name) > 0:
            meta_data['name'] = self._recording_file_name
        if len(self._parameter) > 0:
            meta_data['parameter_set'] = self._parameter

        if database is not None :
            self._database = database
            self._database.put_queue(['data_meta_write', meta_data, self._device_id, str(self._file_uuid), self._id_db])
            # database.put_queue(['data_meta_write', self, meta_data, _path, self._id_db])
                
        with self._filepath.open('wb') as _f:
            f = FileEncoder(_f)

            # magic number
            f.write(self.MAGIC_NUMBER)

            # version number
            f.u32(self._file_version)

            # create time
            f.u64(self._create_time)

            # last time
            if self._last_time is None:
                self._last_time = 0
            f.u64(int(self._last_time))

            # file uuid
            f.write(self._file_uuid.bytes)

            # data format
            f.write(self._data_format)

            # recording device
            device = self._device
            if isinstance(device, str):
                device = json.loads(device)
            _device = {}
            _device['device_name'] = device['device_name']
            _device['device_address'] = device['device_address']
            _device = json_stringify(_device)
            f.encode_string(_device)

            encoder = DeviceConfigurationEncoder()
            encoder.write(self._configuration, f)

            #channel information
            f.u8(self._channel_mask.size)
            f.u16a(self._channel_mask.channels())

            f.write(b'\0')

        self._dirty = False
        self._last_modify_time = self._filepath.stat().st_mtime
        return None

    def update_subfile(self, database = None):
        meta_data = {
            'raw_data': self._recording_sub_file,
            'mini_data': self._recording_sub_mini,
            'channels': str(self._channel_mask.channels()),
            'size': str(self._size),
            'time_duration': str(self._last_time),
            'uuid': str(self._file_uuid),
        }
        _path = str(self._filepath).replace('/', '^')

        if database is not None :
            self._database = database
            self._database.put_queue(['data_meta_write', meta_data, self._device_id, str(self._file_uuid), self._id_db])
            # database.put_queue(['data_meta_write', self, meta_data, _path, self._id_db])
        return None
    
    def update_subfile_time_size(self, database = None):
        meta_data = {
            'channels': str(self._channel_mask.channels()),
            'size': str(self._size),
            'time_duration': str(self._last_time),
            'uuid': str(self._file_uuid),
        }
        _path = str(self._filepath).replace('/', '^')

        if database is not None :
            self._database = database
            self._database.put_queue(['data_meta_write', meta_data, self._device_id, str(self._file_uuid), self._id_db])
            # database.put_queue(['data_meta_write', self, meta_data, _path, self._id_db])
        return None
    
    def update_subfile_raw(self, database = None):
        meta_data = {
            'raw_data': self._recording_sub_file,
        }
        _path = str(self._filepath).replace('/', '^')

        if database is not None :
            self._database = database
            self._database.put_queue(['data_meta_write', meta_data, self._device_id, str(self._file_uuid), self._id_db])
            # database.put_queue(['data_meta_write', self, meta_data, _path, self._id_db])
        return None
    
    def update_subfile_mini(self, database = None):
        meta_data = {
            'mini_data': self._recording_sub_mini,
        }
        _path = str(self._filepath).replace('/', '^')

        if database is not None :
            self._database = database
            self._database.put_queue(['data_meta_write', meta_data, self._device_id, str(self._file_uuid), self._id_db])
            # database.put_queue(['data_meta_write', self, meta_data, _path, self._id_db])
        return None
                
    @property
    def filename(self) -> str:
        """

        :return: filename without file extension
        """
        return self._filepath.with_suffix('').name

    @property
    def filepath(self) -> Path:
        return self._filepath

    @property
    def create_time(self) -> Optional[int]:
        return self._create_time

    @property
    def last_time(self) -> Optional[int]:
        return self._last_time

    @property
    def file_version(self) -> int:
        return self._file_version.value

    @property
    def file_uuid(self) -> uuid.UUID:
        return self._file_uuid

    @property
    def file_size(self) -> int:
        return self._size

    @property
    def data_format(self) -> bytes:
        return self._data_format

    @property
    def device(self) -> str:
        return self._device

    @device.setter
    def device(self, device):
        self._device = device
        self._dirty = True

    @property
    def parameter(self) -> str:
        return self._parameter

    @parameter.setter
    def parameter(self, parameter):
        self._parameter = parameter
        return None

    @property
    def parent(self) -> str:
        return self._parent

    @parent.setter
    def parent(self, parent):
        self._parent = parent

    @property
    def recording_file_name(self) -> str:
        return self._recording_file_name

    @recording_file_name.setter
    def recording_file_name(self, recording_file_name):
        self._recording_file_name = recording_file_name

    @property
    def configuration(self) -> DeviceConfiguration:
        return self._configuration

    @configuration.setter
    def configuration(self, conf: DeviceConfiguration):
        self._configuration = conf
        self._dirty = True
        return None
    
    @property
    def project_id(self) -> int:
        return self._project_id

    @project_id.setter
    def project_id(self, project: int):
        self._project_id = project
        return None

    @property
    def channels(self) -> List[int]:
        return self._channel_mask.channels()

    def update_channels(self, channel: List[int]):
        if self._channel_mask.add_channel(channel):
            self._dirty = True

    @property
    def is_dirty(self) -> bool:
        return self._dirty

    @property
    def is_modified(self) -> bool:
        return self._last_modify_time == 0 or                 not self._filepath.exists() or                 (self._filepath.is_file() and self._filepath.stat().st_mtime > self._last_modify_time)

    @property
    def last_modified(self) -> int:
        return self._last_modify_time

    def new_recording_file(self, _channel: int, _start_time, database = None) -> 'RecordingFile':

        if _channel not in self._raw_serial_number:
            self._raw_serial_number[_channel] = 0
        else:
            self._raw_serial_number[_channel] += 1

        serial_number = self._raw_serial_number[_channel]
        # if _channel is not None:
        #     for rf in self._recording_file_ch:
        #         if rf._channel == _channel:
        #             serial_number = serial_number + 1
        # else:
        #     raise RuntimeError('new file no channel')

        
        n = '{}_{}_{}{}'.format(self.filename, _channel, serial_number, RecordingFile.FILE_EXT)

        p = self._filepath.with_name(n)


        f = RecordingFile(p, database = database)

        f._meta_file = self
        f._file_uuid = self._file_uuid
        f._serial_number = serial_number
        f._data_format = self._data_format
        f._channel = _channel
            
        self._dirty = True

        _name = str(n)
        _path = str(p).replace('/', '^')

        raw_data = {
            'name': _name,
            'path': _path,
            'parent': self._id_db,
            'size': '0',
            'uuid': str(self._file_uuid),
            'serial_number': str(serial_number),
            'data_format': 'rec',
            'channel': int(_channel),
            'start_time': str(_start_time),
        }

        return f, raw_data

    def new_recording_mini(self, _channel: int, _start_time, scale, database = None) -> 'RecordingMini':

        if scale not in self._mini_serial_number:
            self._mini_serial_number[scale] = {}
        
        if _channel not in self._mini_serial_number[scale]:
            self._mini_serial_number[scale][_channel] = 0
        else:
            self._mini_serial_number[scale][_channel] += 1

        serial_number = self._mini_serial_number[scale][_channel]

        
        n = '{}_{}_{}{}'.format(self.filename, _channel, serial_number, RecordingMini.FILE_EXT)

        p = self._filepath.with_name(n)


        f = RecordingMini(p, scale = scale, database = database)

        f._meta_file = self
        f._file_uuid = self._file_uuid
        f._serial_number = serial_number
        f._data_format = self._data_format
        f._channel = _channel
            
        self._dirty = True

        _name = str(n)
        _path = str(p).replace('/', '^')

        mini_data = {
            'name': _name,
            'path': _path,
            'parent': self._id_db,
            'size': '0',
            'uuid': str(self._file_uuid),
            'serial_number': str(serial_number),
            'scale': scale,
            'channel': int(_channel),
            'start_time': str(_start_time),
        }

        return f, mini_data

    def clear_recording_file(self):
        if len(self._channel_mask) > 0:
            self._channel_mask.clear()
            self._dirty = True
        return None

    def change_filepath(self, target_path: Path) -> Path:
        meta_path = self.filepath

        new_name = target_path.with_suffix('').name

        t = target_path.with_name(new_name).with_suffix(RecordingMetaFile.FILE_EXT)
        meta_path.rename(t)

        return t

    def remove_file(self):
        meta_path = self.filepath
        meta_path.unlink()
        return None

    @classmethod
    def remove_meta_file(cls, filepath: Union[str, Path]):
        if isinstance(filepath, str):
            filepath = Path(filepath)

        if not RecordingMetaFile.is_recording_meta_file(filepath):
            raise RuntimeError('not a recording meta file : ' + str(filepath))

        try:
            meta = cls.load(filepath)
            meta.remove_file()
        except:
            # force remove
            parent = filepath.parent
            name = filepath.name

            filepath.unlink()

            for p in list(parent.iterdir()):
                if p.name.startswith(name) and p.name.endswith(RecordingFile.FILE_EXT):
                    p.unlink()
        return None

    def as_json(self) -> JSON:
        ret = {
            'file_name':     str(self._filename),
            'file_path':     str(self._filepath),
            'file_size':     self.file_size,
            'time_duration': self._last_time,
            'create_time': self._create_time,
            'channel':       self.channels,
            'configuration': self._configuration.as_json(),
            'device':        self._device,
        }

        return ret


class RecordingFile:
    """

    **RecordingFile header format**

    ::

        struct {
            u32_B magic_number;
            u32_B version_number
            u64_B create_time;

            # reference meta
            u8 file_uuid[16];
            u8 serial_number;

            u8 data_format[4];

            union {
                # raw data section
                # data_format = b'raw\0'
                struct {
                    u8      raw_data[...]
                }

                # recording data section
                # data_format = b'rec\0'
                struct {
                    struct  RecordingData[...];
                }
            }
        }

    create_time
        unix time stamp

    file_uuid
        meta file uuid.

    serial_number
        the serial number of the split files. for mother file, the number equal to 0.

    """

    __slots__ = ('_filepath', '_create_time', '_file_version', '_file_uuid', '_serial_number', '_data_format',
                 '_meta_file', '_file_object', '_channel', '_status', '_id_db', '_database', '_size')

    FILE_EXT = '.bprf'
    MAGIC_NUMBER = b'BPRF'

    def __init__(self, filepath: Path, id_db = 0, database = None):
        # file path
        self._filepath = filepath
        self._id_db = id_db

        self._database = database

        self._size = 0

        # file header
        self._create_time = int(1000 * time())
        self._file_version = RecordingFileVersion.VERSION_00_02
        self._file_uuid = None
        self._serial_number = None
        self._data_format = None

        self._channel = None

        # file status (open/close)
        self._status = False

        # meta file
        self._meta_file = None

        # cache file object
        self._file_object = None

    @classmethod
    def is_recording_file(cls, path: Union[str, Path]) -> bool:
        if isinstance(path, str):
            path = Path(path)

        if not path.is_file():
            return False

        with path.open('br') as f:
            try:
                return cls._is_recording_file(f)
            except BaseException as e:
                raise IOError('file ' + str(path)) from e

    @classmethod
    def _is_recording_file(cls, io: BinaryIO) -> bool:
        header = io.read(4)

        return header == RecordingFile.MAGIC_NUMBER

    @staticmethod
    def load(filepath: Union[str, Path]):
        if isinstance(filepath, str):
            filepath = Path(filepath)

        if not RecordingFile.is_recording_file(filepath):
            raise IOError('not a recording file : ' + str(filepath))

        f = RecordingFile(filepath, database = self._database)
        f._load()

        return f

    @property
    def filepath(self) -> Path:
        return self._filepath

    @property
    def filename(self) -> str:
        """

        :return: filename with file extension
        """
        return self._filepath.name

    @property
    def create_time(self) -> Optional[int]:
        return self._create_time

    @property
    def file_version(self) -> Optional[int]:
        return self._file_version

    @property
    def file_uuid(self) -> uuid.UUID:
        return self._file_uuid

    @property
    def serial_number(self) -> int:
        return self._serial_number

    @property
    def data_format(self) -> bytes:
        return self._data_format

    def open(self):
        self._status = True
        return None

    def close(self, _end_time):
        if self._database is not None:
            _data = {
                'end_time': str(_end_time),
                'size': str(self._size)
            }

            if self._meta_file._id_db != 0:
                _data['parent'] = self._meta_file._id_db
            self._database.put_queue(['data_raw_update', self._id_db, self._channel, _data])
        self._status = False
        return None

    def update_endtime(self, _end_time):
        if self._database is not None:
            _data = {
                'end_time': str(_end_time),
                'size': str(self._size)
            }
            self._database.put_queue(['data_raw_update', self._id_db, self._channel, _data])
        return None

    def flush(self):
        if self._file_object is not None and self._file_object.writable():
            self._file_object.flush()

    def _ensure_read(self) -> IO:
        return None

    def _ensure_write(self) -> BufferedWriter:
        return None

    def _load(self):
        """load file header"""
        return None

    def _write(self, f: FileEncoder):
        return None

    def read(self, size: Optional[int] = None) -> bytes:
        return None

    # def write(self, content: Union[bytes, RecordingData]) -> int:
    def write(self, content: str, channels: list) -> int:
        if not isinstance(content, str):
            raise RuntimeError('wrong data format : ' + repr(self._data_format))

        self._meta_file.update_channels(channels)
        sz = sys.getsizeof(content)
        self._size += sz
        return sz

    def _write_recording_data(self, data: RecordingData):
        # check channel of data in channel of meta

        meta = self._meta_file

        if meta is not None:
            meta.update_channels(data.channels())
        return None

    def read_block(self) -> Optional[RecordingData]:
        if self._data_format != RecordingFileDataFormat.REC_DATA:
            raise RuntimeError('wrong data format : ' + repr(self._data_format))

        return RecordingData.read_block(self._ensure_read())

    def read_block_iter(self) -> Iterable[RecordingData]:
        if self._data_format != RecordingFileDataFormat.REC_DATA:
            raise RuntimeError('wrong data format : ' + repr(self._data_format))

        auto_close = self._file_object is None

        file = self._ensure_read()

        try:
            data = RecordingData.read_block(file)

            while data is not None:
                yield data
                data = RecordingData.read_block(file)

        finally:
            if auto_close:
                self.close()
        return None

class RecordingMini:

    __slots__ = ('_filepath', '_create_time', '_file_version', '_file_uuid', '_serial_number', '_data_format',
                 '_meta_file', '_file_object', '_channel', '_status', '_id_db', '_database', '_size',
                 '_scale')

    FILE_EXT = '.bpri'
    MAGIC_NUMBER = b'BPRI'

    def __init__(self, filepath: Path, scale = 100, id_db = 0, database = None):
        # file path
        self._filepath = filepath
        self._id_db = id_db

        self._database = database

        self._size = 0

        # file header
        self._create_time = int(1000 * time())
        self._file_version = RecordingFileVersion.VERSION_00_02
        self._file_uuid = None
        self._serial_number = None
        self._data_format = None

        self._channel = None

        # file status (open/close)
        self._status = False

        # meta file
        self._meta_file = None

        # cache file object
        self._file_object = None

        # scale
        self._scale = scale

    @property
    def filepath(self) -> Path:
        return self._filepath

    @property
    def size(self) -> int:
        return self._size

    @property
    def filename(self) -> str:
        """

        :return: filename with file extension
        """
        return self._filepath.name

    @property
    def create_time(self) -> Optional[int]:
        return self._create_time

    @property
    def file_version(self) -> Optional[int]:
        return self._file_version

    @property
    def file_uuid(self) -> uuid.UUID:
        return self._file_uuid

    @property
    def serial_number(self) -> int:
        return self._serial_number

    @property
    def data_format(self) -> bytes:
        return self._data_format

    @property
    def scale(self) -> int:
        return self._scale

    def open(self):
        self._status = True

    def close(self, _end_time):
        if self._database is not None:
            _data = {
                'end_time': str(_end_time),
                'size': str(self._size)
            }

            if self._meta_file._id_db != 0:
                _data['parent'] = self._meta_file._id_db
            self._database.put_queue(['data_mini_update', self._id_db, self._channel, _data])
            # self._database.put_queue(['data_mini_update', self._id_db, self._channel, _data])

    def _load(self):
        """load file header"""
        return None

    def read(self, size: Optional[int] = None) -> bytes:
        return None

    # def write(self, content: Union[bytes, RecordingData]) -> int:
    def write(self, data_mean: list) -> int:

        if len(data_mean) > 0:
            self._size += sys.getsizeof(data_mean)
        return 0


class RecordingFileWriter:
    __slots__ = ('_meta', '_recording_file', '_recording_file_dict', '_channel_list', '_database',
                 'splitting_threshold_time', 'splitting_threshold_size', '_writer_batch_size',
                '_splitting_size', '_time', '_time_now', '_recording_mini_dict',
                 '_data_value_ch', '_close', '_data_mini_ch',
                 '_mini_scale_list', '_time_real_time', '_data_rl', '_data_db',
                 '_raw_save', '_mini_save', '_data_time_ch', '_data_value_ch_for_rl',
                 '_data_time_ch_for_rl', '_device_id', '_send_data', '_data_mqtt_ch', '_id_db_save', '_raw_create_not_done',
                 '_mini_create_not_done')

    def __init__(self, meta: RecordingMetaFile, device_id, database = None):
        self._meta = meta
        # self._recording_file = None
        self._device_id = device_id

        self._database = database

        self._mini_scale_list = [10, 100, 1000]

        self._recording_file_dict = {}
        self._recording_mini_dict = {
            '10': {},
            '100': {},
            '1000': {},
            # '10000': {}
        }

        self._raw_save = {
            'id': {},
            'data': {},
            'end_time': {},
            'size': {}
        }
        self._mini_save = {
            '10': {
                'id': {},
                'start_time': {},
                'data_mean': {},
                'data_random': {},
                'data_bar': {}
            },
            '100': {
                'id': {},
                'start_time': {},
                'data_mean': {},
                'data_random': {},
                'data_bar': {}
            },
            '1000': {
                'id': {},
                'start_time': {},
                'data_mean': {},
                'data_random': {},
                'data_bar': {}
            }
        }

        self._channel_list = []

        # save data chache
        self._data_rl = {}
        self._data_db = {}
        self._data_value_ch = {}
        self._data_value_ch_for_rl = {}
        self._data_time_ch_for_rl = {}
        self._data_time_ch = {}
        self._id_db_save = {}

        # mini data
        self._data_mini_ch = {}

        # mqtt real time
        self._data_mqtt_ch = {}

        self._time = {}
        self._time_now = None
        self._time_real_time = {}

        self._close = False
        self._send_data = {}

        # splitting
        self.splitting_threshold_time = 30 * 60 * 1000  # one minute
        self.splitting_threshold_size = 16 * 1024  #  16 * 16KB

        self._writer_batch_size = 8192 

        self._splitting_size = None

        self._raw_create_not_done = True
        self._mini_create_not_done = True


    @property
    def meta_file(self) -> RecordingMetaFile:
        return self._meta

    @property
    def channel_list(self):
        return self._channel_list

    def __enter__(self):
        self.open()

        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()
        return None

    def channels_update(self, channels):
        channels.sort()
        self._channel_list.extend(channels)
        return self._channel_list

    def open(self):
        self._meta.clear_recording_file()

        if self._meta.is_dirty:
            self._meta.write(database = self._database)

        self._splitting_size = 0
        return None

    def close(self, mqtt_thread):
        self._close = True
        if len(self._recording_file_dict) > 0:
            for ch in self._data_db.keys():
                if len(self._data_rl[ch]) > 0:
                    self._data_rl[ch].append(str(int(self._time_now)))
                    mes = ' '.join(self._data_rl[ch])
                    mqtt_thread[ch].on_message(mes)
                    self._data_rl[ch].clear()
                    self._send_data[ch] = False

                if self._recording_file_dict[ch]._status:
                    _data = ' '.join(self._data_db[ch])
                    self._raw_save['data'][ch] = _data
                    self._raw_save['id'][ch] = self._recording_file_dict[ch]._id_db
                    self._recording_file_dict[ch].write(_data, self._channel_list)
                    self._recording_file_dict[ch].close(self._time_now)
                    self._meta._size += self._recording_file_dict[ch]._size
            # self._data_db.clear()
            if self._database is not None:
                self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data'], self._id_db_save])
                # self._database.put_queue(['data_raw_recording', self._raw_save['id'], self._channel_list, self._raw_save['data']])
            self._recording_file_dict.clear()
        for scale in self._mini_scale_list:
            if len(self._recording_mini_dict[str(scale)]) > 0:
                for ch in self._recording_mini_dict[str(scale)].keys():
                    if self._recording_mini_dict[str(scale)][ch]._status:
                        self._mini_save[str(scale)]['id'][ch] = self._recording_mini_dict[str(scale)][ch]._id_db
                        self._mini_save[str(scale)]['start_time'][ch] = str(self._data_mini_ch[ch][str(scale)]['start_time'])
                        self._mini_save[str(scale)]['data_mean'][ch] = self._data_mini_ch[ch][str(scale)]['mean'].copy()
                        # self._mini_save[str(scale)]['data_random'][ch] = self._data_mini_ch[ch][str(scale)]['random'].copy()
                        # self._mini_save[str(scale)]['data_bar'][ch] = self._data_mini_ch[ch][str(scale)]['bar'].copy()
                        # start_time = str(self._data_mini_ch[ch][str(scale)]['start_time'])
                        # data_mean = self._data_mini_ch[ch][str(scale)]['mean'].copy()
                        # data_random = self._data_mini_ch[ch][str(scale)]['random'].copy()
                        # print('channel:', ch, ', data: ', data_mean)
                        self._recording_mini_dict[str(scale)][ch].write(self._mini_save[str(scale)]['data_mean'][ch])
                        self._recording_mini_dict[str(scale)][ch].close(self._time_now)
                        self._data_mini_ch[ch][str(scale)]['mean'].clear()
                        # self._data_mini_ch[ch][str(scale)]['random'].clear()
                        # del self._data_mini_ch[ch][str(scale)]['mean']
                        # del self._data_mini_ch[ch][str(scale)]['random']
            if self._database is not None:
                self._database.put_queue(['data_mini_recording', self._mini_save[str(scale)]['id'], self._channel_list, self._mini_save[str(scale)]['start_time'], self._mini_save[str(scale)]['data_mean']])
                # self._database.put_queue(['data_mini_recording', self._recording_mini_dict[str(scale)], self._mini_save[str(scale)]['id'], self._channel_list, self._mini_save[str(scale)]['start_time'], self._mini_save[str(scale)]['data_mean'], self._mini_save[str(scale)]['data_random']])
        self._data_mini_ch.clear()
        self._recording_mini_dict.clear()
        # del self._data_mini_ch
        # del self._recording_mini_dict
        gc.collect()

        self._meta._last_time = self._time_now
        self._meta.update_subfile(database = self._database)
        # self._meta.write(database = self._database)
        print('close2')
        return True

    def flush(self):
        if len(self._recording_file_dict) > 0:
            for ch in self._channel_list:
                if ch in self._recording_file_dict:
                    self._recording_file_dict[ch].flush()
        return None

    def get_bar(self, data_ch: list, time_ch: list):
        ret = []
        _first_time = -1
        _last_time = -1
        _max = max(data_ch)
        _max_index = data_ch.index(_max)
        _min = min(data_ch)
        _min_index = data_ch.index(_min)

        if time_ch is not None and len(time_ch) >= 2:
            _first_time = time_ch[0]
            _last_time = time_ch[-1]

        if _max_index < _min_index:
            if _first_time > 0:
                ret.append(str(_first_time))
            ret.append(str(_max))
            if _last_time > 0:
                ret.append(str(_last_time))
            ret.append(str(_min))
        else:
            if _first_time > 0:
                ret.append(str(_first_time))
            ret.append(str(_min))
            if _last_time > 0:
                ret.append(str(_last_time))
            ret.append(str(_max))
        return ret

    def get_data_iter(self, d, mqtt_thread):
        # print('****d size', d.data_size)
        for t, c, v in d.entry_iter():
            if c in self._data_db:
                ### send real-time
                if len(self._data_rl[c]) > 0 and self._send_data[c]:
                    self._data_rl[c].append(str(int(t)))
                    mes = ' '.join(self._data_rl[c])
                    # self._data_mqtt_ch[c] = self._data_rl[c].copy()
                    # self._data_mqtt_ch[c].append(str(int(t)))
                    mqtt_thread[c].on_message(mes)
                    self._data_rl[c].clear()
                    self._send_data[c] = False
                ### send real-time
                
                sample_rate_rl = 1
                # rec data
                if c < 256:
                    sample_rate_rl = int(d.sample_rate / ( 500 * len(self.channel_list) )) #* len(self.channel_list)
                # others
                else:
                    sample_rate_rl = int(d.sample_rate_channel_upper_256 / ( 1000 * len(self.channel_list)))
                if self._close is False:
                    self._data_value_ch[c].append(int(v))
                    self._data_value_ch_for_rl[c].append(int(v))
                    self._data_time_ch_for_rl[c].append(int(t))
                    # self._data_time_ch[c].append(int(t))
                    for scale in self._mini_scale_list:
                        if self._data_mini_ch[c][str(scale)]['start_time'] is None:
                            self._data_mini_ch[c][str(scale)]['start_time'] = str(int(t))
                    
                    if len(self._data_value_ch_for_rl[c]) >= sample_rate_rl:
                        if len(self._data_value_ch_for_rl[c]) == 1:
                            if len(self._data_rl[c]) == 0:
                                self._data_rl[c].append(str(int(t)))
                            # self._data_rl[c].append(str(self._data_time_ch_for_rl[c][0]))
                            self._data_rl[c].append(str(self._data_value_ch_for_rl[c][0]))
                        else:
                            if len(self._data_rl[c]) == 0:
                                self._data_rl[c].append(str(int(t)))
                            # self._data_rl[c].append(str(self._data_time_ch_for_rl[c][0]))
                            # self._data_rl[c].append(str(self._data_value_ch_for_rl[c][0]))
                            _max = max(self._data_value_ch_for_rl[c])
                            _max_index = self._data_value_ch_for_rl[c].index(_max)
                            _min = min(self._data_value_ch_for_rl[c])
                            _min_index = self._data_value_ch_for_rl[c].index(_min)
                            # _mean = mean(self._data_value_ch_for_rl[c])

                            # _first_time = self._data_time_ch_for_rl[c][0]
                            # _last_time = self._data_time_ch_for_rl[c][-1]

                            if _max_index < _min_index:
                                # self._data_rl[c].append(str(_first_time))
                                self._data_rl[c].append(str(_max))
                                # self._data_rl[c].append(str(_last_time))
                                self._data_rl[c].append(str(_min))
                            else:
                                # self._data_rl[c].append(str(_first_time))
                                self._data_rl[c].append(str(_min))
                                # self._data_rl[c].append(str(_last_time))
                                self._data_rl[c].append(str(_max))

                        self._data_value_ch_for_rl[c].clear()
                        self._data_time_ch_for_rl[c].clear()

                    # mini picture
                    if len(self._data_value_ch[c]) >= 10:
                        self._data_mini_ch[c]['10']['mean'].append( int(mean(self._data_value_ch[c][0:9])) )
                        # self._data_mini_ch[c]['10']['random'].append( str(self._data_value_ch[c][random.randint(0,9)]) )
                        # _bar = self.get_bar(self._data_value_ch[c], None)
                        # self._data_mini_ch[c]['10']['bar'].extend(_bar)
                        self._data_value_ch[c].clear()
                        if int(len(self._data_mini_ch[c]['10']['mean']) / 10) - self._data_mini_ch[c]['10']['dec'] > 0:
                            self._data_mini_ch[c]['100']['mean'].append( int(mean(self._data_mini_ch[c]['10']['mean'][-10:])) )
                            # self._data_mini_ch[c]['100']['random'].append( str(self._data_mini_ch[c]['10']['random'][random.randint(-10,-1)]) )
                            # _bar = self.get_bar(self._data_mini_ch[c]['10']['bar'], None)
                            # self._data_mini_ch[c]['100']['bar'].extend(_bar)
                            self._data_mini_ch[c]['10']['dec'] = int(len(self._data_mini_ch[c]['10']['mean']) / 10)
                            if int(len(self._data_mini_ch[c]['100']['mean']) / 10) - self._data_mini_ch[c]['100']['dec'] > 0:
                                self._data_mini_ch[c]['1000']['mean'].append( int(mean(self._data_mini_ch[c]['100']['mean'][-10:])) )
                                # self._data_mini_ch[c]['1000']['random'].append( str(self._data_mini_ch[c]['100']['random'][random.randint(-10,-1)]) )
                                # _bar = self.get_bar(self._data_mini_ch[c]['100']['bar'], None)
                                # self._data_mini_ch[c]['1000']['bar'].extend(_bar)
                                self._data_mini_ch[c]['100']['dec'] = int(len(self._data_mini_ch[c]['100']['mean']) / 10)
                                # if int(len(self._data_mini_ch[c]['1000']['mean']) / 10) - self._data_mini_ch[c]['1000']['dec'] > 0:
                                #     self._data_mini_ch[c]['10000']['mean'].append( int(mean(self._data_mini_ch[c]['1000']['mean'][-10:])) )
                                #     self._data_mini_ch[c]['10000']['random'].append( str(self._data_mini_ch[c]['1000']['random'][random.randint(-10,-1)]) )
                                #     self._data_mini_ch[c]['1000']['dec'] = int(len(self._data_mini_ch[c]['1000']['mean']) / 10)
                    # add normal data
                    self._data_db[c].append(str(int(t)))
                    self._data_db[c].append(str(v))
                    self._time_now = int(t)
        return
    
    # @calculate_time(1)
    def write(self, data: Union[bytes, RecordingData, List[bytes], List[RecordingData]], mqtt_thread) -> int:
        # check size
        ths = self.splitting_threshold_size
        tht = self.splitting_threshold_time

        if self._close is False:
            if len(self._recording_file_dict) == 0:
                self._switch_recording_file()
            elif ths is not None and self._splitting_size > ths * len(self._channel_list):
                self._switch_recording_file()

            for scale in self._mini_scale_list:
                if len(self._recording_mini_dict[str(scale)]) == 0:
                    self._switch_recording_mini(scale = scale)
                elif ths is not None and self._recording_mini_dict[str(scale)][list(self._recording_mini_dict[str(scale)].keys())[0]].size > ths:
                    self._switch_recording_mini(scale = scale)

        # write data

        if isinstance(data, list):
            for d in data:
                if self._close is False:
                    if ths is not None and self._splitting_size > ths * len(self._channel_list):
                        self._switch_recording_file()

                    for scale in self._mini_scale_list:
                        if ths is not None and self._recording_mini_dict[str(scale)][list(self._recording_mini_dict[str(scale)].keys())[0]].size > ths:
                            self._switch_recording_mini(scale = scale)

                sz = 0

                # init
                if len(self._data_value_ch) == 0:
                    for ch in self._channel_list:
                        self._data_value_ch[ch] = []
                        self._data_value_ch_for_rl[ch] = []
                        self._data_time_ch_for_rl[ch] = []
                        self._data_time_ch[ch] = []
                        self._send_data[ch] = False
                        self._data_mqtt_ch[ch] = []
                        self._data_rl[ch] = []
                        self._data_db[ch] = []
                        self._time[ch] = 0
                        self._time_real_time[ch] = 0
                if len(self._data_mini_ch) == 0:
                    for ch in self._channel_list:
                        self._data_mini_ch[ch] = {}
                        for scale in self._mini_scale_list:
                            self._data_mini_ch[ch][str(scale)] = {
                                'start_time': None,
                                'mean': [],
                                'random': [],
                                'bar': [],
                                'dec': 0,
                            }
                
                self.get_data_iter(d, mqtt_thread)

            if len(self._recording_file_dict) > 0:
                for ch in self._data_db.keys():
                    if self._time_now - self._time_real_time[ch] > 1000000:
                        self._send_data[ch] = True
                        self._time_real_time[ch] = self._time_now

            for ch in self._recording_file_dict:
                if self._recording_file_dict[ch]._id_db == 0:
                    return None
            
            for ch in self._data_db.keys():
                for scale in self._mini_scale_list:
                    if  self._recording_mini_dict[str(scale)][ch]._id_db == 0:
                        return None
            
            if self._raw_create_not_done :
                self._raw_create_not_done = False
                self._meta.update_subfile_raw(database = self._database)
            
            if self._mini_create_not_done :
                self._mini_create_not_done = False
                self._meta.update_subfile_mini(database = self._database)

            data_save = False
            mini_save = False
            if len(self._recording_file_dict) > 0:
                for ch in self._data_db.keys():
                    if self._time_now - self._time[ch] > 5000000:
                        if self._recording_file_dict[ch]._status:
                            _data = ' '.join(self._data_db[ch])
                            write_sz = self._recording_file_dict[ch].write(_data, self._channel_list)
                            self._raw_save['data'][ch] = _data
                            self._raw_save['id'][ch] = self._recording_file_dict[ch]._id_db
                            self._raw_save['end_time'][ch] = self._time_now
                            self._raw_save['size'][ch] = self._recording_file_dict[ch]._size
                            self._data_db[ch].clear()
                            self._time[ch] = self._time_now
                            self._meta._last_time = self._time_now
                            self._splitting_size += write_sz
                            data_save = True
                    if len(self._data_mini_ch[ch]['1000']['mean']) >= 10:
                        mini_save = True
                        for scale in self._mini_scale_list:
                            str_mean = [str(int) for int in self._data_mini_ch[ch][str(scale)]['mean']]
                            data_mean = str(self._data_mini_ch[ch][str(scale)]['start_time']) + ' ' + ' '.join(str_mean) + '"***"'
                            self._mini_save[str(scale)]['id'][ch] = self._recording_mini_dict[str(scale)][ch]._id_db
                            self._mini_save[str(scale)]['start_time'][ch] = str(self._data_mini_ch[ch][str(scale)]['start_time'])
                            self._mini_save[str(scale)]['data_mean'][ch] = data_mean
                            # self._mini_save[str(scale)]['data_random'][ch] = self._data_mini_ch[ch][str(scale)]['random'].copy()
                            # self._mini_save[str(scale)]['data_bar'][ch] = self._data_mini_ch[ch][str(scale)]['bar'].copy()
                            self._recording_mini_dict[str(scale)][ch].write(self._mini_save[str(scale)]['data_mean'][ch])
                            self._data_mini_ch[ch][str(scale)]['mean'].clear()
                            # self._data_mini_ch[ch][str(scale)]['random'].clear()
                            self._data_mini_ch[ch][str(scale)]['dec'] = 0
                            self._data_mini_ch[ch][str(scale)]['start_time'] = None

                if data_save is True:
                    if self._database is not None:
                        recording_input = ['data_raw_recording_new', copy(self._raw_save['id']), copy(self._channel_list), copy(self._raw_save['data']), copy(self._raw_save['end_time']), copy(self._raw_save['size'])]
                        self._database.put_queue(recording_input)
                        self._meta.update_subfile_time_size(database = self._database)
                if mini_save is True:
                    if self._database is not None:
                        for scale in self._mini_scale_list:
                            self._database.put_queue(['data_mini_recording_new', copy(self._mini_save[str(scale)]['id']), self._channel_list, copy(self._mini_save[str(scale)]['data_mean'])])
                        self._meta.update_subfile_time_size(database = self._database)
        del data

        return None


    def _switch_recording_file(self):
        self._raw_create_not_done = True

        if len(self._recording_file_dict) > 0:
            for ch in self._recording_file_dict.keys():
                self._id_db_save[ch] = self._recording_file_dict[ch]._id_db
                self._recording_file_dict[ch].close(self._time_now)
                self._meta._size += self._recording_file_dict[ch]._size
                
        start_time = 0
        if self._time_now is not None:
            start_time = self._time_now

        raw_data_dict = {}
        for ch in self._channel_list:
            self._recording_file_dict[ch], raw_data_tmp  = self._meta.new_recording_file(ch, start_time, database = self._database)
            raw_data_dict[ch] = raw_data_tmp
            self._recording_file_dict[ch].open()

        if self._database is not None:
            self._database.put_queue(['data_raw_create', raw_data_dict, self._channel_list, self._device_id])

        self._meta._last_time = self._time_now
        # self._meta.update_subfile(database = self._database)

        self._splitting_size = 0

        return None

    def _switch_recording_mini(self, scale):
        self._mini_create_not_done = True
        
        if len(self._recording_mini_dict[str(scale)]) > 0:
            for ch in self._recording_mini_dict[str(scale)].keys():
                self._recording_mini_dict[str(scale)][ch].close(self._time_now)
                
        start_time = 0
        if self._time_now is not None:
            start_time = self._time_now

        mini_data_dict = {}
        for ch in self._channel_list:
            self._recording_mini_dict[str(scale)][ch], mini_data_tmp = self._meta.new_recording_mini(ch, start_time, scale = scale, database = self._database)
            mini_data_dict[ch] = mini_data_tmp
            self._recording_mini_dict[str(scale)][ch].open()

        if self._database is not None:
            self._database.put_queue(['data_mini_create', mini_data_dict, self._channel_list, scale, self._device_id])
            
        self._meta._last_time = self._time_now
        # self._meta.update_subfile(database = self._database)
        return None

    def update_meta_id(self, _id):
        self._meta._id_db = _id
        return None
    
    def update_raw_dict_id(self, _channel, _id):
        self._recording_file_dict[int(_channel)]._id_db = _id
        if int(_channel) not in self._meta._recording_sub_file:
            self._meta._recording_sub_file[int(_channel)] = []
        self._meta._recording_sub_file[int(_channel)].append(_id)
        return None

    def update_mini_dict_id(self, _scale, _channel, _id):
        self._recording_mini_dict[str(_scale)][int(_channel)]._id_db = _id
        if int(_channel) not in self._meta._recording_sub_mini:
            self._meta._recording_sub_mini[int(_channel)] = {}
        if int(_scale) not in self._meta._recording_sub_mini[int(_channel)]:
            self._meta._recording_sub_mini[int(_channel)][int(_scale)] = []
        self._meta._recording_sub_mini[int(_channel)][int(_scale)].append(_id)
        return None