controller-wisetopdataserver/python/biopro/server/cache.py

import abc
from collections import defaultdict
from pathlib import Path
from time import time
from typing import Union, List, Any, Optional, Dict

from biopro.controller import ControlAPI
from biopro.devlib.device import Device
from biopro.message import (DataMessageResponse,
                            DataMessage,
                            DataMessageEncoder,
                            RecordingFileReader,
                            TimeRangeDataMessageResponse,
                            DataMessageHandler)
from biopro.recording.data import RecordingData
from biopro.recording.file import RecordingMetaFile
from biopro.util.console import hex_table, YELLOW
from biopro.util.json import JsonSerialize, JSON_OBJECT
from biopro.util.lock import Synchronized, synchronized
from biopro.util.logger import LoggerFlag
from biopro.util.socket import SocketClient, SocketClientMacro
from biopro.util.stack import print_exception


class CacheNotifyHandler(metaclass=abc.ABCMeta):
    @abc.abstractmethod
    def notify_data(self, device: int, message: str):
        """data server send a message to control server.

        :param device: device ID
        :param message: message content
        """
        pass


class CacheAPI(CacheNotifyHandler, metaclass=abc.ABCMeta):
    __slots__ = ()

    @abc.abstractmethod
    def clear(self):
        """clear all data

        """
        pass

    @abc.abstractmethod
    def push_data(self, data: Union[bytes, List[bytes]]):
        """data server collect data push to control server.

        :param data:
        """
        pass

    @abc.abstractmethod
    def drop_data(self, *device: int):
        """drop data

        :param device: device IDs
        """
        pass

    @abc.abstractmethod
    def stop_data(self, device: int):
        """data server notify control server that device stopped.

        :param device: device ID
        """
        pass


# noinspection PyAbstractClass
class CacheClient(SocketClient, CacheAPI, metaclass=SocketClientMacro(CacheAPI)):
    __slots__ = ()

    def __init__(self):
        super().__init__(ControlAPI.ADDR)

    def send_command(self, cmd: str, *args: Any, **option) -> Any:
        try:
            return super().send_command(cmd, *args, **option)
        except (ConnectionRefusedError, OSError):
            print_exception()
            return None


class FileHandle(JsonSerialize):
    """
    **json format**

    ::

        file_handle = {
            "path" :str
            "handle" :int
            "library" :str
            "version" :str
            "configuration" : DeviceConfiguration
        }

    """

    __slots__ = '_handle', '_meta', '_path', '_reader', '_open', '_last'

    def __init__(self, handle: int, meta: RecordingMetaFile):
        self._handle = handle
        self._meta = meta
        self._path: Union[str, Path] = meta.filepath
        self._reader: Optional[RecordingFileReader] = None
        self._open = 0
        self._last = time()

    @property
    def file_handle(self) -> int:
        return self._handle

    @property
    def filepath(self) -> Path:
        return self._meta.filepath

    @property
    def meta_file(self) -> RecordingMetaFile:
        return self._meta

    @property
    def reader(self) -> RecordingFileReader:
        self._last = time()

        if self._reader is None:
            self._reader = RecordingFileReader(self._meta)

        return self._reader

    @property
    def last_used_time(self) -> float:
        return self._last

    @property
    def used_count(self) -> int:
        return self._open

    def inc_used(self):
        self._open += 1

    def dec_used(self) -> bool:
        self._open = max(0, self._open - 1)

        return self._open == 0

    def shadow_path(self, path: Union[str, Path]) -> 'FileHandle':
        """change the *path* value in json format

        :param path:
        :return: detached file handle.
        """

        f = FileHandle(self._handle, self._meta)
        f._path = path
        return f

    def as_json(self) -> JSON_OBJECT:
        return {
            'path':          str(self._path),
            'handle':        self._handle,
            'library':       self._meta.configuration.library,
            'version':       self._meta.configuration.library_version,
            'configuration': self._meta.configuration.as_json()
        }


class CacheManager(LoggerFlag, CacheAPI, Synchronized):
    """
    Cache Manager

    handle cache_size && cache_time && buffer && file

    """

    DEFAULT_FILE_HANDLE_TIMEOUT = 10 * 60

    __slots__ = ('_handler',
                 '_cache_size', '_cache_time_at_least', '_cache_time',
                 '_device_block_list', '_buffer',
                 '_file', '_file_handle_counter')

    def __init__(self,
                 handler: CacheNotifyHandler,
                 cache_size: Optional[int] = None,
                 cache_time: Optional[int] = 10 * 1000):
        LoggerFlag.__init__(self, 'CacheManager', YELLOW)
        Synchronized.__init__(self)

        self._handler = handler

        self._cache_size = cache_size  # unit: data count
        self._cache_time_at_least = cache_time  # unit: ms
        self._cache_time = cache_time  # unit: ms

        self._device_block_list: Dict[int, Union[RecordingMetaFile, FileHandle]] = {}

        # data cache
        self._buffer: Dict[int, List[RecordingData]] = defaultdict(list)

        # file handle
        self._file: List[FileHandle] = []

        # always increase
        self._file_handle_counter = 0

    @property
    def cache_size(self) -> Optional[int]:
        return self._cache_size

    @property
    def cache_time(self) -> Optional[int]:
        return self._cache_time

    def shutdown(self):
        self.log_verbose('shutdown')
        self._buffer.clear()
        self._device_block_list.clear()
        self._file = []
        self._file_handle_counter = 0

    def clear(self):
        self.log_verbose('clear')
        self._buffer.clear()
        self._device_block_list.clear()

    def list_open_file(self) -> List[int]:
        self.file_close_not_used()

        return list(map(lambda it: it.file_handle, self._file))

    def get_disable_cache_device_list(self) -> List[int]:
        return list(self._device_block_list.keys())

    def disable_cache(self, device: Union[int, Device], meta: RecordingMetaFile):
        """add device ID to block list, which the manager do not save any data from this device into cache.

        :param device:
        :param meta:
        """
        if isinstance(device, Device):
            device = device.device_id

        self._device_block_list[device] = meta

    def enable_cache(self, device: Union[int, Device]):
        """forget :func:`disable_cache` setting. all device are enable the cache by default.

        :param device:
        :return:
        """
        if isinstance(device, Device):
            device = device.device_id

        try:
            f = self._device_block_list[device]
            del self._device_block_list[device]
        except KeyError:
            pass
        else:
            if isinstance(f, FileHandle):
                self.file_close(f)

    def _get_file_handle(self, handle: Union[int, FileHandle]) -> Optional[FileHandle]:
        if isinstance(handle, FileHandle):
            return handle

        for f in self._file:
            if f.file_handle == handle:
                return f
        return None

    def _file_reader(self, handle: Union[int, FileHandle]) -> Optional[RecordingFileReader]:
        f = self._get_file_handle(handle)

        if f is not None:
            return f.reader

        return None

    def file_open(self, meta: RecordingMetaFile) -> int:
        self.file_close_not_used()

        return self._file_open(meta).file_handle

    @synchronized
    def _file_open(self, meta: RecordingMetaFile) -> FileHandle:
        for f in self._file:
            if f.meta_file == meta:
                break
        else:
            i = self._file_handle_counter
            self._file_handle_counter += 1

            f = FileHandle(i, meta)
            self._file.append(f)

            meta.reload()

        f.inc_used()
        self.log_verbose('open', '[%d*%d]' % (f.file_handle, f.used_count), f.filepath)

        return f

    @synchronized
    def file_close(self, handle: Union[int, FileHandle]):
        f = self._get_file_handle(handle)

        if f is not None:
            if f.dec_used():
                try:
                    self._file.remove(f)
                except ValueError:
                    pass

            self.log_verbose('close', '[%d*%d]' % (handle, f.used_count), f.filepath)

        self.file_close_not_used()

    @synchronized
    def file_close_not_used(self):
        t = time()
        for f in list(self._file):
            if t - f.last_used_time > self.DEFAULT_FILE_HANDLE_TIMEOUT:
                try:
                    self._file.remove(f)
                except ValueError:
                    pass

                self.log_verbose('auto close', '[%d*%d]' % (f.file_handle, f.used_count), f.filepath)

    def file_meta(self, handle: Union[int, FileHandle]) -> Optional[RecordingMetaFile]:
        f = self._get_file_handle(handle)

        if f is not None:
            return f.meta_file

        return None

    def file_info(self, handle: int) -> Optional[FileHandle]:
        return self._get_file_handle(handle)

    def push_data(self, data: Union[bytes, List[bytes]]):
        if isinstance(data, list):
            if len(data) == 0:
                return

            coll: List[RecordingData] = []

            for d in data:
                try:
                    d = RecordingData.decode(d)
                except BaseException as e:
                    self.log_warn(e)
                    hex_table(d)

                else:
                    coll.append(d)

            if len(coll) > 0:
                self._push_data(coll)

        else:
            if len(data) == 0:
                return

            try:
                data = RecordingData.decode(data)
                # self.log(pc(method, YELLOW), '[%d]' % len(self._buffer), '@', data.time_stamp)
            except BaseException as e:
                self.log_warn(e)
                hex_table(data)

            else:
                self._push_data(data)

    def _push_data(self, data: Union[RecordingData, List[RecordingData]]):
        buffer = self._buffer
        time_stamp = None

        if isinstance(data, list):
            for _data in data:  # type: RecordingData
                if _data.device not in self._device_block_list:
                    buffer[_data.device].append(_data)
                    time_stamp = _data.time_stamp

        else:
            if data.device not in self._device_block_list:
                buffer[data.device].append(data)
                time_stamp = data.time_stamp

        # print('cache_time_stamp', time_stamp)

        if time_stamp is None:
            return

        # self.log_verbose(pc('push_data', YELLOW), '[%d]' % len(buffer),
        #                  '@', device_id, buffer[0].time_stamp, '~', buffer[-1].time_stamp)

        # if self._cache_time is not None:
        #     for data_list in buffer.values():
        #         for i, cache in enumerate(data_list):
        #             if time_stamp > cache.time_stamp + self._cache_time:
        #                 print('del2')
        #                 del data_list[:i]
        #                 break

        if self._cache_size is not None:
            # print('2')
            for data_list in buffer.values():
                i = len(data_list) - self._cache_size
                if i > 0:
                    del data_list[:i]

        if self._cache_time is None and self._cache_size is None:
            for data_list in buffer.values():
                i = len(data_list) - 2000
                if i > 0:
                    del data_list[:i]

    def drop_data(self, *device: Union[int, Device]):
        for d in device:
            if isinstance(d, Device):
                d = d.device_id

            try:
                del self._buffer[d]
            except KeyError:
                pass

    def stop_data(self, device: Union[int, Device]):
        if isinstance(device, Device):
            device = device.device_id

        self._handler.notify_data(device, DataMessage.stop(device))

    def notify_data(self, device: Union[int, Device], message: str):
        if isinstance(device, Device):
            device = device.device_id

        self._handler.notify_data(device, DataMessage.message(message, device))

    def get_data_message(self, handler: DataMessageHandler, response: DataMessageResponse):
        file_handle = handler.file_handle

        if file_handle is not None:
            # from file
            self._get_data_from_file(file_handle, handler, response)

        else:
            # from cache
            if len(handler.listen_device) == 0:
                handler.on_error('no device listen')

            else:
                for device in handler.listen_device:

                    if device in self._device_block_list:
                        # device disable the cache function, redirect from file
                        h = self._device_block_list[device]

                        if isinstance(h, RecordingMetaFile):
                            h = self._file_open(h)
                            self._device_block_list[device] = h

                        self._get_data_from_file(h, handler, response)

                    else:
                        self._get_data_in_cache(handler, device, response)

    def _get_data_in_cache(self, handler: DataMessageHandler, device: int, response: DataMessageResponse):
        try:
            buffer = self._buffer[device]
        except KeyError:
            self.log_verbose('no device cache : ' + str(device))

        else:
            if handler.listen_channels(device) is None:
                self.log_verbose('no channel listen for device : ' + str(device))
                return

            if len(buffer) == 0:
                # self.log_verbose('empty device cache : ' + str(device))
                return

                # request time range validating
            response = response.to_time_range(buffer[-1].time_stamp)

            time_duration = int(response.time_duration)

            if time_duration <= 0:
                self.log_verbose("negative time duration for device : " + str(device))
                return

            self._cache_time = max(self._cache_time_at_least, time_duration)
            time_start = response.time_start
            time_stop = response.time_stop

            if time_stop < buffer[0].time_stamp:
                self.log_verbose("request time range too old for device : " + str(device))
                return

                # create encoder
            encoder = handler.new_encoder(response, device)
            assert encoder is not None
            assert isinstance(encoder, DataMessageEncoder)

            # encode message
            message = encoder.message()
            # print('message', message)

            # if message is not None:
            #     handler.on_message(message)

    def _get_data_from_file(self,
                            file_handle: Union[int, FileHandle],
                            handler: DataMessageHandler,
                            response: DataMessageResponse):
        reader = self._file_reader(file_handle)

        if reader is None:
            handler.on_error("file not open")
            return

        if not isinstance(response, TimeRangeDataMessageResponse):
            handler.on_error("request not time range")
            return

        if response.time_duration <= 0:
            handler.on_error("negative time duration")
            return

        encoder = handler.new_encoder(response)

        if encoder is None:
            handler.on_error("no device/channel listen")
            return

        elif not isinstance(encoder, DataMessageEncoder):
            handler.on_error("too many device")
            return
        try:
            encoder.append_file(reader, response.time_start, response.time_stop)
        except Exception as e:
            print(e)

        message = encoder.message()

        # if message is not None:
        #     handler.on_message(message)