Files
controller-wisetopdataserver/python/biopro/file/util.py
T
2021-12-20 14:52:55 +08:00

861 lines
21 KiB
Python

from datetime import datetime
from functools import lru_cache
from pathlib import Path
from shutil import disk_usage
from typing import Union, Optional, Dict, List
from biopro.impl.mount import Partition
from biopro.recording import RecordingMetaFile, RecordingFile
from biopro.util.datetime import datetime_str
from biopro.util.json import JSON_OBJECT, JsonSerialize
from biopro.util.text import Table
from time import time
import os
FILE_LIST_ALL = False
TRASH_DIRECTORY = ".trash"
EXTERNAL_STORAGE_DIRECTORY = "external"
def path_start_with(target_path: Union[str, Path], from_path: Path) -> bool:
if isinstance(target_path, str):
target_path = Path(target_path)
try:
target_path.relative_to(from_path)
return True
except ValueError:
return False
def path_in_trash(path: Union[str, Path]) -> bool:
if isinstance(path, str):
path = Path(path)
return TRASH_DIRECTORY in path.parts
def path_in_external(path: Union[str, Path]) -> bool:
if isinstance(path, str):
path = Path(path)
try:
return EXTERNAL_STORAGE_DIRECTORY == path.parts[0]
except IndexError:
return False
def get_trashed_root(path: Union[str, Path]) -> Optional[Path]:
if isinstance(path, str):
path = Path(path)
if TRASH_DIRECTORY not in path.parts:
return None
else:
parts = list(path.parts)
try:
i = parts.index(TRASH_DIRECTORY)
except ValueError:
pass
else:
del parts[i + 1:]
return Path(*parts)
return None
def resolve_path_if_trashed(path: Union[str, Path]) -> Path:
if isinstance(path, str):
path = Path(path)
if path.exists():
return path
parts = path.parts
for i in range(len(parts) - 1, -1, -1):
tmp = list(parts)
tmp.insert(i, TRASH_DIRECTORY)
p = Path(*tmp)
if p.exists():
return p
return path
def resolve_path_un_trashed(path: Union[str, Path]) -> Path:
if isinstance(path, str):
path = Path(path)
if TRASH_DIRECTORY not in path.parts:
return path
else:
parts = list(path.parts)
try:
parts.remove(TRASH_DIRECTORY)
except ValueError:
pass
return Path(*parts)
@lru_cache(maxsize=32)
def _path_partition(path: Union[str, Path, Partition, 'BaseInfo']) -> 'PartitionInfo':
if isinstance(path, PartitionInfo):
return path
elif isinstance(path, RootInfo):
p = path.mount_partition
if p is None:
raise RuntimeError('Partition not mount : ' + str(path.file_path_origin))
elif isinstance(path, Partition):
p = path
elif isinstance(path, BaseInfo):
p = path.file_path_origin
else:
p = str(path)
if isinstance(p, str):
path = p
if p.startswith('/dev/'):
p = Partition.get_block_device(p)
else:
p = Partition.get_mount_point(p)
if p is None:
raise RuntimeError('Partition not mount : ' + path)
return PartitionInfo(p)
def filesize_str(byte_size: int) -> str:
if byte_size == 0:
return '0B'
u = 'B'
v = float(byte_size)
if v > 1024:
v /= 1024
u = 'KB'
if v > 1024:
v /= 1024
u = 'MB'
if v > 1024:
v /= 1024
u = 'GB'
if v > 1024:
v /= 1024
u = 'TB'
if v >= 100:
return '%.1f%s' % (v, u)
else:
return '%.2f%s' % (v, u)
class BaseInfo(JsonSerialize):
"""basic file information
**json format**
::
BaseInfo = {
"path" :path # file path, relate to storage root
"name" :str # file name with type symbol
"type" :file_type
"size" :int # file size in bytes
"create_time" :datetime
"last_modify" :datetime
"trashed" :bool # does file put into trash can?
"external" :bool # does file locate in external storage
"archived"? :bool # does file archived
} | FileInfo
| DirectoryInfo
| RootInfo
file_type
= "unknown"
| "directory" # directory
| "meta file" # recording meta file
| "data file" # recording data file
datetime :str
= datetime(YYYY,mm,dd,HH,MM.SS)
"""
TYPE_UNKNOWN = 'unknown'
TYPE_DIRECTORY = 'directory'
TYPE_META_FILE = 'meta file'
TYPE_DATA_FILE = 'data file'
TYPE_TAR_FILE = 'archive file'
TYPE_PARTITION = 'partition'
__slots__ = ('_root', '_path')
def __init__(self, root: Path, path: Path):
"""
:param root: storage root
:param path: file path
"""
self._root = root
self._path = path
@property
def root_path(self) -> Path:
return self._root
@property
def file_path(self) -> str:
if self._path == self._root:
return ''
elif self.is_trashed:
return str(self._path.relative_to(self._root)).replace(TRASH_DIRECTORY + '/', '')
else:
return str(self._path.relative_to(self._root))
@property
def parent_path(self) -> str:
if self._path == self._root:
return ''
elif self.is_trashed:
return str(self._path.parent.relative_to(self._root)).replace(TRASH_DIRECTORY + '/', '')
else:
return str(self._path.parent.relative_to(self._root))
@property
def file_path_origin(self) -> Path:
return self._path
@property
def file_name(self) -> str:
if self._path == self._root:
return '/'
ret = self._path.name
ftype = self.file_type
if ftype == self.TYPE_META_FILE:
return ret.replace(RecordingMetaFile.FILE_EXT, '')
elif ftype == self.TYPE_DATA_FILE:
return ret.replace(RecordingFile.FILE_EXT, '')
elif ftype == self.TYPE_DIRECTORY:
return ret + '/'
elif ftype == self.TYPE_TAR_FILE:
return ret.replace(RecordingMetaFile.TAR_FILE_EXT, '')
else:
return ret
@property
def file_type(self) -> str:
return self.TYPE_UNKNOWN
@property
def file_size(self) -> int:
# return self._path.stat().st_size
if self.file_type != self.TYPE_DIRECTORY:
return self._path.stat().st_size
else:
return 0
@property
def create_time(self) -> int:
return self._path.stat().st_ctime
@property
def last_modify(self) -> int:
return self._path.stat().st_mtime
@property
def is_trashed(self) -> bool:
return path_in_trash(self._path)
@property
def is_external(self) -> bool:
return path_in_external(self._path.relative_to(self._root))
@property
def is_archived(self) -> bool:
return self._path.name.endswith(RecordingMetaFile.TAR_FILE_EXT)
def as_json(self) -> JSON_OBJECT:
st = self._path.stat()
return {
'path': self.file_path,
'name': self.file_name,
'type': self.file_type,
'size': self.file_size,
'trashed': self.is_trashed,
'external': self.is_external,
'create_time': datetime_str(datetime.fromtimestamp(st.st_ctime)),
'last_modify': datetime_str(datetime.fromtimestamp(st.st_mtime)),
}
def as_json2(self) -> JSON_OBJECT:
st = self._path.stat()
return {
'path': self.file_path,
'name': self.file_name,
'type': self.file_type,
'size': self.file_size,
'trashed': self.is_trashed,
'external': self.is_external,
'create_time': datetime_str(datetime.fromtimestamp(st.st_ctime)),
'last_modify': datetime_str(datetime.fromtimestamp(st.st_mtime)),
}
def print(self, detail_mode=False, file=None):
t = self._new_print_table(detail_mode)
self._print_table(t, detail_mode=detail_mode)
t.print(file=file)
@classmethod
def _new_print_table(cls, detail_mode: bool):
t = Table(column=4 if detail_mode else 2)
if detail_mode:
t.set_format(0, align_right=False, padding='', split='')
t.set_format(1, align_right=True)
t.set_format(2, align_right=True)
t.set_format(3, align_right=False)
else:
t.set_format(0, align_right=False, padding='', split='')
t.set_format(1, align_right=False)
return t
def _print_table(self, table: Table, detail_mode=False):
d = 'd' if self.file_type == self.TYPE_DIRECTORY else '-'
if self.file_type == self.TYPE_META_FILE:
f = 'm'
elif self.file_type == self.TYPE_DATA_FILE:
f = 'f'
elif self.file_type == self.TYPE_TAR_FILE:
f = 'a'
else:
f = '-'
if self.is_trashed:
x = 'x'
elif self.is_external:
x = 'e'
else:
x = '-'
if detail_mode:
table.append(d + f + x,
filesize_str(self.file_size),
str(datetime.fromtimestamp(self.last_modify)),
self._path.name)
else:
table.append(d + f + x, self._path.name)
def __hash__(self):
return hash(self._path)
def __eq__(self, other):
if self is other:
return True
elif other is None:
return False
elif isinstance(other, BaseInfo):
return self._path == other._path and self.file_type == other.file_type
else:
return False
class FileInfo(BaseInfo):
"""
**json format**
::
FileInfo (BaseInfo) = {
# when "type" = "meta file"
"meta"? = RecordingMetaFile
"export"? = {
export_type = path: str?
}
}
"""
CACHE = {}
__slots__ = '_addition', '_file_type', '_meta', '_export'
def __init__(self, root: Path, path: Path):
"""
:param root: storage root
:param path: file path
:raises FileNotFoundError: if *path* not exist
"""
super().__init__(root, path)
self._addition = {}
if not path.exists():
raise FileNotFoundError(str(path.relative_to(root)))
elif path.is_dir():
self._file_type = self.TYPE_DIRECTORY
elif RecordingMetaFile.is_recording_meta_file(path):
self._file_type = self.TYPE_META_FILE
self._meta = None
self._export = {}
elif RecordingFile.is_recording_file(path):
self._file_type = self.TYPE_DATA_FILE
elif path.name.endswith(RecordingMetaFile.TAR_FILE_EXT):
self._file_type = self.TYPE_TAR_FILE
else:
self._file_type = self.TYPE_UNKNOWN
@property
def file_type(self) -> str:
return self._file_type
@property
def is_meta_file(self) -> bool:
return self._file_type == self.TYPE_META_FILE
@property
def is_directory(self) -> bool:
return self._file_type == self.TYPE_DIRECTORY
@property
def meta_file(self) -> Optional[RecordingMetaFile]:
if hasattr(self, '_meta'):
meta = self._meta
if self._meta is None:
self._meta = meta = RecordingMetaFile.load(self._path)
elif meta.is_modified:
meta.reload()
return meta
else:
return None
@meta_file.setter
def meta_file(self, value: RecordingMetaFile):
if hasattr(self, '_meta'):
self._meta = value
@property
def export(self) -> Dict[str, Optional[Path]]:
if hasattr(self, '_export'):
return self._export
else:
return {}
def add_export_path(self, export_type: str, export_path: Optional[Path]):
"""
:param export_type: export file type
:param export_path: export relative path
"""
if hasattr(self, '_export'):
self._export[export_type] = export_path
def del_export_path(self, export_type: str):
if hasattr(self, '_export'):
del self._export[export_type]
@property
def file_size(self) -> int:
"""file size. If flag ``FILE_LIST_ALL`` is ``False``, the file size of a meta file will include
all relate data files, or a directory will include the file size of its contents.
:return: file size in bytes.
"""
s = 0
if self._file_type == self.TYPE_DIRECTORY:
return 0
elif self._file_type == self.TYPE_META_FILE:
try:
s = self._path.stat().st_size
s = self.meta_file._size
except IOError:
return 0
return s
# noinspection PyTypeChecker
def as_json(self) -> JSON_OBJECT:
print('*****start3', time(), self.file_type)
# if self.file == self.TYPE_META_FILE:
# print(self.meta_file().as_json())
ret = super().as_json()
print('*****end3', time())
ret['archived'] = self.is_archived
ret.update(self._addition)
# if self.file == self.TYPE_META_FILE:
# if self._meta is None:
# self._meta = meta = RecordingMetaFile.load(self._path)
# elif meta.is_modified:
# meta.reload()
# if self._meta is not None:
# ret['meta'] = meta.as_json()
if hasattr(self, '_meta'):
meta = self._meta
if meta is not None:
print('*****start4', time())
ret['meta'] = meta.as_json()
print('*****end4', time())
ret['create_time'] = datetime_str(datetime.fromtimestamp( int(str(ret['meta']['create_time'])[:-3]) ))
ret['export'] = JsonSerialize.to_json(self.export)
return ret
def as_json2(self) -> JSON_OBJECT:
# print('*****start3', time(), self.file_type)
ret = super().as_json2()
# print('*****end3', time())
ret['archived'] = self.is_archived
ret.update(self._addition)
return ret
@classmethod
def get_info(cls, root: Path, path: Path) -> 'FileInfo':
"""factory method. it will cache the result.
:param root: storage root
:param path: file absolute path
:return: file information.
"""
if path not in cls.CACHE:
# print('*****1', str(cls.CACHE))
ret = FileInfo(root, path)
cls.CACHE[path] = ret
else:
# print('*****2', str(cls.CACHE))
ret = cls.CACHE[path]
# ret = FileInfo(root, path)
return ret
@classmethod
def list_info(cls, root: Path, path: Path) -> List['FileInfo']:
"""list the content if *path* is a directory.
:param root: storage root
:param path: directory path
:return: list of file information,
:raises NotADirectoryError: if *path* not a directory
"""
ret = []
for p in path.iterdir():
name = p.name
if name == TRASH_DIRECTORY:
ret.extend(cls.list_info(root, p))
continue
elif name.startswith('.'):
# ignore hidden files
continue
f = cls.get_info(root, p)
if FILE_LIST_ALL:
ret.append(f)
elif f.file_type in (FileInfo.TYPE_DIRECTORY, FileInfo.TYPE_META_FILE, FileInfo.TYPE_TAR_FILE):
ret.append(f)
ret = sorted(ret, key=lambda f: f.create_time)
return ret
@classmethod
def invalid_cache(cls, path: Path):
"""invalid cache at/under the *path*"""
for p in list(cls.CACHE.keys()):
if p == path or path_start_with(p, path):
del cls.CACHE[p]
@classmethod
def invalid_cache_all(cls):
"""invalid all cache"""
cls.CACHE.clear()
class DirectoryInfo(BaseInfo):
"""
**json format**
::
DirectoryInfo (BaseInfo) = {
"type" = "directory"
"list" : [FileInfo]
}
"""
__slots__ = ('_list',)
def __init__(self, root: Path, path: Union[str, Path]):
"""
:param root: storage root
:param path: directory path
:raises NotADirectoryError: if *path* not a directory
"""
if isinstance(path, str):
path = Path(path)
if not path.is_dir():
raise NotADirectoryError(str(path.relative_to(root)))
super().__init__(root, path)
# print('*****start1', time())
self._list = FileInfo.list_info(root, path)
# print('*****end1', time(), self._list)
@property
def file_type(self) -> str:
return self.TYPE_DIRECTORY
@property
def file_list(self) -> List[FileInfo]:
"""
:return: file information list under this directory
"""
return self._list
def as_json(self) -> JSON_OBJECT:
ret = super().as_json()
# noinspection PyTypeChecker
# print('*****start2', time())
ret['list'] = [f.as_json2() for f in self._list]
# print('*****end2', time())
return ret
def print(self, detail_mode=False, file=None):
print('/' + self.file_path + ' :', file=file)
t = self._new_print_table(detail_mode)
for f in sorted(self._list, key=lambda f: f.file_name):
f._print_table(t, detail_mode=detail_mode)
t.print(file=file)
class RootInfo(DirectoryInfo):
"""
**json format**
::
RootInfo (DirectoryInfo) = {
"volume_total" :int # total volume in bytes
"volume_usage" :int # used volume in bytes
"volume_free" :int # free volume in bytes
"mount_point": str?
"mount_device": str?
}
"""
__slots__ = ('_partition',)
def __init__(self, root: Path):
"""
:param root: storage root
:raises NotADirectoryError: if *root* not a directory
"""
super().__init__(root, root)
self._partition = Partition.get_mount_point(root)
@property
def volume_total(self) -> int:
"""disk total volume in bytes"""
total, _, _ = disk_usage(str(self._path))
return total
@property
def volume_usage(self) -> int:
"""disk used volume in bytes"""
_, usage, _ = disk_usage(str(self._path))
return usage
@property
def volume_free(self) -> int:
"""disk free volume in bytes"""
_, _, free = disk_usage(str(self._path))
return free
@property
def mount_point(self) -> Optional[Path]:
p = self._partition
if p is not None:
m = p.mount_path
if m is not None:
return Path(m)
return None
@property
def mount_device(self) -> Optional[str]:
p = self._partition
if p is not None:
return p.fs_path
else:
return None
@property
def mount_partition(self) -> Optional[Partition]:
return self._partition
# noinspection PyTypeChecker
def as_json(self) -> JSON_OBJECT:
ret = super().as_json()
total, usage, free = disk_usage(str(self._path))
p = self._partition
if p is not None:
mp = p.mount_path
md = p.fs_path
else:
mp = None
md = None
ret.update({
'volume_total': total,
'volume_usage': usage,
'volume_free': free,
'mount_point': mp,
'mount_device': md,
})
return ret
class PartitionInfo(BaseInfo):
"""partition information
**json format**
::
PartitionInfo (BaseInfo) = {
"mount_point": str?
"mount_device": str?
}
"""
__slots__ = ('_partition',)
def __init__(self, partition: Partition):
p = Path(partition.fs_path)
super().__init__(p, p)
self._partition = partition
@property
def partition(self) -> Partition:
return self._partition
@property
def file_path(self) -> str:
m = self._partition.mount_path
if m is not None:
return m + '/'
else:
return self._partition.fs_path + '/'
@property
def file_type(self) -> str:
return self.TYPE_PARTITION
@property
def is_trashed(self) -> bool:
return False
def as_json(self) -> JSON_OBJECT:
ret = super().as_json()
ret.update({
'mount_point': self._partition.mount_path,
'mount_device': self._partition.fs_path,
})
return ret
def print(self, detail_mode=False, file=None):
print(self._partition, file=file)
@classmethod
def get_partition(cls, path: Union[str, Path, Partition, BaseInfo]) -> 'PartitionInfo':
return _path_partition(path)