peterlu14
746 lines
23 KiB
Python
746 lines
23 KiB
Python
"""data analysis module.
|
|
|
|
This module is some kinds of out of date and less maintain.
|
|
|
|
"""
|
|
import abc
|
|
from collections import Counter, deque, defaultdict
|
|
from datetime import datetime
|
|
from statistics import mean, median, pstdev
|
|
from typing import Set, List, Iterable, Tuple, Dict, Optional
|
|
|
|
from .cli import *
|
|
from .console import hex_value
|
|
from .iter import zip_next2, flat_iter, counter_diff, counter_total, counter_product_sum
|
|
from .text import list_padding, Table
|
|
|
|
try:
|
|
# noinspection PyStatementEffect
|
|
datetime.fromisoformat
|
|
|
|
|
|
def _from_iso_format(expr: str) -> datetime:
|
|
return datetime.fromisoformat(expr)
|
|
|
|
except AttributeError:
|
|
import re
|
|
|
|
|
|
def _from_iso_format(expr: str) -> datetime:
|
|
m = re.match(r'(\d{4})-(\d{2})-(\d{2}) *(?:(\d{2})(?::(\d{2})(?::(\d{2})(?:\.(\d{3,6}))?)?)?)?', expr)
|
|
|
|
if not m:
|
|
raise ValueError()
|
|
|
|
dy = int(m.group(1))
|
|
dm = int(m.group(2))
|
|
dd = int(m.group(3))
|
|
th = int(m.group(4))
|
|
tm = int(m.group(5))
|
|
ts = int(m.group(6))
|
|
tf = int(m.group(7))
|
|
|
|
return datetime(dy, dm, dd, th, tm, ts, tf)
|
|
|
|
|
|
def _percent(a, b) -> str:
|
|
return '%.2f%%' % (100 * a / b)
|
|
|
|
|
|
def _diff_continuous_data(it, dimension: int = 1, overflow: Optional[int] = None) -> Counter:
|
|
ret = Counter(map(lambda v: v[1] - v[0], zip_next2(it, dimension - 1)))
|
|
|
|
if overflow is not None:
|
|
ret[1] += ret[-overflow + 1]
|
|
del ret[-overflow + 1]
|
|
|
|
return ret
|
|
|
|
|
|
def _max_continuous_data(it, dimension: int = 1, overflow: Optional[int] = None) -> Counter:
|
|
ret = Counter()
|
|
|
|
s = 0
|
|
for a, b in zip_next2(it, dimension - 1):
|
|
if b - a == 1 or a - b + 1 == overflow:
|
|
s += 1
|
|
else:
|
|
ret[s] += 1
|
|
s = 0
|
|
|
|
return ret
|
|
|
|
|
|
# noinspection PyUnusedLocal
|
|
class Main(CliMain):
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
|
|
@cli_flags('-h', '--help', force_return=True)
|
|
def _help(self, opt: str):
|
|
"""print help document"""
|
|
self.print_help()
|
|
|
|
@cli_command('txt')
|
|
def _txt_data_command(self, cmd: str, argv: List[str]):
|
|
"""analysis txt data file"""
|
|
return _TextDataCommand
|
|
|
|
@cli_command("dump")
|
|
def _dump_data_command(self, cmd: str, argv: List[str]):
|
|
"""analysis sync data dump file"""
|
|
return _DumpDataCommand
|
|
|
|
def run(self):
|
|
self.print_help()
|
|
|
|
|
|
# noinspection PyUnusedLocal
|
|
class _AbstractDataCommand(CliSubCommandMain, metaclass=abc.ABCMeta):
|
|
def __init__(self, command: str):
|
|
super().__init__(command)
|
|
|
|
self.data_file: str = None
|
|
|
|
# flag and options
|
|
self.ramp_data = False
|
|
self.avg_data = False
|
|
|
|
self.overview = True
|
|
self.chip_id_table = False
|
|
self.time_diff_table = False
|
|
|
|
self.time_diff_threshold = 10
|
|
self.ramp_data_threshold = 10
|
|
|
|
self.avg_data_time_step = 10
|
|
self.avg_data_duration = 100
|
|
self.avg_data_bin = 10
|
|
|
|
@cli_flags('-h', '--help', force_return=True)
|
|
def _help(self, opt: str):
|
|
"""print help document"""
|
|
self.print_help()
|
|
|
|
@cli_flags('--ramp')
|
|
def _ramp_data(self, opt: str):
|
|
"""analysis ramp data"""
|
|
self.ramp_data = True
|
|
|
|
@cli_flags('--avg', '--average')
|
|
def _avg_data(self, opt: str):
|
|
"""average data"""
|
|
self.avg_data = True
|
|
|
|
@cli_flags('--no-overview')
|
|
def _no_overview(self, opt: str):
|
|
"""do not print overview table"""
|
|
self.overview = False
|
|
|
|
@cli_flags('--chip')
|
|
def _chip_id(self, opt: str):
|
|
"""chip ID table"""
|
|
self.chip_id_table = True
|
|
|
|
@cli_flags('--time-diff')
|
|
def _time_diff_table(self, opt: str):
|
|
"""time diff table"""
|
|
self.time_diff_table = True
|
|
|
|
@cli_options('--time-diff-threshold', value='VALUE')
|
|
def _time_diff_threshold(self, opt: str, value: str):
|
|
"""the threshold to print address in time diff table"""
|
|
self.time_diff_threshold = int(value)
|
|
|
|
@cli_options('--ramp-threshold', value='VALUE')
|
|
def _ramp_data_threshold(self, opt: str, value: str):
|
|
"""the threshold to print address in ramp data table"""
|
|
self.ramp_data_threshold = int(value)
|
|
|
|
@cli_options('--average-time-step', value='VALUE')
|
|
def _avg_data_time_step(self, opt: str, value: str):
|
|
"""the time step for avg data"""
|
|
self.avg_data_time_step = int(value)
|
|
|
|
@cli_options('--average-duration', value='VALUE')
|
|
def _avg_data_duration(self, opt: str, value: str):
|
|
"""the duration to move average for avg data"""
|
|
self.avg_data_duration = int(value)
|
|
|
|
@cli_options('--average-bin', value='VALUE')
|
|
def _avg_data_bin(self, opt: str, value: str):
|
|
"""the bin for avg data"""
|
|
self.avg_data_bin = int(value)
|
|
|
|
@cli_arguments(0, value='FILE')
|
|
def _dump_data_file(self, pos: int, value: str):
|
|
"""set configuration ny recording file"""
|
|
self.data_file = value
|
|
|
|
def run(self):
|
|
if self.data_file is None:
|
|
raise RuntimeError('lost FILE')
|
|
|
|
self.run_collect()
|
|
|
|
if self.overview:
|
|
self.run_table_overview()
|
|
|
|
if self.chip_id_table:
|
|
print()
|
|
self.run_table_chip_id()
|
|
|
|
if self.time_diff_table:
|
|
print()
|
|
self.run_table_time_diff()
|
|
|
|
if self.ramp_data:
|
|
print()
|
|
self.run_table_ramp_data()
|
|
elif self.avg_data:
|
|
print()
|
|
self.run_table_avg_data()
|
|
|
|
@abc.abstractmethod
|
|
def run_collect(self):
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def run_table_overview(self):
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def chip_id_list(self) -> Iterable[int]:
|
|
pass
|
|
|
|
def run_table_chip_id(self):
|
|
counter = Counter(self.chip_id_list())
|
|
total = counter_total(counter)
|
|
|
|
print('chip id, total', total)
|
|
|
|
table = Table('ID', 'count', '(count%)')
|
|
for chip, count in sorted(counter.items(), key=lambda it: it[0]):
|
|
table.append(str(chip), str(count), _percent(count, total))
|
|
|
|
table.set_format(1, split=' : ')
|
|
table.print(' ')
|
|
|
|
@abc.abstractmethod
|
|
def data_time_list(self) -> Iterable[float]:
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def data_time_address(self, delta: float) -> str:
|
|
pass
|
|
|
|
def run_table_time_diff(self):
|
|
time_list = list(self.data_time_list())
|
|
time_diff = list(map(lambda v: round(v, 2),
|
|
map(counter_diff, zip_next2(time_list, 0))))
|
|
time_diff_counter = Counter(time_diff)
|
|
total = counter_total(time_diff_counter)
|
|
|
|
print('time diff [int, ms]', 'total', total)
|
|
|
|
table = Table('diff', 'count', 'count%', 'address (line)')
|
|
for k, v in sorted(time_diff_counter.items(), key=lambda it: abs(it[0])):
|
|
if abs(k) < self.time_diff_threshold:
|
|
a = ''
|
|
else:
|
|
a = self.data_time_address(k)
|
|
|
|
table.append(k, v, _percent(v, total), a)
|
|
|
|
table.set_format(1, split=' : ')
|
|
table.set_format(3, align_right=False)
|
|
|
|
table.print(' ')
|
|
|
|
@abc.abstractmethod
|
|
def data_value_list(self) -> Iterable[float]:
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def data_value_address(self, delta: float) -> str:
|
|
pass
|
|
|
|
def run_table_ramp_data(self):
|
|
counter = _diff_continuous_data(self.data_value_list(), overflow=1024)
|
|
|
|
product = counter_product_sum(counter)
|
|
missing = counter_product_sum(counter, key_mapper=lambda k: k - 1)
|
|
total = counter_total(counter)
|
|
|
|
print('data content, total', product,
|
|
', missing', missing,
|
|
', rate', _percent(missing, product))
|
|
|
|
table = Table('gap', 'count', 'count%', 'total%', 'address (line)')
|
|
|
|
for k, v in sorted(counter.items(), key=lambda it: abs(it[0])):
|
|
if abs(k) < self.ramp_data_threshold:
|
|
a = ''
|
|
else:
|
|
a = self.data_value_address(k)
|
|
|
|
table.append(k, v, _percent(v, total), _percent(k * v, product), a)
|
|
|
|
table.set_format(1, split=' : ')
|
|
table.set_format(4, align_right=False)
|
|
|
|
table.print(' ')
|
|
|
|
#
|
|
print()
|
|
|
|
counter = _max_continuous_data(self.data_value_list(), overflow=1024)
|
|
total = counter_total(counter)
|
|
product = counter_product_sum(counter)
|
|
print('max continuous data',
|
|
', total', product)
|
|
|
|
table = Table('block size', 'count', 'count%', 'total%')
|
|
for k, v in sorted(counter.items(), key=lambda it: it[0], reverse=True):
|
|
table.append(k, v, _percent(v, total), _percent(k * v, product))
|
|
|
|
table.set_format(1, split=' : ')
|
|
|
|
table.print(' ')
|
|
|
|
def run_table_avg_data(self):
|
|
|
|
counter = Counter()
|
|
ls = deque()
|
|
|
|
time_next = None
|
|
|
|
for time, data in zip(self.data_time_list(), self.data_value_list()):
|
|
|
|
if time_next is None:
|
|
ls.append((time, data))
|
|
time_next = time + self.avg_data_time_step
|
|
|
|
elif time < time_next:
|
|
ls.append((time, data))
|
|
continue
|
|
|
|
else:
|
|
old_time = time - self.avg_data_duration
|
|
|
|
try:
|
|
while True:
|
|
if ls[0][0] < old_time:
|
|
ls.popleft()
|
|
else:
|
|
break
|
|
except IndexError:
|
|
pass
|
|
|
|
else:
|
|
if len(ls):
|
|
counter[int(sum(map(lambda it: it[1], ls)) / len(ls) / self.avg_data_bin)] += 1
|
|
|
|
ls.append((time, data))
|
|
time_next += self.avg_data_time_step
|
|
|
|
total = counter_total(counter)
|
|
|
|
print('data average global :', round(mean(self.data_value_list()), 4),
|
|
', total segment :', total,
|
|
', average duration :', self.avg_data_duration, '+', self.avg_data_time_step, 'ms')
|
|
|
|
table = Table('avg range', 'count', 'count%')
|
|
|
|
for k, v in sorted(counter.items(), key=lambda it: -abs(it[1])):
|
|
table.append('[%d, %d)' % (k * self.avg_data_bin, (k + 1) * self.avg_data_bin), v, _percent(v, total))
|
|
|
|
table.set_format(1, split=' : ')
|
|
|
|
table.print(' ')
|
|
|
|
|
|
# noinspection PyUnusedLocal
|
|
class _DumpDataCommand(_AbstractDataCommand):
|
|
def __init__(self, command: str):
|
|
super().__init__(command)
|
|
|
|
self.time_diff_threshold = 1000
|
|
self.ramp_data_threshold = 100
|
|
|
|
# data
|
|
self.sync_count = 0
|
|
self.byte_total = 0
|
|
self.byte_invalid = 0
|
|
self.time_diff = []
|
|
self.group_count: List[int] = []
|
|
self.group_size: List[int] = []
|
|
self.byte_count: List[int] = []
|
|
self.counter_list: List[List[int]] = []
|
|
self._chip_id_list: List[List[int]] = []
|
|
self._data_list: List[List[List[int]]] = []
|
|
self._data_time_list: List[List[float]] = []
|
|
|
|
def run_collect(self):
|
|
with open(self.data_file) as dump:
|
|
self.sync_count = 0
|
|
self.byte_invalid = 0
|
|
self.byte_total = 0
|
|
self.time_diff = []
|
|
self.group_count: List[int] = []
|
|
self.group_size: List[int] = []
|
|
self.byte_count: List[int] = []
|
|
self.counter_list: List[List[int]] = []
|
|
self._chip_id_list: List[List[int]] = []
|
|
self._data_list: List[List[List[int]]] = []
|
|
self._data_time_list: List[List[float]] = []
|
|
|
|
time_stamp = None
|
|
number = 0
|
|
|
|
try:
|
|
for number, line in enumerate(dump):
|
|
line = line.strip()
|
|
|
|
if line.startswith('#'):
|
|
prev_time = time_stamp
|
|
time_stamp = _from_iso_format(line[2:].strip())
|
|
|
|
if prev_time is not None:
|
|
delta = time_stamp - prev_time
|
|
self.time_diff.append(delta.total_seconds() * 1000 + 0.001 * delta.microseconds)
|
|
|
|
else:
|
|
self.sync_count += 1
|
|
data = line.split(' ')
|
|
self.byte_total += len(data)
|
|
self.byte_count.append(len(data))
|
|
|
|
group = self._parsing_group(data)
|
|
if len(group) > 0:
|
|
self.group_count.append(len(group))
|
|
self.group_size.extend(map(len, group))
|
|
else:
|
|
self.byte_invalid += len(data)
|
|
|
|
self.counter_list.append(self._parsing_counter(group))
|
|
self._chip_id_list.append(self._parsing_chip_id(group))
|
|
self._data_list.append(self._parsing_data(group))
|
|
self._data_time_list.append(self._parsing_data_time(group))
|
|
except BaseException as e:
|
|
raise RuntimeError('line ' + str(number + 1), ':', *e.args) from e
|
|
|
|
def run_table_overview(self):
|
|
|
|
print('sync count', self.sync_count, ', total byte', self.byte_total, ', invalid', self.byte_invalid)
|
|
print()
|
|
|
|
data_time_diff = _diff_continuous_data(self._data_time_list, dimension=2)
|
|
data_time_int_diff = list(map(counter_diff,
|
|
map(lambda it: (int(it[0]), int(it[1])),
|
|
zip_next2(self._data_time_list, 1))))
|
|
|
|
table = [
|
|
'',
|
|
'msec / data [f]',
|
|
'msec / data [i]',
|
|
'msec / sync',
|
|
'group / sync',
|
|
'bytes / group',
|
|
'bytes / sync',
|
|
]
|
|
list_padding(table)
|
|
data = [data_time_diff, data_time_int_diff, self.time_diff, self.group_count, self.group_size, self.byte_count]
|
|
|
|
add = ['mean']
|
|
add.extend(map(lambda it: '%.2f' % it, map(mean, data)))
|
|
list_padding(table, add, align_right=True)
|
|
list_padding(table)
|
|
|
|
add = ['median']
|
|
add.extend(map(lambda it: '%.2f' % it, map(median, data)))
|
|
list_padding(table, add, align_right=True)
|
|
list_padding(table)
|
|
|
|
add = ['min']
|
|
add.extend(map(lambda it: '%.2f' % it, map(min, data)))
|
|
list_padding(table, add, align_right=True)
|
|
list_padding(table)
|
|
|
|
add = ['max']
|
|
add.extend(map(lambda it: '%.2f' % it, map(max, data)))
|
|
list_padding(table, add, align_right=True)
|
|
list_padding(table)
|
|
|
|
add = ['std']
|
|
add.extend(map(lambda it: '%.2f' % it, map(pstdev, data)))
|
|
list_padding(table, add, align_right=True)
|
|
list_padding(table)
|
|
for line in table:
|
|
print(line)
|
|
|
|
def chip_id_list(self) -> Iterable[int]:
|
|
return flat_iter(self._chip_id_list)
|
|
|
|
def _run_table_counter(self):
|
|
miss_counter = _diff_continuous_data(self.counter_list, dimension=2, overflow=256)
|
|
|
|
product = counter_product_sum(miss_counter) + 1
|
|
missing = counter_product_sum(miss_counter, key_mapper=lambda k: k - 1)
|
|
total = counter_total(miss_counter)
|
|
|
|
print('header counter, total', product,
|
|
', missing', missing,
|
|
', rate', _percent(missing, product))
|
|
|
|
table = Table('gap', 'count', 'count%', 'total%', 'address (line:group)')
|
|
for k, v in sorted(miss_counter.items(), key=lambda it: (it[1], -it[0]), reverse=True):
|
|
if k == 1:
|
|
a = ''
|
|
else:
|
|
a = ' '.join(map(lambda it: '%d:%d' % it, self._find_diff2(self.counter_list, k)))
|
|
|
|
table.append(k, v, _percent(v, total), _percent(k * v, product), a)
|
|
|
|
table.set_format(1, split=' : ')
|
|
table.set_format(4, align_right=False)
|
|
|
|
table.print(' ')
|
|
|
|
def data_time_list(self) -> Iterable[float]:
|
|
return flat_iter(self._data_time_list)
|
|
|
|
def data_time_address(self, delta: float) -> str:
|
|
return ' '.join(map(lambda it: '%d:%d' % it, self._find_diff2(self._data_time_list, delta, 2)))
|
|
|
|
def data_value_list(self) -> Iterable[float]:
|
|
return flat_iter(flat_iter(self._data_list))
|
|
|
|
def data_value_address(self, delta: float) -> str:
|
|
return ' '.join(map(lambda it: '%d:%d+%d' % it, self._find_diff3(self._data_list, delta)))
|
|
|
|
@staticmethod
|
|
def _parsing_group(data: List[str]) -> List[List[str]]:
|
|
length = len(data)
|
|
|
|
ret = []
|
|
|
|
i = 0
|
|
|
|
while i < length:
|
|
header = data[i]
|
|
if header != 'FF':
|
|
break
|
|
else:
|
|
size = hex_value(data[i + 2])
|
|
ret.append(data[i:i + size + 3])
|
|
i += 3 + size
|
|
|
|
return ret
|
|
|
|
@staticmethod
|
|
def _parsing_counter(data: List[List[str]]) -> List[int]:
|
|
return list(map(lambda g: hex_value(g[1]), data))
|
|
|
|
@staticmethod
|
|
def _parsing_chip_id(data: List[List[str]]) -> List[int]:
|
|
return list(map(lambda g: hex_value(g[3]), data))
|
|
|
|
@staticmethod
|
|
def _parsing_data(data: List[List[str]]) -> List[List[int]]:
|
|
ret = []
|
|
|
|
for group in data:
|
|
tmp = []
|
|
for i in range(9, len(group) - 1, 2):
|
|
value = ((hex_value(group[i]) & 0b0000_1111) << 6) | ((hex_value(group[i + 1]) & 0b1111_1100) >> 2)
|
|
flag = hex_value(group[i + 1]) & 0b0011
|
|
|
|
if flag == 0:
|
|
tmp.append(value - 512)
|
|
|
|
ret.append(tmp)
|
|
|
|
return ret
|
|
|
|
@staticmethod
|
|
def _parsing_data_time(data: List[List[str]]) -> List[float]:
|
|
ret = []
|
|
|
|
for group in data:
|
|
v0 = hex_value(group[5])
|
|
v1 = hex_value(group[6])
|
|
v2 = hex_value(group[7])
|
|
v3 = hex_value(group[8])
|
|
|
|
value = v0 | (v1 << 8) | (v2 << 16) | (v3 << 24)
|
|
|
|
ret.append(value / 32)
|
|
|
|
return ret
|
|
|
|
@staticmethod
|
|
def _find_diff2(ls: List[List[float]], diff_value: float, n=2) -> Iterable[Tuple[int, int]]:
|
|
prev = None
|
|
|
|
for i, group in enumerate(ls):
|
|
for j, value in enumerate(group):
|
|
if prev is not None and round(value - prev, n) == round(diff_value, n):
|
|
yield 2 * (i + 1), j + 1
|
|
prev = value
|
|
|
|
@staticmethod
|
|
def _find_diff3(ls: List[List[List[float]]], diff_value: float, n=2) -> Iterable[Tuple[int, int, int]]:
|
|
prev = None
|
|
|
|
for i, row in enumerate(ls):
|
|
for j, frame in enumerate(row):
|
|
for k, value in enumerate(frame):
|
|
if prev is not None and round(value - prev, n) == round(diff_value, n):
|
|
yield 2 * (i + 1), j + 1, k + 1
|
|
prev = value
|
|
|
|
|
|
# noinspection PyUnusedLocal
|
|
class _TextDataCommand(_AbstractDataCommand):
|
|
def __init__(self, command: str):
|
|
super().__init__(command)
|
|
|
|
self.error_line_table = False
|
|
|
|
#
|
|
self.start_address = None
|
|
self.data_count = 0
|
|
self.time_list: List[int] = []
|
|
self._chip_id_list: List[int] = []
|
|
# self.channel_counter: Dict[int, Counter] = {}
|
|
self._data_list: List[int] = []
|
|
self.comment = []
|
|
self.error_line: Dict[int, Set[str]] = {}
|
|
|
|
@cli_flags('--error-line')
|
|
def _error_line_table(self, opt: str):
|
|
"""error line table"""
|
|
self.error_line_table = True
|
|
|
|
def run(self):
|
|
super().run()
|
|
|
|
if self.error_line_table:
|
|
print()
|
|
self._run_error_line_table()
|
|
|
|
def run_collect(self):
|
|
with open(self.data_file) as dump:
|
|
self.data_count = 0
|
|
self.time_list = []
|
|
self._chip_id_list = []
|
|
# self.channel_counter = defaultdict(Counter)
|
|
self._data_list = []
|
|
self.comment = []
|
|
self.error_line = defaultdict(set)
|
|
|
|
try:
|
|
for number, line in enumerate(dump):
|
|
line = line.strip()
|
|
|
|
if len(line) == 0:
|
|
pass
|
|
|
|
elif line.startswith('#'):
|
|
self.comment.append(line)
|
|
|
|
else:
|
|
if self.start_address is None:
|
|
self.start_address = number
|
|
|
|
self.data_count += 1
|
|
|
|
data = line.split(' ')
|
|
t = float(data[0])
|
|
d = int(data[1])
|
|
c = int(data[2])
|
|
v = float(data[3])
|
|
|
|
self.time_list.append(t)
|
|
self._chip_id_list.append(d)
|
|
# self.channel_counter[d][c] += 1
|
|
self._data_list.append(v)
|
|
|
|
except BaseException as e:
|
|
raise RuntimeError('line ' + str(number + 1), ':', *e.args) from e
|
|
|
|
def run_table_overview(self):
|
|
for comment in self.comment:
|
|
print(comment)
|
|
|
|
print('total', self.data_count)
|
|
|
|
def chip_id_list(self) -> Iterable[int]:
|
|
return self._chip_id_list
|
|
|
|
def data_time_list(self) -> Iterable[float]:
|
|
return self.time_list
|
|
|
|
def data_time_address(self, delta: float) -> str:
|
|
error_line = list(map(lambda it: '%d' % it, self._find_diff2(self.time_list, delta, n=2)))
|
|
ret = ' '.join(map(lambda it: '%d' % it, error_line))
|
|
|
|
if self.error_line_table:
|
|
for line in error_line:
|
|
self.error_line[line].add('time_diff')
|
|
|
|
return ret
|
|
|
|
def data_value_list(self) -> Iterable[float]:
|
|
return self._data_list
|
|
|
|
def data_value_address(self, delta: float) -> str:
|
|
error_line = list(self._find_diff2(self._data_list, delta))
|
|
ret = ' '.join(map(lambda it: '%d' % it, error_line))
|
|
|
|
if self.error_line_table:
|
|
for line in error_line:
|
|
self.error_line[line].add('data_diff')
|
|
|
|
return ret
|
|
|
|
def _run_error_line_table(self):
|
|
error_type_set = list(set(flat_iter(self.error_line.values())))
|
|
error_line: Dict[str, List[int]] = defaultdict(list)
|
|
|
|
for line, error_type in self.error_line.items():
|
|
a = ' '.join(map(lambda e: e if e in error_type else ' ' * len(e), error_type_set))
|
|
error_line[a].append(line)
|
|
|
|
total = sum(map(len, error_line.values()))
|
|
|
|
print('error type',
|
|
', total', total,
|
|
', total lines', self.data_count,
|
|
', rate', _percent(total, self.data_count))
|
|
table = Table('error_type', 'count', 'count%', 'line')
|
|
|
|
for k, v in sorted(error_line.items(), key=lambda it: len(it[1])):
|
|
table.append(k, len(v),
|
|
_percent(len(v), total),
|
|
' '.join(map(str, sorted(v))))
|
|
|
|
table.set_format(1, split=' : ')
|
|
table.set_format(3, align_right=False)
|
|
|
|
table.print(' ')
|
|
|
|
def _find_diff2(self, ls: List[int], diff_value: float, n=0) -> Iterable[int]:
|
|
prev = None
|
|
|
|
for i, value in enumerate(ls):
|
|
if prev is not None and round(value - prev, n) == diff_value:
|
|
yield i + self.start_address
|
|
prev = value
|
|
|
|
|
|
if __name__ == '__main__':
|
|
Main().main()
|