usbmeter-utils/csvproc.py at master · didim99/usbmeter-utils · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
#! /usr/bin/python3

import os

from common import str2ts, ts2str, lines_count

src_dir = "src"
out_dir = "out"
src_ext = ".csv"
default_div = ','
time_div = ':'

reformat = True
add_header = True
decimal_div = ','
value_div = ';'

resample = True
resample_rate = 100     # 2, 5, 10, 20, 50, 100, ...
resample_mode = 'avg'   # start, end, center, avg


def num_fields(point):
    fields = []
    for field in point:
        if type(point[field]) in [int, float]:
            fields.append(field)
    return fields


def resample_avg(data):
    res = {}
    fields = num_fields(data[0])
    for field in data[0]:
        res[field] = 0 if field in fields else None
    for field in fields:
        for point in data:
            res[field] += point[field]
        res[field] /= len(data)
    res['ts'] = data[0]['ts']
    return res


if __name__ == '__main__':
    cwd = os.getcwd()
    src_dir = os.path.join(cwd, src_dir)
    out_dir = os.path.join(cwd, out_dir)

    if resample_rate == 1:
        resample = False
    if not reformat:
        decimal_div = '.'
        value_div = default_div
    time_div_ms = decimal_div if reformat else time_div

    header_line = value_div.join(
        ['time (HH:MM:SS{}MS)'.format(time_div_ms), 'voltage (V)',
         'current (A)', 'D+ (V)', ' D- (V)', 'temp (deg)'])
    formats = {'v': '{:.3f}', 'c': '{:.3f}',
               'd+': '{:.1f}', 'd-': '{:.1f}', 't': '{:.0f}'}

    files = 0
    for name in os.listdir(src_dir):
        if src_ext != os.path.splitext(name)[1]:
            continue

        print(f"Processing: {name}")
        last_line = lines_count(os.path.join(src_dir, name)) - 1
        src_file = open(os.path.join(src_dir, name), "r")
        out_file = open(os.path.join(out_dir, name), "w")

        points = []
        total_read = 0
        processed = written = 0
        for line in src_file:
            line = line.strip()
            if not line:
                total_read += 1
                continue

            if processed == 0:
                if add_header:
                    out_file.write(header_line)
                    out_file.write("\n")
                if "?" in line:
                    total_read += 1
                    processed += 1
                    continue

            line = line.split(default_div)
            ts = str2ts(*line[0].split(time_div))
            point = {'ts': ts, 'v': float(line[1]), 'c': float(line[2]),
                     'd+': float(line[3]), 'd-': float(line[4]),
                     't': None if line[5] == 'null' else int(line[5])}
            points.append(point)

            if not resample or total_read == last_line \
               or len(points) == resample_rate:
                if resample:
                    if resample_mode == 'start':
                        point = points[0]
                    elif resample_mode == 'end':
                        point = points[-1]
                    elif resample_mode == 'center':
                        point = points[len(points) // 2]
                    elif resample_mode == 'avg':
                        point = resample_avg(points)
                    else:
                        exit('Unknown resample mode')

                point['ts'] = ts2str(point['ts'], time_div_ms)
                for field in point:
                    if point[field] is None:
                        point[field] = 'null'
                    elif field in formats:
                        point[field] = formats[field].format(point[field]) \
                            .replace('.', decimal_div)

                out_file.write(value_div.join(point.values()))
                out_file.write("\n")
                written += 1
                points = []

            total_read += 1
            processed += 1
            # break

        print(f"Total lines processed: {processed}")
        print(f"Total lines written: {written}")

        src_file.close()
        out_file.close()
        files += 1

    print(f"Total files processed: {files}")