Selling-Out/parseData.py at master · benjaminpstern/Selling-Out · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
from os import listdir
from os.path import join, realpath, dirname
import sys
from time import mktime, strptime


CHUNKS = 50

def main():
    files = listdir(join(dirname(realpath(__file__)), "data"))
    fileOutName = "parsed.data"
    minStartTime = getMinStartTime(files)
    maxEndTime = getMaxEndTime(files)
    print(minStartTime, maxEndTime)
    for chunkNo in range(CHUNKS):
        writeLines(CHUNKS, chunkNo, minStartTime, maxEndTime, files)


def getMinStartTime(files):
    curMin = float('inf')
    for filename in files:
        realFileName = join(dirname(realpath(__file__)), "data", filename)
        fIn = open(realFileName, "r")
        fIn.readline()
        line = fIn.readline()
        lineSplit = line.split(",")
        date = lineSplit[2]
        time = lineSplit[3]

        year = date[:4]
        month = date[4:6]
        day = date[6:]
        hour = time[:2]
        minute = time[2:4]
        timeString = "{0}.{1}.{2} {3}:{4}:00".format(day, month, year, hour, minute);
        epochTime = int(mktime(strptime(timeString, "%d.%m.%Y %H:%M:%S")))
        if epochTime < curMin:
            curMin = epochTime
    return curMin

def getMaxEndTime(files):
    curMax = 0
    for filename in files:
        realFileName = join(dirname(realpath(__file__)), "data", filename)
        fIn = open(realFileName, "r")
        prevLine = ""
        for line in fIn:
            prevLine = line
        line = prevLine
        lineSplit = line.split(",")
        date = lineSplit[2]
        time = lineSplit[3]

        year = date[:4]
        month = date[4:6]
        day = date[6:]
        hour = time[:2]
        minute = time[2:4]
        timeString = "{0}.{1}.{2} {3}:{4}:00".format(day, month, year, hour, minute);
        epochTime = int(mktime(strptime(timeString, "%d.%m.%Y %H:%M:%S")))
        if epochTime > curMax:
            curMax = epochTime
    return curMax


def writeLines(chunks, chunkNo, minStartTime, maxEndTime, files):
    fOut = open("chunk{0}.data".format(chunkNo), "w")
    lines = {}
    chunkSize = (maxEndTime - minStartTime) // chunks
    minTime = minStartTime + chunkNo * chunkSize
    maxTime = minTime + chunkSize
    for filename in files:
        realFileName = join(dirname(realpath(__file__)), "data", filename)
        fIn = open(realFileName, "r")
        fIn.readline()
        for line in fIn:
            lineSplit = line.split(",")
            name = filename
            date = lineSplit[2]
            time = lineSplit[3]
            close = lineSplit[7]

            year = date[:4]
            month = date[4:6]
            day = date[6:]
            hour = time[:2]
            minute = time[2:4]
            timeString = "{0}.{1}.{2} {3}:{4}:00".format(day, month, year, hour, minute);
            epochTime = int(mktime(strptime(timeString, "%d.%m.%Y %H:%M:%S")))

            if epochTime < minTime:
                continue

            elif epochTime > maxTime:
                break

            if epochTime not in lines:
                lines[epochTime] = []
            lines[epochTime].append(name + ":" + close)
        fIn.close()

    keys = sorted(lines.keys())
    for time in keys:
        line = str(time) + "," + ",".join(lines[time]) + "\n"
        fOut.write(line)

    fOut.close()

if __name__ == "__main__":
    main()