Repository navigation
Expand file tree
/
Copy pathapplication.py
More file actions
363 lines (261 loc) · 9.88 KB
/
Copy pathapplication.py
File metadata and controls
363 lines (261 loc) · 9.88 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
import os
import sys
import csv
from io import StringIO, BytesIO
from redis import Redis
import rq
from flask import Flask, flash, redirect, render_template, request, session, make_response, jsonify, Response, url_for
from flask_session import Session
from tempfile import mkdtemp
from werkzeug.exceptions import default_exceptions, HTTPException, InternalServerError
from mongo.mongo import *
from s3 import upload_file,create_presigned_url
from helpers import *
from hotspots import *
# Configure application
app = Flask(__name__)
# Ensure templates are auto-reloaded
app.config["TEMPLATES_AUTO_RELOAD"] = True
# Setup Redis
REDIS_URL = os.environ.get('REDIS_URL') or 'redis://localhost:6379'
conn = Redis.from_url(REDIS_URL)
task_queue = rq.Queue('scl-tasks', connection=conn)
# Ensure responses aren't cached
@app.after_request
def after_request(response):
response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
response.headers["Expires"] = 0
response.headers["Pragma"] = "no-cache"
return response
# Configure session to use filesystem (instead of signed cookies)
app.config["SESSION_FILE_DIR"] = mkdtemp()
app.config["SESSION_PERMANENT"] = False
app.config["SESSION_TYPE"] = "filesystem"
Session(app)
# Configure MongoDB
MONGO_URL = os.environ.get("MONGO_URL")
MONGO_DB = os.environ.get("MONGO_DB")
mongodb = MongoConnection(url=MONGO_URL, db=MONGO_DB)
def export_csv(params):
"""Generate CSV from MongoDB data and upload it to Amazon S3"""
# Unpack parameters
collection = params['collection']
filename = params['filename']
timezone = params['timezone']
limit = params['limit']
start = params['start']
end = params['end']
# Query MongoDB
rows = mongodb.find(collection=collection, timezone=timezone, limit=limit, start=start, end=end)
# Convert rows to StringIO and upload to Amazon S3
object_name = f"{collection}{start}{end}.csv" if (start or end) else f"{collection}.csv"
data = StringIO()
writer = csv.writer(data, delimiter=",")
writer.writerow(dict(rows[0]).keys())
for row in rows:
writer.writerow(dict(row).values())
upload_file(data, "scl-api", object_name)
# Create presigned url
url = create_presigned_url("scl-api", object_name, filename)
return url
@app.route("/", methods=["GET"])
def index():
"""Home page"""
return render_template("index.html")
@app.route("/data", methods=["GET"])
def data():
"""Data page for downloading data"""
return render_template("data.html")
# @app.route("/map", methods=["GET"])
# def map():
# """Street-level map"""
# return render_template("map.html")
@app.route("/hotspots", methods=["GET"])
def hotspot():
"""Page for calculating hotspots"""
return render_template("hotspots.html")
@app.route("/docs", methods=["GET"])
def docs():
"""Home page for API documentation"""
return render_template("swaggerui.html")
@app.route("/api/hotspots/", methods=["GET", "POST"])
def hotspots_api():
"""Hotspots"""
params = request.get_json() if request.method == "POST" else request.args
collection = params.get("dataset", None)
timezone = params.get("timezone", None)
start = params.get("start", None)
end = params.get("end", None)
# Set timezone
if timezone:
tz = pytz.timezone('US/Eastern')
# Query MongoDB
rows = mongodb.find(collection=collection, timezone=timezone, start=start, end=end)
# Generate list of rows
res = []
for row in rows:
res.append(row)
data = pd.DataFrame(res)
# Calculate hotspots
calculate_hotspots(data)
return jsonify({'redirect': url_for("example")})
@app.route("/example")
def example():
"""Hotspots"""
return render_template("foliumHotspotMap.html")
@app.route("/api/datasets/", methods=["GET"])
def datasets():
"""Gets a list of all available datasets.
:return: List of datasets
:rtype: JSON
"""
# Retrieving datasets must be via GET
if request.method != "GET":
return make_response(jsonify({"error": "GET request required."}), 400)
# Query Mongo for all available datasets
all_collections = list(mongodb.list_collections())
response = {"results": all_collections}
# Return a JSON response
return jsonify(response)
@app.route("/api/latest/", methods=["GET", "POST"])
def latest():
"""Gets latest data from MongoDB.
:param dataset: Dataset to retrieve from
:type dataset: str, required
:param limit: Number of entries to retrieve
:type limit: int, optional
:param timezone:
:type timezone: str, optional
:param start: Start date in format YYYY-MM-DDT20HH:MM:SS
:type start: str, optional
:param end: End date in format YYYY-MM-DDT20HH:MM:SS
:type end: str, optional
:return: List of data
:rtype: JSON
"""
# Parse parameters
params = request.get_json() if request.method == "POST" else request.args
collection = params.get("dataset", None)
filename = params.get("filename", None)
timezone = params.get("timezone", None)
limit = params.get("limit", None)
start = params.get("start", None)
end = params.get("end", None)
# Pack parameters into a dict
params = {'collection': collection, 'filename': filename, 'timezone': timezone, 'limit': limit, 'start': start, 'end': end}
# Ensure start and end date are valid
if start and end:
first_date = list(mongodb.get_start_date(collection=collection))[0]['time']
last_date = list(mongodb.get_end_date(collection=collection))[0]['time']
unix_start = int(string_to_datetime(start, timezone).timestamp())
unix_end = int(string_to_datetime(end, timezone).timestamp())
if unix_start == unix_end:
return make_response(jsonify({"error": "Start date must be larger than end date."}), 400)
if unix_start > unix_end:
return make_response(jsonify({"error": "Start date is larger than end date."}), 400)
if unix_start > last_date or unix_start < first_date:
return make_response(jsonify({"error": "Start date is invalid."}), 400)
if unix_end < first_date:
return make_response(jsonify({"error": "End date is invalid."}), 400)
# Ensure a collection is provided
if not collection:
return make_response(jsonify({
"error": "Collection required."
}), 400)
"""
# Create response from StringIO output
res = make_response(data.getvalue())
res.headers['Content-Disposition'] = "attachment; filename=test.csv"
res.headers['Content-type'] = "text/csv"
"""
if request.method == "POST":
# Launch background task to export CSV
rq_job = task_queue.enqueue_call(func=export_csv,
args=(params,)
)
# Return job id to client
return jsonify({'job_id': rq_job.get_id()})
elif request.method == "GET":
# Query MongoDB
rows = mongodb.find(collection=collection, timezone=timezone, limit=limit, start=start, end=end)
# Generate list of rows
res = []
for row in rows:
res.append(row)
return jsonify({'results': res})
@app.route("/api/latest/<job_id>", methods=["GET"])
def latest_results(job_id):
"""Poll status of requested job"""
# Fetch requested job
rq_job = task_queue.fetch_job(job_id)
# Check if job has failed
if rq_job.get_status() == 'failed':
return jsonify({'error': "Job has failed!"})
# Check if job has finished
if rq_job.is_finished:
url = rq_job.result
return jsonify({'url': url})
else:
return jsonify({'result': rq_job.is_finished})
@app.route("/api/summary/", methods=["GET"])
def summary():
"""Gets a summary of MongoDB collections.
:return: Summary of data
:rtype: JSON
"""
# Parse GET parameters
params = request.args
collection = params.get("collection", None)
result = mongodb.summary(collection)
response = {"results": result}
return jsonify(response)
@app.route("/api/AQ-hotspots/", methods=["GET"])
def aq_hotspots():
"""Gets locations of air pollution hotspots for a given city and a given timeframe.
:param dataset: Dataset to retrieve from
:type dataset: str, required
:return: Location of air pollution hotspots
:rtype: JSON
"""
return jsonify(response)
@app.route("/api/AQ-averages/", methods=["GET"])
def averages():
"""Gets air quality hourly/daily/monthly/yearly averages for a given street segment in a city and a given timeframe.
:param dataset: Dataset to retrieve from
:type dataset: str, required
:return: Air quality averages
:rtype: JSON
"""
return jsonify(response)
@app.route("/api/Noise-averages/", methods=["GET"])
def noise_hotspots():
"""Gets locations of noise pollution hotspots for a given street segment in city and a given timeframe.
:param dataset: Dataset to retrieve from
:type dataset: str, required
:return: Location of noise pollution hotspots
:rtype: JSON
"""
return jsonify(response)
@app.template_filter("format_unix")
def format_unix(date):
"""Format UNIX timestamp"""
monthNames = [
"January", "February", "March", "April", "May", "June",
"July", "August", "September", "October", "November", "December"
]
date = datetime.utcfromtimestamp(date)
month = date.month
day = date.day
year = date.year
hour = date.hour
minutes = date.minute
seconds = date.second
return f"{day} {monthNames[month - 1]} {year}, {hour}:{minutes}:{seconds}"
# def errorhandler(e):
# """Handle error"""
# if not isinstance(e, HTTPException):
# e = InternalServerError()
# return make_response(jsonify({"error": "Sorry, an error has occured."}), 400)
# # Listen for errors
# for code in default_exceptions:
# app.errorhandler(code)(errorhandler)