diff --git a/Dockerfile b/Dockerfile index 9c8bc40..88c4148 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-alpine +FROM python:3.14-alpine WORKDIR /usr/src/app COPY requirements.txt ./ diff --git a/README.md b/README.md index 003c027..03644ce 100644 --- a/README.md +++ b/README.md @@ -27,17 +27,20 @@ python check-url.py ``` ### avec InfluxDB ``` -export "LIST=${PWD}/list.yml" +export "LIST=${PWD}/sources/list.yml" export "INFLUXDB-HOST=http://localhost:8086" -export "INFLUXDB-BUCKET=bucket" -export "INFLUXDB-TOKEN=token" -export "INFLUXDB-ORG=org" +export "INFLUXDB-BUCKET=crawl" +export "INFLUXDB-TOKEN=" +export "INFLUXDB-ORG=" python check-url.py ``` ### Docker + +:warning: Modifier les variables d'environnement dans le docker-compose.yml + ``` -docker-compose build --no-cache && docker-compose up -d +docker compose build --no-cache && docker compose up -d ``` ## Fonctionnalitées futures @@ -57,4 +60,3 @@ Ce projet est sous licence [Apache 2.0](https://www.apache.org/licenses/LICENSE- # Informations sur l'auteur Ce projet a été créé par [PG3](https://pg3.io) en décembre 2018. -Ce projet a été maintenu par [PG3](https://pg3.io) en juillet 2021. diff --git a/check-url.py b/check-url.py index b696dfa..3e89836 100755 --- a/check-url.py +++ b/check-url.py @@ -1,12 +1,11 @@ #!/usr/bin/python import os import time -import re from threading import Thread import queue import yaml import requests - +import sys from influxdb_client import InfluxDBClient from influxdb_client.client.write_api import SYNCHRONOUS @@ -14,117 +13,89 @@ filesource = os.environ['LIST'] except KeyError: print('Missing LIST environnement variable, exiting.') - exit(1) + sys.exit(1) url_data = [] def main(): db_con = False global url_data - global file + global config q = queue.Queue() - file = open_file() + config = parse_config() write_api = connect_to_influxdb() - try: - file['thread'] - except: - for i in range(int(1)): + if 'thread' in config: + for i in range(int(config['thread'])): t = Thread(target=checkurl, args=(q,)) t.daemon = True t.start() else: - for i in range(int(file['thread'])): + for i in range(int(1)): t = Thread(target=checkurl, args=(q,)) t.daemon = True t.start() while 42: - tmp_file = open_file() - if file != tmp_file: - file = tmp_file - print('File has changed : reloading...') - urls = get_url_array(file) + tmp_config = parse_config() + if config != tmp_config: + config = tmp_config + print('config has changed : reloading...') + urls = get_url_array(config) for url in urls: q.put(url) q.join() - fill_limit_data(file) + fill_limit_data(config) for data in url_data: db_con = format_response(write_api, db_con, data[3], data[0], data[1], data[6], data[7], data[5], data[4], data[8]) url_data.clear() - try: - file['delay'] - except KeyError: - time.sleep(30) + if 'delay' in config: + time.sleep(float(config['delay'])) else: - time.sleep(float(file['delay'])) - - -def refresh_file(fQueue): - while 42: - file = open_file() - fQueue.put(file) - try: - refresh = file['refresh'] - except KeyError: time.sleep(30) - else: - time.sleep(refresh) -def fill_limit_data(file): - for line in file['website']: +def fill_limit_data(config): + for site in config['website']: for data in url_data: - if line['url'] == data[3]: - try: - line['search'] - except: + if site['url'] == data[3]: + if 'search' in site: + data.append(site['search']) + else: data.append('') + + if 'warning' in site: + data.append(site['warning']) else: - data.append(line['search']) - - try: - line['warning'] - except: data.append(0.2) + + if 'critical' in site: + data.append(site['critical']) else: - data.append(line['warning']) - - try: - line['critical'] - except: data.append(0.3) - else: - data.append(line['critical']) - try: - line['tags'] - except: - data.append('') + if 'tags' in site: + data.append(site['tags']) else: - data.append(line['tags']) - - try: - line['api'] - except: data.append('') + + if 'api' in site: + data.append(site['api']) else: - data.append(line['api']) + data.append('') -def get_url_array(file): +def get_url_array(config): urls = [] - for url in file['website']: - try: - url['search'] - except: - urls.append([url['url'], False]) - else: + for url in config['website']: + if 'search' in url: urls.append([url['url'], True]) + else: + urls.append([url['url'], False]) return urls @@ -167,13 +138,14 @@ def format_response(write_api, db_con, url, req, timereq, warning, danger, resul return (format_to_json(write_api, db_con, status_code, timereq, url, retcode, message, tags)) -def open_file(): +def parse_config() -> dict: + config = {} with open(filesource, 'r') as stream: try: - file = yaml.load(stream, Loader=yaml.FullLoader) + config = yaml.load(stream, Loader=yaml.FullLoader) except yaml.YAMLError as exc: print(exc) - return file + return config def insert_to_influxdb(write_api, db_con, data_json): @@ -264,4 +236,4 @@ def connect_to_influxdb(): try: main() except KeyboardInterrupt: - exit(1) + sys.exit(1) diff --git a/docker-compose.yml b/docker-compose.yml index 515a456..895e606 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,8 +9,8 @@ services: - "LIST=/sources/list.yml" - "INFLUXDB-HOST=http://influxdb:8086" - "INFLUXDB-BUCKET=crawl" - - "INFLUXDB-TOKEN=Feixa4iePhaeyahji9Eech6ahphaidituy7eeb3TahF6weengo" - - "INFLUXDB-ORG=test" + - "INFLUXDB-TOKEN=" + - "INFLUXDB-ORG=" influxdb: image: influxdb:2.7 @@ -18,8 +18,8 @@ services: - "8086:8086" environment: - DOCKER_INFLUXDB_INIT_MODE=setup - - DOCKER_INFLUXDB_INIT_USERNAME=admin - - DOCKER_INFLUXDB_INIT_PASSWORD=passwordpasswordpassword - - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=Feixa4iePhaeyahji9Eech6ahphaidituy7eeb3TahF6weengo - - DOCKER_INFLUXDB_INIT_ORG=test + - DOCKER_INFLUXDB_INIT_USERNAME=crawl + - DOCKER_INFLUXDB_INIT_PASSWORD= + - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN= + - DOCKER_INFLUXDB_INIT_ORG= - DOCKER_INFLUXDB_INIT_BUCKET=crawl diff --git a/requirements.txt b/requirements.txt index 90a5ff3..cf965a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -queuelib -threaded pyyaml requests -influxdb-client \ No newline at end of file +influxdb-client