Syntax

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-alpine
+FROM python:3.14-alpine
 
 WORKDIR /usr/src/app
 COPY requirements.txt ./

README.md

@@ -27,17 +27,20 @@ python check-url.py
 ```
 ### avec InfluxDB
 ```
-export "LIST=${PWD}/list.yml"
+export "LIST=${PWD}/sources/list.yml"
 export "INFLUXDB-HOST=http://localhost:8086"
-export "INFLUXDB-BUCKET=bucket"
-export "INFLUXDB-TOKEN=token"
-export "INFLUXDB-ORG=org"
+export "INFLUXDB-BUCKET=crawl"
+export "INFLUXDB-TOKEN=<TOKEN_SECRET_INFLUXDB>"
+export "INFLUXDB-ORG=<ORGANISATION>"
 python check-url.py
 ```
 
 ### Docker
+
+:warning: Modifier les variables  d'environnement dans le docker-compose.yml
+
 ```
-docker-compose build --no-cache && docker-compose up -d
+docker compose build --no-cache && docker compose up -d
 ```
 
 ## Fonctionnalitées futures
@@ -57,4 +60,3 @@ Ce projet est sous licence [Apache 2.0](https://www.apache.org/licenses/LICENSE-
 
 # Informations sur l'auteur
 Ce projet a été créé par [PG3](https://pg3.io) en décembre 2018.
-Ce projet a été maintenu par [PG3](https://pg3.io) en juillet 2021.

check-url.py

@@ -1,130 +1,101 @@
 #!/usr/bin/python
 import os
 import time
-import re
 from threading import Thread
 import queue
 import yaml
 import requests
-
+import sys
 from influxdb_client import InfluxDBClient
 from influxdb_client.client.write_api import SYNCHRONOUS
 
 try:
     filesource = os.environ['LIST']
 except KeyError:
     print('Missing LIST environnement variable, exiting.')
-    exit(1)
+    sys.exit(1)
 
 url_data = []
 
 def main():
     db_con = False
     global url_data
-    global file
+    global config
     q = queue.Queue()
-    file = open_file()
+    config = parse_config()
     write_api = connect_to_influxdb()
-    try:
-        file['thread']
-    except:
-        for i in range(int(1)):
+    if 'thread' in config:
+        for i in range(int(config['thread'])):
             t = Thread(target=checkurl, args=(q,))
             t.daemon = True
             t.start()
     else:
-        for i in range(int(file['thread'])):
+        for i in range(int(1)):
             t = Thread(target=checkurl, args=(q,))
             t.daemon = True
             t.start()
 
     while 42:
-        tmp_file = open_file()
-        if file != tmp_file:
-            file = tmp_file
-            print('File has changed : reloading...')
-        urls = get_url_array(file)
+        tmp_config = parse_config()
+        if config != tmp_config:
+            config = tmp_config
+            print('config has changed : reloading...')
+        urls = get_url_array(config)
         for url in urls:
                 q.put(url)
         q.join()
 
-        fill_limit_data(file)
+        fill_limit_data(config)
         for data in url_data:
             db_con = format_response(write_api, db_con, data[3], data[0], data[1], data[6], data[7], data[5], data[4], data[8])
 
         url_data.clear()
 
-        try:
-            file['delay']
-        except KeyError:
-            time.sleep(30)
+        if 'delay' in config:
+            time.sleep(float(config['delay']))
         else:
-            time.sleep(float(file['delay']))
-
-
-def refresh_file(fQueue):
-    while 42:
-        file = open_file()
-        fQueue.put(file)
-        try:
-            refresh = file['refresh']
-        except KeyError:
             time.sleep(30)
-        else: 
-            time.sleep(refresh)
 
 
-def fill_limit_data(file):
-    for line in file['website']:
+def fill_limit_data(config):
+    for site in config['website']:
         for data in url_data:
-            if line['url'] == data[3]:
-                try:
-                    line['search']
-                except:
+            if site['url'] == data[3]:
+                if 'search' in site:
+                    data.append(site['search'])
+                else:
                     data.append('')
+
+                if 'warning' in site:
+                    data.append(site['warning'])
                 else:
-                    data.append(line['search'])
-
-                try:
-                    line['warning']
-                except:
                     data.append(0.2)
+
+                if 'critical' in site:
+                    data.append(site['critical'])
                 else:
-                    data.append(line['warning'])
-
-                try:
-                    line['critical']
-                except:
                     data.append(0.3)
-                else:
-                    data.append(line['critical'])
 
-                try:
-                    line['tags']
-                except:
-                    data.append('')
+                if 'tags' in site:
+                    data.append(site['tags'])
                 else:
-                    data.append(line['tags'])
-
-                try:
-                    line['api']
-                except:
                     data.append('')
+
+                if 'api' in site:
+                    data.append(site['api'])
                 else:
-                    data.append(line['api'])
+                    data.append('')
 
 
 
-def get_url_array(file):
+def get_url_array(config):
     urls = []
 
-    for url in file['website']:
-        try:
-            url['search']
-        except:
-            urls.append([url['url'], False])
-        else:
+    for url in config['website']:
+        if 'search' in url:
             urls.append([url['url'], True])
+        else:
+            urls.append([url['url'], False])
 
     return urls
 
@@ -167,13 +138,14 @@ def format_response(write_api, db_con, url, req, timereq, warning, danger, resul
     return (format_to_json(write_api, db_con, status_code, timereq, url, retcode, message, tags))
 
 
-def open_file():
+def parse_config() -> dict:
+    config = {}
     with open(filesource, 'r') as stream:
         try:
-            file = yaml.load(stream, Loader=yaml.FullLoader)
+            config = yaml.load(stream, Loader=yaml.FullLoader)
         except yaml.YAMLError as exc:
             print(exc)
-    return file
+    return config
 
 
 def insert_to_influxdb(write_api, db_con, data_json):
@@ -264,4 +236,4 @@ def connect_to_influxdb():
         try:
             main()
         except KeyboardInterrupt:
-            exit(1)
+            sys.exit(1)

docker-compose.yml

@@ -9,17 +9,17 @@ services:
       - "LIST=/sources/list.yml"
       - "INFLUXDB-HOST=http://influxdb:8086"
       - "INFLUXDB-BUCKET=crawl"
-      - "INFLUXDB-TOKEN=Feixa4iePhaeyahji9Eech6ahphaidituy7eeb3TahF6weengo"
-      - "INFLUXDB-ORG=test"
+      - "INFLUXDB-TOKEN=<TOKEN_SECRET_INFLUXDB>"
+      - "INFLUXDB-ORG=<ORGANISATION>"
 
   influxdb:
     image: influxdb:2.7
     ports:
       - "8086:8086"
     environment: 
       - DOCKER_INFLUXDB_INIT_MODE=setup
-      - DOCKER_INFLUXDB_INIT_USERNAME=admin
-      - DOCKER_INFLUXDB_INIT_PASSWORD=passwordpasswordpassword
-      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=Feixa4iePhaeyahji9Eech6ahphaidituy7eeb3TahF6weengo
-      - DOCKER_INFLUXDB_INIT_ORG=test
+      - DOCKER_INFLUXDB_INIT_USERNAME=crawl
+      - DOCKER_INFLUXDB_INIT_PASSWORD=<MOT_DE_PASSE_SECRET>
+      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=<TOKEN_SECRET_INFLUXDB>
+      - DOCKER_INFLUXDB_INIT_ORG=<ORGANISATION>
       - DOCKER_INFLUXDB_INIT_BUCKET=crawl

requirements.txt

@@ -1,5 +1,3 @@
-queuelib
-threaded
 pyyaml
 requests
-influxdb-client
+influxdb-client