airflow/leaf-pipeline: add DAG, Docker & LFS config (rebased on origi…

.gitattributes

@@ -0,0 +1,5 @@
+projects/leaf-counting/weights/*.pt filter=lfs diff=lfs merge=lfs -text
+projects/leaf-counting/weights/*.pth filter=lfs diff=lfs merge=lfs -text
+projects/leaf-counting/weights/*.safetensors filter=lfs diff=lfs merge=lfs -text
+# אם יש גם מודלים בתיקיות אחרות (דוגמה):
+projects/Detection_Jobs/**/models/* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,40 +1,84 @@
-# --- Secrets and Certificates ---
-*.env
-*.crt
-*/certs/
-**/certs/
-**/secrets/
-storage_with_mqtt/secrets/
-storage_with_mqtt/mqtt_images/secrets/
-MQTT_IMAGES/secrets/
-services/sounds/sounds_classifier/src/classification/data/
-services/sounds/sounds_classifier/src/classification/models/panns_data/
+# === OS / Editors ===
+.DS_Store
+*.swp
+*.swo
+.idea/
+.vscode/
+*.code-workspace
 
-# --- Python ---
+# === Python ===
 __pycache__/
-*.pyc
+*.py[cod]
 *.pyo
 *.pyd
-*.pytest_cache/
-.venv/
-venv/
+*.egg-info/
+.eggs/
+.build/
+build/
+dist/
+.mypy_cache/
+.pytest_cache/
 .coverage
+.coverage.*
+.cache/
+.ipynb_checkpoints/
 
-# --- VSCode / Editor ---
-.vscode/
-.idea/
+# Virtual envs
+.venv/
+venv/
+env/
 
-# --- Docker / Build ---
-*.log
-*.pid
-*.bak
-*.tmp
-*.swp
-.env.local
+# === Docker / Compose ===
+docker.env
+.env
 .env.*
-!.env.example
+.env.local
+.envrc
 
-# --- OS files ---
-.DS_Store
-Thumbs.db
+# === Airflow (ארטיפקטים/לוגים/DB) ===
+airflow/logs/
+airflow/airflow.db
+airflow/airflow.db-journal
+airflow/*.pid
+airflow/*webserver*.log
+airflow/*webserver*.out
+airflow/*webserver*.err
+airflow/*scheduler*.log
+airflow/*scheduler*.out
+airflow/*scheduler*.err
+airflow/dags/*.bak.*
+airflow/staging/
+
+# === Projects: leaf-counting (ארטיפקטים בלבד) ===
+projects/leaf-counting/out_detect/
+projects/leaf-counting/out_crops/
+projects/leaf-counting/out_pwb/
+projects/leaf-counting/runs_local/
+projects/leaf-counting/.venv/
+projects/leaf-counting/staging/
 
+# === Projects: Detection_Jobs / disease-monitor (ארטיפקטים כלליים) ===
+projects/Detection_Jobs/**/__pycache__/
+projects/Detection_Jobs/**/.mypy_cache/
+projects/Detection_Jobs/**/.pytest_cache/
+projects/disease-monitor/**/__pycache__/
+projects/disease-monitor/**/.mypy_cache/
+projects/disease-monitor/**/.pytest_cache/
+
+# === Secrets / Certs ===
+*.key
+*.pem
+*.crt
+*.p12
+*credentials*.json
+*service_account*.json
+*.secrets.*
+.secrets/
+secrets/
+projects/Detection_Jobs/.git.backup-*.tar.gz
+airflow/dags/leaf-counting/runs_local/
+airflow/dags/leaf-counting/demo_images/
+airflow/dags/leaf-counting/out_*/
+projects/Detection_Jobs/**/models/
+projects/disease-monitor/disease-monitor/alerts.db
+projects/**/.git.backup-*.tar.gz

.gitignore.bak.1762665487

@@ -0,0 +1,76 @@
+# === OS / Editors ===
+.DS_Store
+*.swp
+*.swo
+.idea/
+.vscode/
+*.code-workspace
+
+# === Python ===
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.egg-info/
+.eggs/
+.build/
+build/
+dist/
+.mypy_cache/
+.pytest_cache/
+.coverage
+.coverage.*
+.cache/
+.ipynb_checkpoints/
+
+# Virtual envs
+.venv/
+venv/
+env/
+
+# === Docker / Compose ===
+docker.env
+.env            # סודות מקומיים - לא מעלים (נפרסם .env.example בהמשך)
+
+# === Airflow (רק ארטיפקטים/לוגים/DB) ===
+airflow/logs/
+airflow/airflow.db
+airflow/airflow.db-journal
+airflow/*.pid
+airflow/*webserver*.log
+airflow/*webserver*.out
+airflow/*webserver*.err
+airflow/*scheduler*.log
+airflow/*scheduler*.out
+airflow/*scheduler*.err
+# קבצי גיבוי של DAG (אלו *.bak.*)
+airflow/dags/*.bak.*
+
+# קלט/פלט זמני של Airflow (staging) — לא קוד
+airflow/staging/
+
+# === Projects: leaf-counting (ארטיפקטים בלבד) ===
+projects/leaf-counting/out_detect/
+projects/leaf-counting/out_crops/
+projects/leaf-counting/out_pwb/
+projects/leaf-counting/runs_local/
+projects/leaf-counting/.venv/
+
+# === Projects: Detection_Jobs / disease-monitor (ארטיפקטים כלליים) ===
+projects/Detection_Jobs/**/__pycache__/
+projects/Detection_Jobs/**/.mypy_cache/
+projects/Detection_Jobs/**/.pytest_cache/
+projects/disease-monitor/**/__pycache__/
+projects/disease-monitor/**/.mypy_cache/
+projects/disease-monitor/**/.pytest_cache/
+
+# === Secrets / Certs (למקרה שיש) ===
+*.key
+*.pem
+*.crt
+*.p12
+*credentials*.json
+*service_account*.json
+*.secrets.*
+.secrets/
+secrets/

Dockerfile

@@ -0,0 +1,135 @@
+# # Airflow 2.9.3 על Python 3.12 (Debian)
+# FROM apache/airflow:2.9.3-python3.12
+
+# USER root
+# RUN apt-get update && apt-get install -y --no-install-recommends \
+#     rsync curl && \
+#     rm -rf /var/lib/apt/lists/*
+
+# # נתקין את דרישות הפרויקט (כולל torch/opencv/ultralytics וכו') + awscli/minio
+# USER airflow
+# COPY projects/leaf-counting/requirements.txt /tmp/req.txt
+# RUN pip install --no-cache-dir -r /tmp/req.txt \
+#     && pip install --no-cache-dir awscli minio
+
+# # Airflow כבר מותקן בתמונה – אין מה לעשות כאן מעבר
+# Airflow 2.9.3 על Python 3.12 (Debian)
+# FROM apache/airflow:2.9.3-python3.12
+
+# USER root
+# # מוסיפים ספריות מערכת הנדרשות ל-OpenCV (כולל libGL) + כלי עזר
+# RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+#     libgl1 \
+#     libglib2.0-0 \
+#     libsm6 \
+#     libxext6 \
+#     libxrender1 \
+#     rsync \
+#     curl \
+#     && rm -rf /var/lib/apt/lists/*
+
+# # תקיית עבודה עם הרשאות כתיבה למשתמש airflow (UID 50000)
+# RUN mkdir -p /work /opt/airflow/runs_local \
+#     && chown -R airflow:0 /work /opt/airflow/runs_local \
+#     && chmod -R 775 /work /opt/airflow/runs_local
+
+# # נתקין את דרישות הפרויקט + awscli/minio + opencv-python-headless
+# USER airflow
+# COPY projects/leaf-counting/requirements.txt /tmp/req.txt
+# RUN pip install --no-cache-dir -r /tmp/req.txt && \
+#     pip install --no-cache-dir \
+#       awscli==1.33.0 \
+#       minio==7.2.9 \
+#       opencv-python-headless==4.10.0.84
+# Airflow 2.9.3 על Python 3.12
+# FROM apache/airflow:2.9.3-python3.12
+
+# # ספריה מערכתית ל-OpenCV
+# USER root
+# RUN apt-get update && apt-get install -y --no-install-recommends \
+#       libgl1 \
+#  && rm -rf /var/lib/apt/lists/*
+# USER airflow
+
+# # תלותי ML/עיבוד תמונה + כלים לענן/MinIO
+# # שמירה על תאימות בינארית: numpy<2; OpenCV headless; YOLO (ultralytics)
+# RUN pip install --no-cache-dir \
+#       "numpy==1.26.4 " \
+#       "opencv-python-headless==4.9.0.80" \
+#       "ultralytics==8.*" \
+#       "boto3" "minio" "awscli"
+
+# # (אופציונלי אבל מומלץ ל-CPU): PyTorch CPU מפינים הרשמיים
+# RUN pip install --no-cache-dir \
+#       torch==2.3.1+cpu torchvision==0.18.1+cpu \
+#       --index-url https://download.pytorch.org/whl/cpu
+# FROM apache/airflow:2.9.3-python3.10
+# FROM ghcr.io/apache/airflow:2.9.3-python3.10
+
+# FROM quay.io/apache/airflow:2.9.3-python3.10
+
+
+# USER root
+# RUN apt-get update && apt-get install -y --no-install-recommends libgl1 && \
+#     rm -rf /var/lib/apt/lists/*
+# USER airflow
+
+# RUN python -m pip install --no-cache-dir --upgrade pip
+
+# RUN pip install --no-cache-dir \
+#     --extra-index-url https://download.pytorch.org/whl/cpu \
+#     torch==2.3.1+cpu torchvision==0.18.1+cpu torchaudio==2.3.1+cpu
+
+# RUN pip install --no-cache-dir \
+#     numpy==1.26.4 opencv-python-headless==4.9.0.80 ultralytics==8.2.10 \
+#     boto3 minio awscli
+# FROM python:3.10-slim
+FROM mcr.microsoft.com/devcontainers/python:1-3.10-bullseye
+
+
+# מערכת מינימלית
+# RUN apt-get update && apt-get install -y --no-install-recommends \
+#     libgl1 libglib2.0-0 ffmpeg curl ca-certificates && \
+#     rm -rf /var/lib/apt/lists/*
+# RUN apt-get update && apt-get install -y --no-install-recommends \
+#     libgl1 libglib2.0-0 ffmpeg curl ca-certificates \
+#     util-linux procps \
+#  && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libgl1 libglib2.0-0 ffmpeg curl ca-certificates \
+    util-linux procps \
+ && rm -rf /var/lib/apt/lists/*
+
+
+
+# שדרוג כלי build של פייתון
+RUN python -m pip install --no-cache-dir --upgrade pip wheel setuptools
+
+# === Airflow מ-PyPI עם constraints (עוקף רג'יסטרי) ===
+ENV AIRFLOW_VERSION=2.9.3
+ENV PYTHON_VERSION=3.10
+ENV CONSTRAINT_URL=https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt
+RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
+
+# Providers תואמים ל-2.9.3 דרך אותו constraints
+RUN pip install --no-cache-dir \
+  "apache-airflow-providers-docker" \
+  --constraint "${CONSTRAINT_URL}"
+
+
+# === PyTorch CPU wheels (מהאינדקס הרשמי) ===
+RUN pip install --no-cache-dir \
+  --extra-index-url https://download.pytorch.org/whl/cpu \
+  torch==2.3.1+cpu torchvision==0.18.1+cpu torchaudio==2.3.1+cpu
+
+# === YOLO/עיבוד-תמונה וכלי עזר ===
+RUN pip install --no-cache-dir \
+  numpy==1.26.4 opencv-python-headless==4.9.0.80 ultralytics==8.2.10 \
+  onnx==1.16.1 onnxruntime==1.18.1 \
+  boto3 minio awscli requests tqdm
+
+# סביבת עבודה ומשתמש לא-root
+RUN useradd -ms /bin/bash airflow
+USER airflow
+WORKDIR /opt/airflow