diff --git a/App/schemas/explain_schema.py b/App/schemas/explain_schema.py index 37a51d8..276f806 100644 --- a/App/schemas/explain_schema.py +++ b/App/schemas/explain_schema.py @@ -23,14 +23,29 @@ class ExplainImageRequest(BaseModel): - branch_level: Literal["low","high"] = Field("high", description="브랜치 레벨\nlow: 국소 위조 흔적 포착\nhigh: 전역적 위조 흔적 포착") - explainer_type: str = Field("eigengradcam", description = ("선택 가능한 XAI 기법. low: [hirescam, gradcamelementwise, layercam], ""high: [eigengradcam, gradcamplusplus, xgradcam]")) - display_type: Literal["heatmap", "bbox"] = Field("heatmap", description="시각화 형태. heatmap: 전체 분포, bbox: 위조 의심 영역 사각형") - category: Literal[0, 1] = Field(1, description="판단 클래스 인덱스 (0: Real / 1: Fake)") - overlay_ratio: float = Field(0.5, ge=0.0, le=1.0, description = "Heatmap 투명도 (0: 히트맵만 강조, 1: 원본 이미지 위주)") - threshold: float = Field(0.5, ge=0.5, le=1.0, description="contour/bbox 이진화 임계값 (0.0~1.0)") - aug_smooth: bool = Field(False, description = "TTA(Test Time Augmentation) 적용 여부. 히트맵을 더 객체 중심적으로 정렬") - eigen_smooth: bool = Field(False, description = "PCA 기반 노이즈 제거. 지배적인 패턴만 남김") + branch_level: Literal["low","high"] = Field("high", + description="브랜치 레벨\nlow: 국소 위조 흔적 포착\nhigh: 전역적 위조 흔적 포착") + + explainer_type: str = Field("eigengradcam", + description = ("선택 가능한 XAI 기법. low: [hirescam, gradcamelementwise, layercam], ""high: [eigengradcam, gradcamplusplus, xgradcam]")) + + display_type: Literal["heatmap", "bbox", "heatmap_bbox"] = Field("heatmap", + description="시각화 형태. heatmap: 전체 분포, bbox: 위조 의심 영역 사각형, heatmap_bbox: 위조 의심 사각형 내부에 블러 처리된 히트맵을 중첩") + + category: Literal[0, 1] = Field(1, + description="판단 클래스 인덱스 (0: Real / 1: Fake)") + + overlay_ratio: float = Field(0.5, ge=0.0, le=1.0, + description = "Heatmap 투명도 (0: 히트맵만 강조, 1: 원본 이미지 위주)") + + threshold: float = Field(0.5, ge=0.5, le=1.0, + description="contour/bbox 이진화 임계값 (0.0~1.0)") + + aug_smooth: bool = Field(False, + description = "TTA(Test Time Augmentation) 적용 여부. 히트맵을 더 객체 중심적으로 정렬") + + eigen_smooth: bool = Field(False, + description = "PCA 기반 노이즈 제거. 지배적인 패턴만 남김") @model_validator(mode="after") def validate_explainer_for_branch(self) -> "ExplainImageRequest": diff --git a/App/services/explain_svc.py b/App/services/explain_svc.py index 83acc09..89c2f14 100644 --- a/App/services/explain_svc.py +++ b/App/services/explain_svc.py @@ -44,10 +44,12 @@ def _run_visualization(explainer: CAMExplainer, image_path: str, explain_req_dic if explain_req_dict["display_type"] == "heatmap": return explainer.display_heatmap_on_image(image_path, image_weight=explain_req_dict["overlay_ratio"], threshold=explain_req_dict["threshold"], category=explain_req_dict["category"], aug_smooth=explain_req_dict["aug_smooth"], eigen_smooth=explain_req_dict["eigen_smooth"]) - else: + elif explain_req_dict["display_type"] == "bbox": return explainer.display_bbox_on_image(image_path, threshold=explain_req_dict["threshold"], category=explain_req_dict["category"], aug_smooth=explain_req_dict["aug_smooth"], eigen_smooth=explain_req_dict["eigen_smooth"]) - + else: # display_type == "heatmap_bbox" + return explainer.display_heatmap_bbox_on_image(image_path, threshold=explain_req_dict["threshold"], + category=explain_req_dict["category"], aug_smooth=explain_req_dict["aug_smooth"], eigen_smooth=explain_req_dict["eigen_smooth"]) # 딥페이크 이미지 위조 흔적 시각화 처리 @celery_app.task(name="process_explain_image_task") def process_explain_image_task(user_email: str, diff --git "a/Images/deepfake-xai/\354\204\234\354\226\221\354\235\270_\352\260\200\354\247\234_1.JPG" "b/Images/deepfake-xai/\354\204\234\354\226\221\354\235\270_\352\260\200\354\247\234_1.JPG" deleted file mode 100644 index 85a0e1a..0000000 Binary files "a/Images/deepfake-xai/\354\204\234\354\226\221\354\235\270_\352\260\200\354\247\234_1.JPG" and /dev/null differ diff --git a/Images/deepfake.JPG b/Images/deepfake.JPG deleted file mode 100644 index 4506889..0000000 Binary files a/Images/deepfake.JPG and /dev/null differ diff --git a/Images/ff_compare.png b/Images/ff_compare.png deleted file mode 100644 index f75129b..0000000 Binary files a/Images/ff_compare.png and /dev/null differ diff --git a/Images/model_architecture.JPG b/Images/model_architecture.JPG deleted file mode 100644 index 128e285..0000000 Binary files a/Images/model_architecture.JPG and /dev/null differ diff --git a/Images/ms_eff_gcvit.JPG b/Images/ms_eff_gcvit.JPG deleted file mode 100644 index b1ffced..0000000 Binary files a/Images/ms_eff_gcvit.JPG and /dev/null differ diff --git a/Images/multi_scale.JPG b/Images/multi_scale.JPG deleted file mode 100644 index 01d2f9e..0000000 Binary files a/Images/multi_scale.JPG and /dev/null differ diff --git a/README.md b/README.md index af97bc7..89c9ef3 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@

- DeepGuard Banner + DeepGuard Banner

@@ -143,7 +143,7 @@ Multi Scale Efficient Global Context Vision Transformer is an optimized multi-sc

- +

We utilizes two distinct types of self-attention to capture both long-range and short-range information across feature maps. @@ -154,7 +154,7 @@ We utilizes two distinct types of self-attention to capture both long-range and

- +

## 🧬 Model Zoo @@ -310,35 +310,148 @@ print(f"Deepfake Probability: {result:.4f}") ## 🎨 DeepFake AI Explainability -Deepfake detection requires high reliability and interpretability. DeepGuard integrates a robust Explainable AI (XAI) Toolkit to visualize the decision-making process of our hybrid models +Deepfake detection is only as trustworthy as its explanations. DeepGuard integrates a production-ready XAI Toolkit that visualizes where and why the model flags a face as manipulated — turning a black-box score into actionable forensic evidence. ⭐ Validated on hybrid CNN-ViT architectures, specifically `MS-EffViT` and `MS-EffGCViT`. -⭐ Dual-Branch Analysis: `Low-Level Branch`(focus on local forgery region), `High-Level Branch`(focus on global forgery region) +⭐ Dual-Branch Analysis: Dual-branch design mirrors the model's own multi-scale reasoning -### Deepfake Detection: XAI Methods by Multi-Scale Branch +### 🧠 How Dual-Branch XAI Works -| Branch | Method | 🎯 What it does | +

+ +

+ + +| Branch | Feature Map | Focus | Best For | +| ------ | ----------- | ----- | -------- | +| ![](https://img.shields.io/badge/Low_level-blue?style=flat-square) | High Resolution | Local Forgery artifacts | Skin texture, boundary blending, compression traces | +| ![](https://img.shields.io/badge/High_level-red?style=flat-square) | Low Resolution | Global Semantic Structure | Lighting inconsistency, facial geometry, Shadow artifacts | + +### 📐 XAI Methods + +Each method is assigned to the branch where it performs best empirically. + +| Branch | Method | 🎯 Core Idea | | :--- | :--- | :--- | -| ![](https://img.shields.io/badge/Low_level-blue?style=flat-square) | **HiResCAM** | Like GradCAM but element-wise multiply the activations with the gradients; provably guaranteed faithfulness for certain models | -| ![](https://img.shields.io/badge/Low_level-blue?style=flat-square) | **GradCAMElementWise** | Like GradCAM but element-wise multiply the activations with the gradients then apply a ReLU operation before summing | -| ![](https://img.shields.io/badge/Low_level-blue?style=flat-square) | **LayerCAM** | Spatially weight the activations by positive gradients. Works better especially in lower layers | +| **`low level`** | **HiResCAM** | Like GradCAM but element-wise multiply the activations with the gradients; provably guaranteed faithfulness for certain models | +| **`low level`** | **GradCAMElementWise** | Like GradCAM but element-wise multiply the activations with the gradients then apply a ReLU operation before summing | +| **`low level`** | **LayerCAM** | Spatially weight the activations by positive gradients. Works better especially in lower layers | | --- | --- | --- | --- | -| ![](https://img.shields.io/badge/High_level-red?style=flat-square) | **EigenGradCAM** | Like EigenCAM but with class discrimination: First principle component of Activations*Grad. Looks like GradCAM, but cleaner | -| ![](https://img.shields.io/badge/High_level-red?style=flat-square) | **GradCAM++** | Like GradCAM but uses second order gradients | -| ![](https://img.shields.io/badge/High_level-red?style=flat-square) | **XGradCAM** | Like GradCAM but scale the gradients by the normalized activations | +| **`high level`** | **EigenGradCAM** | Like EigenCAM but with class discrimination: First principle component of Activations*Grad. Looks like GradCAM, but cleaner | +| **`high level`** | **GradCAM++** | Like GradCAM but uses second order gradients | +| **`high level`** | **XGradCAM** | Like GradCAM but scale the gradients by the normalized activations | + +- **`aug_smooth`** applies TTA (horizontal flips) before averaging CAMs → smoother, more object-aligned maps +- **`eigen_smooth`** applies PCA noise reduction → retains dominant forgery pattern only + +### 💡 DeepFake XAI Usage -### Multi Scale Efficient Vision Transformer +**Low-Level Branch — Local Artifact Detection** + +```python +from explainability import HiResCAMExplainer, GradCAMElementWiseExplainer, LayerCAMExplainer + +explainer = HiResCAMExplainer( + model_name = "ms_eff_gcvit_b0", # or ms_eff_vit_b0, ms_eff_gcvit_b5, ms_eff_vit_b5 + dataset = "celeb_df_v2", # or ff++, kodf + branch_level = "low", +) +``` + +**High-Level Branch — Global Semantic Detection** +```python +from explainability import EigenGradCAMExplainer, GradCAMPlusPlusExplainer, XGradCAMExplainer + +explainer = EigenGradCAMExplainer( + model_name = "ms_eff_gcvit_b0", + dataset = "celeb_df_v2", + branch_level = "high", +) +``` + +### 🎨 Visualization Modes + +

+ +

+ +**1. Heatmap — Continuous activation distribution** + +```python +result = explainer.display_heatmap_on_image( + img_path = "path/to/image.jpg", + category = 1, # 0: Real, 1: Fake + threshold = 0.5, # binarization cutoff (0.5~1.0), or "auto" for Otsu + image_weight = 0.5, # 0.0: heatmap only ← → 1.0: original only + aug_smooth = False, # TTA smoothing (not supported on 'pro' models) + eigen_smooth = False, # PCA noise reduction +) +``` + +**2. Bounding Box — Discrete forgery region localization** + +```python +result = explainer.display_bbox_on_image( + img_path = "path/to/image.jpg", + category = 1, + threshold = 0.5, + thickness = 1, + aug_smooth = False, + eigen_smooth = False, +) +``` + +**3. Heatmap + BBox — Full overlay (recommended for reporting)** +```python +result = explainer.display_heatmap_bbox_on_image( + img_path = "path/to/image.jpg", + category = 1, + threshold = 0.5, + image_weight = 0.5, + aug_smooth = False, + eigen_smooth = False, +) +``` + +### 📊 Visual Results + +

+ + + + + + +
+

+ +#### MS-EFF-VIT — Low-Level Branch | Model | Branch-Level | Image | HiresCam | GradCamElementwise | LayerCam | | :--- | :---: | :---: | :---: | :---: | :---: | -| **⚡ ms-eff-vit-b0** | ![](https://img.shields.io/badge/Low_level_Branch-blue?style=flat-square) | | | | | -| **⚡ ms-eff-vit-b5** | ![](https://img.shields.io/badge/Low_level_Branch-blue?style=flat-square) | | | | | +| **⚡ ms-eff-vit-b0** | ![](https://img.shields.io/badge/Low_level_Branch-blue?style=flat-square) | | | | | +| **🔥 ms-eff-vit-b5** | ![](https://img.shields.io/badge/Low_level_Branch-blue?style=flat-square) | | | | | + +#### MS-Eff-ViT — High-Level Branch | Model | Branch-Level | Image | EigenGradCam | GradCamPlusPlus | XGradCam | | :--- | :---: | :---: | :---: | :---: | :---: | -| **⚡ ms-eff-vit-b0** | ![](https://img.shields.io/badge/High_level_Branch-red?style=flat-square) | | | | | -| **🔥 ms-eff-vit-b5** | ![](https://img.shields.io/badge/High_level_Branch-red?style=flat-square) | | | | | +| **⚡ ms-eff-vit-b0** | ![](https://img.shields.io/badge/High_level_Branch-red?style=flat-square) | | | | | +| **🔥 ms-eff-vit-b5** | ![](https://img.shields.io/badge/High_level_Branch-red?style=flat-square) | | | | | + +#### MS-EFF-GCVIT — Low-Level Branch +| Model | Branch-Level | Image | HiresCam | GradCamElementwise | LayerCam | +| :--- | :---: | :---: | :---: | :---: | :---: | +| **⚡ ms-eff-gcvit-b0** | ![](https://img.shields.io/badge/Low_level_Branch-blue?style=flat-square) | | | | | +| **🔥 ms-eff-gcvit-b5** | ![](https://img.shields.io/badge/Low_level_Branch-blue?style=flat-square) | | | | | + +#### MS-Eff-GCViT — High-Level Branch + +| Model | Branch-Level | Image | EigenGradCam | GradCamPlusPlus | XGradCam | +| :--- | :---: | :---: | :---: | :---: | :---: | +| **⚡ ms-eff-gcvit-b0** | ![](https://img.shields.io/badge/High_level_Branch-red?style=flat-square) | | | | | +| **🔥 ms-eff-gcvit-b5** | ![](https://img.shields.io/badge/High_level_Branch-red?style=flat-square) | | | | | ## 📊 Metrics and Evaluation for DeepFake XAI diff --git a/README_KR.md b/README_KR.md index 817282a..6234030 100644 --- a/README_KR.md +++ b/README_KR.md @@ -1,7 +1,7 @@ # Deepfakes Detection (딥페이크 탐지)

- DeepGuard Banner + DeepGuard Banner

@@ -135,7 +135,7 @@ DATA_ROOT/

- +

특징 맵 전체에 걸쳐 장거리(Long-range) 및 단거리(Short-range) 정보를 모두 캡처하기 위해 두 가지 유형의 셀프 어텐션을 활용합니다. @@ -144,7 +144,7 @@ DATA_ROOT/ - **Global Window Attention**: Swin Transformer와 달리, 이 모듈은 로컬 윈도우의 Key, Value와 상호작용하는 글로벌 쿼리(Global-queries)를 사용합니다. 이를 통해 각 로컬 영역이 전역 컨텍스트를 수용하게 함으로써 장거리 의존성을 효과적으로 파악하고 전체 공간 구조에 대한 포괄적인 이해를 제공합니다.

- +

## 🧬 모델 주(Model Zoo) diff --git a/Videos/display_img.py b/Videos/display_img.py deleted file mode 100644 index e412e12..0000000 --- a/Videos/display_img.py +++ /dev/null @@ -1,46 +0,0 @@ -import os -import argparse -import cv2 - -# ------------------------------- -# Display specific frame -# ------------------------------- -def display_img(video_path: str, frame_number: int) -> None: - cap = cv2.VideoCapture(video_path) - if not cap.isOpened(): - raise FileNotFoundError(f"Could not open video file: {video_path}") - - total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - if frame_number < 1 or frame_number > total_frames: - cap.release() - raise ValueError(f"Frame number {frame_number} is out of range (1-{total_frames})") - - cap.set(cv2.CAP_PROP_POS_FRAMES, frame_number - 1) - ret, frame = cap.read() - cap.release() - if not ret or frame is None: - raise RuntimeError(f"Failed to read frame {frame_number} from {video_path}") - - cv2.imshow(f"Frame {frame_number}/{total_frames}", frame) - print("Press any key to close window...") - cv2.waitKey(0) - cv2.destroyAllWindows() - -# ------------------------------- -# Main -# ------------------------------- -def main(): - parser = argparse.ArgumentParser(description="Play a video or display a specific frame") - parser.add_argument("--video_path", type=str, required=True, help="Path to video file") - parser.add_argument("--frame", type=int, default=1, help="Frame number in Video") - - args = parser.parse_args() - print(args) - - if not os.path.exists(args.video_path): - raise FileNotFoundError(f"Video file not found: {args.video_path}") - - display_img(args.video_path, args.frame) - -if __name__ == "__main__": - main() diff --git a/Videos/display_video.py b/Videos/display_video.py deleted file mode 100644 index bbdab81..0000000 --- a/Videos/display_video.py +++ /dev/null @@ -1,67 +0,0 @@ -import os -import argparse -import cv2 - -# ------------------------------- -# Display whole video -# ------------------------------- - -## cv2.CAP_PROP_FPS: Video Frame per Second -## cv2.CAP_PROP_FRAME_COUNT: Video Total Frames -## cv2.CAP_PROP_FRAME_HEIGHT: Frame Height -## cv2.CAP_PROP_FRAME_WIDTH: Frame Width - -def display_video(video_path: str, resize_width: int = None) -> None: - cap = cv2.VideoCapture(video_path) - if not cap.isOpened(): - raise FileNotFoundError(f"Could not open video file: {video_path}") - - fps = cap.get(cv2.CAP_PROP_FPS) - total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - delay = int(1000 / fps) if fps > 0 else 30 - - h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - - print(f"🎬 Playing video: {video_path}") - print(f"📸 FPS: {fps:.2f}, Total_Frames: {total_frames}, Delay: {delay}ms") - print(f"📸 Frame Width: {w}, Frame Height: {h}") - print("Press 'q' to stop playback.") - - while True: - ret, frame = cap.read() - if not ret: - print("✅ Video playback finished.") - break - - if resize_width is not None: - new_h = int(resize_width * h / w) - frame = cv2.resize(frame, (resize_width, new_h)) - - cv2.imshow("Video Playback", frame) - - if cv2.waitKey(delay) & 0xFF == ord('q'): - print("🛑 Video stopped by user.") - break - - cap.release() - cv2.destroyAllWindows() - -# ------------------------------- -# Main -# ------------------------------- -def main(): - parser = argparse.ArgumentParser(description="Play a video or display a specific frame") - parser.add_argument("--video_path", type=str, required=True, help="Path to video file") - parser.add_argument("--width", type=int, default=None, help="Resize width (optional, used only in 'video' mode')") - args = parser.parse_args() - - print(args) - - if not os.path.exists(args.video_path): - raise FileNotFoundError(f"Video file not found: {args.video_path}") - - display_video(args.video_path, args.width) - -if __name__ == "__main__": - main() diff --git a/Videos/sample1.mp4 b/Videos/sample1.mp4 deleted file mode 100644 index 6dd65e4..0000000 Binary files a/Videos/sample1.mp4 and /dev/null differ diff --git a/Videos/sample2.mp4 b/Videos/sample2.mp4 deleted file mode 100644 index d9abaca..0000000 Binary files a/Videos/sample2.mp4 and /dev/null differ diff --git a/Videos/sample3.mp4 b/Videos/sample3.mp4 deleted file mode 100644 index 3db3330..0000000 Binary files a/Videos/sample3.mp4 and /dev/null differ diff --git a/deepguard/MS_EffGCViT.md b/deepguard/MS_EffGCViT.md index 9b46eee..2a5aa3b 100644 --- a/deepguard/MS_EffGCViT.md +++ b/deepguard/MS_EffGCViT.md @@ -14,7 +14,7 @@ This Repository presents the PyTorch implementation of **Multi Scale Efficient G This model is a **frame-level** and **spatial-domain** architecture, designed to perform classification tasks on both **static images** and **video sequences** - + ## 💥 News 💥 @@ -32,23 +32,23 @@ MS_Eff_GCViT achieves state-of-the-art(SOTA) results across deepfake video class ### Test Result of Celeb_DF(v2) - +
Test Result of FaceForensics++ - +
Test Result of KoDF - +
## Model Indroduction Multi Scale Efficient Global Context Vision Transformer is an optimized multi-scale hybrid architecture that integrates CNN-driven spatial inductive bias with hierarchical attention mechanisms to effectively identify subtle(local) artifacts and macro(global) artifacts for robust deepfake forensics." - + ### Part 1: CNN-based Patch Embedding for Spatial Inductive Bias @@ -61,7 +61,7 @@ While traditional Vision Transformers (ViTs) utilize a Linear Projection for pat We utilizes two distinct types of self-attention to capture both long-range and short-range information across feature maps. - + - **Local Window Attention**: this model efficiently captures local textures and precise spatial details while maintaining linear computational complexity relative to the image size. @@ -173,4 +173,30 @@ import deepguard model = timm.create_model("ms_eff_gcvit_b0", pretrained=True, dataset="ff++") model = timm.create_model("ms_eff_gcvit_b5", pretrained=True, dataset="kodf") -``` \ No newline at end of file +``` + +## 📊 Visual Results + +

+ + + + + + +
+

+ +### MS-EFF-GCVIT — Low-Level Branch + +| Model | Branch-Level | Image | HiresCam | GradCamElementwise | LayerCam | +| :--- | :---: | :---: | :---: | :---: | :---: | +| **⚡ ms-eff-gcvit-b0** | ![](https://img.shields.io/badge/Low_level_Branch-blue?style=flat-square) | | | | | +| **🔥 ms-eff-gcvit-b5** | ![](https://img.shields.io/badge/Low_level_Branch-blue?style=flat-square) | | | | | + +### MS-Eff-GCViT — High-Level Branch + +| Model | Branch-Level | Image | EigenGradCam | GradCamPlusPlus | XGradCam | +| :--- | :---: | :---: | :---: | :---: | :---: | +| **⚡ ms-eff-gcvit-b0** | ![](https://img.shields.io/badge/High_level_Branch-red?style=flat-square) | | | | | +| **🔥 ms-eff-gcvit-b5** | ![](https://img.shields.io/badge/High_level_Branch-red?style=flat-square) | | | | | diff --git a/deepguard/MS_EffViT.md b/deepguard/MS_EffViT.md index 6d4a76e..8ee7312 100644 --- a/deepguard/MS_EffViT.md +++ b/deepguard/MS_EffViT.md @@ -26,11 +26,11 @@ MS_Eff_ViT achieves state-of-the-art(SOTA) results across deepfake video classif ### Test Result of Celeb_DF(v2) - +
Test Result of FaceForensics++ - +
## Model Introduction @@ -142,4 +142,20 @@ import deepguard model = timm.create_model("ms_eff_vit_b0", pretrained=True, dataset="celeb_df_v2") model = timm.create_model("ms_eff_vit_b5", pretrained=True, dataset="ff++") -``` \ No newline at end of file +``` + +## 📊 Visual Results + +### MS-EFF-VIT — Low-Level Branch + +| Model | Branch-Level | Image | HiresCam | GradCamElementwise | LayerCam | +| :--- | :---: | :---: | :---: | :---: | :---: | +| **⚡ ms-eff-vit-b0** | ![](https://img.shields.io/badge/Low_level_Branch-blue?style=flat-square) | | | | | +| **🔥 ms-eff-vit-b5** | ![](https://img.shields.io/badge/Low_level_Branch-blue?style=flat-square) | | | | | + +### MS-Eff-ViT — High-Level Branch + +| Model | Branch-Level | Image | EigenGradCam | GradCamPlusPlus | XGradCam | +| :--- | :---: | :---: | :---: | :---: | :---: | +| **⚡ ms-eff-vit-b0** | ![](https://img.shields.io/badge/High_level_Branch-red?style=flat-square) | | | | | +| **🔥 ms-eff-vit-b5** | ![](https://img.shields.io/badge/High_level_Branch-red?style=flat-square) | | | | | diff --git a/docs/architectures/dual_branch.gif b/docs/architectures/dual_branch.gif new file mode 100644 index 0000000..1ad1713 Binary files /dev/null and b/docs/architectures/dual_branch.gif differ diff --git a/docs/architectures/high_branch.gif b/docs/architectures/high_branch.gif new file mode 100644 index 0000000..edc2e81 Binary files /dev/null and b/docs/architectures/high_branch.gif differ diff --git a/docs/architectures/low_branch.gif b/docs/architectures/low_branch.gif new file mode 100644 index 0000000..4bcfb7f Binary files /dev/null and b/docs/architectures/low_branch.gif differ diff --git a/docs/architectures/ms_eff_gcvit.JPG b/docs/architectures/ms_eff_gcvit.JPG new file mode 100644 index 0000000..14af884 Binary files /dev/null and b/docs/architectures/ms_eff_gcvit.JPG differ diff --git a/Images/window_attention.JPG b/docs/architectures/window_attention.JPG similarity index 100% rename from Images/window_attention.JPG rename to docs/architectures/window_attention.JPG diff --git a/Images/celeb_df_v2_gcvit.png b/docs/benchmarks/celeb_df_v2_gcvit.png similarity index 100% rename from Images/celeb_df_v2_gcvit.png rename to docs/benchmarks/celeb_df_v2_gcvit.png diff --git a/Images/celeb_df_v2_vit.png b/docs/benchmarks/celeb_df_v2_vit.png similarity index 100% rename from Images/celeb_df_v2_vit.png rename to docs/benchmarks/celeb_df_v2_vit.png diff --git a/Images/ff_gcvit.png b/docs/benchmarks/ff_gcvit.png similarity index 100% rename from Images/ff_gcvit.png rename to docs/benchmarks/ff_gcvit.png diff --git a/Images/ff_vit.png b/docs/benchmarks/ff_vit.png similarity index 100% rename from Images/ff_vit.png rename to docs/benchmarks/ff_vit.png diff --git a/Images/kodf_gcvit.png b/docs/benchmarks/kodf_gcvit.png similarity index 100% rename from Images/kodf_gcvit.png rename to docs/benchmarks/kodf_gcvit.png diff --git a/Images/deepfake2.png b/docs/samples/deepfake_thumbnails.png similarity index 100% rename from Images/deepfake2.png rename to docs/samples/deepfake_thumbnails.png diff --git a/docs/samples/images/asian/asian_fake_1.JPG b/docs/samples/images/asian/asian_fake_1.JPG new file mode 100644 index 0000000..c835a50 Binary files /dev/null and b/docs/samples/images/asian/asian_fake_1.JPG differ diff --git a/docs/samples/images/asian/asian_fake_2.JPG b/docs/samples/images/asian/asian_fake_2.JPG new file mode 100644 index 0000000..2e5cb74 Binary files /dev/null and b/docs/samples/images/asian/asian_fake_2.JPG differ diff --git a/Images/deepfake/image/asian_real_1.JPG b/docs/samples/images/asian/asian_real_1.JPG similarity index 100% rename from Images/deepfake/image/asian_real_1.JPG rename to docs/samples/images/asian/asian_real_1.JPG diff --git a/Images/deepfake/image/western_fake_1.JPG b/docs/samples/images/western/western_fake_1.JPG similarity index 100% rename from Images/deepfake/image/western_fake_1.JPG rename to docs/samples/images/western/western_fake_1.JPG diff --git a/Images/deepfake/image/western_fake_2.JPG b/docs/samples/images/western/western_fake_2.JPG similarity index 100% rename from Images/deepfake/image/western_fake_2.JPG rename to docs/samples/images/western/western_fake_2.JPG diff --git a/Images/deepfake/image/western_real_1.JPG b/docs/samples/images/western/western_real_1.JPG similarity index 100% rename from Images/deepfake/image/western_real_1.JPG rename to docs/samples/images/western/western_real_1.JPG diff --git a/Images/deepfake/image/western_real_2.JPG b/docs/samples/images/western/western_real_2.JPG similarity index 100% rename from Images/deepfake/image/western_real_2.JPG rename to docs/samples/images/western/western_real_2.JPG diff --git a/Images/deepfake/video/western_fake.mp4 b/docs/samples/videos/western/western_fake.mp4 similarity index 100% rename from Images/deepfake/video/western_fake.mp4 rename to docs/samples/videos/western/western_fake.mp4 diff --git a/Images/deepfake/video/western_real.mp4 b/docs/samples/videos/western/western_real.mp4 similarity index 100% rename from Images/deepfake/video/western_real.mp4 rename to docs/samples/videos/western/western_real.mp4 diff --git a/docs/xai-results/ms_eff_gcvit_b0_high_eigengradcam.JPG b/docs/xai-results/ms_eff_gcvit_b0_high_eigengradcam.JPG new file mode 100644 index 0000000..402b195 Binary files /dev/null and b/docs/xai-results/ms_eff_gcvit_b0_high_eigengradcam.JPG differ diff --git a/docs/xai-results/ms_eff_gcvit_b0_high_gradcamplusplus.JPG b/docs/xai-results/ms_eff_gcvit_b0_high_gradcamplusplus.JPG new file mode 100644 index 0000000..ebf2788 Binary files /dev/null and b/docs/xai-results/ms_eff_gcvit_b0_high_gradcamplusplus.JPG differ diff --git a/docs/xai-results/ms_eff_gcvit_b0_high_xgradcam.JPG b/docs/xai-results/ms_eff_gcvit_b0_high_xgradcam.JPG new file mode 100644 index 0000000..7192d77 Binary files /dev/null and b/docs/xai-results/ms_eff_gcvit_b0_high_xgradcam.JPG differ diff --git a/docs/xai-results/ms_eff_gcvit_b0_low_gradcamelementwise.JPG b/docs/xai-results/ms_eff_gcvit_b0_low_gradcamelementwise.JPG new file mode 100644 index 0000000..b5a003a Binary files /dev/null and b/docs/xai-results/ms_eff_gcvit_b0_low_gradcamelementwise.JPG differ diff --git a/docs/xai-results/ms_eff_gcvit_b0_low_hirescam.JPG b/docs/xai-results/ms_eff_gcvit_b0_low_hirescam.JPG new file mode 100644 index 0000000..e12e9b9 Binary files /dev/null and b/docs/xai-results/ms_eff_gcvit_b0_low_hirescam.JPG differ diff --git a/docs/xai-results/ms_eff_gcvit_b0_low_layercam.JPG b/docs/xai-results/ms_eff_gcvit_b0_low_layercam.JPG new file mode 100644 index 0000000..1241248 Binary files /dev/null and b/docs/xai-results/ms_eff_gcvit_b0_low_layercam.JPG differ diff --git a/docs/xai-results/ms_eff_gcvit_b5_high_eigengradcam.JPG b/docs/xai-results/ms_eff_gcvit_b5_high_eigengradcam.JPG new file mode 100644 index 0000000..1403305 Binary files /dev/null and b/docs/xai-results/ms_eff_gcvit_b5_high_eigengradcam.JPG differ diff --git a/docs/xai-results/ms_eff_gcvit_b5_high_gradcamplusplus.JPG b/docs/xai-results/ms_eff_gcvit_b5_high_gradcamplusplus.JPG new file mode 100644 index 0000000..b6dea9d Binary files /dev/null and b/docs/xai-results/ms_eff_gcvit_b5_high_gradcamplusplus.JPG differ diff --git a/docs/xai-results/ms_eff_gcvit_b5_high_xgradcam.JPG b/docs/xai-results/ms_eff_gcvit_b5_high_xgradcam.JPG new file mode 100644 index 0000000..7bb381a Binary files /dev/null and b/docs/xai-results/ms_eff_gcvit_b5_high_xgradcam.JPG differ diff --git a/docs/xai-results/ms_eff_gcvit_b5_low_gradcamelementwise.JPG b/docs/xai-results/ms_eff_gcvit_b5_low_gradcamelementwise.JPG new file mode 100644 index 0000000..089b17a Binary files /dev/null and b/docs/xai-results/ms_eff_gcvit_b5_low_gradcamelementwise.JPG differ diff --git a/docs/xai-results/ms_eff_gcvit_b5_low_hirescam.JPG b/docs/xai-results/ms_eff_gcvit_b5_low_hirescam.JPG new file mode 100644 index 0000000..f24e9c0 Binary files /dev/null and b/docs/xai-results/ms_eff_gcvit_b5_low_hirescam.JPG differ diff --git a/docs/xai-results/ms_eff_gcvit_b5_low_layercam.JPG b/docs/xai-results/ms_eff_gcvit_b5_low_layercam.JPG new file mode 100644 index 0000000..d9931b5 Binary files /dev/null and b/docs/xai-results/ms_eff_gcvit_b5_low_layercam.JPG differ diff --git a/Images/deepfake-xai/ms_eff_vit_b0_high_eigengradcam.JPG b/docs/xai-results/ms_eff_vit_b0_high_eigengradcam.JPG similarity index 100% rename from Images/deepfake-xai/ms_eff_vit_b0_high_eigengradcam.JPG rename to docs/xai-results/ms_eff_vit_b0_high_eigengradcam.JPG diff --git a/Images/deepfake-xai/ms_eff_vit_b0_high_gradcamplusplus.JPG b/docs/xai-results/ms_eff_vit_b0_high_gradcamplusplus.JPG similarity index 100% rename from Images/deepfake-xai/ms_eff_vit_b0_high_gradcamplusplus.JPG rename to docs/xai-results/ms_eff_vit_b0_high_gradcamplusplus.JPG diff --git a/Images/deepfake-xai/ms_eff_vit_b0_high_xgradcam.JPG b/docs/xai-results/ms_eff_vit_b0_high_xgradcam.JPG similarity index 100% rename from Images/deepfake-xai/ms_eff_vit_b0_high_xgradcam.JPG rename to docs/xai-results/ms_eff_vit_b0_high_xgradcam.JPG diff --git a/Images/deepfake-xai/ms_eff_vit_b0_low_gradcamelementwise.JPG b/docs/xai-results/ms_eff_vit_b0_low_gradcamelementwise.JPG similarity index 100% rename from Images/deepfake-xai/ms_eff_vit_b0_low_gradcamelementwise.JPG rename to docs/xai-results/ms_eff_vit_b0_low_gradcamelementwise.JPG diff --git a/Images/deepfake-xai/ms_eff_vit_b0_low_hirescam.JPG b/docs/xai-results/ms_eff_vit_b0_low_hirescam.JPG similarity index 100% rename from Images/deepfake-xai/ms_eff_vit_b0_low_hirescam.JPG rename to docs/xai-results/ms_eff_vit_b0_low_hirescam.JPG diff --git a/Images/deepfake-xai/ms_eff_vit_b0_low_layercam.JPG b/docs/xai-results/ms_eff_vit_b0_low_layercam.JPG similarity index 100% rename from Images/deepfake-xai/ms_eff_vit_b0_low_layercam.JPG rename to docs/xai-results/ms_eff_vit_b0_low_layercam.JPG diff --git a/Images/deepfake-xai/ms_eff_vit_b5_high_eigengradcam.JPG b/docs/xai-results/ms_eff_vit_b5_high_eigengradcam.JPG similarity index 100% rename from Images/deepfake-xai/ms_eff_vit_b5_high_eigengradcam.JPG rename to docs/xai-results/ms_eff_vit_b5_high_eigengradcam.JPG diff --git a/Images/deepfake-xai/ms_eff_vit_b5_high_gradcamplusplus.JPG b/docs/xai-results/ms_eff_vit_b5_high_gradcamplusplus.JPG similarity index 100% rename from Images/deepfake-xai/ms_eff_vit_b5_high_gradcamplusplus.JPG rename to docs/xai-results/ms_eff_vit_b5_high_gradcamplusplus.JPG diff --git a/Images/deepfake-xai/ms_eff_vit_b5_high_xgradcam.JPG b/docs/xai-results/ms_eff_vit_b5_high_xgradcam.JPG similarity index 100% rename from Images/deepfake-xai/ms_eff_vit_b5_high_xgradcam.JPG rename to docs/xai-results/ms_eff_vit_b5_high_xgradcam.JPG diff --git a/Images/deepfake-xai/ms_eff_vit_b5_low_gradcamelementwise.JPG b/docs/xai-results/ms_eff_vit_b5_low_gradcamelementwise.JPG similarity index 100% rename from Images/deepfake-xai/ms_eff_vit_b5_low_gradcamelementwise.JPG rename to docs/xai-results/ms_eff_vit_b5_low_gradcamelementwise.JPG diff --git a/Images/deepfake-xai/ms_eff_vit_b5_low_hirescam.JPG b/docs/xai-results/ms_eff_vit_b5_low_hirescam.JPG similarity index 100% rename from Images/deepfake-xai/ms_eff_vit_b5_low_hirescam.JPG rename to docs/xai-results/ms_eff_vit_b5_low_hirescam.JPG diff --git a/Images/deepfake-xai/ms_eff_vit_b5_low_layercam.JPG b/docs/xai-results/ms_eff_vit_b5_low_layercam.JPG similarity index 100% rename from Images/deepfake-xai/ms_eff_vit_b5_low_layercam.JPG rename to docs/xai-results/ms_eff_vit_b5_low_layercam.JPG diff --git a/docs/xai-results/xai_demo.gif b/docs/xai-results/xai_demo.gif new file mode 100644 index 0000000..a1d8c34 Binary files /dev/null and b/docs/xai-results/xai_demo.gif differ diff --git a/explainability/explainer/cam_explainer.py b/explainability/explainer/cam_explainer.py index f630055..c5d9a42 100644 --- a/explainability/explainer/cam_explainer.py +++ b/explainability/explainer/cam_explainer.py @@ -4,35 +4,12 @@ import numpy as np from abc import ABC, abstractmethod from explainability.utils.model_targets import BinaryClassifierOutputTarget -from explainability.utils.image import show_cam_on_image, deprocess_image, remove_padding_and_resize +from explainability.utils.image import ( + show_cam_on_image, show_bbox_on_image, + remove_padding_and_resize, draw_label +) from explainability.explainer.base_explainer import BaseExplainer - -def _draw_label(img: np.ndarray, text: str, x: int, y: int, prob_ratio: float = 1.0): - """Confidence label with a clean pill-style background. - - prob_ratio: 0~1, controls bg color (green→red) - """ - font = cv2.FONT_HERSHEY_SIMPLEX - font_scale = 0.52 - thickness = 1 - pad = 4 - - (tw, th), baseline = cv2.getTextSize(text, font, font_scale, thickness) - - rx1, ry1 = x - pad, y - th - pad - rx2, ry2 = x + tw + pad, y + baseline + pad - - r = int(min(255, 2 * prob_ratio * 255)) - g = int(min(255, 2 * (1 - prob_ratio) * 255)) - bg_color = (r, g, 30) # RGB: green → red - - overlay = img.copy() - cv2.rectangle(overlay, (rx1, ry1), (rx2, ry2), bg_color, -1) - cv2.addWeighted(overlay, 0.75, img, 0.25, 0, img) - cv2.putText(img, text, (x, y), font, font_scale, (255, 255, 255), thickness, cv2.LINE_AA) - - class CAMExplainer(BaseExplainer, ABC): def __init__(self, **kwargs): super().__init__(**kwargs) @@ -261,28 +238,10 @@ def display_bbox_on_image(self, img_path: str, **kwargs) -> np.ndarray: eigen_smooth = kwargs.get("eigen_smooth", False) ) - binary_mask = self._get_binary_mask(cam_recovered, kwargs.get("threshold", "auto")) - - contours, _ = cv2.findContours(binary_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - annotated_img = face.copy() - for c in contours: - x, y, w, h = cv2.boundingRect(c) - roi_prob = float(np.max(cam_recovered[y:y + h, x:x + w])) - - r = int(min(255, 2 * roi_prob * 255)) - g = int(min(255, 2 * (1 - roi_prob) * 255)) - box_color = (r, g, 30) # RGB - - cv2.rectangle(annotated_img, (x, y), (x + w, y + h), box_color, kwargs.get("thickness", 1)) - - _draw_label( - annotated_img, - text=f"{roi_prob * 100:.1f}%", - x=x, - y=max(y - 6, 14), - prob_ratio=roi_prob, - ) + + binary_mask = self._get_binary_mask(cam_recovered, kwargs.get("threshold", "auto")) + show_bbox_on_image(annotated_img, cam_recovered, binary_mask, kwargs.get("thickness", 1)) return annotated_img @@ -308,27 +267,53 @@ def display_bbox_from_array(self, face: np.ndarray, **kwargs) -> np.ndarray: eigen_smooth = kwargs.get("eigen_smooth", False) ) + annotated_img = face.copy() + binary_mask = self._get_binary_mask(cam_recovered, kwargs.get("threshold", "auto")) + show_bbox_on_image(annotated_img, cam_recovered, binary_mask, kwargs.get("thickness", 1)) + + return annotated_img + + def display_heatmap_bbox_on_image(self, img_path: str, **kwargs) -> np.ndarray: + cam_recovered, face = self._prepare_cam( + img_path, + category = kwargs.get("category", 1), + aug_smooth = kwargs.get("aug_smooth", False), + eigen_smooth = kwargs.get("eigen_smooth", False) + ) - contours, _ = cv2.findContours(binary_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + binary_mask = self._get_binary_mask(cam_recovered, kwargs.get("threshold", "auto")) + cam_recovered = np.where(binary_mask > 0, cam_recovered, 0.0) + + heatmap = show_cam_on_image( + np.float32(face) / 255.0, + cam_recovered, + use_rgb = True, + colormap = kwargs.get("colormap", cv2.COLORMAP_JET), + image_weight = kwargs.get("image_weight", 0.5), + ) - annotated_img = face.copy() - for c in contours: - x, y, w, h = cv2.boundingRect(c) - roi_prob = float(np.max(cam_recovered[y:y + h, x:x + w])) - - r = int(min(255, 2 * roi_prob * 255)) - g = int(min(255, 2 * (1 - roi_prob) * 255)) - box_color = (r, g, 30) # RGB - - cv2.rectangle(annotated_img, (x, y), (x + w, y + h), box_color, kwargs.get("thickness", 1)) - - _draw_label( - annotated_img, - text=f"{roi_prob * 100:.1f}%", - x=x, - y=max(y - 6, 14), - prob_ratio=roi_prob, + show_bbox_on_image(heatmap, cam_recovered, binary_mask, kwargs.get("thickness", 1)) + return heatmap + + def display_heatmap_bbox_on_image_from_array(self, face: np.ndarray, **kwargs) -> np.ndarray: + cam_recovered, face = self._prepare_cam_from_array( + face, + category = kwargs.get("category", 1), + aug_smooth = kwargs.get("aug_smooth", False), + eigen_smooth = kwargs.get("eigen_smooth", False) ) - - return annotated_img \ No newline at end of file + + binary_mask = self._get_binary_mask(cam_recovered, kwargs.get("threshold", "auto")) + cam_recovered = np.where(binary_mask > 0, cam_recovered, 0.0) + + heatmap = show_cam_on_image( + np.float32(face) / 255.0, + cam_recovered, + use_rgb = True, + colormap = kwargs.get("colormap", cv2.COLORMAP_JET), + image_weight = kwargs.get("image_weight", 0.5), + ) + + show_bbox_on_image(heatmap, cam_recovered, binary_mask, kwargs.get("thickness", 1)) + return heatmap \ No newline at end of file diff --git a/explainability/utils/image.py b/explainability/utils/image.py index 92a5ba4..30cef9c 100644 --- a/explainability/utils/image.py +++ b/explainability/utils/image.py @@ -35,16 +35,20 @@ def show_cam_on_image(img: np.ndarray, cam = cam / np.max(cam) return np.uint8(255 * cam) -def deprocess_image(tensor): - mean = np.array([0.485, 0.456, 0.406]) - std = np.array([0.229, 0.224, 0.225]) +def show_bbox_on_image(annotated_img: np.ndarray, cam_recovered: np.ndarray, binary_mask: np.ndarray, thickness: int = 1): + contours, _ = cv2.findContours(binary_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - # (1, C, H, W -> H, W, C) - img = tensor.squeeze().cpu().numpy().transpose(1, 2, 0) - img = (img * std) + mean - img = np.clip(img, 0, 1) # range(0~1) + for c in contours: + x, y, bw, bh = cv2.boundingRect(c) + roi_prob = float(np.max(cam_recovered[y:y + bh, x:x + bw])) - return np.float32(img) + r = int(min(255, 2 * roi_prob * 255)) + g = int(min(255, 2 * (1 - roi_prob) * 255)) + box_color = (r, g, 30) + + cv2.rectangle(annotated_img, (x, y), (x + bw, y + bh), box_color, thickness) + + draw_label(annotated_img, f"{roi_prob * 100:.0f}%", x, max(y - 5, 12), roi_prob) def remove_padding_and_resize(cam, orig_shape, target_shape): orig_h, orig_w = orig_shape[:2] # (H,W,3) @@ -58,4 +62,25 @@ def remove_padding_and_resize(cam, orig_shape, target_shape): # Remove Padding unpadded_cam = cam[pad_h:pad_h+new_h, pad_w:pad_w+new_w] - return cv2.resize(unpadded_cam, (orig_w, orig_h)) \ No newline at end of file + return cv2.resize(unpadded_cam, (orig_w, orig_h)) + +def draw_label(img: np.ndarray, text: str, x: int, y: int, prob_ratio: float = 1.0): + h, w = img.shape[:2] + font = cv2.FONT_HERSHEY_SIMPLEX + font_scale = max(0.28, min(w,h) / 1000) # 이미지 크기 기반 동적 스케일 + thickness = 1 + pad = 4 + + (tw, th), baseline = cv2.getTextSize(text, font, font_scale, thickness) + + rx1, ry1 = x - pad, y - th - pad + rx2, ry2 = x + tw + pad, y + baseline + pad + + r = int(min(255, 2 * prob_ratio * 255)) + g = int(min(255, 2 * (1 - prob_ratio) * 255)) + bg_color = (r, g, 30) # RGB: green → red + + overlay = img.copy() + cv2.rectangle(overlay, (rx1, ry1), (rx2, ry2), bg_color, -1) + cv2.addWeighted(overlay, 0.6, img, 0.4, 0, img) + cv2.putText(img, text, (x, y), font, font_scale, (255, 255, 255), thickness, cv2.LINE_AA) diff --git a/labels.txt b/labels.txt deleted file mode 100644 index 505ee95..0000000 --- a/labels.txt +++ /dev/null @@ -1,4 +0,0 @@ -label2id = { - "REAL": 0, - "FAKE": 1, -} \ No newline at end of file