-
-
-
+
@@ -43,6 +46,8 @@
- [📈 Model Evaluation](#-model-evaluation) - Benchmarking results
- [💻 Model Usage](#-model-usage) - How to integrate DeepGuard models into your own Python code or via timm
- [🔮 Predict Image & Video](#-predict-image--video) - Simple Inference examples for detecting deepfakes in image and video
+- [🎨 DeepFake AI Explainability](#-deepfake-ai-explainability) - Visualizing model focus using Grad-CAM and attention maps
+- [📊 Metrics and Evaluation for DeepFake XAI](#-metrics-and-evaluation-for-deepfake-xai) - Quantitative assessment of explanation reliability
- [📬 Authors](#-authors)
- [📝 Reference](#-reference)
- [⚖️ License](#-license)
@@ -303,6 +308,41 @@ print(f"Deepfake Probability: {result:.4f}")
```
+## 🎨 DeepFake AI Explainability
+
+Deepfake detection requires high reliability and interpretability. DeepGuard integrates a robust Explainable AI (XAI) Toolkit to visualize the decision-making process of our hybrid models
+
+⭐ Validated on hybrid CNN-ViT architectures, specifically `MS-EffViT` and `MS-EffGCViT`.
+⭐ Dual-Branch Analysis: `Low-Level Branch`(focus on local forgery region), `High-Level Branch`(focus on global forgery region)
+
+### Deepfake Detection: XAI Methods by Multi-Scale Branch
+
+| Branch | Method | 🎯 What it does |
+| :--- | :--- | :--- |
+|  | **HiResCAM** | Like GradCAM but element-wise multiply the activations with the gradients; provably guaranteed faithfulness for certain models |
+|  | **GradCAMElementWise** | Like GradCAM but element-wise multiply the activations with the gradients then apply a ReLU operation before summing |
+|  | **LayerCAM** | Spatially weight the activations by positive gradients. Works better especially in lower layers |
+| --- | --- | --- | --- |
+|  | **EigenGradCAM** | Like EigenCAM but with class discrimination: First principle component of Activations*Grad. Looks like GradCAM, but cleaner |
+|  | **GradCAM++** | Like GradCAM but uses second order gradients |
+|  | **XGradCAM** | Like GradCAM but scale the gradients by the normalized activations |
+
+### Multi Scale Efficient Vision Transformer
+
+| Model | Branch-Level | Image | HiresCam | GradCamElementwise | LayerCam |
+| :--- | :---: | :---: | :---: | :---: | :---: |
+| **⚡ ms-eff-vit-b0** |  | 
| 
| 
| 
|
+| **⚡ ms-eff-vit-b5** |  | | 
| 
| 
|
+
+| Model | Branch-Level | Image | EigenGradCam | GradCamPlusPlus | XGradCam |
+| :--- | :---: | :---: | :---: | :---: | :---: |
+| **⚡ ms-eff-vit-b0** |  | | 
| 
| 
|
+| **🔥 ms-eff-vit-b5** |  | | 
| 
| 
|
+
+
+
+## 📊 Metrics and Evaluation for DeepFake XAI
+
## 📬 Authors
@@ -323,6 +363,7 @@ _**This project was developed as a Senior Graduation Project by the Department o
6. [`deepfake-detection-project-v4`](https://github.com/ameencaslam/deepfake-detection-project-v4) - _Multiple Deep Learning Models by Ameen Caslam_
7. [`Awesome-Deepfake-Detection`](https://github.com/Daisy-Zhang/Awesome-Deepfakes-Detection
) - _A curated list of tools, papers and code by Daisy Zhang_
+8. [`Pytorch-Grad-Cam`](https://github.com/jacobgil/pytorch-grad-cam) - _Advanced Visual Explanations for PyTorch Models_
## ⚖️ License
diff --git a/explainability/__init__.py b/explainability/__init__.py
index e69de29..b547123 100644
--- a/explainability/__init__.py
+++ b/explainability/__init__.py
@@ -0,0 +1,13 @@
+from explainability.explainer.base_explainer import BaseExplainer
+from explainability.explainer.cam_explainer import CAMExplainer
+from explainability.explainer.gradient_explainer import (
+ GradCAMExplainer, GradCAMPlusPlusExplainer, GradCAMElementWiseExplainer,
+ XGradCAMExplainer, HiResCAMExplainer
+)
+from explainability.explainer.eigenvalue_explainer import (
+ EigenCAMExplainer, EigenGradCAMExplainer, KPCACAMExplainer
+)
+from explainability.explainer.perturbation_explainer import (
+ ScoreCAMExplainer, FEMExplainer
+)
+from explainability.explainer.misc_explainer import LayerCAMExplainer
\ No newline at end of file
diff --git a/explainability/base_explainer.py b/explainability/base_explainer.py
deleted file mode 100644
index 1a54186..0000000
--- a/explainability/base_explainer.py
+++ /dev/null
@@ -1,113 +0,0 @@
-import cv2
-import timm
-import torch
-import numpy as np
-from pathlib import Path
-from typing import List, Optional, Tuple
-from preprocess.face_detector import FaceDetector2
-from deepguard.data import get_test_transforms
-from explainability.utils.reshape_transforms import reshape_transform_vit, reshape_transform_gcvit
-
-DATASETS = {"celeb_df_v2", "ff++", "kodf"}
-MODELS = {"ms_eff_vit_b0", "ms_eff_vit_b5", "ms_eff_gcvit_b0", "ms_eff_gcvit_b5"}
-
-class BaseExplainer:
- def __init__(
- self,
- model_name: str, # ms_eff_vit_b0 | ms_eff_vit_b5 | ms_eff_gcvit_b0 | ms_eff_gcvit_b5
- dataset: str, # celeb_df_v2 | ff++ | both
- margin_ratio: float = 0.2,
- conf_thres: float = 0.5,
- category: int = 1, # 0: real | 1: fake
- branch_level: str = "both", # low | high | both
- l_stage_idx: int = -1 # gcvit low branch stage index (0~3)
- ):
- self._validate_inputs(model_name, dataset, branch_level)
-
- self.device = "cuda:0" if torch.cuda.is_available() else 'cpu'
- self.margin_ratio = margin_ratio
- self.category = category
- self.model_name = model_name
- self.branch_level = branch_level
- self.l_stage_idx = l_stage_idx
-
- # Parse Model Metadata
- self.model_variant = model_name.split("_")[-1] # b0, b5
- self.transformer_type = model_name.split("_")[-2] # vit, gcvit
-
- # Model & Tools setup
- self.model = timm.create_model(model_name, pretrained=True, dataset=dataset)
- self.face_detector = FaceDetector2(conf_thres)
- self.img_size = [224,224] if self.model_variant == "b0" else [384,384]
- self.transforms = get_test_transforms(img_size=self.img_size)
-
- self.model.to(self.device).eval()
-
- # Explainability setup
- self.reshape_fn = self._get_reshape_fn()
- self.target_layers = self._resolve_target_layers()
-
- def _validate_inputs(self, model_name: str, dataset: str, branch_level: str):
- if model_name not in MODELS:
- raise ValueError(f"Unsupported model: '{model_name}'. Must be one of {MODELS}")
- if dataset not in DATASETS:
- raise ValueError(f"Invalid dataset: '{dataset}'. Choose from {DATASETS}")
- if branch_level not in ("low", "high", "both"):
- raise ValueError(f"Invalid branch_level: '{branch_level}'. Expected 'low', 'high', or 'both'.")
-
- def _get_reshape_fn(self):
- return reshape_transform_vit if self.transformer_type == "vit" else reshape_transform_gcvit
-
- def _resolve_target_layers(self) -> list[torch.nn.Module]:
- layers = []
-
- if self.transformer_type == "gcvit":
- n_stages = len(self.model.l_gcvit.stages)
- idx = self.l_stage_idx if self.l_stage_idx >= 0 else n_stages + self.l_stage_idx
-
- if not (0 <= idx < n_stages):
- raise IndexError(f"l_stage_idx {self.l_stage_idx} out of range (n={n_stages})")
- if self.branch_level in ("low", "both"):
- layers.append(self.model.l_gcvit.stages[idx].blocks[-1].norm2)
- if self.branch_level in ("high", "both"):
- layers.append(self.model.h_gcvit.stages[0].blocks[-1].norm2)
-
- else: # vit
- if self.branch_level in ("low", "both"):
- layers.append(self.model.l_vit.blocks[-1].norm1)
- if self.branch_level in ("high", "both"):
- layers.append(self.model.h_vit.blocks[-1].norm1)
-
- return layers
-
- def _get_face_bbox(self, img: np.ndarray) -> List[float]:
- result = self.face_detector.detect_batch([img], [1.0])
- return result[0]
-
- def _crop_face(self, img: np.ndarray, bbox: List[float]) -> np.ndarray:
- xmin, ymin, xmax, ymax = bbox
- h, w = img.shape[:2]
- pw = int((xmax - xmin) * self.margin_ratio)
- ph = int((ymax - ymin) * self.margin_ratio)
- return img[
- max(int(ymin - ph), 0) : min(int(ymax + ph), h),
- max(int(xmin - pw), 0) : min(int(xmax + pw), w),
- ]
-
- def _preprocess_img(self, img_path: str) -> np.ndarray:
- img = cv2.imread(img_path)
- img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
- detect_result = self._get_face_bbox(img)
- bbox = detect_result[:4]
- cropped = self._crop_face(img, bbox)
- return cropped
-
- def _get_transform(self, img_path: str):
- face = self._preprocess_img(img_path)
- tensor = self.transforms(image=face)['image'].unsqueeze(0) # (1,3,H,W)
- return face, tensor
-
- def __repr__(self):
- return (f"{self.__class__.__name__}("
- f"model={self.model_name}, branch={self.branch}, "
- f"device={self.device})")
\ No newline at end of file
diff --git a/explainability/cam_evaluator.py b/explainability/cam_evaluator.py
new file mode 100644
index 0000000..d215def
--- /dev/null
+++ b/explainability/cam_evaluator.py
@@ -0,0 +1,48 @@
+import numpy as np
+import torch
+from typing import List, Dict, Literal
+from explainability.utils.model_targets import BinaryClassifierOutputSigmoidTarget
+from explainability.explainer.cam_explainer import CAMExplainer
+from explainability.metrics.road import (
+ ROADLeastRelevantFirstAverage, ROADMostRelevantFirstAverage,
+ ROADCombined
+)
+
+METRIC_REGISTRY = {
+ "MORF": ROADMostRelevantFirstAverage,
+ "LERF": ROADLeastRelevantFirstAverage,
+ "Combined": ROADCombined,
+}
+
+class CAMEvaluator:
+ def __init__(self,
+ cam_explainer: CAMExplainer,
+ cam_metric: Literal["Combined", "MORF", "LERF"] = "Combined", # Combined, MORF, LERF
+ percentiles: List[int] = [10,20,30,40,50,60,70,80,90],
+ seed: int = 2026
+ ):
+ self.seed = seed
+ self.cam_explainer = cam_explainer
+ self.cam_metric_name = cam_metric
+ self.percentiles = percentiles
+ self.cam_metric = METRIC_REGISTRY[cam_metric](percentiles)
+
+ def evaluate(self, img_path: str) -> float:
+ torch.manual_seed(self.seed)
+ np.random.seed(self.seed)
+
+ grayscale_cam, tensor, _ = self.cam_explainer._build_grayscale_cam(img_path)
+ targets = [BinaryClassifierOutputSigmoidTarget()]
+
+ return self.cam_metric(
+ input_tensor = tensor, # (1,C,H,W)
+ cams = grayscale_cam[np.newaxis, ...], #(1,H,W)
+ targets = targets,
+ model = self.cam_explainer.model,
+ )[0]
+
+ def __repr__(self):
+ return (
+ f"CAMEvaluator(explainer={self.cam_explainer.__class__.__name__}, "
+ f"metric={self.cam_metric_name}, percentiles={self.percentiles})"
+ )
\ No newline at end of file
diff --git a/explainability/cam_explainer.py b/explainability/cam_explainer.py
deleted file mode 100644
index aa5fa96..0000000
--- a/explainability/cam_explainer.py
+++ /dev/null
@@ -1,57 +0,0 @@
-import torch
-import cv2
-import numpy as np
-from abc import ABC, abstractmethod
-from explainability.utils.model_targets import BinaryClassifierOutputTarget
-from explainability.utils.image import show_cam_on_image, deprocess_image
-from .base_explainer import BaseExplainer
-class CAMExplainer(BaseExplainer, ABC):
- def __init__(self,
- aug_smooth: bool = False,
- eigen_smooth: bool = False,
- colormap: int = cv2.COLORMAP_JET,
- image_weight: float = 0.5,
- **kwargs):
- super().__init__(**kwargs)
- self.aug_smooth = aug_smooth # aug_smooth has the effect of beter centering the CAM around the objects
- self.eigen_smooth = eigen_smooth # # eigen_smooth has the effect of removing a lot of noise
- self.colormap = colormap
- self.image_weight = image_weight
- self.cam = self._build_cam()
-
- @abstractmethod
- def _build_cam(self):
- ...
-
- def display_heatmap(self, img_path):
- face, tensor = self._get_transform(img_path)
- tensor = tensor.to(self.device) # (1,3,H,W)
- targets = [BinaryClassifierOutputTarget(self.category)]
-
- grayscale_cam = self.cam(
- input_tensor = tensor, # (1,3,H,W)
- targets = targets,
- aug_smooth = self.aug_smooth,
- eigen_smooth = self.eigen_smooth
- ) # (1,H,W)
-
- return grayscale_cam[0], tensor
-
- def display_cam_on_image(self, img_path):
- grayscale_cam, tensor = self.display_heatmap(img_path)
-
- heatmap = show_cam_on_image(
- deprocess_image(tensor), # (H,W,3), range(0~1), np.ndarray
- grayscale_cam, # (H,W), range(0~1), np.ndarray
- use_rgb=True,
- colormap = self.colormap
- image_weight=self.image_weight)
-
- return heatmap # (H,W,3), range(0~255), np.uint8
-
- def display_inverse_cam(self, img_path):
- grayscale_cam, _ = self.display_heatmap(img_path)
-
- return 1 - grayscale_cam # (H,W), range(0~1), np.float32
-
-
\ No newline at end of file
diff --git a/explainability/explainer/base_explainer.py b/explainability/explainer/base_explainer.py
new file mode 100644
index 0000000..dc4fc62
--- /dev/null
+++ b/explainability/explainer/base_explainer.py
@@ -0,0 +1,194 @@
+import cv2
+import timm
+import torch
+import numpy as np
+from pathlib import Path
+from typing import List, Optional, Tuple, Literal, Callable
+from preprocess.face_detector import FaceDetector2
+from deepguard.data import get_test_transforms
+from explainability.utils.reshape_transforms import reshape_transform_vit, reshape_transform_gcvit
+
+class BaseExplainer:
+ def __init__(
+ self,
+ model_name: Literal["ms_eff_vit_b0", "ms_eff_vit_b5", "ms_eff_gcvit_b0", "ms_eff_gcvit_b5"],
+ dataset: Literal["celeb_df_v2", "ff++", "kodf"],
+ margin_ratio: float = 0.2,
+ conf_thres: float = 0.5,
+ branch_level: Literal["low", "high"] = "high",
+ l_stage_idx: int = -1, # gcvit low branch stage index (0~3)
+ block_idx: int = -1, # vit, gcvit block index
+ ):
+
+ self.device = "cuda:0" if torch.cuda.is_available() else 'cpu'
+ self.margin_ratio = margin_ratio
+ self.conf_thres = conf_thres
+ self.model_name = model_name
+ self.dataset = dataset
+ self.branch_level = branch_level
+ self.l_stage_idx = l_stage_idx
+ self.block_idx = block_idx
+
+ # Parse Model Metadata
+ self.model_variant = model_name.split("_")[-1] # b0, b5
+ self.transformer_type = model_name.split("_")[-2] # vit, gcvit
+
+ # Model & Tools setup
+ self.model = timm.create_model(model_name, pretrained=True, dataset=dataset)
+ self.face_detector = FaceDetector2(conf_thres)
+ self.img_size = [224,224] if self.model_variant == "b0" else [384,384]
+ self.transforms = get_test_transforms(img_size=self.img_size)
+
+ self.model.to(self.device).eval()
+
+ # Explainability setup
+ self.reshape_fn = self._get_reshape_fn()
+ self.target_layers = self._resolve_target_layers()
+
+ def _get_reshape_fn(self) -> Callable[[torch.Tensor], torch.Tensor]:
+ """모델 종류에 맞는 Feature map reshape 함수를 반환한다.
+
+ Returns:
+ Callable: 3D/4D 텐서를 GradCAM 연산에 적합한 형태로 변환하는 함수.
+ """
+ return reshape_transform_vit if self.transformer_type == "vit" else reshape_transform_gcvit
+
+ def _resolve_target_layers(self) -> list[torch.nn.Module]:
+ """시각화(XAI)의 대상이 될 Target Layer들을 선택한다.
+
+ Returns:
+ list[torch.nn.Module]: 분석 대상 정규화(Normalization) 레이어 리스트.
+
+ Raises:
+ ValueError: l_stage_idx나 block_idx가 모델 범위를 벗어날 경우 발생.
+ """
+ layers = []
+
+ if self.transformer_type == "gcvit":
+ n_stages = len(self.model.l_gcvit.stages)
+ stage_idx = self.l_stage_idx if self.l_stage_idx >= 0 else n_stages + self.l_stage_idx
+
+ if not (0 <= stage_idx < n_stages):
+ raise ValueError(f"l_stage_idx={self.l_stage_idx} 범위 초과 (stage 수: {n_stages})")
+
+ if self.branch_level in ("low", "both"):
+ n_blocks = len(self.model.l_gcvit.stages[stage_idx].blocks)
+ blk = self._resolve_block_idx(self.block_idx, n_blocks, f"l_gcvit stage[{stage_idx}]")
+ layers.append(self.model.l_gcvit.stages[stage_idx].blocks[blk].norm2)
+
+ if self.branch_level in ("high", "both"):
+ n_blocks = len(self.model.h_gcvit.stages[0].blocks)
+ blk = self._resolve_block_idx(self.block_idx, n_blocks, "h_gcvit stage[0]")
+ layers.append(self.model.h_gcvit.stages[0].blocks[blk].norm2)
+
+ else: # vit
+ if self.branch_level in ("low", "both"):
+ n_blocks = len(self.model.l_vit.blocks)
+ blk = self._resolve_block_idx(self.block_idx, n_blocks, "l_vit")
+ layers.append(self.model.l_vit.blocks[blk].norm1)
+
+ if self.branch_level in ("high", "both"):
+ n_blocks = len(self.model.h_vit.blocks)
+ blk = self._resolve_block_idx(self.block_idx, n_blocks, "h_vit")
+ layers.append(self.model.h_vit.blocks[blk].norm1)
+
+ return layers
+
+ @staticmethod
+ def _resolve_block_idx(block_idx: int, n_blocks: int, name: str) -> int:
+ """음수 인덱싱을 지원하며, 해당 블록의 유효한 인덱스를 계산 및 검증한다.
+
+ Args:
+ block_idx (int): 접근하려는 블록 인덱스 (음수 가능).
+ n_blocks (int): 해당 스테이지/모델의 총 블록 개수.
+ name (str): 에러 메시지 출력용 모듈 이름.
+
+ Returns:
+ int: 정제된 양수 형태의 블록 인덱스.
+
+ Raises:
+ ValueError: 인덱스가 유효 범위를 벗어난 경우 발생.
+ """
+ idx = block_idx if block_idx >= 0 else n_blocks + block_idx
+ if not (0 <= idx < n_blocks):
+ raise ValueError(f"block_idx={block_idx} 범위 초과 ({name} block 수: {n_blocks})")
+ return idx
+
+ def _get_face_bbox(self, img: np.ndarray) -> List[float]:
+ """얼굴 탐지기(YOLO)를 통해 이미지에서 얼굴 바운딩 박스를 좌표를 추출한다.
+
+ Args:
+ img (np.ndarray): HWC 형태의 RGB 이미지 데이터.
+
+ Returns:
+ list[float]: [xmin, ymin, xmax, ymax, confidence] 형태의 좌표 리스트.
+ """
+ result = self.face_detector.detect_batch([img], [1.0])
+ return result[0]
+
+ def _crop_face(self, img: np.ndarray, bbox: List[float]) -> np.ndarray:
+ """마진 비율을 고려하여 바운딩 박스 기준으로 얼굴 영역을 크롭한다.
+
+ Args:
+ img (np.ndarray): 전체 원본 이미지.
+ bbox (list[float]): 탐지된 [xmin, ymin, xmax, ymax] 얼굴 좌표.
+
+ Returns:
+ np.ndarray: 크롭된 얼굴 이미지 영역.
+ """
+ xmin, ymin, xmax, ymax = bbox
+ h, w = img.shape[:2]
+ pw = int((xmax - xmin) * self.margin_ratio)
+ ph = int((ymax - ymin) * self.margin_ratio)
+ return img[
+ max(int(ymin - ph), 0) : min(int(ymax + ph), h),
+ max(int(xmin - pw), 0) : min(int(xmax + pw), w),
+ ]
+
+ def _preprocess_img(self, img_path: str) -> np.ndarray:
+ """이미지 경로를 읽어 얼굴을 탐지하고 크롭하는 전과정을 수행한다.
+
+ Args:
+ img_path (str): 입력 이미지 파일 경로.
+
+ Returns:
+ np.ndarray: 전처리가 완료된 크롭된 얼굴 이미지 (RGB).
+ """
+ img = cv2.imread(img_path)
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+ detect_result = self._get_face_bbox(img)
+ bbox = detect_result[:4]
+ cropped = self._crop_face(img, bbox)
+ return cropped
+
+ def _get_transform(self, img_path: str) -> tuple[np.ndarray, torch.Tensor]:
+ """이미지 경로를 받아 크롭 이미지와 모델 입력용 텐서를 동시에 생성한다.
+
+ Args:
+ img_path (str): 입력 이미지 파일 경로.
+
+ Returns:
+ tuple[np.ndarray, torch.Tensor]: (크롭된 얼굴 이미지, (1, 3, H, W) 형태의 배치 텐서).
+ """
+ face = self._preprocess_img(img_path)
+ tensor = self.transforms(image=face)['image'].unsqueeze(0) # (1,3,H,W)
+ return face, tensor
+
+ def _get_transform_from_array(self, face: np.ndarray) -> tuple[np.ndarray, torch.Tensor]:
+ """이미 크롭된 얼굴 배열을 받아 모델 입력용 텐서로 변환한다.
+
+ Args:
+ face (np.ndarray): 이미 잘려진 얼굴 이미지 배열 (RGB).
+
+ Returns:
+ tuple[np.ndarray, torch.Tensor]: (원본 얼굴 이미지, (1, 3, H, W) 형태의 배치 텐서).
+ """
+ tensor = self.transforms(image=face)['image'].unsqueeze(0) # (1,3,H,W)
+ return face, tensor
+
+ def __repr__(self):
+ return (f"{self.__class__.__name__}("
+ f"model={self.model_name}, dataset={self.dataset}, "
+ f"margin_ratio={self.margin_ratio}, conf_thres={self.conf_thres}, "
+ f"branch_level={self.branch_level}, l_stage_idx={self.l_stage_idx}, block_idx={self.block_idx},"
+ f"device={self.device})")
\ No newline at end of file
diff --git a/explainability/explainer/cam_explainer.py b/explainability/explainer/cam_explainer.py
new file mode 100644
index 0000000..f630055
--- /dev/null
+++ b/explainability/explainer/cam_explainer.py
@@ -0,0 +1,334 @@
+from typing import Union, Literal
+import torch
+import cv2
+import numpy as np
+from abc import ABC, abstractmethod
+from explainability.utils.model_targets import BinaryClassifierOutputTarget
+from explainability.utils.image import show_cam_on_image, deprocess_image, remove_padding_and_resize
+from explainability.explainer.base_explainer import BaseExplainer
+
+
+def _draw_label(img: np.ndarray, text: str, x: int, y: int, prob_ratio: float = 1.0):
+ """Confidence label with a clean pill-style background.
+
+ prob_ratio: 0~1, controls bg color (green→red)
+ """
+ font = cv2.FONT_HERSHEY_SIMPLEX
+ font_scale = 0.52
+ thickness = 1
+ pad = 4
+
+ (tw, th), baseline = cv2.getTextSize(text, font, font_scale, thickness)
+
+ rx1, ry1 = x - pad, y - th - pad
+ rx2, ry2 = x + tw + pad, y + baseline + pad
+
+ r = int(min(255, 2 * prob_ratio * 255))
+ g = int(min(255, 2 * (1 - prob_ratio) * 255))
+ bg_color = (r, g, 30) # RGB: green → red
+
+ overlay = img.copy()
+ cv2.rectangle(overlay, (rx1, ry1), (rx2, ry2), bg_color, -1)
+ cv2.addWeighted(overlay, 0.75, img, 0.25, 0, img)
+ cv2.putText(img, text, (x, y), font, font_scale, (255, 255, 255), thickness, cv2.LINE_AA)
+
+
+class CAMExplainer(BaseExplainer, ABC):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self.cam = self._build_cam()
+
+ @abstractmethod
+ def _build_cam(self) -> any:
+ """하위 클래스에서 각 알고리즘(GradCAM, HiResCAM 등)에 맞는 CAM 객체를 생성한다.
+
+ Returns:
+ Any: pytorch-grad-cam 패키지의 CAM 기반 인스턴스.
+ """
+ pass
+
+ def _build_grayscale_cam(self, img_path: str,
+ category: int = 1,
+ aug_smooth: bool = False,
+ eigen_smooth: bool = False) -> tuple[np.ndarray, torch.Tensor, np.ndarray]:
+ """이미지 경로를 받아 전처리를 수행하고 raw 흑백(Grayscale) CAM을 생성한다.
+
+ Args:
+ img_path (str): 입력 이미지 파일 경로.
+ category (int, optional): 타겟 클래스 인덱스 (위조: 1, 정상: 0). Defaults to 1.
+ aug_smooth (bool, optional): Test-time augmentation 가동 여부. Defaults to False.
+ eigen_smooth (bool, optional): PCA 기반 노이즈 제거 가동 여부. Defaults to False.
+
+ Returns:
+ tuple[np.ndarray, torch.Tensor, np.ndarray]:
+ - grayscale_cam: (H, W) 형태의 0~1 사이로 정규화된 CAM 맵.
+ - tensor: (1, 3, H_model, W_model) 형태의 모델 입력용 GPU 텐서.
+ - face: 크롭된 원본 얼굴 이미지 배열 (RGB).
+ """
+
+ face, tensor = self._get_transform(img_path)
+ tensor = tensor.to(self.device)
+ targets = [BinaryClassifierOutputTarget(category)]
+
+ grayscale_cam = self.cam(
+ input_tensor=tensor,
+ targets=targets,
+ aug_smooth=aug_smooth,
+ eigen_smooth=eigen_smooth,
+ )
+ return grayscale_cam[0], tensor, face
+
+ def _build_grayscale_cam_from_array(self,
+ face: np.ndarray,
+ category: int = 1,
+ aug_smooth: bool = False,
+ eigen_smooth: bool = False) -> tuple[np.ndarray, torch.Tensor, np.ndarray]:
+ """이미 크롭된 얼굴 이미지 배열로부터 raw 흑백(Grayscale) CAM을 생성한다.
+
+ Args:
+ face (np.ndarray): 전처리(크롭)가 완료된 얼굴 이미지 배열 (RGB).
+ category (int, optional): 타겟 클래스 인덱스. Defaults to 1.
+ aug_smooth (bool, optional): Test-time augmentation 가동 여부. Defaults to False.
+ eigen_smooth (bool, optional): PCA 기반 노이즈 제거 가동 여부. Defaults to False.
+
+ Returns:
+ tuple[np.ndarray, torch.Tensor, np.ndarray]:
+ - grayscale_cam: (H, W) 형태의 0~1 사이로 정규화된 CAM 맵.
+ - tensor: (1, 3, H_model, W_model) 형태의 모델 입력용 GPU 텐서.
+ - face: 입력받은 원본 얼굴 이미지 배열.
+ """
+ face, tensor = self._get_transform_from_array(face)
+ tensor = tensor.to(self.device)
+ targets = [BinaryClassifierOutputTarget(category)]
+
+ grayscale_cam = self.cam(
+ input_tensor=tensor,
+ targets=targets,
+ aug_smooth=aug_smooth,
+ eigen_smooth=eigen_smooth,
+ )
+ return grayscale_cam[0], tensor, face
+
+ def _prepare_cam(self, img_path: str,
+ category: int = 1,
+ aug_smooth: bool = False,
+ eigen_smooth: bool = False) -> tuple[np.ndarray, np.ndarray]:
+ """흑백 CAM을 생성하고, 모델 입력 시 발생한 패딩 제거 및 원래 얼굴 해상도로 복원한다.
+
+ Args:
+ img_path (str): 입력 이미지 파일 경로.
+ category (int, optional): 타겟 클래스 인덱스. Defaults to 1.
+ aug_smooth (bool, optional): Test-time augmentation 가동 여부. Defaults to False.
+ eigen_smooth (bool, optional): PCA 기반 노이즈 제거 가동 여부. Defaults to False.
+
+ Returns:
+ tuple[np.ndarray, np.ndarray]:
+ - cam_recovered: 원본 얼굴 크기로 복원된 CAM 맵 (H_face, W_face).
+ - face: 크롭된 원본 얼굴 이미지 배열 (RGB).
+ """
+ grayscale_cam, tensor, face = self._build_grayscale_cam(img_path, category, aug_smooth, eigen_smooth)
+ cam_recovered = remove_padding_and_resize(grayscale_cam, face.shape, tensor.shape)
+ return cam_recovered, face
+
+ def _prepare_cam_from_array(self, face: np.ndarray,
+ category: int = 1,
+ aug_smooth: bool = False,
+ eigen_smooth: bool = False) -> tuple[np.ndarray, np.ndarray]:
+ """얼굴 배열로부터 흑백 CAM을 생성하고 원래 얼굴 해상도로 복원한다.
+
+ Args:
+ face (np.ndarray): 전처리(크롭)가 완료된 얼굴 이미지 배열 (RGB).
+ category (int, optional): 타겟 클래스 인덱스. Defaults to 1.
+ aug_smooth (bool, optional): Test-time augmentation 가동 여부. Defaults to False.
+ eigen_smooth (bool, optional): PCA 기반 노이즈 제거 가동 여부. Defaults to False.
+
+ Returns:
+ tuple[np.ndarray, np.ndarray]:
+ - cam_recovered: 원본 얼굴 크기로 복원된 CAM 맵 (H_face, W_face).
+ - face: 입력받은 원본 얼굴 이미지 배열.
+ """
+ grayscale_cam, tensor, face = self._build_grayscale_cam_from_array(face, category, aug_smooth, eigen_smooth)
+ cam_recovered = remove_padding_and_resize(grayscale_cam, face.shape, tensor.shape)
+ return cam_recovered, face
+
+ def _get_binary_mask(self, cam: np.ndarray, threshold: Union[float, Literal["auto"]]) -> np.ndarray:
+ """입력된 CAM 맵을 기반으로 위조 유력 부위를 이진화(Binary Mask)한다.
+
+ Args:
+ cam (np.ndarray): 0~1 사이 값을 가진 복원된 CAM 맵.
+ threshold (Union[float, 'auto']):
+ - float (0.0~1.0): 상위 백분위수 기준으로 임계값 설정 (예: 0.7 이면 상위 30% 영역 선택).
+ - 'auto': Otsu 알고리즘을 사용해 최적의 임계값을 자동으로 계산.
+
+ Returns:
+ np.ndarray: 0 또는 255 값을 가지는 이진화 마스크 맵 (np.uint8).
+ """
+ if threshold != 'auto':
+ t = np.percentile(cam, threshold * 100)
+ binary_mask = (cam > t).astype(np.uint8) * 255
+ else:
+ _, binary_mask = cv2.threshold(
+ np.uint8(cam * 255), 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU
+ )
+ return binary_mask
+
+ def display_heatmap_on_image(self, img_path: str, **kwargs) -> np.ndarray:
+ """이미지 경로를 입력받아 원본 얼굴 위에 위조 흔적 히트맵을 중첩한 이미지를 생성한다.
+
+ Args:
+ img_path (str): 입력 이미지 파일 경로.
+ **kwargs:
+ category (int): 타겟 클래스 인덱스 (기본값: 1).
+ aug_smooth (bool): Test-time augmentation 가동 여부 (기본값: False).
+ eigen_smooth (bool): PCA 기반 노이즈 제거 가동 여부 (기본값: False).
+ threshold (float | str): 이진화 임계값 (기본값: "auto").
+ colormap (int): OpenCV 컬러맵 상수 (기본값: cv2.COLORMAP_JET).
+ image_weight (float): 원본 이미지 불투명도 가중치 (기본값: 0.5).
+
+ Returns:
+ np.ndarray: 히트맵이 합성된 얼굴 이미지 (RGB, np.uint8).
+ """
+ cam_recovered, face = self._prepare_cam(
+ img_path,
+ category = kwargs.get("category", 1),
+ aug_smooth = kwargs.get("aug_smooth", False),
+ eigen_smooth = kwargs.get("eigen_smooth", False))
+
+ binary_mask = self._get_binary_mask(cam_recovered, kwargs.get("threshold", "auto"))
+ cam_recovered = np.where(binary_mask > 0, cam_recovered, 0.0)
+
+ return show_cam_on_image(
+ np.float32(face) / 255.0,
+ cam_recovered,
+ use_rgb = True,
+ colormap = kwargs.get("colormap", cv2.COLORMAP_JET),
+ image_weight = kwargs.get("image_weight", 0.5),
+ )
+
+ def display_heatmap_from_array(self, face: np.ndarray, **kwargs) -> np.ndarray:
+ """얼굴 배열을 직접 입력받아 원본 얼굴 위에 위조 흔적 히트맵을 중첩한 이미지를 생성한다.
+
+ Args:
+ face (np.ndarray): 전처리(크롭)가 완료된 얼굴 이미지 배열 (RGB).
+ **kwargs:
+ category (int): 타겟 클래스 인덱스 (기본값: 1).
+ aug_smooth (bool): Test-time augmentation 가동 여부 (기본값: False).
+ eigen_smooth (bool): PCA 기반 노이즈 제거 가동 여부 (기본값: False).
+ threshold (float | str): 이진화 임계값 (기본값: "auto").
+ colormap (int): OpenCV 컬러맵 상수 (기본값: cv2.COLORMAP_JET).
+ image_weight (float): 원본 이미지 불투명도 가중치 (기본값: 0.5).
+
+ Returns:
+ np.ndarray: 히트맵이 합성된 얼굴 이미지 (RGB, np.uint8).
+ """
+
+ cam_recovered, face = self._prepare_cam_from_array(
+ face,
+ category = kwargs.get("category", 1),
+ aug_smooth = kwargs.get("aug_smooth", False),
+ eigen_smooth = kwargs.get("eigen_smooth", False))
+
+ binary_mask = self._get_binary_mask(cam_recovered, kwargs.get("threshold", "auto"))
+ cam_recovered = np.where(binary_mask > 0, cam_recovered, 0.0)
+
+ return show_cam_on_image(
+ np.float32(face) / 255.0,
+ cam_recovered,
+ use_rgb = True,
+ colormap = kwargs.get("colormap", cv2.COLORMAP_JET),
+ image_weight = kwargs.get("image_weight", 0.5),
+ )
+
+ def display_bbox_on_image(self, img_path: str, **kwargs) -> np.ndarray:
+ """이미지 경로를 입력받아 위조 의심 부위를 찾아 사각형 바운딩 박스와 확률을 그린다.
+
+ Args:
+ img_path (str): 입력 이미지 파일 경로.
+ **kwargs:
+ category (int): 타겟 클래스 인덱스 (기본값: 1).
+ aug_smooth (bool): Test-time augmentation 가동 여부 (기본값: False).
+ eigen_smooth (bool): PCA 기반 노이즈 제거 가동 여부 (기본값: False).
+ threshold (float | str): 이진화 임계값 (기본값: "auto").
+ thickness (int): 사각형 선 굵기 (기본값: 2).
+
+ Returns:
+ np.ndarray: 바운딩 박스와 어노테이션이 추가된 원본 얼굴 이미지 (RGB).
+ """
+ cam_recovered, face = self._prepare_cam(
+ img_path,
+ category = kwargs.get("category", 1),
+ aug_smooth = kwargs.get("aug_smooth", False),
+ eigen_smooth = kwargs.get("eigen_smooth", False)
+ )
+
+ binary_mask = self._get_binary_mask(cam_recovered, kwargs.get("threshold", "auto"))
+
+ contours, _ = cv2.findContours(binary_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+ annotated_img = face.copy()
+ for c in contours:
+ x, y, w, h = cv2.boundingRect(c)
+ roi_prob = float(np.max(cam_recovered[y:y + h, x:x + w]))
+
+ r = int(min(255, 2 * roi_prob * 255))
+ g = int(min(255, 2 * (1 - roi_prob) * 255))
+ box_color = (r, g, 30) # RGB
+
+ cv2.rectangle(annotated_img, (x, y), (x + w, y + h), box_color, kwargs.get("thickness", 1))
+
+ _draw_label(
+ annotated_img,
+ text=f"{roi_prob * 100:.1f}%",
+ x=x,
+ y=max(y - 6, 14),
+ prob_ratio=roi_prob,
+ )
+
+ return annotated_img
+
+ def display_bbox_from_array(self, face: np.ndarray, **kwargs) -> np.ndarray:
+ """얼굴 배열을 직접 입력받아 위조 의심 부위를 찾아 사각형 바운딩 박스와 확률을 그린다.
+
+ Args:
+ face (np.ndarray): 전처리(크롭)가 완료된 얼굴 이미지 배열 (RGB).
+ **kwargs:
+ category (int): 타겟 클래스 인덱스 (기본값: 1).
+ aug_smooth (bool): Test-time augmentation 가동 여부 (기본값: False).
+ eigen_smooth (bool): PCA 기반 노이즈 제거 가동 여부 (기본값: False).
+ threshold (float | str): 이진화 임계값 (기본값: "auto").
+ thickness (int): 사각형 선 굵기 (기본값: 2).
+
+ Returns:
+ np.ndarray: 바운딩 박스와 어노테이션이 추가된 원본 얼굴 이미지 (RGB).
+ """
+ cam_recovered, face = self._prepare_cam_from_array(
+ face,
+ category = kwargs.get("category", 1),
+ aug_smooth = kwargs.get("aug_smooth", False),
+ eigen_smooth = kwargs.get("eigen_smooth", False)
+ )
+
+ binary_mask = self._get_binary_mask(cam_recovered, kwargs.get("threshold", "auto"))
+
+ contours, _ = cv2.findContours(binary_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+ annotated_img = face.copy()
+ for c in contours:
+ x, y, w, h = cv2.boundingRect(c)
+ roi_prob = float(np.max(cam_recovered[y:y + h, x:x + w]))
+
+ r = int(min(255, 2 * roi_prob * 255))
+ g = int(min(255, 2 * (1 - roi_prob) * 255))
+ box_color = (r, g, 30) # RGB
+
+ cv2.rectangle(annotated_img, (x, y), (x + w, y + h), box_color, kwargs.get("thickness", 1))
+
+ _draw_label(
+ annotated_img,
+ text=f"{roi_prob * 100:.1f}%",
+ x=x,
+ y=max(y - 6, 14),
+ prob_ratio=roi_prob,
+ )
+
+ return annotated_img
\ No newline at end of file
diff --git a/explainability/eigenvalue_explainers.py b/explainability/explainer/eigenvalue_explainer.py
similarity index 69%
rename from explainability/eigenvalue_explainers.py
rename to explainability/explainer/eigenvalue_explainer.py
index 7f53174..9aa2633 100644
--- a/explainability/eigenvalue_explainers.py
+++ b/explainability/explainer/eigenvalue_explainer.py
@@ -1,5 +1,5 @@
-from pytorch_grad_cam import EigenCAM, EigenGradCAM, KPCA_CAM, SegEigenCAM
-from explainability.cam_explainer import CAMExplainer
+from pytorch_grad_cam import EigenCAM, EigenGradCAM, KPCA_CAM
+from explainability.explainer.cam_explainer import CAMExplainer
class EigenCAMExplainer(CAMExplainer):
"""
@@ -31,14 +31,5 @@ def _build_cam(self):
return KPCA_CAM(model=self.model, target_layers=self.target_layers, reshape_transform=self.reshape_fn,
kernel=self.kernel, gamma=self.gamma)
-class SegEigenCAMExplainer(CAMExplainer):
- """
- Like EigenCAM but with gradient weighting (absolute gradients ⊙ activations )
- before SVD and sign correction to fix SVD sign ambiguity
- designed for semantic segmentation
- """
- def _build_cam(self):
- return SegEigenCAM(model=self.model, target_layers=self.target_layers, reshape_transform=self.reshape_fn)
-
-
-
+ def __repr__(self):
+ return super().__repr__().rstrip(")") + f", kernel={self.kernel}, gamma={self.gamma})"
diff --git a/explainability/gradient_explainer.py b/explainability/explainer/gradient_explainer.py
similarity index 56%
rename from explainability/gradient_explainer.py
rename to explainability/explainer/gradient_explainer.py
index f50bff9..3767fc7 100644
--- a/explainability/gradient_explainer.py
+++ b/explainability/explainer/gradient_explainer.py
@@ -1,8 +1,8 @@
from pytorch_grad_cam import (
GradCAM, GradCAMPlusPlus, GradCAMElementWise,
- XGradCAM, HiResCAM, FullGrad, ShapleyCAM, FinerCAM,
+ XGradCAM, HiResCAM
)
-from explainability.cam_explainer import CAMExplainer
+from explainability.explainer.cam_explainer import CAMExplainer
class GradCAMExplainer(CAMExplainer):
@@ -44,29 +44,4 @@ def _build_cam(self):
return HiResCAM(model=self.model, target_layers=self.target_layers, reshape_transform=self.reshape_fn)
-class FullGradExplainer(CAMExplainer):
- """
- Computes the gradients of the biases from all over the network, and then sums them
- """
- def _build_cam(self):
- return FullGrad(model=self.model, target_layers=self.target_layers, reshape_transform=self.reshape_fn)
-
-
-class ShapleyCAMExplainer(CAMExplainer):
- """
- Weight the activations using the gradient and Hessian-vector product.
- """
- def _build_cam(self):
- return ShapleyCAM(model=self.model, target_layers=self.target_layers, reshape_transform=self.reshape_fn)
-
-
-class FinerCAMExplainer(CAMExplainer):
- """
- Improves fine-grained classification by comparing similar classes, suppressing shared features and highlighting discriminative details.
- """
- def __init__(self, base_method=GradCAM, **kwargs):
- self.base_method = base_method
- super().__init__(**kwargs)
-
- def _build_cam(self):
- return FinerCAM(model=self.model, target_layers=self.target_layers, reshape_transform=self.reshape_fn, base_method=self.base_method)
\ No newline at end of file
+
diff --git a/explainability/explainer/misc_explainer.py b/explainability/explainer/misc_explainer.py
new file mode 100644
index 0000000..85cb38f
--- /dev/null
+++ b/explainability/explainer/misc_explainer.py
@@ -0,0 +1,13 @@
+from pytorch_grad_cam import LayerCAM, RandomCAM
+from explainability.explainer.cam_explainer import CAMExplainer
+
+
+class LayerCAMExplainer(CAMExplainer):
+ """
+ Spatially weight the activations by positive gradients.
+ Works better especially in lower layers
+ """
+ def _build_cam(self):
+ return LayerCAM(model=self.model, target_layers=self.target_layers, reshape_transform=self.reshape_fn)
+
+
diff --git a/explainability/explainer/perturbation_explainer.py b/explainability/explainer/perturbation_explainer.py
new file mode 100644
index 0000000..cb460aa
--- /dev/null
+++ b/explainability/explainer/perturbation_explainer.py
@@ -0,0 +1,24 @@
+from pytorch_grad_cam import ScoreCAM, FEM
+from explainability.explainer.cam_explainer import CAMExplainer
+class ScoreCAMExplainer(CAMExplainer):
+ """
+ Perbutate the image by the scaled activation
+ mesaure how the output drops
+ """
+ def _build_cam(self):
+ return ScoreCAM(model=self.model, target_layers=self.target_layers, reshape_transform=self.reshape_fn)
+
+class FEMExplainer(CAMExplainer):
+ """
+ A gradient free method that binarizes activations
+ by an activation > mean + k * std rule
+ """
+ def __init__(self, k: int = 2, **kwargs):
+ self.k = k
+ super().__init__(**kwargs)
+
+ def _build_cam(self):
+ return FEM(model=self.model, target_layers=self.target_layers, reshape_transform=self.reshape_fn, k=self.k)
+
+ def __repr__(self):
+ return super().__repr__().rstrip(")") + f", k={self.k})"
\ No newline at end of file
diff --git a/explainability/metrics/__init__.py b/explainability/metrics/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/explainability/metrics/cam_mult_image.py b/explainability/metrics/cam_mult_image.py
new file mode 100644
index 0000000..c46751e
--- /dev/null
+++ b/explainability/metrics/cam_mult_image.py
@@ -0,0 +1,37 @@
+import torch
+import numpy as np
+from typing import List, Callable
+from explainability.metrics.perturbation_confidence import PerturbationConfidenceMetric
+
+
+def multiply_tensor_with_cam(input_tensor: torch.Tensor,
+ cam: torch.Tensor):
+ """ Multiply an input tensor (after normalization)
+ with a pixel attribution map
+ """
+ return input_tensor * cam
+
+
+class CamMultImageConfidenceChange(PerturbationConfidenceMetric):
+ def __init__(self):
+ super(CamMultImageConfidenceChange,
+ self).__init__(multiply_tensor_with_cam)
+
+
+class DropInConfidence(CamMultImageConfidenceChange):
+ def __init__(self):
+ super(DropInConfidence, self).__init__()
+
+ def __call__(self, *args, **kwargs):
+ scores = super(DropInConfidence, self).__call__(*args, **kwargs)
+ scores = -scores
+ return np.maximum(scores, 0)
+
+
+class IncreaseInConfidence(CamMultImageConfidenceChange):
+ def __init__(self):
+ super(IncreaseInConfidence, self).__init__()
+
+ def __call__(self, *args, **kwargs):
+ scores = super(IncreaseInConfidence, self).__call__(*args, **kwargs)
+ return np.float32(scores > 0)
\ No newline at end of file
diff --git a/explainability/metrics/perturbation_confidence.py b/explainability/metrics/perturbation_confidence.py
new file mode 100644
index 0000000..211ab8a
--- /dev/null
+++ b/explainability/metrics/perturbation_confidence.py
@@ -0,0 +1,109 @@
+import torch
+import numpy as np
+from typing import List, Callable
+
+import numpy as np
+import cv2
+
+
+class PerturbationConfidenceMetric:
+ def __init__(self, perturbation):
+ self.perturbation = perturbation
+
+ def __call__(self, input_tensor: torch.Tensor,
+ cams: np.ndarray,
+ targets: List[Callable],
+ model: torch.nn.Module,
+ return_visualization=False,
+ return_diff=True):
+
+ if return_diff:
+ with torch.no_grad():
+ outputs = model(input_tensor)
+ scores = [target(output).cpu().numpy()
+ for target, output in zip(targets, outputs)]
+ scores = np.float32(scores)
+
+ batch_size = input_tensor.size(0)
+ perturbated_tensors = []
+ for i in range(batch_size):
+ cam = cams[i]
+ tensor = self.perturbation(input_tensor[i, ...].cpu(),
+ torch.from_numpy(cam))
+ tensor = tensor.to(input_tensor.device)
+ perturbated_tensors.append(tensor.unsqueeze(0))
+ perturbated_tensors = torch.cat(perturbated_tensors)
+
+ with torch.no_grad():
+ outputs_after_imputation = model(perturbated_tensors)
+ scores_after_imputation = [
+ target(output).cpu().numpy() for target, output in zip(
+ targets, outputs_after_imputation)]
+ scores_after_imputation = np.float32(scores_after_imputation)
+
+ if return_diff:
+ result = scores_after_imputation - scores
+ else:
+ result = scores_after_imputation
+
+ if return_visualization:
+ return result, perturbated_tensors
+ else:
+ return result
+
+
+class RemoveMostRelevantFirst:
+ def __init__(self, percentile, imputer):
+ self.percentile = percentile
+ self.imputer = imputer
+
+ def __call__(self, input_tensor, mask):
+ imputer = self.imputer
+ if self.percentile != 'auto':
+ threshold = np.percentile(mask.cpu().numpy(), self.percentile)
+ binary_mask = np.float32(mask < threshold)
+ else:
+ _, binary_mask = cv2.threshold(
+ np.uint8(mask * 255), 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+
+ binary_mask = torch.from_numpy(binary_mask)
+ binary_mask = binary_mask.to(mask.device)
+ return imputer(input_tensor, binary_mask)
+
+
+class RemoveLeastRelevantFirst(RemoveMostRelevantFirst):
+ def __init__(self, percentile, imputer):
+ super(RemoveLeastRelevantFirst, self).__init__(percentile, imputer)
+
+ def __call__(self, input_tensor, mask):
+ return super(RemoveLeastRelevantFirst, self).__call__(
+ input_tensor, 1 - mask)
+
+
+class AveragerAcrossThresholds:
+ def __init__(
+ self,
+ imputer,
+ percentiles=[
+ 10,
+ 20,
+ 30,
+ 40,
+ 50,
+ 60,
+ 70,
+ 80,
+ 90]):
+ self.imputer = imputer
+ self.percentiles = percentiles
+
+ def __call__(self,
+ input_tensor: torch.Tensor,
+ cams: np.ndarray,
+ targets: List[Callable],
+ model: torch.nn.Module):
+ scores = []
+ for percentile in self.percentiles:
+ imputer = self.imputer(percentile)
+ scores.append(imputer(input_tensor, cams, targets, model))
+ return np.mean(np.float32(scores), axis=0)
\ No newline at end of file
diff --git a/explainability/metrics/road.py b/explainability/metrics/road.py
new file mode 100644
index 0000000..905f84e
--- /dev/null
+++ b/explainability/metrics/road.py
@@ -0,0 +1,154 @@
+import torch
+import numpy as np
+from scipy.sparse import lil_matrix, csc_matrix
+from scipy.sparse.linalg import spsolve
+from typing import List, Callable
+from explainability.metrics.perturbation_confidence import (
+ PerturbationConfidenceMetric, AveragerAcrossThresholds,
+ RemoveMostRelevantFirst, RemoveLeastRelevantFirst
+)
+
+# The weights of the surrounding pixels
+neighbors_weights = [((1, 1), 1 / 12),
+ ((0, 1), 1 / 6),
+ ((-1, 1), 1 / 12),
+ ((1, -1), 1 / 12),
+ ((0, -1), 1 / 6),
+ ((-1, -1), 1 / 12),
+ ((1, 0), 1 / 6),
+ ((-1, 0), 1 / 6)]
+
+
+class NoisyLinearImputer:
+ def __init__(self,
+ noise: float = 0.01,
+ weighting: List[float] = neighbors_weights):
+ """
+ Noisy linear imputation.
+ noise: magnitude of noise to add (absolute, set to 0 for no noise)
+ weighting: Weights of the neighboring pixels in the computation.
+ List of tuples of (offset, weight)
+ """
+ self.noise = noise
+ self.weighting = neighbors_weights
+
+ @staticmethod
+ def add_offset_to_indices(indices, offset, mask_shape):
+ """ Add the corresponding offset to the indices.
+ Return new indices plus a valid bit-vector. """
+ cord1 = indices % mask_shape[1]
+ cord0 = indices // mask_shape[1]
+ cord0 += offset[0]
+ cord1 += offset[1]
+ valid = ((cord0 < 0) | (cord1 < 0) |
+ (cord0 >= mask_shape[0]) |
+ (cord1 >= mask_shape[1]))
+ return ~valid, indices + offset[0] * mask_shape[1] + offset[1]
+
+ @staticmethod
+ def setup_sparse_system(mask, img, neighbors_weights):
+ """ Vectorized version to set up the equation system.
+ mask: (H, W)-tensor of missing pixels.
+ Image: (H, W, C)-tensor of all values.
+ Return (N,N)-System matrix, (N,C)-Right hand side for each of the C channels.
+ """
+ maskflt = mask.flatten()
+ imgflat = img.reshape((img.shape[0], -1))
+ # Indices that are imputed in the flattened mask:
+ indices = np.argwhere(maskflt == 0).flatten()
+ coords_to_vidx = np.zeros(len(maskflt), dtype=int)
+ coords_to_vidx[indices] = np.arange(len(indices))
+ numEquations = len(indices)
+ # System matrix:
+ A = lil_matrix((numEquations, numEquations))
+ b = np.zeros((numEquations, img.shape[0]))
+ # Sum of weights assigned:
+ sum_neighbors = np.ones(numEquations)
+ for n in neighbors_weights:
+ offset, weight = n[0], n[1]
+ # Take out outliers
+ valid, new_coords = NoisyLinearImputer.add_offset_to_indices(
+ indices, offset, mask.shape)
+ valid_coords = new_coords[valid]
+ valid_ids = np.argwhere(valid == 1).flatten()
+ # Add values to the right hand-side
+ has_values_coords = valid_coords[maskflt[valid_coords] > 0.5]
+ has_values_ids = valid_ids[maskflt[valid_coords] > 0.5]
+ b[has_values_ids, :] -= weight * imgflat[:, has_values_coords].T
+ # Add weights to the system (left hand side)
+# Find coordinates in the system.
+ has_no_values = valid_coords[maskflt[valid_coords] < 0.5]
+ variable_ids = coords_to_vidx[has_no_values]
+ has_no_values_ids = valid_ids[maskflt[valid_coords] < 0.5]
+ A[has_no_values_ids, variable_ids] = weight
+ # Reduce weight for invalid
+ sum_neighbors[np.argwhere(valid == 0).flatten()] = \
+ sum_neighbors[np.argwhere(valid == 0).flatten()] - weight
+
+ A[np.arange(numEquations), np.arange(numEquations)] = -sum_neighbors
+ return A, b
+
+ def __call__(self, img: torch.Tensor, mask: torch.Tensor):
+ """ Our linear inputation scheme. """
+ """
+ This is the function to do the linear infilling
+ img: original image (C,H,W)-tensor;
+ mask: mask; (H,W)-tensor
+
+ """
+ imgflt = img.reshape(img.shape[0], -1)
+ maskflt = mask.reshape(-1)
+ # Indices that need to be imputed.
+ indices_linear = np.argwhere(maskflt == 0).flatten()
+ # Set up sparse equation system, solve system.
+ A, b = NoisyLinearImputer.setup_sparse_system(
+ mask.numpy(), img.numpy(), neighbors_weights)
+ res = torch.tensor(spsolve(csc_matrix(A), b), dtype=torch.float)
+
+ # Fill the values with the solution of the system.
+ img_infill = imgflt.clone()
+ img_infill[:, indices_linear] = res.t() + self.noise * \
+ torch.randn_like(res.t())
+
+ return img_infill.reshape_as(img)
+
+
+class ROADMostRelevantFirst(PerturbationConfidenceMetric):
+ def __init__(self, percentile=80):
+ super(ROADMostRelevantFirst, self).__init__(
+ RemoveMostRelevantFirst(percentile, NoisyLinearImputer()))
+
+
+class ROADLeastRelevantFirst(PerturbationConfidenceMetric):
+ def __init__(self, percentile=20):
+ super(ROADLeastRelevantFirst, self).__init__(
+ RemoveLeastRelevantFirst(percentile, NoisyLinearImputer()))
+
+
+class ROADMostRelevantFirstAverage(AveragerAcrossThresholds):
+ def __init__(self, percentiles=[10, 20, 30, 40, 50, 60, 70, 80, 90]):
+ super(ROADMostRelevantFirstAverage, self).__init__(
+ ROADMostRelevantFirst, percentiles)
+
+
+class ROADLeastRelevantFirstAverage(AveragerAcrossThresholds):
+ def __init__(self, percentiles=[10, 20, 30, 40, 50, 60, 70, 80, 90]):
+ super(ROADLeastRelevantFirstAverage, self).__init__(
+ ROADLeastRelevantFirst, percentiles)
+
+
+class ROADCombined:
+ def __init__(self, percentiles=[10, 20, 30, 40, 50, 60, 70, 80, 90]):
+ self.percentiles = percentiles
+ self.morf_averager = ROADMostRelevantFirstAverage(percentiles)
+ self.lerf_averager = ROADLeastRelevantFirstAverage(percentiles)
+
+ def __call__(self,
+ input_tensor: torch.Tensor,
+ cams: np.ndarray,
+ targets: List[Callable],
+ model: torch.nn.Module):
+
+ scores_lerf = self.lerf_averager(input_tensor, cams, targets, model)
+ scores_morf = self.morf_averager(input_tensor, cams, targets, model)
+ return (scores_lerf - scores_morf) / 2
\ No newline at end of file
diff --git a/explainability/utils/image.py b/explainability/utils/image.py
index 83b90fa..92a5ba4 100644
--- a/explainability/utils/image.py
+++ b/explainability/utils/image.py
@@ -44,4 +44,18 @@ def deprocess_image(tensor):
img = (img * std) + mean
img = np.clip(img, 0, 1) # range(0~1)
- return np.float32(img)
\ No newline at end of file
+ return np.float32(img)
+
+def remove_padding_and_resize(cam, orig_shape, target_shape):
+ orig_h, orig_w = orig_shape[:2] # (H,W,3)
+ target_h, target_w = target_shape[2:] # (1,3,H,W)
+
+ scale = max(target_h, target_w) / max(orig_h, orig_w)
+ new_h, new_w = int(orig_h * scale), int(orig_w * scale)
+
+ pad_h = (target_h - new_h) // 2
+ pad_w = (target_w - new_w) // 2
+
+ # Remove Padding
+ unpadded_cam = cam[pad_h:pad_h+new_h, pad_w:pad_w+new_w]
+ return cv2.resize(unpadded_cam, (orig_w, orig_h))
\ No newline at end of file
diff --git a/explainability/utils/model_targets.py b/explainability/utils/model_targets.py
index 62ba03a..ba63a85 100644
--- a/explainability/utils/model_targets.py
+++ b/explainability/utils/model_targets.py
@@ -1,3 +1,5 @@
+import torch
+
class BinaryClassifierOutputTarget:
def __init__(self, category):
self.category = category
@@ -8,4 +10,7 @@ def __call__(self, model_output):
else:
sign = -1
return model_output * sign
-
\ No newline at end of file
+
+class BinaryClassifierOutputSigmoidTarget:
+ def __call__(self, model_output):
+ return torch.sigmoid(model_output)
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..2d4ad22
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,51 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+# ─────────────────────────────────────────────
+# [project]
+# PyPI에 올라가는 패키지 메타데이터
+# pip install deepguard 했을 때 보이는 정보들
+# ─────────────────────────────────────────────
+[project]
+name = "deepguard" # pip install deepguard 할 때 이 이름
+version = "0.2.1" # pip install deepguard==0.2.1 으로 버전 고정 가능
+description = "Multi-Scale Efficient Vision Transformer for Robust Deepfake Detection"
+readme = "README.md" # PyPI 페이지에 표시될 설명 (마크다운)
+requires-python = ">=3.10" # 이 버전 미만이면 pip install 자체를 막음
+license = { text = "MIT" }
+authors = [
+ { name = "seoyunje", email = "seoyunje2001@gmail.com" }
+]
+classifiers = [
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "License :: OSI Approved :: MIT License",
+ "Operating System :: OS Independent",
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+dependencies = ["torch>=2.10.0",
+ "torchmetrics>=1.6.1",
+ "timm>=1.0.12",
+ "ultralytics>=8.4.41",
+ "grad-cam>=1.5.5"
+ ]
+
+[project.urls]
+Homepage = "https://github.com/HanMoonSub/DeepGuard"
+"Bug Tracker" = "https://github.com/HanMoonSub/DeepGuard/issues"
+"Source Code" = "https://github.com/HanMoonSub/DeepGuard"
+
+
+[tool.setuptools.packages.find]
+
+include = [
+ "deepguard*", # 모델, 레이어, 데이터셋 등 핵심 패키지
+ "inference*", # ImagePredictor, VideoPredictor
+ "explainability*", # CAM 기반 XAI
+ "preprocess*", # 얼굴 검출, 프레임 추출 등
+]
+
+exclude = ["App*", "client*"]
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index ce368d3..a7bc9ab 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,6 +5,7 @@ python-multipart>=0.0.22
pydantic[email]>=2.12.5
aiofiles>=25.1.0
itsdangerous>=2.2.0
+python-dotenv>=0.9.9
# --- [Database & ORM] ---
sqlalchemy>=2.0.48
@@ -16,6 +17,7 @@ redis>=7.4.0
# --- [Security & Auth] ---
bcrypt>=4.0.1
passlib[bcrypt]>=1.7.4
+cryptography>=42.0.0
# --- [Data Analysis & Math] ---
numpy>=1.23.0,<3.0.0
@@ -40,4 +42,7 @@ seaborn>=0.13.2
colorama>=0.4.6
# --- [Distributed Task Queue] ---
-celery>=5.6.3
\ No newline at end of file
+celery>=5.6.3
+
+# --- [Linting & Formatting] ---
+ruff>=0.9.0
\ No newline at end of file
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 16d3c3c..0000000
--- a/setup.py
+++ /dev/null
@@ -1,30 +0,0 @@
-import setuptools
-
-with open("README.md", mode='r', encoding='utf-8') as fh:
- long_description = fh.read()
-
-setuptools.setup(
- name = 'deepguard',
- version = '0.2.1',
- author = 'seoyunje',
- author_email = "seoyunje2001@gmail.com",
- description = "Multi-Scale Efficient Vision Transformer for Robust Deepfake Detection",
- long_description = long_description,
- long_description_content_type ='text/markdown',
- url = 'https://github.com/HanMoonSub/DeepGuard',
- project_urls = {
- "Bug Tracker": "https://github.com/HanMoonSub/DeepGuard/issues",
- "Source Code": "https://github.com/HanMoonSub/DeepGuard",
- },
- classifiers = [
- 'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 3.10',
- 'Programming Language :: Python :: 3.11',
- 'License :: OSI Approved :: MIT License',
- 'Operating System :: OS Independent',
- 'Topic :: Scientific/Engineering :: Artificial Intelligence',
- ],
- packages = setuptools.find_packages(exclude=['Attention', 'preprocess','inference']),
- python_requires = '>=3.10',
- install_requires = []
-)
\ No newline at end of file