diff --git a/BLEU/BLEU.pdf b/BLEU/BLEU.pdf new file mode 100644 index 0000000..1c90d92 Binary files /dev/null and b/BLEU/BLEU.pdf differ diff --git a/BLEU/bleu.py b/BLEU/bleu.py index 2d87fee..41a8790 100644 --- a/BLEU/bleu.py +++ b/BLEU/bleu.py @@ -1,40 +1,52 @@ from collections import Counter import numpy as np + # removes punctuation and adds everything to lowercase and converts to list of list of str -def text_prep(sentences): - if not type(sentences)== list: - raise ValueError("Please enter a list") +def text_prep(sentences:list[str]): + assert type(sentences)==list, "The input to the text-prep function should be a list of sentences." return [["".join(char.lower() for char in word if char.isalnum()) for word in sentence.split()]for sentence in sentences] -def ngram(candidate,references,n): +def ngram(candidate: list[str], references: list[list[str]], n:int): if n<1: raise ValueError("Condition not met : N>=1") candidate_ngram = Counter([tuple(candidate[i:i+n]) for i in range(len(candidate)-n+1)]) + # print(candidate_ngram) max_ref=Counter() for ref in references: ref_ngram=Counter([tuple(ref[i:i+n]) for i in range(len(ref)-n+1)]) for n_gram in ref_ngram: max_ref[n_gram]=max(max_ref[n_gram],ref_ngram[n_gram]) + # print(max_ref) clipped_cnt={k:min(count,max_ref[k]) for k,count in candidate_ngram.items()} return sum(clipped_cnt.values()),sum(candidate_ngram.values()) - -def bp(candidate,references): +def brevity_penalty(candidate:list[str],references:list[list[str]]): if len(candidate)>min(len(ref) for ref in references): return 1 else: return float(np.exp(1-len(candidate)/min(len(ref) for ref in references))) - -def bleu(candidate_seq,reference_seq,n_max): - candidate_seq: list[str] = text_prep([candidate_seq])[0] - references_seq: list[list[str]] = text_prep(reference_seq) +def bleu(candidate_seq: str, reference_seq: list[str], n_max: int): + candidate_seq = text_prep([candidate_seq])[0] + reference_seq = text_prep(reference_seq) precision=[] for n in range(1,n_max+1): p_n,total=ngram(candidate_seq,reference_seq,n) precision.append(p_n/total if total>0 else 0) + # print(f"Precision = {precision}") if all(p==0 for p in precision): return 0 mean=np.exp(np.mean([np.log(p) for p in precision if p>0])) - brev=bp(candidate_seq,reference_seq) - return brev*mean + bp=brevity_penalty(candidate_seq,reference_seq) + return bp*mean + +if __name__=="__main__": + print(bleu("I have fun while playing cricket. I love cricket.", + [ + "I like to play cricket", + "I enjoy playing cricket", + "I play cicket and have fun", + "I love to play cricket", + "I love playing cricket" + ], + 6)) \ No newline at end of file diff --git a/LoRA/LoRA.pdf b/LoRA/LoRA.pdf new file mode 100644 index 0000000..671a865 Binary files /dev/null and b/LoRA/LoRA.pdf differ diff --git a/LoRA/lora.md b/LoRA/lora.md new file mode 100644 index 0000000..c77dc8e --- /dev/null +++ b/LoRA/lora.md @@ -0,0 +1,106 @@ +# ๐Ÿงพ LoRA Module: Low-Rank Adaptation for Neural Network Weights + +This code defines a custom PyTorch module implementing **Low-Rank Adaptation (LoRA)**, which is often used to adapt pretrained models efficiently by injecting low-rank matrices into the weight matrices, instead of fine-tuning the full model. + +--- + +## ๐Ÿ”ง **What the Module Does** + +* Adds a **trainable low-rank modification** to an existing weight matrix `wts`. +* Instead of fine-tuning `wts` directly, it learns two small matrices `A` and `B` whose product is added to `wts`. +* The goal is **parameter-efficient fine-tuning**. + +--- + +## ๐Ÿ“ฆ Code Breakdown + +```python +import torch +import torch.nn as nn + +class LoRA(nn.Module): +``` + +Defines a subclass of `nn.Module` called `LoRA`. + +--- + +### ๐Ÿ”จ `__init__` method + +```python +def __init__(self, in_f, out_f, rank=1, alpha=1, device: str = "cpu"): +``` + +* `in_f`: Input feature dimension of the original weight matrix. +* `out_f`: Output feature dimension of the original weight matrix. +* `rank`: Rank of the low-rank decomposition (smaller means fewer parameters). +* `alpha`: Scaling factor to balance the update magnitude. +* `device`: Device to store the parameters (`"cpu"` or `"cuda"`). + +```python +super().__init__() +self.A = nn.Parameter(torch.zeros((rank, out_f)).to(device)) +self.B = nn.Parameter(torch.zeros((in_f, rank)).to(device)) +``` + +* `A` and `B` are the low-rank trainable matrices. +* The effective update to the weights is `B @ A`, with shape `(in_f, out_f)`. + +```python +self.scale = alpha / rank +self.en = True +``` + +* `scale` adjusts the update size (as recommended in LoRA paper). +* `en`: A flag to enable or disable LoRA injection. + +--- + +### ๐Ÿง  `forward` method + +```python +def forward(self, wts): +``` + +* Takes the original weight matrix `wts` (from some layer in a neural network). +* Returns the modified weight matrix depending on whether LoRA is enabled. + +```python +if self.en: + return wts + torch.matmul(self.B, self.A).view(wts.shape) * self.scale +else: + return wts +``` + +* If enabled: + + * Computes the low-rank update `B @ A`. + * Reshapes it to match `wts` (typically already `(in_f, out_f)`). + * Scales it by `alpha/rank`. + * Adds it to the original weights. +* If disabled: returns the original `wts` unchanged. + +--- + +## โœ… Example Use Case + +```python +w = torch.randn(128, 256) # Example weight matrix +lora = LoRA(in_f=128, out_f=256, rank=4, alpha=16, device='cpu') + +new_w = lora(w) # Modified weights using LoRA +``` + +You can plug `new_w` into another linear layer, or use it to override pretrained weights. + +--- + +## โœ๏ธ Summary + +| Component | Purpose | +| -------------- | ---------------------------------------------- | +| `self.A` | Rank-`r` matrix for projecting to output dim | +| `self.B` | Rank-`r` matrix for projecting from input dim | +| `self.scale` | Scaling factor `alpha / rank` | +| `forward(wts)` | Returns adapted weights: `wts + B @ A * scale` | +| `self.en` | Enables or disables the LoRA modification |