Skip to content

local-support #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pavanran wants to merge 1 commit into
base: main
Choose a base branch
from
Open

local-support #12

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
227 changes: 78 additions & 149 deletions repototxt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,160 +1,89 @@
import os
from github import Github
from tqdm import tqdm

# Set your GitHub token here
GITHUB_TOKEN = os.getenv('GITHUB_TOKEN', 'YOUR TOKEN HERE')

def get_readme_content(repo):
"""
Retrieve the content of the README file.
"""
try:
readme = repo.get_contents("README.md")
return readme.decoded_content.decode('utf-8')
except:
return "README not found."

def traverse_repo_iteratively(repo):
"""
Traverse the repository iteratively to avoid recursion limits for large repositories.
"""
structure = ""
dirs_to_visit = [("", repo.get_contents(""))]
dirs_visited = set()

while dirs_to_visit:
path, contents = dirs_to_visit.pop()
dirs_visited.add(path)
for content in tqdm(contents, desc=f"Processing {path}", leave=False):
if content.type == "dir":
if content.path not in dirs_visited:
structure += f"{path}/{content.name}/\n"
dirs_to_visit.append((f"{path}/{content.name}", repo.get_contents(content.path)))
else:
structure += f"{path}/{content.name}\n"
return structure

def get_file_contents_iteratively(repo):
file_contents = ""
dirs_to_visit = [("", repo.get_contents(""))]
dirs_visited = set()
binary_extensions = [
# Compiled executables and libraries
'.exe', '.dll', '.so', '.a', '.lib', '.dylib', '.o', '.obj',
# Compressed archives
'.zip', '.tar', '.tar.gz', '.tgz', '.rar', '.7z', '.bz2', '.gz', '.xz', '.z', '.lz', '.lzma', '.lzo', '.rz', '.sz', '.dz',
# Application-specific files
'.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.odt', '.ods', '.odp',
# Media files (less common)
'.png', '.jpg', '.jpeg', '.gif', '.mp3', '.mp4', '.wav', '.flac', '.ogg', '.avi', '.mkv', '.mov', '.webm', '.wmv', '.m4a', '.aac',
# Virtual machine and container images
'.iso', '.vmdk', '.qcow2', '.vdi', '.vhd', '.vhdx', '.ova', '.ovf',
# Database files
'.db', '.sqlite', '.mdb', '.accdb', '.frm', '.ibd', '.dbf',
# Java-related files
'.jar', '.class', '.war', '.ear', '.jpi',
# Python bytecode and packages
'.pyc', '.pyo', '.pyd', '.egg', '.whl',
# Other potentially important extensions
'.deb', '.rpm', '.apk', '.msi', '.dmg', '.pkg', '.bin', '.dat', '.data',
'.dump', '.img', '.toast', '.vcd', '.crx', '.xpi', '.lockb', 'package-lock.json', '.svg' ,
'.eot', '.otf', '.ttf', '.woff', '.woff2',
'.ico', '.icns', '.cur',
'.cab', '.dmp', '.msp', '.msm',
'.keystore', '.jks', '.truststore', '.cer', '.crt', '.der', '.p7b', '.p7c', '.p12', '.pfx', '.pem', '.csr',
'.key', '.pub', '.sig', '.pgp', '.gpg',
'.nupkg', '.snupkg', '.appx', '.msix', '.msp', '.msu',
'.deb', '.rpm', '.snap', '.flatpak', '.appimage',
'.ko', '.sys', '.elf',
'.swf', '.fla', '.swc',
'.rlib', '.pdb', '.idb', '.pdb', '.dbg',
'.sdf', '.bak', '.tmp', '.temp', '.log', '.tlog', '.ilk',
'.bpl', '.dcu', '.dcp', '.dcpil', '.drc',
'.aps', '.res', '.rsrc', '.rc', '.resx',
'.prefs', '.properties', '.ini', '.cfg', '.config', '.conf',
'.DS_Store', '.localized', '.svn', '.git', '.gitignore', '.gitkeep',
]

while dirs_to_visit:
path, contents = dirs_to_visit.pop()
dirs_visited.add(path)
for content in tqdm(contents, desc=f"Downloading {path}", leave=False):
if content.type == "dir":
if content.path not in dirs_visited:
dirs_to_visit.append((f"{path}/{content.name}", repo.get_contents(content.path)))
else:
# Check if the file extension suggests it's a binary file
if any(content.name.endswith(ext) for ext in binary_extensions):
file_contents += f"File: {path}/{content.name}\nContent: Skipped binary file\n\n"
import zipfile
import tempfile
import shutil

binary_extensions = {
'.exe', '.dll', '.so', '.a', '.lib', '.dylib', '.o', '.obj',
'.zip', '.tar', '.tar.gz', '.tgz', '.rar', '.7z', '.bz2', '.gz',
'.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
'.png', '.jpg', '.jpeg', '.gif', '.mp3', '.mp4', '.wav',
'.iso', '.db', '.jar', '.class', '.pyc', '.whl',
'.svg', '.ico', '.woff', '.ttf', '.p12', '.pem', '.key', '.pfx',
'.DS_Store', '.gitignore', '.gitkeep', '.log'
}

def is_binary_file(file_path):
ext = os.path.splitext(file_path)[1].lower()
return ext in binary_extensions

def analyze_local_repo(zip_path):
repo_name = os.path.splitext(os.path.basename(zip_path))[0]

with tempfile.TemporaryDirectory() as tmpdir:
with zipfile.ZipFile(zip_path, 'r') as zip_ref:
zip_ref.extractall(tmpdir)

base_dir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) # first folder inside zip

structure = f"Repository Structure: {repo_name}\n"
file_contents = ""
readme_content = ""

for root, dirs, files in os.walk(base_dir):
rel_root = os.path.relpath(root, base_dir)
if rel_root == '.':
rel_root = ''

for d in dirs:
structure += f"{rel_root}/{d}/\n"

for file in files:
file_path = os.path.join(root, file)
rel_path = os.path.join(rel_root, file)

structure += f"{rel_path}\n"

if file.lower() == "readme.md":
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
readme_content = f.read()

if is_binary_file(file_path):
file_contents += f"File: {rel_path}\nContent: Skipped binary file\n\n"
else:
file_contents += f"File: {path}/{content.name}\n"
try:
if content.encoding is None or content.encoding == 'none':
file_contents += "Content: Skipped due to missing encoding\n\n"
else:
try:
decoded_content = content.decoded_content.decode('utf-8')
file_contents += f"Content:\n{decoded_content}\n\n"
except UnicodeDecodeError:
try:
decoded_content = content.decoded_content.decode('latin-1')
file_contents += f"Content (Latin-1 Decoded):\n{decoded_content}\n\n"
except UnicodeDecodeError:
file_contents += "Content: Skipped due to unsupported encoding\n\n"
except (AttributeError, UnicodeDecodeError):
file_contents += "Content: Skipped due to decoding error or missing decoded_content\n\n"
return file_contents

def get_repo_contents(repo_url):
"""
Main function to get repository contents.
"""
repo_name = repo_url.split('/')[-1]
if not GITHUB_TOKEN:
raise ValueError("Please set the 'GITHUB_TOKEN' environment variable or the 'GITHUB_TOKEN' in the script.")
g = Github(GITHUB_TOKEN)
repo = g.get_repo(repo_url.replace('https://github.com/', ''))

print(f"Fetching README for: {repo_name}")
readme_content = get_readme_content(repo)

print(f"\nFetching repository structure for: {repo_name}")
repo_structure = f"Repository Structure: {repo_name}\n"
repo_structure += traverse_repo_iteratively(repo)

print(f"\nFetching file contents for: {repo_name}")
file_contents = get_file_contents_iteratively(repo)

instructions = f"Prompt: Analyze the {repo_name} repository to understand its structure, purpose, and functionality. Follow these steps to study the codebase:\n\n"
instructions += "1. Read the README file to gain an overview of the project, its goals, and any setup instructions.\n\n"
instructions += "2. Examine the repository structure to understand how the files and directories are organized.\n\n"
instructions += "3. Identify the main entry point of the application (e.g., main.py, app.py, index.js) and start analyzing the code flow from there.\n\n"
instructions += "4. Study the dependencies and libraries used in the project to understand the external tools and frameworks being utilized.\n\n"
instructions += "5. Analyze the core functionality of the project by examining the key modules, classes, and functions.\n\n"
instructions += "6. Look for any configuration files (e.g., config.py, .env) to understand how the project is configured and what settings are available.\n\n"
instructions += "7. Investigate any tests or test directories to see how the project ensures code quality and handles different scenarios.\n\n"
instructions += "8. Review any documentation or inline comments to gather insights into the codebase and its intended behavior.\n\n"
instructions += "9. Identify any potential areas for improvement, optimization, or further exploration based on your analysis.\n\n"
instructions += "10. Provide a summary of your findings, including the project's purpose, key features, and any notable observations or recommendations.\n\n"
instructions += "Use the files and contents provided below to complete this analysis:\n\n"

return repo_name, instructions, readme_content, repo_structure, file_contents
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
content = f.read()
file_contents += f"File: {rel_path}\nContent:\n{content}\n\n"
except Exception as e:
file_contents += f"File: {rel_path}\nContent: Skipped due to error: {e}\n\n"

instructions = f"Prompt: Analyze the {repo_name} repository to understand its structure, purpose, and functionality. Follow these steps:\n\n"
instructions += "1. Read the README file to gain an overview of the project.\n"
instructions += "2. Examine the repository structure.\n"
instructions += "3. Identify the main entry point.\n"
instructions += "4. Study dependencies.\n"
instructions += "5. Analyze core functionality.\n"
instructions += "6. Review config files.\n"
instructions += "7. Look for test cases.\n"
instructions += "8. Review comments and documentation.\n"
instructions += "9. Suggest improvements.\n"
instructions += "10. Summarize the findings.\n\n"

if __name__ == '__main__':
repo_url = input("Please enter the GitHub repository URL: ")
try:
repo_name, instructions, readme_content, repo_structure, file_contents = get_repo_contents(repo_url)
output_filename = f'{repo_name}_contents.txt'
with open(output_filename, 'w', encoding='utf-8') as f:
f.write(instructions)
f.write(f"README:\n{readme_content}\n\n")
f.write(repo_structure)
f.write(structure)
f.write('\n\n')
f.write(file_contents)

print(f"Repository contents saved to '{output_filename}'.")
except ValueError as ve:
print(f"Error: {ve}")
except Exception as e:
print(f"An error occurred: {e}")
print("Please check the repository URL and try again.")

# Example usage:
if __name__ == '__main__':
zip_file_path = input("Enter the path to the zipped repo: ")
if not os.path.isfile(zip_file_path):
print("Zip file not found!")
else:
analyze_local_repo(zip_file_path)