diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000..2628b37 --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,12 @@ +# Example scripts + +- [registry_analyze.py](registry_analyze.py) - list all images and tags, create json cache as result +- [create_lstags.py](create_lstags.py) - load cached json, and create [lstags] config + +[lstags]: https://github.com/ivanilves/lstags + +``` +cp cred.example.py cred.prod.py +./registry_analyze.py cred.prod.py +./create_lstags.py cred.prod.py > lstags.yml +``` diff --git a/contrib/create_lstags.py b/contrib/create_lstags.py new file mode 100755 index 0000000..c7c2adc --- /dev/null +++ b/contrib/create_lstags.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +import yaml +import json +import os +import sys +import binascii +from gitlab_registry_usage.cli import human_size +from urllib.parse import urlparse + +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper + +def parse_config(filename): + """Parse configuration file. + + Args: + filename: Path to the configuration file to parse. + Returns: + Dictionary of values defined in the file. + """ + with open(filename) as f: + data = f.read() + compiled = compile(data, filename, "exec") + result = { 'main': sys.modules[__name__] } + eval(compiled, result) + return result + +c = parse_config(sys.argv[1]) + +registry_base_url = c['registry_base_url'] + +def getRegistry(cache_file): + with open(cache_file) as f: + data = json.load(f) + return data + +gitlab_registry = getRegistry(cache_file='{hostname}.json'.format(hostname=urlparse(c['registry_base_url']).netloc)) + +def format_printable(gitlab_registry): + for repository, tags in gitlab_registry['repository_tags'].items(): + if tags is None: + continue + disk_size = human_size(gitlab_registry['repository_disk_sizes'][repository]) + repo_size = human_size(gitlab_registry['repository_sizes'][repository]) + print("\n# {repository} {repo_size} {disk_size}".format(repository=repository, repo_size=repo_size, disk_size=disk_size)) + for tag in tags: + disk_size = human_size(gitlab_registry['tag_disk_sizes'][repository][tag]) + repo_size = human_size(gitlab_registry['tag_sizes'][repository][tag]) + print(" {repository}/{tag} {repo_size} {disk_size}".format(repository=repository, tag=tag, repo_size=repo_size, disk_size=disk_size)) + + print("\n# total: {size}".format(size=human_size(gitlab_registry['total_size']))) + +def is_hex(string): + try: + res = binascii.unhexlify(string) + except binascii.Error: + return False + return True + +def format_lstags(gitlab_registry): + repos = [] + registry = gitlab_registry['registry'] + + for repository, tags in gitlab_registry['repository_tags'].items(): + if tags is None: + continue + if repository.endswith('/builds') or repository.endswith('/build') or repository.endswith('/test') or repository.endswith('/dev'): + print("# Skipped image: {image}".format(image=repository)) + continue + for tag in tags: + image = '{registry}/{repo}={tag}'.format(registry=registry, repo=repository, tag=tag) + if len(tag) == 40 and is_hex(tag): + print("# Skipped tag: {repo}:{tag}".format(repo=repository, tag=tag)) + continue + repos.append(image) + + data = {} + data['lstags'] = {} + data['lstags']['repositories'] = repos + + return yaml.dump(data, Dumper=Dumper, default_flow_style=False) + +print(format_lstags(gitlab_registry)) +#format_printable(gitlab_registry) diff --git a/contrib/cred.example.py b/contrib/cred.example.py new file mode 100644 index 0000000..fbdbf45 --- /dev/null +++ b/contrib/cred.example.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +gitlab_base_url = 'https://gitlab.example.net/' +registry_base_url = 'https://gitlab.example.net:4567/' + +# ldap or access token +username = 'glen' +access_token = 'secret' diff --git a/contrib/registry_analyze.py b/contrib/registry_analyze.py new file mode 100755 index 0000000..24274ee --- /dev/null +++ b/contrib/registry_analyze.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +from gitlab_registry_usage import GitLabRegistry +import json +import os +import sys +from urllib.parse import urlparse + +def parse_config(filename): + """Parse configuration file. + + Args: + filename: Path to the configuration file to parse. + Returns: + Dictionary of values defined in the file. + """ + with open(filename) as f: + data = f.read() + compiled = compile(data, filename, "exec") + result = { 'main': sys.modules[__name__] } + eval(compiled, result) + return result + +c = parse_config(sys.argv[1]) + +gitlab_base_url = c['gitlab_base_url'] +registry_base_url = c['registry_base_url'] +username = c['username'] +access_token = c['access_token'] + +def getRegistry(cache_file, config): + if os.path.exists(cache_file): + print("loading cache: {}".format(cache_file)) + with open(cache_file) as f: + data = json.load(f) + else: + gitlab_registry = GitLabRegistry( + gitlab_base_url, registry_base_url, username, access_token + ) + data = {} + data['registry'] = urlparse(config['registry_base_url']).netloc + data['gitlab_base_url'] = config['gitlab_base_url'] + data['registry_base_url'] = config['registry_base_url'] + data['repository_tags'] = gitlab_registry.repository_tags + data['repository_sizes'] = gitlab_registry.repository_sizes + data['repository_disk_sizes'] = gitlab_registry.repository_disk_sizes + data['tag_sizes'] = gitlab_registry.tag_sizes + data['tag_disk_sizes'] = gitlab_registry.tag_disk_sizes + data['total_size'] = gitlab_registry.total_size + data['total_disk_size'] = gitlab_registry.total_disk_size + + with open(cache_file, 'w') as f: + json.dump(data, f, sort_keys=True, indent=4, separators=(',', ': ')) + return data + +gitlab_registry = getRegistry(cache_file='{hostname}.json'.format(hostname=urlparse(c['registry_base_url']).netloc), config=c) + +for repository in gitlab_registry['repository_tags'].keys(): + repository_tags = gitlab_registry['repository_tags'][repository] + repository_size = gitlab_registry['repository_sizes'][repository] + repository_disk_size = gitlab_registry['repository_disk_sizes'][repository] + tag_sizes = gitlab_registry['tag_sizes'][repository] + tag_disk_sizes = gitlab_registry['tag_disk_sizes'][repository] + if ( + repository_tags is not None and repository_size is not None + and repository_disk_size is not None and tag_sizes is not None + and tag_disk_sizes is not None + ): + print( + '{}: repository size: {}, repository disk size: {}'.format( + repository, repository_size, repository_disk_size + ) + ) + for tag in repository_tags: + print( + '{}: tag size: {}, tag disk size: {}'.format( + tag, tag_sizes[tag], tag_disk_sizes[tag] + ) + ) + else: + print('{}: no further information available'.format(repository)) + print() +print( + ('total size: {}, total disk size: {}').format( + gitlab_registry['total_size'], gitlab_registry['total_disk_size'] + ) +)