From 44677a84310467ee82c6c6b55eebcebb4a24680f Mon Sep 17 00:00:00 2001 From: Ajey K Date: Thu, 1 Oct 2020 22:09:27 +0530 Subject: [PATCH] Added aws refactor changes. --- .../connector/snapshot_aws_refactor.py | 762 ++++++++++++++++++ 1 file changed, 762 insertions(+) create mode 100644 src/processor/connector/snapshot_aws_refactor.py diff --git a/src/processor/connector/snapshot_aws_refactor.py b/src/processor/connector/snapshot_aws_refactor.py new file mode 100644 index 00000000..83797a90 --- /dev/null +++ b/src/processor/connector/snapshot_aws_refactor.py @@ -0,0 +1,762 @@ +""" +snapshot connector file for aws type snapshot. When the 'type' is 'aws', the +resource objects have to be fetched from the AWS interface. +The connection could use python SDK libraries or https calls or other known methods +provided by the provider. +Here in case of AWS, have used boto python SDK released and maintained by AWS. +boto library provides different 'describe_' functions and using this functionality, +the snapshot mentions which describe function has to be called, so 'type' could +be 'security_groups' , 'instances', 'regions' and using python reflection capabilities +a callable from describe_security_groups gets callable for this function to return security groups. +The describe_ functions lots of methods to query AWS resources, since we know what resource we need +we always query by id, so AWS snapshots shall always have {"id": "123de23"} to uniquely +identify the resource object. +""" +import json +import hashlib +import time +import copy +import pymongo +import os +from boto3 import client +from boto3 import Session +from processor.helper.file.file_utils import exists_file +from processor.logging.log_handler import getlogger +from processor.helper.config.rundata_utils import put_in_currentdata, get_dbtests, get_from_currentdata +from processor.helper.json.json_utils import get_field_value, json_from_file,\ + collectiontypes, STRUCTURE, make_snapshots_dir, store_snapshot +from processor.connector.vault import get_vault_data +from processor.helper.config.config_utils import config_value, get_test_json_dir, CUSTOMER +from processor.database.database import insert_one_document, sort_field, get_documents,\ + COLLECTION, DATABASE, DBNAME, get_collection_size, create_indexes +from processor.helper.httpapi.restapi_azure import json_source +from processor.helper.httpapi.restapi_azure import get_client_secret +from processor.connector.snapshot_utils import validate_snapshot_nodes +from processor.connector.arn_parser import arnparse + +logger = getlogger() +_valid_service_names = Session().get_available_services() + + +def _validate_client_name(client_name): + """ + A private function to validate whether a given client provided + in snapshot or aws connector is a valid service in Boto3 + """ + return client_name is not None and client_name.lower() in _valid_service_names + + +def get_aws_data(snapshot_json, snapshot): + """ + The AWS source object to be fetched from database or the filesystem + The initial configuration for database is 'validator' and collection + is 'structures', whereas for the filesystem the path to fetch the + 'structures' is $SOLUTIONDIR/realm/.json + """ + sub_data = snapshot.get_structure_data(snapshot_json) + return sub_data + + +def _get_aws_function(awsclient, node): + """ + A private function to get the function which has to be called by the + boto3 client object to get snapshot data. + """ + describe_function_str = get_aws_describe_function(node) + if describe_function_str: + describe_function = getattr(awsclient, describe_function_str, None) + if describe_function and callable(describe_function): + return describe_function + + function_str = _get_callable_method_from_node(node) + if function_str: + callable_function = getattr(awsclient, function_str, None) + if callable_function and callable(callable_function): + return callable_function + + +def _get_callable_method_from_node(node): + """Callable Method from node using python reflection mechanism""" + _fn_str = None + if node and 'type' in node and node['type']: + _fn_str = node['type'] + return _fn_str + + +def get_aws_describe_function(node): + """Describe function for the node using python reflection mechanism""" + describe_fn_str = None + if node and 'type' in node and node['type']: + describe_fn_str = 'describe_%s' % node['type'] + return describe_fn_str + + +def get_node(awsclient, node, snapshot_source): + """ + Fetch node from aws using connection. In this case using boto API's + describe functions. + """ + + collection = node['collection'] if 'collection' in node else COLLECTION + parts = snapshot_source.split('.') + function_to_call = None + db_record = { + "structure": "aws", + "error": None, + "reference": "", + "source": parts[0], + "path": '', + "timestamp": int(time.time() * 1000), + "queryuser": "", + "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), + "node": node, + "snapshotId": node['snapshotId'], + "collection": collection.replace('.', '').lower(), + "json": {} # Refactor when node is absent it should None, when empty object put it as {} + } + detail_methods = get_field_value(node, "detailMethods") + if detail_methods is None: + function_to_call = _get_aws_function(awsclient, node) + if function_to_call and callable(function_to_call): + queryval = get_field_value(node, 'id') + try: + data = function_to_call(**queryval) + if data: + db_record['json'] = data + checksum = get_checksum(data) + if checksum: + db_record['checksum'] = checksum + else: + put_in_currentdata('errors', data) + logger.info("Describe function does not exist: %s", str(function_to_call)) + db_record['error'] = "Describe function does not exist: %s" % str(function_to_call) + except Exception as ex: + logger.info('Describe function exception: %s', ex) + db_record['error'] = 'Describe function exception: %s' % ex + else: + logger.info('Invalid function exception: %s', str(function_to_call)) + db_record['error'] = 'Invalid function exception: %s' % str(function_to_call) + else: + json_to_put = {} + arn_str = get_field_value(node, "arn") + db_record["path"] = arn_str if arn_str else "" + + arn_obj = arnparse(arn_str) + client_str = arn_obj.service + resourceid = arn_obj.resource + for each_method_str in detail_methods: + function_to_call = getattr(awsclient, each_method_str, None) + if function_to_call and callable(function_to_call): + params = _get_function_kwargs(arn_str, each_method_str, json_to_put) + try: + data = function_to_call(**params) + if data: + json_to_put.update(data) + except Exception as ex: + logger.info('Describe function exception: %s', ex) + db_record['error'] = 'Describe function exception: %s' % ex + else: + logger.info('Invalid function exception: %s', str(function_to_call)) + db_record['error'] = 'Invalid function exception: %s' % str(function_to_call) + db_record['json'] = json_to_put + return db_record + + +def _get_resources_from_list_function(response, method): + """ + Fetches the resources id from different responses + and returns a list of responses. + """ + if method == 'list_buckets': + return [x['Name'] for x in response['Buckets']] + elif method == 'describe_instances': + final_list = [] + for reservation in response['Reservations']: + for instance in reservation['Instances']: + final_list.append(instance['InstanceId']) + return final_list + elif method == 'describe_db_instances': + return [x['DBInstanceIdentifier'] for x in response['DBInstances']] + elif method == 'describe_load_balancers': + return [x['LoadBalancerName'] for x in response['LoadBalancerDescriptions']] + elif method == 'list_certificates': + return [x['CertificateArn'] for x in response['CertificateSummaryList']] + elif method == 'list_stacks': + return [x['StackName'] for x in response['StackSummaries']] + elif method == 'list_trails': + return [x['Name'] for x in response['Trails']] + elif method in ['describe_stacks', 'list_trails']: + return [x['StackName'] for x in response['Stacks']] + elif method == 'get_rest_apis': + return [x['id'] for x in response['items']] + elif method == 'list_users': + return [x['UserName'] for x in response['Users']] + elif method == 'list_roles': + return [x['RoleName'] for x in response['Roles']] + elif method == 'list_hosted_zones': + return [x['Id'] for x in response['HostedZones']] + elif method == 'list_keys': + return [x.get('KeyId') for x in response['Keys']] + elif method == 'list_tables': + return response.get("TableNames") + elif method == 'list_backups': + return [x.get('BackupArn',"") for x in response['BackupSummaries']] + return response.get("TableNames") + elif method == 'list_task_definitions': + return response.get('taskDefinitionArns') + elif method == 'list_clusters': + return response.get("clusters") + elif method == 'describe_replication_groups': + return [x.get('ReplicationGroupId') for x in response['ReplicationGroups']] + elif method == 'list_streams': + return response.get("StreamNames") + elif method == 'list_functions': + return [x.get('FunctionName',"") for x in response['Functions']] + elif method == 'describe_clusters': + return [x.get('ClusterIdentifier',"") for x in response['Clusters']] + elif method == 'list_topics': + return [x.get('TopicArn',"") for x in response['Topics']] + elif method == 'list_queues': + return response.get("QueueUrls") + elif method == 'list_domain_names': + return [x.get('DomainName') for x in response['DomainNames']] + elif method == 'describe_configuration_recorders': + return [x.get('name') for x in response['ConfigurationRecorders']] + elif method == 'list_distributions': + return [x.get('Id') for x in response['DistributionList']['Items']] + elif method == 'describe_vpn_gateways': + return [x.get('VpnGatewayId') for x in response['VpnGateways']] + elif method == 'describe_file_systems': + return [x.get('FileSystemId') for x in response['FileSystems']] + elif method == 'describe_parameters': + return [x.get('Name') for x in response['Parameters']] + elif method == 'describe_cache_subnet_groups': + return [x.get('CacheSubnetGroupName') for x in response['CacheSubnetGroups']] + elif method == 'describe_route_tables': + return [x.get('RouteTableId') for x in response['RouteTables']] + elif method == 'describe_network_acls': + return [x.get('NetworkAclId') for x in response['NetworkAcls']] + elif method == 'describe_event_subscriptions': + return [x.get('EventSubscriptionArn').split(':')[-1] for x in response['EventSubscriptionsList']] + elif method == 'describe_db_snapshots': + return [x.get('DBSnapshotIdentifier') for x in response['DBSnapshots']] + else: + return [] + +def get_all_nodes(awsclient, node, snapshot, connector): + """ Fetch all the nodes from the cloned git repository in the given path.""" + db_records = [] + arn_string = "arn:aws:%s:%s::%s" + collection = node['collection'] if 'collection' in node else COLLECTION + snapshot_source = get_field_value(snapshot, 'source') + parts = snapshot_source.split('.') + d_record = { + "structure": "aws", + "reference": "", + "source": parts[0], + "path": '', + "timestamp": int(time.time() * 1000), + "queryuser": "", + "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), + "node": node, + "snapshotId": None, + "masterSnapshotId": node['masterSnapshotId'], + "collection": collection.replace('.', '').lower(), + "json": {} + } + list_function_name = get_field_value(node, 'listMethod') + if list_function_name: + list_function = getattr(awsclient, list_function_name, None) + if list_function and callable(list_function): + try: + list_kwargs = _get_list_function_kwargs(awsclient.meta._service_model.service_name, list_function_name) + response = list_function(**list_kwargs) + list_of_resources = _get_resources_from_list_function(response, list_function_name) + except Exception as ex: + list_of_resources = [] + detail_methods = get_field_value(node, 'detailMethods') + for each_resource in list_of_resources: + type_list = [] + if "arn:" in each_resource: + resource_arn = each_resource + else: + resource_arn = arn_string %(awsclient.meta._service_model.service_name, + awsclient.meta.region_name, each_resource) + for each_method_str in detail_methods: + each_method = getattr(awsclient, each_method_str, None) + if each_method and callable(each_method): + type_list.append(each_method_str) + db_record = copy.deepcopy(d_record) + db_record['detailMethods'] = type_list + db_record['arn'] = resource_arn + db_records.append(db_record) + + return db_records + +def get_checksum(data): + """ Get the checksum for the AWS data fetched.""" + checksum = None + try: + data_str = json.dumps(data, default=str) + checksum = hashlib.md5(data_str.encode('utf-8')).hexdigest() + except: + pass + return checksum + +def _get_list_function_kwargs(service, function_name): + if service == "cloudformation" and function_name == 'list_stacks': + return { + 'StackStatusFilter' : ['CREATE_IN_PROGRESS', 'CREATE_COMPLETE', 'ROLLBACK_IN_PROGRESS',\ + 'ROLLBACK_COMPLETE', 'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS', \ + 'UPDATE_COMPLETE', 'UPDATE_ROLLBACK_IN_PROGRESS', 'UPDATE_ROLLBACK_FAILED', \ + 'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS', 'UPDATE_ROLLBACK_COMPLETE', \ + 'REVIEW_IN_PROGRESS', 'IMPORT_IN_PROGRESS', 'IMPORT_COMPLETE', 'IMPORT_ROLLBACK_IN_PROGRESS', \ + 'IMPORT_ROLLBACK_FAILED', 'IMPORT_ROLLBACK_COMPLETE'] + } + else: + return {} + +def _get_function_kwargs(arn_str, function_name, existing_json): + """Fetches the correct keyword arguments for different detail functions""" + arn = arnparse(arn_str) + client_str = arn.service + resource_id = arn.resource + { + 's3': {'Bucket' : resource_id}, + 'rds': {'DBInstanceIdentifier': resource_id} if function_name in ["describe_db_instances", "describe_db_snapshots"] else {}, + 'ec2': {'Attribute': 'instanceType', 'InstanceId': resource_id} if function_name == "describe_instance_attribute" else {}, + + } + if client_str == "s3": + return {'Bucket' : resource_id} + elif client_str == "rds" and function_name in ["describe_db_instances",\ + "describe_db_snapshots"]: + return { + 'DBInstanceIdentifier': resource_id + } + elif client_str == "ec2" and function_name == "describe_instance_attribute": + return { + 'Attribute': 'instanceType', + 'InstanceId': resource_id + } + elif client_str == "ec2" and function_name in ["describe_instances", "monitor_instances"]: + return { + 'InstanceIds': [resource_id] + } + elif client_str == "ec2" and function_name == "describe_images": + try: + imageid = existing_json['Reservations'][0]['Instances'][0]['ImageId'] + except: + imageid = "" + return { + 'ImageIds': [imageid] + } + elif client_str == "ec2" and function_name == "describe_volumes": + try: + volumeid = existing_json['Reservations'][0]['Instances'][0]['BlockDeviceMappings'][0]['Ebs']['VolumeId'] + except: + volumeid = "" + return { + 'VolumeIds': [volumeid] + } + elif client_str == "ec2" and function_name == "describe_security_groups": + try: + groups = existing_json['Reservations'][0]['Instances'][0]['SecurityGroups'] + groupsidlist = [x['GroupId'] for x in groups] + except: + volumeid = [] + return { + 'GroupIds': groupsidlist + } + elif client_str == "ec2" and function_name == "describe_vpcs": + try: + vpicid = existing_json['Reservations'][0]['Instances'][0]['VpcId'] + except: + vpicid = "" + return { + 'VpcIds': [vpicid] + } + elif client_str == "ec2" and function_name == "describe_subnets": + try: + subnetid = existing_json['Reservations'][0]['Instances'][0]['SubnetId'] + except: + subnetid = "" + return { + 'SubnetIds': [subnetid] + } + elif client_str == "ec2" and function_name == "describe_snapshots": + try: + ownerid = existing_json['Reservations'][0]['OwnerId'] + except: + ownerid = "" + return { + 'OwnerIds': [ownerid] + } + elif client_str == "ec2" and function_name == "describe_snapshot_attribute": + try: + snapshot_id = existing_json['Snapshots'][0]['SnapshotId'] + except: + snapshot_id = "" + return { + 'SnapshotId': snapshot_id, + 'Attribute' : 'createVolumePermission' + } + elif client_str == "elb" and function_name == "describe_load_balancers": + return { + 'LoadBalancerNames': [resource_id] + } + elif client_str == "elb" and function_name in ["describe_load_balancers", "describe_load_balancer_attributes",\ + "describe_load_balancer_policies"]: + return { + 'LoadBalancerName': resource_id + } + elif client_str == "acm" and function_name == "describe_certificate": + return { + 'CertificateArn': arn_str + } + elif client_str == "cloudformation" and function_name in ["describe_stack_resource",\ + "describe_stack_events", "describe_stacks", "describe_stack_resource_drifts", \ + "get_stack_policy"]: + return { + 'StackName': resource_id + } + elif client_str == "cloudtrail" and function_name == "describe_trails": + return { + 'trailNameList': [resource_id] + } + elif client_str == "cloudtrail" and function_name in ["get_event_selectors",\ + "get_insight_selectors"]: + return { + 'TrailName': resource_id + } + elif client_str == "apigateway" and function_name in ["get_rest_api",\ + "get_documentation_parts", "get_documentation_versions",\ + "get_gateway_responses", "get_models", "get_request_validators",\ + "get_resources", "get_stages"]: + return { + 'restApiId': resource_id + } + elif client_str == "route53" and function_name == "get_hosted_zone": + return { + 'Id': resource_id + } + elif client_str == "route53" and function_name == "list_resource_record_sets": + return { + 'HostedZoneId': resource_id + } + elif client_str == "iam" and function_name in ["get_user", "list_ssh_public_keys", \ + "get_account_summary", "get_account_password_policy", "list_attached_user_policies"]: + return { + 'UserName': resource_id + } + elif client_str == "iam" and function_name == "get_role": + return { + 'RoleName': resource_id + } + elif client_str == "kms" and function_name in ["get_key_rotation_status", "describe_key",]: + return { + 'KeyId': resource_id + } + elif client_str == "dynamodb" and function_name == "describe_table": + return { + 'TableName': resource_id + } + elif client_str == "dynamodb" and function_name == "describe_backup": + return { + 'BackupArn': arn_str + } + elif client_str == "ecs" and function_name == "describe_task_definition": + return { + 'taskDefinition': resource_id + } + elif client_str == "eks" and function_name == "describe_cluster": + return { + 'name': resource_id + } + elif client_str == "elasticache" and function_name == "describe_replication_groups": + return { + 'ReplicationGroupId': resource_id + } + elif client_str == "elasticache" and function_name == "describe_cache_subnet_groups": + return { + 'CacheSubnetGroupName': resource_id + } + elif client_str == "kinesis" and function_name == "describe_stream": + return { + 'StreamName': resource_id + } + elif client_str == "lambda" and function_name == "get_function": + return { + 'FunctionName': resource_id + } + elif client_str == "redshift" and function_name == "describe_clusters": + return { + 'ClusterIdentifier': resource_id + } + elif client_str == "sns" and function_name == "get_topic_attributes": + return { + 'TopicArn': arn_str + } + elif client_str == "sqs" and function_name == "get_queue_attributes": + return { + 'QueueUrl': 'https:{url}'.format(url=resource_id), 'AttributeNames': ['All'] + } + elif client_str == "config" and function_name in ["describe_configuration_recorders", "describe_configuration_recorder_status"]: + return { + 'ConfigurationRecorderNames': [resource_id] + } + elif client_str == "es" and function_name == "describe_elasticsearch_domain": + return { + 'DomainName': resource_id + } + elif client_str == "cloudfront" and function_name == "get_distribution": + return { + 'Id': resource_id + } + elif client_str == "ec2" and function_name == "describe_vpn_gateways": + return { + 'VpnGatewayIds': [resource_id] + } + elif client_str == "efs" and function_name == "describe_file_systems": + return { + 'FileSystemId': resource_id + } + elif client_str=='ec2'and function_name == 'describe_route_tables': + return{ + 'RouteTableIds': [resource_id] + } + elif client_str=='ec2'and function_name == 'describe_network_acls': + return{ + 'NetworkAclIds': [resource_id] + } + elif client_str=='rds'and function_name == 'describe_event_subscriptions': + return{ + 'SubscriptionName': resource_id + } + elif client_str=='rds'and function_name == 'describe_db_snapshot_attributes': + return{ + 'DBSnapshotIdentifier': resource_id + } + else: + return {} + +def _get_aws_client_data_from_node(node, default_client=None, default_region=None): + """ + Fetches client name and region from ARN, then from the node, + then from the connector. + """ + aws_region = client_str = None + arn_str = get_field_value(node, 'arn') + if arn_str: + arn_obj = arnparse(arn_str) + client_str = arn_obj.service + aws_region = arn_obj.region + if not client_str: + client_str = get_field_value(node, 'client') + if not client_str: + logger.info("No client type provided in snapshot, using client type from connector") + client_str = default_client + if not aws_region: + aws_region = get_field_value(node, 'region') + if not aws_region: + logger.info("No region provided in snapshot, using region from connector") + aws_region = default_region + aws_region = aws_region or default_region + client_str = client_str or default_client + return client_str, aws_region + + +def populate_snapshot_aws(snapshot_json, fssnapshot): + """ + Add steps as per new interface. + """ + snapshot_data, valid_snapshotids = fssnapshot.validate_snapshot_ids_in_nodes(snapshot_json) + snapshot_user = get_field_value(snapshot_json, 'testUser') + account_id = get_field_value(snapshot_json, 'accountId') + sub_data = get_aws_data(snapshot_json, fssnapshot) + + if valid_snapshotids and sub_data and fssnapshot.get_snapshot_nodes(snapshot_json): + logger.debug(sub_data) + access_key, secret_access, region, connector_client_str = \ + get_aws_client_data(sub_data, snapshot_user, account_id) + if not access_key: + logger.info("No access_key in the snapshot to access aws resource!...") + raise Exception("No access_key in the snapshot to access aws resource!...") + # return snapshot_data + + # Read the client secrets from envirnment variable + if not secret_access: + secret_access = os.getenv(snapshot_user, None) + if secret_access: + logger.info('Secret Access key from environment variable, Secret: %s', '*' * len(secret_access)) + + # Read the client secrets from the vault + if not secret_access: + secret_access = get_vault_data(access_key) + if secret_access: + logger.info('Secret Access key from vault Secret: %s', '*' * len(secret_access)) + elif get_from_currentdata(CUSTOMER): + logger.error("Secret Access key does not set in a vault") + raise Exception("Secret Access key does not set in a vault") + + if not secret_access: + raise Exception("No `secret-access` key in the connector file to access aws resource!...") + + if access_key and secret_access: + # existing_aws_client = {} + for node in fssnapshot.get_snapshot_nodes(snapshot_json): + validate = node['validate'] if 'validate' in node else True + mastercode = False + if 'snapshotId' in node and validate: + client_str, aws_region = _get_aws_client_data_from_node(node, + default_client=connector_client_str, default_region=region) + if not _validate_client_name(client_str): + logger.error("Invalid Client Name") + return snapshot_data + try: + awsclient = client(client_str.lower(), aws_access_key_id=access_key, + aws_secret_access_key=secret_access, region_name=aws_region) + except Exception as ex: + logger.info('Unable to create AWS client: %s', ex) + awsclient = None + logger.info(awsclient) + if awsclient: + data = get_node(awsclient, node, get_field_value(snapshot_json, "source")) + if data: + error_str = data.pop('error', None) + fssnapshot.store_data_node(data) + + # if get_dbtests(): + # if get_collection_size(data['collection']) == 0: + # #Creating indexes for collection + # create_indexes(data['collection'], + # config_value(DATABASE, DBNAME), + # [('snapshotId', pymongo.ASCENDING), + # ('timestamp', pymongo.DESCENDING)]) + # + # create_indexes( + # data['collection'], + # config_value(DATABASE, DBNAME), + # [ + # ('_id', pymongo.DESCENDING), + # ('timestamp', pymongo.DESCENDING), + # ('snapshotId', pymongo.ASCENDING) + # ]) + # check_key = is_check_keys_required(data) + # insert_one_document(data, data['collection'], dbname, check_key) + # else: + # snapshot_dir = make_snapshots_dir(fssnapshot.) + # if snapshot_dir: + # store_snapshot(snapshot_dir, data) + if 'masterSnapshotId' in node: + snapshot_data[node['snapshotId']] = node['masterSnapshotId'] + else: + snapshot_data[node['snapshotId']] = False if error_str else True + else: + node['status'] = 'inactive' + elif 'masterSnapshotId' in node: + mastercode = True + client_str, aws_region = _get_aws_client_data_from_node(node, + default_client=connector_client_str, default_region=region) + if not _validate_client_name(client_str): + logger.error("Invalid Client Name") + return snapshot_data + if aws_region: + all_regions = [aws_region] + else: + all_regions = Session().get_available_regions(client_str.lower()) + if client_str.lower() in ['s3','cloudtrail']: + all_regions = ['us-west-1'] + logger.info("Length of all regions is %s"%(str(len(all_regions)))) + count = 0 + snapshot_data[node['masterSnapshotId']] = [] + for each_region in all_regions: + logger.info(each_region) + try: + awsclient = client(client_str.lower(), aws_access_key_id=access_key, + aws_secret_access_key=secret_access, region_name=each_region) + except Exception as ex: + logger.info('Unable to create AWS client: %s', ex) + logger.info(awsclient) + if awsclient: + all_data = get_all_nodes(awsclient, node, snapshot_json, sub_data) + if all_data: + for data in all_data: + snapshot_data[node['masterSnapshotId']].append( + { + 'snapshotId': '%s%s' % (node['masterSnapshotId'], str(count)), + 'validate': validate, + 'detailMethods': data['detailMethods'], + 'structure': 'aws', + 'masterSnapshotId': node['masterSnapshotId'], + 'collection': data['collection'], + 'arn' : data['arn'], + 'status' : 'active' + }) + count += 1 + if mastercode: + snapshot_data = eliminate_duplicate_snapshots(snapshot_data) + return snapshot_data + +def is_check_keys_required(data): + try: + data = json.dumps(data) + return True + except Exception: + return False + +def eliminate_duplicate_snapshots(snapshot_data): + data = {} + is_updated = False + for snapshot_id, value in snapshot_data.items(): + is_updated = False + for count, snapshot in enumerate(value): + for sid, sval in data.items(): + for cnt, val in enumerate(sval): + if sid == snapshot_id: + continue + if snapshot['arn'] == val['arn'] and snapshot['detailMethods'] == val['detailMethods']: + is_updated = True + s_id = snapshot_data[snapshot_id][count]['masterSnapshotId'] + if isinstance(val['masterSnapshotId'], str): + data[sid][cnt]['masterSnapshotId'] = [s_id, val['masterSnapshotId']] + elif isinstance(val['masterSnapshotId'], list): + data[sid][cnt]['masterSnapshotId'].append(s_id) + + if not is_updated: + data.update({snapshot_id:value}) + return data + + +def get_aws_client_data(aws_data, snapshot_user, account_id): + """ + AWS client information as required by the Boto client, viz access_key + access_secret, AWS command type like EC2, S3 etc and region + The access_secret is either read from structure json or env variable or keyvault + """ + accesskey = None + secret_access = None + region = None + client_str = None + if aws_data and snapshot_user: + accounts = get_field_value(aws_data, "accounts") + if accounts: + found = False + for account in accounts: + if account_id == get_field_value(account, "account-id"): + users = get_field_value(account, "users") + if users: + for user in users: + if snapshot_user == get_field_value(user, "name"): + found = True + accesskey = get_field_value(user, 'access-key') + secret_access = get_field_value(user, 'secret-access') + region = get_field_value(user, 'region') + client_str = get_field_value(user, 'client') + if client_str and not _validate_client_name(client_str): + logger.error("Invalid Client Name") + break + if found: + break + + return accesskey, secret_access, region, client_str