-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathMakefile
More file actions
124 lines (93 loc) · 4.71 KB
/
Makefile
File metadata and controls
124 lines (93 loc) · 4.71 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
LEADS_QUERY_ENGINE_CONTAINER_NAME=query_engine
_HOW_MANY_UPLOAD_IN_PARALLEL=3
_VIRT_ENV_NAME=leads_query_engine
TARGET_UCLOUD=
dev_virtualenv_create:
bash -c ". $$(which virtualenvwrapper.sh); mkvirtualenv $(_VIRT_ENV_NAME);"
dev_virtualenv_install_packages:
bash -c ". $$(which virtualenvwrapper.sh) ; \
workon $(_VIRT_ENV_NAME) ; \
pip install -U -r requirements.txt ;" \
echo "Use: workon $(_VIRT_ENV_NAME)" ;
dev_virtualenv_printname:
@echo workon $(_VIRT_ENV_NAME)
deploy_saltstack_generate_config:
python tools/salt/prepare_provider_conf.py
deploy_import_leads_deploy_ssh_key:
nova keypair-add --pub_key ~/.ssh/leads_cluster.pub leads_cluster || nova keypair-list
deploy_create_salt_security_group:
nova secgroup-create global_saltstack "allow nodes managed with salstack to communicate" && nova secgroup-add-rule global_saltstack tcp 4505 4506 0.0.0.0/0
# hadoop - 9000 and 9001 and 50070 (NameNode) and 8088 (resourcemanager)
deploy_create_yarn_security_group:
nova secgroup-create internal_yarn "allow YARN nodes to communicate";\
nova secgroup-add-group-rule internal_yarn internal_yarn tcp 9000 9000 ;\
nova secgroup-add-group-rule internal_yarn internal_yarn tcp 9001 9001 ;\
nova secgroup-add-group-rule internal_yarn internal_yarn tcp 50070 50070 ;\
nova secgroup-add-group-rule internal_yarn internal_yarn tcp 8088 8088 ;
deploy_create_ispn_security_group:
nova secgroup-create internal_ispn "allow ISPN nodes to communicate";\
nova secgroup-add-group-rule internal_ispn internal_ispn tcp 54200 54200 ;\
nova secgroup-add-group-rule internal_ispn internal_ispn tcp 55200 55200 ; \
nova secgroup-add-group-rule internal_ispn internal_ispn tcp 11222 11222 ;
deploy_monitoring_security_group:
nova secgroup-create monitoring_group "allow monitoring (netflix vector) trafic";\
nova secgroup-add-group-rule monitoring_group monitoring_group tcp 44323 44323;
deploy_monitoring_security_group_open:
nova secgroup-add-rule monitoring_group tcp 44323 44323 0.0.0.0/24;
deploy_monitoring_security_group_close:
nova secgroup-delete-rule monitoring_group tcp 44323 44323 0.0.0.0/24 ;
list_ucloud:
sudo salt-cloud -c salt --list-providers
list_images:
sudo salt-cloud -c salt --list-sizes $(TARGET_UCLOUD)
generate_ssh_config:
bash tools/salt/generate_ssh_config.sh
test_vagrant_create_local_node:
vagrant up
test_vagrant_install_openstck_cli:
vagrant ssh -c "mkdir -p ~/tools"; \
vagrant ssh -c "set +x; cd ~/tools; virtualenv openstack_cli; \
source openstack_cli/bin/activate;\
pip install -r /vagrant_tools/openstack_cli/requirements.txt"
upload_zips_to_container:
vagrant ssh -c "source ~/tools/openstack_cli/bin/activate; \
cd zips;\
find * -name '*.zip' -type f | xargs -I {} -P $(_HOW_MANY_UPLOAD_IN_PARALLEL) \
swift \
--os-auth-url=$${OS_AUTH_URL} \
--os-username=$${OS_USERNAME} \
--os-password=$${OS_PASSWORD} \
--os-tenant-name=$${OS_TENANT_NAME} \
upload --skip-identical --changed $(LEADS_QUERY_ENGINE_CONTAINER_NAME) {}"
list_container:
vagrant ssh -c " source ~/tools/openstack_cli/bin/activate; \
swift \
--os-auth-url=$${OS_AUTH_URL} \
--os-username=$${OS_USERNAME} \
--os-password=$${OS_PASSWORD} \
--os-tenant-name=$${OS_TENANT_NAME} \
list $(LEADS_QUERY_ENGINE_CONTAINER_NAME)"
# unicrawl archive
TARGET_SWIFT_OBJECT_SWIFT=/v1/AUTH_73e8d4d1688f4e1f86926d4cb897091f/unicrawl/nutch.tgz
# infinispan archive
TARGET_SWIFT_OBJECT_ISPN=/v1/AUTH_73e8d4d1688f4e1f86926d4cb897091f/infinispan/infinispan-server-7.0.1-SNAPSHOT-NEW.tgz
TARGET_SWIFT_ENDPOINT=https://object-hamm5.cloudandheat.com:8080
# adidas objects
TARGET_SWIFT_OBJECT_PROCESSING_RESOURCE=/v1/AUTH_73e8d4d1688f4e1f86926d4cb897091f/adidas/adidas-processing-plugin-resources.tar.gz
TARGET_SWIFT_OBJECT_DOC_CONCEPT_SEARCH=/v1/AUTH_73e8d4d1688f4e1f86926d4cb897091f/adidas/leads-document-concept-search.jar
# default value 30 days
TARGET_SWIFT_VALIDITY_OF_TEMPURL_SEC=24*60*60*30*10
get_swift_tempurl_unicrawl_archive:
$(MAKE) get_swift_tempurl TARGET_SWIFT_OBJECT=$(TARGET_SWIFT_OBJECT_SWIFT)
get_swift_tempurl:
if test '$(SWIFT_TEMPURL_KEY)' = ""; then echo "SWIFT_TEMPURL_KEY must be set"; exit 1; fi; \
unset OS_TENANT_ID ; \
swift post -m "Temp-URL-Key: $(SWIFT_TEMPURL_KEY)" ; \
swift \
tempurl GET $$(echo '$(TARGET_SWIFT_VALIDITY_OF_TEMPURL_SEC)' | bc) $(TARGET_SWIFT_OBJECT) $(SWIFT_TEMPURL_KEY) | xargs -I {} echo $(TARGET_SWIFT_ENDPOINT){}
get_swift_tempurl_ispn_archive:
$(MAKE) get_swift_tempurl TARGET_SWIFT_OBJECT=$(TARGET_SWIFT_OBJECT_ISPN)
get_swift_tempurl_resources_resources:
$(MAKE) get_swift_tempurl TARGET_SWIFT_OBJECT=$(TARGET_SWIFT_OBJECT_PROCESSING_RESOURCE)
get_swift_tempurl_doc_concept_search:
$(MAKE) get_swift_tempurl TARGET_SWIFT_OBJECT=$(TARGET_SWIFT_OBJECT_DOC_CONCEPT_SEARCH)