Hive-Docker/bootstrap.sh at JavaWithARM+TEZ · simhadri-g/Hive-Docker · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
#!/bin/bash

eg='\033[0;32m'
enc='\033[0m'
echoe () {
	OIFS=${IFS}
	IFS='%'
	echo -e $@
	IFS=${OIFS}
}

gprn() {
	echoe "${eg} >> ${1}${enc}"
}


## Setup ENV variables

## export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-arm64"

export HDFS_NAMENODE_USER="root"
export HDFS_SECONDARYNAMENODE_USER="root"
export HDFS_DATANODE_USER="root"
export YARN_RESOURCEMANAGER_USER="root"
export YARN_NODEMANAGER_USER="root"

export HADOOP_HOME="/hadoop"
export HADOOP_ROOT_LOGGER=DEBUG
export HADOOP_COMMON_LIB_NATIVE_DIR="/hadoop/lib/native"
export TEZ_HOME="/tez"
export HADOOP_CLASSPATH=$HADOOP_HOME/share/hadoop/hdfs/*:$TEZ_HOME/*:$TEZ_HOME/lib/*:$HADOOP_CLASSPATH
export TEZ_CONF_DIR=/hive/conf/

## Add it to bashrc for starting hadoop
## echo 'export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"' >> ~/.bashrc
echo 'export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-arm64"' >> ~/.bashrc
echo 'export HADOOP_HOME="/hadoop"' >> ~/.bashrc
echo 'export TEZ_HOME="/tez"' >> ~/.bashrc
echo 'export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:$TEZ_HOME/*:$TEZ_HOME/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*"' >> ~/.bashrc
echo 'export TEZ_CONF_DIR="/hive/conf/"' >> ~/.bashrc

rm /hadoop
ln -sf /hadoop-3.3.6 /hadoop

ln -sf /apache-hive-4.1.0-SNAPSHOT-bin /hive
ln -sf /apache-tez-0.10.3-bin /tez

cp /conf/core-site.xml /hadoop/etc/hadoop
cp /conf/hdfs-site.xml /hadoop/etc/hadoop
cp /conf/hadoop-env.sh /hadoop/etc/hadoop
# cp /conf/mapred-site.xml /hadoop/etc/hadoop
# cp /conf/yarn-site.xml /hadoop/etc/hadoop
cp /conf/hive-site.xml /hive/conf/

cp /mysql-connector-java-8.0.28.jar /hive/lib

gprn "set up mysql"
sudo service mysql start

# Set root password
mysql -uroot -e "set password = PASSWORD('root');"
mysql -uroot -e "grant all privileges on *.* to 'root'@'localhost' identified by 'root';"
mysql -uroot -e "CREATE USER 'hive'@'localhost' IDENTIFIED BY 'hive';"
mysql -uroot -e "GRANT ALL PRIVILEGES ON *.* TO 'hive'@'localhost';"
mysql -uroot -e "CREATE USER 'hive'@'%' IDENTIFIED BY 'hive';"
mysql -uroot -e "GRANT ALL PRIVILEGES ON *.* TO 'hive'@'%';"
mysql -uroot -e "FLUSH PRIVILEGES;"

service ssh start

gprn "start yarn"
hadoop/sbin/start-yarn.sh &
sleep 5

gprn "Formatting name node"
hadoop/bin/hdfs namenode -format

gprn "Start hdfs"
hadoop/sbin/start-dfs.sh

jps

#mkdir -p /hive/warehouse -dbType mysql  -initSchemaTo 4.0.0-alpha-1


gprn "Set up metastore DB"
hive/bin/schematool -userName hive -passWord 'hive' -dbType mysql  -initSchemaTo 4.1.0

gprn "Start HMS server"
hive/bin/hive --service metastore  -p  10000 &
#To attach a debugger use --debug as follows. The remote debugger can be attached on port 8000.
#hive/bin/hive --service metastore --debug -p  10000 &

gprn "Sleep and wait for HMS to be up and running"
sleep 20

gprn "Start HiveServer2"
# hive/bin/hive --service hiveserver2 --hiveconf hive.server2.thrift.port=10001 --hiveconf hive.execution.engine=mr

#To attach a debugger use --debug as follows. The remote debugger can be attached on port 8000.
#hive/bin/hive --service hiveserver2 --debug --hiveconf hive.server2.thrift.port=10001 --hiveconf hive.execution.engine=mr

hive/bin/hive --service hiveserver2  --debug --hiveconf hive.server2.thrift.port=10001 --hiveconf hive.execution.engine=tez

sleep 20000