1、环境配置(都在cdh_install.gz.tar和cdh.gz.tar中)
#安装JDK
rpm -ivh jdk-8u191-linux-x64.rpm
#安装时间同步
yum install ntp
vi /etc/ntp.conf
#将server 0.centos.pool.ntp.org iburst注释
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
#新增
pool 0.pool.ntp.org iburst
pool 1.pool.ntp.org iburst
pool 2.pool.ntp.org iburst
pool 3.pool.ntp.org iburst
server 0.cn.pool.ntp.org
server 1.cn.pool.ntp.org
server 2.cn.pool.ntp.org
server 3.cn.pool.ntp.org
#重启时间同步服务
systemctl restart ntpd
#修改主机名及hosts
vi /etc/hostname
vi /etc/hosts
#安装CDH依赖包
sh /home/publicuser/init.sh
#init.sh内容如下：
#!/bin/bash
yum install -y psmisc MySQL-python at bc bind-libs bind-utils cups-client cups-libs cyrus-sasl-gssapi cyrus-sasl-plain ed fuse fuse-libs httpd httpd-tools keyutils-libs-devel krb5-devel libcom_err-devel libselinux-devel libsepol-devel libverto-devel 2、 创建cloudera-scm用户
sh /home/publicuser/create_user.sh
#脚本内容如下：
#!/bin/bash
check_user()
{
#判断用户是否存在passwd中
i=`cat /etc/passwd | cut -f1 -d':' | grep -w "$1" -c`
if [ $i -le 0 ]; then
echo "User $1 is not in the passwd"
return 0
else
#显示用户存在
echo "User $1 is in then passwd"
return 1
fi
}
check_user "cloudera-scm"
if [ $? -eq 0 ]
then
useradd --system --home=/opt/cm-5.11.2/run/cloudera-scm-server --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
fi-
3. 解压CDH程序包及配置
tar -zxvf cloudera-manager-centos7-cm5.11.2_x86_64.tar.gz
mv cm-5.11.2/ /opt/
mv cloudera/ /opt
cd /opt/cloudera/
mkdir parcels
cd /opt/cloudera/parcel-repo/
#龙蜥环境下需要将parcel文件名中el7修改为unknown；manifest.json中的el7也需要修改为unknown
cp /home/publicuser/cdh/CDH-5.11.2-1.cdh5.11.2.p0.4-unknown.parcel ./
cp /home/publicuser/cdh/CDH-5.11.2-1.cdh5.11.2.p0.4-unknown.parcel.sha ./
cp /home/publicuser/cdh/manifest.json ./
chown cloudera-scm:cloudera-scm /opt/*
#修改CDH agent连接的指向，将localhost修改为server的IP地址或主机名
vi /opt/cm-5.11.2/etc/cloudera-scm-agent/config.ini
#启动CDH agent
/opt/cm-5.11.2/etc/init.d/cloudera-scm-agent start4. CDH界面添加设备
在列表中选中该设备（主机名）以后默认即可加入到CDH集群中。
5. 导入parcel文件
mv CDH-5.11.2-1.cdh5.11.2.p0.4 /opt/cloudera/parcels/- 检查CDH中parcel文件分发情况
parcel激活成功即可通过CDH部署相关大数据组件。
spark安装:
1. 将现有机器上/opt/cloudera/parcels/SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354/ 压缩并上传到新集群需要的节点
2. 将压缩包上传到需要安装spark的机器上
scp -P 11892 SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354.tar.gz publicuser@39.135.1.252:/home/publicuser/
publicuser@39.135.1.251's password:
%$W%6xVeAtSRWVZ
ssh -p 11892 publicuser@39.135.1.251
tar -zxvf SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354.tar.gz
sudo su
mv SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354 /opt/cloudera/parcels/
3. 在对应机器/opt/cloudera/parcels/下解压
4. 然后在cdh页面"主机"->parcel->下面找spark，看进度
CDH-5.11.2-1.cdh5.11.2.p0.4-el7.parcel
CDH-5.11.2-1.cdh5.11.2.p0.4-unknown.parcel
5. 将原集群上/opt/cloudera/parcel-repo/下
SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354-el7.parcel
SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354-el7.parcel.sha
SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354-el7.parcel.torrent
压缩传递到对应机器上，然后解压到/opt/cloudera/parcel-repo/下改unknown
scp -P 11892 el7_parcels.gz.tar publicuser@39.135.1.254:/home/publicuser/
scp -P 11892 op-window.tar.gz publicuser@39.135.1.254:/home/publicuser/
publicuser@39.135.1.251's password:
%$W%6xVeAtSRWVZ
ssh -p 11892 publicuser@39.135.1.251
tar -zxvf SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354.tar.gz
sudo su
mv SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354 /opt/cloudera/parcels/
cp SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354-el7.parcel SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354-unknown.parcel
cp SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354-el7.parcel.sha SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354-unknown.parcel.sha
cp SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354-el7.parcel.torrent SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354-unknown.parcel.torrent
6. 将SPARK2_ON_YARN-2.2.0.cloudera1.jar放到/opt/cloudera/csd(仅主节点需要操作)
安装kafka：
将安装包放到这个目录下:
/opt/cloudera/parcels/
