261 lines
8.5 KiB
Markdown
261 lines
8.5 KiB
Markdown
---
|
||
title: "CentOS7 安装 Cloudera Manager"
|
||
date: 2020-08-16T13:48:37+08:00
|
||
lastmod: 2020-08-16T18:27:00+08:00
|
||
keywords: []
|
||
tags: [cloudera cdh]
|
||
categories: ["hadoop"]
|
||
---
|
||
|
||
# 环境
|
||
|
||
角色 | IP | 主机名 | 服务
|
||
---- | ---- | ---- | ----
|
||
Utility | 192.168.1.100 | cm0.colben.cn | ClouderaManager<br>ClouderaManagerManagementService<br>HiveMetastore
|
||
Gateway | 192.168.1.101 | gw0.colben.cn | GatewayConfiguration<br>HiveServer2<br>Zookeeper
|
||
Master | 192.168.1.102 | m0.colben.cn | NameNode<br>JournalNode<br>FailoverController<br>YarnResourceManager<br>Zookeeper
|
||
Master | 192.168.1.103 | m1.colben.cn | NameNode<br>JournalNode<br>FailoverController<br>YarnResourceManager<br>Zookeeper
|
||
Worker | 192.168.1.104 | w0.colben.cn | DataNode<br>NodeManager
|
||
Worker | 192.168.1.105 | w0.colben.cn | DataNode<br>NodeManager
|
||
Worker | 192.168.1.106 | w0.colben.cn | DataNode<br>NodeManager
|
||
|
||
# 配置 ssh 免密登陆
|
||
|
||
- 在 cm0 上配置 ssh 可免密登陆全部服务器
|
||
```
|
||
ssh-keygen
|
||
seq -f'192.168.1.%g' 100 106 | xargs -L1 ssh-copy-id
|
||
```
|
||
|
||
- 这里的**私钥会在后面通过界面增加主机时用到**
|
||
|
||
# 关闭防火墙和 selinux
|
||
|
||
- 在全部服务器上关闭防火墙
|
||
```
|
||
systemctl stop firewalld
|
||
systemctl disable firewalld
|
||
```
|
||
|
||
- 在全部服务器上关闭 selinux
|
||
```
|
||
setenforce 0
|
||
sed -i '/^SELINUX=/cSELINUX=disabled' /etc/selinux/config
|
||
```
|
||
|
||
# 配置网络名称
|
||
|
||
- 在全部服务器上修改 /etc/hosts，增加如下解析记录
|
||
```
|
||
192.168.1.100 cm0.colben.cn cm0
|
||
192.168.1.101 gw0.colben.cn gw0
|
||
192.168.1.102 m0.colben.cn m0
|
||
192.168.1.103 m1.colben.cn m1
|
||
192.168.1.104 w0.colben.cn w0
|
||
192.168.1.105 w1.colben.cn w1
|
||
192.168.1.106 w2.colben.cn w2
|
||
```
|
||
|
||
- 在全部服务器上修改 /etc/sysconfig/network，增加各自主机名设置
|
||
```
|
||
HOSTNAME=XXXX.colben.cn
|
||
```
|
||
|
||
- 在全部服务器上设置各自主机名
|
||
```
|
||
hostnamectl set-hostname XXXX.colben.cn
|
||
```
|
||
|
||
# 配置时间同步
|
||
|
||
- 在 cm0 上配置修改 /etc/chrony.conf
|
||
```
|
||
server ntp.aliyun.com iburst
|
||
allow 192.168.1.0/24
|
||
```
|
||
|
||
- 在其他服务器上修改 /etc/chrony.conf
|
||
```
|
||
server cm0.colben.cn iburst
|
||
```
|
||
|
||
- 在全部服务器上重启 chronyd 服务
|
||
```
|
||
systemctl restart chronyd
|
||
```
|
||
|
||
# 配置 cloudera manager 内网安装源
|
||
|
||
- 在 cm0 上安装并启动 httpd 服务
|
||
```
|
||
yum install httpd
|
||
systemctl enable httpd
|
||
systemctl start httpd
|
||
```
|
||
|
||
- 在 cm0 上下载 cloudera manager yum 仓库
|
||
```
|
||
mkdir -p /var/www/html/cloudera-repos/cm6/6.3.1
|
||
wget https://archive.cloudera.com/cm6/6.3.1/repo-as-tarball/cm6.3.1-redhat7.tar.gz
|
||
tar zxf cm6.3.1-redhat7.tar.gz -C /var/www/html/cloudera-repos/cm6/6.3.1 --strip-components=1
|
||
chmod -R ugo+rX /var/www/html/cloudera-repos/cm6
|
||
```
|
||
|
||
- 在 cm0 上下载 cdh yum 仓库(**体积较小，需手动升级**)
|
||
```
|
||
mkdir -p /var/www/html/cloudera-repos
|
||
wget --recursive --no-parent --no-host-directories \
|
||
https://archive.cloudera.com/cdh6/6.3.2/redhat7/ -P /var/www/html/cloudera-repos
|
||
wget --recursive --no-parent --no-host-directories \
|
||
https://archive.cloudera.com/gplextras6/6.3.2/redhat7/ -P /var/www/html/cloudera-repos
|
||
chmod -R ugo+rX /var/www/html/cloudera-repos/cdh6
|
||
chmod -R ugo+rX /var/www/html/cloudera-repos/gplextras6
|
||
```
|
||
|
||
- 在 cm0 上下载 cdh parcel 仓库(**体积较大，可通过界面操作自动升级**)
|
||
```
|
||
mkdir -p /var/www/html/cloudera-repos
|
||
wget --recursive --no-parent --no-host-directories \
|
||
https://archive.cloudera.com/cdh6/6.3.2/parcels/ -P /var/www/html/cloudera-repos
|
||
wget --recursive --no-parent --no-host-directories \
|
||
https://archive.cloudera.com/gplextras6/6.3.2/parcels/ -P /var/www/html/cloudera-repos
|
||
chmod -R ugo+rX /var/www/html/cloudera-repos/cdh6
|
||
chmod -R ugo+rX /var/www/html/cloudera-repos/gplextras6
|
||
```
|
||
|
||
- 在 cm0 上创建 cloudera.repo，并重建 yum 缓存
|
||
```
|
||
cat > /etc/yum.repos.d/cloudera.repo <<-EOF
|
||
[cloudera-manager]
|
||
name=cloudera-manager
|
||
baseurl=http://cm0.colben.cn/cloudera-repos/cm6/
|
||
enabled=1
|
||
gpgcheck=0
|
||
EOF
|
||
yum clean all
|
||
yum makecache fast
|
||
```
|
||
|
||
- 通过界面增加其他主机时，cloudera manager 会自动部署该 repo 文件
|
||
|
||
# 安装 jdk
|
||
|
||
- 在 cm0 上安装 jdk
|
||
```
|
||
yum install oracle-j2sdk1.8
|
||
```
|
||
|
||
- 通过界面增加其他主机时，可自动安装 oracle-j2sdk1.8
|
||
|
||
# 安装 cloudera manager
|
||
|
||
- 在 cm0 上安装 cloudera-manager
|
||
```
|
||
yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server
|
||
```
|
||
|
||
# 配置 cloudera manager 数据库
|
||
|
||
- 在 cm0 上安装 mysql，[参考这里](/post/mysql5.7-install/)
|
||
- 官方推荐 my.cnf
|
||
```
|
||
[mysqld]
|
||
transaction-isolation = READ-COMMITTED
|
||
symbolic-links = 0
|
||
sql_mode=STRICT_ALL_TABLES
|
||
|
||
key_buffer_size = 32M
|
||
max_allowed_packet = 32M
|
||
thread_stack = 256K
|
||
thread_cache_size = 64
|
||
query_cache_limit = 8M
|
||
query_cache_size = 64M
|
||
query_cache_type = 1
|
||
max_connections = 600
|
||
|
||
log_bin=/var/lib/mysql/mysql_binary_log
|
||
server_id=1
|
||
binlog_format = mixed
|
||
|
||
read_buffer_size = 2M
|
||
read_rnd_buffer_size = 16M
|
||
sort_buffer_size = 8M
|
||
join_buffer_size = 8M
|
||
|
||
innodb_file_per_table = 1
|
||
innodb_flush_log_at_trx_commit = 2
|
||
innodb_log_buffer_size = 64M
|
||
innodb_buffer_pool_size = 4G
|
||
innodb_thread_concurrency = 8
|
||
innodb_flush_method = O_DIRECT
|
||
innodb_log_file_size = 512M
|
||
```
|
||
|
||
- 在 cm0 上启动 mysql
|
||
```
|
||
systemctl enable mysqld
|
||
systemctl start mysqld
|
||
```
|
||
|
||
- 在 cm0 上的 mysql 中创建数据库和用户
|
||
```
|
||
CREATE DATABASE scm DEFAULT CHARSET utf8 DEFAULT COLLATE utf8_general_ci;
|
||
CREATE DATABASE amon DEFAULT CHARSET utf8 DEFAULT COLLATE utf8_general_ci;
|
||
CREATE DATABASE rman DEFAULT CHARSET utf8 DEFAULT COLLATE utf8_general_ci;
|
||
CREATE DATABASE hue DEFAULT CHARSET utf8 DEFAULT COLLATE utf8_general_ci;
|
||
CREATE DATABASE metastore DEFAULT CHARSET utf8 DEFAULT COLLATE utf8_general_ci;
|
||
CREATE DATABASE sentry DEFAULT CHARSET utf8 DEFAULT COLLATE utf8_general_ci;
|
||
CREATE DATABASE nav DEFAULT CHARSET utf8 DEFAULT COLLATE utf8_general_ci;
|
||
CREATE DATABASE navms DEFAULT CHARSET utf8 DEFAULT COLLATE utf8_general_ci;
|
||
CREATE DATABASE oozie DEFAULT CHARSET utf8 DEFAULT COLLATE utf8_general_ci;
|
||
GRANT ALL ON scm.* to scm@'%' identified by 'Pass-1234';
|
||
GRANT ALL ON amon.* to amon@'%' identified by 'Pass-1234';
|
||
GRANT ALL ON rman.* to rman@'%' identified by 'Pass-1234';
|
||
GRANT ALL ON hue.* to hue@'%' identified by 'Pass-1234';
|
||
GRANT ALL ON metastore.* to hive@'%' identified by 'Pass-1234';
|
||
GRANT ALL ON sentry.* to sentry@'%' identified by 'Pass-1234';
|
||
GRANT ALL ON nav.* to nav@'%' identified by 'Pass-1234';
|
||
GRANT ALL ON navms.* to navms@'%' identified by 'Pass-1234';
|
||
GRANT ALL ON oozie.* to oozie@'%' identified by 'Pass-1234';
|
||
```
|
||
|
||
- 在 cm0 上初始化数据库
|
||
```
|
||
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm
|
||
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql amon amon
|
||
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql rman rman
|
||
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql hue hue
|
||
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql metastore hive
|
||
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql sentry sentry
|
||
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql nav nav
|
||
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql navms navms
|
||
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql oozie oozie
|
||
```
|
||
|
||
- 在全部服务器上安装 mysql connector
|
||
```
|
||
wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz
|
||
tar zxvf mysql-connector-java-5.1.46.tar.gz
|
||
mkdir -p /usr/share/java/
|
||
cp mysql-connector-java-5.1.46/mysql-connector-java-5.1.46-bin.jar \
|
||
/usr/share/java/mysql-connector-java.jar
|
||
```
|
||
|
||
# 启动 cloudera manager
|
||
- 在 cm0 上启动 cloudera manager
|
||
```
|
||
systemctl start cloudera-scm-server
|
||
```
|
||
|
||
- 日志: /var/log/cloudera-scm-server/cloudera-scm-server.log
|
||
- 日志出现如下信息，表示 cloudera-scm-server 启动完成
|
||
```
|
||
INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server.
|
||
```
|
||
|
||
- 浏览器访问
|
||
- 地址: http://cm0.colben.cn:7180/
|
||
- 用户: admin
|
||
- 密码: admin
|
||
|