You've already forked www.colben.cn
174 lines
4.9 KiB
Markdown
174 lines
4.9 KiB
Markdown
---
|
||
title: "hive2 部署"
|
||
date: 2023-05-23T11:00:00+08:00
|
||
lastmod: 2023-05-23T11:00:00+08:00
|
||
keywords: []
|
||
tags: ["hadoop", "hive"]
|
||
categories: ["storage"]
|
||
---
|
||
|
||
## 环境
|
||
操作系统 | 主机名 | 地址 | 运行组件
|
||
---- | ---- | ---- | ----
|
||
Rocky9 | hive-hs20 | 192.168.8.20/24 | Hive Server
|
||
Rocky9 | hive-ms21 | 192.168.8.21/24 | Hive Metastore, MySQL8.0
|
||
|
||
## 服务器初始配置
|
||
- 在**全部主机**上执行如下操作
|
||
- 禁用防火墙
|
||
- 禁用 selinux
|
||
- 配置时间同步
|
||
- 配置主机名解析
|
||
```bash
|
||
echo "192.168.8.1 hdp-nn" >> /etc/hosts
|
||
echo "192.168.8.2 hdp-snn" >> /etc/hosts
|
||
echo "192.168.8.3 hdp-dn" >> /etc/hosts
|
||
echo "192.168.8.10 hdp-slave10" >> /etc/hosts
|
||
echo "192.168.8.11 hdp-slave11" >> /etc/hosts
|
||
echo "192.168.8.20 hdp-hs20" >> /etc/hosts
|
||
echo "192.168.8.21 hdp-ms21" >> /etc/hosts
|
||
```
|
||
|
||
## 创建 mysql 数据库
|
||
- 在 **hive-ms21** 上执行如下操作
|
||
- 部署 mysql8.0，略过
|
||
- 创建用户及其数据库，参考 sql 如下
|
||
```sql
|
||
create user hive@127.7.7.7 identified by 'Hive_1234';
|
||
create database hive default charset utf8mb4;
|
||
grant all on hive.* to hive@127.7.7.7;
|
||
```
|
||
|
||
## 复制 jdk 和 hadoop 环境
|
||
- 在**全部主机**上执行如下操作
|
||
- 从 hdp-nn 复制 jdk 和 hadoop 环境
|
||
```bash
|
||
scp -r hdp-nn:/opt/{jdk,hdp} /opt/
|
||
scp hdp-nn:/etc/profile.d/{jdk,hdp}.sh /etc/profile.d/
|
||
source /etc/profile.d/jdk.sh
|
||
source /etc/profile.d/hdp.sh
|
||
```
|
||
|
||
## 部署 hive 环境
|
||
- 在**全部主机**上执行如下操作
|
||
- 下载 hive 2.3.9 部署包，解压
|
||
```bash
|
||
curl -LO https://archive.apache.org/dist/hive/hive-2.3.9/apache-hive-2.3.9-bin.tar.gz
|
||
tar zxf apache-hive-2.3.9.tar.gz
|
||
mv apache-hive-2.3.9 /opt/hive
|
||
```
|
||
|
||
- 下载 mysql 连接库，解压到 hive 库目录下
|
||
```bash
|
||
curl -LO https://downloads.mysql.com/archives/get/p/3/file/mysql-connector-j-8.0.33.tar.gz
|
||
tar zxf mysql-connector-j-8.0.33.tar.gz mysql-connector-j-8.0.33/mysql-connector-j-8.0.33.jar
|
||
mv mysql-connector-j-8.0.33/mysql-connector-j-8.0.33.jar /opt/hive/lib/
|
||
rm -rf mysql-connector-j-8.0.33*
|
||
```
|
||
|
||
- 配置环境变量
|
||
```bash
|
||
echo 'export HIVE_HOME=/opt/hive' > /etc/profile.d/hive.sh
|
||
echo 'export PATH=$HIVE_HOME/bin:$PATH' >> /etc/profile.d/hive.sh
|
||
source /etc/profile.d/hive.sh
|
||
```
|
||
|
||
### 修改 hive-env.sh
|
||
- 编辑 $HIVE_HOME/conf/hive-env.sh，指定 HADOOP_HOME 环境变量
|
||
```bash
|
||
export HADOOP_HOME=/opt/hdp
|
||
```
|
||
|
||
### 创建 hive-site.xml
|
||
- 创建 $HIVE_HOME/conf/hive-site.xml，参考内容如下
|
||
```xml
|
||
<configuration>
|
||
<property>
|
||
<!-- mysql 地址 -->
|
||
<name>javax.jdo.option.ConnectionURL</name>
|
||
<value>jdbc:mysql://127.7.7.7:3306/hive?createDatabaseIfNotExist=true&useSSL=false</value>
|
||
</property>
|
||
<property>
|
||
<!-- mysql 驱动 -->
|
||
<name>javax.jdo.option.ConnectionDriverName</name>
|
||
<value>com.mysql.cj.jdbc.Driver</value>
|
||
</property>
|
||
<property>
|
||
<!-- mysql 用户 -->
|
||
<name>javax.jdo.option.ConnectionUserName</name>
|
||
<value>hive</value>
|
||
</property>
|
||
<property>
|
||
<!-- mysql 密码 -->
|
||
<name>javax.jdo.option.ConnectionPassword</name>
|
||
<value>Hive_1234</value>
|
||
</property>
|
||
<property>
|
||
<!-- 自动初始化 hive 库 -->
|
||
<name>datanucleus.schema.autoCreateAll</name>
|
||
<value>true</value>
|
||
</property>
|
||
<property>
|
||
<name>hive.cli.print.header</name>
|
||
<value>true</value>
|
||
</property>
|
||
<property>
|
||
<name>hive.cli.print.current.db</name>
|
||
<value>true</value>
|
||
</property>
|
||
<property>
|
||
<!-- hive server 端口 -->
|
||
<name>hive.server2.webui.port</name>
|
||
<value>10002</value>
|
||
</property>
|
||
<property>
|
||
<!-- 数据存储位置(hdfs) -->
|
||
<name>hive.metastore.warehouse.dir</name>
|
||
<value>/hive/warehouse</value>
|
||
</property>
|
||
<property>
|
||
<!-- hive metastore 端口-->
|
||
<name>hive.metastore.uris</name>
|
||
<value>thrift://hive-ms:9083</value>
|
||
</property>
|
||
<!-- hive 使用 tez 引擎
|
||
<property>
|
||
<name>hive.execution.engine</name>
|
||
<value>tez</value>
|
||
</property> -->
|
||
<property>
|
||
<name>tez.am.staging-dir</name>
|
||
<value>/hive/tez/staging</value>
|
||
</property>
|
||
</configuration>
|
||
```
|
||
|
||
## 初始化 hive 库
|
||
- 在 **hive-ms21** 上执行如下操作
|
||
```bash
|
||
schematool -dbType mysql -initSchema
|
||
```
|
||
|
||
## 启动 hive
|
||
- 在 **hive-ms21** 上启动 hive metastore
|
||
```bash
|
||
hive --service metastore
|
||
```
|
||
|
||
- 在 **hive-hs20** 上启动 hive server
|
||
```bash
|
||
hive --service hiveserver2
|
||
```
|
||
|
||
## 客户端连接
|
||
- 本地直接连接
|
||
```bash
|
||
hive
|
||
```
|
||
|
||
- beeline 连接，需要[在 $HADOOP_HOME/etc/hadoop/core-site.xml 中配置 proxyuser](/post/hdp2/#修改-core-sitexml)
|
||
```bash
|
||
beeline -u jdbc:hive2://hive-hs20:10000 -n root
|
||
```
|
||
|