Hive-4.0.0 Docker 01
Prerequisites
- Docker
Launch Hive
export HIVE_VERSION=4.0.0
export POSTGRES_LOCAL_PATH=your_local_path_to_postgres_driver
-
docker-compose.yml
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: '3.9'
services:
postgres:
image: postgres
restart: unless-stopped
container_name: postgres
hostname: postgres
environment:
POSTGRES_DB: 'metastore_db'
POSTGRES_USER: 'hive'
POSTGRES_PASSWORD: 'password'
ports:
- '5432:5432'
volumes:
- hive-db:/var/lib/postgresql
networks:
- hive
metastore:
image: apache/hive:${HIVE_VERSION}
depends_on:
- postgres
restart: unless-stopped
container_name: metastore
hostname: metastore
environment:
DB_DRIVER: postgres
SERVICE_NAME: 'metastore'
SERVICE_OPTS: '-Xmx1G -Djavax.jdo.option.ConnectionDriverName=org.postgresql.Driver
-Djavax.jdo.option.ConnectionURL=jdbc:postgresql://postgres:5432/metastore_db
-Djavax.jdo.option.ConnectionUserName=hive
-Djavax.jdo.option.ConnectionPassword=password'
ports:
- '9083:9083'
volumes:
- warehouse:/opt/hive/data/warehouse
- type: bind
source: ${POSTGRES_LOCAL_PATH}
target: /opt/hive/lib/postgres.jar
networks:
- hive
hiveserver2:
image: apache/hive:${HIVE_VERSION}
depends_on:
- metastore
restart: unless-stopped
container_name: hiveserver2
environment:
HIVE_SERVER2_THRIFT_PORT: 10000
SERVICE_OPTS: '-Xmx1G -Dhive.metastore.uris=thrift://metastore:9083'
IS_RESUME: 'true'
SERVICE_NAME: 'hiveserver2'
ports:
- '10000:10000'
- '10002:10002'
volumes:
- warehouse:/opt/hive/data/warehouse
networks:
- hive
volumes:
hive-db:
warehouse:
networks:
hive:
name: hive
-
docker compose up -d
docker ps -a
Output:
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
088e0373719e apache/hive:4.0.0 "sh -c /entrypoint.sh" 20 minutes ago Up 3 minutes 0.0.0.0:10000->10000/tcp, 9083/tcp, 0.0.0.0:10002->10002/tcp hiveserver2
0cf91a83e260 apache/hive:4.0.0 "sh -c /entrypoint.sh" 20 minutes ago Up 3 minutes 10000/tcp, 0.0.0.0:9083->9083/tcp, 10002/tcp metastore
beff35c5e819 postgres "docker-entrypoint.s…" 20 minutes ago Up 20 minutes 0.0.0.0:5432->5432/tcp postgres
Beeline:
docker exec -it hiveserver2 beeline -u 'jdbc:hive2://hiveserver2:10000/'
docker exec -it hiveserver2 beeline -u 'jdbc:hive2://hiveserver2:10000/'
# If beeline is installed on host machine, HiveServer2 can be simply reached via:
# beeline -u 'jdbc:hive2://localhost:10000/'
...
Connecting to jdbc:hive2://hiveserver2:10000/
Connected to: Apache Hive (version 4.0.0)
Driver: Hive JDBC (version 4.0.0)
Transaction isolation: TRANSACTION_REPEATABLE_READ
Beeline version 4.0.0 by Apache Hive
0: jdbc:hive2://hiveserver2:10000/>
show databases;
0: jdbc:hive2://hiveserver2:10000/> show databases;
INFO : Compiling command(queryId=hive_20250430080937_22a191f2-e4c7-485c-b964-19e0628cc576): show databases
INFO : Semantic Analysis Completed (retrial = false)
INFO : Created Hive schema: Schema(fieldSchemas:[FieldSchema(name:database_name, type:string, comment:from deserializer)], properties:null)
INFO : Completed compiling command(queryId=hive_20250430080937_22a191f2-e4c7-485c-b964-19e0628cc576); Time taken: 0.824 seconds
INFO : Concurrency mode is disabled, not creating a lock manager
INFO : Executing command(queryId=hive_20250430080937_22a191f2-e4c7-485c-b964-19e0628cc576): show databases
INFO : Starting task [Stage-0:DDL] in serial mode
INFO : Completed executing command(queryId=hive_20250430080937_22a191f2-e4c7-485c-b964-19e0628cc576); Time taken: 0.05 seconds
+----------------+
| database_name |
+----------------+
| default |
+----------------+
1 row selected (1.32 seconds)
0: jdbc:hive2://hiveserver2:10000/>
Run some queries:
create table hive_example(a string, b int) partitioned by(c int);
alter table hive_example add partition(c=1);
insert into hive_example partition(c=1) values('a', 1), ('a', 2),('b',3);
select count(distinct a) from hive_example;
select sum(b) from hive_example;
0: jdbc:hive2://hiveserver2:10000/> show tables;
INFO : Compiling command(queryId=hive_20250429090109_b00b0c5a-d603-436c-bda5-52b48c397c85): show tables
INFO : Semantic Analysis Completed (retrial = false)
INFO : Created Hive schema: Schema(fieldSchemas:[FieldSchema(name:tab_name, type:string, comment:from deserializer)], properties:null)
INFO : Completed compiling command(queryId=hive_20250429090109_b00b0c5a-d603-436c-bda5-52b48c397c85); Time taken: 0.011 seconds
INFO : Concurrency mode is disabled, not creating a lock manager
INFO : Executing command(queryId=hive_20250429090109_b00b0c5a-d603-436c-bda5-52b48c397c85): show tables
INFO : Starting task [Stage-0:DDL] in serial mode
INFO : Completed executing command(queryId=hive_20250429090109_b00b0c5a-d603-436c-bda5-52b48c397c85); Time taken: 0.033 seconds
+---------------+
| tab_name |
+---------------+
| hive_example |
+---------------+
1 row selected (0.092 seconds)
HiveServer2
Accessed on browser at http://localhost:10002/
Postgres Metastore
docker exec -it postgres psql -h localhost -U hive -d metastore_db
Output:
psql (17.4 (Debian 17.4-1.pgdg120+2))
Type "help" for help.
metastore_db=#
metastore_db-# \l
List of databases
Name | Owner | Encoding | Locale Provider | Collate | Ctype | Locale | ICU Rules | Access privileges
--------------+-------+----------+-----------------+------------+------------+--------+-----------+-------------------
metastore_db | hive | UTF8 | libc | en_US.utf8 | en_US.utf8 | | |
postgres | hive | UTF8 | libc | en_US.utf8 | en_US.utf8 | | |
template0 | hive | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/hive +
| | | | | | | | hive=CTc/hive
template1 | hive | UTF8 | libc | en_US.utf8 | en_US.utf8 | | | =c/hive +
| | | | | | | | hive=CTc/hive
(4 rows)
metastore_db=# \c metastore_db;
metastore_db=# \dt
Output:
List of relations
Schema | Name | Type | Owner
--------+-------------------------------+-------+-------
public | AUX_TABLE | table | hive
public | BUCKETING_COLS | table | hive
public | CDS | table | hive
public | COLUMNS_V2 | table | hive
public | COMPACTION_METRICS_CACHE | table | hive
public | COMPACTION_QUEUE | table | hive
public | COMPLETED_COMPACTIONS | table | hive
public | COMPLETED_TXN_COMPONENTS | table | hive
public | CTLGS | table | hive
public | DATABASE_PARAMS | table | hive
public | DATACONNECTORS | table | hive
public | DATACONNECTOR_PARAMS | table | hive
public | DBS | table | hive
public | DB_PRIVS | table | hive
public | DC_PRIVS | table | hive
public | DELEGATION_TOKENS | table | hive
public | FUNCS | table | hive
public | FUNC_RU | table | hive
public | GLOBAL_PRIVS | table | hive
public | HIVE_LOCKS | table | hive
:
Hive Hadoop
docker exec -it hiveserver2 bash
Output:
hive@4d687e8196e1:/opt/hive$
hive@4d687e8196e1:/opt/hive$ cd ..
Output:
hive@4d687e8196e1:/opt$
hadoop hive tez