部署Doris3.1.4+DolphinScheduler 3.4.1 | Evergreen Notes返回首页部署Doris3.1.4+DolphinScheduler 3.4.1
一.部署Doris3.1.4
1.1.配置jdk17
略
1.2.更改系统配置
# 1. 关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
# 2. 关闭selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
setenforce 0
# 3. 加大文件句柄(必须)
echo "* soft nofile 2048000" >> /etc/security/limits.conf
echo "* hard nofile 2048000" >> /etc/security/limits.conf
# 4. 加大虚拟内存(必须)
echo "vm.max_map_count=2048000" >> /etc/sysctl.conf
sysctl -p
#5.关闭系统 swap
vi /etc/fstab
#/dev/mapper/rl-swap none swap defaults 0 0
# 关闭 swap,上面改配置是为了永久生效,临时关闭敲只敲这行命令也是可以的
swapoff -a
# 查看内存情况
free -h
1.2.解压启动
1.3.服务注册
最后把 Doris 服务注册到系统服务中,使用 systemctl 命令来管理
Doris 前端服务
二.部署DolphinScheduler 3.4.1
1.准备数据目录
2.准备环境变量
3.docker-compsoe配置文件
4.初始化数据库启动
docker-compose --profile schema up -d
docker-compose --profile all up -d
#6.调高 ulimit
vi /etc/security/limits.conf
文件末尾添加下面这四行配置
* soft nofile 655350
* hard nofile 655350
* soft nproc 655350
* hard nproc 655350
敲 ulimit -n 出现配置后的数量,表示配置完成
tar -xvf apache-doris-3.4.1-bin-x64-noavx2.tar.gz
cd到解压后的目录
sh fe/bin/start_fe.sh --daemon
sh be/bin/start_be.sh --daemon
[Unit]
Description=Doris FE Service
After=network.target
[Service]
Type=forking
User=root
Group=root
Environment="JAVA_HOME=/usr/local/dev/jdk-17.0.17"
ExecStart=/usr/local/dev/apache-doris-3.1.4/fe/bin/start_fe.sh --daemon
ExecStop=/usr/local/dev/apache-doris-3.1.4/fe/bin/stop_fe.sh
Restart=on-failure
LimitNOFILE=655350
LimitNPROC=655350
[Install]
WantedBy=multi-user.target
Doris 后端服务
vim /etc/systemd/system/doris-be.service
[Unit]
Description=Doris BE Service
After=network.target doris-fe.service
[Service]
Type=forking
User=root
Group=root
Environment="JAVA_HOME=/usr/local/dev/jdk-17.0.17"
ExecStart=/usr/local/dev/apache-doris-3.1.4/be/bin/start_be.sh --daemon
ExecStop=/usr/local/dev/apache-doris-3.1.4/be/bin/stop_be.sh
Restart=on-failure
LimitNOFILE=655350
LimitNPROC=655350
[Install]
WantedBy=multi-user.target
刷新服务
systemctl daemon-reload
systemctl start doris-be
systemctl start doris-fe
mkdir -p /data/deploy/dolphinscheduler/postgres_data
mkdir -p /data/deploy/dolphinscheduler/zookeeper_data
mkdir -p /data/deploy/dolphinscheduler/logs
chown -R 999:999 /data/deploy/dolphinscheduler/postgres_data
chmod 700 /data/deploy/dolphinscheduler/postgres_data
chown -R 1000:1000 /data/deploy/dolphinscheduler/zookeeper_data
chmod -R 700 /data/deploy/dolphinscheduler/zookeeper_data
[root@localhost docker]# cat .env
HUB=apache
TAG=3.4.1
TZ=Asia/Shanghai
DATABASE=postgresql
SPRING_JACKSON_TIME_ZONE=UTC
SPRING_DATASOURCE_URL=jdbc:postgresql://dolphinscheduler-postgresql:5432/dolphinscheduler
SPRING_DATASOURCE_USERNAME=root
SPRING_DATASOURCE_PASSWORD=q.EN7RTaZ1
REGISTRY_ZOOKEEPER_CONNECT_STRING=dolphinscheduler-zookeeper:2181
[root@localhost docker]# cat docker-compose.yml
services:
dolphinscheduler-postgresql:
#image: bitnami/postgresql:15.2.0
image: postgres:15
ports:
- "5432:5432"
profiles: ["all", "schema"]
environment:
POSTGRES_USER: root
POSTGRES_PASSWORD: q.EN7RTaZ1
POSTGRES_DB: dolphinscheduler
volumes:
#- dolphinscheduler-postgresql:/bitnami/postgresql
- /data/deploy/dolphinscheduler/postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432"
]
interval: 5s
timeout: 60s
retries: 120
networks:
- dolphinscheduler
dolphinscheduler-zookeeper:
#image: bitnami/zookeeper:3.7.1
image: zookeeper:3.7.1
profiles: ["all"]
environment:
ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes:
#- dolphinscheduler-zookeeper:/bitnami/zookeeper
- /data/deploy/dolphinscheduler/zookeeper_data:/data
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"]
interval: 5s
timeout: 60s
retries: 120
networks:
- dolphinscheduler
dolphinscheduler-schema-initializer:
image: ${HUB}/dolphinscheduler-tools:${TAG}
env_file: .env
profiles: ["schema"]
command: [ tools/bin/upgrade-schema.sh ]
depends_on:
dolphinscheduler-postgresql:
condition: service_healthy
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
networks:
- dolphinscheduler
dolphinscheduler-api:
image: ${HUB}/dolphinscheduler-api:${TAG}
ports:
- "12345:12345"
- "25333:25333"
profiles: ["all"]
env_file: .env
healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
depends_on:
dolphinscheduler-zookeeper:
condition: service_healthy
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
networks:
- dolphinscheduler
dolphinscheduler-alert:
image: ${HUB}/dolphinscheduler-alert-server:${TAG}
profiles: ["all"]
env_file: .env
healthcheck:
test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
depends_on:
dolphinscheduler-zookeeper:
condition: service_healthy
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
dolphinscheduler-master:
image: ${HUB}/dolphinscheduler-master:${TAG}
profiles: ["all"]
env_file: .env
healthcheck:
test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
depends_on:
dolphinscheduler-zookeeper:
condition: service_healthy
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
networks:
- dolphinscheduler
dolphinscheduler-worker:
image: ${HUB}/dolphinscheduler-worker:${TAG}
profiles: ["all"]
env_file: .env
healthcheck:
test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
depends_on:
dolphinscheduler-zookeeper:
condition: service_healthy
volumes:
- dolphinscheduler-worker-data:/tmp/dolphinscheduler
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-shared-local:/opt/soft
- dolphinscheduler-resource-local:/dolphinscheduler
networks:
- dolphinscheduler
networks:
dolphinscheduler:
driver: bridge
volumes:
dolphinscheduler-postgresql:
dolphinscheduler-zookeeper:
dolphinscheduler-worker-data:
dolphinscheduler-logs:
dolphinscheduler-shared-local:
dolphinscheduler-resource-local: