Flink YARN 运行模式 -hadoop集群准备

戒掉贪嗔痴

226人浏览 · 2026-06-29 21:38:08

戒掉贪嗔痴 · 2026-06-29 21:38:08 发布

1.YARN 运行模式

YARN 上部署的过程是:客户端把Flink 应用提交给 yarn 的 ResourceManager ,Yarn 的 ResourceManager
会向 Yarn 的 NodeManager 申请容器。在这些容器上，Flink 会部署 JobManager 和 TaskManager 的实例，
从而启动集群。Flink 会根据运行在 JobManager 上作业所需要的 Slot 数量动态分配 TaskManager 资源。

在将 FLINK 任务部署到 YARN 集群之前，需要确认集群是否安装有 Hadoop,保证Hadoop 版本至少在2.2以上，
并且集群中安装有 HDFS 服务。

#配置环境变量，增加环境变量配置如下:
vim /etc/profile.d/my_env.sh 
HADOOP_HOME=/opt/module/hadoop-3.3.6 
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin 

#Flink需要
export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop 
export HADOOP_CLASSPATH=`hadoop classpath`

source /etc/profile.d/my_env.sh 

#hadoop环境准备  

#java11 的参数和java8有所不同。

vim zookeeper-env.sh 
#!/bin/bash
JAVA_HOME=/opt/module/jdk-11.0.30
ZOO_LOG_DIR=/opt/module/zookeeper-3.6.4/logs
ZOO_LOG4_PROP="WARN,ROLLINGFILE"
JVMFLAGS="-server -Xms1G -Xmx1G -Xmn400m -Xss228k -XX:+UseG1GC 
-XX:G1ReservePercent=25 
-XX:InitiatingHeapOccupancyPercent=30 
-XX:+DisableExplicitGC 
-XX:+HeapDumpOnOutOfMemoryError 
-XX:HeapDumpPath=/opt/module/zookeeper-3.6.4/logs 
-XX:ErrorFile=/opt/module/zookeeper-3.6.4/logs/hs_err_pid%p.log 
-XX:+PrintGCDetails  
-Xloggc:/opt/module/zookeeper-3.6.4/logs/gc.lo


vim ~/.bash_profile
export JAVA_HOME=/opt/module/jdk-11.0.30
PATH=$PATH:$HOME/.local/bin:$HOME/bin:${JAVA_HOME}/bin
export PATH

vim zoo.cfg 
tickTime=2000 
initLimit=30000  
syncLimit=10     
maxClientCnxns=2000  
maxSessionTimeout=60000000 
autopurge.snapRetainCount=10 
autopurge.purgeInterval=1  
globalOutstandingLimit=200 
preAlloSize=131072 
snapCount=3000000  
leaderServes=yes  
dataDir=/opt/module/zookeeper-3.6.4/data/
dataLogDir=/opt/module/zookeeper-3.6.4/log 
clientPort=2181
4lw.commands.whitelist=*   
server.11=hadoop001:2888:3888
server.12=hadoop002:2888:3888
server.13=hadoop003:2888:3888

echo "11">/opt/module/zookeeper-3.6.4/data/myid
echo "12">/opt/module/zookeeper-3.6.4/data/myid
echo "13">/opt/module/zookeeper-3.6.4/data/myid



#vim hadoop-env.sh
cd /opt/module/hadoop-3.3.6/etc/hadoop
vi /opt/module/hadoop-3.3.6/etc/hadoop/hadoop-env.sh
export JAVA_HOME=/opt/module/jdk-11.0.30
export HADOOP_HOME=/opt/module/hadoop-3.3.6
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_YARN_HOME=$HADOOP_HOME
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HDFS_NAMENODE_USER=flink
export HDFS_DATANODE_USER=flink
export HDFS_SECONDARYNAMENODE_USER=flink
export HDFS_JOURNALNODE_USER=flink
export HDFS_ZKFC_USER=flink
export HADOOP_SHELL_EXECNAME=flink
export YARN_RESOURCEMANAGER_USER=flink
export YARN_NODEMANAGER_USER=flink
export YARN_SECURE_DN_USER=flink
export FLINK_NAMENODE_OPTS="-server -Xms1G -Xmx1G -Xmn400m -Xss228k -XX:+UseG1GC -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/module/hadoop-3.3.6/logs/hadoop -XX:ErrorFile=/opt/module/hadoop-3.3.6/logs/hadoop/hs_err_pid%p.log -XX:+PrintGCDetails  -Xloggc:/opt/module/hadoop-3.3.6/logs/hadoop/gc.log ${HADOOP_NAMENODE_OPTS}"
export FLINK_DATANODE_OPTS="-server -Xms1G -Xmx1G -Xmn400m -Xss228k -XX:+UseG1GC -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/module/hadoop-3.3.6/logs/hadoop -XX:ErrorFile=/opt/module/hadoop-3.3.6/logs/hadoop/hs_err_pid%p.log -XX:+PrintGCDetails  -Xloggc:/opt/module/hadoop-3.3.6/logs/hadoop/gc.log ${HADOOP_DATANODE_OPTS}"
export FLINK_NAMENODE_OPTS="-Dhdfs.audit.logger=WARN,DRFAAUDIT -Dhadoop.security.logger=WARN,DRFAS $FLINK_NAMENODE_OPTS"
export FLINK_DATANODE_OPTS="-Dhadoop.security.logger=WARN,DRFAS $FLINK_DATANODE_OPTS"

mkdir -p /opt/module/hadoop-3.3.6/logs/hadoop


###################
stop-dfs.sh
Stopping namenodes on [hadoop001 hadoop002]
hadoop001: WARNING: HADOOP_NAMENODE_OPTS has been replaced by FLINK_NAMENODE_OPTS. Using value of HADOOP_NAMENODE_OPTS.
hadoop002: WARNING: HADOOP_NAMENODE_OPTS has been replaced by FLINK_NAMENODE_OPTS. Using value of HADOOP_NAMENODE_OPTS.
Stopping datanodes
hadoop001: WARNING: HADOOP_DATANODE_OPTS has been replaced by FLINK_DATANODE_OPTS. Using value of HADOOP_DATANODE_OPTS.
hadoop002: WARNING: HADOOP_DATANODE_OPTS has been replaced by FLINK_DATANODE_OPTS. Using value of HADOOP_DATANODE_OPTS.
hadoop003: WARNING: HADOOP_DATANODE_OPTS has been replaced by FLINK_DATANODE_OPTS. Using value of HADOOP_DATANODE_OPTS.
Stopping journal nodes [hadoop003 hadoop002 hadoop001]




vi /opt/module/hadoop-3.3.6/etc/hadoop/core-site.xml
<configuration>
<!--指定hdfs的nameservice为hadoopns,用来指定hdfs的老大，ns为固定属性名，此值可以自己设置-但是后面的值要和此值对应，表示两个namenode-->
	<property>
		<name>fs.defaultFS</name>
		<value>hdfs://hadoopns</value>
	</property>
	<!--用来指定hadoop运行时产生文件的存放目录-->
	<property>
		<name>hadoop.tmp.dir</name>
		<value>/oracle/hadoop/tmp</value>
	</property>
	<!-- 指定zookeeper 地址-->
	<property>
		<name>ha.zookeeper.quorum</name>
		<value>hadoop001:2181,hadoop002:2181,hadoop003:2181</value>
	</property>
	<!--开启hadoop的回收站机制，当删除HDFS中的文件时，文件将会被移到回收站-/usr/<username>/.Trash，这个是hdfs上的目录,在指定的时间过后再对其进行删除 -->
	<property>
		<name>fs.trash.interval</name>
		<!-- 单位是分钟-->
		<value>1440</value>
	</property>
</configuration>



#namenode 的数据目录 和 datanode 的数据目录应该设置不同的目录。
vi hdfs-site.xml
<configuration>
	<!--指定hdfs的nameservice为ns,需要和core-site.xml中的保持一致-->
	<property>
		<name>dfs.nameservices</name>
		<value>hadoopns</value>
	</property>
	<!--ns下面有两个NameNode，分别是nn1,nn2-->
	<property>
		<name>dfs.ha.namenodes.hadoopns</name>
		<value>hdfsnn1,hdfsnn2</value>
	</property>
	<!--nn1的RPC通信地址-->
	<property>
		<name>dfs.namenode.rpc-address.hadoopns.hdfsnn1</name>
		<value>hadoop001:9000</value>
	</property>
	<!-- nn1 的http 通信地址-->
	<property>
		<name>dfs.namenode.http-address.hadoopns.hdfsnn1</name>
		<value>hadoop001:50070</value>
	</property>
	<!-- nn2 的RPC 通信地址-->
	<property>
		<name>dfs.namenode.rpc-address.hadoopns.hdfsnn2</name>
		<value>hadoop002:9000</value>
	</property>
	<!-- nn2 的http 通信地址-->
	<property>
		<name>dfs.namenode.http-address.hadoopns.hdfsnn2</name>
		<value>hadoop002:50070</value>
	</property>
	<!-- 指定NameNode 的edits 元数据在JournalNode 上的存放位置-->
	<property>
		<name>dfs.namenode.shared.edits.dir</name>
		<value>qjournal://hadoop001:8485;hadoop002:8485;hadoop003:8485/hadoopns</value>
	</property>
	<!-- 指定JournalNode 在本地磁盘存放数据的位置-->
	<property>
		<name>dfs.journalnode.edits.dir</name>
		<value>/oracle/hadoop/jdata</value>
	</property>
	<!-- 开启NameNode 失败自动切换-->
	<property>
		<name>dfs.ha.automatic-failover.enabled.hadoopns</name>
		<value>true</value>
	</property>
	<!-- 配置失败自动切换实现方式-->
	<property>
		<name>dfs.client.failover.proxy.provider.hadoopns</name>
		<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
	</property>
	<!-- 配置隔离机制方法，多个机制用换行分割，即每个机制暂用一行-->
	<property>
		<name>dfs.ha.fencing.methods</name>
		<value>sshfence</value>
	</property>
	<!-- 使用sshfence 隔离机制时需要ssh 免登陆，涉及到主备主机的，都要去生成ssh-keygen -t rsa-->
	<property>
		<name>dfs.ha.fencing.ssh.private-key-files</name>
		<value>/home/flink/.ssh/id_rsa</value>
	</property>
	<!-- 配置sshfence 隔离机制超时时间30s-->
	<property>
		<name>dfs.ha.fencing.ssh.connect-timeout</name>
		<value>30000</value>
	</property>
	<property>
		<name>dfs.namenode.name.dir</name>
		<value>file:///oracle/hadoop/data</value>
	</property>
	<!--配置datanode数据存放的位置，可以不配置，如果不配置，默认用的是core-site.xml里配置的hadoop.tmp.dir的路径-->
	<property>
		<name>dfs.datanode.data.dir</name>
		<value>file:///oracle/hadoop/data</value>
	</property>
	<!--HDFS 副本数量-->
	<property>
		<name>dfs.replication</name>
		<value>3</value>
	</property>
	<!--此参数控制是否为集群启用了diskbalancer，磁盘空间数据平衡。如果未启用，则datanode将-拒绝任何执行命令。默认值为false-->
	<property>
		<name>dfs.disk.balancer.enabled</name>
		<value>true</value>
	</property>
	<!--指定一个配置文件，使NameNode过滤配置文件中指定的host-->
	<property>
		<name>dfs.hosts.exclude</name>
		<value>/opt/module/hadoop-3.3.6/etc/hadoop/dfs.hosts.exclude</value>
	</property>
</configuration>

	


--4)修改mapred-site.xml    
	
vim mapred-site.xml
<configuration>
	<!--指定mapreduce 运行在yarn 上-->
	<property>
		<name>mapreduce.framework.name</name>
		<value>yarn</value>
	</property>
	<!--设置JobHistory的服务地址，JobHistory记录了已经完成的mapreduce任务信息并存放在HDFS-指定的目录下，默认未开启，Jobhistory用于查询每个job运行完以后的历史日志信息，是作为-一台单独的服务器运行的，可以在namenode或者datanode的任意一台 启动即可-->
	<property>
		<name>mapreduce.jobhistory.address</name>
		<value>hadoop001:10020</value>
	</property>
	<!-- 指定JobHistory 的Web 访问地址-->
	<property>
		<name>mapreduce.jobhistory.webapp.address</name>
		<value>hadoop001:19888</value>
	</property>
	<!--开启Uber运行模式，Uber运行模式对小作业进行优化，不会给每个任务分别申请Container资源	这些小任务将统一在一个Container中按先执行map任务后执行reduce任务的顺序串行执行-->
	<property>
		<name>mapreduce.job.ubertask.enable</name>
		<value>true</value>
	</property>
	<!--通过 hadoop classpath 得到classpath写到下面，解决在服务器中运行hadoop自带的jar包中的实例报错，可以在后期集群运行后配置，如果不需要再服务器运行，可以不配置下面3个参数$HADOOP_MAPRED_HOME 就是hadoop实际安装路径，必须填写完整的路径，即必须是绝对路径，不能包含变量-->
	<property>
		<name>mapreduce.map.env</name>
		<value>HADOOP_MAPRED_HOME=/opt/module/hadoop-3.3.6/etc/hadoop:
		/opt/module/hadoop-3.3.6/share/hadoop/common/lib/*:/opt/module/hadoop-3.3.6/share/hadoop/common/*
		:/opt/module/hadoop-3.3.6/share/hadoop/hdfs:/opt/module/hadoop-3.3.6/share/hadoop/hdfs/lib/*
		:/opt/module/hadoop-3.3.6/share/hadoop/hdfs/*:/opt/module/hadoop-3.3.6/share/hadoop/mapreduce/lib/*
		:/opt/module/hadoop-3.3.6/share/hadoop/mapreduce/*:/opt/module/hadoop-3.3.6/share/hadoop/yarn
		:/opt/module/hadoop-3.3.6/share/hadoop/yarn/lib/*:/opt/module/hadoop-3.3.6/share/hadoop/yarn/*</value>
	</property>
	<property>
		<name>mapreduce.reduce.env</name>
		<value>HADOOP_MAPRED_HOME=/opt/module/hadoop-3.3.6/etc/hadoop
		:/opt/module/hadoop-3.3.6/share/hadoop/common/lib/*:/opt/module/hadoop-3.3.6/share/hadoop/common/*
		:/opt/module/hadoop-3.3.6/share/hadoop/hdfs:/opt/module/hadoop-3.3.6/share/hadoop/hdfs/lib/*
		:/opt/module/hadoop-3.3.6/share/hadoop/hdfs/*:/opt/module/hadoop-3.3.6/share/hadoop/mapreduce/lib/*
		:/opt/module/hadoop-3.3.6/share/hadoop/mapreduce/*:/opt/module/hadoop-3.3.6/share/hadoop/yarn
		:/opt/module/hadoop-3.3.6/share/hadoop/yarn/lib/*:/opt/module/hadoop-3.3.6/share/hadoop/yarn/*</value>
	</property>
	<property>
		<name>mapreduce.application.classpath</name>
		<value>/opt/module/hadoop-3.3.6/etc/hadoop:/opt/module/hadoop-3.3.6/share/hadoop/common/lib/*
		:/opt/module/hadoop-3.3.6/share/hadoop/common/*:/opt/module/hadoop-3.3.6/share/hadoop/hdfs
		:/opt/module/hadoop-3.3.6/share/hadoop/hdfs/lib/*:/opt/module/hadoop-3.3.6/share/hadoop/hdfs/*
		:/opt/module/hadoop-3.3.6/share/hadoop/mapreduce/lib/*:/opt/module/hadoop-3.3.6/share/hadoop/mapreduce/*
		:/opt/module/hadoop-3.3.6/share/hadoop/yarn:/opt/module/hadoop-3.3.6/share/hadoop/yarn/lib/*
		:/opt/module/hadoop-3.3.6/share/hadoop/yarn/*</value>
	</property>
</configuration>


*/


--5）yarn-site.xml
vim yarn-site.xml
<configuration>

<!-- Site specific YARN configuration properties -->
	<!-- Site specific YARN configuration properties -->
	<!-- 开启RM YARN HA -->
	<property>
		<name>yarn.resourcemanager.ha.enabled</name>
		<value>true</value>
	</property>
		<!-- 指定RM HA 的cluster id -->
	<property>
		<name>yarn.resourcemanager.cluster-id</name>
		<value>yarncluster</value>
	</property>
	<!--指定yarn 的老大resoucemanager 的地址-->
	<property>
		<name>yarn.resourcemanager.hostname</name>
		<value>hadoop001</value>
	</property>
	<!-- 指定RM 的名字-->
	<property>
		<name>yarn.resourcemanager.ha.rm-ids</name>
		<value>yarnrm1,yarnrm2</value>
	</property>
	<!-- 分别指定RM 的地址-->
	<property>
		<name>yarn.resourcemanager.hostname.yarnrm1</name>
		<value>hadoop001</value>
	</property>
	<property>
		<name>yarn.resourcemanager.hostname.yarnrm2</name>
		<value>hadoop002</value>
	</property>
	<!--开启yarn 恢复机制-->
	<property>
		<name>yarn.resourcemanager.recovery.enabled</name>
		<value>true</value>
	</property>
	<!--执行rm恢复机制实现类-->
	<property>
		<name>yarn.resourcemanager.store.class</name>
		<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
	</property>
	<!--指定zk集群地址-->
	<property>
		<name>hadoop.zk.address</name>
		<value>hadoop001:2181,hadoop002:2181,hadoop003:2181</value>
	</property>
	<!--NodeManager 获取数据的方式-->
	<property>
		<name>yarn.nodemanager.aux-services</name>
		<value>mapreduce_shuffle</value>
	</property>
	<!--开启日志聚合-->
	<property>
		<name>yarn.log-aggregation-enable</name>
		<value>true</value>
	</property>
	<!--日志在HDFS上最多保存24小时，这里改成60小时了。-->
	<property>
		<name>yarn.log-aggregation.retain-seconds</name>
		<value>86400</value>
	</property>
	<!--指定RM 的Web 端访问地址-->
	<property>
		<name>yarn.resourcemanager.webapp.address.yarnrm1</name>
		<value>hadoop001:8088</value>
	</property>
	<property>
		<name>yarn.resourcemanager.webapp.address.yarnrm2</name>
		<value>hadoop002:8088</value>
	</property>
	<!--指定一个配置文件，使 Resourcemanager 过滤配置文件中指定的host-->
	<property>
		<name>yarn.resourcemanager.nodes.exclude-path</name>
		<value>/opt/module/hadoop-3.3.6/etc/hadoop/yarn.hosts.exclude</value>
	</property>
	<!--通过hadoop classpath 得到classpath 写到下面，解决在服务器中运行hadoop 自带的
	     	jar 包中的实例报错，可以在后期集群运行后配置，如果不需要在服务器运行，可以不配置下面3
	个参数$HADOOP_MAPRED_HOME 就是hadoop 实际安装路径,必须填写完整的路径，即必须是绝对路径，
	不能包含变量。-->
	<property>
		<name>yarn.application.classpath</name>
		<value>/opt/module/hadoop-3.3.6/etc/hadoop:/opt/module/hadoop-3.3.6/share/hadoop/common/lib/*
		:/opt/module/hadoop-3.3.6/share/hadoop/common/*:/opt/module/hadoop-3.3.6/share/hadoop/hdfs
		:/opt/module/hadoop-3.3.6/share/hadoop/hdfs/lib/*:/opt/module/hadoop-3.3.6/share/hadoop/hdfs/*
		:/opt/module/hadoop-3.3.6/share/hadoop/mapreduce/lib/*:/opt/module/hadoop-3.3.6/share/hadoop/mapreduce/*
		:/opt/module/hadoop-3.3.6/share/hadoop/yarn:/opt/module/hadoop-3.3.6/share/hadoop/yarn/lib/*
		:/opt/module/hadoop-3.3.6/share/hadoop/yarn/*</value>
	</property>
	<!--设置yarn的内存大小，单位是MB，表示该节点上YARN可使用的物理内存总量-->
	<property>
		<name>yarn.nodemanager.resource.memory-mb</name>
		<value>2048</value>
	</property>
	<!--单个任务可申请的最多物理内存量,最小1024-->
	<property>
		<name>yarn.scheduler.maximum-allocation-mb</name>
		<value>512</value>
	</property>
	<!--设置yarn的CPU核数-->
	<property>
		<name>yarn.nodemanager.source.cpu-vcores</name>
		<value>2</value>
	</property>
</configuration>

*/

echo "" > /opt/module/hadoop-3.3.6/etc/hadoop/yarn.hosts.exclude 
echo "" > /opt/module/hadoop-3.3.6/etc/hadoop/dfs.hosts.exclude 

vi yarn-site.xml 
YARN内存利用优化。物理机可以设置，自己的实验就不要设置了。
设置：
<!--设置yarn的内存大小，单位是MB，表示该节点上YARN可使用的物理内存总量-->
<property>
	<name>yarn.nodemanager.resource.memory-mb</name>
	<value>8192</value>
</property>
<!--单个任务可申请的最多物理内存量-->
<property>
	<name>yarn.scheduler.maximum-allocation-mb</name>
	<value>1024</value>
</property>
<!--设置yarn的CPU核数-->
<property>
	<name>yarn.nodemanager.source.cpu-vcores</name>
	<value>8</value>
</property>

修改works文件。
在hadoop3.0之前的版本中，这个文件叫做slaves;

vim workers
hadoop001
hadoop002
hadoop003

--7)修改log4j.properties日志文件。

vi /opt/module/hadoop-3.3.6/etc/hadoop/log4j.properties
hdfs.audit.logger=WARN,console
hdfs.audit.log.maxfilesize=256MB
hdfs.audit.log.maxbackupindex=20
log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false

#log4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFAAUDIT=org.apache.log4j.RollingFileAppender
log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout
log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd

############各个资源分布情况展示;
hadoop的名称服务:hadoopns 
hadoop:的NameNode: hadoop001:hdfsnn1,hadoop001:hdfsnn2 
namenode元数据存放位置:hadoop001,hadoop002,hadoop003 
journalnode数据存放位置:/oracle/hadoop/jdata
namenode数据存放位置:/oracle/hadoop/data
datanode数据存放位置:/oracle/hadoop/data
mapreduce.jobhistory:hadoop001 
yarn的resourcemanager:hadoop001 
hadoop001:yarnrm1,hadoop002:yarnrm2 :8088
zkfc:hadoop001,hadoop002

#启动 hadoop集群，包括  HDFS 和 YARN  
start-dfs.sh 
start-yarn.sh 

#启动 netcat 
nc -l -s 192.168.3.11 7777 

#hdfs启动失败 #原因是 namenode 还没有进行初始化。
[flink@hadoop001 module]$ start-dfs.sh
Starting namenodes on [hadoop001 hadoop002]
hadoop001: ERROR: Cannot set priority of namenode process 42004
hadoop002: ERROR: Cannot set priority of namenode process 39207
Starting datanodes
hadoop001: ERROR: Cannot set priority of datanode process 42149
hadoop002: ERROR: Cannot set priority of datanode process 39314
hadoop003: ERROR: Cannot set priority of datanode process 38585
Starting journal nodes [hadoop003 hadoop002 hadoop001]


#vi /opt/module/hadoop-3.3.6/etc/hadoop/hadoop-env.sh
原因:文件中某些目录不存在，所以导致无法启动。
[flink@hadoop001 ~]$ hdfs --daemon start datanode
ERROR: Cannot set priority of datanode process 54579


[flink@hadoop003 hadoop]$ hdfs haadmin -getAllServiceState
hadoop001:9000                                     active
hadoop002:9000                                     standby

[flink@hadoop003 hadoop]$ hdfs haadmin -getServiceState hdfsnn1
active
[flink@hadoop003 hadoop]$ hdfs haadmin -getServiceState hdfsnn2
standby

[flink@hadoop001 logs]$ yarn rmadmin -getAllServiceState
hadoop001:8033                                     standby
hadoop002:8033                                     active

[flink@hadoop001 logs]$ yarn rmadmin -getServiceState yarnrm1
standby
[flink@hadoop001 logs]$ yarn rmadmin -getServiceState yarnrm2
active


#namenode;
http://192.168.3.11:50070 
http://192.168.3.12:50070 

#historyserver;
mapred --daemon start historyserver 
http://192.168.3.11:19888/jobhistory 

#查看yarn的管理地址：active 
http://192.168.3.11:8088 


http://192.168.3.11:19888/jobhistory


#停止hadoop 集群
#1.停止hdfs 集群
ssh flink@hadoop001 stop-dfs.sh

#2.停止zkfc;
ssh flink@hadoop001 hdfs --daemon stop zkfc
ssh flink@hadoop002 hdfs --daemon stop zkfc

#3.停止yarn
ssh flink@hadoop001 stop-yarn.sh
#4.JobHistory 停止
ssh flink@hadoop001 mapred --daemon stop historyserver

#5.停止zookeeper
ssh flink@hadoop001 source ~/.bash_profile;zkServer.sh stop
ssh flink@hadoop002 source ~/.bash_profile;zkServer.sh stop
ssh flink@hadoop003 source ~/.bash_profile;zkServer.sh stop

Hadoop环境准备。

https://edu.csdn.net/learn/39067/627173?utm_source=2019755004

汇聚全球AI编程工具，助力开发者即刻编程。

更多推荐

使用up8ai.com部署Codex中转API

在 CC-Switch 的聊天界面中直接发送消息，测试是否能正常接收回复。命令，即可启动交互式对话。若能正常收到回复，则 CLI 配置成功。此路径通过图形化客户端 CC-Switch 实现，无需命令行操作。此路径通过 Node.js 和 Codex CLI 工具实现。Codex CLI 的核心配置是设置。1. 下载并安装 CC-Switch。

AI编程社区

Codex++：让 OpenAI Codex 与 ChatGPT 桌面版真正“听话“的外部启动器

AI编程社区

当所有人都在用 TS/Python 写 Agent，我们为什么坚持 Java

SolonCode是一个基于Java实现的开源编码智能体，兼容Java 8至26环境，特别针对企业级需求设计。它选择Java而非主流的TS/Python技术栈，旨在解决企业部署中的四大核心问题：JDK合规性、信创适配（如毕昇JDK与鸿蒙PC）、运维友好性（单进程/系统命令形态）和可扩展性（Java插件机制）。该工具强调与企业现有基础设施无缝集成，支持内网部署、离线安装和CLI/Web双交互模式，并