📜 Hadoop 核心组件配置文件模板

提示

本文件包含了 core-site.xml、hdfs-site.xml、yarn-site.xml 以及 mapred-site.xml 的核心生产配置参数，专供实训参考。

<!-- hadoop_config_template.conf -->
<configuration>
    <!-- 1. 配置 HDFS 主节点（NameNode）的 RPC 通信地址 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:9000</value>
    </property>

    <!-- [在此处直接粘贴你的 hadoop_config_template.conf 文件的其余配置源码] -->
    # Hadoop集群配置文件模板
    # 请根据实际环境修改以下配置

    # ==================== 基础配置 ====================

    # 集群节点配置
    MASTER_HOST="hadoop-master"
    SLAVE_HOSTS=("hadoop-slave1" "hadoop-slave2" "hadoop-slave3")

    # 网络配置
    MASTER_IP="192.168.1.100"
    SLAVE1_IP="192.168.1.101"
    SLAVE2_IP="192.168.1.102"
    SLAVE3_IP="192.168.1.103"

    # Hadoop版本
    HADOOP_VERSION="3.3.4"

    # 用户配置
    HADOOP_USER="hadoop"
    HADOOP_GROUP="hadoop"
    HADOOP_PASSWORD="123456"

    # 安装路径
    HADOOP_HOME="/home/hadoop/hadoop"
    JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk"

    # ==================== 性能配置 ====================

    # JVM堆大小配置
    NAMENODE_HEAPSIZE="2048"
    DATANODE_HEAPSIZE="1024"
    RESOURCEMANAGER_HEAPSIZE="2048"
    NODEMANAGER_HEAPSIZE="1024"

    # HDFS配置
    DFS_REPLICATION="3"
    DFS_BLOCK_SIZE="134217728"  # 128MB
    DFS_NAMENODE_HANDLER_COUNT="20"

    # YARN配置
    YARN_NODEMANAGER_MEMORY="4096"
    YARN_NODEMANAGER_VCORES="4"
    YARN_SCHEDULER_MIN_ALLOC_MB="256"
    YARN_SCHEDULER_MAX_ALLOC_MB="2048"

    # MapReduce配置
    MAPREDUCE_MAP_MEMORY_MB="1024"
    MAPREDUCE_REDUCE_MEMORY_MB="2048"
    MAPREDUCE_MAP_JAVA_OPTS="-Xmx819m"
    MAPREDUCE_REDUCE_JAVA_OPTS="-Xmx1638m"

    # ==================== 端口配置 ====================

    # HDFS端口
    HDFS_NAMENODE_PORT="9000"
    HDFS_DATANODE_PORT="9866"
    HDFS_NAMENODE_HTTP_PORT="9870"
    HDFS_DATANODE_HTTP_PORT="9864"

    # YARN端口
    YARN_RESOURCEMANAGER_PORT="8032"
    YARN_NODEMANAGER_PORT="8042"
    YARN_RESOURCEMANAGER_WEBAPP_PORT="8088"
    YARN_NODEMANAGER_WEBAPP_PORT="8042"

    # MapReduce端口
    MAPREDUCE_JOBHISTORY_PORT="10020"
    MAPREDUCE_JOBHISTORY_WEBAPP_PORT="19888"

    # ==================== 存储配置 ====================

    # 数据目录
    HDFS_NAMENODE_DIR="/home/hadoop/hadoop/data/namenode"
    HDFS_DATANODE_DIR="/home/hadoop/hadoop/data/datanode"
    HDFS_SECONDARY_DIR="/home/hadoop/hadoop/data/namesecondary"
    HADOOP_TMP_DIR="/home/hadoop/hadoop/data/tmp"
    HADOOP_LOG_DIR="/home/hadoop/hadoop/logs"

    # ==================== 安全配置 ====================

    # 是否启用Kerberos认证
    ENABLE_KERBEROS="false"

    # 是否启用SSL/TLS
    ENABLE_SSL="false"

    # 访问控制列表
    HDFS_SUPERUSER="hadoop"
    YARN_ADMIN_USER="hadoop"

    # ==================== 监控配置 ====================

    # 是否启用Ganglia监控
    ENABLE_GANGLIA="false"

    # 是否启用Ambari监控
    ENABLE_AMBARI="false"

    # 日志级别
    LOG_LEVEL="INFO"

    # ==================== 网络配置 ====================

    # 防火墙配置
    FIREWALL_ENABLED="false"

    # 需要开放的端口
    OPEN_PORTS=(
        "9000"   # HDFS NameNode
        "9870"   # HDFS Web UI
        "9866"   # HDFS DataNode
        "9864"   # HDFS DataNode HTTP
        "8032"   # YARN ResourceManager
        "8088"   # YARN Web UI
        "8042"   # YARN NodeManager
        "10020"  # MapReduce JobHistory
        "19888"  # MapReduce Web UI
    )

    # ==================== 备份配置 ====================

    # 是否启用NameNode HA
    ENABLE_HA="false"

    # 备份节点配置
    BACKUP_NAMENODE="hadoop-slave1"

    # ==================== 示例配置 ====================

    # 小规模集群配置 (4节点)
    SMALL_CLUSTER_CONFIG='
    MASTER_HOST="hadoop-master"
    SLAVE_HOSTS=("hadoop-slave1" "hadoop-slave2" "hadoop-slave3")
    DFS_REPLICATION="3"
    YARN_NODEMANAGER_MEMORY="2048"
    YARN_NODEMANAGER_VCORES="2"
    '

    # 中等规模集群配置 (8节点)
    MEDIUM_CLUSTER_CONFIG='
    MASTER_HOST="hadoop-master"
    SLAVE_HOSTS=("hadoop-slave1" "hadoop-slave2" "hadoop-slave3" "hadoop-slave4" "hadoop-slave5" "hadoop-slave6" "hadoop-slave7")
    DFS_REPLICATION="3"
    YARN_NODEMANAGER_MEMORY="4096"
    YARN_NODEMANAGER_VCORES="4"
    '

    # 大规模集群配置 (16节点)
    LARGE_CLUSTER_CONFIG='
    MASTER_HOST="hadoop-master"
    SLAVE_HOSTS=("hadoop-slave1" "hadoop-slave2" "hadoop-slave3" "hadoop-slave4" "hadoop-slave5" "hadoop-slave6" "hadoop-slave7" "hadoop-slave8" "hadoop-slave9" "hadoop-slave10" "hadoop-slave11" "hadoop-slave12" "hadoop-slave13" "hadoop-slave14" "hadoop-slave15")
    DFS_REPLICATION="3"
    YARN_NODEMANAGER_MEMORY="8192"
    YARN_NODEMANAGER_VCORES="8"
    '

    # ==================== 使用说明 ====================

    # 1. 修改基础配置中的主机名和IP地址
    # 2. 根据集群规模选择合适的性能配置
    # 3. 根据安全需求配置安全选项
    # 4. 根据网络环境配置防火墙规则
    # 5. 根据存储需求配置数据目录

    # 配置完成后，将此文件重命名为 hadoop_config.conf
    # 并在部署脚本中引用此配置文件 
</configuration>