Docker-Hadoop ALL-IN-ONE

额，最近折腾机器学习和大数据，之前一直对hadoop有所了解，但是觉得用不上，现在有机遇可以接触一下。遂研究了一下网上的搭建方式。弄明白原理后找了些docker的搭建方案。发现了https://hub.docker.com/r/uhopper/hadoop/ 这个项目还比较火。可以一件部署hadoop生态环境。自带的容器有namenode 、 datanode、 resourcemanager、nodemanager、spark。如果整合其他的也可以通过环境变量引入配置文件的方式来处理。不过目前这几个容器也够用了。

话不多说。上代码

version: '3'
    services:
        namenode:
          image: uhopper/hadoop-namenode:2.8.1
          hostname: namenode
          container_name: namenode
          domainname: hadoop
          net: hadoop
          volumes:
            - /namenode:/hadoop/dfs/name
          environment:
            - CLUSTER_NAME=datanode1
            - CLUSTER_NAME=datanode2
            - CLUSTER_NAME=datanode3

datanode1:
      image: uhopper/hadoop-datanode:2.8.1
      hostname: datanode1
      container_name: datanode1
      domainname: hadoop
      net: hadoop
      volumes:
        - /datanode1:/hadoop/dfs/data
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        
    datanode2:
      image: uhopper/hadoop-datanode:2.8.1
      hostname: datanode2
      container_name: datanode2
      domainname: hadoop
      net: hadoop
      volumes:
        - /datanode2:/hadoop/dfs/data
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020

datanode3:
      image: uhopper/hadoop-datanode:2.8.1
      hostname: datanode3
      container_name: datanode3
      domainname: hadoop
      net: hadoop
      volumes:
        - /datanode3:/hadoop/dfs/data
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    
    resourcemanager:
      image: uhopper/hadoop-resourcemanager:2.8.1
      hostname: resourcemanager
      container_name: resourcemanager
      domainname: hadoop
      net: hadoop
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        - YARN_CONF_yarn_log___aggregation___enable=true
        
    nodemanager:
      image: uhopper/hadoop-nodemanager:2.8.1
      hostname: nodemanager
      container_name: nodemanager
      domainname: hadoop
      net: hadoop
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        - YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
        - YARN_CONF_yarn_log___aggregation___enable=true
        - YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
        
    spark:
      image: uhopper/hadoop-spark:2.8.1
      hostname: spark
      container_name: spark
      domainname: hadoop
      net: hadoop
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        - YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
      command: tail -f /var/log/dmesg

Navigation

Recent Posts

Friend Links