= Apache Hadoop =
[[https://hadoop.apache.org/| Apache Hadoop]] project develops open-source software for reliable, scalable, distributed computing.

The Apache Hadoop software library is a framework that allows for the '''distributed processing of large data sets''' across clusters of computers using simple programming models. It is designed to scale up from single servers to thousands of machines, each offering local computation and storage. 

Hadoop modules:
 * Hadoop Common 
  * The common utilities that support the other Hadoop modules.
 * Hadoop Distributed File System (HDFS)
  * A '''distributed file system''' that provides high-throughput access to application data.
 * Hadoop YARN
  * A framework for '''job scheduling''' and cluster resource management.
 * Hadoop MapReduce
  * A YARN-based system for '''parallel processing of large data sets'''.


== Install ==
{{{#!highlight bash
cd ~/tmp
wget https://archive.apache.org/dist/hadoop/core/hadoop-3.3.1/hadoop-3.3.1.tar.gz
tar tvzf hadoop-3.3.1.tar.gz
tar xvzf hadoop-3.3.1.tar.gz
}}}

=== ~/tmp/hadoop-3.3.1/etc/hadoop/hadoop-env.sh ===
{{{#!highlight bash
export JAVA_HOME=/home/vitor/jdk-11.0.10+9
}}}

=== ~/tmp/hadoop-3.3.1/etc/hadoop/core-site.xml ===
{{{#!highlight xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>fs.default.name</name>
    <value>hdfs://master:9000</value>
  </property>
</configuration>
}}}

=== ~/tmp/hadoop-3.3.1/etc/hadoop/hdfs-site.xml ===
{{{#!highlight xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>dfs.namenode.name.dir</name>
    <value>/tmp/nameNode</value>
  </property>
  <property>
    <name>dfs.namenode.data.dir</name>
    <value>/tmp/dataNode</value>
  </property>
  <property>
    <name>dfs.replication</name>
    <value>2</value>
   </property>
</configuration>
}}}

=== ~/tmp/hadoop-3.3.1/etc/hadoop/mapred-site.xml ===
{{{#!highlight xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
</configuration>
}}}

=== ~/tmp/hadoop-3.3.1/etc/hadoop/yarn-site.xml ===
{{{#!highlight xml
<?xml version="1.0"?>
<configuration>
  <property>
    <name>yarn.acl.enable</name>
    <value>0</value>
  </property>
  <property>
    <name>yarn.resourcemanager.hostname</name>
    <value>master</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
</configuration>
}}}

===  ~/tmp/hadoop-3.3.1/etc/hadoop/slaves ===
{{{#!highlight bash
localhost
}}}

=== ~/.bashrc ===
{{{#!highlight bash
export HADOOP_HOME=/home/vitor/tmp/hadoop-3.3.1
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
}}}

=== Setup HDFS ===
{{{#!highlight bash
ls $HADOOP_HOME/bin/
hdfs dfsadmin -report 
# 2021-07-23 14:07:17,010 WARN fs.FileSystem: Failed to initialize fileystem hdfs://master:9000: 
# java.lang.IllegalArgumentException: java.net.UnknownHostException: master
# report: java.net.UnknownHostException: master
# add 127.0.0.1 master to /etc/hosts
hdfs namenode -format master
hdfs --daemon start namenode
hdfs --daemon start datanode
yarn --daemon start resourcemanager
yarn --daemon start nodemanager
yarn --daemon start proxyserver
mapred --daemon start historyserver
hdfs dfsadmin -report
# http://localhost:9870/
# http://localhost:9870/dfshealth.html#tab-overview
# http://localhost:9870/explorer.html#
# http://localhost:8088/
# http://localhost:8088/cluster
# http://localhost:19888/
hadoop fs -ls /
hadoop fs -ls /tmp 
hadoop fs -mkdir /test
hadoop fs -ls /
}}}