다운받아 사용하시면 됩니다

# 첨부파일을 누르면 링크를 확인할 수 있습니다 
wget 링크
chmod +x 파일
./파일

설치 직후 네트워크 설정하는 스크립트

있으면 좋겠당

자바랑 하둡 설치하는 스크립트

if [ "$EUID" -ne 0 ]; then
  echo usage: \\" sudo ./install_hadoop.sh \\"
  exit
fi

echo \\>\\>\\>\\> INSTALLING JDK ...
apt install openjdk-8-jdk -y
sudo sed -i '$a JAVA_HOME="\\/usr\\/lib\\/jvm\\/java-8-openjdk-arm64\\/"' /etc/environment
wait $! 
echo \\>\\>\\>\\>  DONE !

echo \\>\\>\\>\\>  DOWNLOADING HADOOP "$VERSION" ...
HADOOP_VERSION="hadoop-"$VERSION
HADOOP_VERSION_TAR=$HADOOP_VERSION".tar.gz"
if [ ! -f "$HADOOP_VERSION_TAR" ]; then
        wget "<https://archive.apache.org/dist/hadoop/common/$HADOOP_VERSION/$HADOOP_VERSION_TAR>"
        wait $!
fi
echo \\>\\>\\>\\> DONE !

echo \\>\\>\\>\\> INSTALLING HADOOP ...
rm -rf "$HADOOP_VERSION"
rm -rf hadoop
sudo rm -rf /usr/local/hadoop

tar xzvf "$HADOOP_VERSION_TAR"
wait $!
mv $HADOOP_VERSION hadoop
sudo mv hadoop /usr/local/hadoop
echo \\>\\>\\>\\> DONE !

echo \\>\\>\\>\\> CHANGING PATH ...
sudo sed -i 's/^\\# export JAVA_HOME=.*/export JAVA_HOME=\\/usr\\/lib\\/jvm\\/java-1.8.0-openjdk-arm64/' hadoop/etc/hadoop/hadoop-env.sh
sudo sed -i '/^PATH=/ s/"$/\\:\\/usr\\/local\\/hadoop\\/bin\\:\\/usr\\/local\\/hadoop\\/sbin"/' /etc/environment
wait $!

if [ ${VERSION%.*} -eq 3 ]; then
sudo sed -i '$a\\
HADOOP_COMMON_HOME="/usr/local/hadoop"\\
' /etc/environment
else
sudo sed -i '$a\\
HADOOP_COMMON_HOME="/usr/local/hadoop"\\
' /etc/environment
fi
sudo sed -i '$a\\
HADOOP_PREFIX="/usr/local/hadoop"\\
HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop\\
HADOOP_HDFS_HOME=$HADOOP_PREFIX\\
HADOOP_MAPRED_HOME=$HADOOP_PREFIX\\
HADOOP_YARN_HOME=$HADOOP_PREFIX\\
HADOOP_CMD=$HADOOP_PREFIX/bin/hadoop\\
HADOOP_STREAMING="$HADOOP_PREFIX/share/hadoop/tools/lib/hadoop-streaming-$VERSION.jar"\\
HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_PREFIX/lib/native\\
HADOOP_OPTS="-Djava.library.path=$HADOOP_PREFIX/lib/native"
' /etc/environment
echo \\>\\>\\>\\> DONE !

echo \\>\\>\\>\\> CREATING MASTER ...
adduser --disabled-login --gecos "" ubuntu1 
echo ubuntu1:R2020hjp | chpasswd
wait $!
echo \\>\\>\\>\\> DONE !

echo \\>\\>\\>\\> GRANTING SUPER USER TO MASTER ...
sudo usermod -aG ubuntu1 ubuntu1
sudo chown ubuntu1:root -R /usr/local/hadoop
sudo chmod g+rwx -R /usr/local/hadoop
sudo adduser ubuntu1 sudo
wait $!
echo \\>\\>\\>\\> DONE !

네임노드 하둡 초기화(반복사용 가능)

if [ $USER != "ubuntu1" ]; then
	echo current user is $USER
  echo Script must be run as user:su - ubuntu1
  echo run \\'su - ubuntu1\\' first
	echo then \\" ./hdfs_setup.sh \\"
  exit -1
fi

cd /home/ubuntu1
sudo apt install sshpass

echo CREATING SSH KEY ...
sudo rm -rf .ssh
cat /dev/zero | ssh-keygen -N ''
wait $!

ssh-keyscan -H ubuntu1 >> .ssh/known_hosts
sshpass -p "R2020hjp" ssh ubuntu1@ubuntu1 -y 'exit'
sshpass -p "R2020hjp" ssh-copy-id ubuntu1@ubuntu1
wait $!

#ssh-keyscan -H ubuntu2 >> .ssh/known_hosts
#sshpass -p "R2020hjp" ssh ubuntu1@ubuntu2 -y 'exit'
#sshpass -p "R2020hjp" ssh-copy-id ubuntu1@ubuntu2
#wait $!

#ssh-keyscan -H ubuntu3 >> .ssh/known_hosts
#sshpass -p "R2020hjp" ssh ubuntu1@ubuntu3 -y 'exit'
#sshpass -p "R2020hjp" ssh-copy-id ubuntu1@ubuntu3
#wait $!

ssh-keyscan -H ubuntu4 >> .ssh/known_hosts
sshpass -p "R2020hjp" ssh ubuntu1@ubuntu4 -y 'exit'
sshpass -p "R2020hjp" ssh-copy-id ubuntu1@ubuntu4
wait $!

ssh-keyscan -H ubuntu5 >> .ssh/known_hosts
sshpass -p "R2020hjp" ssh ubuntu1@ubuntu5 -y 'exit'
sshpass -p "R2020hjp" ssh-copy-id ubuntu1@ubuntu5
wait $!
echo DONE !

cd /usr/local/hadoop/etc/hadoop
echo EDITING core-site.xml
if [ ! -f core-site.xml.original ]; then
	cp core-site.xml core-site.xml.original
else
	cp core-site.xml.original core-site.xml
fi
sed -i '19a\\
<property>\\
    <name>fs.defaultFS</name>\\
    <value>hdfs://ubuntu1:9000</value>\\
</property>' core-site.xml
echo DONE !

echo EDITING hdfs-site.xml
if [ ! -f hdfs-site.xml.original ]; then
	cp hdfs-site.xml hdfs-site.xml.original
else
	cp hdfs-site.xml.original hdfs-site.xml
fi
sed -i '19a\\
<property>\\
    <name>dfs.namenode.name.dir</name>\\
    <value>/usr/local/hadoop/data/nameNode</value>\\
</property>\\
<property>\\
    <name>dfs.datanode.data.dir</name>\\
    <value>/usr/local/hadoop/data/dataNode</value>\\
</property>\\
<property>\\
    <name>dfs.replication</name>\\
    <value>2</value>\\
</property>' /usr/local/hadoop/etc/hadoop/hdfs-site.xml
echo DONE !

echo EDITING workers
echo "
ubuntu4
ubuntu5" > /usr/local/hadoop/etc/hadoop/workers
echo DONE !

echo COPYING CONFIGURATION
# scp /usr/local/hadoop/etc/hadoop/* ubuntu2:/usr/local/hadoop/etc/hadoop/
# scp /usr/local/hadoop/etc/hadoop/* ubuntu3:/usr/local/hadoop/etc/hadoop/
scp /usr/local/hadoop/etc/hadoop/* ubuntu4:/usr/local/hadoop/etc/hadoop/
scp /usr/local/hadoop/etc/hadoop/* ubuntu5:/usr/local/hadoop/etc/hadoop/
echo DONE !

YARN 설정하는 스크립트

if [ $USER != "ubuntu1" ]; then
	echo current user is $USER
  echo Script must be run as user:su - ubuntu1
  echo run \\'su - ubuntu1\\' first
	echo then \\" ./yarn_setup.sh \\"
  exit -1
fi

echo SETTING CONF...
cd /usr/local/hadoop/etc/hadoop/
cp yarn-site.xml yarn-site.xml.original
sed -i '/<configuration>/s/$/\\
<property>\\
    <name>yarn.resourcemanager.hostname<\\/name>\\
    <value>'$HOSTNAME'<\\/value>\\
<\\/property>/' yarn-site.xml
# scp /usr/local/hadoop/etc/hadoop/yarn-site.xml ubuntu2:/usr/local/hadoop/etc/hadoop/
# scp /usr/local/hadoop/etc/hadoop/yarn-site.xml ubuntu3:/usr/local/hadoop/etc/hadoop/
scp /usr/local/hadoop/etc/hadoop/yarn-site.xml ubuntu4:/usr/local/hadoop/etc/hadoop/
scp /usr/local/hadoop/etc/hadoop/yarn-site.xml ubuntu5:/usr/local/hadoop/etc/hadoop/
cp yarn-site.xml.original yarn-site.xml
cd
echo DONE !

네임노드랑 데이터노드 포맷하는 스크립트

clean_data.sh

hdfs namenode -format -y
#sshpass -p "R2020hjp" ssh ubuntu1 'echo R2020hjp | sudo -S -s /bin/bash -c "rm -rf /usr/local/hadoop/logs /usr/local/hadoop/data"'
#sshpass -p "R2020hjp" ssh ubuntu2 'echo R2020hjp | sudo -S -s /bin/bash -c "rm -rf /usr/local/hadoop/logs /usr/local/hadoop/data"'
#sshpass -p "R2020hjp" ssh ubuntu3 'echo R2020hjp | sudo -S -s /bin/bash -c "rm -rf /usr/local/hadoop/logs /usr/local/hadoop/data"'
sshpass -p "R2020hjp" ssh ubuntu4 'echo R2020hjp | sudo -S -s /bin/bash -c "rm -rf /usr/local/hadoop/logs /usr/local/hadoop/data"'
sshpass -p "R2020hjp" ssh ubuntu5 'echo R2020hjp | sudo -S -s /bin/bash -c "rm -rf /usr/local/hadoop/logs /usr/local/hadoop/data"'

R설치 (모든 노드), 스크립트 아님, 단계별 실행필요

su - ubuntu1

cd /usr/local/src
sudo wget <https://cran.rstudio.com/src/base/R-4/R-4.0.2.tar.gz>
sudo apt-get update && sudo apt-get install -y gfortran libreadline6-dev libx11-dev libxt-dev \\
                               libpng-dev libjpeg-dev libcairo2-dev xvfb \\
                               libbz2-dev libzstd-dev liblzma-dev \\
                               libcurl4-openssl-dev \\
                               texinfo texlive texlive-fonts-extra \\
                               screen libpcre2-dev \\
                               g++ make

sudo su

sudo su
tar zxvf R-4.0.2.tar.gz
cd R-4.0.2
./configure --enable-R-shlib && make && make install
cd ..
rm -rf R-4.0.2*
exit
cd

Rhadoop 의존 패키지 설치(모든 노드)