Hadoop Replication Pipelining, Replica Placement Policy, Replication Rack Awareness

https://hadoopabcd.wordpress.com/2015/03/17/hdfs-file-blocks-distribution-in-datanodes/

Replication Rack Awareness

1. Default

    默認機架感知是基於腳本的。

        <property>
            <name>net.topology.node.switch.mapping.impl</name>                   
            <value>org.apache.hadoop.net.ScriptBasedMapping</value>
        </property>

        <!-- 配置腳本,默認爲空，說明所有節點都在同一機架。 -->
        <property>
            <name>net.topology.script.file.name</name>
            <value></value>
        </property>

2.創建機架感知類

package com.it18zhang.hadoop.rackaware;

        import org.apache.hadoop.net.DNSToSwitchMapping;

        import java.io.FileOutputStream;
        import java.util.ArrayList;
        import java.util.List;

        /**
         * 自定義機架類
         */
        public class MyDNSwitchMapping implements DNSToSwitchMapping{

            public List<String> resolve(List<String> names) {
                saveNames(names);
                List<String> list = new ArrayList<String>();
                for(String s: names){
                    list.add(extractIp(s)) ;
                }
                return list;
            }

            public void reloadCachedMappings() {
            }

            public void reloadCachedMappings(List<String> names) {
            }

            private String extractIp(String name){
                if(name.startsWith("s")){
                    String ip = name.substring(1) ;
                    int intIp = Integer.parseInt(ip) ;
                    if(intIp < 103){
                        return "/rack1/" + ip ;
                    }
                    else{
                        return "/rack2/" + ip;
                    }
                }
                //
                else if(name.contains(".")){
                    String ip = name.split("\\.")[3] ;
                    int intIp = Integer.parseInt(ip);
                    if (intIp < 103) {
                        return "/rack1/" + ip;
                    } else {
                        return "/rack2/" + ip;
                    }
                }
                return "/rack1/" + name ;
            }

            /**
             * 保存名稱到磁盤
             */
            private void saveNames(List<String> names){
                try {
                    FileOutputStream fos = new FileOutputStream("/home/centos/rack.log",true) ;
                    fos.write("===================\r\n".getBytes());
                    for(String s : names){
                        fos.write((s + "\r\n").getBytes());
                    }
                    fos.flush();
                    fos.close();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }

3.導出jar包

    添加artifacts -> 刪除依賴的jar

4.部署jar到hadoop集羣/soft/hadoop/share/hadoop/common/lib

5.配置core-site.xnml

        <property>
            <name>net.topology.node.switch.mapping.impl</name>
            <value>com.it18zhang.hadoop.rackaware.MyDNSwitchMapping</value>
        </property>

6.重啓集羣，在client上傳文件觀察存放副本情況。