Apache Ranger中Yarn插件使用問題

使用ranger可以控制用戶提交作業。在搭建yarn插件之後，建立新的service的時候，填寫入‘YARN REST URL ’後，點擊test connection，出現了問題，頁面彈出框提示無法通過該url獲得隊列，查看ranger-admin運行日誌也沒有報錯。翻看底層代碼後發現，ranger在對yarn的支持上只支持了capacity調度，對fair調度沒有支持。
當將運行中的集羣的調度改爲capacity調度後，test connection成功。根本原因就是，resourcemanager的rest api http://rm:port/ws/v1/cluster/scheduler在不同的調度器下，得到的json文件不同，由於ranger底層只解析了capacity調度的json文件，沒有解析fair調度的，所以出現了上述問題。
在ranger頁面中點擊test connection之後，對應的底層代碼在YarnClient這個類中，其中的getQueuelist()方法中，通過Gson類中的fromJson(string,class)方法，來將string類型的json字符串序列化爲某個class類的實例，這樣就可以通過面向對象的方式獲得json字符中的各項值。fromJson方法得到的值賦給了YarnSchdulerResponse對象的實例，我們需要修改的就是這個類。在其中添加針對fair調度生成的json文件的解析。出問題的代碼片段

Gson gson = new GsonBuilder().setPrettyPrinting().create();
YarnSchedulerResponse yarnQResponse = gson.fromJson(jsonString, YarnSchedulerResponse.class);
if (yarnQResponse != null) {
        List<String>  yarnQueueList = yarnQResponse.getQueueNames();
if (yarnQueueList != null) {
    for ( String yarnQueueName : yarnQueueList) {
        if ( existingQueueList != null && existingQueueList.contains(yarnQueueName)) {
            continue;
        }
        if (queueNameMatching == null || queueNameMatching.isEmpty()
            || yarnQueueName.startsWith(queueNameMatching)) {
        if (LOG.isDebugEnabled()
        ......                                                         lret.add(yarnQueueName);
}
}
}

問題就處在yarnQueueList這個實例中，在fair調度下，這個List爲空無任何值，也就是沒有拿到任何隊列。
原YarnSchedulerResponse代碼就不貼了，貼上修改過後的整個類的代碼，也很簡單。

import java.util.ArrayList;
import java.util.List;

import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;

import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;

public class YarnSchedulerResponse implements java.io.Serializable{

    private static final long serialVersionUID = 1L;

    private YarnScheduler scheduler = null;

    public YarnScheduler getScheduler() { return scheduler; }

    public List<String> getQueueNames() {
        List<String> ret = new ArrayList<String>();

        if(scheduler != null) {
            scheduler.collectQueueNames(ret);
        }

        return ret;
    }

    @JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY)
    @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL )
    @JsonIgnoreProperties(ignoreUnknown=true)
    @XmlRootElement
    @XmlAccessorType(XmlAccessType.FIELD)
    public static class YarnScheduler implements java.io.Serializable {
        private static final long serialVersionUID = 1L;

        private YarnSchedulerInfo schedulerInfo = null;

        public YarnSchedulerInfo getSchedulerInfo() { return schedulerInfo; }

        public void collectQueueNames(List<String> queueNames) {
            if(schedulerInfo != null) {
                schedulerInfo.collectQueueNames(queueNames,null);
            }
        }
    }

    @JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY)
    @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL )
    @JsonIgnoreProperties(ignoreUnknown=true)
    @XmlRootElement
    @XmlAccessorType(XmlAccessType.FIELD)
    public static class YarnSchedulerInfo implements java.io.Serializable {
        private static final long serialVersionUID = 1L;

        private String type = null;
        public String getType() {return type;}

        private YarnRootQueues rootQueue = null;
        public YarnRootQueues getRootQueue() { return rootQueue; }

        private String     queueName = null;
        private YarnQueues queues    = null;
        public String getQueueName() { return queueName; }
        public YarnQueues getQueues() { return queues; }

        public void collectQueueNames(List<String> queueNames,String parentQueueName) {
            if(type != null && type.equals("fairScheduler")){

                if(rootQueue != null) {
                    rootQueue.collectQueueNames(queueNames);
                } 
            }

            else if(type != null && type.equals("capacityScheduler") || type.equals("capacitySchedulerLeafQueueInfo")){

                if(queueName != null) {
                    String queueFqdn = parentQueueName == null ? queueName : parentQueueName + "." + queueName;

                    queueNames.add(queueFqdn);

                    if(queues != null) {
                        queues.collectQueueNames(queueNames, queueFqdn);
                    }
                } 
            }

            else{
                try {
                    throw new Exception("no Schduler");
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }

    @JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY)
    @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL )
    @JsonIgnoreProperties(ignoreUnknown=true)
    @XmlRootElement
    @XmlAccessorType(XmlAccessType.FIELD)
    public static class YarnRootQueues implements java.io.Serializable {
        private static final long serialVersionUID = 1L;

        private String queueName = null;
        private List<YarnRootQueues> childQueues = null;

        public String getQueueName() { return queueName; }

        public List<YarnRootQueues> getChildQueues() { return childQueues; }

        public void collectQueueNames(List<String> queueNames) {
            if(queueName != null){
               queueNames.add(queueName);

               if(childQueues != null){
                 for(int i = 0;i < childQueues.size();i++)
                 queueNames.add(childQueues.get(i).queueName);
               }
            }
        }
    }

    @JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY)
    @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL )
    @JsonIgnoreProperties(ignoreUnknown=true)
    @XmlRootElement
    @XmlAccessorType(XmlAccessType.FIELD)
    public static class YarnQueues implements java.io.Serializable {
        private static final long serialVersionUID = 1L;

        private List<YarnSchedulerInfo> queue = null;

        public List<YarnSchedulerInfo> getQueue() { return queue; }

        public void collectQueueNames(List<String> queueNames, String parentQueueName) {
            if(queue != null) {
                for(YarnSchedulerInfo schedulerInfo : queue) {
                    schedulerInfo.collectQueueNames(queueNames, parentQueueName);
                }
            }
        }
    }

}