一、題記
Apache kylin 提供了一個非常好用的 Web 界面，通過 web 界面可以將 hive 表裏面的數據同步到 kylin 中。基於 Hive 表構建 model（模型）、cube（多維立方體）。通過配置 Mandatory Dimensions（必要維度）、Hierarchy Dimensions（層級維度）、Joint Dimensions（聯合維度）減少 Cube 的數據量。
二、使用web操作頁面
1、訪問 & 登陸
默認賬號：ADMIN 密碼：KYLIN
2、Kylin 中可用的 Hive 表
雖然 Kylin 使用 SQL 作爲查詢接口並利用 Hive 元數據，Kylin 不會讓用戶查詢所有的 hive 表，因爲到目前爲止它是一個預構建 OLAP (MOLAP) 系統。爲了使表在 Kylin 中可用，使用 “Sync” 方法能夠方便地從 Hive 中同步表。
3、Kylin OLAP Cube
Kylin 的 OLAP Cube 是從星型模式的 Hive 表中獲取的預計算數據集，這是供用戶探索、管理所有 cube 的網頁管理頁面。由菜單欄進入 Model 頁面，系統中所有可用的 cube 將被列出。
4、在網頁上編寫和運行 SQL
Kylin 的網頁版爲用戶提供了一個簡單的查詢工具來運行 SQL 以探索現存的 cube，驗證結果並探索使用下一章中的 Pivot analysis 與可視化的結果集。
5、模型 (model) 構建
kyin 目前只支持星型模型，雪花性模型可以通過預關聯成 hive 寬表的方式轉換成星型模型。
三、JAVA應用
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import org.apache.commons.codec.binary.Base64;
/**
*
* http://kylin.apache.org/docs15/
*
*/
public class KylinHttpUtils {
private static String encoding;
private static final String baseURL = "http://127.0.0.1:7070/kylin/api";
public static String login(String user,String passwd){
String method = "POST";
String para = "/user/authentication";
byte[] key = (user+":"+passwd).getBytes();
encoding = Base64.encodeBase64String(key);
return excute(para,method,null);
}
public static String listQueryableTables(String projectName){
String method = "GET";
String para = "/tables_and_columns?project="+projectName;
return excute(para,method,null);
}
/**
*
* @param offset required int Offset used by pagination
* @param limit required int Cubes per page.
* @param cubeName optional string Keyword for cube names. To find cubes whose name contains this keyword.
* @param projectName optional string Project name.
* @return
*/
public static String listCubes(int offset,
int limit,
String cubeName,
String projectName ){
String method = "GET";
String para = "/cubes?offset="+offset
+"&limit="+limit
+"&cubeName="+cubeName
+"&projectName="+projectName;
return excute(para,method,null);
}
/**
*
* @param cubeName Cube name.
* @return
*/
public static String getCubeDes(String cubeName){
String method = "GET";
String para = "/cube_desc/"+cubeName;
return excute(para,method,null);
}
/**
*
* @param cubeName
* @return
*/
public static String getCube(String cubeName){
String method = "GET";
String para = "/cubes/"+cubeName;
return excute(para,method,null);
}
/**
*
* @param modelName Data model name, by default it should be the same with cube name.
* @return
*/
public static String getDataModel(String modelName){
String method = "GET";
String para = "/model/"+modelName;
return excute(para,method,null);
}
/**
*
* @param cubeName cubeName Cube name.
* @return
*/
public static String enableCube(String cubeName){
String method = "PUT";
String para = "/cubes/"+cubeName+"/enable";
return excute(para,method,null);
}
/**
*
* @param cubeName Cube name.
* @return
*/
public static String disableCube(String cubeName){
String method = "PUT";
String para = "/cubes/"+cubeName+"/disable";
return excute(para,method,null);
}
/**
*
* @param cubeName Cube name.
* @return
*/
public static String purgeCube(String cubeName){
String method = "PUT";
String para = "/cubes/"+cubeName+"/purge";
return excute(para,method,null);
}
/**
*
* @param jobId Job id
* @return
*/
public static String resumeJob(String jobId){
String method = "PUT";
String para = "/jobs/"+jobId+"/resume";
return excute(para,method,null);
}
/**
* startTime - required long Start timestamp of data to build, e.g. 1388563200000 for 2014-1-1
* endTime - required long End timestamp of data to build
* buildType - required string Supported build type: ¡®BUILD¡¯, ¡®MERGE¡¯, ¡®REFRESH¡¯
* @param cubeName Cube name.
* @return
*/
public static String buildCube(String cubeName,String body){
String method = "PUT";
String para = "/cubes/"+cubeName+"/rebuild";
return excute(para,method,body);
}
/**
*
* @param jobId Job id.
* @return
*/
public static String discardJob(String jobId){
String method = "PUT";
String para = "/jobs/"+jobId+"/cancel";
return excute(para,method,null);
}
/**
*
* @param jobId Job id.
* @return
*/
public static String getJobStatus(String jobId){
String method = "GET";
String para = "/jobs/"+jobId;
return excute(para,method,null);
}
/**
*
* @param jobId Job id.
* @param stepId Step id; the step id is composed by jobId with step sequence id;
* for example, the jobId is ¡°fb479e54-837f-49a2-b457-651fc50be110¡±, its 3rd step id
* is ¡°fb479e54-837f-49a2-b457-651fc50be110-3¡±,
* @return
*/
public static String getJobStepOutput(String jobId,String stepId){
String method = "GET";
String para = "/"+jobId+"/steps/"+stepId+"/output";
return excute(para,method,null);
}
/**
*
* @param tableName table name to find.
* @return
*/
public static String getHiveTable(String tableName){
String method = "GET";
String para = "/tables/"+tableName;
return excute(para,method,null);
}
/**
*
* @param tableName table name to find.
* @return
*/
public static String getHiveTableInfo(String tableName){
String method = "GET";
String para = "/tables/"+tableName+"/exd-map";
return excute(para,method,null);
}
/**
*
* @param projectName will list all tables in the project.
* @param extOptional boolean set true to get extend info of table.
* @return
*/
public static String getHiveTables(String projectName,boolean extOptional){
String method = "GET";
String para = "/tables?project="+projectName+"&ext="+extOptional;
return excute(para,method,null);
}
/**
*
* @param tables table names you want to load from hive, separated with comma.
* @param project the project which the tables will be loaded into.
* @return
*/
public static String loadHiveTables(String tables,String project){
String method = "POST";
String para = "/tables/"+tables+"/"+project;
return excute(para,method,null);
}
/**
*
* @param type ¡®METADATA¡¯ or ¡®CUBE¡¯
* @param name Cache key, e.g the cube name.
* @param action ¡®create¡¯, ¡®update¡¯ or ¡®drop¡¯
* @return
*/
public static String wipeCache(String type,String name,String action){
String method = "POST";
String para = "/cache/"+type+"/"+name+"/"+action;
return excute(para,method,null);
}
public static String query(String body){
String method = "POST";
String para = "/query";
return excute(para,method,body);
}
private static String excute(String para,String method,String body){
StringBuilder out = new StringBuilder();
try {
URL url = new URL(baseURL+para);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod(method);
connection.setDoOutput(true);
String value = "Basic " + encoding;
connection.setRequestProperty("Authorization", value);
connection.setRequestProperty("Content-Type","application/json");
if(body !=null){
byte[] outputInBytes = body.getBytes("UTF-8");
OutputStream os = connection.getOutputStream();
os.write(outputInBytes);
os.close();
}
InputStream content = (InputStream)connection.getInputStream();
BufferedReader in = new BufferedReader (new InputStreamReader (content));
String line;
while ((line = in.readLine()) != null) {
out.append(line);
}
in.close();
connection.disconnect();
} catch(Exception e) {
e.printStackTrace();
}
return out.toString();
}
}
public static void main(String[] args){
String output ;
KylinHttpBasic.login("ADMIN","KYLIN");
String body = "{\"sql\":\"select * from FACT_\",\"offset\":0,\"limit\":50000,\"acceptPartial\":false,\"project\":\"my_kylin\"}";
output = KylinHttpBasic.query(body);
System.out.println(output);
}