import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.client.CookieStore;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.*;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.cookie.Cookie;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HTTP;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;
import java.io.*;
import java.math.BigDecimal;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
/**
* jupyter-api
* https://github.com/jupyter/jupyter/wiki/Jupyter-Notebook-Server-API#Notebook-and-file-contents-API
*/
public class JupyterApiUtil {
private static final Logger LOGGER = LoggerFactory.getLogger(HttpUtils.class);
private static final String HOST = "http://";
/**
* 登錄URL
*/
private static final String LOGIN_URL = HOST + "/hub/login?next=";
/**
* 文件操作
*/
private static final String CREATE_URL_PREFIX = "%s/user/%s/api/contents/%s";
public static void main(String[] args) throws IOException, InterruptedException {
String username = "tom";
String password = "123456";
//CookieStore cookieStore = JupyterApiUtil.login(username, password);
String token = "b5829414eb81473082bd1efe0c141c2f";
// System.out.println(token = getToken(cookieStore, username));
if (spawner(username, token) != null) {
// System.out.println(mkdir(username, "ccc/g", token, "new_tom"));
// System.out.println(listPath(username, token, "ccc/g"));
// System.out.println(delete(username, token, "ccc/Untitled Folder"));
// System.out.println(download(username, token, "aaa/hello-ar2.apk"));
long start =System.currentTimeMillis();
System.out.println(upload(username, token, "aaa/aaa111/new.txt", new File("d:\\a.csv")));//136M
System.out.println("time="+(System.currentTimeMillis()-start)/(1000)+"秒");//28s
}
}
/**
* login
*/
public static CookieStore login(String username, String password) {
final HttpPost httpPost = new HttpPost(LOGIN_URL);
List postParameters = new ArrayList<NameValuePair>();
postParameters.add(new BasicNameValuePair("username", username));
postParameters.add(new BasicNameValuePair("password", password));
CloseableHttpClient httpClient = null;
try {
httpPost.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8"));
httpClient = HttpUtils.getHttpClient();
HttpClientContext context = new HttpClientContext();
CloseableHttpResponse response = httpClient.execute(httpPost, context);
CookieStore cookieStore = context.getCookieStore();
return checkStatus(response) && !CollectionUtils.isEmpty(cookieStore.getCookies()) ? cookieStore : null;
} catch (Exception e) {
LOGGER.error("JupyterApiUtil.login", e);
}
return null;
}
/**
* getToken
*/
private static String getToken(CookieStore cookieStore, String username) {
final String token_url_prefix = "%s/hub/api/users/%s/tokens";
String getTokenUrl = String.format(token_url_prefix, HOST, username);
final HttpPost tokenPost = new HttpPost(getTokenUrl);
tokenPost.setHeader("Referer", HOST + "/hub/token");
tokenPost.setHeader("Content-type", "application/json");
tokenPost.setHeader("content-security-policy", "frame-ancestors 'self'; report-uri /hub/security/csp-report; default-src 'none'");
JSONObject jsonObject = new JSONObject();
jsonObject.put("note", "Requested via token page");
StringEntity requestParam = null;
try {
requestParam = new StringEntity(jsonObject.toJSONString());
tokenPost.setEntity(requestParam);
CloseableHttpClient httpClient = HttpUtils.getHttpClient();
HttpClientContext context = new HttpClientContext();
context.setCookieStore(cookieStore);
CloseableHttpResponse response = httpClient.execute(tokenPost, context);
JSONObject json = JSONObject.parseObject(EntityUtils.toString(response.getEntity()));
String token = json.getString("token");
return token;
} catch (Exception e) {
LOGGER.error("JupyterApiUtil.getToken", e);
}
return null;
}
/**
* Swpan
*/
public static String spawner(String userName, String token) {
final String spawner_url_prefix = "%s/hub/spawn/%s";
String spawner_url = String.format(spawner_url_prefix, HOST, userName);
CloseableHttpClient httpClient = HttpUtils.getHttpClient();
final HttpGet httpGetContent = new HttpGet(spawner_url);
httpGetContent.setHeader("Authorization", String.format("token %s", token));
CloseableHttpResponse response = null;
try {
HttpClientContext context = new HttpClientContext();
response = httpClient.execute(httpGetContent, context);
if (checkStatus(response) && context.getCookieStore() != null) {
String _xsrf = null;
for (Cookie cookie : context.getCookieStore().getCookies()) {
if ("_xsrf".equals(cookie.getName())) {
return cookie.getValue();
}
}
}
return null;
} catch (IOException e) {
LOGGER.error("JupyterApiUtil.spawner", e);
}
return null;
}
/**
* list-path
*/
public static JSONObject listPath(String userName, String token, String absPath) {
CloseableHttpClient httpClient = HttpUtils.getHttpClient();
try {
String getUrl = String.format(CREATE_URL_PREFIX, HOST, userName, absPath);
final HttpGet get = new HttpGet(getUrl);
get.setHeader("Authorization", String.format("token %s", token));
CloseableHttpResponse response = httpClient.execute(get);
if (checkStatus(response)) {
return JSON.parseObject(EntityUtils.toString(response.getEntity()));
}
} catch (IOException e) {
LOGGER.error("JupyterApiUtil.delete", e);
}
return null;
}
/**
* @param userName
* @param token
* @param absPath (預)文件名
*/
public static boolean upload(String userName, String token, String absPath, File file) {
boolean uploadFull = false;
CloseableHttpClient httpClient = HttpUtils.getHttpClient();
try {
String tagetFileName = absPath.substring(absPath.lastIndexOf("/") + 1);
String path = checkPath(absPath);
String uploadUrl = String.format(CREATE_URL_PREFIX, HOST, userName, path);
FileInputStream fis = new FileInputStream(file);
FileChannel fc = fis.getChannel();
int bufferSize = 1024 * 1024 * 8;
ByteBuffer buf = ByteBuffer.allocate(bufferSize);
int count = (int) Math.ceil(file.length() / (double) bufferSize);
for (int chunk = 1; chunk <= count; chunk++) {
BigDecimal bg = new BigDecimal((double)(chunk/Double.valueOf(count)));
double f1 = bg.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue();
System.out.println("============"+f1*100 +"%");
fc.read(buf);
buf.flip();
byte[] bytes = new byte[buf.remaining()];
buf.get(bytes);
String content = Base64.getEncoder().encodeToString(bytes);
buf.clear();
final HttpPut put = new HttpPut(uploadUrl);
JSONObject jsonObject = new JSONObject();
jsonObject.put("type", "file");
jsonObject.put("format", "base64");
jsonObject.put("name", tagetFileName);
jsonObject.put("chunk", chunk);
if (chunk == count) {
jsonObject.put("chunk", -1);
}
jsonObject.put("content", content);
jsonObject.put("path", absPath);
StringEntity requestParam = new StringEntity(jsonObject.toJSONString());
put.setEntity(requestParam);
put.setHeader("Authorization", String.format("token %s", token));
CloseableHttpResponse response = httpClient.execute(put);
if (checkStatus(response)) {
if (chunk == count) {
uploadFull = true;
break;
}
} else {
LOGGER.error("JupyterApiUtil.http.upload.fail");
break;
}
}
} catch (IOException e) {
LOGGER.error("JupyterApiUtil.upload", e);
}
return uploadFull;
}
/**
* download
*/
public static InputStream download(String userName, String token, String absPath) {
CloseableHttpClient httpClient = HttpUtils.getHttpClient();
try {
String _xsrf = spawner(userName, token);
_xsrf = checkPath(_xsrf);
String download_url = String.format("%s/user/%s/files/%s?_xsrf=%s", HOST, userName, absPath, _xsrf);
System.out.println(download_url);
final HttpGet get = new HttpGet(download_url);
CloseableHttpResponse response = httpClient.execute(get);
if (checkStatus(response)) {
return response.getEntity().getContent();
}
} catch (Exception e) {
LOGGER.error("JupyterApiUtil.download", e);
}
return null;
}
/**
* @param userName
* @param token
* @param absPath 絕對路徑
*/
public static boolean delete(String userName, String token, String absPath) {
CloseableHttpClient httpClient = HttpUtils.getHttpClient();
try {
absPath = checkPath(absPath);
String deleteUrl = String.format(CREATE_URL_PREFIX, HOST, userName, absPath);
final HttpDelete delete = new HttpDelete(deleteUrl);
delete.setHeader("Authorization", String.format("token %s", token));
CloseableHttpResponse response = httpClient.execute(delete);
if (!checkStatus(response)) {
return false;
}
return true;
} catch (IOException e) {
LOGGER.error("JupyterApiUtil.delete", e);
}
return false;
}
/**
* @param userName 用戶名
* @param parentPath 父目錄
* @param token
* @param targetDirName 創建目錄名稱
*/
public static boolean mkdir(String userName, String parentPath, String token, String targetDirName) {
/*預創建臨時目錄*/
/*修改目錄名稱*/
String pre_create_url = String.format(CREATE_URL_PREFIX, HOST, userName, parentPath);
final HttpPost post = new HttpPost(pre_create_url);
JSONObject jsonObject = new JSONObject();
jsonObject.put("path", parentPath);
jsonObject.put("type", "directory");
CloseableHttpClient httpClient = HttpUtils.getHttpClient();
try {
StringEntity requestParam = new StringEntity(jsonObject.toJSONString());
post.setEntity(requestParam);
post.setHeader("Authorization", String.format("token %s", token));
//step1
CloseableHttpResponse createTmpDirResp = httpClient.execute(post);
String preDirPath = null;
String respString = EntityUtils.toString(createTmpDirResp.getEntity());
JSONObject data = JSON.parseObject(respString);
if (!checkStatus(createTmpDirResp) || data == null || (preDirPath = data.getString("path")) == null) {
return false;
}
preDirPath = checkPath(preDirPath);
//step2
String pre_rename_url = String.format(CREATE_URL_PREFIX, HOST, userName, preDirPath);
final HttpPatch patch = new HttpPatch(pre_rename_url);
jsonObject = new JSONObject();
jsonObject.put("path", parentPath + "/" + targetDirName);
requestParam = new StringEntity(jsonObject.toJSONString(), HTTP.UTF_8);
patch.setEntity(requestParam);
patch.setHeader("Authorization", String.format("token %s", token));
CloseableHttpResponse renameResp = httpClient.execute(patch);
respString = EntityUtils.toString(renameResp.getEntity());
data = JSON.parseObject(respString);
String error = null;
if (!checkStatus(createTmpDirResp) || data == null || (error = data.getString("message")) != null) {
if (error != null) {
LOGGER.error("JupyterApiUtil.mkdir", error);
}
return false;
}
return true;
} catch (Exception e) {
LOGGER.error("JupyterApiUtil.mkdir", e);
}
return false;
}
private static String checkPath(String preDirPath) throws UnsupportedEncodingException {
return URLEncoder.encode(preDirPath, "UTF-8").replaceAll("\\+", "%20");
}
public static boolean checkStatus(CloseableHttpResponse response) {
int code = response.getStatusLine().getStatusCode();
return HttpStatus.SC_OK == code || HttpStatus.SC_CREATED == code || HttpStatus.SC_NO_CONTENT == code;
}
}
