大家好，我是不溫卜火，是一名計算機學院大數據專業大二的學生，暱稱來源於成語—
不溫不火，本意是希望自己性情溫和。作爲一名互聯網行業的小白，博主寫博客一方面是爲了記錄自己的學習過程，另一方面是總結自己所犯的錯誤希望能夠幫助到很多和自己一樣處於起步階段的萌新。但由於水平有限，博客中難免會有一些錯誤出現，有紕漏之處懇請各位大佬不吝賜教！暫時只有csdn這一個平臺，博客主頁：https://buwenbuhuo.blog.csdn.net/
此篇爲大家帶來的是HBase項目實戰系列(1) | Weibo項目簡易版(附全代碼)。
標註：
此處爲反爬蟲標記：讀者可自行忽略
原文地址：https://buwenbuhuo.blog.csdn.net/
1. 需求分析
1.微博內容的瀏覽，數據庫表設計
2.用戶社交體現：關注用戶，取關用戶
3.拉取關注的人的微博內容
微博表的分析
- 1.
- 2. 項目所需要的表
![2]()
2. 代碼實現
1. 代碼設計總覽
// 一.創建
1.創建命名空間以及表名的定義
2.創建微博內容表
3.創建用戶關係表
4.創建用戶微博內容接收郵件表
// 二. 測試
5.發佈微博內容
6.添加關注用戶
7.移除（取關）用戶
8.獲取關注的人的微博內容
2.
2. 創建項目及添加依賴
- 創建項目結構如下圖
- 依賴
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.3.1</version>
</dependency>
</dependencies>
3. 創建
- 1. WeiboDao
package com.buwenbuhuo.hbase.weibo.dao;
import com.buwenbuhuo.hbase.weibo.constant.Names;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* @author 卜溫不火
* @create 2020-05-13 21:25
* com.buwenbuhuo.hbase.weibo.dao - the name of the target package where the new class or interface will be created.
* weibo0513 - the name of the current project.
*/
public class WeiboDao {
public static Connection connection = null;
static {
try {
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "hadoop002,hadoop003,hadoop004");
connection = ConnectionFactory.createConnection(conf);
}catch (IOException e){
e.printStackTrace();
}
}
public void createNamespace(String namespace) throws IOException {
Admin admin = connection.getAdmin();
NamespaceDescriptor namespce = NamespaceDescriptor.create(namespace).build();
admin.createNamespace(namespce);
admin.close();
}
public void createTable(String tableName, String... families) throws IOException {
// 因爲下面的存在，此處可以省略
createTable(tableName,1,families);
}
public void createTable(String tableName,Integer versions, String... families) throws IOException {
Admin admin = connection.getAdmin();
HTableDescriptor table = new HTableDescriptor(TableName.valueOf(tableName));
for (String family : families) {
HColumnDescriptor familyDesc = new HColumnDescriptor(family);
familyDesc.setMaxVersions(versions);
table.addFamily(familyDesc);
}
admin.createTable(table);
admin.close();
}
public void putCell(String tableName, String rowKey, String family, String column, String value) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Put put = new Put(Bytes.toBytes(rowKey));
put.addColumn(Bytes.toBytes(family),Bytes.toBytes(column),Bytes.toBytes(value));
table.put(put);
table.close();
}
public List<String> getRowKeysByPrefix(String tableName, String prefix) throws IOException {
ArrayList<String> list = new ArrayList<>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.setRowPrefixFilter(Bytes.toBytes(prefix));
ResultScanner scanner = table.getScanner(scan);
for (Result result : scanner) {
byte[] row = result.getRow();
String rowKey = Bytes.toString(row);
list.add(rowKey);
}
scanner.close();
table.close();
return list;
}
public void putCells(String tableName, List<String> rowKeys, String family, String column, String value) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
ArrayList<Put> puts = new ArrayList<>();
// 遍歷RowKeys
for (String rowKey : rowKeys) {
Put put = new Put(Bytes.toBytes(rowKey));
put.addColumn(Bytes.toBytes(family),Bytes.toBytes(column),Bytes.toBytes(value));
puts.add(put);
}
table.put(puts);
table.close();
}
public List<String> getRowKeysByRange(String tableName, String startRow, String stopRow) throws IOException {
List<String> list = new ArrayList<>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan(Bytes.toBytes(startRow), Bytes.toBytes(stopRow));
ResultScanner scanner = table.getScanner(scan);
for (Result result : scanner) {
byte[] row = result.getRow();
String rowKey = Bytes.toString(row);
list.add(rowKey);
}
scanner.close();
table.close();
return list;
}
public void deleteRow(String tableName, String rowKey) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Delete delete = new Delete(Bytes.toBytes(rowKey));
table.delete(delete);
table.close();
}
public void deleteCells(String tableName, String rowKey, String family, String column) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Delete delete = new Delete(Bytes.toBytes(rowKey));
delete.addColumns(Bytes.toBytes(family),Bytes.toBytes(column));
table.delete(delete);
table.close();
}
public List<String> getCellsByPrefix(String tableName, String prefix, String family, String column) throws IOException {
List<String> list = new ArrayList<>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.setRowPrefixFilter(Bytes.toBytes(prefix));
scan.addColumn(Bytes.toBytes(family),Bytes.toBytes(column));
ResultScanner scanner = table.getScanner(scan);
for (Result result : scanner) {
Cell[] cells = result.rawCells();
list.add(Bytes.toString(CellUtil.cloneValue(cells[0])));
}
scanner.close();
table.close();
return list;
}
public List<String> getFamilyByRowKey(String tableName, String rowKey, String family) throws IOException {
List<String> list = new ArrayList<>();
Table table = connection.getTable(TableName.valueOf(tableName));
Get get = new Get(Bytes.toBytes(rowKey));
get.setMaxVersions(Names.INBOX_DATA_VERSIONS);
get.addFamily(Bytes.toBytes(family));
Result result = table.get(get);
for (Cell cell : result.rawCells()) {
list.add(Bytes.toString(CellUtil.cloneValue(cell)));
}
table.close();
return list;
}
public List<String> getCellsByRowKey(String tableName, List<String> rowKeys, String family, String column) throws IOException {
List<String> weibos = new ArrayList<>();
Table table = connection.getTable(TableName.valueOf(tableName));
List<Get> gets = new ArrayList<>();
for (String rowKey : rowKeys) {
Get get = new Get(Bytes.toBytes(rowKey));
get.addColumn(Bytes.toBytes(family),Bytes.toBytes(column));
gets.add(get);
}
Result[] results = table.get(gets);
for (Result result : results) {
String weibo = Bytes.toString(CellUtil.cloneValue(result.rawCells()[0]));
weibos.add(weibo);
}
table.close();
return weibos;
}
}
- 2. WeiboService
package com.buwenbuhuo.hbase.weibo.service;
import com.buwenbuhuo.hbase.weibo.constant.Names;
import com.buwenbuhuo.hbase.weibo.dao.WeiboDao;
import javax.naming.Name;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* @author 卜溫不火
* @create 2020-05-13 21:27
* com.buwenbuhuo.hbase.weibo.service - the name of the target package where the new class or interface will be created.
* weibo0513 - the name of the current project.
*/
public class WeiboService {
private WeiboDao dao = new WeiboDao();
public void init() throws IOException {
//1) 創建命名空間以及表名的定義
dao.createNamespace(Names.NAMESPACE_WEIBO);
//2) 創建微博內容表
dao.createTable(Names.TABLE_WEIBO,Names.WEIBO_FAMILY_DATA);
//3) 創建用戶關係表
dao.createTable(Names.TABLE_RELATION,Names.RELATION_FAMILY_DATA);
//4) 創建用戶微博內容接收郵件表
dao.createTable(Names.TABLE_INBOX,Names.INBOX_DATA_VERSIONS,Names.INBOX_FAMILY_DATA);
}
public void publish(String star, String content) throws IOException {
// 1. 在weibo表中插入一條數據
String rowKey = star + "_" + System.currentTimeMillis();
dao.putCell(Names.TABLE_WEIBO,rowKey,Names.WEIBO_FAMILY_DATA,Names.WEIBO_COLUMN_CONTENT,content);
// 2. 從relation表中獲取star的所有fansID (默認有粉絲邏輯有些問題)
String prefix = star+":followedby:";
List<String> list = dao.getRowKeysByPrefix(Names.TABLE_RELATION,prefix);
if (list.size()<= 0){
return;
}
List<String> fansIds = new ArrayList<>();
// 遍歷
for (String row : list) {
String[] split = row.split(":");
// 獲取粉絲ID
fansIds.add(split[2]);
}
// 3. 向所有fans的inbox中插入本條weibo的id
// 循環調用 or 批量調用
dao.putCells(Names.TABLE_INBOX,fansIds,Names.INBOX_FAMILY_DATA,star,rowKey);
}
public void follow(String fans, String star) throws IOException {
// 1. 向relation表中插入兩條數據
String rowKey1 = fans + ":follow:" + star;
String rowKey2 = star + ":followedby:" + fans;
String time = System.currentTimeMillis() + "";
dao.putCell(Names.TABLE_RELATION,rowKey1,Names.RELATION_FAMILY_DATA,Names.RELATION_COLUMN_TIME,time);
dao.putCell(Names.TABLE_RELATION,rowKey2,Names.RELATION_FAMILY_DATA,Names.RELATION_COLUMN_TIME,time);
// 2. 從weibo表中獲取star的近期weibo
// 拿取所有
String startRow = star;
String stopRow = star + "|";
List<String> list = dao.getRowKeysByRange(Names.TABLE_WEIBO,startRow,stopRow);
// 判斷
if (list.size() <= 0){
return;
}
// 獲取近期的weibo
// 使用三元運算符進行判斷
int fromIndex = list.size() > Names.INBOX_DATA_VERSIONS?list.size()-Names.INBOX_DATA_VERSIONS:0;
List<String> recentWeiboIds = list.subList(fromIndex, list.size());
// 3. 向fans的inbox表中插入star的近期weiboId
for (String recentWeiboId : recentWeiboIds) {
dao.putCell(Names.TABLE_INBOX,fans, Names.INBOX_FAMILY_DATA,star,recentWeiboId);
}
}
public void unFollow(String fans, String star) throws IOException {
// 1. 刪除relation表中的兩條數據
String rowKey1 = fans + ":follow:" + star;
String rowKey2 = star + ":followedby:" + fans;
dao.deleteRow(Names.TABLE_RELATION,rowKey1);
dao.deleteRow(Names.TABLE_RELATION,rowKey2);
// 2. 刪除inbox表中的一列
dao.deleteCells(Names.TABLE_INBOX,fans,Names.INBOX_FAMILY_DATA,star);
}
public List<String> getAllWeiboByUserId(String star) throws IOException {
return dao.getCellsByPrefix(Names.TABLE_WEIBO,star,Names.WEIBO_FAMILY_DATA,Names.WEIBO_COLUMN_CONTENT);
}
public List<String> getAllRecentWeibos(String fans) throws IOException {
// 1. 從inbox中獲取fans的所有的star的近期weiboId
List<String> list = dao.getFamilyByRowKey(Names.TABLE_INBOX,fans,Names.INBOX_FAMILY_DATA);
// 2. 根據weiboID去weibo表中查詢內容
return dao.getCellsByRowKey(Names.TABLE_WEIBO,list,Names.WEIBO_FAMILY_DATA,Names.WEIBO_COLUMN_CONTENT);
}
}
- 3. Names
package com.buwenbuhuo.hbase.weibo.constant;
/**
* @author 卜溫不火
* @create 2020-05-13 23:28
* com.buwenbuhuo.hbase.weibo.constant - the name of the target package where the new class or interface will be created.
* weibo0513 - the name of the current project.
*/
public class Names {
public final static String NAMESPACE_WEIBO = "weibo";
public final static String TABLE_WEIBO = "weibo:weibo";
public final static String TABLE_RELATION = "weibo:relation";
public final static String TABLE_INBOX = "weibo:inbox";
public final static String WEIBO_FAMILY_DATA = "data";
public final static String RELATION_FAMILY_DATA = "data";
public final static String INBOX_FAMILY_DATA = "data";
public final static String WEIBO_COLUMN_CONTENT = "content";
public final static String RELATION_COLUMN_TIME = "time";
public final static Integer INBOX_DATA_VERSIONS = 3;
}
- 4. WeiboController
package com.buwenbuhuo.hbase.weibo.controller;
import com.buwenbuhuo.hbase.weibo.service.WeiboService;
import java.io.IOException;
import java.util.List;
/**
* @author 卜溫不火
* @create 2020-05-13 21:27
* com.buwenbuhuo.hbase.weibo.controller - the name of the target package where the new class or interface will be created.
* weibo0513 - the name of the current project.
*/
public class WeiboController {
private WeiboService service = new WeiboService();
public void init() throws IOException {
service.init();
}
//5) 發佈微博內容
public void publish(String star,String content) throws IOException {
service.publish(star,content);
}
//6) 添加關注用戶
public void follow(String fans,String star) throws IOException {
service.follow(fans,star);
}
//7) 移除（取關）用戶
public void unFollow(String fans,String star) throws IOException {
service.unFollow(fans,star);
}
//8) 獲取關注的人的微博內容
// 8.1 獲取某個明星的所有weibo
public List<String> getAllWeibosByUserID(String star) throws IOException {
return service.getAllWeiboByUserId(star);
}
// 8.2 獲取關注的所有star的近期weibo
public List<String> getAllRecentWeibos(String fans) throws IOException {
return service.getAllRecentWeibos(fans);
}
}
- 5. WeiboAPP
package com.buwenbuhuo.hbase.weibo;
import com.buwenbuhuo.hbase.weibo.controller.WeiboController;
import java.io.IOException;
import java.util.List;
/**
* @author 卜溫不火
* @create 2020-05-13 21:24
* com.buwenbuhuo.hbase.weibo - the name of the target package where the new class or interface will be created.
* weibo0513 - the name of the current project.
*/
public class WeiboAPP {
private static WeiboController controller = new WeiboController();
public static void main(String[] args) throws IOException {
// 1. 創建表的初始化
// controller.init();
// 2. 發微博(發五條微博)
// controller.publish("buwenbuhuo","Happy 1");
// controller.publish("buwenbuhuo","Happy 2");
// controller.publish("buwenbuhuo","Happy 3");
// controller.publish("buwenbuhuo","Happy 4");
// controller.publish("buwenbuhuo","Happy 5");
// 3. 關注微博
// controller.follow("1002","buwenbuhuo");
// controller.follow("1003","buwenbuhuo");
// 4. 獲取微博內容
// 最新的消息(獲取)
// List<String> allRecentWeibos = controller.getAllRecentWeibos("1002");
// 查看數據
// for (String allRecentWeibo : allRecentWeibos) {
// System.out.println(allRecentWeibo);
// }
// 5. 取關微博
// controller.unFollow("1002","buwenbuhuo");
// 查看數據
// for (String allRecentWeibo : allRecentWeibos) {
// System.out.println(allRecentWeibo);
// }
// 6. 獲取某一個人的所有微博
// List<String> allWeibosByUserID = controller.getAllWeibosByUserID("buwenbuhuo");
//
// for (String s : allWeibosByUserID) {
// System.out.println(s);
// }
}
}
4. 測試
- 1. 建表的初始化
// 1. 創建表的初始化
controller.init();
- 2. 發微博
// 2. 發微博(發五條微博)
controller.publish("buwenbuhuo","Happy 1");
controller.publish("buwenbuhuo","Happy 2");
controller.publish("buwenbuhuo","Happy 3");
controller.publish("buwenbuhuo","Happy 4");
controller.publish("buwenbuhuo","Happy 5");
// 查看weibo
hbase(main):002:0> scan 'weibo:weibo'
- 3. 關注微博
// 3. 關注微博
controller.follow("buwen","buwenbuhuo");
controller.follow("buhuo","buwenbuhuo");
// 查看是否關注
hbase(main):004:0> scan 'weibo:relation'
- 4. 獲取微博內容
// 最新的消息(獲取)
List<String> allRecentWeibos = controller.getAllRecentWeibos("1002");
// 查看數據
for (String allRecentWeibo : allRecentWeibos) {
System.out.println(allRecentWeibo);
}
- 5. 取關微博
controller.unFollow("1002","buwenbuhuo");
List<String> allRecentWeibos = controller.getAllRecentWeibos("1002");
// 查看數據
for (String allRecentWeibo : allRecentWeibos) {
System.out.println(allRecentWeibo);
}
hbase(main):014:0> scan 'weibo:relation'
- 6. 獲取所有人的微博
// 6. 獲取某一個人的所有微博
List<String> allWeibosByUserID = controller.getAllWeibosByUserID("buwenbuhuo");
for (String s : allWeibosByUserID) {
System.out.println(s);
}
本次的分享就到這裏了,
好書不厭讀百回，熟讀課思子自知。而我想要成爲全場最靚的仔，就必須堅持通過學習來獲取更多知識，用知識改變命運，用博客見證成長，用行動證明我在努力。
如果我的博客對你有幫助、如果你喜歡我的博客內容，請“點贊” “評論”“收藏”一鍵三連哦！聽說點讚的人運氣不會太差，每一天都會元氣滿滿呦！如果實在要白嫖的話，那祝你開心每一天，歡迎常來我博客看看。
碼字不易，大家的支持就是我堅持下去的動力。點贊後不要忘了關注我哦！
