平時任務實在比較忙,只能先記錄簡單步驟,後續再寫詳細說明。
win7下eclipse中開發統一局域網內hbase程序簡單操作步驟
win7下管理統一局域網內的集羣
1.3臺ubuntu服務器作爲實驗機器,一臺win7作爲管理機器,網線都在同一個交換機上。
2.3臺主機關閉防火牆,並開通ssh服務。
3.在win7上安裝xshell 用來進行命令操作3臺主機。安裝winscp來進行文件上傳等操作。
ubuntu和win7安裝配置
一、運行平臺
hbase服務器端:Ubuntu 14.04 64位;Hadoop2.8.1;HBASE1.3.1;JAVA 1.8;
hbase客服端:windows64位;JAVA1.8;eclipse客戶端 ;
二、linux服務器端環境配置
1、 安裝java 1.8軟件(略)
1)下載java軟件
2)安裝java 8
3)驗證安裝的java版本
java –version #執行後會輸出java版本信息
3、 安裝配置Hadoop
1)下載
2)安裝和配置安裝參考
3、 安裝配置HBASE
1)下載hbase-1.3.1
2)解壓
3)配置hbase分佈式安裝參考
到此,hbase服務器端的配置已完成。
三、windows客服端配置
1、下載安裝java 1.8並且配置好環境變量。
2、下載安裝eclipse,最好爲最新版。
3、客戶端java程序設置
1)找到運行hbase程序所需要的jar包,我通過pom.xml文件導入
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>storm_hbase</groupId>
<artifactId>lab</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>lab</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<storm.version>1.0.1</storm.version>
<!-- 開發調試時配置爲compile,topology打包時配置爲provided -->
<storm.scope>provided</storm.scope>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
<version>1.7</version>
<scope>system</scope>
<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${storm.version}</version>
<scope>${storm.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.2.2</version>
</dependency>
</dependencies>
</project>
2)將hbase文件下conf文件下的hbase-site.xml文件拷貝一份,放入爲其單獨建一個文件夾中。
3)將hadoop和hbase下的配置文件複製到項目下自己建立的config文件裏,並且導入。
5)Windows下hosts文件
在win7下C:\Windows\System32\drivers\etc中找到hosts文件,添加集羣的hostname解析。其中,前半部分表示IP地址,後半部分表示機器名字,根據實際情況進行修改。 這裏不要寫錯,可以先通過ping命令測試是否eclipse能和集羣聯通。
6)測試代碼:首先通過hbase shell建立表和列族結構,然後運行程序,注意程序中的表名要和自己建立的表名一致。
在hbase shell中可以看到,程序計算結果已經寫入。
最後貼出代碼,代碼不是自己寫的,也是網上的大神寫的,可以根據需要進行改動。
topo
package storm_hbase.lab;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.StormSubmitter;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
public class PersistentWordCount {
private static final String WORD_SPOUT = "WORD_SPOUT";
private static final String COUNT_BOLT = "COUNT_BOLT";
private static final String HBASE_BOLT = "HBASE_BOLT";
public static void main(String[] args) throws Exception {
// System.setProperty("hadoop.home.dir","E:/BaiduYunDownload");
Config config = new Config();
WordSpout spout = new WordSpout();
WordCounter bolt = new WordCounter();
MyHBaseBolt hbase = new MyHBaseBolt();
// wordSpout ==> countBolt ==> HBaseBolt
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(WORD_SPOUT, spout, 1);
builder.setBolt(COUNT_BOLT, bolt, 1).shuffleGrouping(WORD_SPOUT);
builder.setBolt(HBASE_BOLT, hbase, 10).fieldsGrouping(COUNT_BOLT, new Fields("word"));
if (args.length == 0) {
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("word", config, builder.createTopology());
Thread.sleep(10000);
cluster.killTopology("word");
cluster.shutdown();
System.exit(0);
} else {
config.setNumWorkers(3);
StormSubmitter.submitTopology(args[0], config, builder.createTopology());
}
}}
spout
package storm_hbase.lab;
import java.util.Map;
import java.util.Random;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
public class WordSpout extends BaseRichSpout {
private SpoutOutputCollector collector;
private static final String[] MSGS = new String[]{
"Storm", "HBase", "Integration", "example", "by ", "aloo", "in", "Aug",
};
private static final Random random = new Random();
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word"));
}
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
this.collector = collector;
}
public void nextTuple() {
String word = MSGS[random.nextInt(8)];
collector.emit(new Values(word));
System.out.println(word);
}
}
wordcountbolt
package storm_hbase.lab;
import java.util.HashMap;
import java.util.Map;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
public class WordCounter extends BaseBasicBolt {
private Map<String, Integer> _counts = new HashMap<String, Integer>();
public void execute(Tuple tuple, BasicOutputCollector collector) {
String word = tuple.getString(0);
int count;
if(_counts.containsKey(word)){
count = _counts.get(word);
} else {
count = 0;
}
count ++;
_counts.put(word, count);
collector.emit(new Values(word, count));
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word", "count"));
}
}
hbasebolt
package storm_hbase.lab;
import java.io.IOException;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Tuple;
public class MyHBaseBolt extends BaseBasicBolt {
private Connection connection;
private Table table;
public void prepare(Map stormConf, TopologyContext context) {
Configuration config = HBaseConfiguration.create();
//
// HBaseAdmin admin = new HBaseAdmin(config);
// admin.createTable("WordCount");
try {
connection = ConnectionFactory.createConnection(config);
//示例都是對同一個table進行操作,因此直接將Table對象的創建放在了prepare,在bolt執行過程中可以直接重用。
table = connection.getTable(TableName.valueOf("wordcount"));
} catch (IOException e) {
//do something to handle exception
}
}
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
//從tuple中獲取單詞
String word = tuple.getString(0);
//從tuple中獲取計數,這裏轉換爲String只是爲了示例運行後存入hbase的計數值能夠直觀顯示。
String count = tuple.getInteger(1).toString();
try {
//以各個單詞作爲row key
System.out.println(word);
Put put = new Put(Bytes.toBytes(word));
//將被計數的單詞寫入cf:words列
put.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("words"), Bytes.toBytes(word));
//將單詞的計數寫入cf:counts列
put.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("counts"), Bytes.toBytes(count));
table.put(put);
} catch (IOException e) {
//do something to handle exception
}
}
@Override
public void cleanup() {
//關閉table
try {
if(table != null) table.close();
} catch (Exception e){
//do something to handle exception
} finally {
//在finally中關閉connection
try {
connection.close();
} catch (IOException e) {
//do something to handle exception
}
}
}
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
//示例中本bolt不向外發射數據,所以沒有再做聲明
}
}