如何查看Hadoop運行過程中產生日誌
hadoop的日誌主要是MapReduce程序,運行過程中,產生的一些數據日誌,除了系統的日誌外,還包含一些我們自己在測試時候,或者線上環境輸出的日誌,這部分日誌通常會被放在userlogs這個文件夾下面,我們可以在mapred-site.xml裏面配置運行日誌的輸出目錄,散仙測試文件內容如下:
<?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <!-- Put site-specific property overrides in this file. --> <configuration> <!-- jobtracker的master地址--> <property> <name>mapred.job.tracker</name> <value>192.168.75.130:9001</value> </property> <property> <!-- hadoop的日誌輸出指定目錄--> <name>mapred.local.dir</name> <value>/root/hadoop1.2/mylogs</value> </property> </configuration>配置好,日誌目錄後,我們就可以把這個配置文件,分發到各個節點上,然後啓動hadoop。
下面我們看來下在eclipse環境中如何調試,散仙在setup,map和reduce方法中,分別使用System打印了一些數據,當我們使用local方式跑MR程序時候,日誌並不會被記錄下來,而是直接會在控制檯打印,散仙的測試代碼如下:
package com.qin.testdistributed;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.net.URI;
import java.util.Scanner;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.log4j.pattern.LogEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.qin.operadb.WriteMapDB;
/**
* 測試hadoop的全局共享文件
* 使用DistributedCached
*
* 大數據技術交流羣: 37693216
* @author qindongliang
*
* ***/
public class TestDistributed {
private static Logger logger=LoggerFactory.getLogger(TestDistributed.class);
private static class FileMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
Path path[]=null;
/**
* Map函數前調用
*
* */
@Override
protected void setup(Context context)
throws IOException, InterruptedException {
logger.info("開始啓動setup了哈哈哈哈");
// System.out.println("運行了.........");
Configuration conf=context.getConfiguration();
path=DistributedCache.getLocalCacheFiles(conf);
System.out.println("獲取的路徑是: "+path[0].toString());
// FileSystem fs = FileSystem.get(conf);
FileSystem fsopen= FileSystem.getLocal(conf);
// FSDataInputStream in = fsopen.open(path[0]);
// System.out.println(in.readLine());
// for(Path tmpRefPath : path) {
// if(tmpRefPath.toString().indexOf("ref.png") != -1) {
// in = reffs.open(tmpRefPath);
// break;
// }
// }
// FileReader reader=new FileReader("file://"+path[0].toString());
// File f=new File("file://"+path[0].toString());
// FSDataInputStream in=fs.open(new Path(path[0].toString()));
// Scanner scan=new Scanner(in);
// while(scan.hasNext()){
// System.out.println(Thread.currentThread().getName()+"掃描的內容: "+scan.next());
// }
// scan.close();
//
// System.out.println("size: "+path.length);
}
@Override
protected void map(LongWritable key, Text value,Context context)
throws IOException, InterruptedException {
// System.out.println("map aaa");
//logger.info("Map裏的任務");
System.out.println("map裏輸出了");
// logger.info();
context.write(new Text(""), new IntWritable(0));
}
@Override
protected void cleanup(Context context)
throws IOException, InterruptedException {
logger.info("清空任務了。。。。。。");
}
}
private static class FileReduce extends Reducer<Object, Object, Object, Object>{
@Override
protected void reduce(Object arg0, Iterable<Object> arg1,
Context arg2)throws IOException, InterruptedException {
System.out.println("我是reduce裏面的東西");
}
}
public static void main(String[] args)throws Exception {
JobConf conf=new JobConf(TestDistributed.class);
//conf.set("mapred.local.dir", "/root/hadoop");
//Configuration conf=new Configuration();
// conf.set("mapred.job.tracker","192.168.75.130:9001");
//讀取person中的數據字段
//conf.setJar("tt.jar");
//注意這行代碼放在最前面,進行初始化,否則會報
String inputPath="hdfs://192.168.75.130:9000/root/input";
String outputPath="hdfs://192.168.75.130:9000/root/outputsort";
Job job=new Job(conf, "a");
DistributedCache.addCacheFile(new URI("hdfs://192.168.75.130:9000/root/input/f1.txt"), job.getConfiguration());
job.setJarByClass(TestDistributed.class);
System.out.println("運行模式: "+conf.get("mapred.job.tracker"));
/**設置輸出表的的信息 第一個參數是job任務,第二個參數是表名,第三個參數字段項**/
FileSystem fs=FileSystem.get(job.getConfiguration());
Path pout=new Path(outputPath);
if(fs.exists(pout)){
fs.delete(pout, true);
System.out.println("存在此路徑, 已經刪除......");
}
/**設置Map類**/
// job.setOutputKeyClass(Text.class);
//job.setOutputKeyClass(IntWritable.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setMapperClass(FileMapper.class);
job.setReducerClass(FileReduce.class);
FileInputFormat.setInputPaths(job, new Path(inputPath)); //輸入路徑
FileOutputFormat.setOutputPath(job, new Path(outputPath));//輸出路徑
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
Local模式下輸出如下: 運行模式: local
存在此路徑, 已經刪除......
WARN - NativeCodeLoader.<clinit>(52) | Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
WARN - JobClient.copyAndConfigureFiles(746) | Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.
WARN - JobClient.copyAndConfigureFiles(870) | No job jar file set. User classes may not be found. See JobConf(Class) or JobConf#setJar(String).
INFO - FileInputFormat.listStatus(237) | Total input paths to process : 1
WARN - LoadSnappy.<clinit>(46) | Snappy native library not loaded
INFO - TrackerDistributedCacheManager.downloadCacheObject(423) | Creating f1.txt in /root/hadoop1.2/hadooptmp/mapred/local/archive/9070031930820799196_1788685676_88844454/192.168.75.130/root/input-work-186410214545932656 with rwxr-xr-x
INFO - TrackerDistributedCacheManager.downloadCacheObject(463) | Cached hdfs://192.168.75.130:9000/root/input/f1.txt as /root/hadoop1.2/hadooptmp/mapred/local/archive/9070031930820799196_1788685676_88844454/192.168.75.130/root/input/f1.txt
INFO - TrackerDistributedCacheManager.localizePublicCacheObject(486) | Cached hdfs://192.168.75.130:9000/root/input/f1.txt as /root/hadoop1.2/hadooptmp/mapred/local/archive/9070031930820799196_1788685676_88844454/192.168.75.130/root/input/f1.txt
INFO - JobClient.monitorAndPrintJob(1380) | Running job: job_local479869714_0001
INFO - LocalJobRunner$Job.run(340) | Waiting for map tasks
INFO - LocalJobRunner$Job$MapTaskRunnable.run(204) | Starting task: attempt_local479869714_0001_m_000000_0
INFO - Task.initialize(534) | Using ResourceCalculatorPlugin : null
INFO - MapTask.runNewMapper(729) | Processing split: hdfs://192.168.75.130:9000/root/input/f1.txt:0+31
INFO - MapTask$MapOutputBuffer.<init>(949) | io.sort.mb = 100
INFO - MapTask$MapOutputBuffer.<init>(961) | data buffer = 79691776/99614720
INFO - MapTask$MapOutputBuffer.<init>(962) | record buffer = 262144/327680
INFO - TestDistributed$FileMapper.setup(57) | 開始啓動setup了哈哈哈哈
獲取的路徑是: /root/hadoop1.2/hadooptmp/mapred/local/archive/9070031930820799196_1788685676_88844454/192.168.75.130/root/input/f1.txt
map裏輸出了
map裏輸出了
INFO - TestDistributed$FileMapper.cleanup(107) | 清空任務了。。。。。。
INFO - MapTask$MapOutputBuffer.flush(1289) | Starting flush of map output
INFO - MapTask$MapOutputBuffer.sortAndSpill(1471) | Finished spill 0
INFO - Task.done(858) | Task:attempt_local479869714_0001_m_000000_0 is done. And is in the process of commiting
INFO - LocalJobRunner$Job.statusUpdate(466) |
INFO - Task.sendDone(970) | Task 'attempt_local479869714_0001_m_000000_0' done.
INFO - LocalJobRunner$Job$MapTaskRunnable.run(229) | Finishing task: attempt_local479869714_0001_m_000000_0
INFO - LocalJobRunner$Job.run(348) | Map task executor complete.
INFO - Task.initialize(534) | Using ResourceCalculatorPlugin : null
INFO - LocalJobRunner$Job.statusUpdate(466) |
INFO - Merger$MergeQueue.merge(408) | Merging 1 sorted segments
INFO - Merger$MergeQueue.merge(491) | Down to the last merge-pass, with 1 segments left of total size: 16 bytes
INFO - LocalJobRunner$Job.statusUpdate(466) |
我是reduce裏面的東西
INFO - Task.done(858) | Task:attempt_local479869714_0001_r_000000_0 is done. And is in the process of commiting
INFO - LocalJobRunner$Job.statusUpdate(466) |
INFO - Task.commit(1011) | Task attempt_local479869714_0001_r_000000_0 is allowed to commit now
INFO - FileOutputCommitter.commitTask(173) | Saved output of task 'attempt_local479869714_0001_r_000000_0' to hdfs://192.168.75.130:9000/root/outputsort
INFO - LocalJobRunner$Job.statusUpdate(466) | reduce > reduce
INFO - Task.sendDone(970) | Task 'attempt_local479869714_0001_r_000000_0' done.
INFO - JobClient.monitorAndPrintJob(1393) | map 100% reduce 100%
INFO - JobClient.monitorAndPrintJob(1448) | Job complete: job_local479869714_0001
INFO - Counters.log(585) | Counters: 18
INFO - Counters.log(587) | File Output Format Counters
INFO - Counters.log(589) | Bytes Written=0
INFO - Counters.log(587) | File Input Format Counters
INFO - Counters.log(589) | Bytes Read=31
INFO - Counters.log(587) | FileSystemCounters
INFO - Counters.log(589) | FILE_BYTES_READ=454
INFO - Counters.log(589) | HDFS_BYTES_READ=124
INFO - Counters.log(589) | FILE_BYTES_WRITTEN=138372
INFO - Counters.log(587) | Map-Reduce Framework
INFO - Counters.log(589) | Map output materialized bytes=20
INFO - Counters.log(589) | Map input records=2
INFO - Counters.log(589) | Reduce shuffle bytes=0
INFO - Counters.log(589) | Spilled Records=4
INFO - Counters.log(589) | Map output bytes=10
INFO - Counters.log(589) | Total committed heap usage (bytes)=455475200
INFO - Counters.log(589) | Combine input records=0
INFO - Counters.log(589) | SPLIT_RAW_BYTES=109
INFO - Counters.log(589) | Reduce input records=2
INFO - Counters.log(589) | Reduce input groups=1
INFO - Counters.log(589) | Combine output records=0
INFO - Counters.log(589) | Reduce output records=0
INFO - Counters.log(589) | Map output records=2
下面,我們將程序,提交成hadoop集羣上運行進行測試,注意在集羣上運行,日誌信息就不會在控制檯顯示了,我們需要去自己定義的日誌目錄下,找到最新提交 的那個下,然後就可以查看我們的日誌信息了。 查看stdout裏面的內容如下:
獲取的路徑是: /root/hadoop1.2/mylogs/taskTracker/distcache/2726204645197711229_1788685676_88844454/192.168.75.130/root/input/f1.txt map裏輸出了 map裏輸出了注意,map裏面的日誌需要去xxxmxxx和xxxrxxx裏面去找:
當然,除了這種方式外,我們還可以直接通過50030端口在web頁面上進行查看,截圖示例如下:
至此,我們已經散仙已經介紹完了,這兩種方式,Hadoop在執行過程中,日誌會被隨機分到任何一臺節點上,我們可能不能確定本次提交的任務日誌輸出到底放在那裏,但是我們可以通過在50030的web頁面上,查看最新的一次任務,一般是最下面的任務,是最新提交的,通過頁面上的連接我們就可以,查看到具體的本次任務的日誌情況被隨機分發到那個節點上了,然後就可以去具體的 節點上獲取了。