1 pom文件配置
<dependencies>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-configuration</artifactId>
<version>1.5.2</version>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>1.5.2</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.26</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>1.0.27</version>
</dependency>
</dependencies>2 java代碼實現
package cn.gcks;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.flume.Channel;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.Transaction;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
/**
* Flume收集文件到MySQL
*
* @author 林志明
*
*/
public class MySQLSink extends AbstractSink implements Configurable {
private Logger logger = LoggerFactory.getLogger(MySQLSink.class);
private String hostname;
private String port;
private String databaseName;
private String tableName;
private String user;
private String password;
private PreparedStatement preparedStatement;
private Connection conn;
private int batchSize;
private String columnName;
private String fieldSeparator;
private int columnNumber;
public MySQLSink() {
logger.info("MysqlSink start...");
}
Pattern pattern = null;
@Override
public void configure(Context context) {
hostname = context.getString("hostname");
Preconditions.checkNotNull(hostname, "hostname must be set!!");
port = context.getString("port");
Preconditions.checkNotNull(port, "port must be set!!");
databaseName = context.getString("databaseName");
Preconditions.checkNotNull(databaseName, "databaseName must be set!!");
tableName = context.getString("tableName");
Preconditions.checkNotNull(tableName, "tableName must be set!!");
user = context.getString("user");
Preconditions.checkNotNull(user, "user must be set!!");
password = context.getString("password");
Preconditions.checkNotNull(password, "password must be set!!");
batchSize = context.getInteger("batchSize", 100);
Preconditions.checkNotNull(batchSize > 0, "batchSize must be a positive number!!");
columnName = context.getString("column_name");
Preconditions.checkNotNull(columnName, "column_name must be set!!");
fieldSeparator = context.getString("field_separator");
Preconditions.checkNotNull(fieldSeparator, "field_separator must be set!!");
pattern = Pattern.compile(fieldSeparator);
}
private String sql = null;
@Override
public void start() {
super.start();
String url = CommonUtils.append("jdbc:mysql://", hostname, ":", port, "/", databaseName,
"?Unicode=true&characterEncoding=UTF-8");
try {
conn = DataSourceFactory.getConnection(url, user, password);
conn.setAutoCommit(false);
// 創建一個Statement對象
String[] columns = columnName.split(",");
String values = null;
StringBuffer stringBuffer = new StringBuffer();
int i = 0;
columnNumber = columns.length;
while (i < columnNumber) {
stringBuffer.append("?").append(",");
i++;
}
values = stringBuffer.substring(0, stringBuffer.length() - 1).toString();
sql = "insert into " + tableName + " (" + columnName + ") values (" + values + ")";
} catch (SQLException e) {
e.printStackTrace();
System.exit(1);
}
}
@Override
public void stop() {
super.stop();
DataSourceFactory.closeCon(preparedStatement, conn);
}
@Override
public Status process() throws EventDeliveryException {
try {
preparedStatement = conn.prepareStatement(sql);
logger.info("===preparedStatement Alive===");
} catch (SQLException e1) {
e1.printStackTrace();
}
Status result = Status.READY;
Channel channel = getChannel();
Transaction transaction = channel.getTransaction();
Event event;
String content;
List<String> actions = Lists.newArrayList();
transaction.begin();
try {
for (int i = 0; i < batchSize; i++) {
event = channel.take();
if (event != null) {
content = new String(event.getBody());
actions.add(content);
} else {
result = Status.BACKOFF;
break;
}
}
if (actions.size() > 0) {
preparedStatement.clearBatch();
for (String temp : actions) {
String[] datas = {};
if (temp != null && temp.trim().length() != 0) {
datas = pattern.split(temp);
}
for (int i = 0; i < columnNumber; i++) {
if (i > datas.length - 1) {
preparedStatement.setString(i + 1, null);
} else {
preparedStatement.setString(i + 1, datas[i]);
}
}
preparedStatement.addBatch();
}
preparedStatement.executeBatch();
conn.commit();
}
transaction.commit();
} catch (Throwable e) {
try {
transaction.rollback();
} catch (Exception e2) {
logger.error("Exception in rollback. Rollback might not have been" + "successful.", e2);
}
logger.error("Failed to commit transaction." + "Transaction rolled back.", e);
Throwables.propagate(e);
} finally {
transaction.close();
}
return result;
}
}package cn.gcks;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import org.apache.log4j.Logger;
import com.alibaba.druid.pool.DruidDataSource;
/**
* 數據庫連接
*
* @author linzm
*
*/
public class DataSourceFactory {
private static Logger log = Logger.getLogger(DataSourceFactory.class);
private static DruidDataSource dataSource = null;
/**
* 創建數據源
*
* @return
*/
public static DruidDataSource getDataSource(String url, String userName, String password) throws Exception {
if (dataSource == null) {
log.info("數據庫連接信息：[url:" + url + ",userName:" + userName + ",password:" + password + "]");
dataSource = new DruidDataSource();
dataSource.setUrl(url);
dataSource.setUsername(userName);
dataSource.setPassword(password);
dataSource.setMaxActive(15);// 設置最大併發數
dataSource.setInitialSize(2);// 數據庫初始化時，創建的連接個數
dataSource.setMaxWait(60000);
dataSource.setMinIdle(1);// 最小空閒連接數
dataSource.setTimeBetweenEvictionRunsMillis(5 * 60 * 1000);// 5分鐘檢測一次是否有死掉的線程
dataSource.setMinEvictableIdleTimeMillis(300000);// 空閒連接60秒中後釋放
dataSource.setTestWhileIdle(true);
// 檢測連接有效性
dataSource.setTestOnBorrow(true);
dataSource.setValidationQuery("select 1");
dataSource.setPoolPreparedStatements(true);
dataSource.setMaxOpenPreparedStatements(15);
}
return dataSource;
}
/**
* 釋放數據源
*/
public static void shutDownDataSource() throws Exception {
if (dataSource != null) {
dataSource.close();
}
}
/**
* 獲取數據庫連接
*
* @return
*/
public static Connection getConnection(String url, String userName, String password) {
Connection con = null;
try {
if (dataSource != null) {
con = dataSource.getConnection();
} else {
con = getDataSource(url, userName, password).getConnection();
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return con;
}
/**
* 關閉連接
*/
public static void closeCon(PreparedStatement ps, Connection con) {
if (ps != null) {
try {
ps.close();
} catch (Exception e) {
log.error("預編譯SQL語句對象PreparedStatement關閉異常！" + e.getMessage(), e);
}
}
if (con != null) {
try {
con.close();
} catch (Exception e) {
log.error("關閉連接對象Connection異常！" + e.getMessage(), e);
}
}
}
}package cn.gcks;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class CommonUtils {
public static final String EMPTY = "";
/**
* Represents a failed index search.
*
* @since 2.1
*/
public static final int INDEX_NOT_FOUND = -1;
/**
* <p>
* The maximum size to which the padding constant(s) can expand.
* </p>
*/
// private static final int PAD_LIMIT = 8192;
/**
* <p>
* Checks if a String is whitespace, empty ("") or null.
* </p>
* <p>
* <pre>
* StringUtils.isBlank(null) = true
* StringUtils.isBlank("") = true
* StringUtils.isBlank(" ") = true
* StringUtils.isBlank("bob") = false
* StringUtils.isBlank(" bob ") = false
* </pre>
*
* @param str the String to check, may be null
* @return <code>true</code> if the String is null, empty or whitespace
* @since 2.0
*/
public static boolean isBlank(String str) {
int strLen;
if (str == null || (strLen = str.length()) == 0) {
return true;
}
for (int i = 0; i < strLen; i++) {
if ((Character.isWhitespace(str.charAt(i)) == false)) {
return false;
}
}
return true;
}
/**
* <p>
* Checks if a String is whitespace, empty ("") or null.
* </p>
* <p>
* <pre>
* StringUtils.isBlank(null) = true
* StringUtils.isBlank("") = true
* StringUtils.isBlank(" ") = true
* StringUtils.isBlank("bob") = false
* StringUtils.isBlank(" bob ") = false
* </pre>
*
* @param str the String to check, may be null
* @return <code>true</code> if the String is null, empty or whitespace
* @since 2.0
*/
public static boolean isNotBlank(String str) {
return !isBlank(str);
}
public static Long parseLong(String str) {
try {
return Long.parseLong(str);
} catch (Exception ex) {
return 0L;
}
}
public static String toCNDateTimeString(Date date) {
SimpleDateFormat sf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
return sf.format(date);
}
public static int strLengthContainBlank(String str) {
return isBlank(str) ? 0 : str.length();
}
/**
* <p>
* Gets a substring from the specified String avoiding exceptions.
* </p>
* <p>
* <p>
* A negative start position can be used to start/end <code>n</code>
* characters from the end of the String.
* </p>
* <p>
* <p>
* The returned substring starts with the character in the
* <code>start</code> position and ends before the <code>end</code>
* position. All position counting is zero-based -- i.e., to start at the
* beginning of the string use <code>start = 0</code>. Negative start and
* end positions can be used to specify offsets relative to the end of the
* String.
* </p>
* <p>
* <p>
* If <code>start</code> is not strictly to the left of <code>end</code>, ""
* is returned.
* </p>
* <p>
* <pre>
* StringUtils.substring(null, *, *) = null
* StringUtils.substring("", * , *) = "";
* StringUtils.substring("abc", 0, 2) = "ab"
* StringUtils.substring("abc", 2, 0) = ""
* StringUtils.substring("abc", 2, 4) = "c"
* StringUtils.substring("abc", 4, 6) = ""
* StringUtils.substring("abc", 2, 2) = ""
* StringUtils.substring("abc", -2, -1) = "b"
* StringUtils.substring("abc", -4, 2) = "ab"
* </pre>
*
* @param str the String to get the substring from, may be null
* @param start the position to start from, negative means count back from the
* end of the String by this many characters
* @param end the position to end at (exclusive), negative means count back
* from the end of the String by this many characters
* @return substring from start position to end positon, <code>null</code>
* if null String input
*/
public static String substring(String str, int start, int end) {
if (str == null) {
return null;
}
// handle negatives
if (end < 0) {
end = str.length() + end; // remember end is negative
}
if (start < 0) {
start = str.length() + start; // remember start is negative
}
// check length next
if (end > str.length()) {
end = str.length();
}
// if start is greater than end, return ""
if (start > end) {
return EMPTY;
}
if (start < 0) {
start = 0;
}
if (end < 0) {
end = 0;
}
return str.substring(start, end);
}
public static String append(String... strings) {
StringBuffer stringBuffer = new StringBuffer();
for (String string : strings) {
stringBuffer.append(string);
}
return stringBuffer.toString();
}
public static String appendWithOperator(String operator, Object... objs) {
StringBuffer stringBuffer = new StringBuffer();
for (Object obj : objs) {
if (obj instanceof Object[]) {
Object[] arr = (Object[]) obj;
for (Object obj2 : arr) {
stringBuffer.append(obj2).append(operator);
}
} else {
stringBuffer.append(obj).append(operator);
}
}
return substring(stringBuffer.toString(), 0, -1);
}
public static String appendWithOperatorReplaceString(String operator, String nullReplaceString, Object... objs) {
StringBuffer stringBuffer = new StringBuffer();
for (Object obj : objs) {
if (obj != null && isBlank(obj.toString())) {
obj = nullReplaceString;
}
if (obj instanceof Object[]) {
Object[] arr = (Object[]) obj;
for (Object obj2 : arr) {
if (obj2 == null) {
obj2 = nullReplaceString;
}
stringBuffer.append(obj2).append(operator);
}
} else {
stringBuffer.append(obj).append(operator);
}
}
return substring(stringBuffer.toString(), 0, -1);
}
public static boolean equals(String str1, String str2) {
return str1 == null ? str2 == null : str1.equals(str2);
}
public static boolean notEquals(String str1, String str2) {
return !(str1 == null ? str2 == null : str1.equals(str2));
}
/**
* <p>
* Checks if String contains a search String, handling <code>null</code>.
* This method uses {@link String#indexOf(String)}.
* </p>
* <p>
* <p>
* A <code>null</code> String will return <code>false</code>.
* </p>
* <p>
* <pre>
* StringUtils.contains(null, *) = false
* StringUtils.contains(*, null) = false
* StringUtils.contains("", "") = true
* StringUtils.contains("abc", "") = true
* StringUtils.contains("abc", "a") = true
* StringUtils.contains("abc", "z") = false
* </pre>
*
* @param str the String to check, may be null
* @param searchStr the String to find, may be null
* @return true if the String contains the search String, false if not or
* <code>null</code> string input
* @since 2.0
*/
public static boolean contains(String str, String searchStr) {
if (str == null || searchStr == null) {
return false;
}
return str.indexOf(searchStr) >= 0;
}
public static String[] split(String str, String operator, Pattern pattern) {
if (str == null || operator == null) {
return null;
} else {
if (contains(str, operator)) {
return pattern.split(str);
} else {
return null;
}
}
}
// StripAll
// -----------------------------------------------------------------------
/**
* <p>
* Strips whitespace from the start and end of every String in an array.
* Whitespace is defined by {@link Character#isWhitespace(char)}.
* </p>
* <p>
* <p>
* A new array is returned each time, except for length zero. A
* <code>null</code> array will return <code>null</code>. An empty array
* will return itself. A <code>null</code> array entry will be ignored.
* </p>
* <p>
* <pre>
* StringUtils.stripAll(null) = null
* StringUtils.stripAll([]) = []
* StringUtils.stripAll(["abc", " abc"]) = ["abc", "abc"]
* StringUtils.stripAll(["abc ", null]) = ["abc", null]
* </pre>
*
* @param strs the array to remove whitespace from, may be null
* @return the stripped Strings, <code>null</code> if null array input
*/
public static String[] stripAll(String[] strs) {
return stripAll(strs, null);
}
/**
* <p>
* Strips any of a set of characters from the start and end of every String
* in an array.
* </p>
* Whitespace is defined by {@link Character#isWhitespace(char)}.
* </p>
* <p>
* <p>
* A new array is returned each time, except for length zero. A
* <code>null</code> array will return <code>null</code>. An empty array
* will return itself. A <code>null</code> array entry will be ignored. A
* <code>null</code> stripChars will strip whitespace as defined by
* {@link Character#isWhitespace(char)}.
* </p>
* <p>
* <pre>
* StringUtils.stripAll(null, *) = null
* StringUtils.stripAll([], *) = []
* StringUtils.stripAll(["abc", " abc"], null) = ["abc", "abc"]
* StringUtils.stripAll(["abc ", null], null) = ["abc", null]
* StringUtils.stripAll(["abc ", null], "yz") = ["abc ", null]
* StringUtils.stripAll(["yabcz", null], "yz") = ["abc", null]
* </pre>
*
* @param strs the array to remove characters from, may be null
* @param stripChars the characters to remove, null treated as whitespace
* @return the stripped Strings, <code>null</code> if null array input
*/
public static String[] stripAll(String[] strs, String stripChars) {
int strsLen;
if (strs == null || (strsLen = strs.length) == 0) {
return strs;
}
String[] newArr = new String[strsLen];
for (int i = 0; i < strsLen; i++) {
newArr[i] = strip(strs[i], stripChars);
}
return newArr;
}
/**
* <p>
* Strips any of a set of characters from the start and end of a String.
* This is similar to {@link String#trim()} but allows the characters to be
* stripped to be controlled.
* </p>
* <p>
* <p>
* A <code>null</code> input String returns <code>null</code>. An empty
* string ("") input returns the empty string.
* </p>
* <p>
* <p>
* If the stripChars String is <code>null</code>, whitespace is stripped as
* defined by {@link Character#isWhitespace(char)}. Alternatively use
* {@link #strip(String)}.
* </p>
* <p>
* <pre>
* StringUtils.strip(null, *) = null
* StringUtils.strip("", *) = ""
* StringUtils.strip("abc", null) = "abc"
* StringUtils.strip(" abc", null) = "abc"
* StringUtils.strip("abc ", null) = "abc"
* StringUtils.strip(" abc ", null) = "abc"
* StringUtils.strip(" abcyx", "xyz") = " abc"
* </pre>
*
* @param str the String to remove characters from, may be null
* @param stripChars the characters to remove, null treated as whitespace
* @return the stripped String, <code>null</code> if null String input
*/
public static String strip(String str, String stripChars) {
if (isEmpty(str)) {
return str;
}
str = stripStart(str, stripChars);
return stripEnd(str, stripChars);
}
/**
* <p>
* Strips any of a set of characters from the start of a String.
* </p>
* <p>
* <p>
* A <code>null</code> input String returns <code>null</code>. An empty
* string ("") input returns the empty string.
* </p>
* <p>
* <p>
* If the stripChars String is <code>null</code>, whitespace is stripped as
* defined by {@link Character#isWhitespace(char)}.
* </p>
* <p>
* <pre>
* StringUtils.stripStart(null, *) = null
* StringUtils.stripStart("", *) = ""
* StringUtils.stripStart("abc", "") = "abc"
* StringUtils.stripStart("abc", null) = "abc"
* StringUtils.stripStart(" abc", null) = "abc"
* StringUtils.stripStart("abc ", null) = "abc "
* StringUtils.stripStart(" abc ", null) = "abc "
* StringUtils.stripStart("yxabc ", "xyz") = "abc "
* </pre>
*
* @param str the String to remove characters from, may be null
* @param stripChars the characters to remove, null treated as whitespace
* @return the stripped String, <code>null</code> if null String input
*/
public static String stripStart(String str, String stripChars) {
int strLen;
if (str == null || (strLen = str.length()) == 0) {
return str;
}
int start = 0;
if (stripChars == null) {
while ((start != strLen) && Character.isWhitespace(str.charAt(start))) {
start++;
}
} else if (stripChars.length() == 0) {
return str;
} else {
while ((start != strLen) && (stripChars.indexOf(str.charAt(start)) != INDEX_NOT_FOUND)) {
start++;
}
}
return str.substring(start);
}
/**
* <p>
* Strips any of a set of characters from the end of a String.
* </p>
* <p>
* <p>
* A <code>null</code> input String returns <code>null</code>. An empty
* string ("") input returns the empty string.
* </p>
* <p>
* <p>
* If the stripChars String is <code>null</code>, whitespace is stripped as
* defined by {@link Character#isWhitespace(char)}.
* </p>
* <p>
* <pre>
* StringUtils.stripEnd(null, *) = null
* StringUtils.stripEnd("", *) = ""
* StringUtils.stripEnd("abc", "") = "abc"
* StringUtils.stripEnd("abc", null) = "abc"
* StringUtils.stripEnd(" abc", null) = " abc"
* StringUtils.stripEnd("abc ", null) = "abc"
* StringUtils.stripEnd(" abc ", null) = " abc"
* StringUtils.stripEnd(" abcyx", "xyz") = " abc"
* StringUtils.stripEnd("120.00", ".0") = "12"
* </pre>
*
* @param str the String to remove characters from, may be null
* @param stripChars the set of characters to remove, null treated as whitespace
* @return the stripped String, <code>null</code> if null String input
*/
public static String stripEnd(String str, String stripChars) {
int end;
if (str == null || (end = str.length()) == 0) {
return str;
}
if (stripChars == null) {
while ((end != 0) && Character.isWhitespace(str.charAt(end - 1))) {
end--;
}
} else if (stripChars.length() == 0) {
return str;
} else {
while ((end != 0) && (stripChars.indexOf(str.charAt(end - 1)) != INDEX_NOT_FOUND)) {
end--;
}
}
return str.substring(0, end);
}
// Empty checks
// -----------------------------------------------------------------------
/**
* <p>
* Checks if a String is empty ("") or null.
* </p>
* <p>
* <pre>
* StringUtils.isEmpty(null) = true
* StringUtils.isEmpty("") = true
* StringUtils.isEmpty(" ") = false
* StringUtils.isEmpty("bob") = false
* StringUtils.isEmpty(" bob ") = false
* </pre>
* <p>
* <p>
* NOTE: This method changed in Lang version 2.0. It no longer trims the
* String. That functionality is available in isBlank().
* </p>
*
* @param str the String to check, may be null
* @return <code>true</code> if the String is empty or null
*/
public static boolean isEmpty(String str) {
return str == null || str.length() == 0;
}
// Stripping
// -----------------------------------------------------------------------
/**
* <p>
* Strips whitespace from the start and end of a String.
* </p>
* <p>
* <p>
* This is similar to {@link (String)} but removes whitespace.
* Whitespace is defined by {@link Character#isWhitespace(char)}.
* </p>
* <p>
* <p>
* A <code>null</code> input String returns <code>null</code>.
* </p>
* <p>
* <pre>
* StringUtils.strip(null) = null
* StringUtils.strip("") = ""
* StringUtils.strip(" ") = ""
* StringUtils.strip("abc") = "abc"
* StringUtils.strip(" abc") = "abc"
* StringUtils.strip("abc ") = "abc"
* StringUtils.strip(" abc ") = "abc"
* StringUtils.strip(" ab c ") = "ab c"
* </pre>
*
* @param str the String to remove whitespace from, may be null
* @return the stripped String, <code>null</code> if null String input
*/
public static String strip(String str) {
return strip(str, null);
}
/**
* <p>
* Converts a String to lower case as per {@link String#toLowerCase()}.
* </p>
* <p>
* <p>
* A <code>null</code> input String returns <code>null</code>.
* </p>
* <p>
* <pre>
* StringUtils.lowerCase(null) = null
* StringUtils.lowerCase("") = ""
* StringUtils.lowerCase("aBc") = "abc"
* </pre>
* <p>
* <p>
* <strong>Note:</strong> As described in the documentation for
* {@link String#toLowerCase()}, the result of this method is affected by
* the current locale. For platform-independent case transformations, the
* method {@link (String, Locale)} should be used with a specific
* locale (e.g. {@link Locale#ENGLISH}).
* </p>
*
* @param str the String to lower case, may be null
* @return the lower cased String, <code>null</code> if null String input
*/
public static String toLowerCase(String str) {
if (isBlank(str)) {
return str;
}
return str.toLowerCase();
}
public static String normalDateToConnectorDate (String time){
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
SimpleDateFormat format2 = new SimpleDateFormat("yyyyMMddHHmmss");
Date date = null;
try {
date = format.parse(time);
} catch (ParseException e) {
e.toString();
return null;
}
return format2.format(date);
}
}
3 打包java代碼，上傳jar包和mysql驅動包到flume的lib目錄
進入源碼的根目錄執行mvn install -DskipTests -Dtar
4 創建MySQL表
CREATE TABLE `test_flume_sink` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`content` varchar(255) DEFAULT NULL,
`log2` bigint(25) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
5 修改flume配置文件
agent.sinks.mysqlSink.type = cn.bfire.MySQLSink
agent.sinks.mysqlSink.hostname=1.1.1.1
agent.sinks.mysqlSink.port=3306
agent.sinks.mysqlSink.databaseName=bigdata
agent.sinks.mysqlSink.tableName=test_flume_sink
agent.sinks.mysqlSink.user=mi
agent.sinks.mysqlSink.password=11111111
agent.sinks.mysqlSink.column_name=content,log2
agent.sinks.mysqlSink.field_separator=\\|6 flume收集文件內容
abc|ddd
eee|fff
aaa,ggg
ddd|fff
sss|jjj
fff,fff
aaaaaa|gg|fffff
aaaaa,bbbbbbbbbb,ccccc
dddddd|eeeeeee|aa
xxxx|eeee
zzz|aaa,bbb|xxzz
7 MySQL表收集結果
8 遇到的問題【如果使用flume1.6.0下面的內容屬於無聊】
錯誤日誌：
java.lang.IllegalStateException: File should not roll when commit is outstanding.
at org.apache.flume.client.avro.ReliableSpoolingFileEventReader.readEvents(ReliableSpoolingFileEventReader.java:221)
at org.apache.flume.source.SpoolDirectorySource$SpoolDirectoryRunnable.run(SpoolDirectorySource.java:227)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)原因 ： flume收集目錄中有文件大小爲0
不過，這是flume1.6.0之前版本的BUG
https://issues.apache.org/jira/browse/FLUME-1934
http://flume.apache.org/releases/1.6.0.html
