大數據 java01 hive udf函數（手機號碼脫敏）

Hive UDF 函數

1 POM 文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>填寫自己的組織名稱</groupId>
    <artifactId>udf</artifactId>
    <version>1.0-SNAPSHOT</version>
    <properties>
        <project.build.sourceEncoding>UTF8</project.build.sourceEncoding>
        <!--Hadoop版本更改成自己的版本-->
        <hadoop.version>2.6.0-cdh5.13.3</hadoop.version>
        <hive.version>1.1.0-cdh5.13.3</hive.version>
    </properties>

    <repositories>
        <!--加入Hadoop原生態的maven倉庫的地址-->
        <repository>
            <id>Apache Hadoop</id>
            <name>Apache Hadoop</name>
            <url>https://repo1.maven.org/maven2/</url>
        </repository>
        <!--加入cdh的maven倉庫的地址-->
        <repository>
            <id>cloudera</id>
            <name>cloudera</name>
            <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
        </repository>
    </repositories>

    <dependencies>
        <!--添加hadoop依賴-->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <!--添加hive依賴-->
        <dependency>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-exec</artifactId>
            <version>${hive.version}</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <!--這部分可有可無,加上的話則直接生成可運行jar包-->
                    <!--<archive>-->
                        <!--<manifest>-->
                            <!--<mainClass>填寫自己的組織名稱.PhoneUnlookUdf</mainClass>-->
                        <!--</manifest>-->
                    <!--</archive>-->
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>


</project>

2.UDF 函數

package 填寫自己的組織名稱;

import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;

// 上傳udf jar到集羣 hdfs dfs -put udf-1.0-SNAPSHOT-jar-with-dependencies.jar /data/data_coe/data_asset/prod/db/tmp/udf/
// 修改文件權限 hdfs dfs -chmod -R 777 hdfs://idc-nn/data/data_coe/data_asset/prod/db/tmp/udf/
//註冊udf函數 create function tmp.pul as '填寫自己的組織名稱.PhoneUnlookUdf' using jar 'hdfs://idc-nn/data/data_coe/data_asset/prod/db/tmp/udf/udf-1.0-SNAPSHOT-jar-with-dependencies.jar

public class PhoneUnlookUdf extends UDF {
//重寫evaluate方法
    public String evaluate(String phone){
        if (phone.length() == 11){
            String res = phone.substring(0, 3) + "****" + phone.substring(7, phone.length());
            System.out.println(res);
            return res;
        } else {
            return phone;
        }

    }
}

3 利用idea打包

先點clean，在點package

4 添加hive udf函數

集羣的某些問題，不能直接通過添加服務器上本地文件到hive增加udf；需要將文件上傳到hdfs，然後定義udf函數。

4.1 上傳jar包到集羣

// 上傳udf jar到集羣 hdfs dfs -put udf-1.0-SNAPSHOT-jar-with-dependencies.jar /data/data_coe/data_asset/prod/db/tmp/udf/

4.2 修改集羣hdfs文件權限

// 修改文件權限 hdfs dfs -chmod -R 777 hdfs://idc-nn/data/data_coe/data_asset/prod/db/tmp/udf/

4.3 註冊UDF

//註冊udf函數 create function tmp.pul as 'cn.mcd.com.PhoneUnlookUdf' using jar 'hdfs://idc-nn/data/data_coe/data_asset/prod/db/tmp/udf/udf-1.0-SNAPSHOT-jar-with-dependencies.jar

4.4 使用UDF

···
打開集羣hive客戶端：
select tmp.pul(phone) from tmp.tmp_order limit 3;
···