linkis重编译适配cdh

感谢各位webank大佬的指导！希望社区发展更快更好！

1、系统环境：

redhat7、cdh5.15.1、spark2.3.0、开启了sentry、没有开启kerberos

2、获取源码

在linkis的github上有一个linkis-cdh5.6的分支

https://github.com/WeBankFinTech/Linkis/release

 

 

 

 

 点击进入这位大佬wForget的github，https://github.com/wForget/Linkis，下载源码（怀疑这位大佬改过源码，所以为了方便，直接下载这位大佬的，我的是cdh5.15）

3、修改元数据板块（我的cdh集群采用了sentry做hive权限控制，源码采用的是hive自带的权限控制（只通过jdbc），不修改无法展示hive元数据）

3、主要修改读取元数据的部分（如果你用sentry控制hive权限，可以拷贝覆盖）

<?xml version="1.0" encoding="UTF-8" ?>
<!--
  ~ Copyright 2019 WeBank
  ~
  ~ Licensed under the Apache License, Version 2.0 (the "License");
  ~  you may not use this file except in compliance with the License.
  ~ You may obtain a copy of the License at
  ~
  ~ http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  -->

<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.webank.wedatasphere.linkis.metadata.hive.dao.HiveMetaDao">

    <select id="getLocationByDbAndTable" resultType="java.lang.String" parameterType="map">
        select LOCATION from SDS where SD_ID in (
        select SD_ID from `TBLS`
        where TBL_NAME = #{tableName,jdbcType=VARCHAR}
        and DB_ID in (select DB_ID from `DBS` where NAME = #{dbName,jdbcType=VARCHAR})
        )
    </select>

    <select id="getDbsByUser" resultType="java.lang.String" parameterType="java.lang.String">
        <!--select NAME from(
        select t2.NAME  as NAME
        from DB_PRIVS t1, DBS t2
        where (lcase(t1.PRINCIPAL_NAME) = #{userName,jdbcType=VARCHAR}
        OR t1.PRINCIPAL_NAME IN (SELECT ROLE FROM(SELECT r.ROLE_NAME AS ROLE, u.PRINCIPAL_NAME AS USER FROM ROLES r LEFT JOIN (SELECT * FROM ROLE_MAP WHERE PRINCIPAL_TYPE = 'USER') u ON r.ROLE_ID = u.ROLE_ID)AS T where T.USER = #{userName,jdbcType=VARCHAR}))
        and lcase(t1.DB_PRIV) in ('select','all') and t1.DB_ID =t2.DB_ID
        union all
        select t3.NAME as NAME
        from TBL_PRIVS t1, TBLS t2 , DBS t3
        where t1.TBL_ID=t2.TBL_ID and lcase(t1.TBL_PRIV) in ('select','all') and (
        lcase(t1.PRINCIPAL_NAME) = #{userName,jdbcType=VARCHAR} or lcase(t1.PRINCIPAL_NAME) in (SELECT ROLE FROM(SELECT r.ROLE_NAME AS ROLE, u.PRINCIPAL_NAME AS USER FROM ROLES r LEFT JOIN (SELECT * FROM ROLE_MAP WHERE PRINCIPAL_TYPE = 'USER') u ON r.ROLE_ID = u.ROLE_ID)AS T where T.USER = #{userName,jdbcType=VARCHAR}))
        and t2.DB_ID=t3.DB_ID) a
        GROUP BY NAME
        order by NAME-->
        select name from DBS
    </select>

    <select id="getTablesByDbNameAndUser" resultType="map"  parameterType="map">
        <!--select t2.TBL_NAME as NAME, t2.TBL_TYPE as TYPE, t2.CREATE_TIME as CREATE_TIME, t2.LAST_ACCESS_TIME as LAST_ACCESS_TIME, t2.OWNER as OWNER
        from DB_PRIVS t1,TBLS t2, DBS t3
        where  t1.DB_ID =t3.DB_ID
        and t2.DB_ID=t3.DB_ID
        and lcase(t1.DB_PRIV) in ('select','all')
        and lcase(t1.PRINCIPAL_NAME) = #{userName,jdbcType=VARCHAR}
        and t3.NAME = #{dbName,jdbcType=VARCHAR}
        union
        select t2.TBL_NAME as NAME, t2.TBL_TYPE as TYPE, t2.CREATE_TIME as CREATE_TIME, t2.LAST_ACCESS_TIME as LAST_ACCESS_TIME, t2.OWNER as OWNER
        from DB_PRIVS t1,TBLS t2, DBS t3
        where  t1.DB_ID =t3.DB_ID
        and t2.DB_ID=t3.DB_ID
        and lcase(t1.DB_PRIV) in ('select','all')
        and lcase(t1.PRINCIPAL_NAME) in (select ROLE_NAME from ROLES where ROLE_ID in (select ROLE_ID from ROLE_MAP where PRINCIPAL_NAME = #{userName,jdbcType=VARCHAR}))
        and t3.NAME = #{dbName,jdbcType=VARCHAR}
        union
        select t2.TBL_NAME as NAME, t2.TBL_TYPE as TYPE, t2.CREATE_TIME as CREATE_TIME, t2.LAST_ACCESS_TIME as LAST_ACCESS_TIME, t2.OWNER as OWNER
        from TBL_PRIVS t1, TBLS t2 , DBS t3
        where t1.TBL_ID=t2.TBL_ID
        and t2.DB_ID=t3.DB_ID
        and lcase(t1.TBL_PRIV) in ('select','all')
        and t1.PRINCIPAL_NAME = #{userName,jdbcType=VARCHAR}
        and t3.NAME = #{dbName,jdbcType=VARCHAR}
        union
        select t2.TBL_NAME as NAME, t2.TBL_TYPE as TYPE, t2.CREATE_TIME as CREATE_TIME, t2.LAST_ACCESS_TIME as LAST_ACCESS_TIME, t2.OWNER as OWNER
        from TBL_PRIVS t1, TBLS t2 , DBS t3
        where t1.TBL_ID=t2.TBL_ID
        and t2.DB_ID=t3.DB_ID
        and lcase(t1.TBL_PRIV) in ('select','all')
        and t1.PRINCIPAL_NAME in (select ROLE_NAME from ROLES where ROLE_ID in (select ROLE_ID from ROLE_MAP where PRINCIPAL_NAME = #{userName,jdbcType=VARCHAR}))
        and t3.NAME = #{dbName,jdbcType=VARCHAR}
        order by NAME;-->
        select t2.TBL_NAME as NAME, t2.TBL_TYPE as TYPE, t2.CREATE_TIME as CREATE_TIME, t2.LAST_ACCESS_TIME as LAST_ACCESS_TIME, t2.OWNER as OWNER
        from TBLS t2 , DBS t3
        where
        t2.DB_ID=t3.DB_ID
        and t3.NAME = #{dbName,jdbcType=VARCHAR}
    </select>

    <select id="getPartitionSize" resultType="java.lang.Long"  parameterType="map">
        select PARAM_VALUE from PARTITION_PARAMS
        where PARAM_KEY = 'totalSize'
        and PART_ID in (
        select PART_ID from PARTITIONS
        where PART_NAME = #{partitionName,jdbcType=VARCHAR}
        and TBL_ID in(
        select TBL_ID from `TBLS`
        where TBL_NAME = #{tableName,jdbcType=VARCHAR}
        and DB_ID in (select DB_ID from `DBS` where NAME = #{dbName,jdbcType=VARCHAR})
        )
        );
    </select>

    <select id="getPartitions" resultType="java.lang.String"  parameterType="map">
        select PART_NAME from PARTITIONS
        where TBL_ID in(
        select TBL_ID from `TBLS`
        where TBL_NAME = #{tableName,jdbcType=VARCHAR}
        and DB_ID in (select DB_ID from `DBS` where NAME = #{dbName,jdbcType=VARCHAR})
        );
    </select>

    <select id="getColumns" resultType="map"  parameterType="map">
        SELECT COMMENT, COLUMN_NAME, TYPE_NAME FROM COLUMNS_V2
        where CD_ID in(
        select CD_ID from SDS where SD_ID in (
        select SD_ID from `TBLS`
        where TBL_NAME = #{tableName,jdbcType=VARCHAR}
        and DB_ID in (select DB_ID from `DBS` where NAME = #{dbName,jdbcType=VARCHAR})
        )
        ) order by INTEGER_IDX asc;
    </select>

    <select id="getPartitionKeys" resultType="map"  parameterType="map">
        select PKEY_COMMENT, PKEY_NAME, PKEY_TYPE from PARTITION_KEYS
        where TBL_ID in(
        select TBL_ID from `TBLS`
        where TBL_NAME = #{tableName,jdbcType=VARCHAR}
        and DB_ID in (select DB_ID from `DBS` where NAME = #{dbName,jdbcType=VARCHAR})
        );
    </select>

    <select id="getTableComment" resultType="String">
        SELECT
        tp.PARAM_VALUE
        FROM
        `DBS` d
        LEFT JOIN `TBLS` t ON d.DB_ID = t.DB_ID
        LEFT JOIN `TABLE_PARAMS` tp ON t.TBL_ID = tp.TBL_ID
        WHERE
        tp.PARAM_KEY = 'comment'
        AND d.`NAME` = #{DbName}
        AND t.TBL_NAME = #{tableName}
    </select>

</mapper>

 4、修改pom.xml

 4.1修改版本部分（将hive和hadoop版本指向cdh5.15.1）

 

4.2修改netty版本（我的是spark2.3.0，如果不修改，会有兼容性问题，在执行spark引擎时报错）

 

 5、配置多maven源，科 学 上网

首先保证编译主机可以访问外网，然后给maven配置多maven源,修改settings.xml里面<profiles></profiles>和<activeProfiles></activeProfiles>模块，添加cloudera官网提供的maven源

 

  <profiles>
    <profile>
      <id>boundlessgeo</id>
      <repositories>
        <repository>
          <id>boundlessgeo</id>
          <url>https://repo.boundlessgeo.com/main/</url>
          <releases>
            <enabled>true</enabled>
          </releases>
          <snapshots>
            <enabled>true</enabled>
            <updatePolicy>always</updatePolicy>
          </snapshots>
        </repository>
      </repositories>
    </profile>
    <profile>
      <id>aliyun</id>
      <repositories>
        <repository>
          <id>aliyun</id>
          <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
          <releases>
            <enabled>true</enabled>
          </releases>
          <snapshots>
            <enabled>true</enabled>
            <updatePolicy>always</updatePolicy>
          </snapshots>
        </repository>
      </repositories>
    </profile>
    <profile>
      <id>maven-central</id>
      <repositories>
        <repository>
          <id>maven-central</id>
          <url>http://central.maven.org/maven2/</url>
          <releases>
            <enabled>true</enabled>
          </releases>
          <snapshots>
            <enabled>true</enabled>
            <updatePolicy>always</updatePolicy>
          </snapshots>
        </repository>
      </repositories>
    </profile>
  <profile>
      <id>cloudera</id>
      <repositories>
        <repository>
          <id>cloudera</id>
          <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
          <releases>
            <enabled>true</enabled>
          </releases>
          <snapshots>
            <enabled>true</enabled>
            <updatePolicy>always</updatePolicy>
          </snapshots>
        </repository>
      </repositories>
    </profile>
  </profiles>

<activeProfiles>
    <activeProfile>boundlessgeo</activeProfile>
    <activeProfile>aliyun</activeProfile>
    <activeProfile>maven-central</activeProfile>
  <activeProfile>cloudera</activeProfile>
</activeProfiles>

 

6、编译，安装

mvn  -clean  install

mvn  package

 

7、体验（hive sql  、spark sql都可以用，权限也能满足需求，scala也可以用，不过没有开启kerberos，不安全）