Spark JdbcRDD 簡單使用

package org.apache.spark.sql.sources

import org.apache.spark.SparkContext
import java.sql.{ResultSet, DriverManager}
import org.apache.spark.rdd.JdbcRDD

/**
 * @author luogankun
 * Created by spark on 14-12-25.
 */
object JdbcTest extends  App{
    val sc = new SparkContext("local[2]", "demo")

    def getConnection() = {
      Class.forName("com.mysql.jdbc.Driver").newInstance()
      DriverManager.getConnection("jdbc:mysql://hadoop000:3306/hive", "root", "root")
    }

    def flatValue(result: ResultSet) = {
      (result.getInt("TBL_ID"), result.getString("TBL_NAME"))
    }

    //select * from TBLS WHERE TBL_ID>=1 AND TBL_ID<=10
    val data = new JdbcRDD(
      sc,
      getConnection,
      "select * from TBLS where TBL_ID >= ? and TBL_ID <= ?",
      1,
      10,
      2,
      flatValue
    )

    println(data.collect().toList)

    sc.stop
}

執行報錯：

查看JdbcRDD代碼發現，sql語句一定要帶上2個條件：

這個使用起來不太方便，最近需要找時間將JdbcRDD優化下，以便后續更方便的在jdbc external data source中能使用JdbcRDD。