spark-sql性能優化之——多線程實現多Job並發執行

直接上代碼

val spark = SparkSession.builder()
    .appName("name")
    .master("local[2]")
    .getOrCreate()

  val df = spark.read.json("src\\main\\resources\\json.txt")

  df.show()


  //沒有多線程處理的情況，連續執行兩個Action操作，生成兩個Job
  df.rdd.saveAsTextFile("")
  df.rdd.saveAsTextFile("")


  //用Executor實現多線程方式處理Job
  val dfList = Array(df,df)
  val executorService = Executors.newFixedThreadPool(2)
  for(df <- dfList) {
    executorService.submit(new Callable[Boolean]() {
      def call() : Boolean  = {
        df.show()
        true
      }
    })
  }

  executorService.shutdown()

  spark.stop()