Spark讀取文件統計行數

 

package sy3

import org.apache.spark.SparkContext
import org.apache.spark.SparkConf

object tjhs {
  def main(args: Array[String]) {
    val logFile = "E:/IntelliJ IDEA 2019.3.3/WorkSpace/MyScala/src/main/scala/sy3/test1"
    //val logFile = "hdfs://192.168.249.128:9000/mapreduce/input1/in1.txt"
    val conf = new SparkConf().setAppName("Simple Application")
    val sc = new SparkContext(conf)
    val logData = sc.textFile(logFile, 2)
    val num = logData.count()
    println("這個文件有 %d 行!".format(num))
  }
}