spark dataframe 轉換為json

調用spark 的隱式轉換

scala> import spark.implicits._
import spark.implicits._

scala> val df=sc.parallelize(Array(("a",1),("b",2),("c",3)))

scala> df.show()
+---+---+                                                                       
| _1| _2|
+---+---+
|  a|  1|
|  b|  2|
|  c|  3|
+---+---+

scala> val jsonStr=df.toJSON.collect()
jsonStr: Array[String] = Array({"_1":"a","_2":1}, {"_1":"b","_2":2}, {"_1":"c","_2":3})

使用scala JSON方法

從dataframe 轉換為Array

scala> df.show()
+---+---+                                                                       
| _1| _2|
+---+---+
|  a|  1|
|  b|  2|
|  c|  3|
+---+---+

scala> val dfarr=df.collect().map{case org.apache.spark.sql.Row(x:String,y:Int)=>(x,y)}
dfarr: Array[(String, Int)] = Array((a,1), (b,2), (c,3))

從Array轉換成JSONObject

scala> val jsonData:Array[JSONObject] = dfarr.map{ i =>JSONObject(Map(i._1 -> i._2))}
jsonData: Array[scala.util.parsing.json.JSONObject] = Array({"a" : 1}, {"b" : 2}, {"c" : 3})

scala> 

從JSONObject轉換為JSONArray

scala> val jsonArray:JSONArray=new JSONArray(jsonData.toList)
jsonArray: scala.util.parsing.json.JSONArray = [{"a" : 1}, {"b" : 2}, {"c" : 3}]