You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by ev...@apache.org on 2017/08/04 18:23:53 UTC
incubator-spot git commit: SPOT-212: [Ingest] [Proxy] bluecoat.py
parser not able to save into Hive table Replaced saveAsTable syntax
(deprecated for Hive tables in Spark 2.0) with insertInto
Repository: incubator-spot
Updated Branches:
refs/heads/master 781ccf777 -> 1e9833e19
SPOT-212: [Ingest] [Proxy] bluecoat.py parser not able to save into Hive table
Replaced saveAsTable syntax (deprecated for Hive tables in Spark 2.0) with insertInto
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/1e9833e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/1e9833e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/1e9833e1
Branch: refs/heads/master
Commit: 1e9833e191c85330c25d089db12e56fbdff21947
Parents: 781ccf7
Author: Deon Griessel <dg...@searchtechnologies.com>
Authored: Tue Aug 1 19:30:46 2017 +0000
Committer: Deon Griessel <dg...@searchtechnologies.com>
Committed: Tue Aug 1 19:30:46 2017 +0000
----------------------------------------------------------------------
spot-ingest/pipelines/proxy/bluecoat.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/1e9833e1/spot-ingest/pipelines/proxy/bluecoat.py
----------------------------------------------------------------------
diff --git a/spot-ingest/pipelines/proxy/bluecoat.py b/spot-ingest/pipelines/proxy/bluecoat.py
index de9ca6c..31d89ca 100644
--- a/spot-ingest/pipelines/proxy/bluecoat.py
+++ b/spot-ingest/pipelines/proxy/bluecoat.py
@@ -121,7 +121,7 @@ def save_data(rdd,sqc,db,db_table,topic):
sqc.setConf("hive.exec.dynamic.partition", "true")
sqc.setConf("hive.exec.dynamic.partition.mode", "nonstrict")
hive_table = "{0}.{1}".format(db,db_table)
- df.write.saveAsTable(hive_table,format="parquet",mode="append",partitionBy=('y','m','d','h'))
+ df.write.format("parquet").mode("append").insertInto(hive_table)
else:
print("------------------------LISTENING KAFKA TOPIC:{0}------------------------".format(topic))