You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2018/03/19 19:28:49 UTC
[40/42] incubator-spot git commit: corrected data insertion
corrected data insertion
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/f722127c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/f722127c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/f722127c
Branch: refs/heads/SPOT-181_ODM
Commit: f722127ca410a88355cc3f6d952739845c7499ce
Parents: a166efc
Author: tpltnt <tp...@dropcut.net>
Authored: Mon Feb 5 19:05:13 2018 +0100
Committer: tpltnt <tp...@dropcut.net>
Committed: Mon Feb 5 19:05:13 2018 +0100
----------------------------------------------------------------------
spot-ingest/pipelines/proxy/bluecoat.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/f722127c/spot-ingest/pipelines/proxy/bluecoat.py
----------------------------------------------------------------------
diff --git a/spot-ingest/pipelines/proxy/bluecoat.py b/spot-ingest/pipelines/proxy/bluecoat.py
index 01b9922..d476733 100644
--- a/spot-ingest/pipelines/proxy/bluecoat.py
+++ b/spot-ingest/pipelines/proxy/bluecoat.py
@@ -168,7 +168,7 @@ def save_data(rdd, sqc, db, db_table, topic):
sqc.setConf("hive.exec.dynamic.partition", "true")
sqc.setConf("hive.exec.dynamic.partition.mode", "nonstrict")
hive_table = "{0}.{1}".format(db, db_table)
- df.write.format("parquet").mode("append").insertInto(hive_table)
+ df.write.format("parquet").mode("append").partitionBy('y', 'm', 'd', 'h').insertInto(hive_table)
else:
print("------------------------LISTENING KAFKA TOPIC:{0}------------------------".format(topic))