You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/05/09 07:32:13 UTC
[1/2] kylin git commit: KYLIN-1660 Streaming/kafka config not match
with table name
Repository: kylin
Updated Branches:
refs/heads/master ecbff227a -> f65e7d1bd
KYLIN-1660 Streaming/kafka config not match with table name
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f65e7d1b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f65e7d1b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f65e7d1b
Branch: refs/heads/master
Commit: f65e7d1bd23d095bc6c60c5cd2eee4505ece0cfa
Parents: 0c0bb07
Author: shaofengshi <sh...@apache.org>
Authored: Mon May 9 15:21:46 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon May 9 15:32:01 2016 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/StreamingController.java | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f65e7d1b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index 932ad51..3cb1d72 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -114,6 +114,8 @@ public class StreamingController extends BasicController {
throw new BadRequestException("Failed to add streaming table.");
}
+ streamingConfig.setName(tableDesc.getIdentity());
+ kafkaConfig.setName(tableDesc.getIdentity());
try {
if (StringUtils.isEmpty(streamingConfig.getName())) {
logger.info("StreamingConfig should not be empty.");
@@ -168,7 +170,7 @@ public class StreamingController extends BasicController {
@RequestMapping(value = "", method = { RequestMethod.PUT })
@ResponseBody
- public StreamingRequest updateModelDesc(@RequestBody StreamingRequest streamingRequest) throws JsonProcessingException {
+ public StreamingRequest updateStreamingConfig(@RequestBody StreamingRequest streamingRequest) throws JsonProcessingException {
StreamingConfig streamingConfig = deserializeSchemalDesc(streamingRequest);
KafkaConfig kafkaConfig = deserializeKafkaSchemalDesc(streamingRequest);
@@ -220,10 +222,10 @@ public class StreamingController extends BasicController {
logger.debug("Saving TableDesc " + streamingRequest.getTableData());
desc = JsonUtil.readValue(streamingRequest.getTableData(), TableDesc.class);
} catch (JsonParseException e) {
- logger.error("The TableDesc definition is not valid.", e);
+ logger.error("The TableDesc definition is invalid.", e);
updateRequest(streamingRequest, false, e.getMessage());
} catch (JsonMappingException e) {
- logger.error("The data TableDesc definition is not valid.", e);
+ logger.error("The data TableDesc definition is invalid.", e);
updateRequest(streamingRequest, false, e.getMessage());
} catch (IOException e) {
logger.error("Failed to deal with the request.", e);
@@ -238,10 +240,10 @@ public class StreamingController extends BasicController {
logger.debug("Saving StreamingConfig " + streamingRequest.getStreamingConfig());
desc = JsonUtil.readValue(streamingRequest.getStreamingConfig(), StreamingConfig.class);
} catch (JsonParseException e) {
- logger.error("The StreamingConfig definition is not valid.", e);
+ logger.error("The StreamingConfig definition is invalid.", e);
updateRequest(streamingRequest, false, e.getMessage());
} catch (JsonMappingException e) {
- logger.error("The data StreamingConfig definition is not valid.", e);
+ logger.error("The data StreamingConfig definition is invalid.", e);
updateRequest(streamingRequest, false, e.getMessage());
} catch (IOException e) {
logger.error("Failed to deal with the request.", e);
@@ -257,10 +259,10 @@ public class StreamingController extends BasicController {
logger.debug("Saving KafkaConfig " + streamingRequest.getKafkaConfig());
desc = JsonUtil.readValue(streamingRequest.getKafkaConfig(), KafkaConfig.class);
} catch (JsonParseException e) {
- logger.error("The KafkaConfig definition is not valid.", e);
+ logger.error("The KafkaConfig definition is invalid.", e);
updateRequest(streamingRequest, false, e.getMessage());
} catch (JsonMappingException e) {
- logger.error("The data KafkaConfig definition is not valid.", e);
+ logger.error("The data KafkaConfig definition is invalid.", e);
updateRequest(streamingRequest, false, e.getMessage());
} catch (IOException e) {
logger.error("Failed to deal with the request.", e);
[2/2] kylin git commit: KYLIN-1650 Filesystem closed - in Cube Build
Step 2
Posted by sh...@apache.org.
KYLIN-1650 Filesystem closed - in Cube Build Step 2
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0c0bb078
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0c0bb078
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0c0bb078
Branch: refs/heads/master
Commit: 0c0bb078f3de11b0b2a0e4644b9134836727e59b
Parents: ecbff22
Author: shaofengshi <sh...@apache.org>
Authored: Mon May 9 14:20:14 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon May 9 15:32:01 2016 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0c0bb078/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index c07738d..2649492 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -44,6 +44,8 @@ import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*/
@@ -62,6 +64,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<Text, Text, NullWri
private boolean isStatistics = false;
private boolean outputTouched = false;
private KylinConfig cubeConfig;
+ protected static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
@Override
protected void setup(Context context) throws IOException {
@@ -100,7 +103,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<Text, Text, NullWri
if (isStatistics == false) {
colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, key.getLength() - 1)));
if (colValues.size() == 1000000) { //spill every 1 million
- System.out.println("spill values to disk...");
+ logger.info("spill values to disk...");
outputDistinctValues(col, colValues, context);
colValues.clear();
}
@@ -138,10 +141,10 @@ public class FactDistinctColumnsReducer extends KylinReducer<Text, Text, NullWri
try {
if (fs.exists(outputFile)) {
out = fs.append(outputFile);
- System.out.println("append file " + outputFile);
+ logger.info("append file " + outputFile);
} else {
out = fs.create(outputFile);
- System.out.println("create file " + outputFile);
+ logger.info("create file " + outputFile);
}
for (ByteArray value: values) {
@@ -150,7 +153,6 @@ public class FactDistinctColumnsReducer extends KylinReducer<Text, Text, NullWri
}
} finally {
IOUtils.closeQuietly(out);
- IOUtils.closeQuietly(fs);
outputTouched = true;
}
}