You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2013/11/15 06:50:18 UTC
git commit: TAJO-309: Remove unused fields in FileFragment. (jihoon)
Updated Branches:
refs/heads/master 943899360 -> 67ae6bc7a
TAJO-309: Remove unused fields in FileFragment. (jihoon)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/67ae6bc7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/67ae6bc7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/67ae6bc7
Branch: refs/heads/master
Commit: 67ae6bc7ac6c99ed0ee203698254d44e1c1cda93
Parents: 9438993
Author: Jihoon Son <ji...@apache.org>
Authored: Fri Nov 15 14:49:44 2013 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Fri Nov 15 14:49:44 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../src/main/proto/CatalogProtos.proto | 10 ++---
.../tajo/master/querymaster/QueryUnit.java | 12 +-----
.../apache/tajo/storage/TestFileFragment.java | 4 --
.../tajo/storage/AbstractStorageManager.java | 4 +-
.../tajo/storage/fragment/FileFragment.java | 41 +++-----------------
.../apache/tajo/storage/v2/TestCSVScanner.java | 2 +-
7 files changed, 15 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/67ae6bc7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3e55e2e..6128a7b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -63,6 +63,8 @@ Release 0.2.0 - unreleased
IMPROVEMENTS
+ TAJO-309: Remove unused fields in FileFragment. (jihoon)
+
TAJO-297: Rename JDBC variables in CatalogConstants to be more generic. (jaehwa)
TAJO-287: Improve Fragment to be more generic. (hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/67ae6bc7/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index 9c57bf3..acfa4fb 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -87,12 +87,10 @@ message FragmentProto {
}
message FileFragmentProto {
- required string id = 1;
- required string path = 2;
- required int64 startOffset = 3;
- required int64 length = 4;
- optional TableStatsProto stats = 5;
- optional bool distCached = 6 [default = false];
+ required string id = 1;
+ required string path = 2;
+ required int64 startOffset = 3;
+ required int64 length = 4;
repeated string hosts = 7;
repeated int32 diskIds = 8;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/67ae6bc7/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java
index 5c6f79f..cf8d514 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java
@@ -136,12 +136,11 @@ public class QueryUnit implements EventHandler<TaskEvent> {
public void setDataLocations(FileFragment fragment) {
String[] hosts = fragment.getHosts();
- int[] blockCount = fragment.getHostsBlockCount();
int[] volumeIds = fragment.getDiskIds();
this.dataLocations = new ArrayList<DataLocation>(hosts.length);
for (int i = 0; i < hosts.length; i++) {
- this.dataLocations.add(new DataLocation(hosts[i], blockCount[i], volumeIds[i]));
+ this.dataLocations.add(new DataLocation(hosts[i], volumeIds[i]));
}
}
@@ -531,12 +530,10 @@ public class QueryUnit implements EventHandler<TaskEvent> {
public static class DataLocation {
private String host;
- private int blockCount; // for Non-Splittable
private int volumeId;
- public DataLocation(String host, int blockCount, int volumeId) {
+ public DataLocation(String host, int volumeId) {
this.host = host;
- this.blockCount = blockCount;
this.volumeId = volumeId;
}
@@ -544,10 +541,6 @@ public class QueryUnit implements EventHandler<TaskEvent> {
return host;
}
- public int getBlockCount() {
- return blockCount;
- }
-
public int getVolumeId() {
return volumeId;
}
@@ -556,7 +549,6 @@ public class QueryUnit implements EventHandler<TaskEvent> {
public String toString() {
return "DataLocation{" +
"host=" + host +
- ", blocks=" + blockCount +
", volumeId=" + volumeId +
'}';
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/67ae6bc7/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestFileFragment.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestFileFragment.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestFileFragment.java
index 684cb58..e6ff7a3 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestFileFragment.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestFileFragment.java
@@ -43,11 +43,9 @@ public class TestFileFragment {
@Test
public final void testGetAndSetFields() {
FileFragment fragment1 = new FileFragment("table1_1", new Path(path, "table0"), 0, 500);
- fragment1.setDistCached();
assertEquals("table1_1", fragment1.getTableName());
assertEquals(new Path(path, "table0"), fragment1.getPath());
- assertTrue(fragment1.isDistCached());
assertTrue(0 == fragment1.getStartKey());
assertTrue(500 == fragment1.getEndKey());
}
@@ -55,12 +53,10 @@ public class TestFileFragment {
@Test
public final void testGetProtoAndRestore() {
FileFragment fragment = new FileFragment("table1_1", new Path(path, "table0"), 0, 500);
- fragment.setDistCached();
FileFragment fragment1 = FragmentConvertor.convert(FileFragment.class, fragment.getProto());
assertEquals("table1_1", fragment1.getTableName());
assertEquals(new Path(path, "table0"), fragment1.getPath());
- assertTrue(fragment.isDistCached());
assertTrue(0 == fragment1.getStartKey());
assertTrue(500 == fragment1.getEndKey());
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/67ae6bc7/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
index efdd023..6c584d8 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
@@ -474,14 +474,12 @@ public abstract class AbstractStorageManager {
});
String[] hosts = new String[blkLocations[0].getHosts().length];
- int[] hostsBlockCount = new int[blkLocations[0].getHosts().length];
for (int i = 0; i < hosts.length; i++) {
Map.Entry<String, Integer> entry = entries.get((entries.size() - 1) - i);
hosts[i] = entry.getKey();
- hostsBlockCount[i] = entry.getValue();
}
- return new FileFragment(fragmentId, file, start, length, hosts, hostsBlockCount);
+ return new FileFragment(fragmentId, file, start, length, hosts);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/67ae6bc7/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java
index dc6228c..bbf98e3 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java
@@ -39,10 +39,8 @@ public class FileFragment implements Fragment, Comparable<FileFragment> {
@Expose private Path uri; // required
@Expose private Long startOffset; // required
@Expose private Long length; // required
- @Expose private boolean distCached = false; // optional
private String[] hosts; // Datanode hostnames
- @Expose private int[] hostsBlockCount; // list of block count of hosts
@Expose private int[] diskIds;
public FileFragment(ByteString raw) throws InvalidProtocolBufferException {
@@ -59,11 +57,9 @@ public class FileFragment implements Fragment, Comparable<FileFragment> {
}
// Non splittable
- public FileFragment(String tableName, Path uri, long start, long length, String[] hosts,
- int[] hostsBlockCount) {
+ public FileFragment(String tableName, Path uri, long start, long length, String[] hosts) {
this.set(tableName, uri, start, length, null, null);
this.hosts = hosts;
- this.hostsBlockCount = hostsBlockCount;
}
public FileFragment(String fragmentId, Path path, long start, long length) {
@@ -84,9 +80,6 @@ public class FileFragment implements Fragment, Comparable<FileFragment> {
proto.getStartOffset(), proto.getLength(),
proto.getHostsList().toArray(new String[]{}),
diskIds);
- if (proto.hasDistCached() && proto.getDistCached()) {
- distCached = true;
- }
}
private void set(String tableName, Path path, long start,
@@ -111,18 +104,6 @@ public class FileFragment implements Fragment, Comparable<FileFragment> {
}
/**
- * Get the list of hosts block count
- * if a fragment given multiple block, it returned 'host0:3, host1:1 ...'
- */
- public int[] getHostsBlockCount() {
- if (hostsBlockCount == null) {
- this.hostsBlockCount = new int[getHosts().length];
- Arrays.fill(this.hostsBlockCount, 1);
- }
- return hostsBlockCount;
- }
-
- /**
* Get the list of Disk Ids
* Unknown disk is -1. Others 0 ~ N
*/
@@ -154,14 +135,6 @@ public class FileFragment implements Fragment, Comparable<FileFragment> {
return this.length;
}
- public Boolean isDistCached() {
- return this.distCached;
- }
-
- public void setDistCached() {
- this.distCached = true;
- }
-
/**
*
* The offset range of tablets <b>MUST NOT</b> be overlapped.
@@ -191,8 +164,7 @@ public class FileFragment implements Fragment, Comparable<FileFragment> {
FileFragment t = (FileFragment) o;
if (getPath().equals(t.getPath())
&& TUtil.checkEquals(t.getStartKey(), this.getStartKey())
- && TUtil.checkEquals(t.getEndKey(), this.getEndKey())
- && TUtil.checkEquals(t.isDistCached(), this.isDistCached())) {
+ && TUtil.checkEquals(t.getEndKey(), this.getEndKey())) {
return true;
}
}
@@ -201,18 +173,16 @@ public class FileFragment implements Fragment, Comparable<FileFragment> {
@Override
public int hashCode() {
- return Objects.hashCode(tableName, uri, startOffset, length, isDistCached());
+ return Objects.hashCode(tableName, uri, startOffset, length);
}
public Object clone() throws CloneNotSupportedException {
FileFragment frag = (FileFragment) super.clone();
frag.tableName = tableName;
frag.uri = uri;
- frag.distCached = distCached;
frag.diskIds = diskIds;
frag.hosts = hosts;
- frag.hostsBlockCount = hostsBlockCount;
-
+
return frag;
}
@@ -220,7 +190,7 @@ public class FileFragment implements Fragment, Comparable<FileFragment> {
public String toString() {
return "\"fragment\": {\"id\": \""+ tableName +"\", \"path\": "
+getPath() + "\", \"start\": " + this.getStartKey() + ",\"length\": "
- + getEndKey() + ", \"distCached\": " + distCached + "}" ;
+ + getEndKey() + "}" ;
}
public FragmentProto getProto() {
@@ -229,7 +199,6 @@ public class FileFragment implements Fragment, Comparable<FileFragment> {
builder.setStartOffset(this.startOffset);
builder.setLength(this.length);
builder.setPath(this.uri.toString());
- builder.setDistCached(this.distCached);
if(diskIds != null) {
List<Integer> idList = new ArrayList<Integer>();
for(int eachId: diskIds) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/67ae6bc7/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
index ae60ddb..c356548 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
@@ -145,7 +145,7 @@ public class TestCSVScanner {
long startOffset = (64 * 1024 * 1024) * scanCount;
long length = Math.min(64 * 1024 * 1024, fileLength - startOffset);
- FileFragment fragment = new FileFragment("Test", path, startOffset, length, null, null);
+ FileFragment fragment = new FileFragment("Test", path, startOffset, length, null);
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment, schema);
scanner.init();
Tuple tuple = null;