You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2007/03/17 23:15:24 UTC
svn commit: r519461 [3/3] - in /lucene/hadoop/branches/branch-0.12: ./ bin/
docs/ lib/ src/c++/librecordio/ src/c++/librecordio/test/
src/contrib/streaming/src/java/org/apache/hadoop/streaming/
src/docs/src/documentation/content/xdocs/ src/java/org/apa...
Modified: lucene/hadoop/branches/branch-0.12/src/docs/src/documentation/content/xdocs/index.xml
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/docs/src/documentation/content/xdocs/index.xml?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/docs/src/documentation/content/xdocs/index.xml (original)
+++ lucene/hadoop/branches/branch-0.12/src/docs/src/documentation/content/xdocs/index.xml Sat Mar 17 15:15:19 2007
@@ -15,6 +15,15 @@
<title>News</title>
<section>
+ <title>17 March, 2007: release 0.12.1 available</title>
+ <p>For details see the <a
+ href="http://tinyurl.com/2kynuc">release notes</a>. The release can
+ be obtained from <a
+ href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
+ nearby mirror</a>.
+ </p> </section>
+
+ <section>
<title>2 March, 2007: release 0.12.0 available</title>
<p>For details see the <a
href="http://tinyurl.com/2vp248">release notes</a>. The release can
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSDataset.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSDataset.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSDataset.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSDataset.java Sat Mar 17 15:15:19 2007
@@ -42,17 +42,13 @@
class FSDir {
File dir;
int numBlocks = 0;
- int myIdx = 0;
FSDir children[];
- FSDir siblings[];
-
+ int lastChildIdx = 0;
/**
*/
- public FSDir(File dir, int myIdx, FSDir[] siblings)
+ public FSDir(File dir)
throws IOException {
this.dir = dir;
- this.myIdx = myIdx;
- this.siblings = siblings;
this.children = null;
if (! dir.exists()) {
if (! dir.mkdirs()) {
@@ -74,36 +70,61 @@
int curdir = 0;
for (int idx = 0; idx < files.length; idx++) {
if (files[idx].isDirectory()) {
- children[curdir] = new FSDir(files[idx], curdir, children);
+ children[curdir] = new FSDir(files[idx]);
curdir++;
}
}
}
}
}
+
+ public File addBlock( Block b, File src ) throws IOException {
+ //First try without creating subdirectories
+ File file = addBlock( b, src, false, false );
+ return ( file != null ) ? file : addBlock( b, src, true, true );
+ }
- /**
- */
- public File addBlock(Block b, File src) throws IOException {
+ private File addBlock( Block b, File src, boolean createOk,
+ boolean resetIdx ) throws IOException {
if (numBlocks < maxBlocksPerDir) {
File dest = new File(dir, b.getBlockName());
src.renameTo(dest);
numBlocks += 1;
return dest;
- } else {
- if (siblings != null && myIdx != (siblings.length-1)) {
- File dest = siblings[myIdx+1].addBlock(b, src);
- if (dest != null) { return dest; }
- }
- if (children == null) {
- children = new FSDir[maxBlocksPerDir];
- for (int idx = 0; idx < maxBlocksPerDir; idx++) {
- children[idx] = new FSDir(
- new File(dir, "subdir"+idx), idx, children);
+ }
+
+ if ( lastChildIdx < 0 && resetIdx ) {
+ //reset so that all children will be checked
+ lastChildIdx = random.nextInt( children.length );
+ }
+
+ if ( lastChildIdx >= 0 && children != null ) {
+ //Check if any child-tree has room for a block.
+ for (int i=0; i < children.length; i++) {
+ int idx = ( lastChildIdx + i )%children.length;
+ File file = children[idx].addBlock( b, src, false, resetIdx );
+ if ( file != null ) {
+ lastChildIdx = idx;
+ return file;
}
}
- return children[0].addBlock(b, src);
+ lastChildIdx = -1;
+ }
+
+ if ( !createOk ) {
+ return null;
+ }
+
+ if ( children == null || children.length == 0 ) {
+ children = new FSDir[maxBlocksPerDir];
+ for (int idx = 0; idx < maxBlocksPerDir; idx++) {
+ children[idx] = new FSDir( new File(dir, "subdir"+idx) );
+ }
}
+
+ //now pick a child randomly for creating a new set of subdirs.
+ lastChildIdx = random.nextInt( children.length );
+ return children[ lastChildIdx ].addBlock( b, src, true, false );
}
/**
@@ -171,13 +192,57 @@
}
void clearPath(File f) {
- if (dir.compareTo(f) == 0) numBlocks--;
- else {
- if ((siblings != null) && (myIdx != (siblings.length - 1)))
- siblings[myIdx + 1].clearPath(f);
- else if (children != null)
- children[0].clearPath(f);
+ String root = dir.getAbsolutePath();
+ String dir = f.getAbsolutePath();
+ if ( dir.startsWith( root ) ) {
+ String[] dirNames = dir.substring( root.length() ).
+ split( File.separator + "subdir" );
+ if ( clearPath( f, dirNames, 1 ) )
+ return;
+ }
+ clearPath( f, null, -1 );
+ }
+
+ /*
+ * dirNames is an array of string integers derived from
+ * usual directory structure data/subdirN/subdirXY/subdirM ...
+ * If dirName array is non-null, we only check the child at
+ * the children[dirNames[idx]]. This avoids iterating over
+ * children in common case. If directory structure changes
+ * in later versions, we need to revisit this.
+ */
+ private boolean clearPath( File f, String[] dirNames, int idx ) {
+ if ( ( dirNames == null || idx == dirNames.length ) &&
+ dir.compareTo(f) == 0) {
+ numBlocks--;
+ return true;
+ }
+
+ if ( dirNames != null ) {
+ //guess the child index from the directory name
+ if ( idx > ( dirNames.length - 1 ) || children == null ) {
+ return false;
+ }
+ int childIdx;
+ try {
+ childIdx = Integer.parseInt( dirNames[idx] );
+ } catch ( NumberFormatException ignored ) {
+ // layout changed? we could print a warning.
+ return false;
+ }
+ return ( childIdx >= 0 && childIdx < children.length ) ?
+ children[childIdx].clearPath( f, dirNames, idx+1 ) : false;
+ }
+
+ //guesses failed. back to blind iteration.
+ if ( children != null ) {
+ for(int i=0; i < children.length; i++) {
+ if ( children[i].clearPath( f, null, -1 ) ){
+ return true;
+ }
+ }
}
+ return false;
}
public String toString() {
@@ -203,7 +268,7 @@
this.usableDiskPct = conf.getFloat("dfs.datanode.du.pct",
(float) USABLE_DISK_PCT_DEFAULT);
this.dir = dir;
- this.dataDir = new FSDir(new File(dir, "data"), 0, null);
+ this.dataDir = new FSDir(new File(dir, "data"));
this.tmpDir = new File(dir, "tmp");
if (tmpDir.exists()) {
FileUtil.fullyDelete(tmpDir);
@@ -361,6 +426,7 @@
private int maxBlocksPerDir = 0;
private HashMap<Block,FSVolume> volumeMap = null;
private HashMap<Block,File> blockMap = null;
+ static Random random = new Random();
/**
* An FSDataset has a directory where it loads its data files.
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSNamesystem.java Sat Mar 17 15:15:19 2007
@@ -155,7 +155,7 @@
// We also store pending replication-orders.
// Set of: Block
//
- private UnderReplicationBlocks neededReplications = new UnderReplicationBlocks();
+ private UnderReplicatedBlocks neededReplications = new UnderReplicatedBlocks();
private PendingReplicationBlocks pendingReplications;
//
@@ -334,12 +334,12 @@
* Blocks have replication priority, with priority 0 indicating the highest
* Blocks have only one replicas has the highest
*/
- private class UnderReplicationBlocks {
+ private class UnderReplicatedBlocks {
private static final int LEVEL = 3;
TreeSet<Block>[] priorityQueues = new TreeSet[LEVEL];
/* constructor */
- UnderReplicationBlocks() {
+ UnderReplicatedBlocks() {
for(int i=0; i<LEVEL; i++) {
priorityQueues[i] = new TreeSet<Block>();
}
@@ -369,7 +369,7 @@
*/
private int getPriority(Block block,
int curReplicas, int expectedReplicas) {
- if (curReplicas>=expectedReplicas) {
+ if (curReplicas<=0 || curReplicas>=expectedReplicas) {
return LEVEL; // no need to replicate
} else if(curReplicas==1) {
return 0; // highest priority
@@ -414,16 +414,14 @@
/* remove a block from a under replication queue */
synchronized boolean remove(Block block,
int oldReplicas, int oldExpectedReplicas) {
- if(oldExpectedReplicas <= oldReplicas) {
- return false;
- }
int priLevel = getPriority(block, oldReplicas, oldExpectedReplicas);
return remove(block, priLevel);
}
/* remove a block from a under replication queue given a priority*/
private boolean remove(Block block, int priLevel ) {
- if( priorityQueues[priLevel].remove(block) ) {
+ if( priLevel >= 0 && priLevel < LEVEL
+ && priorityQueues[priLevel].remove(block) ) {
NameNode.stateChangeLog.debug(
"BLOCK* NameSystem.UnderReplicationBlock.remove: "
+ "Removing block " + block.getBlockName()
@@ -1562,6 +1560,7 @@
DatanodeDescriptor nodeDescr
= new DatanodeDescriptor( nodeReg, networkLocation, hostName );
unprotectedAddDatanode( nodeDescr );
+ clusterMap.add(nodeDescr);
getEditLog().logAddDatanode( nodeDescr );
// also treat the registration message as a heartbeat
@@ -1890,7 +1889,6 @@
void unprotectedAddDatanode( DatanodeDescriptor nodeDescr ) {
datanodeMap.put( nodeDescr.getStorageID(), nodeDescr );
- clusterMap.add(nodeDescr);
NameNode.stateChangeLog.debug(
"BLOCK* NameSystem.unprotectedAddDatanode: "
+ "node " + nodeDescr.getName() + " is added to datanodeMap." );
@@ -2115,16 +2113,19 @@
return block;
// filter out containingNodes that are marked for decommission.
- int numCurrentReplica = countContainingNodes(containingNodes);
+ int numCurrentReplica = countContainingNodes(containingNodes)
+ + pendingReplications.getNumReplicas(block);
// check whether safe replication is reached for the block
// only if it is a part of a files
incrementSafeBlockCount( numCurrentReplica );
-
+
// handle underReplication/overReplication
short fileReplication = fileINode.getReplication();
- if(neededReplications.contains(block)) {
- neededReplications.update(block, curReplicaDelta, 0);
+ if (numCurrentReplica >= fileReplication) {
+ neededReplications.remove(block);
+ } else {
+ neededReplications.update(block, curReplicaDelta, 0);
}
proccessOverReplicatedBlock( block, fileReplication );
return block;
@@ -2501,6 +2502,7 @@
* that are marked for decommission.
*/
private int countContainingNodes(Collection<DatanodeDescriptor> nodelist) {
+ if( nodelist == null ) return 0;
int count = 0;
for (Iterator<DatanodeDescriptor> it = nodelist.iterator();
it.hasNext(); ) {
@@ -2638,17 +2640,21 @@
filterDecommissionedNodes(containingNodes);
int numCurrentReplica = nodes.size() +
pendingReplications.getNumReplicas(block);
- DatanodeDescriptor targets[] = replicator.chooseTarget(
+ if (numCurrentReplica >= fileINode.getReplication()) {
+ it.remove();
+ } else {
+ DatanodeDescriptor targets[] = replicator.chooseTarget(
Math.min( fileINode.getReplication() - numCurrentReplica,
needed),
datanodeMap.get(srcNode.getStorageID()),
nodes, null, blockSize);
- if (targets.length > 0) {
- // Build items to return
- replicateBlocks.add(block);
- numCurrentReplicas.add(new Integer(numCurrentReplica));
- replicateTargetSets.add(targets);
- needed -= targets.length;
+ if (targets.length > 0) {
+ // Build items to return
+ replicateBlocks.add(block);
+ numCurrentReplicas.add(new Integer(numCurrentReplica));
+ replicateTargetSets.add(targets);
+ needed -= targets.length;
+ }
}
}
}
@@ -3528,6 +3534,8 @@
NameNode.stateChangeLog.info("STATE* Network topology has "
+clusterMap.getNumOfRacks()+" racks and "
+clusterMap.getNumOfLeaves()+ " datanodes");
+ NameNode.stateChangeLog.info("STATE* UnderReplicatedBlocks has "
+ +neededReplications.size()+" blocks" );
}
/**
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java Sat Mar 17 15:15:19 2007
@@ -135,11 +135,15 @@
long checksumBoundary = desired/bytesPerSum*bytesPerSum;
if(checksumBoundary != getPos()) {
datas.seek(checksumBoundary);
- sums.seek(HEADER_LENGTH + 4*(checksumBoundary/bytesPerSum));
+ if(sums != null) {
+ sums.seek(HEADER_LENGTH + 4*(checksumBoundary/bytesPerSum));
+ }
}
- sum.reset();
- inSum = 0;
+ if(sums != null) {
+ sum.reset();
+ inSum = 0;
+ }
// scan to desired position
int delta = (int)(desired - checksumBoundary);
@@ -217,14 +221,14 @@
datas.seek(oldPos);
if (seekToNewSource(oldPos)) {
- // Neither the data stream nor the checksum stream are being read
- // from different sources, meaning we'll still get a checksum error
- // if we try to do the read again. We throw an exception instead.
- throw ce;
+ // Since at least one of the sources is different,
+ // the read might succeed, so we'll retry.
+ retry = true;
} else {
- // Since at least one of the sources is different,
- // the read might succeed, so we'll retry.
- retry = true;
+ // Neither the data stream nor the checksum stream are being read
+ // from different sources, meaning we'll still get a checksum error
+ // if we try to do the read again. We throw an exception instead.
+ throw ce;
}
}
}
@@ -307,7 +311,7 @@
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
- return datas.seekToNewSource(targetPos) &&
+ return datas.seekToNewSource(targetPos) ||
sums.seekToNewSource(targetPos/bytesPerSum);
}
@@ -397,8 +401,10 @@
public void close() throws IOException {
writeSum();
- sums.close();
- super.close();
+ if(sums != null) {
+ sums.close();
+ }
+ out.close();
}
public static long getChecksumLength(long size, int bytesPerSum) {
@@ -599,7 +605,7 @@
} else {
Path[] srcs = listPaths(src);
for (Path srcFile : srcs) {
- copyToLocalFile(srcFile, dst, copyCrc);
+ copyToLocalFile(srcFile, new Path(dst, srcFile.getName()), copyCrc);
}
}
}
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/FSDataOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/FSDataOutputStream.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/FSDataOutputStream.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/FSDataOutputStream.java Sat Mar 17 15:15:19 2007
@@ -43,6 +43,10 @@
return position; // return cached position
}
+ public void close() throws IOException {
+ flush();
+ out.close();
+ }
}
private static class Buffer extends BufferedOutputStream {
@@ -62,6 +66,11 @@
buf[count++] = (byte)b;
}
}
+
+ public void close() throws IOException {
+ flush();
+ out.close();
+ }
}
public FSDataOutputStream(OutputStream out, int bufferSize)
@@ -76,5 +85,10 @@
public long getPos() throws IOException {
return ((Buffer)out).getPos();
+ }
+
+ public void close() throws IOException {
+ flush();
+ out.close();
}
}
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableComparator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableComparator.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableComparator.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableComparator.java Sat Mar 17 15:15:19 2007
@@ -187,7 +187,7 @@
i = i << 8;
i = i | (bytes[start+1+idx] & 0xFF);
}
- return (isNegative ? (i | 0x8000000000000000L) : i);
+ return (isNegative ? (i ^ -1L) : i);
}
/**
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableUtils.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableUtils.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableUtils.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableUtils.java Sat Mar 17 15:15:19 2007
@@ -279,7 +279,7 @@
int len = -112;
if (i < 0) {
- i &= 0x7FFFFFFFFFFFFFFFL; // reset the sign bit
+ i ^= -1L; // take one's complement'
len = -120;
}
@@ -320,7 +320,7 @@
i = i << 8;
i = i | (b & 0xFF);
}
- return (isNegative ? (i | 0x8000000000000000L) : i);
+ return (isNegative ? (i ^ -1L) : i);
}
/**
@@ -345,7 +345,7 @@
int len = -112;
if (i < 0) {
- i &= 0x7FFFFFFFFFFFFFFFL; // reset the sign bit
+ i ^= -1L; // take one's complement'
len = -120;
}
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/Counters.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/Counters.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/Counters.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/Counters.java Sat Mar 17 15:15:19 2007
@@ -21,13 +21,13 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.MissingResourceException;
import java.util.ResourceBundle;
-import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.commons.logging.*;
@@ -174,7 +174,7 @@
* @return Set of counter names.
*/
public synchronized Collection<String> getGroupNames() {
- return counters.keySet();
+ return new ArrayList<String>(counters.keySet());
}
/**
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobClient.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobClient.java Sat Mar 17 15:15:19 2007
@@ -617,11 +617,14 @@
private static void displayTaskLogs(String taskId, String baseUrl)
throws IOException {
- // Copy tasks's stdout of the JobClient
- getTaskLogs(taskId, new URL(baseUrl+"&filter=stdout"), System.out);
-
- // Copy task's stderr to stderr of the JobClient
- getTaskLogs(taskId, new URL(baseUrl+"&filter=stderr"), System.err);
+ // The tasktracker for a 'failed/killed' job might not be around...
+ if (baseUrl != null) {
+ // Copy tasks's stdout of the JobClient
+ getTaskLogs(taskId, new URL(baseUrl+"&filter=stdout"), System.out);
+
+ // Copy task's stderr to stderr of the JobClient
+ getTaskLogs(taskId, new URL(baseUrl+"&filter=stderr"), System.err);
+ }
}
private static void getTaskLogs(String taskId, URL taskLogUrl,
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobInProgress.java Sat Mar 17 15:15:19 2007
@@ -315,26 +315,29 @@
ttStatus.getHttpPort() + "/tasklog.jsp?plaintext=true&taskid=" +
status.getTaskId() + "&all=true";
}
-
+
+ TaskCompletionEvent taskEvent = null;
if (state == TaskStatus.State.SUCCEEDED) {
- this.taskCompletionEvents.add( new TaskCompletionEvent(
- taskCompletionEventTracker,
- status.getTaskId(),
- tip.idWithinJob(),
- status.getIsMap(),
- TaskCompletionEvent.Status.SUCCEEDED,
- httpTaskLogLocation ));
+ taskEvent = new TaskCompletionEvent(
+ taskCompletionEventTracker,
+ status.getTaskId(),
+ tip.idWithinJob(),
+ status.getIsMap(),
+ TaskCompletionEvent.Status.SUCCEEDED,
+ httpTaskLogLocation
+ );
tip.setSuccessEventNumber(taskCompletionEventTracker);
completedTask(tip, status, metrics);
} else if (state == TaskStatus.State.FAILED ||
state == TaskStatus.State.KILLED) {
- this.taskCompletionEvents.add( new TaskCompletionEvent(
- taskCompletionEventTracker,
- status.getTaskId(),
- tip.idWithinJob(),
- status.getIsMap(),
- TaskCompletionEvent.Status.FAILED,
- httpTaskLogLocation ));
+ taskEvent = new TaskCompletionEvent(
+ taskCompletionEventTracker,
+ status.getTaskId(),
+ tip.idWithinJob(),
+ status.getIsMap(),
+ TaskCompletionEvent.Status.FAILED,
+ httpTaskLogLocation
+ );
// Get the event number for the (possibly) previously successful
// task. If there exists one, then set that status to OBSOLETE
int eventNumber;
@@ -348,9 +351,13 @@
failedTask(tip, status.getTaskId(), status, status.getTaskTracker(),
wasRunning, wasComplete);
}
- }
- taskCompletionEventTracker++;
+ // Add the 'complete' task i.e. successful/failed
+ if (taskEvent != null) {
+ this.taskCompletionEvents.add(taskEvent);
+ taskCompletionEventTracker++;
+ }
+ }
//
// Update JobInProgress status
@@ -936,8 +943,8 @@
return null;
}
- public TaskCompletionEvent[] getTaskCompletionEvents(int fromEventId,
- int maxEvents) {
+ synchronized public TaskCompletionEvent[] getTaskCompletionEvents(
+ int fromEventId, int maxEvents) {
TaskCompletionEvent[] events = TaskCompletionEvent.EMPTY_ARRAY;
if( taskCompletionEvents.size() > fromEventId) {
int actualMax = Math.min(maxEvents,
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/LineRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/LineRecordReader.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/LineRecordReader.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/LineRecordReader.java Sat Mar 17 15:15:19 2007
@@ -113,7 +113,7 @@
if (start == end) {
return 0.0f;
} else {
- return (pos - start) / (end - start);
+ return Math.min(1.0f, (pos - start) / (float)(end - start));
}
}
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java Sat Mar 17 15:15:19 2007
@@ -144,6 +144,12 @@
*/
private Map<Integer, MapOutputLocation> retryFetches = new HashMap();
+ /**
+ * a TreeSet for needed map outputs
+ */
+ private Set <Integer> neededOutputs =
+ Collections.synchronizedSet(new TreeSet<Integer>());
+
/** Represents the result of an attempt to copy a map output */
private class CopyResult {
@@ -152,7 +158,10 @@
// the size of the file copied, -1 if the transfer failed
private final long size;
-
+
+ //a flag signifying whether a copy result is obsolete
+ private static final int OBSOLETE = -2;
+
CopyResult(MapOutputLocation loc, long size) {
this.loc = loc;
this.size = size;
@@ -160,6 +169,9 @@
public int getMapId() { return loc.getMapId(); }
public boolean getSuccess() { return size >= 0; }
+ public boolean isObsolete() {
+ return size == OBSOLETE;
+ }
public long getSize() { return size; }
public String getHost() { return loc.getHost(); }
public MapOutputLocation getLocation() { return loc; }
@@ -284,7 +296,9 @@
*/
private long copyOutput(MapOutputLocation loc
) throws IOException, InterruptedException {
-
+ if (!neededOutputs.contains(loc.getMapId())) {
+ return CopyResult.OBSOLETE;
+ }
String reduceId = reduceTask.getTaskId();
LOG.info(reduceId + " Copying " + loc.getMapTaskId() +
" output from " + loc.getHost() + ".");
@@ -297,16 +311,28 @@
tmpFilename = loc.getFile(inMemFileSys, localFileSys, shuffleMetrics,
tmpFilename, reduceTask.getPartition(),
STALLED_COPY_TIMEOUT);
+ if (!neededOutputs.contains(loc.getMapId())) {
+ if (tmpFilename != null) {
+ FileSystem fs = tmpFilename.getFileSystem(conf);
+ fs.delete(tmpFilename);
+ }
+ return CopyResult.OBSOLETE;
+ }
if (tmpFilename == null)
throw new IOException("File " + finalFilename + "-" + id +
" not created");
long bytes = -1;
// lock the ReduceTaskRunner while we do the rename
synchronized (ReduceTaskRunner.this) {
- // if we can't rename the file, something is broken (and IOException
- // will be thrown). This file could have been created in the inmemory
+ // This file could have been created in the inmemory
// fs or the localfs. So need to get the filesystem owning the path.
FileSystem fs = tmpFilename.getFileSystem(conf);
+ if (!neededOutputs.contains(loc.getMapId())) {
+ fs.delete(tmpFilename);
+ return CopyResult.OBSOLETE;
+ }
+ // if we can't rename the file, something is broken (and IOException
+ // will be thrown).
if (!fs.rename(tmpFilename, finalFilename)) {
fs.delete(tmpFilename);
throw new IOException("failure to rename map output " + tmpFilename);
@@ -332,6 +358,7 @@
mergeInProgress = true;
m.start();
}
+ neededOutputs.remove(loc.getMapId());
}
return bytes;
}
@@ -424,7 +451,6 @@
this.mapOutputFile.removeAll(reduceTask.getTaskId());
final int numOutputs = reduceTask.getNumMaps();
- List neededOutputs = new ArrayList(numOutputs);
Map<Integer, MapOutputLocation> knownOutputs =
new HashMap<Integer, MapOutputLocation>();
int numInFlight = 0, numCopied = 0;
@@ -484,23 +510,12 @@
List <MapOutputLocation> locs = queryJobTracker(fromEventId,
jobClient);
- // remove discovered outputs from needed list
- // and put them on the known list
- int gotLocs = (locs == null ? 0 : locs.size());
+ // put discovered them on the known list
for (int i=0; i < locs.size(); i++) {
- // check whether we actually need an output. It could happen
- // that a map task that successfully ran earlier got lost, but
- // if we already have copied the output of that unfortunate task
- // we need not copy it again from the new TT (we will ignore
- // the event for the new rescheduled execution)
- if(neededOutputs.remove(new Integer(locs.get(i).getMapId()))) {
- knownOutputs.put(new Integer(locs.get(i).getMapId()), locs.get(i));
- }
- else gotLocs--; //we don't need this output
-
+ knownOutputs.put(new Integer(locs.get(i).getMapId()), locs.get(i));
}
LOG.info(reduceTask.getTaskId() +
- " Got " + gotLocs +
+ " Got " + locs.size() +
" new map outputs from jobtracker and " + retryFetches.size() +
" map outputs from previous failures");
// clear the "failed" fetches hashmap
@@ -575,9 +590,13 @@
copyPhase.setStatus("copy (" + numCopied + " of " + numOutputs +
" at " +
mbpsFormat.format(transferRate) + " MB/s)");
+ } else if (cr.isObsolete()) {
+ //ignore
+ LOG.info(reduceTask.getTaskId() +
+ " Ignoring obsolete copy result for Map Task: " +
+ cr.getLocation().getMapTaskId() + " from host: " +
+ cr.getHost());
} else {
- // this copy failed, put it back onto neededOutputs
- neededOutputs.add(new Integer(cr.getMapId()));
retryFetches.put(new Integer(cr.getMapId()), cr.getLocation());
// wait a random amount of time for next contact
@@ -600,7 +619,6 @@
if (cr.getHost().equals(loc.getHost())) {
retryFetches.put(new Integer(loc.getMapId()), loc);
locIt.remove();
- neededOutputs.add(new Integer(loc.getMapId()));
}
}
}
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/SequenceFileRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/SequenceFileRecordReader.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/SequenceFileRecordReader.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/SequenceFileRecordReader.java Sat Mar 17 15:15:19 2007
@@ -44,7 +44,8 @@
if (split.getStart() > in.getPosition())
in.sync(split.getStart()); // sync to start
- more = in.getPosition() < end;
+ this.start = in.getPosition();
+ more = start < end;
}
@@ -104,7 +105,7 @@
if (end == start) {
return 0.0f;
} else {
- return (in.getPosition() - start) / (end - start);
+ return Math.min(1.0f, (in.getPosition() - start) / (float)(end - start));
}
}
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java Sat Mar 17 15:15:19 2007
@@ -323,7 +323,7 @@
//Map<String,MetricValue> metricUpdates = record.getMetricTable();
Map metricUpdates = record.getMetricTable();
- RecordMap recordMap = (RecordMap) bufferedData.get(recordName);
+ RecordMap recordMap = getRecordMap(recordName);
synchronized (recordMap) {
MetricMap metricMap = (MetricMap) recordMap.get(tagTable);
if (metricMap == null) {
@@ -349,6 +349,10 @@
}
}
+ private synchronized RecordMap getRecordMap(String recordName) {
+ return (RecordMap) bufferedData.get(recordName);
+ }
+
/**
* Adds two numbers, coercing the second to the type of the first.
*
@@ -382,7 +386,7 @@
String recordName = record.getRecordName();
TagMap tagTable = record.getTagTable();
- RecordMap recordMap = (RecordMap) bufferedData.get(recordName);
+ RecordMap recordMap = getRecordMap(recordName);
synchronized (recordMap) {
recordMap.remove(tagTable);
}
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java Sat Mar 17 15:15:19 2007
@@ -64,6 +64,9 @@
* @throws MetricsException if the tagName conflicts with the configuration
*/
public void setTag(String tagName, String tagValue) {
+ if (tagValue == null) {
+ tagValue = "";
+ }
tagTable.put(tagName, tagValue);
}
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/net/NetworkTopology.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/net/NetworkTopology.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/net/NetworkTopology.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/net/NetworkTopology.java Sat Mar 17 15:15:19 2007
@@ -197,9 +197,7 @@
}
}
if(parentNode==null) {
- throw new IllegalArgumentException( n.getName()
- + ", which is located at "
- + parent+", is not a decendent of " + currentPath);
+ return false;
}
// remove n from the parent node
boolean isRemoved = parentNode.remove( n );
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Record.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Record.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Record.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Record.java Sat Mar 17 15:15:19 2007
@@ -18,16 +18,62 @@
package org.apache.hadoop.record;
+import java.io.DataInput;
+import java.io.DataOutput;
import java.io.IOException;
+import org.apache.hadoop.io.WritableComparable;
/**
- * Interface that is implemented by generated classes.
+ * Abstract class that is extended by generated classes.
*
* @author Milind Bhandarkar
*/
-public interface Record extends Cloneable {
- void serialize(OutputArchive archive, String tag)
- throws IOException;
- void deserialize(InputArchive archive, String tag)
- throws IOException;
+public abstract class Record implements WritableComparable, Cloneable {
+
+ /**
+ * Serialize a record with tag (ususally field name)
+ * @param rout Record output destination
+ * @param tag record tag (Used only in tagged serialization e.g. XML)
+ */
+ public abstract void serialize(RecordOutput rout, String tag)
+ throws IOException;
+
+ /**
+ * Deserialize a record with a tag (usually field name)
+ * @param rin Record input source
+ * @param tag Record tag (Used only in tagged serialization e.g. XML)
+ */
+ public abstract void deserialize(RecordInput rin, String tag)
+ throws IOException;
+
+ // inheric javadoc
+ public abstract int compareTo (final Object peer) throws ClassCastException;
+
+ /**
+ * Serialize a record without a tag
+ * @param rout Record output destination
+ */
+ public void serialize(RecordOutput rout) throws IOException {
+ this.serialize(rout, "");
+ }
+
+ /**
+ * Deserialize a record without a tag
+ * @param rin Record input source
+ */
+ public void deserialize(RecordInput rin) throws IOException {
+ this.deserialize(rin, "");
+ }
+
+ // inherit javadoc
+ public void write(final DataOutput out) throws java.io.IOException {
+ BinaryRecordOutput bout = new BinaryRecordOutput(out);
+ this.serialize(bout);
+ }
+
+ // inherit javadoc
+ public void readFields(final DataInput din) throws java.io.IOException {
+ BinaryRecordInput rin = new BinaryRecordInput(din);
+ this.deserialize(rin);
+ }
}
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Utils.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Utils.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Utils.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Utils.java Sat Mar 17 15:15:19 2007
@@ -51,11 +51,15 @@
} else if (ch == '&') {
sb.append("&");
} else if (ch == '%') {
- sb.append("%25");
- } else if (ch < 0x20) {
+ sb.append("%0025");
+ } else if (ch < 0x20 ||
+ (ch > 0xD7FF && ch < 0xE000) ||
+ (ch > 0xFFFD)) {
sb.append("%");
- sb.append(hexchars[ch/16]);
- sb.append(hexchars[ch%16]);
+ sb.append(hexchars[(ch & 0xF000) >> 12]);
+ sb.append(hexchars[(ch & 0x0F00) >> 8]);
+ sb.append(hexchars[(ch & 0x00F0) >> 4]);
+ sb.append(hexchars[(ch & 0x000F)]);
} else {
sb.append(ch);
}
@@ -67,9 +71,9 @@
if (ch >= '0' && ch <= '9') {
return ch - '0';
} else if (ch >= 'A' && ch <= 'F') {
- return ch - 'A';
+ return ch - 'A' + 10;
} else if (ch >= 'a' && ch <= 'f') {
- return ch - 'a';
+ return ch - 'a' + 10;
}
return 0;
}
@@ -84,15 +88,16 @@
for (int idx = 0; idx < s.length();) {
char ch = s.charAt(idx++);
if (ch == '%') {
- char ch1 = s.charAt(idx++);
- char ch2 = s.charAt(idx++);
- char res = (char)(h2c(ch1)*16 + h2c(ch2));
+ int ch1 = h2c(s.charAt(idx++)) << 12;
+ int ch2 = h2c(s.charAt(idx++)) << 8;
+ int ch3 = h2c(s.charAt(idx++)) << 4;
+ int ch4 = h2c(s.charAt(idx++));
+ char res = (char)(ch1 | ch2 | ch3 | ch4);
sb.append(res);
} else {
sb.append(ch);
}
}
-
return sb.toString();
}
@@ -196,7 +201,7 @@
for (int idx = 0; idx < blen; idx++) {
char c1 = s.charAt(2*idx);
char c2 = s.charAt(2*idx+1);
- barr[idx] = Byte.parseByte(""+c1+c2, 16);
+ barr[idx] = (byte)Integer.parseInt(""+c1+c2, 16);
}
return new Buffer(barr);
}
@@ -230,9 +235,162 @@
for (int idx = 0; idx < blen; idx++) {
char c1 = s.charAt(2*idx+1);
char c2 = s.charAt(2*idx+2);
- barr[idx] = Byte.parseByte(""+c1+c2, 16);
+ barr[idx] = (byte)Integer.parseInt(""+c1+c2, 16);
}
return new Buffer(barr);
+ }
+
+ private static int utf8LenForCodePoint(final int cpt) throws IOException {
+ if (cpt >=0 && cpt <= 0x7F) {
+ return 1;
+ }
+ if (cpt >= 0x80 && cpt <= 0x07FF) {
+ return 2;
+ }
+ if ((cpt >= 0x0800 && cpt < 0xD800) ||
+ (cpt > 0xDFFF && cpt <= 0xFFFD)) {
+ return 3;
+ }
+ if (cpt >= 0x10000 && cpt <= 0x10FFFF) {
+ return 4;
+ }
+ throw new IOException("Illegal Unicode Codepoint "+
+ Integer.toHexString(cpt)+" in string.");
+ }
+
+ private static final int B10 = Integer.parseInt("10000000", 2);
+ private static final int B110 = Integer.parseInt("11000000", 2);
+ private static final int B1110 = Integer.parseInt("11100000", 2);
+ private static final int B11110 = Integer.parseInt("11110000", 2);
+ private static final int B11 = Integer.parseInt("11000000", 2);
+ private static final int B111 = Integer.parseInt("11100000", 2);
+ private static final int B1111 = Integer.parseInt("11110000", 2);
+ private static final int B11111 = Integer.parseInt("11111000", 2);
+
+ private static int writeUtf8(int cpt, final byte[] bytes, final int offset)
+ throws IOException {
+ if (cpt >=0 && cpt <= 0x7F) {
+ bytes[offset] = (byte) cpt;
+ return 1;
+ }
+ if (cpt >= 0x80 && cpt <= 0x07FF) {
+ bytes[offset+1] = (byte) (B10 | (cpt & 0x3F));
+ cpt = cpt >> 6;
+ bytes[offset] = (byte) (B110 | (cpt & 0x1F));
+ return 2;
+ }
+ if ((cpt >= 0x0800 && cpt < 0xD800) ||
+ (cpt > 0xDFFF && cpt <= 0xFFFD)) {
+ bytes[offset+2] = (byte) (B10 | (cpt & 0x3F));
+ cpt = cpt >> 6;
+ bytes[offset+1] = (byte) (B10 | (cpt & 0x3F));
+ cpt = cpt >> 6;
+ bytes[offset] = (byte) (B1110 | (cpt & 0x0F));
+ return 3;
+ }
+ if (cpt >= 0x10000 && cpt <= 0x10FFFF) {
+ bytes[offset+3] = (byte) (B10 | (cpt & 0x3F));
+ cpt = cpt >> 6;
+ bytes[offset+2] = (byte) (B10 | (cpt & 0x3F));
+ cpt = cpt >> 6;
+ bytes[offset+1] = (byte) (B10 | (cpt & 0x3F));
+ cpt = cpt >> 6;
+ bytes[offset] = (byte) (B11110 | (cpt & 0x07));
+ return 4;
+ }
+ throw new IOException("Illegal Unicode Codepoint "+
+ Integer.toHexString(cpt)+" in string.");
+ }
+
+ static void toBinaryString(final DataOutput out, final String str)
+ throws IOException {
+ final int strlen = str.length();
+ byte[] bytes = new byte[strlen*4]; // Codepoints expand to 4 bytes max
+ int utf8Len = 0;
+ int idx = 0;
+ while(idx < strlen) {
+ final int cpt = str.codePointAt(idx);
+ idx += Character.isSupplementaryCodePoint(cpt) ? 2 : 1;
+ utf8Len += writeUtf8(cpt, bytes, utf8Len);
+ }
+ writeVInt(out, utf8Len);
+ out.write(bytes, 0, utf8Len);
+ }
+
+ static boolean isValidCodePoint(int cpt) {
+ return !((cpt > 0x10FFFF) ||
+ (cpt >= 0xD800 && cpt <= 0xDFFF) ||
+ (cpt >= 0xFFFE && cpt <=0xFFFF));
+ }
+
+ private static int utf8ToCodePoint(int b1, int b2, int b3, int b4) {
+ int cpt = 0;
+ cpt = (((b1 & ~B11111) << 18) |
+ ((b2 & ~B11) << 12) |
+ ((b3 & ~B11) << 6) |
+ (b4 & ~B11));
+ return cpt;
+ }
+
+ private static int utf8ToCodePoint(int b1, int b2, int b3) {
+ int cpt = 0;
+ cpt = (((b1 & ~B1111) << 12) | ((b2 & ~B11) << 6) | (b3 & ~B11));
+ return cpt;
+ }
+
+ private static int utf8ToCodePoint(int b1, int b2) {
+ int cpt = 0;
+ cpt = (((b1 & ~B111) << 6) | (b2 & ~B11));
+ return cpt;
+ }
+
+ private static void checkB10(int b) throws IOException {
+ if ((b & B11) != B10) {
+ throw new IOException("Invalid UTF-8 representation.");
+ }
+ }
+
+ static String fromBinaryString(final DataInput din) throws IOException {
+ final int utf8Len = readVInt(din);
+ final byte[] bytes = new byte[utf8Len];
+ din.readFully(bytes);
+ int len = 0;
+ // For the most commmon case, i.e. ascii, numChars = utf8Len
+ StringBuilder sb = new StringBuilder(utf8Len);
+ while(len < utf8Len) {
+ int cpt = 0;
+ final int b1 = bytes[len++] & 0xFF;
+ if (b1 <= 0x7F) {
+ cpt = b1;
+ } else if ((b1 & B11111) == B11110) {
+ int b2 = bytes[len++] & 0xFF;
+ checkB10(b2);
+ int b3 = bytes[len++] & 0xFF;
+ checkB10(b3);
+ int b4 = bytes[len++] & 0xFF;
+ checkB10(b4);
+ cpt = utf8ToCodePoint(b1, b2, b3, b4);
+ } else if ((b1 & B1111) == B1110) {
+ int b2 = bytes[len++] & 0xFF;
+ checkB10(b2);
+ int b3 = bytes[len++] & 0xFF;
+ checkB10(b3);
+ cpt = utf8ToCodePoint(b1, b2, b3);
+ } else if ((b1 & B111) == B110) {
+ int b2 = bytes[len++] & 0xFF;
+ checkB10(b2);
+ cpt = utf8ToCodePoint(b1, b2);
+ } else {
+ throw new IOException("Invalid UTF-8 byte "+Integer.toHexString(b1)+
+ " at offset "+(len-1)+" in length of "+utf8Len);
+ }
+ if (!isValidCodePoint(cpt)) {
+ throw new IOException("Illegal Unicode Codepoint "+
+ Integer.toHexString(cpt)+" in stream.");
+ }
+ sb.appendCodePoint(cpt);
+ }
+ return sb.toString();
}
/** Parse a float from a byte array. */
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/CppGenerator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/CppGenerator.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/CppGenerator.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/CppGenerator.java Sat Mar 17 15:15:19 2007
@@ -46,20 +46,21 @@
FileWriter cc = new FileWriter(name+".cc");
FileWriter hh = new FileWriter(name+".hh");
- hh.write("#ifndef __"+name.toUpperCase().replace('.','_')+"__\n");
- hh.write("#define __"+name.toUpperCase().replace('.','_')+"__\n");
+ String fileName = (new File(name)).getName();
+ hh.write("#ifndef __"+fileName.toUpperCase().replace('.','_')+"__\n");
+ hh.write("#define __"+fileName.toUpperCase().replace('.','_')+"__\n");
hh.write("#include \"recordio.hh\"\n");
for (Iterator<JFile> iter = ilist.iterator(); iter.hasNext();) {
hh.write("#include \""+iter.next().getName()+".hh\"\n");
}
- cc.write("#include \""+name+".hh\"\n");
+ cc.write("#include \""+fileName+".hh\"\n");
for (Iterator<JRecord> iter = rlist.iterator(); iter.hasNext();) {
iter.next().genCppCode(hh, cc, options);
}
- hh.write("#endif //"+name.toUpperCase().replace('.','_')+"__\n");
+ hh.write("#endif //"+fileName.toUpperCase().replace('.','_')+"__\n");
hh.close();
cc.close();
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JBuffer.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JBuffer.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JBuffer.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JBuffer.java Sat Mar 17 15:15:19 2007
@@ -33,10 +33,6 @@
super("org.apache.hadoop.record.Buffer", "Buffer", "org.apache.hadoop.record.Buffer");
}
- void genWriteMethod(CodeBuffer cb, String fname, String tag) {
- cb.append("a_.writeBuffer("+fname+",\""+tag+"\");\n");
- }
-
void genCompareTo(CodeBuffer cb, String fname, String other) {
cb.append("ret = "+fname+".compareTo("+other+");\n");
}
@@ -65,7 +61,7 @@
cb.append("int z1 = org.apache.hadoop.record.Utils.getVIntSize(i1);\n");
cb.append("int z2 = org.apache.hadoop.record.Utils.getVIntSize(i2);\n");
cb.append("s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
- cb.append("int r1 = org.apache.hadoop.record.Utils.compareBytes(b1,s1,l1,b2,s2,l2);\n");
+ cb.append("int r1 = org.apache.hadoop.record.Utils.compareBytes(b1,s1,i1,b2,s2,i2);\n");
cb.append("if (r1 != 0) { return (r1<0)?-1:0; }\n");
cb.append("s1+=i1; s2+=i2; l1-=i1; l1-=i2;\n");
cb.append("}\n");
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JMap.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JMap.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JMap.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JMap.java Sat Mar 17 15:15:19 2007
@@ -79,14 +79,14 @@
}
cb.append("{\n");
incrLevel();
- cb.append("org.apache.hadoop.record.Index "+getId("midx")+" = a_.startMap(\""+tag+"\");\n");
+ cb.append("org.apache.hadoop.record.Index "+getId("midx")+" = a.startMap(\""+tag+"\");\n");
cb.append(fname+"=new "+getType()+"();\n");
cb.append("for (; !"+getId("midx")+".done(); "+getId("midx")+".incr()) {\n");
key.genReadMethod(cb, getId("k"),getId("k"),true);
value.genReadMethod(cb, getId("v"),getId("v"),true);
cb.append(fname+".put("+getId("k")+","+getId("v")+");\n");
cb.append("}\n");
- cb.append("a_.endMap(\""+tag+"\");\n");
+ cb.append("a.endMap(\""+tag+"\");\n");
decrLevel();
cb.append("}\n");
}
@@ -100,7 +100,7 @@
key.getWrapperType()+","+value.getWrapperType()+">> ";
cb.append("{\n");
incrLevel();
- cb.append("a_.startMap("+fname+",\""+tag+"\");\n");
+ cb.append("a.startMap("+fname+",\""+tag+"\");\n");
cb.append(setType+getId("es")+" = "+fname+".entrySet();\n");
cb.append("for("+iterType+getId("midx")+" = "+getId("es")+".iterator(); "+getId("midx")+".hasNext(); ) {\n");
cb.append(entryType+getId("me")+" = "+getId("midx")+".next();\n");
@@ -109,7 +109,7 @@
key.genWriteMethod(cb, getId("k"),getId("k"));
value.genWriteMethod(cb, getId("v"),getId("v"));
cb.append("}\n");
- cb.append("a_.endMap("+fname+",\""+tag+"\");\n");
+ cb.append("a.endMap("+fname+",\""+tag+"\");\n");
cb.append("}\n");
decrLevel();
}
@@ -161,7 +161,7 @@
/** Creates a new instance of JMap */
public JMap(JType t1, JType t2) {
setJavaType(new JavaMap(t1.getJavaType(), t2.getJavaType()));
- setCppType(new CppType(" ::std::map<"+t1.getCppType().getType()+","+
+ setCppType(new CppCompType(" ::std::map<"+t1.getCppType().getType()+","+
t2.getCppType().getType()+">"));
setCType(new CType());
keyType = t1;
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JRecord.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JRecord.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JRecord.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JRecord.java Sat Mar 17 15:15:19 2007
@@ -55,11 +55,11 @@
cb.append(fullName+" "+fname+";\n");
}
cb.append(fname+"= new "+fullName+"();\n");
- cb.append("a_.readRecord("+fname+",\""+tag+"\");\n");
+ cb.append(fname+".deserialize(a,\""+tag+"\");\n");
}
void genWriteMethod(CodeBuffer cb, String fname, String tag) {
- cb.append("a_.writeRecord("+fname+",\""+tag+"\");\n");
+ cb.append(fname+".serialize(a,\""+tag+"\");\n");
}
void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
@@ -100,9 +100,7 @@
cb.append("// File generated by hadoop record compiler. Do not edit.\n");
cb.append("package "+module+";\n\n");
cb.append("public class "+name+
- " implements org.apache.hadoop.record.Record");
- cb.append(", org.apache.hadoop.io.WritableComparable");
- cb.append(" {\n");
+ " extends org.apache.hadoop.record.Record {\n");
for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
JField<JavaType> jf = i.next();
@@ -138,57 +136,44 @@
type.genGetSet(cb, name);
}
cb.append("public void serialize("+
- "final org.apache.hadoop.record.OutputArchive a_, final String tag)\n"+
+ "final org.apache.hadoop.record.RecordOutput a, final String tag)\n"+
"throws java.io.IOException {\n");
- cb.append("a_.startRecord(this,tag);\n");
+ cb.append("a.startRecord(this,tag);\n");
for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
JField<JavaType> jf = i.next();
String name = jf.getName();
JavaType type = jf.getType();
type.genWriteMethod(cb, name, name);
}
- cb.append("a_.endRecord(this,tag);\n");
+ cb.append("a.endRecord(this,tag);\n");
cb.append("}\n");
cb.append("public void deserialize("+
- "final org.apache.hadoop.record.InputArchive a_, final String tag)\n"+
+ "final org.apache.hadoop.record.RecordInput a, final String tag)\n"+
"throws java.io.IOException {\n");
- cb.append("a_.startRecord(tag);\n");
+ cb.append("a.startRecord(tag);\n");
for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
JField<JavaType> jf = i.next();
String name = jf.getName();
JavaType type = jf.getType();
type.genReadMethod(cb, name, name, false);
}
- cb.append("a_.endRecord(tag);\n");
+ cb.append("a.endRecord(tag);\n");
cb.append("}\n");
cb.append("public String toString() {\n");
cb.append("try {\n");
cb.append("java.io.ByteArrayOutputStream s =\n");
cb.append(" new java.io.ByteArrayOutputStream();\n");
- cb.append("org.apache.hadoop.record.CsvOutputArchive a_ = \n");
- cb.append(" new org.apache.hadoop.record.CsvOutputArchive(s);\n");
- cb.append("this.serialize(a_,\"\");\n");
+ cb.append("org.apache.hadoop.record.CsvRecordOutput a = \n");
+ cb.append(" new org.apache.hadoop.record.CsvRecordOutput(s);\n");
+ cb.append("this.serialize(a);\n");
cb.append("return new String(s.toByteArray(), \"UTF-8\");\n");
cb.append("} catch (Throwable ex) {\n");
cb.append("throw new RuntimeException(ex);\n");
cb.append("}\n");
cb.append("}\n");
- cb.append("public void write(final java.io.DataOutput out)\n"+
- "throws java.io.IOException {\n");
- cb.append("org.apache.hadoop.record.BinaryOutputArchive archive =\n"+
- "new org.apache.hadoop.record.BinaryOutputArchive(out);\n");
- cb.append("this.serialize(archive, \"\");\n");
- cb.append("}\n");
-
- cb.append("public void readFields(final java.io.DataInput in)\n"+
- "throws java.io.IOException {\n");
- cb.append("org.apache.hadoop.record.BinaryInputArchive archive =\n"+
- "new org.apache.hadoop.record.BinaryInputArchive(in);\n");
- cb.append("this.deserialize(archive, \"\");\n");
- cb.append("}\n");
cb.append("public int compareTo (final Object peer_) throws ClassCastException {\n");
cb.append("if (!(peer_ instanceof "+name+")) {\n");
cb.append("throw new ClassCastException(\"Comparing different types of records.\");\n");
@@ -253,7 +238,7 @@
cb.append("}\n");
cb.append("public static class Comparator extends"+
- " org.apache.hadoop.io.WritableComparator {\n");
+ " org.apache.hadoop.record.RecordComparator {\n");
cb.append("public Comparator() {\n");
cb.append("super("+name+".class);\n");
cb.append("}\n");
@@ -295,7 +280,7 @@
cb.append("}\n");
cb.append("}\n\n");
cb.append("static {\n");
- cb.append("org.apache.hadoop.io.WritableComparator.define("
+ cb.append("org.apache.hadoop.record.RecordComparator.define("
+name+".class, new Comparator());\n");
cb.append("}\n");
cb.append("}\n");
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JString.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JString.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JString.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JString.java Sat Mar 17 15:15:19 2007
@@ -48,7 +48,7 @@
cb.append("int z1 = org.apache.hadoop.record.Utils.getVIntSize(i1);\n");
cb.append("int z2 = org.apache.hadoop.record.Utils.getVIntSize(i2);\n");
cb.append("s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
- cb.append("int r1 = org.apache.hadoop.record.Utils.compareBytes(b1,s1,l1,b2,s2,l2);\n");
+ cb.append("int r1 = org.apache.hadoop.record.Utils.compareBytes(b1,s1,i1,b2,s2,i2);\n");
cb.append("if (r1 != 0) { return (r1<0)?-1:0; }\n");
cb.append("s1+=i1; s2+=i2; l1-=i1; l1-=i2;\n");
cb.append("}\n");
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JType.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JType.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JType.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JType.java Sat Mar 17 15:15:19 2007
@@ -80,14 +80,14 @@
}
void genWriteMethod(CodeBuffer cb, String fname, String tag) {
- cb.append("a_.write"+methodSuffix+"("+fname+",\""+tag+"\");\n");
+ cb.append("a.write"+methodSuffix+"("+fname+",\""+tag+"\");\n");
}
void genReadMethod(CodeBuffer cb, String fname, String tag, boolean decl) {
if (decl) {
cb.append(name+" "+fname+";\n");
}
- cb.append(fname+"=a_.read"+methodSuffix+"(\""+tag+"\");\n");
+ cb.append(fname+"=a.read"+methodSuffix+"(\""+tag+"\");\n");
}
void genCompareTo(CodeBuffer cb, String fname, String other) {
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JVector.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JVector.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JVector.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JVector.java Sat Mar 17 15:15:19 2007
@@ -75,13 +75,13 @@
}
cb.append("{\n");
incrLevel();
- cb.append("org.apache.hadoop.record.Index "+getId("vidx")+" = a_.startVector(\""+tag+"\");\n");
+ cb.append("org.apache.hadoop.record.Index "+getId("vidx")+" = a.startVector(\""+tag+"\");\n");
cb.append(fname+"=new "+getType()+"();\n");
cb.append("for (; !"+getId("vidx")+".done(); "+getId("vidx")+".incr()) {\n");
element.genReadMethod(cb, getId("e"), getId("e"), true);
cb.append(fname+".add("+getId("e")+");\n");
cb.append("}\n");
- cb.append("a_.endVector(\""+tag+"\");\n");
+ cb.append("a.endVector(\""+tag+"\");\n");
decrLevel();
cb.append("}\n");
}
@@ -89,13 +89,13 @@
void genWriteMethod(CodeBuffer cb, String fname, String tag) {
cb.append("{\n");
incrLevel();
- cb.append("a_.startVector("+fname+",\""+tag+"\");\n");
+ cb.append("a.startVector("+fname+",\""+tag+"\");\n");
cb.append("int "+getId("len")+" = "+fname+".size();\n");
cb.append("for(int "+getId("vidx")+" = 0; "+getId("vidx")+"<"+getId("len")+"; "+getId("vidx")+"++) {\n");
cb.append(element.getType()+" "+getId("e")+" = "+fname+".get("+getId("vidx")+");\n");
element.genWriteMethod(cb, getId("e"), getId("e"));
cb.append("}\n");
- cb.append("a_.endVector("+fname+",\""+tag+"\");\n");
+ cb.append("a.endVector("+fname+",\""+tag+"\");\n");
cb.append("}\n");
decrLevel();
}
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java Sat Mar 17 15:15:19 2007
@@ -43,6 +43,10 @@
System.exit(driver(args));
}
+ public static void usage() {
+ System.err.println("Usage: rcc --language [java|c++] ddl-files");
+ }
+
public static int driver(String[] args) {
for (int i=0; i<args.length; i++) {
if ("-l".equalsIgnoreCase(args[i]) ||
@@ -64,7 +68,7 @@
}
}
if (recFiles.size() == 0) {
- System.err.println("No record files specified. Exiting.");
+ usage();
return 1;
}
for (int i=0; i<recFiles.size(); i++) {
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj Sat Mar 17 15:15:19 2007
@@ -47,6 +47,10 @@
System.exit(driver(args));
}
+ public static void usage() {
+ System.err.println("Usage: rcc --language [java|c++] ddl-files");
+ }
+
public static int driver(String[] args) {
for (int i=0; i<args.length; i++) {
if ("-l".equalsIgnoreCase(args[i]) ||
@@ -68,7 +72,7 @@
}
}
if (recFiles.size() == 0) {
- System.err.println("No record files specified. Exiting.");
+ usage();
return 1;
}
for (int i=0; i<recFiles.size(); i++) {
Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/package.html
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/package.html?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/package.html (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/package.html Sat Mar 17 15:15:19 2007
@@ -479,9 +479,9 @@
private:
- int32_t mI32;
- double mD;
- std::string mS;
+ int32_t I32;
+ double D;
+ std::string S;
public:
@@ -491,14 +491,14 @@
virtual bool operator==(const RI& peer) const;
virtual bool operator<(const RI& peer) const;
- virtual int32_t getI32(void) const { return mI32; }
- virtual void setI32(int32_t v) { mI32 = v; }
+ virtual int32_t getI32(void) const { return I32; }
+ virtual void setI32(int32_t v) { I32 = v; }
- virtual double getD(void) const { return mD; }
- virtual void setD(double v) { mD = v; }
+ virtual double getD(void) const { return D; }
+ virtual void setD(double v) { D = v; }
- virtual std::string& getS(void) const { return mS; }
- virtual const std::string& getS(void) const { return mS; }
+ virtual std::string& getS(void) const { return S; }
+ virtual const std::string& getS(void) const { return S; }
virtual std::string type(void) const;
virtual std::string signature(void) const;
@@ -507,8 +507,6 @@
virtual void serialize(hadoop::OArchive& a) const;
virtual void deserialize(hadoop::IArchive& a);
-
- virtual bool validate(void);
};
} // end namespace inclrec
@@ -531,9 +529,9 @@
private:
- std::vector<float> mVF;
- inclrec::RI mRec;
- std::string mBuf;
+ std::vector<float> VF;
+ inclrec::RI Rec;
+ std::string Buf;
public:
@@ -595,11 +593,11 @@
long int64_t long
float float float
double double double
-ustring std::string Text
-buffer std::string java.io.ByteArrayOutputStream
+ustring std::string java.lang.String
+buffer std::string org.apache.hadoop.record.Buffer
class type class type class type
-vector<type> std::vector<type> java.util.ArrayList
-map<type,type> std::map<type,type> java.util.TreeMap
+vector<type> std::vector<type> java.util.ArrayList<type>
+map<type,type> std::map<type,type> java.util.TreeMap<type,type>
</code></pre>
<h2>Data encodings</h2>
Modified: lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/FromCpp.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/FromCpp.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/FromCpp.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/FromCpp.java Sat Mar 17 15:15:19 2007
@@ -18,7 +18,6 @@
package org.apache.hadoop.record;
-import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -58,9 +57,9 @@
r1.setVectorVal(new ArrayList<String>());
r1.setMapVal(new TreeMap<String,String>());
FileInputStream istream = new FileInputStream(tmpfile);
- RecordReader in = new RecordReader(istream, "binary");
+ BinaryRecordInput in = new BinaryRecordInput(istream);
RecRecord1 r2 = new RecRecord1();
- in.read(r2);
+ r2.deserialize(in, "");
istream.close();
assertTrue(r1.equals(r2));
} catch (IOException ex) {
@@ -84,9 +83,9 @@
r1.setVectorVal(new ArrayList<String>());
r1.setMapVal(new TreeMap<String,String>());
FileInputStream istream = new FileInputStream(tmpfile);
- RecordReader in = new RecordReader(istream, "csv");
+ CsvRecordInput in = new CsvRecordInput(istream);
RecRecord1 r2 = new RecRecord1();
- in.read(r2);
+ r2.deserialize(in, "");
istream.close();
assertTrue(r1.equals(r2));
} catch (IOException ex) {
@@ -110,9 +109,9 @@
r1.setVectorVal(new ArrayList<String>());
r1.setMapVal(new TreeMap<String,String>());
FileInputStream istream = new FileInputStream(tmpfile);
- RecordReader in = new RecordReader(istream, "xml");
+ XmlRecordInput in = new XmlRecordInput(istream);
RecRecord1 r2 = new RecRecord1();
- in.read(r2);
+ r2.deserialize(in, "");
istream.close();
assertTrue(r1.equals(r2));
} catch (IOException ex) {
Modified: lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/TestRecordIO.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/TestRecordIO.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/TestRecordIO.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/TestRecordIO.java Sat Mar 17 15:15:19 2007
@@ -47,7 +47,7 @@
try {
tmpfile = File.createTempFile("hadooprec", ".dat");
FileOutputStream ostream = new FileOutputStream(tmpfile);
- RecordWriter out = new RecordWriter(ostream, "binary");
+ BinaryRecordOutput out = new BinaryRecordOutput(ostream);
RecRecord1 r1 = new RecRecord1();
r1.setBoolVal(true);
r1.setByteVal((byte)0x66);
@@ -62,12 +62,12 @@
RecRecord0 r0 = new RecRecord0();
r0.setStringVal("other random text");
r1.setRecordVal(r0);
- out.write(r1);
+ r1.serialize(out, "");
ostream.close();
FileInputStream istream = new FileInputStream(tmpfile);
- RecordReader in = new RecordReader(istream, "binary");
+ BinaryRecordInput in = new BinaryRecordInput(istream);
RecRecord1 r2 = new RecRecord1();
- in.read(r2);
+ r2.deserialize(in, "");
istream.close();
tmpfile.delete();
assertTrue("Serialized and deserialized records do not match.", r1.equals(r2));
@@ -81,7 +81,7 @@
try {
tmpfile = File.createTempFile("hadooprec", ".txt");
FileOutputStream ostream = new FileOutputStream(tmpfile);
- RecordWriter out = new RecordWriter(ostream, "csv");
+ CsvRecordOutput out = new CsvRecordOutput(ostream);
RecRecord1 r1 = new RecRecord1();
r1.setBoolVal(true);
r1.setByteVal((byte)0x66);
@@ -96,12 +96,12 @@
RecRecord0 r0 = new RecRecord0();
r0.setStringVal("other random text");
r1.setRecordVal(r0);
- out.write(r1);
+ r1.serialize(out, "");
ostream.close();
FileInputStream istream = new FileInputStream(tmpfile);
- RecordReader in = new RecordReader(istream, "csv");
+ CsvRecordInput in = new CsvRecordInput(istream);
RecRecord1 r2 = new RecRecord1();
- in.read(r2);
+ r2.deserialize(in, "");
istream.close();
tmpfile.delete();
assertTrue("Serialized and deserialized records do not match.", r1.equals(r2));
@@ -140,7 +140,7 @@
try {
tmpfile = File.createTempFile("hadooprec", ".xml");
FileOutputStream ostream = new FileOutputStream(tmpfile);
- RecordWriter out = new RecordWriter(ostream, "xml");
+ XmlRecordOutput out = new XmlRecordOutput(ostream);
RecRecord1 r1 = new RecRecord1();
r1.setBoolVal(true);
r1.setByteVal((byte)0x66);
@@ -148,19 +148,19 @@
r1.setDoubleVal(1.5234);
r1.setIntVal(4567);
r1.setLongVal(0x5a5a5a5a5a5aL);
- r1.setStringVal("ran\002dom < %text<&more");
+ r1.setStringVal("ran\002dom < %text<&more\uffff");
r1.setBufferVal(new Buffer());
r1.setVectorVal(new ArrayList<String>());
r1.setMapVal(new TreeMap<String,String>());
RecRecord0 r0 = new RecRecord0();
r0.setStringVal("other %rando\007m & >&more text");
r1.setRecordVal(r0);
- out.write(r1);
+ r1.serialize(out, "");
ostream.close();
FileInputStream istream = new FileInputStream(tmpfile);
- RecordReader in = new RecordReader(istream, "xml");
+ XmlRecordInput in = new XmlRecordInput(istream);
RecRecord1 r2 = new RecRecord1();
- in.read(r2);
+ r2.deserialize(in, "");
istream.close();
tmpfile.delete();
assertTrue("Serialized and deserialized records do not match.", r1.equals(r2));
Modified: lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/ToCpp.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/ToCpp.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/ToCpp.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/ToCpp.java Sat Mar 17 15:15:19 2007
@@ -20,8 +20,6 @@
import java.io.IOException;
import junit.framework.*;
-import org.apache.hadoop.record.RecordWriter;
-import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.util.ArrayList;
@@ -46,9 +44,9 @@
public void testBinary() {
File tmpfile;
try {
- tmpfile = new File("/temp/hadooptemp.dat");
+ tmpfile = new File("/tmp/hadooptemp.dat");
FileOutputStream ostream = new FileOutputStream(tmpfile);
- RecordWriter out = new RecordWriter(ostream, "binary");
+ BinaryRecordOutput out = new BinaryRecordOutput(ostream);
RecRecord1 r1 = new RecRecord1();
r1.setBoolVal(true);
r1.setByteVal((byte)0x66);
@@ -60,7 +58,7 @@
r1.setBufferVal(new Buffer());
r1.setVectorVal(new ArrayList<String>());
r1.setMapVal(new TreeMap<String,String>());
- out.write(r1);
+ r1.serialize(out, "");
ostream.close();
} catch (IOException ex) {
ex.printStackTrace();
@@ -70,9 +68,9 @@
public void testCsv() {
File tmpfile;
try {
- tmpfile = new File("/temp/hadooptemp.txt");
+ tmpfile = new File("/tmp/hadooptemp.txt");
FileOutputStream ostream = new FileOutputStream(tmpfile);
- RecordWriter out = new RecordWriter(ostream, "csv");
+ CsvRecordOutput out = new CsvRecordOutput(ostream);
RecRecord1 r1 = new RecRecord1();
r1.setBoolVal(true);
r1.setByteVal((byte)0x66);
@@ -84,7 +82,7 @@
r1.setBufferVal(new Buffer());
r1.setVectorVal(new ArrayList<String>());
r1.setMapVal(new TreeMap<String,String>());
- out.write(r1);
+ r1.serialize(out, "");
ostream.close();
} catch (IOException ex) {
ex.printStackTrace();
@@ -94,9 +92,9 @@
public void testXml() {
File tmpfile;
try {
- tmpfile = new File("/temp/hadooptemp.xml");
+ tmpfile = new File("/tmp/hadooptemp.xml");
FileOutputStream ostream = new FileOutputStream(tmpfile);
- RecordWriter out = new RecordWriter(ostream, "xml");
+ XmlRecordOutput out = new XmlRecordOutput(ostream);
RecRecord1 r1 = new RecRecord1();
r1.setBoolVal(true);
r1.setByteVal((byte)0x66);
@@ -108,7 +106,7 @@
r1.setBufferVal(new Buffer());
r1.setVectorVal(new ArrayList<String>());
r1.setMapVal(new TreeMap<String,String>());
- out.write(r1);
+ r1.serialize(out, "");
ostream.close();
} catch (IOException ex) {
ex.printStackTrace();
Modified: lucene/hadoop/branches/branch-0.12/src/webapps/job/jobfailures.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/webapps/job/jobfailures.jsp?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/webapps/job/jobfailures.jsp (original)
+++ lucene/hadoop/branches/branch-0.12/src/webapps/job/jobfailures.jsp Sat Mar 17 15:15:19 2007
@@ -24,7 +24,7 @@
TaskTrackerStatus taskTracker = tracker.getTaskTracker(taskTrackerName);
out.print("<tr><td>" + statuses[i].getTaskId() +
"</td><td><a href=\"/taskdetails.jsp?jobid="+ jobId +
- "&taskid=" + tipId + "\">" + tipId +
+ "&tipid=" + tipId + "\">" + tipId +
"</a></td>");
if (taskTracker == null) {
out.print("<td>" + taskTrackerName + "</td>");