You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by jn...@apache.org on 2014/05/15 10:10:09 UTC
svn commit: r1594812 [2/5] - in /nutch/branches/2.x: ./ ivy/ src/gora/
src/java/org/apache/nutch/api/ src/java/org/apache/nutch/crawl/
src/java/org/apache/nutch/fetcher/ src/java/org/apache/nutch/host/
src/java/org/apache/nutch/indexer/ src/java/org/ap...
Modified: nutch/branches/2.x/src/java/org/apache/nutch/storage/Mark.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/storage/Mark.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/storage/Mark.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/storage/Mark.java Thu May 15 08:10:07 2014
@@ -29,7 +29,7 @@ public enum Mark {
}
public void putMark(WebPage page, Utf8 markValue) {
- page.putToMarkers(name, markValue);
+ page.getMarkers().put(name, markValue);
}
public void putMark(WebPage page, String markValue) {
@@ -37,11 +37,11 @@ public enum Mark {
}
public Utf8 removeMark(WebPage page) {
- return page.removeFromMarkers(name);
+ return (Utf8) page.getMarkers().put(name, null);
}
public Utf8 checkMark(WebPage page) {
- return page.getFromMarkers(name);
+ return (Utf8) page.getMarkers().get(name);
}
/**
@@ -50,8 +50,8 @@ public enum Mark {
* @return If the mark was present.
*/
public Utf8 removeMarkIfExist(WebPage page) {
- if (page.getFromMarkers(name) != null) {
- return page.removeFromMarkers(name);
+ if (checkMark(page) != null) {
+ return removeMark(page);
}
return null;
}
Modified: nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/storage/ParseStatus.java Thu May 15 08:10:07 2014
@@ -1,113 +1,416 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+/**
+ * Autogenerated by Avro
*
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-
-package org.apache.nutch.storage;
-
-import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.avro.Protocol;
-import org.apache.avro.Schema;
-import org.apache.avro.AvroRuntimeException;
-import org.apache.avro.Protocol;
-import org.apache.avro.util.Utf8;
-import org.apache.avro.ipc.AvroRemoteException;
-import org.apache.avro.generic.GenericArray;
-import org.apache.avro.specific.FixedSize;
-import org.apache.avro.specific.SpecificExceptionBase;
-import org.apache.avro.specific.SpecificRecordBase;
-import org.apache.avro.specific.SpecificRecord;
-import org.apache.avro.specific.SpecificFixed;
-import org.apache.gora.persistency.StateManager;
-import org.apache.gora.persistency.impl.PersistentBase;
-import org.apache.gora.persistency.impl.StateManagerImpl;
-import org.apache.gora.persistency.StatefulHashMap;
-import org.apache.gora.persistency.ListGenericArray;
-
+ * DO NOT EDIT DIRECTLY
+ */
+package org.apache.nutch.storage;
@SuppressWarnings("all")
-public class ParseStatus extends PersistentBase {
- public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"ParseStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"majorCode\",\"type\":\"int\"},{\"name\":\"minorCode\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"}}]}");
+public class ParseStatus extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent {
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ParseStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"majorCode\",\"type\":\"int\",\"default\":0},{\"name\":\"minorCode\",\"type\":\"int\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]}]}");
+
+ /** Enum containing all data bean's fields. */
public static enum Field {
- MAJOR_CODE(0,"majorCode"),
- MINOR_CODE(1,"minorCode"),
- ARGS(2,"args"),
+ __G__DIRTY(0, "__g__dirty"),
+ MAJOR_CODE(1, "majorCode"),
+ MINOR_CODE(2, "minorCode"),
+ ARGS(3, "args"),
;
+ /**
+ * Field's index.
+ */
private int index;
+
+ /**
+ * Field's name.
+ */
private String name;
+
+ /**
+ * Field's constructor
+ * @param index field's index.
+ * @param name field's name.
+ */
Field(int index, String name) {this.index=index;this.name=name;}
+
+ /**
+ * Gets field's index.
+ * @return int field's index.
+ */
public int getIndex() {return index;}
+
+ /**
+ * Gets field's name.
+ * @return String field's name.
+ */
public String getName() {return name;}
+
+ /**
+ * Gets field's attributes to string.
+ * @return String field's attributes to string.
+ */
public String toString() {return name;}
};
- public static final String[] _ALL_FIELDS = {"majorCode","minorCode","args",};
- static {
- PersistentBase.registerFields(ParseStatus.class, _ALL_FIELDS);
- }
+
+ public static final String[] _ALL_FIELDS = {
+ "__g__dirty",
+ "majorCode",
+ "minorCode",
+ "args",
+ };
+
+ /** Bytes used to represent weather or not a field is dirty. */
+ private java.nio.ByteBuffer __g__dirty = java.nio.ByteBuffer.wrap(new byte[1]);
private int majorCode;
private int minorCode;
- private GenericArray<Utf8> args;
- public ParseStatus() {
- this(new StateManagerImpl());
- }
- public ParseStatus(StateManager stateManager) {
- super(stateManager);
- args = new ListGenericArray<Utf8>(getSchema().getField("args").schema());
- }
- public ParseStatus newInstance(StateManager stateManager) {
- return new ParseStatus(stateManager);
- }
- public Schema getSchema() { return _SCHEMA; }
- public Object get(int _field) {
- switch (_field) {
- case 0: return majorCode;
- case 1: return minorCode;
- case 2: return args;
- default: throw new AvroRuntimeException("Bad index");
+ private java.util.List<CharSequence> args;
+ public org.apache.avro.Schema getSchema() { return SCHEMA$; }
+ // Used by DatumWriter. Applications should not call.
+ public Object get(int field$) {
+ switch (field$) {
+ case 0: return __g__dirty;
+ case 1: return majorCode;
+ case 2: return minorCode;
+ case 3: return args;
+ default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
+
+ // Used by DatumReader. Applications should not call.
@SuppressWarnings(value="unchecked")
- public void put(int _field, Object _value) {
- if(isFieldEqual(_field, _value)) return;
- getStateManager().setDirty(this, _field);
- switch (_field) {
- case 0:majorCode = (Integer)_value; break;
- case 1:minorCode = (Integer)_value; break;
- case 2:args = (GenericArray<Utf8>)_value; break;
- default: throw new AvroRuntimeException("Bad index");
+ public void put(int field$, Object value) {
+ switch (field$) {
+ case 0: __g__dirty = (java.nio.ByteBuffer)(value); break;
+ case 1: majorCode = (Integer)(value); break;
+ case 2: minorCode = (Integer)(value); break;
+ case 3: args = (java.util.List<CharSequence>)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)value)); break;
+ default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
- public int getMajorCode() {
- return (Integer) get(0);
+
+ /**
+ * Gets the value of the 'majorCode' field.
+ */
+ public Integer getMajorCode() {
+ return majorCode;
+ }
+
+ /**
+ * Sets the value of the 'majorCode' field.
+ * @param value the value to set.
+ */
+ public void setMajorCode(Integer value) {
+ this.majorCode = value;
+ setDirty(1);
+ }
+
+ /**
+ * Checks the dirty status of the 'majorCode' field. A field is dirty if it represents a change that has not yet been written to the database.
+ * @param value the value to set.
+ */
+ public boolean isMajorCodeDirty(Integer value) {
+ return isDirty(1);
+ }
+
+ /**
+ * Gets the value of the 'minorCode' field.
+ */
+ public Integer getMinorCode() {
+ return minorCode;
}
- public void setMajorCode(int value) {
- put(0, value);
+
+ /**
+ * Sets the value of the 'minorCode' field.
+ * @param value the value to set.
+ */
+ public void setMinorCode(Integer value) {
+ this.minorCode = value;
+ setDirty(2);
+ }
+
+ /**
+ * Checks the dirty status of the 'minorCode' field. A field is dirty if it represents a change that has not yet been written to the database.
+ * @param value the value to set.
+ */
+ public boolean isMinorCodeDirty(Integer value) {
+ return isDirty(2);
+ }
+
+ /**
+ * Gets the value of the 'args' field.
+ */
+ public java.util.List<CharSequence> getArgs() {
+ return args;
+ }
+
+ /**
+ * Sets the value of the 'args' field.
+ * @param value the value to set.
+ */
+ public void setArgs(java.util.List<CharSequence> value) {
+ this.args = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper(value);
+ setDirty(3);
+ }
+
+ /**
+ * Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database.
+ * @param value the value to set.
+ */
+ public boolean isArgsDirty(java.util.List<CharSequence> value) {
+ return isDirty(3);
}
- public int getMinorCode() {
- return (Integer) get(1);
+
+ /** Creates a new ParseStatus RecordBuilder */
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ /** Creates a new ParseStatus RecordBuilder by copying an existing Builder */
+ public static Builder newBuilder(Builder other) {
+ return new Builder(other);
+ }
+
+ /** Creates a new ParseStatus RecordBuilder by copying an existing ParseStatus instance */
+ public static Builder newBuilder(ParseStatus other) {
+ return new Builder(other);
+ }
+
+ private static java.nio.ByteBuffer deepCopyToWriteOnlyBuffer(
+ java.nio.ByteBuffer input) {
+ java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity());
+ int position = input.position();
+ input.reset();
+ int mark = input.position();
+ int limit = input.limit();
+ input.rewind();
+ input.limit(input.capacity());
+ copy.put(input);
+ input.rewind();
+ copy.rewind();
+ input.position(mark);
+ input.mark();
+ copy.position(mark);
+ copy.mark();
+ input.position(position);
+ copy.position(position);
+ input.limit(limit);
+ copy.limit(limit);
+ return copy.asReadOnlyBuffer();
+ }
+
+ /**
+ * RecordBuilder for ParseStatus instances.
+ */
+ public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<ParseStatus>
+ implements org.apache.avro.data.RecordBuilder<ParseStatus> {
+
+ private java.nio.ByteBuffer __g__dirty;
+ private int majorCode;
+ private int minorCode;
+ private java.util.List<CharSequence> args;
+
+ /** Creates a new Builder */
+ private Builder() {
+ super(ParseStatus.SCHEMA$);
+ }
+
+ /** Creates a Builder by copying an existing Builder */
+ private Builder(Builder other) {
+ super(other);
+ }
+
+ /** Creates a Builder by copying an existing ParseStatus instance */
+ private Builder(ParseStatus other) {
+ super(ParseStatus.SCHEMA$);
+ if (isValidValue(fields()[0], other.__g__dirty)) {
+ this.__g__dirty = (java.nio.ByteBuffer) data().deepCopy(fields()[0].schema(), other.__g__dirty);
+ fieldSetFlags()[0] = true;
+ }
+ if (isValidValue(fields()[1], other.majorCode)) {
+ this.majorCode = (Integer) data().deepCopy(fields()[1].schema(), other.majorCode);
+ fieldSetFlags()[1] = true;
+ }
+ if (isValidValue(fields()[2], other.minorCode)) {
+ this.minorCode = (Integer) data().deepCopy(fields()[2].schema(), other.minorCode);
+ fieldSetFlags()[2] = true;
+ }
+ if (isValidValue(fields()[3], other.args)) {
+ this.args = (java.util.List<CharSequence>) data().deepCopy(fields()[3].schema(), other.args);
+ fieldSetFlags()[3] = true;
+ }
+ }
+
+ /** Gets the value of the 'majorCode' field */
+ public Integer getMajorCode() {
+ return majorCode;
+ }
+
+ /** Sets the value of the 'majorCode' field */
+ public Builder setMajorCode(int value) {
+ validate(fields()[1], value);
+ this.majorCode = value;
+ fieldSetFlags()[1] = true;
+ return this;
+ }
+
+ /** Checks whether the 'majorCode' field has been set */
+ public boolean hasMajorCode() {
+ return fieldSetFlags()[1];
+ }
+
+ /** Clears the value of the 'majorCode' field */
+ public Builder clearMajorCode() {
+ fieldSetFlags()[1] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'minorCode' field */
+ public Integer getMinorCode() {
+ return minorCode;
+ }
+
+ /** Sets the value of the 'minorCode' field */
+ public Builder setMinorCode(int value) {
+ validate(fields()[2], value);
+ this.minorCode = value;
+ fieldSetFlags()[2] = true;
+ return this;
+ }
+
+ /** Checks whether the 'minorCode' field has been set */
+ public boolean hasMinorCode() {
+ return fieldSetFlags()[2];
+ }
+
+ /** Clears the value of the 'minorCode' field */
+ public Builder clearMinorCode() {
+ fieldSetFlags()[2] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'args' field */
+ public java.util.List<CharSequence> getArgs() {
+ return args;
+ }
+
+ /** Sets the value of the 'args' field */
+ public Builder setArgs(java.util.List<CharSequence> value) {
+ validate(fields()[3], value);
+ this.args = value;
+ fieldSetFlags()[3] = true;
+ return this;
+ }
+
+ /** Checks whether the 'args' field has been set */
+ public boolean hasArgs() {
+ return fieldSetFlags()[3];
+ }
+
+ /** Clears the value of the 'args' field */
+ public Builder clearArgs() {
+ args = null;
+ fieldSetFlags()[3] = false;
+ return this;
+ }
+
+ @Override
+ public ParseStatus build() {
+ try {
+ ParseStatus record = new ParseStatus();
+ record.__g__dirty = fieldSetFlags()[0] ? this.__g__dirty : (java.nio.ByteBuffer) java.nio.ByteBuffer.wrap(new byte[1]);
+ record.majorCode = fieldSetFlags()[1] ? this.majorCode : (Integer) defaultValue(fields()[1]);
+ record.minorCode = fieldSetFlags()[2] ? this.minorCode : (Integer) defaultValue(fields()[2]);
+ record.args = fieldSetFlags()[3] ? this.args : (java.util.List<CharSequence>) new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)defaultValue(fields()[3]));
+ return record;
+ } catch (Exception e) {
+ throw new org.apache.avro.AvroRuntimeException(e);
+ }
+ }
}
- public void setMinorCode(int value) {
- put(1, value);
+
+ public Tombstone getTombstone(){
+ return TOMBSTONE;
}
- @SuppressWarnings("unchecked")
- public GenericArray<Utf8> getArgs() {
- return (GenericArray<Utf8>) get(2);
+
+ public ParseStatus newInstance(){
+ return newBuilder().build();
}
- public void addToArgs(Utf8 element) {
- getStateManager().setDirty(this, 2);
- args.add(element);
+
+ private static final Tombstone TOMBSTONE = new Tombstone();
+
+ public static final class Tombstone extends ParseStatus implements org.apache.gora.persistency.Tombstone {
+
+ private Tombstone() { }
+
+ /**
+ * Gets the value of the 'majorCode' field.
+ */
+ public Integer getMajorCode() {
+ throw new UnsupportedOperationException("Get is not supported on tombstones");
+ }
+
+ /**
+ * Sets the value of the 'majorCode' field.
+ * @param value the value to set.
+ */
+ public void setMajorCode(Integer value) {
+ throw new UnsupportedOperationException("Set is not supported on tombstones");
+ }
+
+ /**
+ * Checks the dirty status of the 'majorCode' field. A field is dirty if it represents a change that has not yet been written to the database.
+ * @param value the value to set.
+ */
+ public boolean isMajorCodeDirty(Integer value) {
+ throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ }
+
+ /**
+ * Gets the value of the 'minorCode' field.
+ */
+ public Integer getMinorCode() {
+ throw new UnsupportedOperationException("Get is not supported on tombstones");
+ }
+
+ /**
+ * Sets the value of the 'minorCode' field.
+ * @param value the value to set.
+ */
+ public void setMinorCode(Integer value) {
+ throw new UnsupportedOperationException("Set is not supported on tombstones");
+ }
+
+ /**
+ * Checks the dirty status of the 'minorCode' field. A field is dirty if it represents a change that has not yet been written to the database.
+ * @param value the value to set.
+ */
+ public boolean isMinorCodeDirty(Integer value) {
+ throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ }
+
+ /**
+ * Gets the value of the 'args' field.
+ */
+ public java.util.List<CharSequence> getArgs() {
+ throw new UnsupportedOperationException("Get is not supported on tombstones");
+ }
+
+ /**
+ * Sets the value of the 'args' field.
+ * @param value the value to set.
+ */
+ public void setArgs(java.util.List<CharSequence> value) {
+ throw new UnsupportedOperationException("Set is not supported on tombstones");
+ }
+
+ /**
+ * Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database.
+ * @param value the value to set.
+ */
+ public boolean isArgsDirty(java.util.List<CharSequence> value) {
+ throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ }
+
+
}
-}
+
+}
\ No newline at end of file
Modified: nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/storage/ProtocolStatus.java Thu May 15 08:10:07 2014
@@ -1,122 +1,428 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+/**
+ * Autogenerated by Avro
*
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-
-package org.apache.nutch.storage;
-
-import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.avro.Protocol;
-import org.apache.avro.Schema;
-import org.apache.avro.AvroRuntimeException;
-import org.apache.avro.Protocol;
-import org.apache.avro.util.Utf8;
-import org.apache.avro.ipc.AvroRemoteException;
-import org.apache.avro.generic.GenericArray;
-import org.apache.avro.specific.FixedSize;
-import org.apache.avro.specific.SpecificExceptionBase;
-import org.apache.avro.specific.SpecificRecordBase;
-import org.apache.avro.specific.SpecificRecord;
-import org.apache.avro.specific.SpecificFixed;
-import org.apache.gora.persistency.StateManager;
-import org.apache.gora.persistency.impl.PersistentBase;
-import org.apache.gora.persistency.impl.StateManagerImpl;
-import org.apache.gora.persistency.StatefulHashMap;
-import org.apache.gora.persistency.ListGenericArray;
+ * DO NOT EDIT DIRECTLY
+ */
+package org.apache.nutch.storage;
+
import org.apache.nutch.protocol.ProtocolStatusUtils;
@SuppressWarnings("all")
-public class ProtocolStatus extends PersistentBase {
- public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"code\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"lastModified\",\"type\":\"long\"}]}");
+public class ProtocolStatus extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent {
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"code\",\"type\":\"int\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]},{\"name\":\"lastModified\",\"type\":\"long\",\"default\":0}]}");
+
+ /** Enum containing all data bean's fields. */
public static enum Field {
- CODE(0,"code"),
- ARGS(1,"args"),
- LAST_MODIFIED(2,"lastModified"),
+ __G__DIRTY(0, "__g__dirty"),
+ CODE(1, "code"),
+ ARGS(2, "args"),
+ LAST_MODIFIED(3, "lastModified"),
;
+ /**
+ * Field's index.
+ */
private int index;
+
+ /**
+ * Field's name.
+ */
private String name;
+
+ /**
+ * Field's constructor
+ * @param index field's index.
+ * @param name field's name.
+ */
Field(int index, String name) {this.index=index;this.name=name;}
+
+ /**
+ * Gets field's index.
+ * @return int field's index.
+ */
public int getIndex() {return index;}
+
+ /**
+ * Gets field's name.
+ * @return String field's name.
+ */
public String getName() {return name;}
+
+ /**
+ * Gets field's attributes to string.
+ * @return String field's attributes to string.
+ */
public String toString() {return name;}
};
- public static final String[] _ALL_FIELDS = {"code","args","lastModified",};
- static {
- PersistentBase.registerFields(ProtocolStatus.class, _ALL_FIELDS);
- }
+
+ public static final String[] _ALL_FIELDS = {
+ "__g__dirty",
+ "code",
+ "args",
+ "lastModified",
+ };
+
+ /** Bytes used to represent weather or not a field is dirty. */
+ private java.nio.ByteBuffer __g__dirty = java.nio.ByteBuffer.wrap(new byte[1]);
private int code;
- private GenericArray<Utf8> args;
+ private java.util.List<CharSequence> args;
private long lastModified;
- public ProtocolStatus() {
- this(new StateManagerImpl());
- }
- public ProtocolStatus(StateManager stateManager) {
- super(stateManager);
- args = new ListGenericArray<Utf8>(getSchema().getField("args").schema());
- }
- public ProtocolStatus newInstance(StateManager stateManager) {
- return new ProtocolStatus(stateManager);
- }
- public Schema getSchema() { return _SCHEMA; }
- public Object get(int _field) {
- switch (_field) {
- case 0: return code;
- case 1: return args;
- case 2: return lastModified;
- default: throw new AvroRuntimeException("Bad index");
+ public org.apache.avro.Schema getSchema() { return SCHEMA$; }
+ // Used by DatumWriter. Applications should not call.
+ public Object get(int field$) {
+ switch (field$) {
+ case 0: return __g__dirty;
+ case 1: return code;
+ case 2: return args;
+ case 3: return lastModified;
+ default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
+
+ // Used by DatumReader. Applications should not call.
@SuppressWarnings(value="unchecked")
- public void put(int _field, Object _value) {
- if(isFieldEqual(_field, _value)) return;
- getStateManager().setDirty(this, _field);
- switch (_field) {
- case 0:code = (Integer)_value; break;
- case 1:args = (GenericArray<Utf8>)_value; break;
- case 2:lastModified = (Long)_value; break;
- default: throw new AvroRuntimeException("Bad index");
- }
- }
- public int getCode() {
- return (Integer) get(0);
- }
- public void setCode(int value) {
- put(0, value);
- }
- @SuppressWarnings("unchecked")
- public GenericArray<Utf8> getArgs() {
- return (GenericArray<Utf8>) get(1);
- }
- public void addToArgs(Utf8 element) {
- getStateManager().setDirty(this, 1);
- args.add(element);
+ public void put(int field$, Object value) {
+ switch (field$) {
+ case 0: __g__dirty = (java.nio.ByteBuffer)(value); break;
+ case 1: code = (Integer)(value); break;
+ case 2: args = (java.util.List<CharSequence>)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)value)); break;
+ case 3: lastModified = (Long)(value); break;
+ default: throw new org.apache.avro.AvroRuntimeException("Bad index");
+ }
}
- public long getLastModified() {
- return (Long) get(2);
+
+ /**
+ * Gets the value of the 'code' field.
+ */
+ public Integer getCode() {
+ return code;
}
- public void setLastModified(long value) {
- put(2, value);
+
+ /**
+ * Sets the value of the 'code' field.
+ * @param value the value to set.
+ */
+ public void setCode(Integer value) {
+ this.code = value;
+ setDirty(1);
+ }
+
+ /**
+ * Checks the dirty status of the 'code' field. A field is dirty if it represents a change that has not yet been written to the database.
+ * @param value the value to set.
+ */
+ public boolean isCodeDirty(Integer value) {
+ return isDirty(1);
+ }
+
+ /**
+ * Gets the value of the 'args' field.
+ */
+ public java.util.List<CharSequence> getArgs() {
+ return args;
+ }
+
+ /**
+ * Sets the value of the 'args' field.
+ * @param value the value to set.
+ */
+ public void setArgs(java.util.List<CharSequence> value) {
+ this.args = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper(value);
+ setDirty(2);
+ }
+
+ /**
+ * Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database.
+ * @param value the value to set.
+ */
+ public boolean isArgsDirty(java.util.List<CharSequence> value) {
+ return isDirty(2);
+ }
+
+ /**
+ * Gets the value of the 'lastModified' field.
+ */
+ public Long getLastModified() {
+ return lastModified;
+ }
+
+ /**
+ * Sets the value of the 'lastModified' field.
+ * @param value the value to set.
+ */
+ public void setLastModified(Long value) {
+ this.lastModified = value;
+ setDirty(3);
}
/**
+ * Checks the dirty status of the 'lastModified' field. A field is dirty if it represents a change that has not yet been written to the database.
+ * @param value the value to set.
+ */
+ public boolean isLastModifiedDirty(Long value) {
+ return isDirty(3);
+ }
+
+ /**
* A convenience method which returns a successful {@link ProtocolStatus}.
+ *
* @return the {@link ProtocolStatus} value for 200 (success).
*/
public boolean isSuccess() {
- return code == ProtocolStatusUtils.SUCCESS;
+ return code == ProtocolStatusUtils.SUCCESS;
+ }
+
+ /** Creates a new ProtocolStatus RecordBuilder */
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ /** Creates a new ProtocolStatus RecordBuilder by copying an existing Builder */
+ public static Builder newBuilder(Builder other) {
+ return new Builder(other);
+ }
+
+ /** Creates a new ProtocolStatus RecordBuilder by copying an existing ProtocolStatus instance */
+ public static Builder newBuilder(ProtocolStatus other) {
+ return new Builder(other);
+ }
+
+ private static java.nio.ByteBuffer deepCopyToWriteOnlyBuffer(
+ java.nio.ByteBuffer input) {
+ java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity());
+ int position = input.position();
+ input.reset();
+ int mark = input.position();
+ int limit = input.limit();
+ input.rewind();
+ input.limit(input.capacity());
+ copy.put(input);
+ input.rewind();
+ copy.rewind();
+ input.position(mark);
+ input.mark();
+ copy.position(mark);
+ copy.mark();
+ input.position(position);
+ copy.position(position);
+ input.limit(limit);
+ copy.limit(limit);
+ return copy.asReadOnlyBuffer();
}
-}
+
+ /**
+ * RecordBuilder for ProtocolStatus instances.
+ */
+ public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<ProtocolStatus>
+ implements org.apache.avro.data.RecordBuilder<ProtocolStatus> {
+
+ private java.nio.ByteBuffer __g__dirty;
+ private int code;
+ private java.util.List<CharSequence> args;
+ private long lastModified;
+
+ /** Creates a new Builder */
+ private Builder() {
+ super(ProtocolStatus.SCHEMA$);
+ }
+
+ /** Creates a Builder by copying an existing Builder */
+ private Builder(Builder other) {
+ super(other);
+ }
+
+ /** Creates a Builder by copying an existing ProtocolStatus instance */
+ private Builder(ProtocolStatus other) {
+ super(ProtocolStatus.SCHEMA$);
+ if (isValidValue(fields()[0], other.__g__dirty)) {
+ this.__g__dirty = (java.nio.ByteBuffer) data().deepCopy(fields()[0].schema(), other.__g__dirty);
+ fieldSetFlags()[0] = true;
+ }
+ if (isValidValue(fields()[1], other.code)) {
+ this.code = (Integer) data().deepCopy(fields()[1].schema(), other.code);
+ fieldSetFlags()[1] = true;
+ }
+ if (isValidValue(fields()[2], other.args)) {
+ this.args = (java.util.List<CharSequence>) data().deepCopy(fields()[2].schema(), other.args);
+ fieldSetFlags()[2] = true;
+ }
+ if (isValidValue(fields()[3], other.lastModified)) {
+ this.lastModified = (Long) data().deepCopy(fields()[3].schema(), other.lastModified);
+ fieldSetFlags()[3] = true;
+ }
+ }
+
+ /** Gets the value of the 'code' field */
+ public Integer getCode() {
+ return code;
+ }
+
+ /** Sets the value of the 'code' field */
+ public Builder setCode(int value) {
+ validate(fields()[1], value);
+ this.code = value;
+ fieldSetFlags()[1] = true;
+ return this;
+ }
+
+ /** Checks whether the 'code' field has been set */
+ public boolean hasCode() {
+ return fieldSetFlags()[1];
+ }
+
+ /** Clears the value of the 'code' field */
+ public Builder clearCode() {
+ fieldSetFlags()[1] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'args' field */
+ public java.util.List<CharSequence> getArgs() {
+ return args;
+ }
+
+ /** Sets the value of the 'args' field */
+ public Builder setArgs(java.util.List<CharSequence> value) {
+ validate(fields()[2], value);
+ this.args = value;
+ fieldSetFlags()[2] = true;
+ return this;
+ }
+
+ /** Checks whether the 'args' field has been set */
+ public boolean hasArgs() {
+ return fieldSetFlags()[2];
+ }
+
+ /** Clears the value of the 'args' field */
+ public Builder clearArgs() {
+ args = null;
+ fieldSetFlags()[2] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'lastModified' field */
+ public Long getLastModified() {
+ return lastModified;
+ }
+
+ /** Sets the value of the 'lastModified' field */
+ public Builder setLastModified(long value) {
+ validate(fields()[3], value);
+ this.lastModified = value;
+ fieldSetFlags()[3] = true;
+ return this;
+ }
+
+ /** Checks whether the 'lastModified' field has been set */
+ public boolean hasLastModified() {
+ return fieldSetFlags()[3];
+ }
+
+ /** Clears the value of the 'lastModified' field */
+ public Builder clearLastModified() {
+ fieldSetFlags()[3] = false;
+ return this;
+ }
+
+ @Override
+ public ProtocolStatus build() {
+ try {
+ ProtocolStatus record = new ProtocolStatus();
+ record.__g__dirty = fieldSetFlags()[0] ? this.__g__dirty : (java.nio.ByteBuffer) java.nio.ByteBuffer.wrap(new byte[1]);
+ record.code = fieldSetFlags()[1] ? this.code : (Integer) defaultValue(fields()[1]);
+ record.args = fieldSetFlags()[2] ? this.args : (java.util.List<CharSequence>) new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)defaultValue(fields()[2]));
+ record.lastModified = fieldSetFlags()[3] ? this.lastModified : (Long) defaultValue(fields()[3]);
+ return record;
+ } catch (Exception e) {
+ throw new org.apache.avro.AvroRuntimeException(e);
+ }
+ }
+ }
+
+ public Tombstone getTombstone(){
+ return TOMBSTONE;
+ }
+
+ public ProtocolStatus newInstance(){
+ return newBuilder().build();
+ }
+
+ private static final Tombstone TOMBSTONE = new Tombstone();
+
+ public static final class Tombstone extends ProtocolStatus implements org.apache.gora.persistency.Tombstone {
+
+ private Tombstone() { }
+
+ /**
+ * Gets the value of the 'code' field.
+ */
+ public Integer getCode() {
+ throw new UnsupportedOperationException("Get is not supported on tombstones");
+ }
+
+ /**
+ * Sets the value of the 'code' field.
+ * @param value the value to set.
+ */
+ public void setCode(Integer value) {
+ throw new UnsupportedOperationException("Set is not supported on tombstones");
+ }
+
+ /**
+ * Checks the dirty status of the 'code' field. A field is dirty if it represents a change that has not yet been written to the database.
+ * @param value the value to set.
+ */
+ public boolean isCodeDirty(Integer value) {
+ throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ }
+
+ /**
+ * Gets the value of the 'args' field.
+ */
+ public java.util.List<CharSequence> getArgs() {
+ throw new UnsupportedOperationException("Get is not supported on tombstones");
+ }
+
+ /**
+ * Sets the value of the 'args' field.
+ * @param value the value to set.
+ */
+ public void setArgs(java.util.List<CharSequence> value) {
+ throw new UnsupportedOperationException("Set is not supported on tombstones");
+ }
+
+ /**
+ * Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database.
+ * @param value the value to set.
+ */
+ public boolean isArgsDirty(java.util.List<CharSequence> value) {
+ throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ }
+
+ /**
+ * Gets the value of the 'lastModified' field.
+ */
+ public Long getLastModified() {
+ throw new UnsupportedOperationException("Get is not supported on tombstones");
+ }
+
+ /**
+ * Sets the value of the 'lastModified' field.
+ * @param value the value to set.
+ */
+ public void setLastModified(Long value) {
+ throw new UnsupportedOperationException("Set is not supported on tombstones");
+ }
+
+ /**
+ * Checks the dirty status of the 'lastModified' field. A field is dirty if it represents a change that has not yet been written to the database.
+ * @param value the value to set.
+ */
+ public boolean isLastModifiedDirty(Long value) {
+ throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ }
+
+
+ }
+
+}
\ No newline at end of file
Modified: nutch/branches/2.x/src/java/org/apache/nutch/storage/StorageUtils.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/storage/StorageUtils.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/storage/StorageUtils.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/storage/StorageUtils.java Thu May 15 08:10:07 2014
@@ -16,10 +16,6 @@
******************************************************************************/
package org.apache.nutch.storage;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Iterator;
-
import org.apache.gora.mapreduce.GoraMapper;
import org.apache.gora.mapreduce.GoraOutputFormat;
import org.apache.gora.mapreduce.GoraReducer;
@@ -33,6 +29,10 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.nutch.metadata.Nutch;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+
/**
* Entry point to Gora store/mapreduce functionality.
* Translates the concept of "crawlid" to the corresponding Gora support.
@@ -52,23 +52,23 @@ public class StorageUtils {
@SuppressWarnings("unchecked")
public static <K, V extends Persistent> DataStore<K, V> createWebStore(Configuration conf,
Class<K> keyClass, Class<V> persistentClass) throws ClassNotFoundException, GoraException {
-
- String schema = null;
+
+ String crawlId = conf.get(Nutch.CRAWL_ID_KEY, "");
+ String schemaPrefix = "";
+ if (!crawlId.isEmpty()) {
+ schemaPrefix = crawlId + "_";
+ }
+
+ String schema;
if (WebPage.class.equals(persistentClass)) {
schema = conf.get("storage.schema.webpage", "webpage");
+ conf.set("preferred.schema.name", schemaPrefix + "webpage");
} else if (Host.class.equals(persistentClass)) {
schema = conf.get("storage.schema.host", "host");
+ conf.set("preferred.schema.name", schemaPrefix + "host");
} else {
throw new UnsupportedOperationException("Unable to create store for class " + persistentClass);
}
-
- String crawlId = conf.get(Nutch.CRAWL_ID_KEY, "");
-
- if (!crawlId.isEmpty()) {
- conf.set("schema.prefix", crawlId + "_");
- } else {
- conf.set("schema.prefix", "");
- }
Class<? extends DataStore<K, V>> dataStoreClass =
(Class<? extends DataStore<K, V>>) getDataStoreClass(conf);