You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ma...@apache.org on 2015/05/02 08:20:43 UTC

svn commit: r1677277 - in /tika/trunk: ./ tika-core/src/main/java/org/apache/tika/detect/ tika-core/src/main/resources/org/apache/tika/detect/ tika-core/src/test/java/org/apache/tika/detect/ tika-core/src/test/resources/org/apache/tika/detect/ tika-cor...

Author: mattmann
Date: Sat May  2 06:20:42 2015
New Revision: 1677277

URL: http://svn.apache.org/r1677277
Log:
Fix for TIKA-1582 Mime Detection based on neural networks with Byte-frequency-histogram contributed by  LukeLiush <ha...@gmail.com>. This closes #36

Added:
    tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNExampleModelDetector.java
    tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModel.java
    tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModelBuilder.java
    tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModel.java
    tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java
    tika/trunk/tika-core/src/main/resources/org/apache/tika/detect/
    tika/trunk/tika-core/src/main/resources/org/apache/tika/detect/tika-example.nnmodel
    tika/trunk/tika-core/src/test/java/org/apache/tika/detect/MimeDetectionWithNNTest.java
    tika/trunk/tika-core/src/test/resources/org/apache/tika/detect/
    tika/trunk/tika-core/src/test/resources/org/apache/tika/detect/tika-example.nnmodel
    tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/GLDAS_CLM10SUBP_3H.A19790202.0000.001.grb   (with props)
    tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/gdas1.forecmwf.2014062612.grib2   (with props)
Modified:
    tika/trunk/CHANGES.txt

Modified: tika/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/tika/trunk/CHANGES.txt?rev=1677277&r1=1677276&r2=1677277&view=diff
==============================================================================
--- tika/trunk/CHANGES.txt (original)
+++ tika/trunk/CHANGES.txt Sat May  2 06:20:42 2015
@@ -1,5 +1,9 @@
 Release 1.9 - Current Development
 
+  * An algorithm for using Byte Histogram frequencies to construct
+    a Neural Network and to perform MIME detection was added
+    (TIKA-1582).
+
   * A Bayesian algorithm for MIME detection by probabilistic
     means was added (TIKA-1517).
 

Added: tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNExampleModelDetector.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNExampleModelDetector.java?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNExampleModelDetector.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNExampleModelDetector.java Sat May  2 06:20:42 2015
@@ -0,0 +1,174 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.detect;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+
+import org.apache.tika.mime.MediaType;
+
+public class NNExampleModelDetector extends TrainedModelDetector {
+	private static final String EXAMPLE_NNMODEL_FILE = "tika-example.nnmodel";
+
+	private static final long serialVersionUID = 1L;
+
+	public NNExampleModelDetector() {
+		super();
+	}
+
+	public NNExampleModelDetector(final File modelFile) {
+		loadDefaultModels(modelFile);
+	}
+
+	@Override
+	public void loadDefaultModels(InputStream modelStream) {
+	    BufferedReader bReader = null;
+	    try{
+                bReader = new BufferedReader(
+			         new InputStreamReader(modelStream, "UTF-8"));
+	    }
+	    catch(UnsupportedEncodingException e){
+                e.printStackTrace();
+	    }
+
+		NNTrainedModelBuilder nnBuilder = new NNTrainedModelBuilder();
+		String line;
+		try {
+			while ((line = bReader.readLine()) != null) {
+				line = line.trim();
+				if (line.startsWith("#")) {
+					readDescription(nnBuilder, line);
+				} else {
+					readNNParams(nnBuilder, line);
+					// add this model into map of trained models.
+					super.registerModels(nnBuilder.getType(), nnBuilder.build());
+				}
+
+			}
+		} catch (IOException e) {
+			throw new RuntimeException(
+					"Unable to read the default media type registry", e);
+		}
+
+	}
+
+	/**
+	 * this method gets overwritten to register load neural network models
+	 */
+	@Override
+	public void loadDefaultModels(ClassLoader classLoader) {
+		if (classLoader == null) {
+			classLoader = TrainedModelDetector.class.getClassLoader();
+		}
+
+		// This allows us to replicate class.getResource() when using
+		// the classloader directly
+		String classPrefix = TrainedModelDetector.class.getPackage().getName()
+				.replace('.', '/')
+				+ "/";
+
+		// Get the core URL, and all the extensions URLs
+		URL modelURL = classLoader.getResource(classPrefix
+				+ EXAMPLE_NNMODEL_FILE);
+		InputStream stream = null;
+		try {
+			stream = modelURL.openStream();
+			loadDefaultModels(stream);
+
+		} catch (IOException e) {
+			throw new RuntimeException(
+					"Unable to read the default media type registry", e);
+		} finally {
+			try {
+				stream.close();
+			} catch (IOException e) {
+				throw new RuntimeException(
+						"Unable to read the default media type registry", e);
+			}
+		}
+
+	}
+
+	/**
+	 * read the comments where the model configuration is written, e.g the
+	 * number of inputs, hiddens and output please ensure the first char in the
+	 * given string is # In this example grb model file, there are 4 elements 1)
+	 * type 2) number of input units 3) number of hidden units. 4) number of
+	 * output units.
+	 * 
+	 */
+	private void readDescription(final NNTrainedModelBuilder builder,
+			final String line) {
+		int numInputs = 0;
+		int numHidden = 0;
+		int numOutputs = 0;
+		String tline = line;
+		String[] sarr = tline.split("\t");
+
+		try {
+			MediaType type = MediaType.parse(sarr[1]);
+			numInputs = Integer.parseInt(sarr[2]);
+			numHidden = Integer.parseInt(sarr[3]);
+			numOutputs = Integer.parseInt(sarr[4]);
+			builder.setNumOfInputs(numInputs);
+			builder.setNumOfHidden(numHidden);
+			builder.setNumOfOutputs(numOutputs);
+			builder.setType(type);
+		} catch (Exception e) {
+			e.printStackTrace();
+			throw new RuntimeException(
+					"Unable to parse the model configuration", e);
+		}
+
+	}
+
+	/**
+	 * Read the next line for the model parameters and populate the build which
+	 * later will be used to instantiate the instance of TrainedModel
+	 * 
+	 * @param builder
+	 * @param line
+	 */
+	private void readNNParams(final NNTrainedModelBuilder builder,
+			final String line) {
+		String tline = line;
+		String[] sarr = tline.split("\t");
+		int n = sarr.length;
+		float[] params = new float[n];
+		try {
+			int i = 0;
+			for (String fstr : sarr) {
+				params[i] = Float.parseFloat(fstr);
+				i++;
+			}
+			builder.setParams(params);
+		} catch (Exception e) {
+			e.printStackTrace();
+			throw new RuntimeException(
+					"Unable to parse the model configuration", e);
+		}
+
+	}
+
+}

Added: tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModel.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModel.java?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModel.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModel.java Sat May  2 06:20:42 2015
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.detect;
+
+public class NNTrainedModel extends TrainedModel {
+
+	private int numOfInputs;
+	private int numOfHidden;
+	private int numOfOutputs;
+	private float[][] Theta1, Theta2;
+
+	public NNTrainedModel(final int nInput, final int nHidden,
+			final int nOutput, final float[] nn_params) {
+		this.numOfInputs = nInput;
+		this.numOfHidden = nHidden;
+		this.numOfOutputs = nOutput;
+		this.Theta1 = new float[numOfHidden][numOfInputs + 1];
+		this.Theta2 = new float[numOfOutputs][numOfHidden + 1];
+		populateThetas(nn_params);
+	}
+
+	// convert the vector params to the 2 thetas.
+	private void populateThetas(final float[] nn_params) {
+		int m = this.Theta1.length;
+		int n = this.Theta1[0].length;
+		int i, j, k = 0;
+		for (i = 0; i < n; i++) {
+			for (j = 0; j < m; j++) {
+				this.Theta1[j][i] = nn_params[k];
+				k++;
+			}
+		}
+
+		m = this.Theta2.length;
+		n = this.Theta2[0].length;
+		for (i = 0; i < n; i++) {
+			for (j = 0; j < m; j++) {
+				this.Theta2[j][i] = nn_params[k];
+				k++;
+			}
+		}
+	}
+
+	@Override
+	public double predict(double[] unseen) {
+		// TODO Auto-generated method stub
+		return 0;
+	}
+
+	/**
+	 * The given input vector of unseen is m=(256 + 1) * n= 1 this returns a
+	 * prediction probability
+	 */
+	@Override
+	public float predict(float[] unseen) {
+		// please ensure the unseen in size consistent with theta1
+
+		int i, j;
+		int m = this.Theta1.length;
+		int n = this.Theta1[0].length;
+		float[] hh = new float[m + 1];// hidden unit summation
+		hh[0] = 1;
+		for (i = 0; i < m; i++) {
+			double h = 0;
+			for (j = 0; j < n; j++) {
+				h += this.Theta1[i][j] * unseen[j];
+			}
+			// apply sigmoid
+			h = 1.0 / (1.0 + Math.exp(-h));
+			hh[i+1] = (float)h;
+		}
+
+		m = this.Theta2.length;
+		n = this.Theta2[0].length;
+		float[] oo = new float[m];
+		for (i = 0; i < m; i++) {
+			double o = 0;
+			for (j = 0; j < n; j++) {
+				o += this.Theta2[i][j] * hh[j];
+			}
+			// apply sigmoid
+			o = 1.0 / (1.0 + Math.exp(-o));
+			oo[i] = (float)o;
+		}
+
+		return oo[0];
+	}
+}

Added: tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModelBuilder.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModelBuilder.java?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModelBuilder.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModelBuilder.java Sat May  2 06:20:42 2015
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.tika.detect;
+
+import org.apache.tika.mime.MediaType;
+
+public class NNTrainedModelBuilder {
+	private MediaType type;
+
+	private int numOfInputs;
+	private int numOfHidden;
+	private int numOfOutputs;
+
+	private float[] params;
+
+	public MediaType getType() {
+		return this.type;
+	}
+
+	public int getNumOfInputs() {
+		return numOfInputs;
+	}
+
+	public int getNumOfHidden() {
+		return numOfHidden;
+	}
+
+	public int getNumOfOutputs() {
+		return numOfOutputs;
+	}
+
+	public float[] getParams() {
+		return params;
+	}
+
+	public void setType(final MediaType type) {
+		this.type = type;
+	}
+
+	public void setNumOfInputs(final int numOfInputs) {
+		this.numOfInputs = numOfInputs;
+	}
+
+	public void setNumOfHidden(final int numOfHidden) {
+		this.numOfHidden = numOfHidden;
+	}
+
+	public void setNumOfOutputs(final int numOfOutputs) {
+		this.numOfOutputs = numOfOutputs;
+	}
+
+	public void setParams(float[] params) {
+		this.params = params;
+	}
+
+	public NNTrainedModel build() {
+		return new NNTrainedModel(numOfInputs, numOfHidden, numOfOutputs,
+				params);
+	}
+}

Added: tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModel.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModel.java?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModel.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModel.java Sat May  2 06:20:42 2015
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+
+public abstract class TrainedModel {
+	
+	public abstract double predict(double[] input);
+	public abstract float predict(float[] input);
+}

Added: tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java Sat May  2 06:20:42 2015
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.OutputStreamWriter;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Writer;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.io.TemporaryResources;
+
+public abstract class TrainedModelDetector implements Detector {
+	private final Map<MediaType, TrainedModel> MODEL_MAP = new HashMap<MediaType, TrainedModel>();
+
+	private static final long serialVersionUID = 1L;
+
+	public TrainedModelDetector() {
+		loadDefaultModels(getClass().getClassLoader());
+	}
+
+	public int getMinLength() {
+		return Integer.MAX_VALUE;
+	}
+
+	public MediaType detect(InputStream input, Metadata metadata)
+			throws IOException {
+		// convert to byte-histogram
+		if (input != null) {
+			input.mark(getMinLength());
+			float[] histogram = readByteFrequencies(input);
+			// writeHisto(histogram); //on testing purpose
+			/*
+			 * iterate the map to find out the one that gives the higher
+			 * prediction value.
+			 */
+			Iterator<MediaType> iter = MODEL_MAP.keySet().iterator();
+			float threshold = 0.5f;// probability threshold, any value below the
+									// threshold will be considered as
+									// MediaType.OCTET_STREAM
+			float maxprob = threshold;
+			MediaType maxType = MediaType.OCTET_STREAM;
+			while (iter.hasNext()) {
+				MediaType key = iter.next();
+				TrainedModel model = MODEL_MAP.get(key);
+				float prob = model.predict(histogram);
+				if (maxprob < prob) {
+					maxprob = prob;
+					maxType = key;
+				}
+			}
+			input.reset();
+			return maxType;
+		}
+		return null;
+	}
+
+	/**
+	 * read the inputstream and build a byte frequence histogram
+	 * 
+	 * @param input
+	 * @return
+	 * @throws IOException
+	 */
+	protected float[] readByteFrequencies(final InputStream input)
+			throws IOException {
+
+		ReadableByteChannel inputChannel = null;
+		try {
+			inputChannel = Channels.newChannel(input);
+			// long inSize = inputChannel.size();
+			float histogram[] = new float[257];
+			histogram[0] = 1;
+
+			// create buffer with capacity of maxBufSize bytes
+			ByteBuffer buf = ByteBuffer.allocate(1024 * 5);
+			int bytesRead = inputChannel.read(buf); // read into buffer.
+
+			float max = -1;
+			while (bytesRead != -1) {
+
+				buf.flip(); // make buffer ready for read
+
+				while (buf.hasRemaining()) {
+					byte byt = buf.get();
+					int idx = byt;
+					idx++;
+					if (byt < 0) {
+						idx = 256 + idx;
+						histogram[idx]++;
+					} else {
+						histogram[idx]++;
+					}
+					max = max < histogram[idx] ? histogram[idx] : max;
+				}
+
+				buf.clear(); // make buffer ready for writing
+				bytesRead = inputChannel.read(buf);
+			}
+
+			int i;
+			for (i = 1; i < histogram.length; i++) {
+				histogram[i] /= max;
+				histogram[i] = (float) Math.sqrt(histogram[i]);
+			}
+
+			return histogram;
+		} finally {
+			// inputChannel.close();
+		}
+
+	}
+
+	/**
+	 * on testing purpose; this method write the histogram vector to a file.
+	 * 
+	 * @param histogram
+	 * @throws IOException
+	 */
+	private synchronized void writeHisto(final float[] histogram)
+			throws IOException {
+	        String histPath = new TemporaryResources().createTemporaryFile().getAbsolutePath();
+	        Writer writer = new OutputStreamWriter(new FileOutputStream(histPath),"UTF-8");
+		int n = histogram.length;// excluding the last one for storing the
+									// max value
+		for (int i = 0; i < n; i++) {
+			writer.write(new StringBuffer().append(histogram[i]).append("\t")
+					.toString());
+			// writer.write(i + "\t");
+		}
+
+		writer.write("\r\n");
+		writer.flush();
+	}
+
+	public void loadDefaultModels(final File modelFile) {
+		FileInputStream in = null;
+		try {
+			in = new FileInputStream(modelFile);
+			loadDefaultModels(in);
+		} catch (FileNotFoundException e) {
+			throw new RuntimeException(
+					"Unable to read the default media type registry", e);
+		} finally {
+			try {
+				in.close();
+			} catch (IOException e) {
+				throw new RuntimeException(
+						"Unable to read the default media type registry", e);
+			}
+		}
+	}
+
+	public abstract void loadDefaultModels(final InputStream modelStream);
+
+	public abstract void loadDefaultModels(final ClassLoader classLoader);
+
+	protected void registerModels(MediaType type, TrainedModel model) {
+		MODEL_MAP.put(type, model);
+	}
+
+}

Added: tika/trunk/tika-core/src/main/resources/org/apache/tika/detect/tika-example.nnmodel
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/resources/org/apache/tika/detect/tika-example.nnmodel?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/resources/org/apache/tika/detect/tika-example.nnmodel (added)
+++ tika/trunk/tika-core/src/main/resources/org/apache/tika/detect/tika-example.nnmodel Sat May  2 06:20:42 2015
@@ -0,0 +1,2 @@
+#nn	application/x-grib	256	2	1	0.0208829625450076
+-0.05145498	5.127922	-3.078527	-3.988763	-0.2107816	0.8629464	-0.0002683103	3.268235	-0.1322597	2.256234	0.1585416	2.448373	-0.181859	-2.230397	-0.1796804	-3.778733	0.04216435	-5.247452	-0.07137667	4.015641	0.3369519	-2.232862	0.9872297	3.434345	0.01537457	-2.99547	0.05939847	-0.4262142	0.6431364	4.015367	-0.1052946	-0.5597504	-0.02748792	1.406441	-0.00300846	1.756262	0.09294704	-1.096173	-0.006158207	-1.221678	-0.02583992	-3.162681	0.08686034	1.269717	-0.01359701	-4.19245	-0.1400172	-3.274052	-0.0260648	-2.134415	0.02505031	-0.7995273	-0.01464903	-5.054871	0.06927952	-1.541063	-0.1199119	-1.126015	0.1847894	0.3805432	0.1132536	-1.962234	0.1468857	-0.5322409	-0.1450888	-0.1371484	3.097646	7.200917	0.17761	-0.5256572	0.8017177	0.8039271	0.2925865	3.100624	0.1743965	4.369426	0.1317674	-0.833718	0.220852	-4.685311	0.220108	1.81378	0.283003	2.38208	0.3301778	-3.338501	0.2043622	2.858757	0.1801032	-2.616795	0.5628204	1.599731	0.6208351	1.161804	0.8223767	-2.614317	0.9187943	-2.446297	1.3
 50831	2.777917	0.8146963	-0.9438954	0.8172006	-0.3445128	0.6723926	0.9863181	0.5705485	2.406452	0.6185437	-1.202793	0.6450134	1.808502	0.4259292	-7.941289	0.6099652	3.878562	0.4132371	0.98963	0.3298724	-2.366126	0.3319441	-0.8838808	0.6854118	3.12115	0.525016	-1.727541	0.7185858	1.830994	0.03694244	-3.686851	-0.09525806	3.558239	0.4775859	1.931869	0.293615	-3.693917	0.4294216	-5.695489	0.3452209	0.6723521	0.4919632	-0.7649217	0.3818015	2.153055	0.3088684	-0.655546	0.2931553	-1.343124	0.4890785	-0.7017448	0.1028709	-2.578852	0.1097999	-0.3793636	0.3154411	-2.476895	0.3301697	-1.578032	0.4468783	0.9338115	0.3876319	0.1819231	0.3975481	2.377385	0.07344178	-0.5816987	0.4968894	-3.409108	0.4824005	-0.2253791	0.538568	0.3602099	0.1838869	0.9865417	0.2429371	-0.2590284	0.1997649	-2.825891	0.1690305	-2.876223	0.1508797	-2.025365	0.0745675	1.403832	0.1730641	-3.619998	0.02000108	-1.345008	0.1982651	-4.596162	0.08308789	-0.08156122	0.2391607	0.06438381	-0.01053187	-3.724878	1.342193	0.8912752
 	0.6741032	-2.038227	1.02059	1.314822	1.046652	-0.09340967	1.746544	1.748925	0.7121063	2.712003	0.7263532	-0.4646987	0.8260612	-1.657031	1.326838	2.972081	0.3091417	-0.0007440437	0.4395537	0.2231857	1.106737	-1.463802	0.912596	2.458881	1.365523	2.234517	1.385396	-0.4650663	0.7839583	2.282071	0.2386854	0.9784987	1.273242	0.6254676	1.276118	2.447344	1.575805	4.515861	0.8364591	0.6976352	0.5248962	-2.899523	0.5404893	-1.394514	0.3592697	-0.9920228	0.6932542	-0.180825	0.2744942	-0.6203262	0.2417071	-1.224851	0.1811886	-0.8900446	0.226887	-3.733405	0.1735546	-1.547938	0.112908	-8.991437	0.1115473	-4.406975	0.1178586	2.36029	0.1271653	-2.011151	0.08239845	-0.1951623	0.04392552	-3.046913	0.05452365	-1.516051	0.1397218	1.094332	0.1264289	1.933882	0.1752298	-1.989354	0.1306208	-0.9046116	0.1277338	2.303931	0.1250716	-1.277528	0.1262729	2.467187	0.1100633	-0.01313957	0.2080954	0.3508277	0.2128501	0.8266836	0.1570878	-3.358276	0.1242704	0.5336581	0.2453244	1.62488	0.1280062	-2.309668	0.1366247
 	-2.19346	0.1809374	-1.296981	0.1988072	1.969289	0.108966	1.910149	0.1144941	1.112991	0.1896516	3.369206	0.1071244	-1.499693	0.2437382	-0.4268651	0.2060418	2.251279	0.1191459	-0.03434586	0.173683	2.421464	0.1856368	0.8361177	0.1358361	-1.145463	0.1938676	-0.5617768	0.2014952	0.9808323	0.1261575	3.778169	0.2198753	-2.851743	0.2107957	1.588071	0.2140176	-1.231923	0.2426582	3.296147	0.1962984	1.245569	0.2223071	1.98822	0.1688889	0.5244448	0.1442813	0.5712083	0.1423984	1.580774	0.1360883	3.032353	0.1284363	0.2361942	0.2330592	0.7644741	0.1345135	-0.2284832	0.177801	0.9684574	0.1846398	-0.8217517	0.1690076	-0.1556177	0.2140797	0.9469408	0.1869065	1.540043	0.2084914	1.695791	0.1887449	0.8586018	0.2197045	2.042192	0.1317169	-0.8347201	0.1562218	-1.021943	0.1533783	1.968931	0.1491463	0.9810904	0.1098978	1.055438	0.2106167	1.022286	0.199176	0.9168212	0.03894011	-0.3426502	0.1671793	-3.521834	0.1534467	-0.2374157	0.2013961	1.556842	0.1250643	2.738925	0.1591011	1.886823	0.1626787	-0.2158393	0.
 2056642	3.486286	0.1987337	2.471281	0.1744332	1.152822	0.1951747	-0.989242	0.1155556	0.5867982	0.08321315	2.675081	0.1303141	-2.584626	0.1609869	-1.254465	0.122685	-2.27439	0.1415207	1.642059	0.1458468	-2.672928	0.1453881	-5.031017	0.1312302	0.2756784	0.1731334	0.8576683	0.1351911	-2.385648	0.2008109	-0.2893286	0.2062319	2.538498	0.191823	-1.739293	0.1671868	-1.622128	0.1850325	2.469388	0.2514207	2.117658	0.2225232	2.403703	0.1996269	-1.039887	0.122257	-0.6789801	0.1378511	-4.054801	0.08070226	0.2997747	0.1447362	2.234673	0.1876959	2.499773	0.1622439	0.5804301	0.1345898	0.07422866	0.09716025	-0.8710974	0.1657741	-1.370054	0.1313368	-1.671961	0.1861922	-0.9497792	0.2265888	-1.500299	0.2406933	-0.639602	0.2041045	-0.02993134	0.1440134	1.211941	0.1917053	-2.045351	0.1995012	0.4286667	0.2419866	1.319268	0.05873292	3.849816	0.1642288	3.828327	0.2147195	-0.1579602	0.2119238	-1.37588	0.142787	1.026216	0.2192122	-0.106043	0.2299229	0.5020428	0.1666804	-1.020357	0.03175325	-4.244395	0.141780
 8	1.535539	0.2701716	2.742496	0.2501491	1.629031	0.08584132	1.70437	0.1870297	6.293977	0.1240707	6.969766	-1.00584	-2.592914	8.495434	8.769945	-32.94919
\ No newline at end of file

Added: tika/trunk/tika-core/src/test/java/org/apache/tika/detect/MimeDetectionWithNNTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/test/java/org/apache/tika/detect/MimeDetectionWithNNTest.java?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/test/java/org/apache/tika/detect/MimeDetectionWithNNTest.java (added)
+++ tika/trunk/tika-core/src/test/java/org/apache/tika/detect/MimeDetectionWithNNTest.java Sat May  2 06:20:42 2015
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.mime.MimeDetectionTest;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MimeDetectionWithNNTest {
+
+	private Detector detector;
+
+	/** @inheritDoc */
+	@Before
+	public void setUp() {
+		detector = new NNExampleModelDetector();
+	}
+
+	/**
+	 * The test case only works on the detector that only has grb model as
+	 * currently the grb model is used as an example; if more models are added
+	 * in the TrainedModelDetector, the following tests will need to modified to reflect
+	 * the corresponding type instead of test-equal with the "OCTET_STREAM";
+	 * 
+	 * @throws Exception
+	 */
+	@Test
+	public void testDetection() throws Exception {
+		String octetStream_str = MediaType.OCTET_STREAM.toString();
+		String grb_str = "application/x-grib";
+
+		testFile(grb_str, "gdas1.forecmwf.2014062612.grib2");
+		testFile(grb_str, "GLDAS_CLM10SUBP_3H.A19790202.0000.001.grb");
+
+		testFile(octetStream_str, "circles.svg");
+		testFile(octetStream_str, "circles-with-prefix.svg");
+		testFile(octetStream_str, "datamatrix.png");
+		testFile(octetStream_str, "test.html");
+		testFile(octetStream_str, "test-iso-8859-1.xml");
+		testFile(octetStream_str, "test-utf8.xml");
+		testFile(octetStream_str, "test-utf8-bom.xml");
+		testFile(octetStream_str, "test-utf16le.xml");
+		testFile(octetStream_str, "test-utf16be.xml");
+		testFile(octetStream_str, "test-long-comment.xml");
+		testFile(octetStream_str, "stylesheet.xsl");
+		testUrl(octetStream_str,
+				"http://www.ai.sri.com/daml/services/owl-s/1.2/Process.owl",
+				"test-difficult-rdf1.xml");
+		testUrl(octetStream_str, "http://www.w3.org/2002/07/owl#",
+				"test-difficult-rdf2.xml");
+		// add evil test from TIKA-327
+		testFile(octetStream_str, "test-tika-327.html");
+		// add another evil html test from TIKA-357
+		testFile(octetStream_str, "testlargerbuffer.html");
+		// test fragment of HTML with <div> (TIKA-1102)
+		testFile(octetStream_str, "htmlfragment");
+		// test binary CGM detection (TIKA-1170)
+		testFile(octetStream_str, "plotutils-bin-cgm-v3.cgm");
+		// test HTML detection of malformed file, previously identified as
+		// image/cgm (TIKA-1170)
+		testFile(octetStream_str, "test-malformed-header.html.bin");
+
+		// test GCMD Directory Interchange Format (.dif) TIKA-1561
+		testFile(octetStream_str, "brwNIMS_2014.dif");
+	}
+
+	private void testUrl(String expected, String url, String file)
+			throws IOException {
+		InputStream in = MimeDetectionTest.class.getResourceAsStream(file);
+		testStream(expected, url, in);
+	}
+
+	private void testFile(String expected, String filename) throws IOException {
+
+		InputStream in = MimeDetectionTest.class.getResourceAsStream(filename);
+		testStream(expected, filename, in);
+	}
+
+	private void testStream(String expected, String urlOrFileName,
+			InputStream in) throws IOException {
+		assertNotNull("Test stream: [" + urlOrFileName + "] is null!", in);
+		if (!in.markSupported()) {
+			in = new java.io.BufferedInputStream(in);
+		}
+		try {
+			Metadata metadata = new Metadata();
+			String mime = this.detector.detect(in, metadata).toString();
+			assertEquals(
+					urlOrFileName + " is not properly detected: detected.",
+					expected, mime);
+
+			// Add resource name and test again
+			// metadata.set(Metadata.RESOURCE_NAME_KEY, urlOrFileName);
+			mime = this.detector.detect(in, metadata).toString();
+			assertEquals(urlOrFileName
+					+ " is not properly detected after adding resource name.",
+					expected, mime);
+		} finally {
+			in.close();
+		}
+	}
+
+	private void assertNotNull(String string, InputStream in) {
+		// TODO Auto-generated method stub
+
+	}
+
+	/**
+	 * Test for type detection of empty documents.
+	 */
+	@Test
+	public void testEmptyDocument() throws IOException {
+		assertEquals(MediaType.OCTET_STREAM, detector.detect(
+				new ByteArrayInputStream(new byte[0]), new Metadata()));
+
+	}
+
+}

Added: tika/trunk/tika-core/src/test/resources/org/apache/tika/detect/tika-example.nnmodel
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/test/resources/org/apache/tika/detect/tika-example.nnmodel?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/test/resources/org/apache/tika/detect/tika-example.nnmodel (added)
+++ tika/trunk/tika-core/src/test/resources/org/apache/tika/detect/tika-example.nnmodel Sat May  2 06:20:42 2015
@@ -0,0 +1,2 @@
+#nn	application/x-grib	256	2	1	0.0208829625450076
+-0.05145498	5.127922	-3.078527	-3.988763	-0.2107816	0.8629464	-0.0002683103	3.268235	-0.1322597	2.256234	0.1585416	2.448373	-0.181859	-2.230397	-0.1796804	-3.778733	0.04216435	-5.247452	-0.07137667	4.015641	0.3369519	-2.232862	0.9872297	3.434345	0.01537457	-2.99547	0.05939847	-0.4262142	0.6431364	4.015367	-0.1052946	-0.5597504	-0.02748792	1.406441	-0.00300846	1.756262	0.09294704	-1.096173	-0.006158207	-1.221678	-0.02583992	-3.162681	0.08686034	1.269717	-0.01359701	-4.19245	-0.1400172	-3.274052	-0.0260648	-2.134415	0.02505031	-0.7995273	-0.01464903	-5.054871	0.06927952	-1.541063	-0.1199119	-1.126015	0.1847894	0.3805432	0.1132536	-1.962234	0.1468857	-0.5322409	-0.1450888	-0.1371484	3.097646	7.200917	0.17761	-0.5256572	0.8017177	0.8039271	0.2925865	3.100624	0.1743965	4.369426	0.1317674	-0.833718	0.220852	-4.685311	0.220108	1.81378	0.283003	2.38208	0.3301778	-3.338501	0.2043622	2.858757	0.1801032	-2.616795	0.5628204	1.599731	0.6208351	1.161804	0.8223767	-2.614317	0.9187943	-2.446297	1.3
 50831	2.777917	0.8146963	-0.9438954	0.8172006	-0.3445128	0.6723926	0.9863181	0.5705485	2.406452	0.6185437	-1.202793	0.6450134	1.808502	0.4259292	-7.941289	0.6099652	3.878562	0.4132371	0.98963	0.3298724	-2.366126	0.3319441	-0.8838808	0.6854118	3.12115	0.525016	-1.727541	0.7185858	1.830994	0.03694244	-3.686851	-0.09525806	3.558239	0.4775859	1.931869	0.293615	-3.693917	0.4294216	-5.695489	0.3452209	0.6723521	0.4919632	-0.7649217	0.3818015	2.153055	0.3088684	-0.655546	0.2931553	-1.343124	0.4890785	-0.7017448	0.1028709	-2.578852	0.1097999	-0.3793636	0.3154411	-2.476895	0.3301697	-1.578032	0.4468783	0.9338115	0.3876319	0.1819231	0.3975481	2.377385	0.07344178	-0.5816987	0.4968894	-3.409108	0.4824005	-0.2253791	0.538568	0.3602099	0.1838869	0.9865417	0.2429371	-0.2590284	0.1997649	-2.825891	0.1690305	-2.876223	0.1508797	-2.025365	0.0745675	1.403832	0.1730641	-3.619998	0.02000108	-1.345008	0.1982651	-4.596162	0.08308789	-0.08156122	0.2391607	0.06438381	-0.01053187	-3.724878	1.342193	0.8912752
 	0.6741032	-2.038227	1.02059	1.314822	1.046652	-0.09340967	1.746544	1.748925	0.7121063	2.712003	0.7263532	-0.4646987	0.8260612	-1.657031	1.326838	2.972081	0.3091417	-0.0007440437	0.4395537	0.2231857	1.106737	-1.463802	0.912596	2.458881	1.365523	2.234517	1.385396	-0.4650663	0.7839583	2.282071	0.2386854	0.9784987	1.273242	0.6254676	1.276118	2.447344	1.575805	4.515861	0.8364591	0.6976352	0.5248962	-2.899523	0.5404893	-1.394514	0.3592697	-0.9920228	0.6932542	-0.180825	0.2744942	-0.6203262	0.2417071	-1.224851	0.1811886	-0.8900446	0.226887	-3.733405	0.1735546	-1.547938	0.112908	-8.991437	0.1115473	-4.406975	0.1178586	2.36029	0.1271653	-2.011151	0.08239845	-0.1951623	0.04392552	-3.046913	0.05452365	-1.516051	0.1397218	1.094332	0.1264289	1.933882	0.1752298	-1.989354	0.1306208	-0.9046116	0.1277338	2.303931	0.1250716	-1.277528	0.1262729	2.467187	0.1100633	-0.01313957	0.2080954	0.3508277	0.2128501	0.8266836	0.1570878	-3.358276	0.1242704	0.5336581	0.2453244	1.62488	0.1280062	-2.309668	0.1366247
 	-2.19346	0.1809374	-1.296981	0.1988072	1.969289	0.108966	1.910149	0.1144941	1.112991	0.1896516	3.369206	0.1071244	-1.499693	0.2437382	-0.4268651	0.2060418	2.251279	0.1191459	-0.03434586	0.173683	2.421464	0.1856368	0.8361177	0.1358361	-1.145463	0.1938676	-0.5617768	0.2014952	0.9808323	0.1261575	3.778169	0.2198753	-2.851743	0.2107957	1.588071	0.2140176	-1.231923	0.2426582	3.296147	0.1962984	1.245569	0.2223071	1.98822	0.1688889	0.5244448	0.1442813	0.5712083	0.1423984	1.580774	0.1360883	3.032353	0.1284363	0.2361942	0.2330592	0.7644741	0.1345135	-0.2284832	0.177801	0.9684574	0.1846398	-0.8217517	0.1690076	-0.1556177	0.2140797	0.9469408	0.1869065	1.540043	0.2084914	1.695791	0.1887449	0.8586018	0.2197045	2.042192	0.1317169	-0.8347201	0.1562218	-1.021943	0.1533783	1.968931	0.1491463	0.9810904	0.1098978	1.055438	0.2106167	1.022286	0.199176	0.9168212	0.03894011	-0.3426502	0.1671793	-3.521834	0.1534467	-0.2374157	0.2013961	1.556842	0.1250643	2.738925	0.1591011	1.886823	0.1626787	-0.2158393	0.
 2056642	3.486286	0.1987337	2.471281	0.1744332	1.152822	0.1951747	-0.989242	0.1155556	0.5867982	0.08321315	2.675081	0.1303141	-2.584626	0.1609869	-1.254465	0.122685	-2.27439	0.1415207	1.642059	0.1458468	-2.672928	0.1453881	-5.031017	0.1312302	0.2756784	0.1731334	0.8576683	0.1351911	-2.385648	0.2008109	-0.2893286	0.2062319	2.538498	0.191823	-1.739293	0.1671868	-1.622128	0.1850325	2.469388	0.2514207	2.117658	0.2225232	2.403703	0.1996269	-1.039887	0.122257	-0.6789801	0.1378511	-4.054801	0.08070226	0.2997747	0.1447362	2.234673	0.1876959	2.499773	0.1622439	0.5804301	0.1345898	0.07422866	0.09716025	-0.8710974	0.1657741	-1.370054	0.1313368	-1.671961	0.1861922	-0.9497792	0.2265888	-1.500299	0.2406933	-0.639602	0.2041045	-0.02993134	0.1440134	1.211941	0.1917053	-2.045351	0.1995012	0.4286667	0.2419866	1.319268	0.05873292	3.849816	0.1642288	3.828327	0.2147195	-0.1579602	0.2119238	-1.37588	0.142787	1.026216	0.2192122	-0.106043	0.2299229	0.5020428	0.1666804	-1.020357	0.03175325	-4.244395	0.141780
 8	1.535539	0.2701716	2.742496	0.2501491	1.629031	0.08584132	1.70437	0.1870297	6.293977	0.1240707	6.969766	-1.00584	-2.592914	8.495434	8.769945	-32.94919
\ No newline at end of file

Added: tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/GLDAS_CLM10SUBP_3H.A19790202.0000.001.grb
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/GLDAS_CLM10SUBP_3H.A19790202.0000.001.grb?rev=1677277&view=auto
==============================================================================
Binary file - no diff available.

Propchange: tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/GLDAS_CLM10SUBP_3H.A19790202.0000.001.grb
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/gdas1.forecmwf.2014062612.grib2
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/gdas1.forecmwf.2014062612.grib2?rev=1677277&view=auto
==============================================================================
Binary file - no diff available.

Propchange: tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/gdas1.forecmwf.2014062612.grib2
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream