You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ma...@apache.org on 2015/05/02 08:20:43 UTC
svn commit: r1677277 - in /tika/trunk: ./
tika-core/src/main/java/org/apache/tika/detect/
tika-core/src/main/resources/org/apache/tika/detect/
tika-core/src/test/java/org/apache/tika/detect/
tika-core/src/test/resources/org/apache/tika/detect/ tika-cor...
Author: mattmann
Date: Sat May 2 06:20:42 2015
New Revision: 1677277
URL: http://svn.apache.org/r1677277
Log:
Fix for TIKA-1582 Mime Detection based on neural networks with Byte-frequency-histogram contributed by LukeLiush <ha...@gmail.com>. This closes #36
Added:
tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNExampleModelDetector.java
tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModel.java
tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModelBuilder.java
tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModel.java
tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java
tika/trunk/tika-core/src/main/resources/org/apache/tika/detect/
tika/trunk/tika-core/src/main/resources/org/apache/tika/detect/tika-example.nnmodel
tika/trunk/tika-core/src/test/java/org/apache/tika/detect/MimeDetectionWithNNTest.java
tika/trunk/tika-core/src/test/resources/org/apache/tika/detect/
tika/trunk/tika-core/src/test/resources/org/apache/tika/detect/tika-example.nnmodel
tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/GLDAS_CLM10SUBP_3H.A19790202.0000.001.grb (with props)
tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/gdas1.forecmwf.2014062612.grib2 (with props)
Modified:
tika/trunk/CHANGES.txt
Modified: tika/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/tika/trunk/CHANGES.txt?rev=1677277&r1=1677276&r2=1677277&view=diff
==============================================================================
--- tika/trunk/CHANGES.txt (original)
+++ tika/trunk/CHANGES.txt Sat May 2 06:20:42 2015
@@ -1,5 +1,9 @@
Release 1.9 - Current Development
+ * An algorithm for using Byte Histogram frequencies to construct
+ a Neural Network and to perform MIME detection was added
+ (TIKA-1582).
+
* A Bayesian algorithm for MIME detection by probabilistic
means was added (TIKA-1517).
Added: tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNExampleModelDetector.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNExampleModelDetector.java?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNExampleModelDetector.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNExampleModelDetector.java Sat May 2 06:20:42 2015
@@ -0,0 +1,174 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.detect;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+
+import org.apache.tika.mime.MediaType;
+
+public class NNExampleModelDetector extends TrainedModelDetector {
+ private static final String EXAMPLE_NNMODEL_FILE = "tika-example.nnmodel";
+
+ private static final long serialVersionUID = 1L;
+
+ public NNExampleModelDetector() {
+ super();
+ }
+
+ public NNExampleModelDetector(final File modelFile) {
+ loadDefaultModels(modelFile);
+ }
+
+ @Override
+ public void loadDefaultModels(InputStream modelStream) {
+ BufferedReader bReader = null;
+ try{
+ bReader = new BufferedReader(
+ new InputStreamReader(modelStream, "UTF-8"));
+ }
+ catch(UnsupportedEncodingException e){
+ e.printStackTrace();
+ }
+
+ NNTrainedModelBuilder nnBuilder = new NNTrainedModelBuilder();
+ String line;
+ try {
+ while ((line = bReader.readLine()) != null) {
+ line = line.trim();
+ if (line.startsWith("#")) {
+ readDescription(nnBuilder, line);
+ } else {
+ readNNParams(nnBuilder, line);
+ // add this model into map of trained models.
+ super.registerModels(nnBuilder.getType(), nnBuilder.build());
+ }
+
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "Unable to read the default media type registry", e);
+ }
+
+ }
+
+ /**
+ * this method gets overwritten to register load neural network models
+ */
+ @Override
+ public void loadDefaultModels(ClassLoader classLoader) {
+ if (classLoader == null) {
+ classLoader = TrainedModelDetector.class.getClassLoader();
+ }
+
+ // This allows us to replicate class.getResource() when using
+ // the classloader directly
+ String classPrefix = TrainedModelDetector.class.getPackage().getName()
+ .replace('.', '/')
+ + "/";
+
+ // Get the core URL, and all the extensions URLs
+ URL modelURL = classLoader.getResource(classPrefix
+ + EXAMPLE_NNMODEL_FILE);
+ InputStream stream = null;
+ try {
+ stream = modelURL.openStream();
+ loadDefaultModels(stream);
+
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "Unable to read the default media type registry", e);
+ } finally {
+ try {
+ stream.close();
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "Unable to read the default media type registry", e);
+ }
+ }
+
+ }
+
+ /**
+ * read the comments where the model configuration is written, e.g the
+ * number of inputs, hiddens and output please ensure the first char in the
+ * given string is # In this example grb model file, there are 4 elements 1)
+ * type 2) number of input units 3) number of hidden units. 4) number of
+ * output units.
+ *
+ */
+ private void readDescription(final NNTrainedModelBuilder builder,
+ final String line) {
+ int numInputs = 0;
+ int numHidden = 0;
+ int numOutputs = 0;
+ String tline = line;
+ String[] sarr = tline.split("\t");
+
+ try {
+ MediaType type = MediaType.parse(sarr[1]);
+ numInputs = Integer.parseInt(sarr[2]);
+ numHidden = Integer.parseInt(sarr[3]);
+ numOutputs = Integer.parseInt(sarr[4]);
+ builder.setNumOfInputs(numInputs);
+ builder.setNumOfHidden(numHidden);
+ builder.setNumOfOutputs(numOutputs);
+ builder.setType(type);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException(
+ "Unable to parse the model configuration", e);
+ }
+
+ }
+
+ /**
+ * Read the next line for the model parameters and populate the build which
+ * later will be used to instantiate the instance of TrainedModel
+ *
+ * @param builder
+ * @param line
+ */
+ private void readNNParams(final NNTrainedModelBuilder builder,
+ final String line) {
+ String tline = line;
+ String[] sarr = tline.split("\t");
+ int n = sarr.length;
+ float[] params = new float[n];
+ try {
+ int i = 0;
+ for (String fstr : sarr) {
+ params[i] = Float.parseFloat(fstr);
+ i++;
+ }
+ builder.setParams(params);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException(
+ "Unable to parse the model configuration", e);
+ }
+
+ }
+
+}
Added: tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModel.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModel.java?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModel.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModel.java Sat May 2 06:20:42 2015
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.detect;
+
+public class NNTrainedModel extends TrainedModel {
+
+ private int numOfInputs;
+ private int numOfHidden;
+ private int numOfOutputs;
+ private float[][] Theta1, Theta2;
+
+ public NNTrainedModel(final int nInput, final int nHidden,
+ final int nOutput, final float[] nn_params) {
+ this.numOfInputs = nInput;
+ this.numOfHidden = nHidden;
+ this.numOfOutputs = nOutput;
+ this.Theta1 = new float[numOfHidden][numOfInputs + 1];
+ this.Theta2 = new float[numOfOutputs][numOfHidden + 1];
+ populateThetas(nn_params);
+ }
+
+ // convert the vector params to the 2 thetas.
+ private void populateThetas(final float[] nn_params) {
+ int m = this.Theta1.length;
+ int n = this.Theta1[0].length;
+ int i, j, k = 0;
+ for (i = 0; i < n; i++) {
+ for (j = 0; j < m; j++) {
+ this.Theta1[j][i] = nn_params[k];
+ k++;
+ }
+ }
+
+ m = this.Theta2.length;
+ n = this.Theta2[0].length;
+ for (i = 0; i < n; i++) {
+ for (j = 0; j < m; j++) {
+ this.Theta2[j][i] = nn_params[k];
+ k++;
+ }
+ }
+ }
+
+ @Override
+ public double predict(double[] unseen) {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ /**
+ * The given input vector of unseen is m=(256 + 1) * n= 1 this returns a
+ * prediction probability
+ */
+ @Override
+ public float predict(float[] unseen) {
+ // please ensure the unseen in size consistent with theta1
+
+ int i, j;
+ int m = this.Theta1.length;
+ int n = this.Theta1[0].length;
+ float[] hh = new float[m + 1];// hidden unit summation
+ hh[0] = 1;
+ for (i = 0; i < m; i++) {
+ double h = 0;
+ for (j = 0; j < n; j++) {
+ h += this.Theta1[i][j] * unseen[j];
+ }
+ // apply sigmoid
+ h = 1.0 / (1.0 + Math.exp(-h));
+ hh[i+1] = (float)h;
+ }
+
+ m = this.Theta2.length;
+ n = this.Theta2[0].length;
+ float[] oo = new float[m];
+ for (i = 0; i < m; i++) {
+ double o = 0;
+ for (j = 0; j < n; j++) {
+ o += this.Theta2[i][j] * hh[j];
+ }
+ // apply sigmoid
+ o = 1.0 / (1.0 + Math.exp(-o));
+ oo[i] = (float)o;
+ }
+
+ return oo[0];
+ }
+}
Added: tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModelBuilder.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModelBuilder.java?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModelBuilder.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/detect/NNTrainedModelBuilder.java Sat May 2 06:20:42 2015
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.tika.detect;
+
+import org.apache.tika.mime.MediaType;
+
+public class NNTrainedModelBuilder {
+ private MediaType type;
+
+ private int numOfInputs;
+ private int numOfHidden;
+ private int numOfOutputs;
+
+ private float[] params;
+
+ public MediaType getType() {
+ return this.type;
+ }
+
+ public int getNumOfInputs() {
+ return numOfInputs;
+ }
+
+ public int getNumOfHidden() {
+ return numOfHidden;
+ }
+
+ public int getNumOfOutputs() {
+ return numOfOutputs;
+ }
+
+ public float[] getParams() {
+ return params;
+ }
+
+ public void setType(final MediaType type) {
+ this.type = type;
+ }
+
+ public void setNumOfInputs(final int numOfInputs) {
+ this.numOfInputs = numOfInputs;
+ }
+
+ public void setNumOfHidden(final int numOfHidden) {
+ this.numOfHidden = numOfHidden;
+ }
+
+ public void setNumOfOutputs(final int numOfOutputs) {
+ this.numOfOutputs = numOfOutputs;
+ }
+
+ public void setParams(float[] params) {
+ this.params = params;
+ }
+
+ public NNTrainedModel build() {
+ return new NNTrainedModel(numOfInputs, numOfHidden, numOfOutputs,
+ params);
+ }
+}
Added: tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModel.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModel.java?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModel.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModel.java Sat May 2 06:20:42 2015
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+
+public abstract class TrainedModel {
+
+ public abstract double predict(double[] input);
+ public abstract float predict(float[] input);
+}
Added: tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java Sat May 2 06:20:42 2015
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.OutputStreamWriter;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Writer;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.io.TemporaryResources;
+
+public abstract class TrainedModelDetector implements Detector {
+ private final Map<MediaType, TrainedModel> MODEL_MAP = new HashMap<MediaType, TrainedModel>();
+
+ private static final long serialVersionUID = 1L;
+
+ public TrainedModelDetector() {
+ loadDefaultModels(getClass().getClassLoader());
+ }
+
+ public int getMinLength() {
+ return Integer.MAX_VALUE;
+ }
+
+ public MediaType detect(InputStream input, Metadata metadata)
+ throws IOException {
+ // convert to byte-histogram
+ if (input != null) {
+ input.mark(getMinLength());
+ float[] histogram = readByteFrequencies(input);
+ // writeHisto(histogram); //on testing purpose
+ /*
+ * iterate the map to find out the one that gives the higher
+ * prediction value.
+ */
+ Iterator<MediaType> iter = MODEL_MAP.keySet().iterator();
+ float threshold = 0.5f;// probability threshold, any value below the
+ // threshold will be considered as
+ // MediaType.OCTET_STREAM
+ float maxprob = threshold;
+ MediaType maxType = MediaType.OCTET_STREAM;
+ while (iter.hasNext()) {
+ MediaType key = iter.next();
+ TrainedModel model = MODEL_MAP.get(key);
+ float prob = model.predict(histogram);
+ if (maxprob < prob) {
+ maxprob = prob;
+ maxType = key;
+ }
+ }
+ input.reset();
+ return maxType;
+ }
+ return null;
+ }
+
+ /**
+ * read the inputstream and build a byte frequence histogram
+ *
+ * @param input
+ * @return
+ * @throws IOException
+ */
+ protected float[] readByteFrequencies(final InputStream input)
+ throws IOException {
+
+ ReadableByteChannel inputChannel = null;
+ try {
+ inputChannel = Channels.newChannel(input);
+ // long inSize = inputChannel.size();
+ float histogram[] = new float[257];
+ histogram[0] = 1;
+
+ // create buffer with capacity of maxBufSize bytes
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 5);
+ int bytesRead = inputChannel.read(buf); // read into buffer.
+
+ float max = -1;
+ while (bytesRead != -1) {
+
+ buf.flip(); // make buffer ready for read
+
+ while (buf.hasRemaining()) {
+ byte byt = buf.get();
+ int idx = byt;
+ idx++;
+ if (byt < 0) {
+ idx = 256 + idx;
+ histogram[idx]++;
+ } else {
+ histogram[idx]++;
+ }
+ max = max < histogram[idx] ? histogram[idx] : max;
+ }
+
+ buf.clear(); // make buffer ready for writing
+ bytesRead = inputChannel.read(buf);
+ }
+
+ int i;
+ for (i = 1; i < histogram.length; i++) {
+ histogram[i] /= max;
+ histogram[i] = (float) Math.sqrt(histogram[i]);
+ }
+
+ return histogram;
+ } finally {
+ // inputChannel.close();
+ }
+
+ }
+
+ /**
+ * on testing purpose; this method write the histogram vector to a file.
+ *
+ * @param histogram
+ * @throws IOException
+ */
+ private synchronized void writeHisto(final float[] histogram)
+ throws IOException {
+ String histPath = new TemporaryResources().createTemporaryFile().getAbsolutePath();
+ Writer writer = new OutputStreamWriter(new FileOutputStream(histPath),"UTF-8");
+ int n = histogram.length;// excluding the last one for storing the
+ // max value
+ for (int i = 0; i < n; i++) {
+ writer.write(new StringBuffer().append(histogram[i]).append("\t")
+ .toString());
+ // writer.write(i + "\t");
+ }
+
+ writer.write("\r\n");
+ writer.flush();
+ }
+
+ public void loadDefaultModels(final File modelFile) {
+ FileInputStream in = null;
+ try {
+ in = new FileInputStream(modelFile);
+ loadDefaultModels(in);
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(
+ "Unable to read the default media type registry", e);
+ } finally {
+ try {
+ in.close();
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "Unable to read the default media type registry", e);
+ }
+ }
+ }
+
+ public abstract void loadDefaultModels(final InputStream modelStream);
+
+ public abstract void loadDefaultModels(final ClassLoader classLoader);
+
+ protected void registerModels(MediaType type, TrainedModel model) {
+ MODEL_MAP.put(type, model);
+ }
+
+}
Added: tika/trunk/tika-core/src/main/resources/org/apache/tika/detect/tika-example.nnmodel
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/resources/org/apache/tika/detect/tika-example.nnmodel?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/resources/org/apache/tika/detect/tika-example.nnmodel (added)
+++ tika/trunk/tika-core/src/main/resources/org/apache/tika/detect/tika-example.nnmodel Sat May 2 06:20:42 2015
@@ -0,0 +1,2 @@
+#nn application/x-grib 256 2 1 0.0208829625450076
+-0.05145498 5.127922 -3.078527 -3.988763 -0.2107816 0.8629464 -0.0002683103 3.268235 -0.1322597 2.256234 0.1585416 2.448373 -0.181859 -2.230397 -0.1796804 -3.778733 0.04216435 -5.247452 -0.07137667 4.015641 0.3369519 -2.232862 0.9872297 3.434345 0.01537457 -2.99547 0.05939847 -0.4262142 0.6431364 4.015367 -0.1052946 -0.5597504 -0.02748792 1.406441 -0.00300846 1.756262 0.09294704 -1.096173 -0.006158207 -1.221678 -0.02583992 -3.162681 0.08686034 1.269717 -0.01359701 -4.19245 -0.1400172 -3.274052 -0.0260648 -2.134415 0.02505031 -0.7995273 -0.01464903 -5.054871 0.06927952 -1.541063 -0.1199119 -1.126015 0.1847894 0.3805432 0.1132536 -1.962234 0.1468857 -0.5322409 -0.1450888 -0.1371484 3.097646 7.200917 0.17761 -0.5256572 0.8017177 0.8039271 0.2925865 3.100624 0.1743965 4.369426 0.1317674 -0.833718 0.220852 -4.685311 0.220108 1.81378 0.283003 2.38208 0.3301778 -3.338501 0.2043622 2.858757 0.1801032 -2.616795 0.5628204 1.599731 0.6208351 1.161804 0.8223767 -2.614317 0.9187943 -2.446297 1.3
50831 2.777917 0.8146963 -0.9438954 0.8172006 -0.3445128 0.6723926 0.9863181 0.5705485 2.406452 0.6185437 -1.202793 0.6450134 1.808502 0.4259292 -7.941289 0.6099652 3.878562 0.4132371 0.98963 0.3298724 -2.366126 0.3319441 -0.8838808 0.6854118 3.12115 0.525016 -1.727541 0.7185858 1.830994 0.03694244 -3.686851 -0.09525806 3.558239 0.4775859 1.931869 0.293615 -3.693917 0.4294216 -5.695489 0.3452209 0.6723521 0.4919632 -0.7649217 0.3818015 2.153055 0.3088684 -0.655546 0.2931553 -1.343124 0.4890785 -0.7017448 0.1028709 -2.578852 0.1097999 -0.3793636 0.3154411 -2.476895 0.3301697 -1.578032 0.4468783 0.9338115 0.3876319 0.1819231 0.3975481 2.377385 0.07344178 -0.5816987 0.4968894 -3.409108 0.4824005 -0.2253791 0.538568 0.3602099 0.1838869 0.9865417 0.2429371 -0.2590284 0.1997649 -2.825891 0.1690305 -2.876223 0.1508797 -2.025365 0.0745675 1.403832 0.1730641 -3.619998 0.02000108 -1.345008 0.1982651 -4.596162 0.08308789 -0.08156122 0.2391607 0.06438381 -0.01053187 -3.724878 1.342193 0.8912752
0.6741032 -2.038227 1.02059 1.314822 1.046652 -0.09340967 1.746544 1.748925 0.7121063 2.712003 0.7263532 -0.4646987 0.8260612 -1.657031 1.326838 2.972081 0.3091417 -0.0007440437 0.4395537 0.2231857 1.106737 -1.463802 0.912596 2.458881 1.365523 2.234517 1.385396 -0.4650663 0.7839583 2.282071 0.2386854 0.9784987 1.273242 0.6254676 1.276118 2.447344 1.575805 4.515861 0.8364591 0.6976352 0.5248962 -2.899523 0.5404893 -1.394514 0.3592697 -0.9920228 0.6932542 -0.180825 0.2744942 -0.6203262 0.2417071 -1.224851 0.1811886 -0.8900446 0.226887 -3.733405 0.1735546 -1.547938 0.112908 -8.991437 0.1115473 -4.406975 0.1178586 2.36029 0.1271653 -2.011151 0.08239845 -0.1951623 0.04392552 -3.046913 0.05452365 -1.516051 0.1397218 1.094332 0.1264289 1.933882 0.1752298 -1.989354 0.1306208 -0.9046116 0.1277338 2.303931 0.1250716 -1.277528 0.1262729 2.467187 0.1100633 -0.01313957 0.2080954 0.3508277 0.2128501 0.8266836 0.1570878 -3.358276 0.1242704 0.5336581 0.2453244 1.62488 0.1280062 -2.309668 0.1366247
-2.19346 0.1809374 -1.296981 0.1988072 1.969289 0.108966 1.910149 0.1144941 1.112991 0.1896516 3.369206 0.1071244 -1.499693 0.2437382 -0.4268651 0.2060418 2.251279 0.1191459 -0.03434586 0.173683 2.421464 0.1856368 0.8361177 0.1358361 -1.145463 0.1938676 -0.5617768 0.2014952 0.9808323 0.1261575 3.778169 0.2198753 -2.851743 0.2107957 1.588071 0.2140176 -1.231923 0.2426582 3.296147 0.1962984 1.245569 0.2223071 1.98822 0.1688889 0.5244448 0.1442813 0.5712083 0.1423984 1.580774 0.1360883 3.032353 0.1284363 0.2361942 0.2330592 0.7644741 0.1345135 -0.2284832 0.177801 0.9684574 0.1846398 -0.8217517 0.1690076 -0.1556177 0.2140797 0.9469408 0.1869065 1.540043 0.2084914 1.695791 0.1887449 0.8586018 0.2197045 2.042192 0.1317169 -0.8347201 0.1562218 -1.021943 0.1533783 1.968931 0.1491463 0.9810904 0.1098978 1.055438 0.2106167 1.022286 0.199176 0.9168212 0.03894011 -0.3426502 0.1671793 -3.521834 0.1534467 -0.2374157 0.2013961 1.556842 0.1250643 2.738925 0.1591011 1.886823 0.1626787 -0.2158393 0.
2056642 3.486286 0.1987337 2.471281 0.1744332 1.152822 0.1951747 -0.989242 0.1155556 0.5867982 0.08321315 2.675081 0.1303141 -2.584626 0.1609869 -1.254465 0.122685 -2.27439 0.1415207 1.642059 0.1458468 -2.672928 0.1453881 -5.031017 0.1312302 0.2756784 0.1731334 0.8576683 0.1351911 -2.385648 0.2008109 -0.2893286 0.2062319 2.538498 0.191823 -1.739293 0.1671868 -1.622128 0.1850325 2.469388 0.2514207 2.117658 0.2225232 2.403703 0.1996269 -1.039887 0.122257 -0.6789801 0.1378511 -4.054801 0.08070226 0.2997747 0.1447362 2.234673 0.1876959 2.499773 0.1622439 0.5804301 0.1345898 0.07422866 0.09716025 -0.8710974 0.1657741 -1.370054 0.1313368 -1.671961 0.1861922 -0.9497792 0.2265888 -1.500299 0.2406933 -0.639602 0.2041045 -0.02993134 0.1440134 1.211941 0.1917053 -2.045351 0.1995012 0.4286667 0.2419866 1.319268 0.05873292 3.849816 0.1642288 3.828327 0.2147195 -0.1579602 0.2119238 -1.37588 0.142787 1.026216 0.2192122 -0.106043 0.2299229 0.5020428 0.1666804 -1.020357 0.03175325 -4.244395 0.141780
8 1.535539 0.2701716 2.742496 0.2501491 1.629031 0.08584132 1.70437 0.1870297 6.293977 0.1240707 6.969766 -1.00584 -2.592914 8.495434 8.769945 -32.94919
\ No newline at end of file
Added: tika/trunk/tika-core/src/test/java/org/apache/tika/detect/MimeDetectionWithNNTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/test/java/org/apache/tika/detect/MimeDetectionWithNNTest.java?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/test/java/org/apache/tika/detect/MimeDetectionWithNNTest.java (added)
+++ tika/trunk/tika-core/src/test/java/org/apache/tika/detect/MimeDetectionWithNNTest.java Sat May 2 06:20:42 2015
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.mime.MimeDetectionTest;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MimeDetectionWithNNTest {
+
+ private Detector detector;
+
+ /** @inheritDoc */
+ @Before
+ public void setUp() {
+ detector = new NNExampleModelDetector();
+ }
+
+ /**
+ * The test case only works on the detector that only has grb model as
+ * currently the grb model is used as an example; if more models are added
+ * in the TrainedModelDetector, the following tests will need to modified to reflect
+ * the corresponding type instead of test-equal with the "OCTET_STREAM";
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDetection() throws Exception {
+ String octetStream_str = MediaType.OCTET_STREAM.toString();
+ String grb_str = "application/x-grib";
+
+ testFile(grb_str, "gdas1.forecmwf.2014062612.grib2");
+ testFile(grb_str, "GLDAS_CLM10SUBP_3H.A19790202.0000.001.grb");
+
+ testFile(octetStream_str, "circles.svg");
+ testFile(octetStream_str, "circles-with-prefix.svg");
+ testFile(octetStream_str, "datamatrix.png");
+ testFile(octetStream_str, "test.html");
+ testFile(octetStream_str, "test-iso-8859-1.xml");
+ testFile(octetStream_str, "test-utf8.xml");
+ testFile(octetStream_str, "test-utf8-bom.xml");
+ testFile(octetStream_str, "test-utf16le.xml");
+ testFile(octetStream_str, "test-utf16be.xml");
+ testFile(octetStream_str, "test-long-comment.xml");
+ testFile(octetStream_str, "stylesheet.xsl");
+ testUrl(octetStream_str,
+ "http://www.ai.sri.com/daml/services/owl-s/1.2/Process.owl",
+ "test-difficult-rdf1.xml");
+ testUrl(octetStream_str, "http://www.w3.org/2002/07/owl#",
+ "test-difficult-rdf2.xml");
+ // add evil test from TIKA-327
+ testFile(octetStream_str, "test-tika-327.html");
+ // add another evil html test from TIKA-357
+ testFile(octetStream_str, "testlargerbuffer.html");
+ // test fragment of HTML with <div> (TIKA-1102)
+ testFile(octetStream_str, "htmlfragment");
+ // test binary CGM detection (TIKA-1170)
+ testFile(octetStream_str, "plotutils-bin-cgm-v3.cgm");
+ // test HTML detection of malformed file, previously identified as
+ // image/cgm (TIKA-1170)
+ testFile(octetStream_str, "test-malformed-header.html.bin");
+
+ // test GCMD Directory Interchange Format (.dif) TIKA-1561
+ testFile(octetStream_str, "brwNIMS_2014.dif");
+ }
+
+ private void testUrl(String expected, String url, String file)
+ throws IOException {
+ InputStream in = MimeDetectionTest.class.getResourceAsStream(file);
+ testStream(expected, url, in);
+ }
+
+ private void testFile(String expected, String filename) throws IOException {
+
+ InputStream in = MimeDetectionTest.class.getResourceAsStream(filename);
+ testStream(expected, filename, in);
+ }
+
+ private void testStream(String expected, String urlOrFileName,
+ InputStream in) throws IOException {
+ assertNotNull("Test stream: [" + urlOrFileName + "] is null!", in);
+ if (!in.markSupported()) {
+ in = new java.io.BufferedInputStream(in);
+ }
+ try {
+ Metadata metadata = new Metadata();
+ String mime = this.detector.detect(in, metadata).toString();
+ assertEquals(
+ urlOrFileName + " is not properly detected: detected.",
+ expected, mime);
+
+ // Add resource name and test again
+ // metadata.set(Metadata.RESOURCE_NAME_KEY, urlOrFileName);
+ mime = this.detector.detect(in, metadata).toString();
+ assertEquals(urlOrFileName
+ + " is not properly detected after adding resource name.",
+ expected, mime);
+ } finally {
+ in.close();
+ }
+ }
+
+ private void assertNotNull(String string, InputStream in) {
+ // TODO Auto-generated method stub
+
+ }
+
+ /**
+ * Test for type detection of empty documents.
+ */
+ @Test
+ public void testEmptyDocument() throws IOException {
+ assertEquals(MediaType.OCTET_STREAM, detector.detect(
+ new ByteArrayInputStream(new byte[0]), new Metadata()));
+
+ }
+
+}
Added: tika/trunk/tika-core/src/test/resources/org/apache/tika/detect/tika-example.nnmodel
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/test/resources/org/apache/tika/detect/tika-example.nnmodel?rev=1677277&view=auto
==============================================================================
--- tika/trunk/tika-core/src/test/resources/org/apache/tika/detect/tika-example.nnmodel (added)
+++ tika/trunk/tika-core/src/test/resources/org/apache/tika/detect/tika-example.nnmodel Sat May 2 06:20:42 2015
@@ -0,0 +1,2 @@
+#nn application/x-grib 256 2 1 0.0208829625450076
+-0.05145498 5.127922 -3.078527 -3.988763 -0.2107816 0.8629464 -0.0002683103 3.268235 -0.1322597 2.256234 0.1585416 2.448373 -0.181859 -2.230397 -0.1796804 -3.778733 0.04216435 -5.247452 -0.07137667 4.015641 0.3369519 -2.232862 0.9872297 3.434345 0.01537457 -2.99547 0.05939847 -0.4262142 0.6431364 4.015367 -0.1052946 -0.5597504 -0.02748792 1.406441 -0.00300846 1.756262 0.09294704 -1.096173 -0.006158207 -1.221678 -0.02583992 -3.162681 0.08686034 1.269717 -0.01359701 -4.19245 -0.1400172 -3.274052 -0.0260648 -2.134415 0.02505031 -0.7995273 -0.01464903 -5.054871 0.06927952 -1.541063 -0.1199119 -1.126015 0.1847894 0.3805432 0.1132536 -1.962234 0.1468857 -0.5322409 -0.1450888 -0.1371484 3.097646 7.200917 0.17761 -0.5256572 0.8017177 0.8039271 0.2925865 3.100624 0.1743965 4.369426 0.1317674 -0.833718 0.220852 -4.685311 0.220108 1.81378 0.283003 2.38208 0.3301778 -3.338501 0.2043622 2.858757 0.1801032 -2.616795 0.5628204 1.599731 0.6208351 1.161804 0.8223767 -2.614317 0.9187943 -2.446297 1.3
50831 2.777917 0.8146963 -0.9438954 0.8172006 -0.3445128 0.6723926 0.9863181 0.5705485 2.406452 0.6185437 -1.202793 0.6450134 1.808502 0.4259292 -7.941289 0.6099652 3.878562 0.4132371 0.98963 0.3298724 -2.366126 0.3319441 -0.8838808 0.6854118 3.12115 0.525016 -1.727541 0.7185858 1.830994 0.03694244 -3.686851 -0.09525806 3.558239 0.4775859 1.931869 0.293615 -3.693917 0.4294216 -5.695489 0.3452209 0.6723521 0.4919632 -0.7649217 0.3818015 2.153055 0.3088684 -0.655546 0.2931553 -1.343124 0.4890785 -0.7017448 0.1028709 -2.578852 0.1097999 -0.3793636 0.3154411 -2.476895 0.3301697 -1.578032 0.4468783 0.9338115 0.3876319 0.1819231 0.3975481 2.377385 0.07344178 -0.5816987 0.4968894 -3.409108 0.4824005 -0.2253791 0.538568 0.3602099 0.1838869 0.9865417 0.2429371 -0.2590284 0.1997649 -2.825891 0.1690305 -2.876223 0.1508797 -2.025365 0.0745675 1.403832 0.1730641 -3.619998 0.02000108 -1.345008 0.1982651 -4.596162 0.08308789 -0.08156122 0.2391607 0.06438381 -0.01053187 -3.724878 1.342193 0.8912752
0.6741032 -2.038227 1.02059 1.314822 1.046652 -0.09340967 1.746544 1.748925 0.7121063 2.712003 0.7263532 -0.4646987 0.8260612 -1.657031 1.326838 2.972081 0.3091417 -0.0007440437 0.4395537 0.2231857 1.106737 -1.463802 0.912596 2.458881 1.365523 2.234517 1.385396 -0.4650663 0.7839583 2.282071 0.2386854 0.9784987 1.273242 0.6254676 1.276118 2.447344 1.575805 4.515861 0.8364591 0.6976352 0.5248962 -2.899523 0.5404893 -1.394514 0.3592697 -0.9920228 0.6932542 -0.180825 0.2744942 -0.6203262 0.2417071 -1.224851 0.1811886 -0.8900446 0.226887 -3.733405 0.1735546 -1.547938 0.112908 -8.991437 0.1115473 -4.406975 0.1178586 2.36029 0.1271653 -2.011151 0.08239845 -0.1951623 0.04392552 -3.046913 0.05452365 -1.516051 0.1397218 1.094332 0.1264289 1.933882 0.1752298 -1.989354 0.1306208 -0.9046116 0.1277338 2.303931 0.1250716 -1.277528 0.1262729 2.467187 0.1100633 -0.01313957 0.2080954 0.3508277 0.2128501 0.8266836 0.1570878 -3.358276 0.1242704 0.5336581 0.2453244 1.62488 0.1280062 -2.309668 0.1366247
-2.19346 0.1809374 -1.296981 0.1988072 1.969289 0.108966 1.910149 0.1144941 1.112991 0.1896516 3.369206 0.1071244 -1.499693 0.2437382 -0.4268651 0.2060418 2.251279 0.1191459 -0.03434586 0.173683 2.421464 0.1856368 0.8361177 0.1358361 -1.145463 0.1938676 -0.5617768 0.2014952 0.9808323 0.1261575 3.778169 0.2198753 -2.851743 0.2107957 1.588071 0.2140176 -1.231923 0.2426582 3.296147 0.1962984 1.245569 0.2223071 1.98822 0.1688889 0.5244448 0.1442813 0.5712083 0.1423984 1.580774 0.1360883 3.032353 0.1284363 0.2361942 0.2330592 0.7644741 0.1345135 -0.2284832 0.177801 0.9684574 0.1846398 -0.8217517 0.1690076 -0.1556177 0.2140797 0.9469408 0.1869065 1.540043 0.2084914 1.695791 0.1887449 0.8586018 0.2197045 2.042192 0.1317169 -0.8347201 0.1562218 -1.021943 0.1533783 1.968931 0.1491463 0.9810904 0.1098978 1.055438 0.2106167 1.022286 0.199176 0.9168212 0.03894011 -0.3426502 0.1671793 -3.521834 0.1534467 -0.2374157 0.2013961 1.556842 0.1250643 2.738925 0.1591011 1.886823 0.1626787 -0.2158393 0.
2056642 3.486286 0.1987337 2.471281 0.1744332 1.152822 0.1951747 -0.989242 0.1155556 0.5867982 0.08321315 2.675081 0.1303141 -2.584626 0.1609869 -1.254465 0.122685 -2.27439 0.1415207 1.642059 0.1458468 -2.672928 0.1453881 -5.031017 0.1312302 0.2756784 0.1731334 0.8576683 0.1351911 -2.385648 0.2008109 -0.2893286 0.2062319 2.538498 0.191823 -1.739293 0.1671868 -1.622128 0.1850325 2.469388 0.2514207 2.117658 0.2225232 2.403703 0.1996269 -1.039887 0.122257 -0.6789801 0.1378511 -4.054801 0.08070226 0.2997747 0.1447362 2.234673 0.1876959 2.499773 0.1622439 0.5804301 0.1345898 0.07422866 0.09716025 -0.8710974 0.1657741 -1.370054 0.1313368 -1.671961 0.1861922 -0.9497792 0.2265888 -1.500299 0.2406933 -0.639602 0.2041045 -0.02993134 0.1440134 1.211941 0.1917053 -2.045351 0.1995012 0.4286667 0.2419866 1.319268 0.05873292 3.849816 0.1642288 3.828327 0.2147195 -0.1579602 0.2119238 -1.37588 0.142787 1.026216 0.2192122 -0.106043 0.2299229 0.5020428 0.1666804 -1.020357 0.03175325 -4.244395 0.141780
8 1.535539 0.2701716 2.742496 0.2501491 1.629031 0.08584132 1.70437 0.1870297 6.293977 0.1240707 6.969766 -1.00584 -2.592914 8.495434 8.769945 -32.94919
\ No newline at end of file
Added: tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/GLDAS_CLM10SUBP_3H.A19790202.0000.001.grb
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/GLDAS_CLM10SUBP_3H.A19790202.0000.001.grb?rev=1677277&view=auto
==============================================================================
Binary file - no diff available.
Propchange: tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/GLDAS_CLM10SUBP_3H.A19790202.0000.001.grb
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/gdas1.forecmwf.2014062612.grib2
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/gdas1.forecmwf.2014062612.grib2?rev=1677277&view=auto
==============================================================================
Binary file - no diff available.
Propchange: tika/trunk/tika-core/src/test/resources/org/apache/tika/mime/gdas1.forecmwf.2014062612.grib2
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream