You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ni...@apache.org on 2017/09/21 20:40:26 UTC
systemml git commit: [SYSTEMML-1928] Removed Tensorboard
visualization code from Caffe2DML
Repository: systemml
Updated Branches:
refs/heads/master 32620f294 -> a4962f39a
[SYSTEMML-1928] Removed Tensorboard visualization code from Caffe2DML
- This avoids compilation error on powerpc.
Closes #671.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/a4962f39
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/a4962f39
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/a4962f39
Branch: refs/heads/master
Commit: a4962f39a065bce89dd1c45635e318b9440e5491
Parents: 32620f2
Author: Niketan Pansare <np...@us.ibm.com>
Authored: Thu Sep 21 13:37:49 2017 -0700
Committer: Niketan Pansare <np...@us.ibm.com>
Committed: Thu Sep 21 13:37:49 2017 -0700
----------------------------------------------------------------------
docs/beginners-guide-caffe2dml.md | 2 +-
pom.xml | 23 +-
src/assembly/extra.xml | 2 -
src/assembly/extra/LICENSE | 210 -------------------
.../udf/lib/Caffe2DMLVisualizeWrapper.java | 66 ------
.../apache/sysml/utils/TensorboardLogger.java | 177 ----------------
src/main/proto/tensorflow/event.proto | 82 --------
src/main/proto/tensorflow/summary.proto | 103 ---------
src/main/python/LICENSE | 210 -------------------
src/main/python/systemml/mllearn/estimators.py | 33 +--
.../org/apache/sysml/api/dl/Caffe2DML.scala | 8 -
.../org/apache/sysml/api/dl/DMLGenerator.scala | 89 +-------
12 files changed, 5 insertions(+), 1000 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/docs/beginners-guide-caffe2dml.md
----------------------------------------------------------------------
diff --git a/docs/beginners-guide-caffe2dml.md b/docs/beginners-guide-caffe2dml.md
index 12b21d4..220e02c 100644
--- a/docs/beginners-guide-caffe2dml.md
+++ b/docs/beginners-guide-caffe2dml.md
@@ -231,7 +231,7 @@ Dml.g4 ---> antlr ---> DmlLexer.java, DmlListener.java, DmlParser.java ---
caffe.proto ---> protoc ---> target/generated-sources/caffe/Caffe.java ---> parse caffe_network.proto, caffe_solver.proto
```
-Again, the SystemML engine doesnot invoke (or depend on) Caffe and TensorFlow for any of its runtime operators.
+Again, the SystemML engine doesnot invoke (or depend on) Caffe for any of its runtime operators.
Since the grammar files for the respective APIs (i.e. `caffe.proto`) are used by SystemML,
we include their licenses in our jar files.
http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7e04af5..c19e11c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -270,9 +270,6 @@
</archive>
<excludes>
<exclude>**/caffe/*</exclude>
- <exclude>**/org/tensorflow/*</exclude>
- <exclude>**/org/tensorflow/framework/*</exclude>
- <exclude>**/org/tensorflow/util/*</exclude>
<exclude>**/org/apache/sysml/api/dl/*</exclude>
</excludes>
</configuration>
@@ -366,22 +363,6 @@
</outputDirectories>
</configuration>
</execution>
- <execution>
- <id>tf-sources</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <protocVersion>3.0.0</protocVersion> <!-- 2.4.1, 2.5.0, 2.6.1, 3.0.0 -->
- <inputDirectories>
- <include>src/main/proto/tensorflow</include>
- </inputDirectories>
- <outputDirectories>
- <include>src/main/java</include>
- </outputDirectories>
- </configuration>
- </execution>
</executions>
</plugin>
@@ -884,8 +865,6 @@
<exclude>src/main/cpp/kernels/SystemML.ptx</exclude>
<!-- Proto files -->
<exclude>src/main/proto/caffe/caffe.proto</exclude>
- <exclude>src/main/proto/tensorflow/event.proto</exclude>
- <exclude>src/main/proto/tensorflow/summary.proto</exclude>
<exclude>scripts/nn/examples/caffe2dml/models/**/*</exclude>
<!-- Test Validation files -->
<exclude>src/test/scripts/functions/jmlc/**/*.impute</exclude>
@@ -1045,7 +1024,7 @@
org.apache.sysml.runtime.matrix.data (for MatrixIndexes, MatrixBlock, etc)
org.apache.sysml.udf
-->
- <excludePackageNames>caffe:org.tensorflow:org.apache.sysml.conf:org.apache.sysml.debug:org.apache.sysml.hops:org.apache.sysml.lops:org.apache.sysml.parser:org.apache.sysml.runtime.controlprogram:org.apache.sysml.runtime.functionobjects:org.apache.sysml.runtime.instructions.cp:org.apache.sysml.runtime.instructions.cpfile:org.apache.sysml.runtime.instructions.mr:org.apache.sysml.runtime.instructions.spark.data:org.apache.sysml.runtime.instructions.spark.functions:org.apache.sysml.runtime.io:org.apache.sysml.runtime.matrix.data.hadoopfix:org.apache.sysml.runtime.matrix.mapred:org.apache.sysml.runtime.matrix.operators:org.apache.sysml.runtime.matrix.sort:org.apache.sysml.runtime.transform:org.apache.sysml.runtime.util:org.apache.sysml.utils:org.apache.sysml.yarn</excludePackageNames>
+ <excludePackageNames>caffe:org.apache.sysml.conf:org.apache.sysml.debug:org.apache.sysml.hops:org.apache.sysml.lops:org.apache.sysml.parser:org.apache.sysml.runtime.controlprogram:org.apache.sysml.runtime.functionobjects:org.apache.sysml.runtime.instructions.cp:org.apache.sysml.runtime.instructions.cpfile:org.apache.sysml.runtime.instructions.mr:org.apache.sysml.runtime.instructions.spark.data:org.apache.sysml.runtime.instructions.spark.functions:org.apache.sysml.runtime.io:org.apache.sysml.runtime.matrix.data.hadoopfix:org.apache.sysml.runtime.matrix.mapred:org.apache.sysml.runtime.matrix.operators:org.apache.sysml.runtime.matrix.sort:org.apache.sysml.runtime.transform:org.apache.sysml.runtime.util:org.apache.sysml.utils:org.apache.sysml.yarn</excludePackageNames>
<additionalparam>${javadoc.opts}</additionalparam>
</configuration>
<executions>
http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/assembly/extra.xml
----------------------------------------------------------------------
diff --git a/src/assembly/extra.xml b/src/assembly/extra.xml
index c781615..9495b86 100644
--- a/src/assembly/extra.xml
+++ b/src/assembly/extra.xml
@@ -43,8 +43,6 @@
<directory>${basedir}/target/classes</directory>
<includes>
<include>**/caffe/*</include>
- <include>**/org/tensorflow/framework/*</include>
- <include>**/org/tensorflow/util/*</include>
<include>**/org/apache/sysml/api/dl/*</include>
</includes>
<outputDirectory>.</outputDirectory>
http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/assembly/extra/LICENSE
----------------------------------------------------------------------
diff --git a/src/assembly/extra/LICENSE b/src/assembly/extra/LICENSE
index c495849..e08b593 100644
--- a/src/assembly/extra/LICENSE
+++ b/src/assembly/extra/LICENSE
@@ -253,216 +253,6 @@ license and copyright terms herein.
===============================================================================
-The proto files (src/main/proto/tensorflow/event.proto and src/main/proto/tensorflow/summary.proto) is part of TensorFlow project,
-which is used to generate org.tensorflow package.
-TensorFlow are distributed under the Apache License 2.0.
-
-Copyright 2017 The TensorFlow Authors. All rights reserved.
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright 2017, The TensorFlow Authors.
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-===============================================================================
-
The following compile-scope dependencies come under the MIT License
JCuda (jcuda.org)
http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/java/org/apache/sysml/udf/lib/Caffe2DMLVisualizeWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/Caffe2DMLVisualizeWrapper.java b/src/main/java/org/apache/sysml/udf/lib/Caffe2DMLVisualizeWrapper.java
deleted file mode 100644
index 15c867b..0000000
--- a/src/main/java/org/apache/sysml/udf/lib/Caffe2DMLVisualizeWrapper.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.udf.lib;
-
-import org.apache.sysml.udf.FunctionParameter;
-import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.Scalar;
-import org.apache.sysml.udf.Scalar.ScalarValueType;
-import org.apache.sysml.utils.TensorboardLogger;
-
-public class Caffe2DMLVisualizeWrapper extends PackageFunction
-{
- private static final long serialVersionUID = 1L;
- private Scalar _ret;
-
- @Override
- public int getNumFunctionOutputs() {
- return 1;
- }
-
- @Override
- public FunctionParameter getFunctionOutput(int pos) {
- if (pos == 0)
- return _ret;
-
- throw new RuntimeException(
- "Invalid function output being requested");
- }
-
- @Override
- public void execute() {
- String layerName = ((Scalar) this.getFunctionInput(0)).getValue();
- String varType = ((Scalar) this.getFunctionInput(1)).getValue();
- String aggFn = ((Scalar) this.getFunctionInput(2)).getValue();
- double x = Double.parseDouble(((Scalar) this.getFunctionInput(3)).getValue());
- double y = Double.parseDouble(((Scalar) this.getFunctionInput(4)).getValue());
- String logDir = ((Scalar) this.getFunctionInput(5)).getValue();
-
- String key = null;
- if(aggFn.equals("training_loss") || aggFn.equals("validation_loss") ||
- aggFn.equals("training_accuracy") || aggFn.equals("validation_accuracy"))
- key = aggFn;
- else
- key = aggFn + "_" + varType + "_" + layerName;
- TensorboardLogger.writeScalar(logDir, key, (long)x, (float)y);
- _ret = new Scalar(ScalarValueType.Double, String.valueOf(1));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/java/org/apache/sysml/utils/TensorboardLogger.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/TensorboardLogger.java b/src/main/java/org/apache/sysml/utils/TensorboardLogger.java
deleted file mode 100644
index 245d757..0000000
--- a/src/main/java/org/apache/sysml/utils/TensorboardLogger.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.utils;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.zip.Checksum;
-import org.tensorflow.framework.Summary;
-import org.tensorflow.util.Event;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-
-public class TensorboardLogger {
- private static Crc32c crc32 = new Crc32c();
-
- /**
- * Writes scalar of given value in tensorboard format
- *
- * @param logDir log directory of tensorboard
- * @param tag scalar tag (for example: training_loss, validation_loss, ...)
- * @param step usually the iteration number
- * @param value value of the scalar
- */
- public static void writeScalar(String logDir, String tag, long step, float value) {
- String filePath = logDir + File.separator + "tfevents.event_systemml_scalar";
- try {
- FileOutputStream outputStream = new FileOutputStream(filePath, true);
- Event event = Event.newBuilder()
- .setWallTime(System.currentTimeMillis() / 1e3)
- .setStep(step)
- .setSummary(Summary.newBuilder().addValue(
- Summary.Value.newBuilder().setTag(tag).setSimpleValue(value)
- ).build())
- .build();
- byte[] eventString = event.toByteArray();
- byte[] header = reverse(Longs.toByteArray((long)eventString.length));
- write(outputStream, header);
- write(outputStream, eventString);
- outputStream.close();
- }
- catch(IOException e) {
- throw new RuntimeException("Error writing event in tensorboard directory:" + filePath, e);
- }
- }
-
- private static void write(FileOutputStream outputStream, byte[] byteString) throws IOException {
- outputStream.write(byteString);
- outputStream.write(reverse(Ints.toByteArray((int)maskedCRC32(byteString))));
- }
-
- private static byte[] reverse(byte[] nums) {
- byte[] reversed = new byte[nums.length];
- for (int i=0; i<nums.length; i++) {
- reversed[i] = nums[nums.length - 1 - i];
- }
- return reversed;
- }
-
- private static long maskedCRC32(byte[] data){
- crc32.reset();
- crc32.update(data, 0, data.length);
- long x = u32(crc32.getValue());
- return u32(((x >> 15) | u32(x << 17)) + 0xa282ead8);
- }
-
- private static long u32(long x){
- return x & 0xffffffff;
- }
-}
-
-class Crc32c implements Checksum {
- private static final int[] crcTable = {
- 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4,
- 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB,
- 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,
- 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24,
- 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B,
- 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384,
- 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54,
- 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B,
- 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A,
- 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35,
- 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5,
- 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA,
- 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45,
- 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A,
- 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A,
- 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595,
- 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48,
- 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957,
- 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687,
- 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198,
- 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927,
- 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38,
- 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8,
- 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7,
- 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096,
- 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789,
- 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859,
- 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46,
- 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9,
- 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6,
- 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36,
- 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829,
- 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C,
- 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93,
- 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043,
- 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C,
- 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3,
- 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC,
- 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C,
- 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033,
- 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652,
- 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D,
- 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D,
- 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982,
- 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D,
- 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622,
- 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2,
- 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED,
- 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530,
- 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F,
- 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF,
- 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0,
- 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F,
- 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540,
- 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90,
- 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F,
- 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE,
- 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1,
- 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321,
- 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E,
- 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81,
- 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E,
- 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,
- 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351,
- };
-
- private int crc = ~0;
-
- public void update(byte[] buffer, int offset, int length) {
- for (int i = offset; i < offset + length; i++) {
- crc = crc32c(crc, buffer[i]);
- }
- }
- public long getValue() {
- return (crc ^ 0xFFFFFFFFL) & 0xFFFFFFFFL;
- }
- public void reset() {
- crc = ~0;
- }
- private static int crc32c(int crc, int b) {
- return crc >>> 8 ^ crcTable[(crc ^ b & 0xFF) & 0xFF];
- }
- public void update(int arg0) {
- throw new RuntimeException("Not implemented");
- }
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/proto/tensorflow/event.proto
----------------------------------------------------------------------
diff --git a/src/main/proto/tensorflow/event.proto b/src/main/proto/tensorflow/event.proto
deleted file mode 100644
index c1816ff..0000000
--- a/src/main/proto/tensorflow/event.proto
+++ /dev/null
@@ -1,82 +0,0 @@
-syntax = "proto3";
-
-package tensorflow;
-option cc_enable_arenas = true;
-option java_outer_classname = "EventProtos";
-option java_multiple_files = true;
-option java_package = "org.tensorflow.util";
-
-import "summary.proto";
-
-// Protocol buffer representing an event that happened during
-// the execution of a Brain model.
-message Event {
- // Timestamp of the event.
- double wall_time = 1;
-
- // Global step of the event.
- int64 step = 2;
-
- oneof what {
- // An event file was started, with the specified version.
- // This is use to identify the contents of the record IO files
- // easily. Current version is "brain.Event:2". All versions
- // start with "brain.Event:".
- string file_version = 3;
- // An encoded version of a GraphDef.
- bytes graph_def = 4;
- // A summary was generated.
- Summary summary = 5;
- // The user output a log message. Not all messages are logged, only ones
- // generated via the Python tensorboard_logging module.
- LogMessage log_message = 6;
- // The state of the session which can be used for restarting after crashes.
- SessionLog session_log = 7;
- // The metadata returned by running a session.run() call.
- TaggedRunMetadata tagged_run_metadata = 8;
- // An encoded version of a MetaGraphDef.
- bytes meta_graph_def = 9;
- }
-}
-
-// Protocol buffer used for logging messages to the events file.
-message LogMessage {
- enum Level {
- UNKNOWN = 0;
- // Note: The logging level 10 cannot be named DEBUG. Some software
- // projects compile their C/C++ code with -DDEBUG in debug builds. So the
- // C++ code generated from this file should not have an identifier named
- // DEBUG.
- DEBUGGING = 10;
- INFO = 20;
- WARN = 30;
- ERROR = 40;
- FATAL = 50;
- }
- Level level = 1;
- string message = 2;
-}
-
-// Protocol buffer used for logging session state.
-message SessionLog {
- enum SessionStatus {
- STATUS_UNSPECIFIED = 0;
- START = 1;
- STOP = 2;
- CHECKPOINT = 3;
- }
-
- SessionStatus status = 1;
- // This checkpoint_path contains both the path and filename.
- string checkpoint_path = 2;
- string msg = 3;
-}
-
-// For logging the metadata output for a single session.run() call.
-message TaggedRunMetadata {
- // Tag name associated with this metadata.
- string tag = 1;
- // Byte-encoded version of the `RunMetadata` proto in order to allow lazy
- // deserialization.
- bytes run_metadata = 2;
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/proto/tensorflow/summary.proto
----------------------------------------------------------------------
diff --git a/src/main/proto/tensorflow/summary.proto b/src/main/proto/tensorflow/summary.proto
deleted file mode 100644
index 29011fd..0000000
--- a/src/main/proto/tensorflow/summary.proto
+++ /dev/null
@@ -1,103 +0,0 @@
-syntax = "proto3";
-
-package tensorflow;
-option cc_enable_arenas = true;
-option java_outer_classname = "SummaryProtos";
-option java_multiple_files = true;
-option java_package = "org.tensorflow.framework";
-
-// import "tensorflow/core/framework/tensor.proto";
-
-// Metadata associated with a series of Summary data
-message SummaryDescription {
- // Hint on how plugins should process the data in this series.
- // Supported values include "scalar", "histogram", "image", "audio"
- string type_hint = 1;
-}
-
-// Serialization format for histogram module in
-// core/lib/histogram/histogram.h
-message HistogramProto {
- double min = 1;
- double max = 2;
- double num = 3;
- double sum = 4;
- double sum_squares = 5;
-
- // Parallel arrays encoding the bucket boundaries and the bucket values.
- // bucket(i) is the count for the bucket i. The range for
- // a bucket is:
- // i == 0: -DBL_MAX .. bucket_limit(0)
- // i != 0: bucket_limit(i-1) .. bucket_limit(i)
- repeated double bucket_limit = 6 [packed = true];
- repeated double bucket = 7 [packed = true];
-};
-
-// A Summary is a set of named values to be displayed by the
-// visualizer.
-//
-// Summaries are produced regularly during training, as controlled by
-// the "summary_interval_secs" attribute of the training operation.
-// Summaries are also produced at the end of an evaluation.
-message Summary {
- message Image {
- // Dimensions of the image.
- int32 height = 1;
- int32 width = 2;
- // Valid colorspace values are
- // 1 - grayscale
- // 2 - grayscale + alpha
- // 3 - RGB
- // 4 - RGBA
- // 5 - DIGITAL_YUV
- // 6 - BGRA
- int32 colorspace = 3;
- // Image data in encoded format. All image formats supported by
- // image_codec::CoderUtil can be stored here.
- bytes encoded_image_string = 4;
- }
-
- message Audio {
- // Sample rate of the audio in Hz.
- float sample_rate = 1;
- // Number of channels of audio.
- int64 num_channels = 2;
- // Length of the audio in frames (samples per channel).
- int64 length_frames = 3;
- // Encoded audio data and its associated RFC 2045 content type (e.g.
- // "audio/wav").
- bytes encoded_audio_string = 4;
- string content_type = 5;
- }
-
- message Value {
- // Name of the node that output this summary; in general, the name of a
- // TensorSummary node. If the node in question has multiple outputs, then
- // a ":\d+" suffix will be appended, like "some_op:13".
- // Might not be set for legacy summaries (i.e. those not using the tensor
- // value field)
- string node_name = 7;
-
- // Tag name for the data. Will only be used by legacy summaries
- // (ie. those not using the tensor value field)
- // For legacy summaries, will be used as the title of the graph
- // in the visualizer.
- //
- // Tag is usually "op_name:value_name", where "op_name" itself can have
- // structure to indicate grouping.
- string tag = 1;
-
- // Value associated with the tag.
- oneof value {
- float simple_value = 2;
- bytes obsolete_old_style_histogram = 3;
- Image image = 4;
- HistogramProto histo = 5;
- Audio audio = 6;
- // TensorProto tensor = 8;
- }
- }
-
- // Set of values for the summary.
- repeated Value value = 1;
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/python/LICENSE
----------------------------------------------------------------------
diff --git a/src/main/python/LICENSE b/src/main/python/LICENSE
index 02d679c..ed7417c 100644
--- a/src/main/python/LICENSE
+++ b/src/main/python/LICENSE
@@ -296,213 +296,3 @@ CONTRIBUTION AGREEMENT
By contributing to the BVLC/caffe repository through pull-request, comment,
or otherwise, the contributor releases their content to the
license and copyright terms herein.
-
-===============================================================================
-
-The proto files (src/main/proto/tensorflow/event.proto and src/main/proto/tensorflow/summary.proto) is part of TensorFlow project,
-which is used to generate org.tensorflow package.
-TensorFlow are distributed under the Apache License 2.0.
-
-Copyright 2017 The TensorFlow Authors. All rights reserved.
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright 2017, The TensorFlow Authors.
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/python/systemml/mllearn/estimators.py
----------------------------------------------------------------------
diff --git a/src/main/python/systemml/mllearn/estimators.py b/src/main/python/systemml/mllearn/estimators.py
index 1df1bb8..50412aa 100644
--- a/src/main/python/systemml/mllearn/estimators.py
+++ b/src/main/python/systemml/mllearn/estimators.py
@@ -45,7 +45,6 @@ def assemble(sparkSession, pdf, inputCols, outputCol):
class BaseSystemMLEstimator(Estimator):
features_col = 'features'
label_col = 'label'
- do_visualize = False
def set_features_col(self, colName):
"""
@@ -756,7 +755,7 @@ class Caffe2DML(BaseSystemMLClassifier):
>>> caffe2DML = Caffe2DML(spark, 'lenet_solver.proto').set(max_iter=500)
>>> caffe2DML.fit(X, y)
"""
- def __init__(self, sparkSession, solver, input_shape, transferUsingDF=False, tensorboard_log_dir=None):
+ def __init__(self, sparkSession, solver, input_shape, transferUsingDF=False):
"""
Performs training/prediction for a given caffe network.
@@ -766,7 +765,6 @@ class Caffe2DML(BaseSystemMLClassifier):
solver: caffe solver file path
input_shape: 3-element list (number of channels, input height, input width)
transferUsingDF: whether to pass the input dataset via PySpark DataFrame (default: False)
- tensorboard_log_dir: directory to store the event logs (default: None, we use a temporary directory)
"""
self.sparkSession = sparkSession
self.sc = sparkSession._sc
@@ -779,9 +777,6 @@ class Caffe2DML(BaseSystemMLClassifier):
self.estimator = self.sc._jvm.org.apache.sysml.api.dl.Caffe2DML(self.sc._jsc.sc(), solver, str(input_shape[0]), str(input_shape[1]), str(input_shape[2]))
self.transferUsingDF = transferUsingDF
self.setOutputRawPredictionsToFalse = False
- self.visualize_called = False
- if tensorboard_log_dir is not None:
- self.estimator.setTensorBoardLogDir(tensorboard_log_dir)
def load(self, weights=None, sep='/', ignore_weights=None, eager=False):
"""
@@ -831,32 +826,6 @@ class Caffe2DML(BaseSystemMLClassifier):
else:
raise TypeError('Please use spark session of type pyspark.sql.session.SparkSession in the constructor')
- def visualize(self, layerName=None, varType='weight', aggFn='mean'):
- """
- Use this to visualize the training procedure (requires validation_percentage to be non-zero).
- When one provides no argument to this method, we visualize training and validation loss.
-
- Parameters
- ----------
- layerName: Name of the layer in the Caffe prototype
- varType: should be either 'weight', 'bias', 'dweight', 'dbias', 'output' or 'doutput'
- aggFn: should be either 'sum', 'mean', 'var' or 'sd'
- """
- valid_vis_var_types = ['weight', 'bias', 'dweight', 'dbias', 'output', 'doutput']
- valid_vis_aggFn = [ 'sum', 'mean', 'var', 'sd' ]
- if layerName is not None and varType is not None and aggFn is not None:
- # Visualize other values
- if varType not in valid_vis_var_types:
- raise ValueError('The second argument should be either weight, bias, dweight, dbias, output or doutput')
- if aggFn not in valid_vis_aggFn:
- raise ValueError('The third argument should be either sum, mean, var, sd.')
- if self.visualize_called:
- self.estimator.visualizeLoss()
- self.estimator.visualizeLayer(layerName, varType, aggFn)
- else:
- self.estimator.visualizeLoss()
- self.visualize_called = True
- return self
class Keras2DML(Caffe2DML):
"""
http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala
----------------------------------------------------------------------
diff --git a/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala b/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala
index 91373bc..03b9a3b 100644
--- a/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala
+++ b/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala
@@ -303,9 +303,6 @@ class Caffe2DML(val sc: SparkContext,
// TODO: throw error or warning if user tries to set solver_mode == GPU instead of using setGPU method
- // Method called by Python mllearn to visualize variable of certain layer
- def visualizeLayer(layerName: String, varType: String, aggFn: String): Unit = visualizeLayer(net, layerName, varType, aggFn)
-
def getTrainAlgo(): String = if (inputs.containsKey("$train_algo")) inputs.get("$train_algo") else "minibatch"
def getTestAlgo(): String = if (inputs.containsKey("$test_algo")) inputs.get("$test_algo") else "minibatch"
@@ -374,9 +371,6 @@ class Caffe2DML(val sc: SparkContext,
}
// ----------------------------------------------------------------------------
- // Check if this is necessary
- if (doVisualize) tabDMLScript.append("print(" + asDMLString("Visualization counter:") + " + viz_counter)")
-
val trainingScript = tabDMLScript.toString()
// Print script generation time and the DML script on stdout
System.out.println("Time taken to generate training script from Caffe proto: " + ((System.nanoTime() - startTrainingTime) * 1e-9) + " seconds.")
@@ -522,7 +516,6 @@ class Caffe2DML(val sc: SparkContext,
tabDMLScript.append(
print(dmlConcat(asDMLString("Iter:"), "iter", asDMLString(", training loss:"), "training_loss", asDMLString(", training accuracy:"), "training_accuracy"))
)
- appendTrainingVisualizationBody(dmlScript, numTabs)
printClassificationReport
}
} else {
@@ -615,7 +608,6 @@ class Caffe2DML(val sc: SparkContext,
tabDMLScript.append(
print(dmlConcat(asDMLString("Iter:"), "iter", asDMLString(", validation loss:"), "validation_loss", asDMLString(", validation accuracy:"), "validation_accuracy"))
)
- appendValidationVisualizationBody(dmlScript, numTabs)
}
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/scala/org/apache/sysml/api/dl/DMLGenerator.scala
----------------------------------------------------------------------
diff --git a/src/main/scala/org/apache/sysml/api/dl/DMLGenerator.scala b/src/main/scala/org/apache/sysml/api/dl/DMLGenerator.scala
index b68d493..304f788 100644
--- a/src/main/scala/org/apache/sysml/api/dl/DMLGenerator.scala
+++ b/src/main/scala/org/apache/sysml/api/dl/DMLGenerator.scala
@@ -188,81 +188,8 @@ trait NextBatchGenerator extends TabbedDMLGenerator {
assignBatch(tabDMLScript, "Xb", Caffe2DML.XVal, "yb", Caffe2DML.yVal, "", Caffe2DML.numValidationImages, "iVal")
}
-trait VisualizeDMLGenerator extends TabbedDMLGenerator {
- var doVisualize = false
- var _tensorboardLogDir: String = null
- def setTensorBoardLogDir(log: String): Unit = _tensorboardLogDir = log
- def tensorboardLogDir: String = {
- if (_tensorboardLogDir == null) {
- _tensorboardLogDir = java.io.File.createTempFile("temp", System.nanoTime().toString()).getAbsolutePath
- }
- _tensorboardLogDir
- }
- def visualizeLoss(): Unit = {
- checkTensorBoardDependency()
- doVisualize = true
- // Visualize for both training and validation
- visualize(" ", " ", "training_loss", "iter", "training_loss", true)
- visualize(" ", " ", "training_accuracy", "iter", "training_accuracy", true)
- visualize(" ", " ", "validation_loss", "iter", "validation_loss", false)
- visualize(" ", " ", "validation_accuracy", "iter", "validation_accuracy", false)
- }
- val visTrainingDMLScript: StringBuilder = new StringBuilder
- val visValidationDMLScript: StringBuilder = new StringBuilder
- def checkTensorBoardDependency(): Unit =
- try {
- if (!doVisualize)
- Class.forName("com.google.protobuf.GeneratedMessageV3")
- } catch {
- case _: ClassNotFoundException =>
- throw new DMLRuntimeException(
- "To use visualize() feature, you will have to include protobuf-java-3.2.0.jar in your classpath. Hint: you can download the jar from http://central.maven.org/maven2/com/google/protobuf/protobuf-java/3.2.0/protobuf-java-3.2.0.jar"
- )
- }
- private def visualize(layerName: String, varType: String, aggFn: String, x: String, y: String, isTraining: Boolean) = {
- val dmlScript = if (isTraining) visTrainingDMLScript else visValidationDMLScript
- dmlScript.append(
- "viz_counter1 = visualize(" +
- commaSep(asDMLString(layerName), asDMLString(varType), asDMLString(aggFn), x, y, asDMLString(tensorboardLogDir))
- + ");\n"
- )
- dmlScript.append("viz_counter = viz_counter + viz_counter1\n")
- }
- def visualizeLayer(net: CaffeNetwork, layerName: String, varType: String, aggFn: String): Unit = {
- // 'weight', 'bias', 'dweight', 'dbias', 'output' or 'doutput'
- // 'sum', 'mean', 'var' or 'sd'
- checkTensorBoardDependency()
- doVisualize = true
- if (net.getLayers.filter(_.equals(layerName)).size == 0)
- throw new DMLRuntimeException("Cannot visualize the layer:" + layerName)
- val dmlVar = {
- val l = net.getCaffeLayer(layerName)
- varType match {
- case "weight" => l.weight
- case "bias" => l.bias
- case "dweight" => l.dWeight
- case "dbias" => l.dBias
- case "output" => l.out
- // case "doutput" => l.dX
- case _ => throw new DMLRuntimeException("Cannot visualize the variable of type:" + varType)
- }
- }
- if (dmlVar == null)
- throw new DMLRuntimeException("Cannot visualize the variable of type:" + varType)
- // Visualize for both training and validation
- visualize(layerName, varType, aggFn, "iter", aggFn + "(" + dmlVar + ")", true)
- visualize(layerName, varType, aggFn, "iter", aggFn + "(" + dmlVar + ")", false)
- }
-
- def appendTrainingVisualizationBody(dmlScript: StringBuilder, numTabs: Int): Unit =
- if (doVisualize)
- tabDMLScript(dmlScript, numTabs).append(visTrainingDMLScript.toString)
- def appendValidationVisualizationBody(dmlScript: StringBuilder, numTabs: Int): Unit =
- if (doVisualize)
- tabDMLScript(dmlScript, numTabs).append(visValidationDMLScript.toString)
-}
-trait DMLGenerator extends SourceDMLGenerator with NextBatchGenerator with VisualizeDMLGenerator {
+trait DMLGenerator extends SourceDMLGenerator with NextBatchGenerator {
// Also makes "code reading" possible for Caffe2DML :)
var dmlScript = new StringBuilder
var numTabs = 0
@@ -270,9 +197,6 @@ trait DMLGenerator extends SourceDMLGenerator with NextBatchGenerator with Visua
dmlScript.clear()
alreadyImported.clear()
numTabs = 0
- visTrainingDMLScript.clear()
- visValidationDMLScript.clear()
- doVisualize = false
}
// -------------------------------------------------------------------------------------------------
// Helper functions that calls super class methods and simplifies the code of this trait
@@ -341,16 +265,7 @@ trait DMLGenerator extends SourceDMLGenerator with NextBatchGenerator with Visua
if (isTraining) {
// Append external built-in function headers:
- // 1. visualize external built-in function header
- if (doVisualize) {
- tabDMLScript.append(
- "visualize = externalFunction(String layerName, String varType, String aggFn, Double x, Double y, String logDir) return (Double B) " +
- "implemented in (classname=\"org.apache.sysml.udf.lib.Caffe2DMLVisualizeWrapper\",exectype=\"mem\"); \n"
- )
- tabDMLScript.append("viz_counter = 0\n")
- System.out.println("Please use the following command for visualizing: tensorboard --logdir=" + tensorboardLogDir)
- }
- // 2. update_nesterov external built-in function header
+ // 1. update_nesterov external built-in function header
if (Caffe2DML.USE_NESTEROV_UDF) {
tabDMLScript.append(
"update_nesterov = externalFunction(matrix[double] X, matrix[double] dX, double lr, double mu, matrix[double] v, double lambda) return (matrix[double] X, matrix[double] v) implemented in (classname=\"org.apache.sysml.udf.lib.SGDNesterovUpdate\",exectype=\"mem\"); \n"