You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by cs...@apache.org on 2018/12/03 18:40:40 UTC
[incubator-openwhisk-runtime-go] branch master updated: removed
filetype library - added a version number (#63)
This is an automated email from the ASF dual-hosted git repository.
csantanapr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk-runtime-go.git
The following commit(s) were added to refs/heads/master by this push:
new 410d006 removed filetype library - added a version number (#63)
410d006 is described below
commit 410d00617f5e4b4f7931a97a4d21d8c19952cf25
Author: Sciabarra.com ltd <30...@users.noreply.github.com>
AuthorDate: Mon Dec 3 19:40:35 2018 +0100
removed filetype library - added a version number (#63)
---
build.gradle | 1 -
main/proxy.go | 12 ++++++-
openwhisk/compiler.go | 30 ++---------------
openwhisk/extractor.go | 8 +----
openwhisk/extractor_test.go | 29 ++++++++++++++--
openwhisk/filetype.go | 60 +++++++++++++++++++++++++++++++++
openwhisk/filetype_test.go | 64 ++++++++++++++++++++++++++++++++++++
openwhisk/util_test.go | 16 ++-------
build.gradle => openwhisk/version.go | 42 ++---------------------
9 files changed, 170 insertions(+), 92 deletions(-)
diff --git a/build.gradle b/build.gradle
index b8f5383..a59e2a1 100644
--- a/build.gradle
+++ b/build.gradle
@@ -30,7 +30,6 @@ plugins {
dependencies {
golang {
- build 'github.com/h2non/filetype@v1.0.5'
build 'github.com/sirupsen/logrus@v1.1.0'
test 'github.com/stretchr/testify@v1.2.1'
}
diff --git a/main/proxy.go b/main/proxy.go
index 2511ab6..1fb9cdf 100644
--- a/main/proxy.go
+++ b/main/proxy.go
@@ -20,6 +20,7 @@ import (
"archive/zip"
"bytes"
"flag"
+ "fmt"
"io/ioutil"
"log"
"os"
@@ -27,6 +28,9 @@ import (
"github.com/apache/incubator-openwhisk-runtime-go/openwhisk"
)
+// flag to show version
+var version = flag.Bool("version", false, "show version")
+
// flag to enable debug
var debug = flag.Bool("debug", false, "enable debug output")
@@ -69,6 +73,12 @@ func extractAndCompile(ap *openwhisk.ActionProxy) {
func main() {
flag.Parse()
+ // show version number
+ if *version {
+ fmt.Println("OpenWhisk ActionLoop Proxy", openwhisk.Version)
+ return
+ }
+
// debugging
if *debug {
// set debugging flag, propagated to the actions
@@ -86,7 +96,7 @@ func main() {
}
// start the balls rolling
- openwhisk.Debug("OpenWhisk Go Proxy: starting")
+ openwhisk.Debug("OpenWhisk ActionLoop Proxy %s: starting", openwhisk.Version)
ap.Start(8080)
}
diff --git a/openwhisk/compiler.go b/openwhisk/compiler.go
index 412a67b..f03c30d 100644
--- a/openwhisk/compiler.go
+++ b/openwhisk/compiler.go
@@ -23,19 +23,8 @@ import (
"os"
"os/exec"
"runtime"
-
- "gopkg.in/h2non/filetype.v1"
)
-// this is only to let test run on OSX
-// it only recognizes OSX Mach 64 bit executable
-// (magic number: facefeed + 64bit flag)
-var mach64Type = filetype.NewType("mach", "darwin/mach")
-
-func mach64Matcher(buf []byte) bool {
- return len(buf) > 4 && buf[0] == 0xcf && buf[1] == 0xfa && buf[2] == 0xed && buf[3] == 0xfe
-}
-
// check if the file exists and it is already compiled
func isCompiled(file string) bool {
_, err := os.Stat(file)
@@ -48,24 +37,9 @@ func isCompiled(file string) bool {
Debug(err.Error())
return false
}
- // if this is mac add a matcher for mac
- if runtime.GOOS == "darwin" {
- filetype.AddMatcher(mach64Type, mach64Matcher)
- }
- kind, err := filetype.Match(buf)
- Debug("isCompiled: %s kind=%s", file, kind)
- if err != nil {
- Debug(err.Error())
- return false
- }
- if kind.Extension == "elf" {
- return true
- }
- if kind.Extension == "mach" {
- return true
- }
- return false
+ // check if it is an executable
+ return IsExecutable(buf, runtime.GOOS)
}
// CompileAction will compile an anction in source format invoking a compiler
diff --git a/openwhisk/extractor.go b/openwhisk/extractor.go
index ca751f8..8140151 100644
--- a/openwhisk/extractor.go
+++ b/openwhisk/extractor.go
@@ -26,8 +26,6 @@ import (
"os"
"path/filepath"
"strconv"
-
- "gopkg.in/h2non/filetype.v1"
)
func unzip(src []byte, dest string) error {
@@ -110,12 +108,8 @@ func (ap *ActionProxy) ExtractAction(buf *[]byte, suffix string) (string, error)
ap.currentDir++
newDir := fmt.Sprintf("%s/%d/%s", ap.baseDir, ap.currentDir, suffix)
os.MkdirAll(newDir, 0755)
- kind, err := filetype.Match(*buf)
- if err != nil {
- return "", err
- }
file := newDir + "/exec"
- if kind.Extension == "zip" {
+ if IsZip(*buf) {
Debug("Extract Action, assuming a zip")
return file, unzip(*buf, newDir)
}
diff --git a/openwhisk/extractor_test.go b/openwhisk/extractor_test.go
index 8e90ed7..891d7d2 100644
--- a/openwhisk/extractor_test.go
+++ b/openwhisk/extractor_test.go
@@ -18,6 +18,7 @@
package openwhisk
import (
+ "fmt"
"io/ioutil"
"os"
"testing"
@@ -41,7 +42,7 @@ func TestExtractActionTest_exe(t *testing.T) {
// match exe
file, _ := ioutil.ReadFile("_test/exec")
ap.ExtractAction(&file, "bin")
- assert.Equal(t, detect("./action/x2", "bin/exec"), "elf")
+ assert.Equal(t, detectExecutable("./action/x2", "bin/exec"), true)
}
func TestExtractActionTest_zip(t *testing.T) {
@@ -52,7 +53,7 @@ func TestExtractActionTest_zip(t *testing.T) {
// match exe
file, _ := ioutil.ReadFile("_test/exec.zip")
ap.ExtractAction(&file, "bin")
- assert.Equal(t, detect("./action/x3", "bin/exec"), "elf")
+ assert.Equal(t, detectExecutable("./action/x3", "bin/exec"), true)
assert.Nil(t, exists("./action/x3", "bin/etc"))
assert.Nil(t, exists("./action/x3", "bin/dir/etc"))
}
@@ -71,3 +72,27 @@ func TestHighestDir(t *testing.T) {
assert.Equal(t, highestDir("./_test/first"), 3)
assert.Equal(t, highestDir("./_test/second"), 17)
}
+
+// Issue #62 sample zip
+func ExampleBadZip() {
+ buf := []byte{
+ 0x50, 0x4b, 0x03, 0x04, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0xb0, 0x81, 0x4d, 0x2d, 0xf6,
+ 0xa5, 0x66, 0x48, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x09, 0x00, 0x1c, 0x00, 0x69, 0x6e,
+ 0x64, 0x65, 0x78, 0x2e, 0x70, 0x68, 0x70, 0x55, 0x54, 0x09, 0x00, 0x03, 0x51, 0x05, 0x03, 0x5c,
+ 0x54, 0x05, 0x03, 0x5c, 0x75, 0x78, 0x0b, 0x00, 0x01, 0x04, 0xf5, 0x01, 0x00, 0x00, 0x04, 0x14,
+ 0x00, 0x00, 0x00, 0x3c, 0x3f, 0x70, 0x68, 0x70, 0x0a, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f,
+ 0x6e, 0x20, 0x6e, 0x69, 0x61, 0x6d, 0x28, 0x61, 0x72, 0x72, 0x61, 0x79, 0x20, 0x24, 0x61, 0x72,
+ 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e,
+ 0x20, 0x5b, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x20, 0x3d, 0x3e, 0x20, 0x22, 0x69, 0x74, 0x20,
+ 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x22, 0x5d, 0x3b, 0x0a, 0x7d, 0x0a, 0x50, 0x4b, 0x01, 0x02, 0x1e,
+ 0x03, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0xb0, 0x81, 0x4d, 0x2d, 0xf6, 0xa5, 0x66, 0x48,
+ 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x09, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+ 0x00, 0x00, 0x00, 0xb4, 0x81, 0x00, 0x00, 0x00, 0x00, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x70,
+ 0x68, 0x70, 0x55, 0x54, 0x05, 0x00, 0x03, 0x51, 0x05, 0x03, 0x5c, 0x75, 0x78, 0x0b, 0x00, 0x01,
+ 0x04, 0xf5, 0x01, 0x00, 0x00, 0x04, 0x14, 0x00, 0x00, 0x00, 0x50, 0x4b, 0x05, 0x06, 0x00, 0x00,
+ 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x8b, 0x00, 0x00, 0x00, 0x00, 0x00,
+ }
+ fmt.Printf("%t", IsZip(buf))
+ // Output:
+ // true
+}
diff --git a/openwhisk/filetype.go b/openwhisk/filetype.go
new file mode 100644
index 0000000..f4d6c3d
--- /dev/null
+++ b/openwhisk/filetype.go
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package openwhisk
+
+// IsElf checks for a Linux executable
+func IsElf(buf []byte) bool {
+ return len(buf) > 52 &&
+ buf[0] == 0x7F && buf[1] == 0x45 &&
+ buf[2] == 0x4C && buf[3] == 0x46
+}
+
+// IsExe checks for a Windows executable
+func IsExe(buf []byte) bool {
+ return len(buf) > 1 &&
+ buf[0] == 0x4D && buf[1] == 0x5A
+}
+
+// IsMach64 checks for OSX executable
+func IsMach64(buf []byte) bool {
+ return len(buf) > 4 &&
+ buf[0] == 0xcf && buf[1] == 0xfa &&
+ buf[2] == 0xed && buf[3] == 0xfe
+}
+
+// IsExecutable check if it is an executable, according the current runtime
+func IsExecutable(buf []byte, runtime string) bool {
+ switch runtime {
+ case "darwin":
+ return IsMach64(buf)
+ case "linux":
+ return IsElf(buf)
+ case "windows":
+ return IsExe(buf)
+ default:
+ return false
+ }
+}
+
+// IsZip checks if it is a zip file
+func IsZip(buf []byte) bool {
+ return len(buf) > 3 &&
+ buf[0] == 0x50 && buf[1] == 0x4B &&
+ (buf[2] == 0x3 || buf[2] == 0x5 || buf[2] == 0x7) &&
+ (buf[3] == 0x4 || buf[3] == 0x6 || buf[3] == 0x8)
+}
diff --git a/openwhisk/filetype_test.go b/openwhisk/filetype_test.go
new file mode 100644
index 0000000..5f016c6
--- /dev/null
+++ b/openwhisk/filetype_test.go
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package openwhisk
+
+import "fmt"
+
+var zipFile = []byte{
+ 0x50, 0x4b, 0x03, 0x04, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x60, 0x83, 0x4d, 0x00, 0x00,
+}
+
+var linuxFile = []byte{
+ 0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+}
+
+var darwinFile = []byte{
+ 0xcf, 0xfa, 0xed, 0xfe, 0x07, 0x00, 0x00, 0x01, 0x03, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+}
+
+var windowsFile = []byte{
+ 0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00,
+}
+
+func Example_filetype() {
+ fmt.Printf("%t\n%t\n", IsElf(linuxFile), IsElf(zipFile))
+ fmt.Printf("%t\n%t\n", IsMach64(darwinFile), IsMach64(zipFile))
+ fmt.Printf("%t\n%t\n", IsExe(windowsFile), IsExe(zipFile))
+ fmt.Printf("%t\n%t\n", IsZip(zipFile), IsExe(linuxFile))
+ fmt.Printf("%t\n%t\n", IsExecutable(linuxFile, "linux"), IsExecutable(zipFile, "linux"))
+ fmt.Printf("%t\n%t\n", IsExecutable(windowsFile, "windows"), IsExecutable(zipFile, "windows"))
+ fmt.Printf("%t\n%t\n", IsExecutable(darwinFile, "darwin"), IsExecutable(zipFile, "darwin"))
+ // Output:
+ // true
+ // false
+ // true
+ // false
+ // true
+ // false
+ // true
+ // false
+ // true
+ // false
+ // true
+ // false
+ // true
+ // false
+
+}
diff --git a/openwhisk/util_test.go b/openwhisk/util_test.go
index 1593760..59fd5f3 100644
--- a/openwhisk/util_test.go
+++ b/openwhisk/util_test.go
@@ -34,8 +34,6 @@ import (
"strings"
"testing"
"time"
-
- "gopkg.in/h2non/filetype.v1"
)
func startTestServer(compiler string) (*httptest.Server, string, *os.File) {
@@ -153,20 +151,10 @@ func exists(dir, filename string) error {
return err
}
-// definig a type to recognizing also mac executables
-var pseudoElfForMacType = filetype.NewType("elf", "darwin/mach")
-
-func pseudoElfForMacMatcher(buf []byte) bool {
- return len(buf) > 4 && ((buf[0] == 0xcf && buf[1] == 0xfa && buf[2] == 0xed && buf[3] == 0xfe) ||
- (buf[0] == 0xce && buf[1] == 0xfa && buf[2] == 0xed && buf[3] == 0xfe))
-}
-
-func detect(dir, filename string) string {
+func detectExecutable(dir, filename string) bool {
path := fmt.Sprintf("%s/%d/%s", dir, highestDir(dir), filename)
file, _ := ioutil.ReadFile(path)
- filetype.AddMatcher(pseudoElfForMacType, pseudoElfForMacMatcher)
- kind, _ := filetype.Match(file)
- return kind.Extension
+ return IsExecutable(file, runtime.GOOS)
}
func waitabit() {
diff --git a/build.gradle b/openwhisk/version.go
similarity index 52%
copy from build.gradle
copy to openwhisk/version.go
index b8f5383..27af90b 100644
--- a/build.gradle
+++ b/openwhisk/version.go
@@ -14,43 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package openwhisk
-buildscript {
- repositories {
- jcenter()
- }
- dependencies {
- classpath "cz.alenkacz:gradle-scalafmt:${gradle.scalafmt.version}"
- }
-}
-
-plugins {
- id 'com.github.blindpirate.gogradle' version '0.8.1'
-}
-
-dependencies {
- golang {
- build 'github.com/h2non/filetype@v1.0.5'
- build 'github.com/sirupsen/logrus@v1.1.0'
- test 'github.com/stretchr/testify@v1.2.1'
- }
-}
-
-
-subprojects {
- apply plugin: 'scalafmt'
- scalafmt.configFilePath = gradle.scalafmt.config
-}
-
-golang {
- packagePath = 'github.com/apache/incubator-openwhisk-runtime-go'
- goVersion = '1.11.2'
-}
-
-
-build.dependsOn vendor
-
-build {
- targetPlatform = ['linux-amd64']
- go 'build -o actionProxyLoop/proxy main/proxy.go'
-}
+// Version number - internal
+var Version = "1.0.1"