You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/03/31 01:24:12 UTC
[incubator-opendal] branch main updated: refactor(bindings/nodejs): Polish benchmark to make it more readable (#1810)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 85c3785d refactor(bindings/nodejs): Polish benchmark to make it more readable (#1810)
85c3785d is described below
commit 85c3785d5ff2543278325788916da43fe2dc91e9
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Mar 31 09:24:05 2023 +0800
refactor(bindings/nodejs): Polish benchmark to make it more readable (#1810)
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
bindings/nodejs/benchmark/README.md | 17 +++++++
bindings/nodejs/benchmark/index.js | 95 +++++++++++++++++++++++++++++++++++--
bindings/nodejs/benchmark/lib.js | 79 ------------------------------
bindings/nodejs/benchmark/read.js | 45 ------------------
bindings/nodejs/benchmark/write.js | 46 ------------------
bindings/nodejs/package.json | 1 +
6 files changed, 109 insertions(+), 174 deletions(-)
diff --git a/bindings/nodejs/benchmark/README.md b/bindings/nodejs/benchmark/README.md
new file mode 100644
index 00000000..9be0a942
--- /dev/null
+++ b/bindings/nodejs/benchmark/README.md
@@ -0,0 +1,17 @@
+# OpenDAL Node.js Bindings Benchmark
+
+This benchmark is test against the opendal and aws js sdk.
+
+To run the benchmark, please make sure the following env have been set correctly.
+
+- AWS_S3_ENDPOINT: the endpoint of the s3 service
+- AWS_S3_REGION: the region of the s3 service
+- AWS_ACCESS_KEY_ID: the access key of the s3 service
+- AWS_SECRET_ACCESS_KEY: the secret key of the s3 service
+- AWS_BUCKET: the bucket name of the s3 service
+
+To run the benchmark:
+
+```shell
+yarn run bench
+```
diff --git a/bindings/nodejs/benchmark/index.js b/bindings/nodejs/benchmark/index.js
index 26bc0cbb..ff90cbd9 100644
--- a/bindings/nodejs/benchmark/index.js
+++ b/bindings/nodejs/benchmark/index.js
@@ -17,11 +17,98 @@
* under the License.
*/
-const read = require('./read.js')
-const write = require('./write.js')
+const { Operator } = require('../index.js')
+const { S3Client, PutObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3')
+const { suite, add, cycle, complete } = require('benny')
+const crypto = require('node:crypto')
+
+const endpoint = process.env.AWS_S3_ENDPOINT
+const region = process.env.AWS_S3_REGION
+const accessKeyId = process.env.AWS_ACCESS_KEY_ID
+const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY
+const bucket = process.env.AWS_BUCKET
+
+const opendalClient = new Operator('s3', {
+ root: '/',
+ bucket,
+ endpoint,
+})
+
+const s3Client = new S3Client({
+ endpoint,
+ region,
+ credentials: {
+ accessKeyId,
+ secretAccessKey,
+ },
+})
+
+const testCases = [
+ { name: '4kb', content: Buffer.alloc(4 * 1024, 'opendal', 'utf8') },
+ { name: '256kb', content: Buffer.alloc(256 * 1024, 'opendal', 'utf8') },
+ { name: '4mb', content: Buffer.alloc(4 * 1024 * 1024, 'opendal', 'utf8') },
+ { name: '16mb', content: Buffer.alloc(16 * 1024 * 1024, 'opendal', 'utf8') },
+]
+
+async function benchRead() {
+ const uuid = crypto.randomUUID()
+ await testCases
+ .map((v) => async () => {
+ const filename = `${uuid}_${v.name}_read_bench.txt`
+ await opendalClient.write(filename, v.content)
+
+ return suite(
+ `read (${v.name})`,
+ add(`opendal read (${v.name})`, async () => {
+ await opendalClient.read(filename).then((v) => v.toString('utf-8'))
+ }),
+ add(`s3 read (${v.name})`, async () => {
+ const command = new GetObjectCommand({
+ Key: filename,
+ Bucket: bucket,
+ })
+ await s3Client.send(command).then((v) => v.Body.transformToString('utf-8'))
+ }),
+ cycle(),
+ complete(),
+ )
+ })
+ .reduce((p, v) => p.then(() => v()), Promise.resolve())
+}
+
+async function benchWrite() {
+ const uuid = crypto.randomUUID()
+ await testCases
+ .map(
+ (v) => () =>
+ suite(
+ `write (${v.name})`,
+ add(`opendal write (${v.name})`, async () => {
+ let count = 0
+ return async () => opendalClient.write(`${uuid}_${count++}_${v.name}_opendal.txt`, v.content)
+ }),
+ add(`s3 write (${v.name})`, async () => {
+ let count = 0
+
+ return async () => {
+ const command = new PutObjectCommand({
+ Bucket: bucket,
+ Key: `${uuid}_${count++}_${v.name}_s3.txt`,
+ Body: v.content,
+ })
+ await s3Client.send(command)
+ }
+ }),
+ cycle(),
+ complete(),
+ ),
+ )
+ .reduce((p, v) => p.then(() => v()), Promise.resolve())
+}
async function bench() {
- await write()
- await read()
+ await benchRead()
+ await benchWrite()
}
+
bench()
diff --git a/bindings/nodejs/benchmark/lib.js b/bindings/nodejs/benchmark/lib.js
deleted file mode 100644
index 5df3c75d..00000000
--- a/bindings/nodejs/benchmark/lib.js
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-const { Operator } = require('../index.js')
-const { S3Client, PutObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3')
-
-const endpoint = process.env.AWS_S3_ENDPOINT
-const region = process.env.AWS_S3_REGION
-const accessKeyId = process.env.AWS_ACCESS_KEY_ID
-const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY
-const bucket = 'benchmark'
-
-const file_4kb = Buffer.alloc(4 * 1024, 'opendal', 'utf8')
-const file_256kb = Buffer.alloc(256 * 1024, 'opendal', 'utf8')
-const file_4mb = Buffer.alloc(4 * 1024 * 1024, 'opendal', 'utf8')
-const file_16mb = Buffer.alloc(16 * 1024 * 1024, 'opendal', 'utf8')
-
-const testFiles = [
- { name: '4kb', file: file_4kb },
- { name: '256kb', file: file_256kb },
- { name: '4mb', file: file_4mb },
- { name: '16mb', file: file_16mb },
-]
-
-const opendal = new Operator('s3', {
- root: '/',
- bucket,
- endpoint,
-})
-
-const client = new S3Client({
- endpoint,
- region,
- credentials: {
- accessKeyId,
- secretAccessKey,
- },
-})
-
-module.exports.opendal = {
- read: (path) => opendal.read(path),
- write: (path, data) => opendal.write(path, data),
-}
-
-module.exports.s3 = {
- read: (path) => {
- const command = new GetObjectCommand({
- Key: path,
- Bucket: bucket,
- })
- return client.send(command)
- },
- write: (path, data) => {
- const command = new PutObjectCommand({
- Body: data,
- Key: path,
- Bucket: bucket,
- })
- return client.send(command)
- },
-}
-
-module.exports.testFiles = testFiles
diff --git a/bindings/nodejs/benchmark/read.js b/bindings/nodejs/benchmark/read.js
deleted file mode 100644
index 50f0b50f..00000000
--- a/bindings/nodejs/benchmark/read.js
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-const { suite, add, cycle, complete } = require('benny')
-const { s3, opendal, testFiles } = require('./lib.js')
-const crypto = require('node:crypto')
-
-async function bench() {
- const uuid = crypto.randomUUID()
- await testFiles
- .map((v) => async () => {
- const filename = `${uuid}_${v.name}_read_bench.txt`
- await opendal.write(filename, v.file)
- return suite(
- `read (${v.name})`,
- add(`s3 read (${v.name})`, async () => {
- await s3.read(filename).then((v) => v.Body.transformToString('utf-8'))
- }),
- add(`opendal read (${v.name})`, async () => {
- await opendal.read(filename).then((v) => v.toString('utf-8'))
- }),
- cycle(),
- complete(),
- )
- })
- .reduce((p, v) => p.then(() => v()), Promise.resolve())
-}
-
-module.exports = bench
diff --git a/bindings/nodejs/benchmark/write.js b/bindings/nodejs/benchmark/write.js
deleted file mode 100644
index f186deb4..00000000
--- a/bindings/nodejs/benchmark/write.js
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-const { suite, add, cycle, complete } = require('benny')
-const { s3, opendal, testFiles } = require('./lib.js')
-const crypto = require('node:crypto')
-
-async function bench() {
- const uuid = crypto.randomUUID()
- await testFiles
- .map(
- (v) => () =>
- suite(
- `write (${v.name})`,
- add(`opendal write (${v.name})`, async () => {
- let count = 0
- return async () => opendal.write(`${uuid}_${count++}_${v.name}_opendal.txt`, v.file)
- }),
- add(`s3 write (${v.name})`, async () => {
- let count = 0
- return async () => s3.write(`${uuid}_${count++}_${v.name}_s3.txt`, v.file)
- }),
- cycle(),
- complete(),
- ),
- )
- .reduce((p, v) => p.then(() => v()), Promise.resolve())
-}
-
-module.exports = bench
diff --git a/bindings/nodejs/package.json b/bindings/nodejs/package.json
index 92009da7..7d66eb2c 100644
--- a/bindings/nodejs/package.json
+++ b/bindings/nodejs/package.json
@@ -66,6 +66,7 @@
"format": "prettier --write .",
"prepublishOnly": "napi prepublish -t npm",
"test": "cucumber-js",
+ "bench": "node ./benchmark/index.js",
"version": "napi version"
},
"prettier": {