You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by ch...@apache.org on 2023/10/25 10:05:46 UTC
[singa] 01/01: Merge pull request #1115 from apache/dev-postgresql
This is an automated email from the ASF dual-hosted git repository.
chrishkchris pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/singa.git
commit e62ebfc2242e8e9b621b874abaa7746bc7987d16
Merge: 61a810f2 17ac0ada
Author: Chris Yeung <38...@users.noreply.github.com>
AuthorDate: Wed Oct 25 18:05:38 2023 +0800
Merge pull request #1115 from apache/dev-postgresql
Merge dev_postgresql into the master branch
CMakeLists.txt | 6 +-
LICENSE | 56 ++
NOTICE | 13 +
RELEASE_NOTES | 37 ++
doap/doap_SINGA.rdf | 63 ++
examples/armnet/README.md | 24 +
examples/cifar_distributed_cnn/run-rtx.sh | 4 +
examples/cnn/run.sh | 2 +
examples/malaria_cnn/README.md | 44 ++
examples/malaria_cnn/data/malaria.py | 122 ++++
examples/malaria_cnn/model/cnn.py | 94 +++
examples/malaria_cnn/model/mlp.py | 85 +++
examples/malaria_cnn/run.sh | 20 +
examples/mlp_postgresql/README.md | 23 +
.../Dockerfile | 76 +++
.../README.md | 167 +++++
.../documents/dev_guide.md | 251 +++++++
.../documents/image-20231020174425377.png | Bin 0 -> 302750 bytes
.../documents/image-20231020174945226.png | Bin 0 -> 26176 bytes
.../internal/cache-service/cache_service.py | 186 ++++++
.../internal/cache-service/trigger_cache_svc.py | 36 +
.../internal/ml/model_selection/README.md | 290 +++++++++
.../internal/ml/model_selection/config.ini | 104 +++
.../documents/imgs/image-20230421214835152.png | Bin 0 -> 334613 bytes
.../documents/imgs/image-20230421220338391.png | Bin 0 -> 198916 bytes
.../documents/imgs/image-20230421220443231.png | Bin 0 -> 163980 bytes
.../documents/imgs/image-20230702035554579.png | Bin 0 -> 284014 bytes
.../documents/imgs/image-20230702035622198.png | Bin 0 -> 108797 bytes
.../documents/imgs/image-20230702035639502.png | Bin 0 -> 79366 bytes
.../documents/imgs/image-20230702035806963.png | Bin 0 -> 151276 bytes
.../documents/imgs/image-20230722202555763.png | Bin 0 -> 149863 bytes
.../documents/imgs/image-20230722205244718.png | Bin 0 -> 230631 bytes
.../documents/imgs/image-20230724111325368.png | Bin 0 -> 202344 bytes
.../documents/imgs/image-20230724111659545.png | Bin 0 -> 164741 bytes
.../internal/ml/model_selection/eva_service.py | 96 +++
.../internal/ml/model_selection/exps/README.md | 40 ++
.../ml/model_selection/exps/draw_img_lib.py | 724 +++++++++++++++++++++
.../ml/model_selection/exps/draw_tab_lib.py | 215 ++++++
.../exps/nas_bench_tabular/0.train_one_model.py | 96 +++
.../nas_bench_tabular/2.seq_train_dist_online.py | 163 +++++
.../exps/nas_bench_tabular/2.seq_train_online.py | 118 ++++
.../exps/nas_bench_tabular/4.seq_score_online.py | 134 ++++
.../exps/nas_bench_tabular/measure_ecdf.py | 136 ++++
.../exps/nas_bench_tabular/measure_param_auc.py | 144 ++++
.../internal/ml/model_selection/init_env | 12 +
.../internal/ml/model_selection/main.py | 78 +++
.../internal/ml/model_selection/pg_interface.py | 635 ++++++++++++++++++
.../internal/ml/model_selection/requirement.txt | 54 ++
.../scripts/anytime_img_w_baseline.sh | 58 ++
.../ml/model_selection/scripts/anytime_tab.sh | 142 ++++
.../model_selection/scripts/baseline_system_img.sh | 61 ++
.../model_selection/scripts/baseline_system_tab.sh | 83 +++
.../scripts/benchmark_weight_sharing.sh | 41 ++
.../scripts/database/load_data_to_db.sh | 85 +++
.../scripts/latency_embedding_cache.sh | 138 ++++
.../scripts/latency_embedding_cache_concurrent.sh | 155 +++++
.../scripts/latency_phase1_cpu_gpu.sh | 227 +++++++
.../scripts/latency_phase1_in_db.sh | 77 +++
.../ml/model_selection/scripts/latency_phase2.sh | 77 +++
.../scripts/micro_budget_aware_alg.sh | 60 ++
.../model_selection/scripts/micro_nku_tradeoff.sh | 179 +++++
.../scripts/micro_score_metrics_relation.sh | 54 ++
.../scripts/micro_search_strategy.sh | 70 ++
.../scripts/nas-bench-img/convert_api_2_json.sh | 29 +
.../scripts/nas-bench-img/explore_all_models.sh | 77 +++
.../scripts/nas-bench-img/score_all_models.sh | 75 +++
.../nas-bench-tabular/score_all_modesl_criteo.sh | 43 ++
.../nas-bench-tabular/score_all_modesl_frappe.sh | 44 ++
.../nas-bench-tabular/score_all_modesl_uci.sh | 44 ++
.../nas-bench-tabular/train_all_models_criteo.sh | 63 ++
.../train_all_models_criteo_distirbuted.sh | 64 ++
.../nas-bench-tabular/train_all_models_diabetes.sh | 63 ++
.../nas-bench-tabular/train_all_models_frappe.sh | 61 ++
.../nas-bench-tabular/train_one_model_dev.sh | 41 ++
.../nas-bench-tabular/train_params_tune_criteo.sh | 162 +++++
.../train_params_tune_diabetes.sh | 86 +++
.../scripts/pre_processing/pre_processing_data.sh | 63 ++
.../internal/ml/model_selection/shared_config.py | 112 ++++
.../internal/ml/model_selection/src/__init__.py | 17 +
.../ml/model_selection/src/common/constant.py | 80 +++
.../ml/model_selection/src/common/structure.py | 107 +++
.../ml/model_selection/src/controller/__init__.py | 33 +
.../ml/model_selection/src/controller/controler.py | 191 ++++++
.../src/controller/core/__init__.py | 17 +
.../model_selection/src/controller/core/metrics.py | 46 ++
.../model_selection/src/controller/core/sample.py | 46 ++
.../src/controller/sampler_EA/__init__.py | 17 +
.../src/controller/sampler_RL/__init__.py | 18 +
.../src/controller/sampler_all/__init__.py | 17 +
.../src/controller/sampler_all/seq_sampler.py | 50 ++
.../src/controller/sampler_ea/regularized_ea.py | 148 +++++
.../src/controller/sampler_rand/__init__.py | 17 +
.../src/controller/sampler_rand/random_sample.py | 40 ++
.../sampler_rl/reinforcement_learning.py | 65 ++
.../model_selection/src/dataset_utils/__init__.py | 17 +
.../dataset_utils/download_critero_and_avazu.py | 59 ++
.../src/dataset_utils/sequence_dataloader.py | 96 +++
.../src/dataset_utils/stream_dataloader.py | 96 +++
.../ml/model_selection/src/eva_engine/__init__.py | 28 +
.../model_selection/src/eva_engine/coordinator.py | 116 ++++
.../src/eva_engine/phase1/__init__.py | 17 +
.../src/eva_engine/phase1/algo/__init__.py | 18 +
.../src/eva_engine/phase1/algo/prune_synflow.py | 425 ++++++++++++
.../src/eva_engine/phase1/algo/singa_ms/README.md | 18 +
.../phase1/algo/singa_ms/cnn_ms/README.md | 46 ++
.../cnn_ms/autograd/cifar10_multiprocess.py} | 37 +-
.../algo/singa_ms/cnn_ms/autograd/mnist_cnn.py | 304 +++++++++
.../algo/singa_ms/cnn_ms/autograd/mnist_dist.py} | 19 +-
.../cnn_ms/autograd/mnist_multiprocess.py} | 33 +-
.../singa_ms/cnn_ms/autograd/resnet_cifar10.py | 292 +++++++++
.../algo/singa_ms/cnn_ms/autograd/resnet_dist.py | 87 +++
.../cnn_ms/autograd/sparsification_mnist.py | 45 ++
.../algo/singa_ms/cnn_ms/autograd/xceptionnet.py | 303 +++++++++
.../phase1/algo/singa_ms/cnn_ms/benchmark.py | 121 ++++
.../phase1/algo/singa_ms/cnn_ms/data/cifar10.py | 89 +++
.../phase1/algo/singa_ms/cnn_ms/data/cifar100.py | 81 +++
.../algo/singa_ms/cnn_ms/data/download_cifar10.py | 49 ++
.../algo/singa_ms/cnn_ms/data/download_cifar100.py | 26 +
.../algo/singa_ms/cnn_ms/data/download_mnist.py | 49 ++
.../phase1/algo/singa_ms/cnn_ms/data/mnist.py | 91 +++
.../phase1/algo/singa_ms/cnn_ms/model/alexnet.py | 119 ++++
.../phase1/algo/singa_ms/cnn_ms/model/cnn.py | 90 +++
.../phase1/algo/singa_ms/cnn_ms/model/resnet.py | 300 +++++++++
.../algo/singa_ms/cnn_ms/model/xceptionnet.py | 311 +++++++++
.../algo/singa_ms/cnn_ms/pkg_model_code/model.py | 357 ++++++++++
.../eva_engine/phase1/algo/singa_ms/cnn_ms}/run.sh | 2 +
.../phase1/algo/singa_ms/cnn_ms/train_cnn.py | 564 ++++++++++++++++
.../phase1/algo/singa_ms/cnn_ms/train_mpi.py | 91 +++
.../phase1/algo/singa_ms/cnn_ms/train_ms_model.py | 592 +++++++++++++++++
.../algo/singa_ms/cnn_ms/train_multiprocess.py | 111 ++++
.../phase1/algo/singa_ms/ms_model_mlp/model.py | 226 +++++++
.../phase1/algo/singa_ms/ms_model_mlp/native.py | 137 ++++
.../eva_engine/phase1/algo/singa_ms/msmlp/model.py | 217 ++++++
.../phase1/algo/singa_ms/msmlp/native.py | 137 ++++
.../model_selection/src/eva_engine/phase1/vote.py | 133 ++++
.../src/eva_engine/phase2/__init__.py | 17 +
.../src/eva_engine/phase2/algo/__init__.py | 17 +
.../src/eva_engine/phase2/algo/trainer.py | 553 ++++++++++++++++
.../src/eva_engine/phase2/run_sr.py | 144 ++++
.../src/eva_engine/phase2/run_uniform.py | 94 +++
.../ml/model_selection/src/logger/__init__.py | 41 ++
.../ml/model_selection/src/query_api/README.md | 20 +
.../ml/model_selection/src/query_api/__init__.py | 17 +
.../src/query_api/img_explore_ea.py | 96 +++
.../src/query_api/img_train_baseline.py | 129 ++++
.../ml/model_selection/src/query_api/interface.py | 141 ++++
.../model_selection/src/query_api/query_api_img.py | 295 +++++++++
.../model_selection/src/query_api/query_api_mlp.py | 163 +++++
.../ml/model_selection/src/query_api/singleton.py | 31 +
.../model_selection/src/search_space/__init__.py | 17 +
.../src/search_space/core/__init__.py | 17 +
.../src/search_space/core/model_params.py | 41 ++
.../src/search_space/init_search_space.py | 55 ++
.../src/search_space/mlp_api/__init__.py | 17 +
.../src/search_space/mlp_api/model_params.py | 34 +
.../src/search_space/mlp_api/rl_policy.py | 36 +
.../src/search_space/mlp_api/space.py | 643 ++++++++++++++++++
.../src/search_space/utils/__init__.py | 18 +
.../src/search_space/utils/weight_initializers.py | 78 +++
.../ml/model_selection/src/third_pkg/__init__.py | 18 +
.../src/third_pkg/darts_lib/__init__.py | 17 +
.../src/third_pkg/darts_lib/genotypes.py | 36 +
.../src/third_pkg/darts_lib/model.py | 308 +++++++++
.../src/third_pkg/darts_lib/util_convert.py | 126 ++++
.../src/third_pkg/sp101_lib/__init__.py | 17 +
.../src/third_pkg/sp101_lib/graph_util.py | 168 +++++
.../src/third_pkg/sp101_lib/model_spec.py | 343 ++++++++++
.../src/third_pkg/sp101_lib/nb101_api.py | 481 ++++++++++++++
.../src/third_pkg/sp201_lib/nasbench2.py | 117 ++++
.../src/third_pkg/sp201_lib/nasbench2_ops.py | 160 +++++
.../ml/model_selection/src/tools/__init__.py | 17 +
.../ml/model_selection/src/tools/compute.py | 137 ++++
.../ml/model_selection/src/tools/correlation.py | 133 ++++
.../ml/model_selection/src/tools/io_tools.py | 60 ++
.../ml/model_selection/src/tools/res_measure.py | 102 +++
.../internal/ml/model_selection/src/tools/utils.py | 472 ++++++++++++++
.../internal/pg_extension/.cargo/config.toml | 3 +
.../internal/pg_extension/Cargo.toml | 39 ++
.../internal/pg_extension/pg_extension.control | 5 +
.../internal/pg_extension/sql/filter_phase.sql | 54 ++
.../pg_extension/sql/model_selection_cpu.sql | 50 ++
.../sql/model_selection_cpu_workloads.sql | 51 ++
.../pg_extension/sql/model_selection_dev.sql | 99 +++
.../pg_extension/sql/model_selection_trails.sql | 50 ++
.../sql/model_selection_trails_workloads.sql | 51 ++
.../pg_extension/sql/pg_extension--0.1.0.sql | 160 +++++
.../pg_extension/src/bindings/ml_register.rs | 76 +++
.../internal/pg_extension/src/bindings/mod.rs | 26 +
.../internal/pg_extension/src/bindings/model.rs | 39 ++
.../internal/pg_extension/src/bindings/ms.rs | 228 +++++++
.../internal/pg_extension/src/lib.rs | 192 ++++++
.../internal/pg_extension/test/lib.rs | 21 +
.../requirement.txt | 32 +
examples/model_selection_psql/README.md | 22 +
.../{cnn => model_selection_psql/ms_mlp}/run.sh | 20 +-
examples/model_selection_psql/ms_mlp/train_cnn.py | 329 ++++++++++
examples/model_selection_psql/ms_mlp/train_mlp.py | 588 +++++++++++++++++
examples/model_selection_psql/ms_mlp/train_mpi.py | 91 +++
.../model_selection_psql/ms_mlp/train_ms_model.py | 584 +++++++++++++++++
.../ms_mlp/train_multiprocess.py | 111 ++++
.../model_selection_psql/ms_model_mlp/model.py | 224 +++++++
.../model_selection_psql/ms_model_mlp/native.py | 137 ++++
examples/model_selection_psql/msmlp/model.py | 209 ++++++
examples/model_selection_psql/msmlp/native.py | 137 ++++
.../model_selection_psql/pkg_model_code/model.py | 366 +++++++++++
setup.py | 2 +-
tool/conda/singa/meta.yaml | 2 +-
.../devel/centos6/cuda10/Dockerfile.manylinux2014 | 4 +-
208 files changed, 23101 insertions(+), 67 deletions(-)