You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by zh...@apache.org on 2023/11/04 05:47:23 UTC
(singa) 01/01: Add implementations for singa model selection
This is an automated email from the ASF dual-hosted git repository.
zhaojing pushed a commit to branch singa-ms
in repository https://gitbox.apache.org/repos/asf/singa.git
commit f9c6023b7840f874b7cfb68e5a907c14b03bd1f8
Author: zhaojing <zh...@comp.nus.edu.sg>
AuthorDate: Sat Nov 4 12:20:37 2023 +0800
Add implementations for singa model selection
---
.../ml/model_selection/exp_result_singa/.gitkeep | 0
.../sampled_models_10000_models.json | 1 +
.../exps/nas_bench_tabular/sampled_models_all.json | 1 +
.../ml/model_selection/exps/shared_args.py | 204 ++++++
.../scripts/nas-bench-tabular/convert_data_format | 11 +
.../score_mlp_sp_frappe_batch_size_32_cpu.json | 1 +
.../ml/model_selection/src/common/__init__.py | 9 +
.../src/controller/sampler_bohb/bohb_or.py | 285 ++++++++
.../model_selection/src/dataset_utils/dataset.py | 181 +++++
.../src/dataset_utils/h5py_dataset.py | 40 ++
.../src/dataset_utils/imagenet16.py | 137 ++++
.../src/dataset_utils/save_load_torch.py | 87 +++
.../src/dataset_utils/structure_data_loader.py | 258 +++++++
.../src/eva_engine/phase1/algo/alg_base.py | 131 ++++
.../src/eva_engine/phase1/concurrent_evaluator.py | 199 ++++++
.../src/eva_engine/phase1/evaluator.py | 393 +++++++++++
.../src/eva_engine/phase1/run_phase1.py | 145 ++++
.../src/eva_engine/phase2/evaluator.py | 82 +++
.../src/eva_engine/phase2/run_sh.py | 156 +++++
.../ml/model_selection/src/eva_engine/run_ms.py | 309 +++++++++
.../src/search_space/core/rl_policy.py | 5 +
.../model_selection/src/search_space/core/space.py | 174 +++++
.../src/third_pkg/darts_lib/operations.py | 114 +++
.../src/third_pkg/darts_lib/utils.py | 342 +++++++++
.../src/third_pkg/models/CifarDenseNet.py | 117 ++++
.../src/third_pkg/models/CifarResNet.py | 180 +++++
.../src/third_pkg/models/CifarWideResNet.py | 115 ++++
.../src/third_pkg/models/ImageNet_MobileNetV2.py | 117 ++++
.../src/third_pkg/models/ImageNet_ResNet.py | 217 ++++++
.../src/third_pkg/models/SharedUtils.py | 37 +
.../src/third_pkg/models/__init__.py | 335 +++++++++
.../src/third_pkg/models/cell_infers/__init__.py | 5 +
.../src/third_pkg/models/cell_infers/cells.py | 155 +++++
.../third_pkg/models/cell_infers/nasnet_cifar.py | 118 ++++
.../third_pkg/models/cell_infers/tiny_network.py | 63 ++
.../src/third_pkg/models/cell_operations.py | 553 +++++++++++++++
.../src/third_pkg/models/cell_searchs/__init__.py | 33 +
.../third_pkg/models/cell_searchs/_test_module.py | 14 +
.../third_pkg/models/cell_searchs/generic_model.py | 366 ++++++++++
.../src/third_pkg/models/cell_searchs/genotypes.py | 280 ++++++++
.../third_pkg/models/cell_searchs/search_cells.py | 267 +++++++
.../models/cell_searchs/search_model_darts.py | 122 ++++
.../cell_searchs/search_model_darts_nasnet.py | 178 +++++
.../models/cell_searchs/search_model_enas.py | 114 +++
.../models/cell_searchs/search_model_enas_utils.py | 74 ++
.../models/cell_searchs/search_model_gdas.py | 142 ++++
.../cell_searchs/search_model_gdas_frc_nasnet.py | 200 ++++++
.../cell_searchs/search_model_gdas_nasnet.py | 197 ++++++
.../models/cell_searchs/search_model_random.py | 102 +++
.../models/cell_searchs/search_model_setn.py | 178 +++++
.../cell_searchs/search_model_setn_nasnet.py | 205 ++++++
.../src/third_pkg/models/clone_weights.py | 74 ++
.../src/third_pkg/models/initialization.py | 16 +
.../models/shape_infers/InferCifarResNet.py | 287 ++++++++
.../models/shape_infers/InferCifarResNet_depth.py | 263 +++++++
.../models/shape_infers/InferCifarResNet_width.py | 277 ++++++++
.../models/shape_infers/InferImagenetResNet.py | 324 +++++++++
.../models/shape_infers/InferMobileNetV2.py | 176 +++++
.../models/shape_infers/InferTinyCellNet.py | 65 ++
.../src/third_pkg/models/shape_infers/__init__.py | 9 +
.../third_pkg/models/shape_infers/shared_utils.py | 5 +
.../models/shape_searchs/SearchCifarResNet.py | 760 ++++++++++++++++++++
.../shape_searchs/SearchCifarResNet_depth.py | 515 ++++++++++++++
.../shape_searchs/SearchCifarResNet_width.py | 619 +++++++++++++++++
.../models/shape_searchs/SearchImagenetResNet.py | 766 +++++++++++++++++++++
.../models/shape_searchs/SearchSimResNet_width.py | 466 +++++++++++++
.../third_pkg/models/shape_searchs/SoftSelect.py | 128 ++++
.../src/third_pkg/models/shape_searchs/__init__.py | 9 +
.../shape_searchs/generic_size_tiny_cell_model.py | 209 ++++++
.../src/third_pkg/sp101_lib/base_ops.py | 68 ++
.../src/third_pkg/sp101_lib/model.py | 277 ++++++++
.../src/third_pkg/sp201_lib/__init__.py | 42 ++
.../src/third_pkg/sp201_lib/api_201.py | 274 ++++++++
.../src/third_pkg/sp201_lib/api_utils.py | 751 ++++++++++++++++++++
.../internal/ml/model_slicing/.gitkeep | 0
.../internal/pg_extension/.gitignore | 6 +
.../pg_extension/sql/pg_extension--0.1.0.sql | 160 +++++
77 files changed, 14295 insertions(+)
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/exp_result_singa/.gitkeep b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/exp_result_singa/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/exps/nas_bench_tabular/sampled_models_10000_models.json b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/exps/nas_bench_tabular/sampled_models_10000_models.json
new file mode 100644
index 00000000..022c7d1b
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/exps/nas_bench_tabular/sampled_models_10000_models.json
@@ -0,0 +1 @@
+{"0": "8-8-8-8", "1": "8-8-8-16", "2": "8-8-8-32", "3": "8-8-8-48", "4": "8-8-8-96", "5": "8-8-8-112", "6": "8-8-8-144", "7": "8-8-8-176", "8": "8-8-8-240", "9": "8-8-8-384", "10": "8-8-16-8", "11": "8-8-16-16", "12": "8-8-16-32", "13": "8-8-16-48", "14": "8-8-16-96", "15": "8-8-16-112", "16": "8-8-16-144", "17": "8-8-16-176", "18": "8-8-16-240", "19": "8-8-16-384", "20": "8-8-32-8", "21": "8-8-32-16", "22": "8-8-32-32", "23": "8-8-32-48", "24": "8-8-32-96", "25": "8-8-32-112", "26": "8- [...]
\ No newline at end of file
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/exps/nas_bench_tabular/sampled_models_all.json b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/exps/nas_bench_tabular/sampled_models_all.json
new file mode 100644
index 00000000..ef320c9a
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/exps/nas_bench_tabular/sampled_models_all.json
@@ -0,0 +1 @@
+{"0": "8-8-8-8", "1": "8-8-8-16", "2": "8-8-8-24", "3": "8-8-8-32", "4": "8-8-8-48", "5": "8-8-8-64", "6": "8-8-8-80", "7": "8-8-8-96", "8": "8-8-8-112", "9": "8-8-8-128", "10": "8-8-8-144", "11": "8-8-8-160", "12": "8-8-8-176", "13": "8-8-8-192", "14": "8-8-8-208", "15": "8-8-8-224", "16": "8-8-8-240", "17": "8-8-8-256", "18": "8-8-8-384", "19": "8-8-8-512", "20": "8-8-16-8", "21": "8-8-16-16", "22": "8-8-16-24", "23": "8-8-16-32", "24": "8-8-16-48", "25": "8-8-16-64", "26": "8-8-16-80" [...]
\ No newline at end of file
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/exps/shared_args.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/exps/shared_args.py
new file mode 100644
index 00000000..5e9fea16
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/exps/shared_args.py
@@ -0,0 +1,204 @@
+import argparse
+import os
+import random
+import numpy as np
+import torch
+
+
+def seed_everything(seed=2022):
+ ''' [reference] https://gist.github.com/KirillVladimirov/005ec7f762293d2321385580d3dbe335 '''
+ random.seed(seed)
+ os.environ['PYTHONHASHSEED'] = str(seed)
+ np.random.seed(seed)
+ torch.manual_seed(seed)
+ torch.cuda.manual_seed(seed)
+ torch.backends.cudnn.deterministic = True
+
+
+def str2bool(v):
+ if isinstance(v, bool):
+ return v
+ if v.lower() in ('yes', 'true', 't', 'y', '1'):
+ return True
+ elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+ return False
+ else:
+ raise argparse.ArgumentTypeError('Boolean value expected.')
+
+
+def sampler_args(parser):
+ # define search space,
+ parser.add_argument('--search_space', type=str, default="mlp_sp",
+ help='[nasbench101, nasbench201, mlp_sp]')
+ # EA sampler's parameters,
+ parser.add_argument('--population_size', type=int, default=10, help="The learning rate for REINFORCE.")
+ parser.add_argument('--sample_size', type=int, default=3, help="The momentum value for EMA.")
+ parser.add_argument('--simple_score_sum', default='True', type=str2bool,
+ help="Sum multiple TFMEM score or use Global Rank")
+
+
+def space201_101_share_args(parser):
+ parser.add_argument('--api_loc', type=str, default="NAS-Bench-201-v1_1-096897.pth",
+ help='which search space file to use, ['
+ 'nasbench101: nasbench_only108.pkl'
+ 'nasbench201: NAS-Bench-201-v1_1-096897.pth'
+ ' ... ]')
+
+ parser.add_argument('--init_channels', default=16, type=int, help='output channels of stem convolution')
+ parser.add_argument('--bn', type=int, default=1, help="If use batch norm in network 1 = true, 0 = false")
+
+
+def nb101_args(parser):
+ parser.add_argument('--num_stacks', default=3, type=int, help='#stacks of modules')
+ parser.add_argument('--num_modules_per_stack', default=3, type=int, help='# modules per stack')
+
+
+def nb201_args(parser):
+ parser.add_argument('--init_w_type', type=str, default='none',
+ help='weight initialization (before pruning) type [none, xavier, kaiming, zero]')
+ parser.add_argument('--init_b_type', type=str, default='none',
+ help='bias initialization (before pruning) type [none, xavier, kaiming, zero]')
+ parser.add_argument('--arch_size', type=int, default=1,
+ help='How many node the architecture has at least')
+
+
+def mlp_args(parser):
+ parser.add_argument('--num_layers', default=4, type=int, help='# hidden layers')
+ parser.add_argument('--hidden_choice_len', default=20, type=int, help=
+ 'number of hidden layer choices, 10 for criteo, 20 for others')
+
+
+def mlp_trainner_args(parser):
+ parser.add_argument('--epoch', type=int, default=20,
+ help='number of maximum epochs, '
+ 'frappe: 20, uci_diabetes: 40, criteo: 10'
+ 'nb101: 108, nb201: 200')
+
+ parser.add_argument('--batch_size', type=int, default=32, help='batch size')
+ parser.add_argument('--lr', type=float, default=0.001, help="learning reate")
+ parser.add_argument('--patience', type=int, default=1, help='number of epochs for stopping training')
+ # parser.add_argument('--eval_freq', type=int, default=10000, help='max number of batches to train per epoch')
+
+ parser.add_argument('--iter_per_epoch', type=int, default=200,
+ help="None, "
+ "200 for frappe, uci_diabetes, "
+ "2000 for criteo")
+
+ # MLP model config
+ parser.add_argument('--nfeat', type=int, default=5500,
+ help='the number of features, '
+ 'frappe: 5500, '
+ 'uci_diabetes: 369,'
+ 'criteo: 2100000')
+ parser.add_argument('--nfield', type=int, default=10,
+ help='the number of fields, '
+ 'frappe: 10, '
+ 'uci_diabetes: 43,'
+ 'criteo: 39')
+ parser.add_argument('--nemb', type=int, default=10,
+ help='embedding size 10')
+
+ # MLP train config
+ parser.add_argument('--report_freq', type=int, default=30, help='report frequency')
+ parser.add_argument('--workers', default=1, type=int, help='data loading workers')
+
+
+def data_set_config(parser):
+ parser.add_argument('--base_dir', type=str, default="../exp_data/",
+ help='path of data and result parent folder')
+ # define search space,
+ parser.add_argument('--dataset', type=str, default='frappe',
+ help='cifar10, cifar100, ImageNet16-120 '
+ 'frappe, criteo, uci_diabetes')
+
+ parser.add_argument('--num_labels', type=int, default=2,
+ help='[10, 100, 120],'
+ '[2, 2, 2]')
+
+
+def seq_train_all_params(parser):
+ parser.add_argument('--worker_id', type=int, default=0, help='start from 0')
+ parser.add_argument('--total_workers', type=int, default=120,
+ help='total number of workers, each train some models')
+ parser.add_argument('--total_models_per_worker', type=int, default=-1, help='How many models to evaluate')
+ parser.add_argument('--pre_partitioned_file',
+ default="./internal/ml/model_selection/exps/sampled_data/sampled_models_all.json",
+ type=str, help='all models with id')
+
+
+def dis_train_all_models(parser):
+ parser.add_argument('--worker_each_gpu', default=6, type=int, help='num worker each gpu')
+ parser.add_argument('--gpu_num', default=8, type=int, help='num GPus')
+
+
+# tune interval and schedule NK rate such that it can produce a good result
+def tune_interval_NK_rate(parser):
+ parser.add_argument('--kn_rate', default=-1, type=int, help="default N/K = 100")
+
+
+def db4nas(parser):
+ parser.add_argument('--db_name', default="pg_extension", type=str)
+ parser.add_argument('--db_user', default="postgres", type=str)
+ parser.add_argument('--db_host', default="localhost", type=str)
+ parser.add_argument('--db_port', default=28814, type=int)
+
+
+def anytime_exp_set(parser):
+ parser.add_argument('--only_phase1', default='False', type=str2bool)
+ parser.add_argument('--is_simulate', default='True', type=str2bool,
+ help='Use pre-computed result or run online')
+
+
+def system_performance_exp(parser):
+ parser.add_argument('--models_explore', default=-1, type=int, help='# models to explore in the filtering phase')
+ parser.add_argument('--tfmem', default="express_flow", type=str, help='the matrix t use, all_matrix')
+ parser.add_argument('--embedding_cache_filtering', default='False', type=str2bool,
+ help='Cache embedding for MLP in filtering phase?')
+ parser.add_argument('--concurrency', default=1, type=int, help='number of worker in filtering phase')
+
+
+def parse_arguments():
+ parser = argparse.ArgumentParser(description='system')
+
+ # job config
+ parser.add_argument('--log_name', type=str, default="main_T_100s")
+ parser.add_argument('--budget', type=int, default=100, help="in second")
+
+ # define base dir, where it stores apis, datasets, logs, etc,
+ parser.add_argument('--device', type=str, default="cpu")
+ parser.add_argument('--local_rank', type=int, default=1, help="local rank")
+
+ parser.add_argument('--log_folder', default="log_debug", type=str)
+
+ parser.add_argument('--result_dir', default="./internal/ml/model_selection/exp_result/", type=str,
+ help='path to store exp outputs')
+ parser.add_argument('--num_points', default=12, type=int, help='num GPus')
+
+ sampler_args(parser)
+
+ nb101_args(parser)
+ nb201_args(parser)
+ space201_101_share_args(parser)
+
+ mlp_args(parser)
+ data_set_config(parser)
+ mlp_trainner_args(parser)
+ seq_train_all_params(parser)
+ dis_train_all_models(parser)
+
+ tune_interval_NK_rate(parser)
+
+ db4nas(parser)
+ anytime_exp_set(parser)
+
+ system_performance_exp(parser)
+
+ # tmp
+ parser.add_argument('--max_load', type=int, default=-1, help="Max Loading time")
+
+ # refinement server
+ parser.add_argument('--url', type=str, default=-1, help="Max Loading time")
+
+ seed_everything()
+
+ return parser.parse_args()
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/scripts/nas-bench-tabular/convert_data_format b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/scripts/nas-bench-tabular/convert_data_format
new file mode 100644
index 00000000..4ee20e54
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/scripts/nas-bench-tabular/convert_data_format
@@ -0,0 +1,11 @@
+
+
+
+
+python src/storage/save_load_torch.py --nfield=39 --dataset=criteo_full
+
+python src/storage/save_load_torch.py --nfield=10 --dataset=frappe
+
+python src/storage/save_load_torch.py --nfield=43 --dataset=uci_diabetes
+
+
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/singa_exp_result_result/score_mlp_sp_frappe_batch_size_32_cpu.json b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/singa_exp_result_result/score_mlp_sp_frappe_batch_size_32_cpu.json
new file mode 100644
index 00000000..20c6d5eb
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/singa_exp_result_result/score_mlp_sp_frappe_batch_size_32_cpu.json
@@ -0,0 +1 @@
+{"8-8-8-8": {"synflow": 63.11601734161377}, "8-8-8-16": {"synflow": 49.97195529937744}, "8-8-8-24": {"synflow": 12.703484743833542}, "8-8-8-32": {"synflow": 20.45815622806549}, "8-8-8-48": {"synflow": 73.69489860534668}, "8-8-8-64": {"synflow": 74.26036071777344}, "8-8-8-80": {"synflow": 40.1631133556366}, "8-8-8-96": {"synflow": 5.263346729800105}, "8-8-8-112": {"synflow": 23.08770751953125}, "8-8-8-128": {"synflow": 3.2491158824414015}, "8-8-8-144": {"synflow": 5.015950359404087}, "8-8 [...]
\ No newline at end of file
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/common/__init__.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/common/__init__.py
new file mode 100644
index 00000000..47a25294
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/common/__init__.py
@@ -0,0 +1,9 @@
+
+
+import torch
+
+ENABLE_GPU = False
+
+if torch.cuda.is_available():
+ ENABLE_GPU = False
+
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/controller/sampler_bohb/bohb_or.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/controller/sampler_bohb/bohb_or.py
new file mode 100644
index 00000000..97266f6e
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/controller/sampler_bohb/bohb_or.py
@@ -0,0 +1,285 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020 #
+###################################################################
+# BOHB: Robust and Efficient Hyperparameter Optimization at Scale #
+# required to install hpbandster ##################################
+# pip install hpbandster ##################################
+###################################################################
+# OMP_NUM_THREADS=4 python exps/NATS-algos/bohb.py --search_space tss --dataset cifar10 --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3 --rand_seed 1
+# OMP_NUM_THREADS=4 python exps/NATS-algos/bohb.py --search_space sss --dataset cifar10 --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3 --rand_seed 1
+###################################################################
+import os, sys, time, random, argparse, collections
+from copy import deepcopy
+
+from src.tools.env_tools import prepare_seed
+from src.logger import logger
+
+from models import CellStructure, get_search_spaces
+
+# BOHB: Robust and Efficient Hyperparameter Optimization at Scale, ICML 2018
+import ConfigSpace
+from hpbandster.optimizers.bohb import BOHB
+import hpbandster.core.nameserver as hpns
+from hpbandster.core.worker import Worker
+
+from nats_bench import create
+
+
+def time_string():
+ ISOTIMEFORMAT = "%Y-%m-%d %X"
+ string = "[{:}]".format(time.strftime(ISOTIMEFORMAT, time.gmtime(time.time())))
+ return string
+
+
+def get_topology_config_space(search_space, max_nodes=4):
+ cs = ConfigSpace.ConfigurationSpace()
+ # edge2index = {}
+ for i in range(1, max_nodes):
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ cs.add_hyperparameter(
+ ConfigSpace.CategoricalHyperparameter(node_str, search_space)
+ )
+ return cs
+
+
+def get_size_config_space(search_space):
+ cs = ConfigSpace.ConfigurationSpace()
+ for ilayer in range(search_space["numbers"]):
+ node_str = "layer-{:}".format(ilayer)
+ cs.add_hyperparameter(
+ ConfigSpace.CategoricalHyperparameter(node_str, search_space["candidates"])
+ )
+ return cs
+
+
+def config2topology_func(max_nodes=4):
+ def config2structure(config):
+ genotypes = []
+ for i in range(1, max_nodes):
+ xlist = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ op_name = config[node_str]
+ xlist.append((op_name, j))
+ genotypes.append(tuple(xlist))
+ return CellStructure(genotypes)
+
+ return config2structure
+
+
+def config2size_func(search_space):
+ def config2structure(config):
+ channels = []
+ for ilayer in range(search_space["numbers"]):
+ node_str = "layer-{:}".format(ilayer)
+ channels.append(str(config[node_str]))
+ return ":".join(channels)
+
+ return config2structure
+
+
+class MyWorker(Worker):
+ def __init__(self, *args, convert_func=None, dataset=None, api=None, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.convert_func = convert_func
+ self._dataset = dataset
+ self._api = api
+ self.total_times = []
+ self.trajectory = []
+
+ def compute(self, config, budget, **kwargs):
+ arch = self.convert_func(config)
+ accuracy, latency, time_cost, total_time = self._api.simulate_train_eval(
+ arch, self._dataset, iepoch=int(budget) - 1, hp="12"
+ )
+ self.trajectory.append((accuracy, arch))
+ self.total_times.append(total_time)
+ return {"loss": 100 - accuracy, "info": self._api.query_index_by_arch(arch)}
+
+
+def main(xargs, api):
+ import torch
+ torch.set_num_threads(4)
+ prepare_seed(xargs.rand_seed)
+
+ logger.info("{:} use api : {:}".format(time_string(), api))
+ api.reset_time()
+ search_space = get_search_spaces(xargs.search_space, "nats-bench")
+ if xargs.search_space == "tss":
+ cs = get_topology_config_space(search_space)
+ config2structure = config2topology_func()
+ else:
+ cs = get_size_config_space(search_space)
+ config2structure = config2size_func(search_space)
+
+ hb_run_id = "0"
+
+ NS = hpns.NameServer(run_id=hb_run_id, host="localhost", port=0)
+ ns_host, ns_port = NS.start()
+ num_workers = 1
+
+ workers = []
+ for i in range(num_workers):
+ w = MyWorker(
+ nameserver=ns_host,
+ nameserver_port=ns_port,
+ convert_func=config2structure,
+ dataset=xargs.dataset,
+ api=api,
+ run_id=hb_run_id,
+ id=i,
+ )
+ w.run(background=True)
+ workers.append(w)
+
+ start_time = time.time()
+ bohb = BOHB(
+ configspace=cs,
+ run_id=hb_run_id,
+ eta=3,
+ min_budget=1,
+ max_budget=12,
+ nameserver=ns_host,
+ nameserver_port=ns_port,
+ num_samples=xargs.num_samples,
+ random_fraction=xargs.random_fraction,
+ bandwidth_factor=xargs.bandwidth_factor,
+ ping_interval=10,
+ min_bandwidth=xargs.min_bandwidth,
+ )
+
+ results = run(xargs.n_iters, min_n_workers=num_workers)
+
+ bohb.shutdown(shutdown_workers=True)
+ NS.shutdown()
+
+ # print('There are {:} runs.'.format(len(results.get_all_runs())))
+ # workers[0].total_times
+ # workers[0].trajectory
+ current_best_index = []
+ for idx in range(len(workers[0].trajectory)):
+ trajectory = workers[0].trajectory[: idx + 1]
+ arch = max(trajectory, key=lambda x: x[0])[1]
+ current_best_index.append(api.query_index_by_arch(arch))
+
+ best_arch = max(workers[0].trajectory, key=lambda x: x[0])[1]
+ logger.log(
+ "Best found configuration: {:} within {:.3f} s".format(
+ best_arch, workers[0].total_times[-1]
+ )
+ )
+ info = api.query_info_str_by_arch(
+ best_arch, "200" if xargs.search_space == "tss" else "90"
+ )
+ logger.log("{:}".format(info))
+ logger.log("-" * 100)
+ logger.close()
+
+ return logger.log_dir, current_best_index, workers[0].total_times
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ "BOHB: Robust and Efficient Hyperparameter Optimization at Scale"
+ )
+ parser.add_argument(
+ "--dataset",
+ default="cifar10",
+ type=str,
+ choices=["cifar10", "cifar100", "ImageNet16-120"],
+ help="Choose between Cifar10/100 and ImageNet-16.",
+ )
+ # general arg
+ parser.add_argument(
+ "--search_space",
+ default="tss",
+ type=str,
+ choices=["tss", "sss"],
+ help="Choose the search space.",
+ )
+ parser.add_argument(
+ "--time_budget",
+ type=int,
+ default=20000,
+ help="The total time cost budge for searching (in seconds).",
+ )
+ parser.add_argument(
+ "--loops_if_rand", type=int, default=500, help="The total runs for evaluation."
+ )
+ # BOHB
+ parser.add_argument(
+ "--strategy",
+ default="sampling",
+ type=str,
+ nargs="?",
+ help="optimization strategy for the acquisition function",
+ )
+ parser.add_argument(
+ "--min_bandwidth",
+ default=0.3,
+ type=float,
+ nargs="?",
+ help="minimum bandwidth for KDE",
+ )
+ parser.add_argument(
+ "--num_samples",
+ default=64,
+ type=int,
+ nargs="?",
+ help="number of samples for the acquisition function",
+ )
+ parser.add_argument(
+ "--random_fraction",
+ default=0.33,
+ type=float,
+ nargs="?",
+ help="fraction of random configurations",
+ )
+ parser.add_argument(
+ "--bandwidth_factor",
+ default=3,
+ type=int,
+ nargs="?",
+ help="factor multiplied to the bandwidth",
+ )
+ parser.add_argument(
+ "--n_iters",
+ default=300,
+ type=int,
+ nargs="?",
+ help="number of iterations for optimization method",
+ )
+ # log
+ parser.add_argument(
+ "--save_dir",
+ type=str,
+ default="./output/search",
+ help="Folder to save checkpoints and log.",
+ )
+ parser.add_argument("--rand_seed", type=int, default=-1, help="manual seed")
+ args = parser.parse_args()
+
+ api = create(None, args.search_space, fast_mode=False, verbose=False)
+
+ args.save_dir = os.path.join(
+ "{:}-{:}".format(args.save_dir, args.search_space),
+ "{:}-T{:}".format(args.dataset, args.time_budget),
+ "BOHB",
+ )
+ print("save-dir : {:}".format(args.save_dir))
+
+ import torch
+ if args.rand_seed < 0:
+ save_dir, all_info = None, collections.OrderedDict()
+ for i in range(args.loops_if_rand):
+ print("{:} : {:03d}/{:03d}".format(time_string(), i, args.loops_if_rand))
+ args.rand_seed = random.randint(1, 100000)
+ save_dir, all_archs, all_total_times = main(args, api)
+ all_info[i] = {"all_archs": all_archs, "all_total_times": all_total_times}
+ save_path = save_dir / "results.pth"
+ print("save into {:}".format(save_path))
+ torch.save(all_info, save_path)
+ else:
+ main(args, api)
+
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/dataset.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/dataset.py
new file mode 100644
index 00000000..37d3c8ca
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/dataset.py
@@ -0,0 +1,181 @@
+
+import torch
+from torch import tensor
+from torch.utils.data import DataLoader
+from torchvision.datasets import MNIST, CIFAR10, CIFAR100, SVHN
+from torchvision.transforms import Compose
+from torchvision import transforms
+from .imagenet16 import *
+
+
+def get_dataloader(train_batch_size: int, test_batch_size: int, dataset: int,
+ num_workers: int, datadir: str, resize=None) -> (DataLoader, DataLoader, int):
+ """
+ Load CIFAR or imagenet datasets
+ :param train_batch_size:
+ :param test_batch_size:
+ :param dataset: ImageNet16, cifar, svhn, ImageNet1k, mnist
+ :param num_workers:
+ :param datadir:
+ :param resize:
+ :return:
+ """
+
+ class_num = 0
+ mean = []
+ std = []
+ pad = 0
+
+ if 'ImageNet16' in dataset:
+ mean = [x / 255 for x in [122.68, 116.66, 104.01]]
+ std = [x / 255 for x in [63.22, 61.26 , 65.09]]
+ size, pad = 16, 2
+ elif 'cifar' in dataset:
+ mean = (0.4914, 0.4822, 0.4465)
+ std = (0.2023, 0.1994, 0.2010)
+ size, pad = 32, 4
+ elif 'svhn' in dataset:
+ mean = (0.5, 0.5, 0.5)
+ std = (0.5, 0.5, 0.5)
+ size, pad = 32, 0
+ elif dataset == 'ImageNet1k':
+ from .h5py_dataset import H5Dataset
+ size, pad = 224, 2
+ mean = (0.485, 0.456, 0.406)
+ std = (0.229, 0.224, 0.225)
+ #resize = 256
+
+ if resize is None:
+ resize = size
+
+ train_transform = transforms.Compose([
+ transforms.RandomCrop(size, padding=pad),
+ transforms.Resize(resize),
+ transforms.RandomHorizontalFlip(),
+ transforms.ToTensor(),
+ transforms.Normalize(mean, std),
+ ])
+
+ test_transform = transforms.Compose([
+ transforms.Resize(resize),
+ transforms.ToTensor(),
+ transforms.Normalize(mean, std),
+ ])
+
+ if dataset == 'cifar10':
+ class_num = 10
+ train_dataset = CIFAR10(datadir, True, train_transform, download=True)
+ test_dataset = CIFAR10(datadir, False, test_transform, download=True)
+ elif dataset == 'cifar100':
+ class_num = 100
+ train_dataset = CIFAR100(datadir, True, train_transform, download=True)
+ test_dataset = CIFAR100(datadir, False, test_transform, download=True)
+ elif dataset == 'svhn':
+ class_num = 10
+ train_dataset = SVHN(datadir, split='train', transform=train_transform, download=True)
+ test_dataset = SVHN(datadir, split='test', transform=test_transform, download=True)
+ elif dataset == 'ImageNet16-120':
+ class_num = 120
+ train_dataset = ImageNet16(os.path.join(datadir, 'ImageNet16'), True , train_transform, 120)
+ test_dataset = ImageNet16(os.path.join(datadir, 'ImageNet16'), False, test_transform , 120)
+ elif dataset == 'ImageNet1k':
+ class_num = 120
+ train_dataset = H5Dataset(os.path.join(datadir, 'imagenet-train-256.h5'), transform=train_transform)
+ test_dataset = H5Dataset(os.path.join(datadir, 'imagenet-val-256.h5'), transform=test_transform)
+ elif dataset == 'mnist':
+ data_transform = Compose([transforms.ToTensor()])
+ # Normalise? transforms.Normalize((0.1307,), (0.3081,))
+ train_dataset = MNIST("_dataset", True, data_transform, download=True)
+ test_dataset = MNIST("_dataset", False, data_transform, download=True)
+ else:
+ raise ValueError('There are no more cifars or imagenets.')
+
+ train_loader = DataLoader(
+ train_dataset,
+ train_batch_size,
+ shuffle=True,
+ # num_workers=num_workers,
+ # pin_memory=False
+ )
+ test_loader = DataLoader(
+ test_dataset,
+ test_batch_size,
+ shuffle=False,
+ # num_workers=num_workers,
+ # pin_memory=False
+ )
+
+ print("dataset load done")
+
+ return train_loader, test_loader, class_num
+
+
+def get_mini_batch(dataloader: DataLoader, sample_alg: str, batch_size: int, num_classes: int) -> (tensor, tensor):
+ """
+ Get a mini-batch of data,
+ :param dataloader: DataLoader
+ :param sample_alg: random or grasp
+ :param batch_size: batch_size
+ :param num_classes: num_classes
+ :return: two tensor
+ """
+
+ if sample_alg == 'random':
+ inputs, targets = _get_some_data(dataloader, batch_size=batch_size)
+ elif sample_alg == 'grasp':
+ inputs, targets = _get_some_data_grasp(dataloader, num_classes, samples_per_class=batch_size//num_classes)
+ else:
+ raise NotImplementedError(f'dataload {sample_alg} is not supported')
+
+ return inputs, targets
+
+
+def _get_some_data(train_dataloader: DataLoader, batch_size: int) -> (torch.tensor, torch.tensor):
+ """
+ Randomly sample some data, some class may not be sampled
+ :param train_dataloader: torch dataLoader
+ :param batch_size: batch_size of the data.
+ :return:
+ """
+ traindata = []
+
+ dataloader_iter = iter(train_dataloader)
+ traindata.append(next(dataloader_iter))
+
+ inputs = torch.cat([a for a, _ in traindata])
+ targets = torch.cat([b for _, b in traindata])
+ inputs = inputs
+ targets = targets
+ return inputs, targets
+
+
+def _get_some_data_grasp(train_dataloader: DataLoader, num_classes: int,
+ samples_per_class: int) -> (torch.tensor, torch.tensor):
+ """
+ Sample some data while guarantee example class has equal number of samples.
+ :param train_dataloader: torch dataLoader
+ :param num_classes: number of class
+ :param samples_per_class: how many samples for eacl class.
+ :return:
+ """
+
+ datas = [[] for _ in range(num_classes)]
+ labels = [[] for _ in range(num_classes)]
+ mark = dict()
+ dataloader_iter = iter(train_dataloader)
+ while True:
+ inputs, targets = next(dataloader_iter)
+ for idx in range(inputs.shape[0]):
+ x, y = inputs[idx:idx + 1], targets[idx:idx + 1]
+ category = y.item()
+ if len(datas[category]) == samples_per_class:
+ mark[category] = True
+ continue
+ datas[category].append(x)
+ labels[category].append(y)
+ if len(mark) == num_classes:
+ break
+
+ x = torch.cat([torch.cat(_, 0) for _ in datas])
+ y = torch.cat([torch.cat(_) for _ in labels]).view(-1)
+ return x, y
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/h5py_dataset.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/h5py_dataset.py
new file mode 100644
index 00000000..411e4b76
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/h5py_dataset.py
@@ -0,0 +1,40 @@
+import h5py
+import numpy as np
+from PIL import Image
+
+import torch
+from torch.utils.data import Dataset, DataLoader
+
+
+class H5Dataset(Dataset):
+ def __init__(self, h5_path, transform=None):
+ self.h5_path = h5_path
+ self.h5_file = None
+ self.length = len(h5py.File(h5_path, 'r'))
+ self.transform = transform
+
+ def __getitem__(self, index):
+
+ # loading in getitem allows us to use multiple processes for data loading
+ # because hdf5 files aren't pickelable so can't transfer them across processes
+ # https://discuss.pytorch.org/t/hdf5-a-data-format-for-pytorch/40379
+ # https://discuss.pytorch.org/t/dataloader-when-num-worker-0-there-is-bug/25643/16
+ # TODO possible look at __getstate__ and __setstate__ as a more elegant solution
+ if self.h5_file is None:
+ self.h5_file = h5py.File(self.h5_path, 'r')
+
+ record = self.h5_file[str(index)]
+
+ if self.transform:
+ x = Image.fromarray(record['data'][()])
+ x = self.transform(x)
+ else:
+ x = torch.from_numpy(record['data'][()])
+
+ y = record['target'][()]
+ y = torch.from_numpy(np.asarray(y))
+
+ return (x, y)
+
+ def __len__(self):
+ return self.length
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/imagenet16.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/imagenet16.py
new file mode 100644
index 00000000..c49428c0
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/imagenet16.py
@@ -0,0 +1,137 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+##################################################
+
+import os, sys, hashlib
+import numpy as np
+from PIL import Image
+import torch.utils.data as data
+
+if sys.version_info[0] == 2:
+ import cPickle as pickle
+else:
+ import pickle
+
+
+def calculate_md5(fpath, chunk_size=1024 * 1024):
+ md5 = hashlib.md5()
+ with open(fpath, 'rb') as f:
+ for chunk in iter(lambda: f.read(chunk_size), b''):
+ md5.update(chunk)
+ return md5.hexdigest()
+
+
+def check_md5(fpath, md5, **kwargs):
+ return md5 == calculate_md5(fpath, **kwargs)
+
+
+def check_integrity(fpath, md5=None):
+ if not os.path.isfile(fpath): return False
+ if md5 is None:
+ return True
+ else:
+ return check_md5(fpath, md5)
+
+
+class ImageNet16(data.Dataset):
+ # http://image-net.org/download-images
+ # A Downsampled Variant of ImageNet as an Alternative to the CIFAR datasets
+ # https://arxiv.org/pdf/1707.08819.pdf
+
+ train_list = [
+ ['train_data_batch_1', '27846dcaa50de8e21a7d1a35f30f0e91'],
+ ['train_data_batch_2', 'c7254a054e0e795c69120a5727050e3f'],
+ ['train_data_batch_3', '4333d3df2e5ffb114b05d2ffc19b1e87'],
+ ['train_data_batch_4', '1620cdf193304f4a92677b695d70d10f'],
+ ['train_data_batch_5', '348b3c2fdbb3940c4e9e834affd3b18d'],
+ ['train_data_batch_6', '6e765307c242a1b3d7d5ef9139b48945'],
+ ['train_data_batch_7', '564926d8cbf8fc4818ba23d2faac7564'],
+ ['train_data_batch_8', 'f4755871f718ccb653440b9dd0ebac66'],
+ ['train_data_batch_9', 'bb6dd660c38c58552125b1a92f86b5d4'],
+ ['train_data_batch_10', '8f03f34ac4b42271a294f91bf480f29b'],
+ ]
+ valid_list = [
+ ['val_data', '3410e3017fdaefba8d5073aaa65e4bd6'],
+ ]
+
+ def __init__(self, root, train, transform, use_num_of_class_only=None):
+ self.root = root
+ self.transform = transform
+ self.train = train # training set or valid set
+ if not self._check_integrity(): raise RuntimeError('Dataset not found or corrupted.')
+
+ if self.train:
+ downloaded_list = self.train_list
+ else:
+ downloaded_list = self.valid_list
+ self.data = []
+ self.targets = []
+
+ # now load the picked numpy arrays
+ for i, (file_name, checksum) in enumerate(downloaded_list):
+ file_path = os.path.join(self.root, file_name)
+ # print ('Load {:}/{:02d}-th : {:}'.format(i, len(downloaded_list), file_path))
+ with open(file_path, 'rb') as f:
+ if sys.version_info[0] == 2:
+ entry = pickle.load(f)
+ else:
+ entry = pickle.load(f, encoding='latin1')
+ self.data.append(entry['data'])
+ self.targets.extend(entry['labels'])
+ self.data = np.vstack(self.data).reshape(-1, 3, 16, 16)
+ self.data = self.data.transpose((0, 2, 3, 1)) # convert to HWC
+ if use_num_of_class_only is not None:
+ assert isinstance(use_num_of_class_only,
+ int) and use_num_of_class_only > 0 and use_num_of_class_only < 1000, 'invalid use_num_of_class_only : {:}'.format(
+ use_num_of_class_only)
+ new_data, new_targets = [], []
+ for I, L in zip(self.data, self.targets):
+ if 1 <= L <= use_num_of_class_only:
+ new_data.append(I)
+ new_targets.append(L)
+ self.data = new_data
+ self.targets = new_targets
+ # self.mean.append(entry['mean'])
+ # self.mean = np.vstack(self.mean).reshape(-1, 3, 16, 16)
+ # self.mean = np.mean(np.mean(np.mean(self.mean, axis=0), axis=1), axis=1)
+ # print ('Mean : {:}'.format(self.mean))
+ # temp = self.data - np.reshape(self.mean, (1, 1, 1, 3))
+ # std_data = np.std(temp, axis=0)
+ # std_data = np.mean(np.mean(std_data, axis=0), axis=0)
+ # print ('Std : {:}'.format(std_data))
+
+ def __getitem__(self, index):
+ img, target = self.data[index], self.targets[index] - 1
+
+ img = Image.fromarray(img)
+
+ if self.transform is not None:
+ img = self.transform(img)
+
+ return img, target
+
+ def __len__(self):
+ return len(self.data)
+
+ def _check_integrity(self):
+ root = self.root
+ for fentry in (self.train_list + self.valid_list):
+ filename, md5 = fentry[0], fentry[1]
+ fpath = os.path.join(root, filename)
+ if not check_integrity(fpath, md5):
+ return False
+ return True
+
+
+if __name__ == '__main__':
+ train = ImageNet16('/data02/dongxuanyi/.torch/cifar.python/ImageNet16', True, None)
+ valid = ImageNet16('/data02/dongxuanyi/.torch/cifar.python/ImageNet16', False, None)
+
+ print(len(train))
+ print(len(valid))
+ image, label = train[111]
+ trainX = ImageNet16('/data02/dongxuanyi/.torch/cifar.python/ImageNet16', True, None, 200)
+ validX = ImageNet16('/data02/dongxuanyi/.torch/cifar.python/ImageNet16', False, None, 200)
+ print(len(trainX))
+ print(len(validX))
+ # import pdb; pdb.set_trace()
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/save_load_torch.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/save_load_torch.py
new file mode 100644
index 00000000..3bad80b3
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/save_load_torch.py
@@ -0,0 +1,87 @@
+import argparse
+
+from tqdm import tqdm
+import torch
+import os
+import glob
+
+
+def decode_libsvm(line):
+ columns = line.split(' ')
+ map_func = lambda pair: (int(pair[0]), float(pair[1]))
+ id, value = zip(*map(lambda col: map_func(col.split(':')), columns[1:]))
+ sample = {'id': torch.LongTensor(id),
+ 'value': torch.FloatTensor(value),
+ 'y': float(columns[0])}
+ return sample
+
+
+def _save_data(data_dir, fname, nfields, namespace):
+ with open(fname) as f:
+ sample_lines = sum(1 for line in f)
+
+ feat_id = torch.LongTensor(sample_lines, nfields)
+ feat_value = torch.FloatTensor(sample_lines, nfields)
+ y = torch.FloatTensor(sample_lines)
+
+ nsamples = 0
+ with tqdm(total=sample_lines) as pbar:
+ with open(fname) as fp:
+ line = fp.readline()
+ while line:
+ try:
+ sample = decode_libsvm(line)
+ feat_id[nsamples] = sample['id']
+ feat_value[nsamples] = sample['value']
+ y[nsamples] = sample['y']
+ nsamples += 1
+ except Exception:
+ print(f'incorrect data format line "{line}" !')
+ line = fp.readline()
+ pbar.update(1)
+ print(f'# {nsamples} data samples loaded...')
+
+ # save the tensors to disk
+ torch.save(feat_id, f'{data_dir}/{namespace}_feat_id.pt')
+ torch.save(feat_value, f'{data_dir}/{namespace}_feat_value.pt')
+ torch.save(y, f'{data_dir}/{namespace}_y.pt')
+
+
+def parse_arguments():
+ parser = argparse.ArgumentParser(description='FastAutoNAS')
+
+ parser.add_argument('--nfield', type=int, default=10,
+ help='the number of fields, frappe: 10, uci_diabetes: 43, criteo: 39')
+
+ parser.add_argument('--dataset', type=str, default='frappe',
+ help='cifar10, cifar100, ImageNet16-120, frappe, criteo, uci_diabetes')
+
+ return parser.parse_args()
+
+
+def load_data(data_dir, namespace):
+ feat_id = torch.load(f'{data_dir}/{namespace}_feat_id.pt')
+ feat_value = torch.load(f'{data_dir}/{namespace}_feat_value.pt')
+ y = torch.load(f'{data_dir}/{namespace}_y.pt')
+
+ print(f'# {int(y.shape[0])} data samples loaded...')
+
+ return feat_id, feat_value, y, int(y.shape[0])
+
+
+if __name__ == "__main__":
+ args = parse_arguments()
+
+ _data_dir = os.path.join("../exp_data/data/structure_data", args.dataset)
+
+ train_name_space = "decoded_train"
+ valid_name_space = "decoded_valid"
+ # save
+ train_file = glob.glob("%s/tr*libsvm" % _data_dir)[0]
+ val_file = glob.glob("%s/va*libsvm" % _data_dir)[0]
+ _save_data(_data_dir, train_file, args.nfield, train_name_space)
+ _save_data(_data_dir, val_file, args.nfield, valid_name_space)
+
+ # read
+ # load_data(data_dir, train_name_space)
+ # load_data(data_dir, valid_name_space)
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/structure_data_loader.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/structure_data_loader.py
new file mode 100644
index 00000000..9db306e7
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/dataset_utils/structure_data_loader.py
@@ -0,0 +1,258 @@
+
+from tqdm import tqdm
+import torch
+from torch.utils.data import Dataset, DataLoader, TensorDataset
+import os
+import glob
+import numpy as np
+import sklearn.model_selection
+from scipy.io.arff import loadarff
+
+
+def load_data(data_dir, namespace):
+ print(f'# loading data from '
+ f'{data_dir}/{namespace}_feat_id.pt, '
+ f'{data_dir}/{namespace}_feat_value.pt'
+ f'{data_dir}/{namespace}_y.pt ......')
+
+ feat_id = torch.load(f'{data_dir}/{namespace}_feat_id.pt')
+ feat_value = torch.load(f'{data_dir}/{namespace}_feat_value.pt')
+ y = torch.load(f'{data_dir}/{namespace}_y.pt')
+
+ print(f'# {int(y.shape[0])} data samples loaded...')
+
+ return feat_id, feat_value, y, int(y.shape[0])
+
+
+class LibsvmDatasetReadOnce(Dataset):
+ """ Dataset loader for Libsvm data format """
+
+ def __init__(self, fname):
+ parent_directory = os.path.dirname(fname)
+ if "train" in fname:
+ namespace = "decoded_train"
+ elif "valid" in fname:
+ namespace = "decoded_valid"
+ else:
+ raise
+ self.feat_id, self.feat_value, self.y, self.nsamples = load_data(parent_directory, namespace)
+
+ print(f'# {self.nsamples} data samples loaded...')
+
+ def __len__(self):
+ return self.nsamples
+
+ def __getitem__(self, idx):
+ return {'id': self.feat_id[idx],
+ 'value': self.feat_value[idx],
+ 'y': self.y[idx]}
+
+
+class LibsvmDataset(Dataset):
+ """ Dataset loader for Libsvm data format """
+
+ def __init__(self, fname, nfields, max_load=-1):
+
+ def decode_libsvm(line):
+ columns = line.split(' ')
+ map_func = lambda pair: (int(pair[0]), float(pair[1]))
+ id, value = zip(*map(lambda col: map_func(col.split(':')), columns[1:]))
+ sample = {'id': torch.LongTensor(id),
+ 'value': torch.FloatTensor(value),
+ 'y': float(columns[0])}
+ return sample
+
+ with open(fname) as f:
+ sample_lines = sum(1 for line in f)
+
+ self.feat_id = torch.LongTensor(sample_lines, nfields)
+ self.feat_value = torch.FloatTensor(sample_lines, nfields)
+ self.y = torch.FloatTensor(sample_lines)
+
+ self.nsamples = 0
+ with tqdm(total=sample_lines) as pbar:
+ with open(fname) as fp:
+ line = fp.readline()
+ while line:
+ if max_load > 0 and self.nsamples > max_load:
+ break
+ try:
+ sample = decode_libsvm(line)
+ self.feat_id[self.nsamples] = sample['id']
+ self.feat_value[self.nsamples] = sample['value']
+ self.y[self.nsamples] = sample['y']
+ self.nsamples += 1
+ except Exception:
+ print(f'incorrect data format line "{line}" !')
+ line = fp.readline()
+ pbar.update(1)
+ print(f'# {self.nsamples} data samples loaded...')
+
+ def __len__(self):
+ return self.nsamples
+
+ def __getitem__(self, idx):
+ return {'id': self.feat_id[idx],
+ 'value': self.feat_value[idx],
+ 'y': self.y[idx]}
+
+
+def libsvm_dataloader(args, data_dir, nfield, batch_size):
+ print("Loading data from ", data_dir)
+ workers = args.workers
+ train_file_name = f"{data_dir}/train.libsvm"
+ valid_file_name = f"{data_dir}/valid.libsvm"
+ test_file_name = f"{data_dir}/test.libsvm"
+ print(f"using train={train_file_name}, valid={valid_file_name}")
+ # read the converted file
+ if args.device == "cpu":
+ train_loader = DataLoader(LibsvmDatasetReadOnce(train_file_name),
+ batch_size=batch_size,
+ shuffle=True)
+ val_loader = DataLoader(LibsvmDatasetReadOnce(valid_file_name),
+ batch_size=batch_size * 8,
+ shuffle=False)
+
+ else:
+ train_loader = DataLoader(LibsvmDatasetReadOnce(train_file_name),
+ batch_size=batch_size,
+ shuffle=True,
+ num_workers=workers,
+ pin_memory=False)
+
+ val_loader = DataLoader(LibsvmDatasetReadOnce(valid_file_name),
+ batch_size=batch_size * 8,
+ shuffle=False,
+ num_workers=workers,
+ pin_memory=False)
+
+ return train_loader, val_loader, val_loader
+
+
+def libsvm_dataloader_ori(args):
+ data_dir = args.base_dir + args.dataset
+ print(data_dir)
+ train_file = glob.glob("%s/tr*libsvm" % data_dir)[0]
+ val_file = glob.glob("%s/va*libsvm" % data_dir)[0]
+ test_file = glob.glob("%s/te*libsvm" % data_dir)[0]
+
+ train_loader = DataLoader(LibsvmDataset(train_file, args.nfield, args.max_load),
+ batch_size=args.batch_size, shuffle=True,
+ num_workers=args.workers, pin_memory=True)
+ val_loader = DataLoader(LibsvmDataset(val_file, args.nfield, args.max_load),
+ batch_size=args.batch_size, shuffle=False,
+ num_workers=args.workers, pin_memory=True)
+ # test_loader = DataLoader(LibsvmDataset(test_file, args.nfield),
+ # batch_size=args.batch_size, shuffle=False,
+ # num_workers=args.workers, pin_memory=True)
+
+ return train_loader, val_loader, val_loader
+
+
+class UCILibsvmDataset(Dataset):
+ """ Dataset loader for loading UCI dataset of Libsvm format """
+
+ def __init__(self, X, y):
+ assert X.shape[0] == y.shape[0]
+ self.nsamples, self.nfeat = X.shape
+
+ self.feat_id = torch.LongTensor(self.nsamples, self.nfeat)
+ self.feat_value = torch.FloatTensor(self.nsamples, self.nfeat)
+ self.y = torch.FloatTensor(self.nsamples)
+
+ with tqdm(total=self.nsamples) as pbar:
+ id = torch.LongTensor(range(self.nfeat))
+ for idx in range(self.nsamples):
+ self.feat_id[idx] = id
+ self.feat_value[idx] = torch.FloatTensor(X[idx])
+ self.y[idx] = y[idx]
+
+ pbar.update(1)
+ print(f'Data loader: {self.nsamples} data samples')
+
+ def __len__(self):
+ return self.nsamples
+
+ def __getitem__(self, idx):
+ return {'id': self.feat_id[idx],
+ 'value': self.feat_value[idx],
+ 'y': self.y[idx]}
+
+
+def uci_loader(data_dir, batch_size, valid_perc=0., libsvm=False, workers=4):
+ '''
+ :param data_dir: Path to load the uci dataset
+ :param batch_size: Batch size
+ :param valid_perc: valid percentage split from train (default 0, whole train set)
+ :param libsvm: Libsvm loader of format {'id', 'value', 'y'}
+ :param workers: the number of subprocesses to load data
+ :return: train/valid/test loader, train_loader.nclass
+ '''
+
+ def uci_validation_set(X, y, split_perc=0.2):
+ return sklearn.model_selection.train_test_split(
+ X, y, test_size=split_perc, random_state=0)
+
+ def make_loader(X, y, transformer=None, batch_size=64):
+ if transformer is None:
+ transformer = sklearn.preprocessing.StandardScaler()
+ transformer.fit(X)
+ X = transformer.transform(X)
+ if libsvm:
+ return DataLoader(UCILibsvmDataset(X, y),
+ batch_size=batch_size,
+ shuffle=transformer is None,
+ num_workers=workers, pin_memory=True
+ ), transformer
+ else:
+ return DataLoader(
+ dataset=TensorDataset(*[torch.from_numpy(e) for e in [X, y]]),
+ batch_size=batch_size,
+ shuffle=transformer is None,
+ num_workers=workers, pin_memory=True
+ ), transformer
+
+ def uci_folder_to_name(f):
+ return f.split('/')[-1]
+
+ def line_to_idx(l):
+ return np.array([int(e) for e in l.split()], dtype=np.int32)
+
+ def load_uci_dataset(folder, train=True):
+ full_file = f'{folder}/{uci_folder_to_name(folder)}.arff'
+ if os.path.exists(full_file):
+ data = loadarff(full_file)
+ train_idx, test_idx = [line_to_idx(l) for l in open(f'{folder}/conxuntos.dat').readlines()]
+ assert len(set(train_idx) & set(test_idx)) == 0
+ all_idx = list(train_idx) + list(test_idx)
+ assert len(all_idx) == np.max(all_idx) + 1
+ assert np.min(all_idx) == 0
+ if train:
+ data = (data[0][train_idx], data[1])
+ else:
+ data = (data[0][test_idx], data[1])
+ else:
+ typename = 'train' if train else 'test'
+ filename = f'{folder}/{uci_folder_to_name(folder)}_{typename}.arff'
+ data = loadarff(filename)
+ assert data[1].types() == ['numeric'] * (len(data[1].types()) - 1) + ['nominal']
+ X = np.array(data[0][data[1].names()[:-1]].tolist())
+ y = np.array([int(e) for e in data[0][data[1].names()[-1]]])
+ nclass = len(data[1]['clase'][1])
+ return X.astype(np.float32), y, nclass
+
+ Xtrain, ytrain, nclass = load_uci_dataset(data_dir)
+ if valid_perc > 0:
+ Xtrain, Xvalid, ytrain, yvalid = uci_validation_set(Xtrain, ytrain, split_perc=valid_perc)
+ train_loader, _ = make_loader(Xtrain, ytrain, batch_size=batch_size)
+ valid_loader, _ = make_loader(Xvalid, yvalid, batch_size=batch_size)
+ else:
+ train_loader, _ = make_loader(Xtrain, ytrain, batch_size=batch_size)
+ valid_loader = train_loader
+
+ print(f'{uci_folder_to_name(data_dir)}: {len(ytrain)} training samples loaded.')
+ Xtest, ytest, _ = load_uci_dataset(data_dir, False)
+ test_loader, _ = make_loader(Xtest, ytest, batch_size=batch_size)
+ print(f'{uci_folder_to_name(data_dir)}: {len(ytest)} testing samples loaded.')
+ train_loader.nclass = nclass
+ return train_loader, valid_loader, test_loader
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase1/algo/alg_base.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase1/algo/alg_base.py
new file mode 100644
index 00000000..a7d9fe11
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase1/algo/alg_base.py
@@ -0,0 +1,131 @@
+import math
+import time
+from abc import abstractmethod
+import torch
+from torch import nn
+
+
+class Evaluator:
+ def __init__(self):
+ pass
+
+ @abstractmethod
+ def evaluate(self, arch: nn.Module,
+ device: str,
+ batch_data: object, batch_labels: torch.Tensor,
+ space_name: str
+ ) -> float:
+ """
+ Score each architecture with predefined architecture and data
+ :param arch: architecture to be scored
+ :param device: cpu or gpu
+ :param batch_data: a mini batch of data, [ batch_size, channel, W, H ] or dict for structure data
+ :param batch_labels: a mini batch of labels
+ :param space_name: string
+ :return: score
+ """
+ raise NotImplementedError
+
+ def evaluate_wrapper(self, arch, device: str, space_name: str,
+ batch_data: torch.tensor,
+ batch_labels: torch.tensor) -> (float, float):
+ """
+ :param arch: architecture to be scored
+ :param device: cpu or GPU
+ :param space_name: search space name
+ :param batch_data: a mini batch of data, [ batch_size, channel, W, H ]
+ :param batch_labels: a mini batch of labels
+ :return: score, timeUsage
+ """
+
+ arch.train()
+ # arch.zero_grad()
+
+ # measure scoring time
+ if "cuda" in device:
+ torch.cuda.synchronize()
+ # use this will not need cuda.sync
+ # starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
+ # starter.record()
+ starter, ender = time.time(), time.time()
+ else:
+ starter, ender = time.time(), time.time()
+
+ # score
+ score = self.evaluate(arch, device, batch_data, batch_labels, space_name)
+
+ if "cuda" in device:
+ # ender.record()
+ # implicitly waits for the event to be marked as complete before calculating the time difference
+ # curr_time = starter.elapsed_time(ender)
+ torch.cuda.synchronize()
+ ender = time.time()
+ curr_time = ender - starter
+ else:
+ ender = time.time()
+ curr_time = ender - starter
+
+ if math.isnan(score):
+ if score > 0:
+ score = 1e8
+ else:
+ score = -1e8
+ if math.isinf(score):
+ if score > 0:
+ score = 1e8
+ else:
+ score = -1e8
+
+ return score, curr_time
+
+ def evaluate_wrapper_origin(self, arch, device: str, space_name: str,
+ batch_data: torch.tensor,
+ batch_labels: torch.tensor) -> (float, float):
+ """
+ :param arch: architecture to be scored
+ :param device: cpu or GPU
+ :param space_name: search space name
+ :param batch_data: a mini batch of data, [ batch_size, channel, W, H ]
+ :param batch_labels: a mini batch of labels
+ :return: score, timeUsage
+ """
+
+ arch.train()
+ arch.zero_grad()
+
+ # measure scoring time
+ if "cuda" in device:
+ torch.cuda.synchronize()
+ # use this will not need cuda.sync
+ # starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
+ # starter.record()
+ starter, ender = time.time(), time.time()
+ else:
+ starter, ender = time.time(), time.time()
+
+ # score
+ score = self.evaluate(arch, device, batch_data, batch_labels, space_name)
+
+ if "cuda" in device:
+ # ender.record()
+ # implicitly waits for the event to be marked as complete before calculating the time difference
+ # curr_time = starter.elapsed_time(ender)
+ torch.cuda.synchronize()
+ ender = time.time()
+ curr_time = ender - starter
+ else:
+ ender = time.time()
+ curr_time = ender - starter
+
+ if math.isnan(score):
+ if score > 0:
+ score = 1e8
+ else:
+ score = -1e8
+ if math.isinf(score):
+ if score > 0:
+ score = 1e8
+ else:
+ score = -1e8
+
+ return score, curr_time
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase1/concurrent_evaluator.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase1/concurrent_evaluator.py
new file mode 100644
index 00000000..c346a8ef
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase1/concurrent_evaluator.py
@@ -0,0 +1,199 @@
+# this is for checking the flops and params
+try:
+ from thop import profile
+except:
+ pass
+from src.common.constant import Config, CommonVars
+from src.common.structure import ModelAcquireData
+from src.eva_engine import evaluator_register
+from src.query_api.interface import SimulateScore
+from src.dataset_utils import dataset
+from torch.utils.data import DataLoader
+import torch
+import time
+from torch import nn
+from src.search_space.core.space import SpaceWrapper
+from multiprocessing import Manager
+import gc
+
+
+class ConcurrentP1Evaluator:
+
+ def __init__(self, device: str, num_label: int, dataset_name: str,
+ search_space_ins: SpaceWrapper,
+ train_loader: DataLoader, is_simulate: bool, metrics: str = CommonVars.ExpressFlow,
+ enable_cache: bool = False):
+ """
+ :param device:
+ :param num_label:
+ :param dataset_name:
+ :param search_space_ins:
+ :param search_space_ins:
+ :param train_loader:
+ :param is_simulate:
+ :param metrics: which TFMEM to use?
+ :param enable_cache: if cache embedding for scoring? only used on structued data
+ """
+ self.metrics = metrics
+ self.is_simulate = is_simulate
+
+ self.dataset_name = dataset_name
+
+ self.search_space_ins = search_space_ins
+
+ self.device = device
+ self.num_labels = num_label
+
+ self.score_getter = None
+
+ # get one mini batch
+ if not self.is_simulate:
+ if self.dataset_name in [Config.c10, Config.c100, Config.imgNet]:
+ # for img data
+ self.mini_batch, self.mini_batch_targets = dataset.get_mini_batch(
+ dataloader=train_loader,
+ sample_alg="random",
+ batch_size=32,
+ num_classes=self.num_labels)
+ self.mini_batch.to(self.device)
+ self.mini_batch_targets.to(self.device)
+ elif self.dataset_name in [Config.Criteo, Config.Frappe, Config.UCIDataset]:
+ # this is structure data
+ batch = iter(train_loader).__next__()
+ target = batch['y'].type(torch.LongTensor).to(self.device)
+ batch['id'] = batch['id'].to(self.device)
+ batch['value'] = batch['value'].to(self.device)
+ self.mini_batch = batch
+ self.mini_batch_targets = target.to(self.device)
+ else:
+ raise NotImplementedError
+
+ print("GC the large train data loader")
+ del train_loader
+ # Force garbage collection
+ gc.collect()
+
+ self.time_usage = {
+ "latency": 0.0,
+ "io_latency": 0.0,
+ "compute_latency": 0.0,
+ "track_compute": [], # compute time
+ "track_io_model_init": [], # init model weight
+ "track_io_model_load": [], # load into GPU/CPU
+ "track_io_model_release_each_50": [], # context switch
+ "track_io_data": [], # context switch
+ }
+
+ # this is to do the expeirment
+ self.enable_cache = enable_cache
+ if self.enable_cache:
+ # todo: warmup for concurrent usage. this is only test for MLP with embedding.
+ new_model = self.search_space_ins.new_arch_scratch_with_default_setting("8-8-8-8", bn=False)
+ new_model.init_embedding()
+ # shared embedding
+ manager = Manager()
+ self.model_cache = manager.dict()
+ self.model_cache["model"] = new_model.embedding
+ self.get_cache_data = self._get_cache_data_enabled
+ self.set_cache_data = self._set_cache_data_enabled
+ else:
+ # this is the baseline, independently run
+ self.get_cache_data = self._get_cache_data_disabled
+ self.set_cache_data = self._set_cache_data_disabled
+
+ def _get_cache_data_enabled(self):
+ return self.model_cache["model"]
+
+ def _set_cache_data_enabled(self, data):
+ self.model_cache["model"] = data
+
+ def _get_cache_data_disabled(self):
+ return None
+
+ def _set_cache_data_disabled(self, data):
+ pass
+
+ def if_cuda_avaiable(self):
+ if "cuda" in self.device:
+ return True
+ else:
+ return False
+
+ def p1_evaluate(self, data_str: str) -> dict:
+ """
+ :param data_str: encoded ModelAcquireData
+ :return:
+ """
+
+ model_acquire = ModelAcquireData.deserialize(data_str)
+ return self._p1_evaluate_online(model_acquire)
+
+ def _p1_evaluate_online(self, model_acquire: ModelAcquireData) -> dict:
+
+ model_encoding = model_acquire.model_encoding
+
+ # score using only one metrics
+ if self.metrics == CommonVars.PRUNE_SYNFLOW or self.metrics == CommonVars.ExpressFlow:
+ bn = False
+ else:
+ bn = True
+
+ # measure model load time
+ begin = time.time()
+ new_model = self.search_space_ins.new_arch_scratch_with_default_setting(model_encoding, bn=bn)
+
+ # mlp have embedding layer, which can be cached, optimization!
+ if self.search_space_ins.name == Config.MLPSP:
+ if self.enable_cache:
+ new_model.init_embedding(self.get_cache_data())
+ if self.get_cache_data() is None:
+ self.set_cache_data(new_model.embedding.to(self.device))
+ else:
+ new_model.init_embedding()
+
+ self.time_usage["track_io_model_init"].append(time.time() - begin)
+
+ begin = time.time()
+ new_model = new_model.to(self.device)
+ if self.if_cuda_avaiable():
+ torch.cuda.synchronize()
+ self.time_usage["track_io_model_load"].append(time.time() - begin)
+
+ # measure data load time
+ begin = time.time()
+ mini_batch = self.data_pre_processing(self.metrics, new_model)
+ self.time_usage["track_io_data"].append(time.time() - begin)
+
+ _score, curr_time = evaluator_register[self.metrics].evaluate_wrapper(
+ arch=new_model,
+ device=self.device,
+ space_name=self.search_space_ins.name,
+ batch_data=mini_batch,
+ batch_labels=self.mini_batch_targets)
+
+ self.time_usage["track_compute"].append(curr_time)
+
+ del new_model
+ model_score = {self.metrics: _score}
+ return model_score
+
+ def data_pre_processing(self, metrics: str, new_model: nn.Module):
+ """
+ To measure the io/compute time more acccuretely, we pick the data pre_processing here.
+ """
+
+ # for those two metrics, we use all one embedding for efficiency (as in their paper)
+ if metrics in [CommonVars.ExpressFlow, CommonVars.PRUNE_SYNFLOW]:
+ if isinstance(self.mini_batch, torch.Tensor):
+ feature_dim = list(self.mini_batch[0, :].shape)
+ # add one dimension to feature dim, [1] + [3, 32, 32] = [1, 3, 32, 32]
+ mini_batch = torch.ones([1] + feature_dim).float().to(self.device)
+ else:
+ # this is for the tabular data,
+ mini_batch = new_model.generate_all_ones_embedding().float().to(self.device)
+ else:
+ mini_batch = self.mini_batch
+
+ if self.if_cuda_avaiable():
+ torch.cuda.synchronize()
+ return mini_batch
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase1/evaluator.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase1/evaluator.py
new file mode 100644
index 00000000..30e1e244
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase1/evaluator.py
@@ -0,0 +1,393 @@
+# this is for checking the flops and params
+try:
+ from thop import profile
+except:
+ pass
+from src.common.constant import Config, CommonVars
+from src.common.structure import ModelAcquireData
+from src.eva_engine import evaluator_register
+from src.query_api.interface import SimulateScore
+from src.dataset_utils import dataset
+from torch.utils.data import DataLoader
+import torch
+import time
+from torch import nn
+from src.search_space.core.space import SpaceWrapper
+import psycopg2
+from typing import Any, List, Dict, Tuple
+from src.logger import logger
+
+
+class P1Evaluator:
+
+ def __init__(self, device: str, num_label: int, dataset_name: str,
+ search_space_ins: SpaceWrapper,
+ train_loader: DataLoader, is_simulate: bool, metrics: str = CommonVars.ExpressFlow,
+ enable_cache: bool = False, db_config: Dict = None,
+ data_retrievel: str = "sql"):
+ """
+ :param device:
+ :param num_label:
+ :param dataset_name:
+ :param search_space_ins:
+ :param search_space_ins:
+ :param train_loader:
+ :param is_simulate:
+ :param metrics: which TFMEM to use?
+ :param enable_cache: if cache embedding for scoring? only used on structued data
+ :param db_config: how to connect to databaes
+ :param data_retrievel: sql or spi
+ """
+ self.metrics = metrics
+ self.is_simulate = is_simulate
+ # used only is_simulate = True
+ self.score_getter = None
+
+ # dataset settings
+ self.dataset_name = dataset_name
+ self.train_loader = train_loader
+ self.num_labels = num_label
+
+ self.search_space_ins = search_space_ins
+
+ self.device = device
+
+ # this is to do the expeirment
+ self.enable_cache = enable_cache
+ self.model_cache = None
+
+ # performance records
+ self.time_usage = {
+ "latency": 0.0,
+ "io_latency": 0.0,
+ "compute_latency": 0.0,
+
+ "track_compute": [], # compute time
+ "track_io_model_init": [], # init model weight
+ "track_io_model_load": [], # load model into GPU/CPU
+ "track_io_res_load": [], # load result into GPU/CPU
+ "track_io_model_release_each_50": [], # context switch
+ "track_io_model_release": [], # context switch
+ "track_io_data_retrievel": [], # context switch
+ "track_io_data_preprocess": [], # context switch
+ }
+
+ self.db_config = db_config
+ self.last_id = -1
+ self.data_retrievel = data_retrievel
+
+ def if_cuda_avaiable(self):
+ if "cuda" in self.device:
+ return True
+ else:
+ return False
+
+ def p1_evaluate(self, data_str: str) -> dict:
+ """
+ :param data_str: encoded ModelAcquireData
+ :return:
+ """
+
+ model_acquire = ModelAcquireData.deserialize(data_str)
+
+ if self.is_simulate:
+ if self.metrics == "jacflow":
+ return self._p1_evaluate_simu_jacflow(model_acquire)
+ else:
+ return self._p1_evaluate_simu(model_acquire)
+ else:
+ return self._p1_evaluate_online(model_acquire)
+
+ def measure_model_flops(self, data_str: str, batch_size: int, channel_size: int):
+ # todo: check the package
+ mini_batch, mini_batch_targets, _ = self.retrievel_data(None)
+ model_acquire = ModelAcquireData.deserialize(data_str)
+ model_encoding = model_acquire.model_encoding
+ new_model = self.search_space_ins.new_arch_scratch_with_default_setting(model_encoding, bn=True)
+ if self.search_space_ins.name == Config.MLPSP:
+ new_model.init_embedding(requires_grad=True)
+ new_model = new_model.to(self.device)
+ flops, params = profile(new_model, inputs=(mini_batch,))
+ print('FLOPs = ' + str(flops / 1000 ** 3) + 'G')
+ print('Params = ' + str(params / 1000 ** 2) + 'M')
+
+ # # 1. Score NasWot
+ # new_model = self.search_space_ins.new_arch_scratch_with_default_setting(model_encoding, bn=True)
+ # new_model = new_model.to(self.device)
+ # naswot_score, _ = evaluator_register[CommonVars.NAS_WOT].evaluate_wrapper(
+ # arch=new_model,
+ # device=self.device,
+ # space_name = self.search_space_ins.name,
+ # batch_data=self.mini_batch,
+ # batch_labels=self.mini_batch_targets)
+ #
+ # # 2. Score SynFlow
+ # new_model = self.search_space_ins.new_arch_scratch_with_default_setting(model_encoding, bn=False)
+ # new_model = new_model.to(self.device)
+ # synflow_score, _ = evaluator_register[CommonVars.PRUNE_SYNFLOW].evaluate_wrapper(
+ # arch=new_model,
+ # device=self.device,
+ # space_name = self.search_space_ins.name,
+ # batch_data=self.mini_batch,
+ # batch_labels=self.mini_batch_targets)
+ #
+ # # 3. combine the result and return
+ # model_score = {CommonVars.NAS_WOT: naswot_score,
+ # CommonVars.PRUNE_SYNFLOW: synflow_score}
+
+ def _p1_evaluate_online(self, model_acquire: ModelAcquireData) -> dict:
+
+ model_encoding = model_acquire.model_encoding
+
+ # 1. Get a batch of data
+ mini_batch, mini_batch_targets, data_load_time_usage, data_pre_process_time = self.retrievel_data(model_acquire)
+ self.time_usage["track_io_data_retrievel"].append(data_load_time_usage)
+
+ # 2. Score all tfmem
+ if self.metrics == CommonVars.ALL_EVALUATOR:
+ model_score = {}
+ for alg, score_evaluator in evaluator_register.items():
+ if alg == CommonVars.PRUNE_SYNFLOW or alg == CommonVars.ExpressFlow:
+ bn = False
+ else:
+ bn = True
+ new_model = self.search_space_ins.new_arch_scratch_with_default_setting(model_encoding, bn=bn)
+ if self.search_space_ins.name == Config.MLPSP:
+ new_model.init_embedding()
+ new_model = new_model.to(self.device)
+
+ mini_batch = self.data_pre_processing(mini_batch, self.metrics, new_model)
+
+ _score, _ = score_evaluator.evaluate_wrapper(
+ arch=new_model,
+ device=self.device,
+ space_name=self.search_space_ins.name,
+ batch_data=mini_batch,
+ batch_labels=mini_batch_targets)
+
+ _score = _score.item()
+ model_score[alg] = abs(_score)
+
+ # clear the cache
+ if "cuda" in self.device:
+ torch.cuda.empty_cache()
+
+ # 2. score using only one metrics
+ else:
+ if self.metrics == CommonVars.PRUNE_SYNFLOW or self.metrics == CommonVars.ExpressFlow:
+ bn = False
+ else:
+ bn = True
+ # measure model load time
+ begin = time.time()
+ new_model = self.search_space_ins.new_arch_scratch_with_default_setting(model_encoding, bn=bn)
+
+ # mlp have embedding layer, which can be cached, optimization!
+ # if self.search_space_ins.name == Config.MLPSP:
+ # if self.enable_cache:
+ # new_model.init_embedding(self.model_cache)
+ # if self.model_cache is None:
+ # self.model_cache = new_model.embedding.to(self.device)
+ # else:
+ # # init embedding every time created a new model
+ # new_model.init_embedding()
+
+ self.time_usage["track_io_model_init"].append(time.time() - begin)
+
+ if self.if_cuda_avaiable():
+ begin = time.time()
+ new_model = new_model.to(self.device)
+ torch.cuda.synchronize()
+ self.time_usage["track_io_model_load"].append(time.time() - begin)
+ else:
+ self.time_usage["track_io_model_load"].append(0)
+
+ # measure data load time
+ begin = time.time()
+ mini_batch = self.data_pre_processing(mini_batch, self.metrics, new_model)
+ self.time_usage["track_io_data_preprocess"].append(data_pre_process_time + time.time() - begin)
+
+ _score, compute_time = evaluator_register[self.metrics].evaluate_wrapper(
+ arch=new_model,
+ device=self.device,
+ space_name=self.search_space_ins.name,
+ batch_data=mini_batch,
+ batch_labels=mini_batch_targets)
+
+ self.time_usage["track_compute"].append(compute_time)
+
+ if self.if_cuda_avaiable():
+ begin = time.time()
+ _score = _score.item()
+ torch.cuda.synchronize()
+ self.time_usage["track_io_res_load"].append(time.time() - begin)
+
+ else:
+ _score = _score.item()
+ self.time_usage["track_io_res_load"].append(0)
+
+ model_score = {self.metrics: abs(_score)}
+ del new_model
+ return model_score
+
+ def _p1_evaluate_simu_jacflow(self, model_acquire: ModelAcquireData) -> dict:
+ """
+ This involves get rank, and get jacflow
+ """
+ if self.score_getter is None:
+ self.score_getter = SimulateScore(space_name=self.search_space_ins.name,
+ dataset_name=self.dataset_name)
+
+ model_score = self.score_getter.query_tfmem_rank_score(arch_id=model_acquire.model_id)
+
+ return model_score
+
+ def _p1_evaluate_simu(self, model_acquire: ModelAcquireData) -> dict:
+ """
+ This involves simulate get alls core,
+ """
+ if self.score_getter is None:
+ self.score_getter = SimulateScore(space_name=self.search_space_ins.name,
+ dataset_name=self.dataset_name)
+
+ score = self.score_getter.query_all_tfmem_score(arch_id=model_acquire.model_id)
+ model_score = {self.metrics: abs(score[self.metrics])}
+ return model_score
+
+ def retrievel_data(self, model_acquire):
+ if not self.is_simulate:
+ if self.dataset_name in [Config.c10, Config.c100, Config.imgNet]:
+ if self.train_loader is None:
+ raise f"self.train_loader is None for {self.dataset_name}"
+ # for img data
+ begin = time.time()
+ mini_batch, mini_batch_targets = dataset.get_mini_batch(
+ dataloader=self.train_loader,
+ sample_alg="random",
+ batch_size=32,
+ num_classes=self.num_labels)
+ mini_batch.to(self.device)
+ mini_batch_targets.to(self.device)
+ # wait for moving data to GPU
+ if self.if_cuda_avaiable():
+ torch.cuda.synchronize()
+ time_usage = time.time() - begin
+ # todo: here is inaccurate
+ return mini_batch, mini_batch_targets, time_usage, 0
+ elif self.dataset_name in [Config.Criteo, Config.Frappe, Config.UCIDataset]:
+ if self.train_loader is None:
+ if self.data_retrievel == "sql":
+ batch, time_usage = self._retrievel_from_db_sql()
+ data_tensor, y_tensor, process_time = self.batch_data_pre_processing(batch)
+ return data_tensor, y_tensor, time_usage, process_time
+ elif self.data_retrievel == "spi":
+ batch, time_usage = self._retrievel_from_db_spi(model_acquire)
+ data_tensor, y_tensor, process_time = self.batch_data_pre_processing(batch)
+ return data_tensor, y_tensor, time_usage, process_time
+ else:
+ # this is structure data
+ begin = time.time()
+ batch = iter(self.train_loader).__next__()
+ target = batch['y'].type(torch.LongTensor).to(self.device)
+ batch['id'] = batch['id'].to(self.device)
+ batch['value'] = batch['value'].to(self.device)
+
+ # wait for moving data to GPU
+ if self.if_cuda_avaiable():
+ torch.cuda.synchronize()
+ time_usage = time.time() - begin
+ return batch, target, time_usage, 0
+ else:
+ raise NotImplementedError
+
+ def _retrievel_from_db_sql(self):
+
+ begin_time = time.time()
+ with psycopg2.connect(database=self.db_config["db_name"],
+ user=self.db_config["db_user"],
+ host=self.db_config["db_host"],
+ port=self.db_config["db_port"]) as conn:
+ # fetch and preprocess data from PostgreSQL
+ cur = conn.cursor()
+
+ cur.execute(f"SELECT * FROM {self.dataset_name}_train WHERE id > {self.last_id} LIMIT 32")
+ rows = cur.fetchall()
+
+ if rows:
+ # Update last_id with max id of fetched rows
+ self.last_id = max(row[0] for row in rows) # assuming 'id' is at index 0
+ else:
+ # If no more new rows, reset last_id to start over scan and return 'end_position'
+ self.last_id = -1
+
+ # block until a free slot is available
+ time_usage = time.time() - begin_time
+ return rows, time_usage
+
+ def _retrievel_from_db_spi(self, model_acquire):
+ batch = model_acquire.spi_mini_batch
+ time_usage = model_acquire.spi_seconds
+ return batch, time_usage
+
+ def data_pre_processing(self, mini_batch, metrics: str, new_model: nn.Module):
+
+ # for those two metrics, we use all one embedding for efficiency (as in their paper)
+ if metrics in [CommonVars.ExpressFlow, CommonVars.PRUNE_SYNFLOW]:
+ if isinstance(mini_batch, torch.Tensor):
+ feature_dim = list(mini_batch[0, :].shape)
+ # add one dimension to feature dim, [1] + [3, 32, 32] = [1, 3, 32, 32]
+ mini_batch = torch.ones([1] + feature_dim).float().to(self.device)
+ else:
+ # this is for the tabular data,
+ mini_batch = new_model.generate_all_ones_embedding().float().to(self.device)
+ else:
+ # for others, skip preprocessing
+ pass
+
+ # wait for moving data to GPU
+ if self.if_cuda_avaiable():
+ torch.cuda.synchronize()
+ return mini_batch
+
+ def batch_data_pre_processing(self, queryed_rows: List[Tuple]):
+ """
+ mini_batch_data: [('0', '0', '123:123', '123:123', '123:123',)
+ """
+
+ def decode_libsvm(columns):
+ map_func = lambda pair: (int(pair[0]), float(pair[1]))
+ # 0 is id, 1 is label
+ id, value = zip(*map(lambda col: map_func(col.split(':')), columns[2:]))
+ sample = {'id': list(id),
+ 'value': list(value),
+ 'y': int(columns[1])}
+ return sample
+
+ def pre_processing(mini_batch_data: List[Tuple]):
+ """
+ mini_batch_data: [('0', '0', '123:123', '123:123', '123:123',)
+ """
+ sample_lines = len(mini_batch_data)
+ feat_id = []
+ feat_value = []
+ y = []
+
+ for i in range(sample_lines):
+ row_value = mini_batch_data[i]
+ sample = decode_libsvm(list(row_value))
+ feat_id.append(sample['id'])
+ feat_value.append(sample['value'])
+ y.append(sample['y'])
+ return {'id': feat_id, 'value': feat_value, 'y': y}
+
+ begin = time.time()
+ batch = pre_processing(queryed_rows)
+ id_tensor = torch.FloatTensor(batch['id']).to(self.device)
+ value_tensor = torch.FloatTensor(batch['value']).to(self.device)
+ y_tensor = torch.LongTensor(batch['y']).to(self.device)
+ data_tensor = {'id': id_tensor, 'value': value_tensor, 'y': y_tensor}
+ # wait for moving data to GPU
+ if self.if_cuda_avaiable():
+ torch.cuda.synchronize()
+ duration = time.time() - begin
+ return data_tensor, y_tensor, duration
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase1/run_phase1.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase1/run_phase1.py
new file mode 100644
index 00000000..8389c72a
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase1/run_phase1.py
@@ -0,0 +1,145 @@
+import json
+
+from src.common.structure import ModelAcquireData, ModelEvaData
+from src.controller.controler import SampleController
+from src.controller.sampler_all.seq_sampler import SequenceSampler
+
+from src.eva_engine.phase1.evaluator import P1Evaluator
+from src.logger import logger
+from src.query_api.img_explore_ea import fetch_from_db
+from torch.utils.data import DataLoader
+from src.controller.sampler_ea.regularized_ea import RegularizedEASampler
+from src.search_space.core.space import SpaceWrapper
+
+
+# this is for image only
+def p1_evaluate_query(space_name, dataset, run_id, N, K) -> (list, float):
+ """
+ :param space_name:
+ :param dataset:
+ :param run_id:
+ :param N:
+ :param K:
+ :return: return list of models and time usage.
+ """
+ arch_id, candidates, current_time = fetch_from_db(space_name, dataset, run_id, N)
+ return candidates[-K:], current_time
+
+
+class RunPhase1:
+
+ def __init__(self, args, K: int, N: int, search_space_ins: SpaceWrapper,
+ train_loader: DataLoader = None, is_simulate: bool = False):
+ """
+ Each model selection job will init one class here.
+ :param args: space, population_size, sample_size
+ :param K: K models return in 1st phase
+ :param N: N models eval in total
+ :param search_space_ins:
+ """
+
+ # return K models
+ self.K = K
+ # explore N models
+ self.N = N
+
+ self.args = args
+
+ self.search_space_ins = search_space_ins
+
+ # seq: init the search strategy and controller,
+ if self.N >= len(self.search_space_ins):
+ print("Explore all models")
+ strategy = SequenceSampler(self.search_space_ins)
+ else:
+ strategy = RegularizedEASampler(self.search_space_ins,
+ population_size=self.args.population_size,
+ sample_size=self.args.sample_size)
+ self.sampler = SampleController(strategy)
+
+ # generate db config
+ db_config = {
+ "db_name": self.args.db_name,
+ "db_user": self.args.db_user,
+ "db_host": self.args.db_host,
+ "db_port": self.args.db_port,
+ }
+
+ # seq: init the phase 1 evaluator,
+ self._evaluator = P1Evaluator(device=self.args.device,
+ num_label=self.args.num_labels,
+ dataset_name=self.args.dataset,
+ search_space_ins=self.search_space_ins,
+ train_loader=train_loader,
+ is_simulate=is_simulate,
+ metrics=self.args.tfmem,
+ db_config=db_config)
+
+ def run_phase1(self) -> (list, list, list, list):
+ """
+ Controller explore n models, and return the top K models.
+ :return:
+ """
+
+ # those two are used to track performance trace
+ # current best model id
+ trace_highest_scored_models_id = []
+ # current highest score
+ trace_highest_score = []
+ explored_n = 1
+ model_eva = ModelEvaData()
+
+ while explored_n <= self.N:
+ # generate new model
+ arch_id, arch_micro = self.sampler.sample_next_arch()
+ # this is for sequence sampler.
+ if arch_id is None:
+ break
+ model_encoding = self.search_space_ins.serialize_model_encoding(arch_micro)
+
+ explored_n += 1
+
+ # run the model selection
+ model_acquire_data = ModelAcquireData(model_id=str(arch_id),
+ model_encoding=model_encoding,
+ is_last=False)
+ data_str = model_acquire_data.serialize_model()
+
+ # update the shared model eval res
+ try:
+ model_eva.model_id = str(arch_id)
+ model_eva.model_score = self._evaluator.p1_evaluate(data_str)
+ except KeyError:
+ # when it is simulate, it could be keyerror
+ continue
+
+ if explored_n % 100 == 0:
+ logger.info("3. [trails] Phase 1: filter phase explored " + str(explored_n) +
+ " model, model_id = " + model_eva.model_id +
+ " model_scores = " + json.dumps(model_eva.model_score))
+ print("3. [trails] Phase 1: filter phase explored " + str(explored_n) +
+ " model, model_id = " + model_eva.model_id +
+ " model_scores = " + json.dumps(model_eva.model_score))
+
+ ranked_score = self.sampler.fit_sampler(model_eva.model_id,
+ model_eva.model_score,
+ simple_score_sum=self.args.simple_score_sum)
+
+ # this is to measure the value of metrix, sum of two value.
+ if len(trace_highest_score) == 0:
+ trace_highest_score.append(ranked_score)
+ trace_highest_scored_models_id.append(str(arch_id))
+ else:
+ if ranked_score > trace_highest_score[-1]:
+ trace_highest_score.append(ranked_score)
+ trace_highest_scored_models_id.append(str(arch_id))
+ else:
+ trace_highest_score.append(trace_highest_score[-1])
+ trace_highest_scored_models_id.append(trace_highest_scored_models_id[-1])
+
+ logger.info("3. [trails] Phase 1: filter phase explored " + str(explored_n) +
+ " model, model_id = " + model_eva.model_id +
+ " model_scores = " + json.dumps(model_eva.model_score))
+ # return the top K models
+ return self.sampler.get_current_top_k_models(self.K), self.sampler.get_current_top_k_models(-1), \
+ trace_highest_score, trace_highest_scored_models_id
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase2/evaluator.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase2/evaluator.py
new file mode 100644
index 00000000..d304d787
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase2/evaluator.py
@@ -0,0 +1,82 @@
+from src.common.constant import Config
+from src.eva_engine.phase2.algo.trainer import ModelTrainer
+from src.logger import logger
+from src.query_api.interface import SimulateTrain
+from src.search_space.core.space import SpaceWrapper
+from torch.utils.data import DataLoader
+
+
+class P2Evaluator:
+
+ def __init__(self,
+ search_space_ins: SpaceWrapper,
+ dataset: str,
+ is_simulate: bool = True,
+ train_loader: DataLoader = None,
+ val_loader: DataLoader = None,
+ args=None):
+ """
+ :param search_space_ins:
+ :param dataset:
+ :param is_simulate: train or not, default query from API.
+ """
+ self.search_space_ins = search_space_ins
+
+ # dataset name
+ self.dataset = dataset
+ self.is_simulate = is_simulate
+ self.acc_getter = None
+
+ # for training only
+ self.train_loader = train_loader
+ self.val_loader = val_loader
+ self.args = args
+
+ def p2_evaluate(self, cand: str, epoch_per_model: int) -> (float, float):
+ """
+ :param cand: candidate id
+ :param epoch_per_model: epoch for each model
+ :return:
+ """
+ # if it's simulate or it's image dataset
+ if self.is_simulate or self.search_space_ins.name in [Config.NB101, Config.NB201]:
+ return self._evaluate_query(cand, epoch_per_model)
+ else:
+ return self._evaluate_train(cand, epoch_per_model)
+
+ def _evaluate_query(self, cand: str, epoch_per_model: int) -> (float, float):
+ """
+ :param cand: the candidate to evaluate
+ :param epoch_per_model: how many resource it can use, epoch number
+ :return:
+ """
+ if self.acc_getter is None:
+ self.acc_getter = SimulateTrain(space_name=self.search_space_ins.name)
+
+ acc, time_usage = self.acc_getter.get_ground_truth(arch_id=cand, epoch_num=epoch_per_model, dataset=self.dataset)
+
+ return acc, time_usage
+
+ def _evaluate_train(self, cand: str, epoch_per_model: int) -> (float, float):
+ """
+ :param cand: the candidate to evaluate
+ :param epoch_per_model: how many resource it can use, epoch number
+ :return:
+ """
+ model = self.search_space_ins.new_architecture(cand)
+ if self.search_space_ins.name == Config.MLPSP:
+ model.init_embedding(cached_embedding=None, requires_grad=True)
+ model.to(self.args.device)
+ valid_auc, total_run_time, train_log = ModelTrainer.fully_train_arch(
+ model=model,
+ use_test_acc=False,
+ epoch_num=epoch_per_model,
+ train_loader=self.train_loader,
+ val_loader=self.val_loader,
+ test_loader=self.val_loader,
+ args=self.args)
+
+ logger.info(f' ----- model id: {cand}, Val_AUC : {valid_auc} Total running time: '
+ f'{total_run_time}-----')
+
+ return valid_auc, total_run_time
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase2/run_sh.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase2/run_sh.py
new file mode 100644
index 00000000..ef038737
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/phase2/run_sh.py
@@ -0,0 +1,156 @@
+from copy import copy
+
+from src.common.constant import Config
+from src.eva_engine.phase2.evaluator import P2Evaluator
+
+# successive halving
+from src.logger import logger
+from src.search_space.core.space import SpaceWrapper
+from torch.utils.data import DataLoader
+
+
+class BudgetAwareControllerSH:
+
+ @staticmethod
+ def pre_calculate_epoch_required(eta: int, max_unit_per_model: int, K: int, U: int):
+ if K == 1:
+ return 0
+
+ cur_cand_num = K
+ cur_epoch = min(U, max_unit_per_model) # Limit the current epoch to max_unit_per_model
+ total_epochs = 0
+
+ while cur_cand_num > 1 and cur_epoch < max_unit_per_model:
+ total_epochs += cur_cand_num * cur_epoch
+ # Prune models
+ cur_cand_num = int(cur_cand_num * (1 / eta))
+ # Increase the training epoch for the remaining models
+ cur_epoch = min(cur_epoch * eta, max_unit_per_model)
+
+ # If the models are fully trained and there is more than one candidate, add these final evaluations to the total
+ if cur_cand_num > 1 and cur_epoch >= max_unit_per_model:
+ total_epochs += cur_cand_num * max_unit_per_model
+
+ return total_epochs
+
+ def __init__(self,
+ search_space_ins: SpaceWrapper, dataset_name: str,
+ eta, time_per_epoch,
+ train_loader: DataLoader = None,
+ val_loader: DataLoader = None,
+ args=None,
+ is_simulate: bool = True):
+ """
+ :param search_space_ins:
+ :param dataset_name:
+ :param time_per_epoch:
+ :param is_simulate:
+ :param eta: 1/mu to keep in each iteration
+ """
+ self.is_simulate = is_simulate
+ self._evaluator = P2Evaluator(search_space_ins, dataset_name,
+ is_simulate=is_simulate,
+ train_loader=train_loader, val_loader=val_loader,
+ args=args)
+ self.eta = eta
+ self.max_unit_per_model = args.epoch
+ self.time_per_epoch = time_per_epoch
+ self.name = "SUCCHALF"
+
+ def schedule_budget_per_model_based_on_T(self, space_name, fixed_time_budget, K_):
+ # for benchmarking only phase 2
+
+ # try different K and U combinations
+ # only consider 15625 arches in this paper
+ # min_budget_required: when K = 1, N = min_budget_required * 1
+ if space_name == Config.NB101:
+ U_options = [4, 12, 16, 108]
+ else:
+ U_options = list(range(1, 200))
+
+ history = []
+
+ for U in U_options:
+ real_time_used = \
+ BudgetAwareControllerSH.pre_calculate_epoch_required(
+ self.eta, self.max_unit_per_model, K_, U) * self.time_per_epoch
+
+ if real_time_used > fixed_time_budget:
+ break
+ else:
+ history.append(U)
+ if len(history) == 0:
+ raise f"{fixed_time_budget} is too small for current config"
+ return history[-1]
+
+ def pre_calculate_time_required(self, K, U):
+ all_epoch = BudgetAwareControllerSH.pre_calculate_epoch_required(self.eta, self.max_unit_per_model, K, U)
+ return all_epoch, all_epoch * self.time_per_epoch
+
+ def run_phase2(self, U: int, candidates_m: list) -> (str, float, float):
+ if len(candidates_m) == 1:
+ best_perform, _ = self._evaluator.p2_evaluate(candidates_m[0], self.max_unit_per_model)
+ return candidates_m[0], best_perform, 0
+
+ eta = self.eta
+ max_unit_per_model = self.max_unit_per_model
+
+ cur_cand_num = len(candidates_m)
+ cur_epoch = min(U, max_unit_per_model) # Limit the current epoch to max_unit_per_model
+ total_epochs = 0
+
+ while cur_cand_num > 1 and cur_epoch < max_unit_per_model:
+ logger.info(f"4. [trails] Running phase2: train {len(candidates_m)} models each with {cur_epoch} epochs")
+ scores = []
+ # Evaluate all models
+ for cand in candidates_m:
+ score, _ = self._evaluator.p2_evaluate(cand, cur_epoch)
+ scores.append((score, cand))
+ total_epochs += cur_epoch
+
+ # Sort models based on score
+ scores.sort(reverse=True, key=lambda x: x[0])
+
+ # Prune models, at lease keep one model
+ cur_cand_num = max(int(cur_cand_num * (1 / eta)), 1)
+ candidates_m = [x[1] for x in scores[:cur_cand_num]]
+
+ # Increase the training epoch for the remaining models
+ cur_epoch = min(cur_epoch * eta, max_unit_per_model)
+
+ # If the models are fully trained and there is more than one candidate, select the top one
+ if cur_cand_num > 1 and cur_epoch >= max_unit_per_model:
+ logger.info(
+ f"4. [trails] Running phase2: train {len(candidates_m)} models each with {max_unit_per_model} epochs")
+ scores = []
+ for cand in candidates_m:
+ score, _ = self._evaluator.p2_evaluate(cand, max_unit_per_model)
+ scores.append((score, cand))
+ total_epochs += cur_epoch
+ scores.sort(reverse=True, key=lambda x: x[0])
+ candidates_m = [scores[0][1]]
+
+ # only return the performance when simulating, skip the training, just return model
+ if self.is_simulate:
+ logger.info(
+ f"5. [trails] Phase2 Done, Select {candidates_m[0]}, "
+ f"simulate={self.is_simulate}. Acqure the ground truth")
+ best_perform, _ = self._evaluator.p2_evaluate(candidates_m[0], self.max_unit_per_model)
+ else:
+ logger.info(
+ f"5. [trails] Phase2 Done, Select {candidates_m[0]}, "
+ f"simulate={self.is_simulate}, Skip training")
+ best_perform = 0
+ # Return the best model and the total epochs used
+ return candidates_m[0], best_perform, total_epochs
+
+
+if __name__ == "__main__":
+ 'frappe: 20, uci_diabetes: 40, criteo: 10'
+ 'nb101: 108, nb201: 200'
+ k_options = [1, 2, 4, 8, 16]
+ u_options = [1, 2, 4, 8, 16]
+ print(f"k={10}, u={8}, total_epoch = {BudgetAwareControllerSH.pre_calculate_epoch_required(3, 20, 10, 8)}")
+ for k in k_options:
+ for u in u_options:
+ print(f"k={k}, u={u}, total_epoch = {BudgetAwareControllerSH.pre_calculate_epoch_required(3, 20, k, u)}")
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/run_ms.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/run_ms.py
new file mode 100644
index 00000000..9e1d08e0
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/eva_engine/run_ms.py
@@ -0,0 +1,309 @@
+import time
+
+from typing import Set, List
+
+from src.eva_engine import coordinator
+from src.eva_engine.phase1.run_phase1 import RunPhase1, p1_evaluate_query
+from torch.utils.data import DataLoader
+from src.eva_engine.phase2.run_sh import BudgetAwareControllerSH
+from src.logger import logger
+from src.search_space.init_search_space import init_search_space
+from src.query_api.interface import profile_NK_trade_off
+
+
+class RunModelSelection:
+
+ def __init__(self, search_space_name: str, args, is_simulate: bool = False):
+ self.args = args
+
+ self.eta = 3
+ self.is_simulate = is_simulate
+ # basic
+ self.search_space_name = search_space_name
+ self.dataset = self.args.dataset
+
+ # p2 evaluator
+ self.sh = None
+
+ # instance of the search space.
+ self.search_space_ins = init_search_space(self.args)
+
+ def select_model_simulate(self, budget: float, run_id: int = 0, only_phase1: bool = False, run_workers: int = 1):
+ """
+ This is for image data only
+ """
+
+ # 0. profiling dataset and search space, get t1 and t2
+
+ score_time_per_model, train_time_per_epoch, N_K_ratio = self.search_space_ins.profiling(self.dataset)
+ self.sh = BudgetAwareControllerSH(
+ search_space_ins=self.search_space_ins,
+ dataset_name=self.dataset,
+ eta=self.eta,
+ time_per_epoch=train_time_per_epoch,
+ args=self.args,
+ is_simulate=self.is_simulate)
+
+ # 1. run coordinator to schedule
+ K, U, N, B1_planed_time, B2_planed_time, B2_all_epoch = coordinator.schedule(self.dataset, self.sh, budget,
+ score_time_per_model,
+ train_time_per_epoch,
+ run_workers,
+ self.search_space_ins,
+ N_K_ratio,
+ only_phase1)
+
+ print(f"Budget = {budget}, N={N}, K={K}")
+
+ # 2. run phase 1 to score N models
+ k_models, B1_actual_time_use = p1_evaluate_query(self.search_space_name, self.dataset, run_id, N, K)
+
+ # 3. run phase-2 to determine the final model
+ best_arch, best_arch_performance, B2_actual_epoch_use = self.sh.run_phase2(U, k_models)
+ # print("best model returned from Phase2 = ", k_models)
+
+ return best_arch, B1_actual_time_use + B2_actual_epoch_use * train_time_per_epoch, \
+ B1_planed_time + B2_planed_time, B2_all_epoch
+
+ def select_model_online_clean(self, budget: float, data_loader: List[DataLoader],
+ only_phase1: bool = False, run_workers: int = 1):
+ """
+ Select model online for structured data.
+ :param budget: time budget
+ :param data_loader: time budget
+ :param only_phase1:
+ :param run_workers:
+ :return:
+ """
+ begin_time = time.time()
+ score_time_per_model = self.profile_filtering(data_loader)
+ train_time_per_epoch = self.profile_refinement(data_loader)
+ K, U, N = self.coordination(budget, score_time_per_model, train_time_per_epoch, only_phase1)
+ k_models, all_models, p1_trace_highest_score, p1_trace_highest_scored_models_id = self.filtering_phase(
+ N, K, train_loader=data_loader[0])
+ best_arch, best_arch_performance, _ = self.refinement_phase(
+ U, k_models, data_loader[0], data_loader[1])
+
+ end_time = time.time()
+ real_time_usage = end_time - begin_time
+
+ return best_arch, best_arch_performance, real_time_usage, all_models, \
+ p1_trace_highest_score, p1_trace_highest_scored_models_id
+
+ def select_model_online(self, budget: float, data_loader: List[DataLoader],
+ only_phase1: bool = False, run_workers: int = 1):
+ """
+ Select model online for structured data.
+ :param budget: time budget
+ :param data_loader: time budget
+ :param only_phase1:
+ :param run_workers:
+ :return:
+ """
+
+ train_loader, valid_loader, test_loader = data_loader
+
+ logger.info(f"0. [trails] Begin model selection, is_simulate={self.is_simulate} ... ")
+ begin_time = time.time()
+
+ logger.info("1. [trails] Begin profiling.")
+ # 0. profiling dataset and search space, get t1 and t2
+ score_time_per_model, train_time_per_epoch, N_K_ratio = self.search_space_ins.profiling(
+ self.dataset,
+ train_loader,
+ valid_loader,
+ self.args,
+ is_simulate=self.is_simulate)
+
+ self.sh = BudgetAwareControllerSH(
+ search_space_ins=self.search_space_ins,
+ dataset_name=self.dataset,
+ eta=self.eta,
+ time_per_epoch=train_time_per_epoch,
+ is_simulate=self.is_simulate,
+ train_loader=train_loader,
+ val_loader=valid_loader,
+ args=self.args)
+
+ # 1. run coordinator to schedule
+ logger.info("2. [trails] Begin scheduling...")
+ K, U, N, B1_planed_time, B2_planed_time, B2_all_epoch = coordinator.schedule(self.dataset, self.sh, budget,
+ score_time_per_model,
+ train_time_per_epoch,
+ run_workers,
+ self.search_space_ins,
+ N_K_ratio,
+ only_phase1)
+
+ print(f"Budget = {budget}, N={N}, K={K}")
+
+ # 2. run phase 1 to score N models
+ logger.info("3. [trails] Begin to run phase1: filter phase")
+ # lazy loading the search space if needed.
+
+ # run phase-1 to get the K models.
+ p1_runner = RunPhase1(
+ args=self.args,
+ K=K, N=N,
+ search_space_ins=self.search_space_ins,
+ train_loader=train_loader,
+ is_simulate=self.is_simulate)
+
+ k_models, all_models, p1_trace_highest_score, p1_trace_highest_scored_models_id \
+ = p1_runner.run_phase1()
+
+ logger.info("4. [trails] Begin to run phase2: refinement phase")
+
+ # 3. run phase-2 to determine the final model
+ best_arch, best_arch_performance, B2_actual_epoch_use = self.sh.run_phase2(U, k_models)
+ # print("best model returned from Phase2 = ", k_models)
+ end_time = time.time()
+ real_time_usage = end_time - begin_time
+ planned_time_usage = B1_planed_time + B2_planed_time
+ logger.info("5. [trails] Real time Usage = " + str(real_time_usage)
+ + ", Final selected model = " + str(best_arch)
+ + ", planned time usage = " + str(planned_time_usage)
+ )
+ # best arch returned,
+ # time usage, epoch trained,
+ # p1 ea trace
+ return best_arch, best_arch_performance, \
+ real_time_usage, planned_time_usage, B2_all_epoch, \
+ all_models, p1_trace_highest_score, p1_trace_highest_scored_models_id
+
+ def schedule_only(self, budget: float, data_loader: List[DataLoader],
+ only_phase1: bool = False, run_workers: int = 1):
+ """
+ Select model online
+ :param budget: time budget
+ :param data_loader: time budget
+ :param only_phase1:
+ :param run_workers:
+ :return:
+ """
+
+ train_loader, valid_loader, test_loader = data_loader
+ self.search_space_ins.load()
+
+ logger.info("0. [trails] Begin model selection ... ")
+ begin_time = time.time()
+
+ logger.info("1. [trails] Begin profiling.")
+ # 0. profiling dataset and search space, get t1 and t2
+ score_time_per_model, train_time_per_epoch, N_K_ratio = self.search_space_ins.profiling(
+ self.dataset,
+ train_loader,
+ valid_loader,
+ self.args,
+ is_simulate=self.is_simulate)
+
+ self.sh = BudgetAwareControllerSH(
+ search_space_ins=self.search_space_ins,
+ dataset_name=self.dataset,
+ eta=self.eta,
+ time_per_epoch=train_time_per_epoch,
+ is_simulate=self.is_simulate,
+ train_loader=train_loader,
+ val_loader=valid_loader,
+ args=self.args)
+
+ # 1. run coordinator to schedule
+ logger.info("2. [trails] Begin scheduling...")
+ K, U, N, B1_planed_time, B2_planed_time, B2_all_epoch = coordinator.schedule(self.dataset, self.sh, budget,
+ score_time_per_model,
+ train_time_per_epoch,
+ run_workers,
+ self.search_space_ins,
+ N_K_ratio,
+ only_phase1)
+
+ return N
+
+ #############################################
+ # to support in-database model selection
+ #############################################
+
+ def profile_filtering(self, data_loader: List[DataLoader] = [None, None, None]):
+ logger.info("0. [trails] Begin profile_filtering...")
+ begin_time = time.time()
+ train_loader, valid_loader, test_loader = data_loader
+ score_time_per_model = self.search_space_ins.profiling_score_time(
+ self.dataset,
+ train_loader,
+ valid_loader,
+ self.args,
+ is_simulate=self.is_simulate)
+ logger.info(f"0. [trails] profile_filtering Done, time_usage = {time.time() - begin_time}")
+ return score_time_per_model
+
+ def profile_refinement(self, data_loader: List[DataLoader] = [None, None, None]):
+ logger.info("0. [trails] Begin profile_refinement...")
+ begin_time = time.time()
+ train_loader, valid_loader, test_loader = data_loader
+ train_time_per_epoch = self.search_space_ins.profiling_train_time(
+ self.dataset,
+ train_loader,
+ valid_loader,
+ self.args,
+ is_simulate=self.is_simulate)
+ logger.info(f"0. [trails] profile_refinement Done, time_usage = {time.time() - begin_time}")
+ return train_time_per_epoch
+
+ def coordination(self, budget: float, score_time_per_model: float, train_time_per_epoch: float, only_phase1: bool):
+ logger.info("1. [trails] Begin coordination...")
+ begin_time = time.time()
+ sh = BudgetAwareControllerSH(
+ search_space_ins=self.search_space_ins,
+ dataset_name=self.dataset,
+ eta=self.eta,
+ time_per_epoch=train_time_per_epoch,
+ is_simulate=self.is_simulate,
+ train_loader=None,
+ val_loader=None,
+ args=self.args)
+ n_k_ratio = profile_NK_trade_off(self.dataset)
+ K, U, N, B1_planed_time, B2_planed_time, B2_all_epoch = coordinator.schedule(
+ self.dataset, sh, budget,
+ score_time_per_model,
+ train_time_per_epoch,
+ 1,
+ self.search_space_ins,
+ n_k_ratio,
+ only_phase1)
+
+ logger.info(f"1. [trails] Coordination Done, time_usage = {time.time() - begin_time}")
+ return K, U, N
+
+ def filtering_phase(self, N, K, train_loader=None):
+ logger.info("2. [trails] Begin filtering_phase...")
+ begin_time = time.time()
+ p1_runner = RunPhase1(
+ args=self.args,
+ K=K, N=N,
+ search_space_ins=self.search_space_ins,
+ train_loader=train_loader,
+ is_simulate=self.is_simulate)
+
+ k_models, all_models, p1_trace_highest_score, p1_trace_highest_scored_models_id \
+ = p1_runner.run_phase1()
+ logger.info(f"2. [trails] filtering_phase Done, time_usage = {time.time() - begin_time}")
+ print(f"2. [trails] filtering_phase Done, time_usage = {time.time() - begin_time}")
+ return k_models, all_models, p1_trace_highest_score, p1_trace_highest_scored_models_id
+
+ def refinement_phase(self, U, k_models, train_loader=None, valid_loader=None, train_time_per_epoch=None):
+ logger.info("3. [trails] Begin refinement...")
+ begin_time = time.time()
+ self.sh = BudgetAwareControllerSH(
+ search_space_ins=self.search_space_ins,
+ dataset_name=self.dataset,
+ eta=self.eta,
+ time_per_epoch=train_time_per_epoch,
+ is_simulate=self.is_simulate,
+ train_loader=train_loader,
+ val_loader=valid_loader,
+ args=self.args)
+ best_arch, best_arch_performance, B2_actual_epoch_use = self.sh.run_phase2(U, k_models)
+ logger.info(
+ f"3. [trails] refinement phase Done, time_usage = {time.time() - begin_time}, "
+ f"epoches_used = {B2_actual_epoch_use}")
+ return best_arch, best_arch_performance, B2_actual_epoch_use
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/search_space/core/rl_policy.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/search_space/core/rl_policy.py
new file mode 100644
index 00000000..3b21fb60
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/search_space/core/rl_policy.py
@@ -0,0 +1,5 @@
+import torch.nn as nn
+
+
+class RLPolicyBase(nn.Module):
+ pass
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/search_space/core/space.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/search_space/core/space.py
new file mode 100644
index 00000000..9a7ba828
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/search_space/core/space.py
@@ -0,0 +1,174 @@
+from abc import abstractmethod
+from typing import Generator
+
+from torch.utils.data import DataLoader
+from src.search_space.core.model_params import ModelMacroCfg, ModelMicroCfg
+
+
+class SpaceWrapper:
+
+ def __init__(self, cfg: ModelMacroCfg, name: str):
+ self.model_cfg = cfg
+ self.name = name
+
+ @abstractmethod
+ def sample_all_models(self) -> Generator[str, None, None]:
+ """
+ Sample all models, return a list of arch ids
+ """
+ raise NotImplementedError
+
+ """serialize and deserialize"""
+
+ @classmethod
+ def serialize_model_encoding(cls, arch_micro: ModelMicroCfg) -> str:
+ raise NotImplementedError
+
+ @classmethod
+ def deserialize_model_encoding(cls, model_encoding) -> ModelMicroCfg:
+ raise NotImplementedError
+
+ @classmethod
+ def new_arch_scratch(cls, arch_macro: ModelMacroCfg, arch_micro: ModelMicroCfg, bn: bool = True):
+ """
+ Args:
+ arch_macro: macro setting for one architecture
+ arch_micro: micro setting for one architecture
+ bn: true or false
+ Returns:
+ """
+ raise NotImplementedError
+
+ def new_arch_scratch_with_default_setting(self, model_encoding: str, bn: bool):
+ """
+ Use the current search space's macro setting.
+ Args:
+ model_encoding: str of the model encoding
+ bn: true or false
+ Returns:
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ def load(self):
+ """
+ Load the related API
+ Returns:
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ def profiling(self, dataset: str,
+ train_loader: DataLoader = None, val_loader: DataLoader = None,
+ args=None, is_simulate: bool = False) -> (float, float, int):
+ """
+ Profile the training and scoring time.
+ Args:
+ dataset:
+ train_loader:
+ val_loader
+ args:
+ is_simulate:
+ Returns:
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ def micro_to_id(self, arch_struct: ModelMicroCfg) -> str:
+ raise NotImplementedError
+
+ """init new architecture"""
+
+ @abstractmethod
+ def new_architecture(self, arch_id: str):
+ """
+ Generate an architecture with arch id
+ :return:
+ """
+ raise NotImplementedError
+
+ def new_architecture_with_micro_cfg(self, arch_micro: ModelMicroCfg):
+ """
+ Generate an architecture with arch_micro
+ :return:
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ def __len__(self):
+ """
+ How many architectures the space has
+ :return:
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ def get_arch_size(self, architecture):
+ """
+ Get how many edges in each cell of the architecture.
+ :return:
+ """
+ raise NotImplementedError
+
+ def update_bn_flag(self, bn: bool):
+ """
+ Update architecture's bn,
+ :param bn:
+ :return:
+ """
+ self.model_cfg.bn = bn
+
+ """Below is for integrating space with various sampler"""
+
+ def random_architecture_id(self) -> (str, ModelMicroCfg):
+ """
+ Random generate architecture id, cell structure, supporting RN, RL, R
+ :param max_nodes: how many nodes in this cell
+ :return:
+ """
+ raise NotImplementedError
+
+ def mutate_architecture(self, parent_arch: ModelMicroCfg) -> (str, ModelMicroCfg):
+ """
+ Mutate architecture, this is to support EA sampler
+ :rtype: object
+ :return:
+ """
+ raise NotImplementedError
+
+ def get_reinforcement_learning_policy(self, lr_rate):
+ """
+ This is fpr reinforcement learning policy sampler
+ :return:
+ """
+ raise NotImplementedError
+
+ """In-RDBMS Helper Functions"""
+
+ def profiling_score_time(self, dataset: str, train_loader: DataLoader = None, val_loader: DataLoader = None,
+ args=None, is_simulate: bool = False) -> float:
+ """
+ Profile the scoring time.
+ Args:
+ dataset:
+ train_loader:
+ val_loader
+ args:
+ is_simulate:
+ Returns:
+ """
+ raise NotImplementedError
+
+ def profiling_train_time(self, dataset: str, train_loader: DataLoader = None, val_loader: DataLoader = None,
+ args=None, is_simulate: bool = False) -> float:
+ """
+ Profile the training time.
+ Args:
+ dataset:
+ train_loader:
+ val_loader
+ args:
+ is_simulate:
+ Returns:
+ """
+ raise NotImplementedError
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/darts_lib/operations.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/darts_lib/operations.py
new file mode 100644
index 00000000..412a2724
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/darts_lib/operations.py
@@ -0,0 +1,114 @@
+import torch
+import torch.nn as nn
+
+OPS = {
+ 'none': lambda C, stride, affine: Zero(stride),
+ 'avg_pool_3x3': lambda C, stride, affine: nn.AvgPool2d(3, stride=stride, padding=1, count_include_pad=False),
+ 'max_pool_3x3': lambda C, stride, affine: nn.MaxPool2d(3, stride=stride, padding=1),
+ 'skip_connect': lambda C, stride, affine: Identity() if stride == 1 else FactorizedReduce(C, C, affine=affine),
+ 'sep_conv_3x3': lambda C, stride, affine: SepConv(C, C, 3, stride, 1, affine=affine),
+ 'sep_conv_5x5': lambda C, stride, affine: SepConv(C, C, 5, stride, 2, affine=affine),
+ 'sep_conv_7x7': lambda C, stride, affine: SepConv(C, C, 7, stride, 3, affine=affine),
+ 'dil_conv_3x3': lambda C, stride, affine: DilConv(C, C, 3, stride, 2, 2, affine=affine),
+ 'dil_conv_5x5': lambda C, stride, affine: DilConv(C, C, 5, stride, 4, 2, affine=affine),
+ 'conv_7x1_1x7': lambda C, stride, affine: nn.Sequential(
+ nn.ReLU(inplace=False),
+ nn.Conv2d(C, C, (1, 7), stride=(1, stride),
+ padding=(0, 3), bias=False),
+ nn.Conv2d(C, C, (7, 1), stride=(stride, 1),
+ padding=(3, 0), bias=False),
+ nn.BatchNorm2d(C, affine=affine)
+ ),
+}
+
+
+class ReLUConvBN(nn.Module):
+
+ def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True):
+ super(ReLUConvBN, self).__init__()
+ self.op = nn.Sequential(
+ nn.ReLU(inplace=False),
+ nn.Conv2d(C_in, C_out, kernel_size, stride=stride,
+ padding=padding, bias=False),
+ nn.BatchNorm2d(C_out, affine=affine)
+ )
+
+ def forward(self, x):
+ return self.op(x)
+
+
+class DilConv(nn.Module):
+
+ def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine=True):
+ super(DilConv, self).__init__()
+ self.op = nn.Sequential(
+ nn.ReLU(inplace=False),
+ nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=stride,
+ padding=padding, dilation=dilation, groups=C_in, bias=False),
+ nn.Conv2d(C_in, C_out, kernel_size=1, padding=0, bias=False),
+ nn.BatchNorm2d(C_out, affine=affine),
+ )
+
+ def forward(self, x):
+ return self.op(x)
+
+
+class SepConv(nn.Module):
+
+ def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True):
+ super(SepConv, self).__init__()
+ self.op = nn.Sequential(
+ nn.ReLU(inplace=False),
+ nn.Conv2d(C_in, C_in, kernel_size=kernel_size,
+ stride=stride, padding=padding, groups=C_in, bias=False),
+ nn.Conv2d(C_in, C_in, kernel_size=1, padding=0, bias=False),
+ nn.BatchNorm2d(C_in, affine=affine),
+ nn.ReLU(inplace=False),
+ nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=1,
+ padding=padding, groups=C_in, bias=False),
+ nn.Conv2d(C_in, C_out, kernel_size=1, padding=0, bias=False),
+ nn.BatchNorm2d(C_out, affine=affine),
+ )
+
+ def forward(self, x):
+ return self.op(x)
+
+
+class Identity(nn.Module):
+
+ def __init__(self):
+ super(Identity, self).__init__()
+
+ def forward(self, x):
+ return x
+
+
+class Zero(nn.Module):
+
+ def __init__(self, stride):
+ super(Zero, self).__init__()
+ self.stride = stride
+
+ def forward(self, x):
+ if self.stride == 1:
+ return x.mul(0.)
+ return x[:, :, ::self.stride, ::self.stride].mul(0.)
+
+
+class FactorizedReduce(nn.Module):
+
+ def __init__(self, C_in, C_out, affine=True):
+ super(FactorizedReduce, self).__init__()
+ assert C_out % 2 == 0
+ self.relu = nn.ReLU(inplace=False)
+ self.conv_1 = nn.Conv2d(C_in, C_out // 2, 1,
+ stride=2, padding=0, bias=False)
+ self.conv_2 = nn.Conv2d(C_in, C_out // 2, 1,
+ stride=2, padding=0, bias=False)
+ self.bn = nn.BatchNorm2d(C_out, affine=affine)
+
+ def forward(self, x):
+ x = self.relu(x)
+ out = torch.cat([self.conv_1(x), self.conv_2(x[:, :, 1:, 1:])], dim=1)
+ out = self.bn(out)
+ return out
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/darts_lib/utils.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/darts_lib/utils.py
new file mode 100644
index 00000000..3ea554ba
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/darts_lib/utils.py
@@ -0,0 +1,342 @@
+import os
+import sys
+import time
+import warnings
+import numpy as np
+import torch
+import shutil
+import logging
+import random
+
+import torchvision.datasets as dset
+import torchvision.transforms as transforms
+from torch.autograd import Variable
+import torch.nn.functional as F
+import torch.nn as nn
+
+warnings.filterwarnings("error")
+
+
+class AvgrageMeter(object):
+
+ def __init__(self):
+ self.reset()
+
+ def reset(self):
+ self.avg = 0
+ self.sum = 0
+ self.cnt = 0
+
+ def update(self, val, n=1):
+ self.sum += val * n
+ self.cnt += n
+ self.avg = self.sum / self.cnt
+
+
+def accuracy(output, target, topk=(1,)):
+ maxk = max(topk)
+ batch_size = target.size(0)
+
+ _, pred = output.topk(maxk, 1, True, True)
+ pred = pred.t()
+ correct = pred.eq(target.reshape(1, -1).expand_as(pred))
+
+ res = []
+ for k in topk:
+ correct_k = correct[:k].reshape(-1).float().sum(0)
+ res.append(correct_k.mul_(100.0/batch_size))
+ return res
+
+
+class Cutout(object):
+ def __init__(self, length):
+ self.length = length
+
+ def __call__(self, img):
+ h, w = img.size(1), img.size(2)
+ mask = np.ones((h, w), np.float32)
+ y = np.random.randint(h)
+ x = np.random.randint(w)
+
+ y1 = np.clip(y - self.length // 2, 0, h)
+ y2 = np.clip(y + self.length // 2, 0, h)
+ x1 = np.clip(x - self.length // 2, 0, w)
+ x2 = np.clip(x + self.length // 2, 0, w)
+
+ mask[y1: y2, x1: x2] = 0.
+ mask = torch.from_numpy(mask)
+ mask = mask.expand_as(img)
+ img *= mask
+ return img
+
+
+def _data_transforms_cifar10(args):
+ CIFAR_MEAN = [0.49139968, 0.48215827, 0.44653124]
+ CIFAR_STD = [0.24703233, 0.24348505, 0.26158768]
+
+ train_transform = transforms.Compose([
+ transforms.RandomCrop(32, padding=4),
+ transforms.RandomHorizontalFlip(),
+ transforms.ToTensor(),
+ transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
+ ])
+ if args.cutout:
+ train_transform.transforms.append(Cutout(args.cutout_length))
+
+ valid_transform = transforms.Compose([
+ transforms.ToTensor(),
+ transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
+ ])
+ return train_transform, valid_transform
+
+
+def _get_cifar10(args):
+ train_transform, valid_transform = _data_transforms_cifar10(args)
+ train_data = dset.CIFAR10(
+ root=args.data, train=True, download=True, transform=train_transform
+ )
+ valid_data = dset.CIFAR10(
+ root=args.data, train=False, download=True, transform=valid_transform
+ )
+
+ train_queue = torch.utils.data.DataLoader(
+ train_data,
+ batch_size=args.batch_size,
+ shuffle=True,
+ pin_memory=True,
+ num_workers=4,
+ )
+
+ valid_queue = torch.utils.data.DataLoader(
+ valid_data,
+ batch_size=args.batch_size,
+ shuffle=False,
+ pin_memory=True,
+ num_workers=4,
+ )
+ return train_queue, valid_queue
+
+
+def _data_transforms_cifar100(args):
+ CIFAR_MEAN = [0.5070751592371323, 0.48654887331495095, 0.4409178433670343]
+ CIFAR_STD = [0.2673342858792401, 0.2564384629170883, 0.27615047132568404]
+
+ train_transform = transforms.Compose([
+ transforms.RandomCrop(32, padding=4),
+ transforms.RandomHorizontalFlip(),
+ transforms.ToTensor(),
+ transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
+ ])
+ if args.cutout:
+ train_transform.transforms.append(Cutout(args.cutout_length))
+
+ valid_transform = transforms.Compose([
+ transforms.ToTensor(),
+ transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
+ ])
+ return train_transform, valid_transform
+
+
+def _get_cifar100(args):
+ train_transform, valid_transform = _data_transforms_cifar100(args)
+ train_data = dset.CIFAR100(
+ root=args.data, train=True, download=True, transform=train_transform
+ )
+ valid_data = dset.CIFAR100(
+ root=args.data, train=False, download=True, transform=valid_transform
+ )
+
+ train_queue = torch.utils.data.DataLoader(
+ train_data,
+ batch_size=args.batch_size,
+ shuffle=True,
+ pin_memory=True,
+ num_workers=4,
+ )
+
+ valid_queue = torch.utils.data.DataLoader(
+ valid_data,
+ batch_size=args.batch_size,
+ shuffle=False,
+ pin_memory=True,
+ num_workers=4,
+ )
+ return train_queue, valid_queue
+
+
+def _get_imagenet(args):
+ traindir = os.path.join(args.data, 'train')
+ valdir = os.path.join(args.data, 'val')
+ normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
+ std=[0.229, 0.224, 0.225])
+
+ train_dataset = dset.ImageFolder(
+ traindir,
+ transforms.Compose([
+ transforms.RandomResizedCrop(224),
+ transforms.RandomHorizontalFlip(),
+ transforms.ColorJitter(
+ brightness=0.4,
+ contrast=0.4,
+ saturation=0.4,
+ hue=0.2),
+ transforms.ToTensor(),
+ normalize,
+ ]))
+
+ num_train = len(train_dataset)
+ indices = list(range(num_train))
+ random.shuffle(indices)
+ split = int(np.floor(args.train_portion * num_train))
+
+ train_loader = torch.utils.data.DataLoader(
+ train_dataset, batch_size=args.batch_size,
+ sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
+ num_workers=4,pin_memory=True, drop_last=True)
+
+ val_loader = torch.utils.data.DataLoader(
+ train_dataset, batch_size=args.batch_size,
+ sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:2*split]),
+ num_workers=4,pin_memory=True, drop_last=True)
+
+ return train_loader, val_loader
+
+def _get_dist_imagenet(args):
+ traindir = os.path.join(args.data_dir, 'train')
+ valdir = os.path.join(args.data_dir, 'val')
+ normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
+ std=[0.229, 0.224, 0.225])
+
+ train_dataset = dset.ImageFolder(
+ traindir,
+ transforms.Compose([
+ transforms.RandomResizedCrop(224),
+ transforms.RandomHorizontalFlip(),
+ transforms.ColorJitter(
+ brightness=0.4,
+ contrast=0.4,
+ saturation=0.4,
+ hue=0.2),
+ transforms.ToTensor(),
+ normalize,
+ ]))
+
+ sampler = torch.utils.data.distributed.DistributedSampler(
+ train_dataset, num_replicas=args.gpu_num, rank=args.local_rank)
+
+ train_loader = torch.utils.data.DataLoader(
+ train_dataset, batch_size=args.batch_size//args.gpu_num, num_workers=max(args.gpu_num*2, 4),
+ pin_memory=True, drop_last=True, sampler=sampler)
+
+ val_loader = torch.utils.data.DataLoader(
+ dset.ImageFolder(valdir, transforms.Compose([
+ transforms.Resize(256),
+ transforms.CenterCrop(224),
+ transforms.ToTensor(),
+ normalize,
+ ])),
+ batch_size=args.batch_size, shuffle=False,
+ num_workers=4, pin_memory=True)
+
+ return train_loader, val_loader, sampler
+
+
+
+def count_parameters_in_MB(model):
+ return np.sum([np.prod(v.size()) for n, v in model.named_parameters() if 'auxiliary' not in n])/1e6
+
+
+def save_checkpoint(state, is_best, save):
+ filename = os.path.join(save, 'checkpoint.pth.tar')
+ torch.save(state, filename)
+ if is_best:
+ best_filename = os.path.join(save, 'model_best.pth.tar')
+ shutil.copyfile(filename, best_filename)
+
+
+def save(model, model_path):
+ torch.save(model.state_dict(), model_path)
+
+
+def load(model, model_path):
+ model.load_state_dict(torch.load(model_path))
+
+
+def drop_path(x, drop_prob, dims=(0,)):
+ var_size = [1 for _ in range(x.dim())]
+ for i in dims:
+ var_size[i] = x.size(i)
+ if drop_prob > 0.:
+ keep_prob = 1.-drop_prob
+ mask = Variable(torch.cuda.FloatTensor(
+ *var_size).bernoulli_(keep_prob))
+ x.div_(keep_prob)
+ x.mul_(mask)
+ return x
+
+
+def create_exp_dir(path, scripts_to_save=None):
+ if not os.path.exists(path):
+ os.makedirs(path)
+ print('Experiment dir : {}'.format(path))
+
+ if scripts_to_save is not None:
+ os.makedirs(os.path.join(path, 'tools'))
+ for script in scripts_to_save:
+ dst_file = os.path.join(path, 'tools', os.path.basename(script))
+ shutil.copyfile(script, dst_file)
+
+
+class Performance(object):
+ def __init__(self, path):
+ self.path = path
+ self.data = None
+
+ def update(self, alphas_normal, alphas_reduce, val_loss):
+ a_normal = F.softmax(alphas_normal, dim=-1)
+ # print("alpha normal size: ", a_normal.data.size())
+ a_reduce = F.softmax(alphas_reduce, dim=-1)
+ # print("alpha reduce size: ", a_reduce.data.size())
+ data = np.concatenate([a_normal.data.reshape(-1),
+ a_reduce.data.reshape(-1),
+ np.array([val_loss.data])]).reshape(1, -1)
+ if self.data is not None:
+ self.data = np.concatenate([self.data, data], axis=0)
+ else:
+ self.data = data
+
+ def save(self):
+ np.save(self.path, self.data)
+
+
+def logger(log_dir, need_time=True, need_stdout=False):
+ log = logging.getLogger(__name__)
+ log.setLevel(logging.DEBUG)
+ fh = logging.FileHandler(log_dir)
+ fh.setLevel(logging.DEBUG)
+ formatter = logging.Formatter(fmt='%(asctime)s %(message)s', datefmt='%m/%d/%Y-%I:%M:%S')
+ if need_stdout:
+ ch = logging.StreamHandler(sys.stdout)
+ ch.setLevel(logging.DEBUG)
+ log.addHandler(ch)
+ if need_time:
+ fh.setFormatter(formatter)
+ if need_stdout:
+ ch.setFormatter(formatter)
+ log.addHandler(fh)
+ return log
+
+class CrossEntropyLabelSmooth(nn.Module):
+
+ def __init__(self, num_classes, epsilon):
+ super(CrossEntropyLabelSmooth, self).__init__()
+ self.num_classes = num_classes
+ self.epsilon = epsilon
+ self.logsoftmax = nn.LogSoftmax(dim=1)
+
+ def forward(self, inputs, targets):
+ log_probs = self.logsoftmax(inputs)
+ targets = torch.zeros_like(log_probs).scatter_(1, targets.unsqueeze(1), 1)
+ targets = (1 - self.epsilon) * targets + self.epsilon / self.num_classes
+ loss = (-targets * log_probs).mean(0).sum()
+ return loss
\ No newline at end of file
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/CifarDenseNet.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/CifarDenseNet.py
new file mode 100644
index 00000000..eaf8e981
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/CifarDenseNet.py
@@ -0,0 +1,117 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+##################################################
+import math, torch
+import torch.nn as nn
+import torch.nn.functional as F
+from .initialization import initialize_resnet
+
+
+class Bottleneck(nn.Module):
+ def __init__(self, nChannels, growthRate):
+ super(Bottleneck, self).__init__()
+ interChannels = 4 * growthRate
+ self.bn1 = nn.BatchNorm2d(nChannels)
+ self.conv1 = nn.Conv2d(nChannels, interChannels, kernel_size=1, bias=False)
+ self.bn2 = nn.BatchNorm2d(interChannels)
+ self.conv2 = nn.Conv2d(
+ interChannels, growthRate, kernel_size=3, padding=1, bias=False
+ )
+
+ def forward(self, x):
+ out = self.conv1(F.relu(self.bn1(x)))
+ out = self.conv2(F.relu(self.bn2(out)))
+ out = torch.cat((x, out), 1)
+ return out
+
+
+class SingleLayer(nn.Module):
+ def __init__(self, nChannels, growthRate):
+ super(SingleLayer, self).__init__()
+ self.bn1 = nn.BatchNorm2d(nChannels)
+ self.conv1 = nn.Conv2d(
+ nChannels, growthRate, kernel_size=3, padding=1, bias=False
+ )
+
+ def forward(self, x):
+ out = self.conv1(F.relu(self.bn1(x)))
+ out = torch.cat((x, out), 1)
+ return out
+
+
+class Transition(nn.Module):
+ def __init__(self, nChannels, nOutChannels):
+ super(Transition, self).__init__()
+ self.bn1 = nn.BatchNorm2d(nChannels)
+ self.conv1 = nn.Conv2d(nChannels, nOutChannels, kernel_size=1, bias=False)
+
+ def forward(self, x):
+ out = self.conv1(F.relu(self.bn1(x)))
+ out = F.avg_pool2d(out, 2)
+ return out
+
+
+class DenseNet(nn.Module):
+ def __init__(self, growthRate, depth, reduction, nClasses, bottleneck):
+ super(DenseNet, self).__init__()
+
+ if bottleneck:
+ nDenseBlocks = int((depth - 4) / 6)
+ else:
+ nDenseBlocks = int((depth - 4) / 3)
+
+ self.message = "CifarDenseNet : block : {:}, depth : {:}, reduction : {:}, growth-rate = {:}, class = {:}".format(
+ "bottleneck" if bottleneck else "basic",
+ depth,
+ reduction,
+ growthRate,
+ nClasses,
+ )
+
+ nChannels = 2 * growthRate
+ self.conv1 = nn.Conv2d(3, nChannels, kernel_size=3, padding=1, bias=False)
+
+ self.dense1 = self._make_dense(nChannels, growthRate, nDenseBlocks, bottleneck)
+ nChannels += nDenseBlocks * growthRate
+ nOutChannels = int(math.floor(nChannels * reduction))
+ self.trans1 = Transition(nChannels, nOutChannels)
+
+ nChannels = nOutChannels
+ self.dense2 = self._make_dense(nChannels, growthRate, nDenseBlocks, bottleneck)
+ nChannels += nDenseBlocks * growthRate
+ nOutChannels = int(math.floor(nChannels * reduction))
+ self.trans2 = Transition(nChannels, nOutChannels)
+
+ nChannels = nOutChannels
+ self.dense3 = self._make_dense(nChannels, growthRate, nDenseBlocks, bottleneck)
+ nChannels += nDenseBlocks * growthRate
+
+ self.act = nn.Sequential(
+ nn.BatchNorm2d(nChannels), nn.ReLU(inplace=True), nn.AvgPool2d(8)
+ )
+ self.fc = nn.Linear(nChannels, nClasses)
+
+ self.apply(initialize_resnet)
+
+ def get_message(self):
+ return self.message
+
+ def _make_dense(self, nChannels, growthRate, nDenseBlocks, bottleneck):
+ layers = []
+ for i in range(int(nDenseBlocks)):
+ if bottleneck:
+ layers.append(Bottleneck(nChannels, growthRate))
+ else:
+ layers.append(SingleLayer(nChannels, growthRate))
+ nChannels += growthRate
+ return nn.Sequential(*layers)
+
+ def forward(self, inputs):
+ out = self.conv1(inputs)
+ out = self.trans1(self.dense1(out))
+ out = self.trans2(self.dense2(out))
+ out = self.dense3(out)
+ features = self.act(out)
+ features = features.view(features.size(0), -1)
+ out = self.fc(features)
+ return features, out
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/CifarResNet.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/CifarResNet.py
new file mode 100644
index 00000000..7ab777fc
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/CifarResNet.py
@@ -0,0 +1,180 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from .initialization import initialize_resnet
+from .SharedUtils import additive_func
+
+
+class Downsample(nn.Module):
+ def __init__(self, nIn, nOut, stride):
+ super(Downsample, self).__init__()
+ assert stride == 2 and nOut == 2 * nIn, "stride:{} IO:{},{}".format(
+ stride, nIn, nOut
+ )
+ self.in_dim = nIn
+ self.out_dim = nOut
+ self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0)
+ self.conv = nn.Conv2d(nIn, nOut, kernel_size=1, stride=1, padding=0, bias=False)
+
+ def forward(self, x):
+ x = self.avg(x)
+ out = self.conv(x)
+ return out
+
+
+class ConvBNReLU(nn.Module):
+ def __init__(self, nIn, nOut, kernel, stride, padding, bias, relu):
+ super(ConvBNReLU, self).__init__()
+ self.conv = nn.Conv2d(
+ nIn, nOut, kernel_size=kernel, stride=stride, padding=padding, bias=bias
+ )
+ self.bn = nn.BatchNorm2d(nOut)
+ if relu:
+ self.relu = nn.ReLU(inplace=True)
+ else:
+ self.relu = None
+ self.out_dim = nOut
+ self.num_conv = 1
+
+ def forward(self, x):
+ conv = self.conv(x)
+ bn = self.bn(conv)
+ if self.relu:
+ return self.relu(bn)
+ else:
+ return bn
+
+
+class ResNetBasicblock(nn.Module):
+ expansion = 1
+
+ def __init__(self, inplanes, planes, stride):
+ super(ResNetBasicblock, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+ self.conv_a = ConvBNReLU(inplanes, planes, 3, stride, 1, False, True)
+ self.conv_b = ConvBNReLU(planes, planes, 3, 1, 1, False, False)
+ if stride == 2:
+ self.downsample = Downsample(inplanes, planes, stride)
+ elif inplanes != planes:
+ self.downsample = ConvBNReLU(inplanes, planes, 1, 1, 0, False, False)
+ else:
+ self.downsample = None
+ self.out_dim = planes
+ self.num_conv = 2
+
+ def forward(self, inputs):
+
+ basicblock = self.conv_a(inputs)
+ basicblock = self.conv_b(basicblock)
+
+ if self.downsample is not None:
+ residual = self.downsample(inputs)
+ else:
+ residual = inputs
+ out = additive_func(residual, basicblock)
+ return F.relu(out, inplace=True)
+
+
+class ResNetBottleneck(nn.Module):
+ expansion = 4
+
+ def __init__(self, inplanes, planes, stride):
+ super(ResNetBottleneck, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+ self.conv_1x1 = ConvBNReLU(inplanes, planes, 1, 1, 0, False, True)
+ self.conv_3x3 = ConvBNReLU(planes, planes, 3, stride, 1, False, True)
+ self.conv_1x4 = ConvBNReLU(
+ planes, planes * self.expansion, 1, 1, 0, False, False
+ )
+ if stride == 2:
+ self.downsample = Downsample(inplanes, planes * self.expansion, stride)
+ elif inplanes != planes * self.expansion:
+ self.downsample = ConvBNReLU(
+ inplanes, planes * self.expansion, 1, 1, 0, False, False
+ )
+ else:
+ self.downsample = None
+ self.out_dim = planes * self.expansion
+ self.num_conv = 3
+
+ def forward(self, inputs):
+
+ bottleneck = self.conv_1x1(inputs)
+ bottleneck = self.conv_3x3(bottleneck)
+ bottleneck = self.conv_1x4(bottleneck)
+
+ if self.downsample is not None:
+ residual = self.downsample(inputs)
+ else:
+ residual = inputs
+ out = additive_func(residual, bottleneck)
+ return F.relu(out, inplace=True)
+
+
+class CifarResNet(nn.Module):
+ def __init__(self, block_name, depth, num_classes, zero_init_residual):
+ super(CifarResNet, self).__init__()
+
+ # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
+ if block_name == "ResNetBasicblock":
+ block = ResNetBasicblock
+ assert (depth - 2) % 6 == 0, "depth should be one of 20, 32, 44, 56, 110"
+ layer_blocks = (depth - 2) // 6
+ elif block_name == "ResNetBottleneck":
+ block = ResNetBottleneck
+ assert (depth - 2) % 9 == 0, "depth should be one of 164"
+ layer_blocks = (depth - 2) // 9
+ else:
+ raise ValueError("invalid block : {:}".format(block_name))
+
+ self.message = "CifarResNet : Block : {:}, Depth : {:}, Layers for each block : {:}".format(
+ block_name, depth, layer_blocks
+ )
+ self.num_classes = num_classes
+ self.channels = [16]
+ self.layers = nn.ModuleList([ConvBNReLU(3, 16, 3, 1, 1, False, True)])
+ for stage in range(3):
+ for iL in range(layer_blocks):
+ iC = self.channels[-1]
+ planes = 16 * (2 ** stage)
+ stride = 2 if stage > 0 and iL == 0 else 1
+ module = block(iC, planes, stride)
+ self.channels.append(module.out_dim)
+ self.layers.append(module)
+ self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iC={:3d}, oC={:3d}, stride={:}".format(
+ stage,
+ iL,
+ layer_blocks,
+ len(self.layers) - 1,
+ iC,
+ module.out_dim,
+ stride,
+ )
+
+ self.avgpool = nn.AvgPool2d(8)
+ self.classifier = nn.Linear(module.out_dim, num_classes)
+ assert (
+ sum(x.num_conv for x in self.layers) + 1 == depth
+ ), "invalid depth check {:} vs {:}".format(
+ sum(x.num_conv for x in self.layers) + 1, depth
+ )
+
+ self.apply(initialize_resnet)
+ if zero_init_residual:
+ for m in self.modules():
+ if isinstance(m, ResNetBasicblock):
+ nn.init.constant_(m.conv_b.bn.weight, 0)
+ elif isinstance(m, ResNetBottleneck):
+ nn.init.constant_(m.conv_1x4.bn.weight, 0)
+
+ def get_message(self):
+ return self.message
+
+ def forward(self, inputs):
+ x = inputs
+ for i, layer in enumerate(self.layers):
+ x = layer(x)
+ features = self.avgpool(x)
+ features = features.view(features.size(0), -1)
+ logits = self.classifier(features)
+ return features, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/CifarWideResNet.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/CifarWideResNet.py
new file mode 100644
index 00000000..62e97c32
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/CifarWideResNet.py
@@ -0,0 +1,115 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from .initialization import initialize_resnet
+
+
+class WideBasicblock(nn.Module):
+ def __init__(self, inplanes, planes, stride, dropout=False):
+ super(WideBasicblock, self).__init__()
+
+ self.bn_a = nn.BatchNorm2d(inplanes)
+ self.conv_a = nn.Conv2d(
+ inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False
+ )
+
+ self.bn_b = nn.BatchNorm2d(planes)
+ if dropout:
+ self.dropout = nn.Dropout2d(p=0.5, inplace=True)
+ else:
+ self.dropout = None
+ self.conv_b = nn.Conv2d(
+ planes, planes, kernel_size=3, stride=1, padding=1, bias=False
+ )
+
+ if inplanes != planes:
+ self.downsample = nn.Conv2d(
+ inplanes, planes, kernel_size=1, stride=stride, padding=0, bias=False
+ )
+ else:
+ self.downsample = None
+
+ def forward(self, x):
+
+ basicblock = self.bn_a(x)
+ basicblock = F.relu(basicblock)
+ basicblock = self.conv_a(basicblock)
+
+ basicblock = self.bn_b(basicblock)
+ basicblock = F.relu(basicblock)
+ if self.dropout is not None:
+ basicblock = self.dropout(basicblock)
+ basicblock = self.conv_b(basicblock)
+
+ if self.downsample is not None:
+ x = self.downsample(x)
+
+ return x + basicblock
+
+
+class CifarWideResNet(nn.Module):
+ """
+ ResNet optimized for the Cifar dataset, as specified in
+ https://arxiv.org/abs/1512.03385.pdf
+ """
+
+ def __init__(self, depth, widen_factor, num_classes, dropout):
+ super(CifarWideResNet, self).__init__()
+
+ # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
+ assert (depth - 4) % 6 == 0, "depth should be one of 20, 32, 44, 56, 110"
+ layer_blocks = (depth - 4) // 6
+ print(
+ "CifarPreResNet : Depth : {} , Layers for each block : {}".format(
+ depth, layer_blocks
+ )
+ )
+
+ self.num_classes = num_classes
+ self.dropout = dropout
+ self.conv_3x3 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)
+
+ self.message = "Wide ResNet : depth={:}, widen_factor={:}, class={:}".format(
+ depth, widen_factor, num_classes
+ )
+ self.inplanes = 16
+ self.stage_1 = self._make_layer(
+ WideBasicblock, 16 * widen_factor, layer_blocks, 1
+ )
+ self.stage_2 = self._make_layer(
+ WideBasicblock, 32 * widen_factor, layer_blocks, 2
+ )
+ self.stage_3 = self._make_layer(
+ WideBasicblock, 64 * widen_factor, layer_blocks, 2
+ )
+ self.lastact = nn.Sequential(
+ nn.BatchNorm2d(64 * widen_factor), nn.ReLU(inplace=True)
+ )
+ self.avgpool = nn.AvgPool2d(8)
+ self.classifier = nn.Linear(64 * widen_factor, num_classes)
+
+ self.apply(initialize_resnet)
+
+ def get_message(self):
+ return self.message
+
+ def _make_layer(self, block, planes, blocks, stride):
+
+ layers = []
+ layers.append(block(self.inplanes, planes, stride, self.dropout))
+ self.inplanes = planes
+ for i in range(1, blocks):
+ layers.append(block(self.inplanes, planes, 1, self.dropout))
+
+ return nn.Sequential(*layers)
+
+ def forward(self, x):
+ x = self.conv_3x3(x)
+ x = self.stage_1(x)
+ x = self.stage_2(x)
+ x = self.stage_3(x)
+ x = self.lastact(x)
+ x = self.avgpool(x)
+ features = x.view(x.size(0), -1)
+ outs = self.classifier(features)
+ return features, outs
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/ImageNet_MobileNetV2.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/ImageNet_MobileNetV2.py
new file mode 100644
index 00000000..814ab39f
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/ImageNet_MobileNetV2.py
@@ -0,0 +1,117 @@
+# MobileNetV2: Inverted Residuals and Linear Bottlenecks, CVPR 2018
+from torch import nn
+from .initialization import initialize_resnet
+
+
+class ConvBNReLU(nn.Module):
+ def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1):
+ super(ConvBNReLU, self).__init__()
+ padding = (kernel_size - 1) // 2
+ self.conv = nn.Conv2d(
+ in_planes,
+ out_planes,
+ kernel_size,
+ stride,
+ padding,
+ groups=groups,
+ bias=False,
+ )
+ self.bn = nn.BatchNorm2d(out_planes)
+ self.relu = nn.ReLU6(inplace=True)
+
+ def forward(self, x):
+ out = self.conv(x)
+ out = self.bn(out)
+ out = self.relu(out)
+ return out
+
+
+class InvertedResidual(nn.Module):
+ def __init__(self, inp, oup, stride, expand_ratio):
+ super(InvertedResidual, self).__init__()
+ self.stride = stride
+ assert stride in [1, 2]
+
+ hidden_dim = int(round(inp * expand_ratio))
+ self.use_res_connect = self.stride == 1 and inp == oup
+
+ layers = []
+ if expand_ratio != 1:
+ # pw
+ layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1))
+ layers.extend(
+ [
+ # dw
+ ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim),
+ # pw-linear
+ nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),
+ nn.BatchNorm2d(oup),
+ ]
+ )
+ self.conv = nn.Sequential(*layers)
+
+ def forward(self, x):
+ if self.use_res_connect:
+ return x + self.conv(x)
+ else:
+ return self.conv(x)
+
+
+class MobileNetV2(nn.Module):
+ def __init__(
+ self, num_classes, width_mult, input_channel, last_channel, block_name, dropout
+ ):
+ super(MobileNetV2, self).__init__()
+ if block_name == "InvertedResidual":
+ block = InvertedResidual
+ else:
+ raise ValueError("invalid block name : {:}".format(block_name))
+ inverted_residual_setting = [
+ # t, c, n, s
+ [1, 16, 1, 1],
+ [6, 24, 2, 2],
+ [6, 32, 3, 2],
+ [6, 64, 4, 2],
+ [6, 96, 3, 1],
+ [6, 160, 3, 2],
+ [6, 320, 1, 1],
+ ]
+
+ # building first layer
+ input_channel = int(input_channel * width_mult)
+ self.last_channel = int(last_channel * max(1.0, width_mult))
+ features = [ConvBNReLU(3, input_channel, stride=2)]
+ # building inverted residual blocks
+ for t, c, n, s in inverted_residual_setting:
+ output_channel = int(c * width_mult)
+ for i in range(n):
+ stride = s if i == 0 else 1
+ features.append(
+ block(input_channel, output_channel, stride, expand_ratio=t)
+ )
+ input_channel = output_channel
+ # building last several layers
+ features.append(ConvBNReLU(input_channel, self.last_channel, kernel_size=1))
+ # make it nn.Sequential
+ self.features = nn.Sequential(*features)
+
+ # building classifier
+ self.classifier = nn.Sequential(
+ nn.Dropout(dropout),
+ nn.Linear(self.last_channel, num_classes),
+ )
+ self.message = "MobileNetV2 : width_mult={:}, in-C={:}, last-C={:}, block={:}, dropout={:}".format(
+ width_mult, input_channel, last_channel, block_name, dropout
+ )
+
+ # weight initialization
+ self.apply(initialize_resnet)
+
+ def get_message(self):
+ return self.message
+
+ def forward(self, inputs):
+ features = self.features(inputs)
+ vectors = features.mean([2, 3])
+ predicts = self.classifier(vectors)
+ return features, predicts
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/ImageNet_ResNet.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/ImageNet_ResNet.py
new file mode 100644
index 00000000..66d830a0
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/ImageNet_ResNet.py
@@ -0,0 +1,217 @@
+# Deep Residual Learning for Image Recognition, CVPR 2016
+import torch.nn as nn
+from .initialization import initialize_resnet
+
+
+def conv3x3(in_planes, out_planes, stride=1, groups=1):
+ return nn.Conv2d(
+ in_planes,
+ out_planes,
+ kernel_size=3,
+ stride=stride,
+ padding=1,
+ groups=groups,
+ bias=False,
+ )
+
+
+def conv1x1(in_planes, out_planes, stride=1):
+ return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
+
+
+class BasicBlock(nn.Module):
+ expansion = 1
+
+ def __init__(
+ self, inplanes, planes, stride=1, downsample=None, groups=1, base_width=64
+ ):
+ super(BasicBlock, self).__init__()
+ if groups != 1 or base_width != 64:
+ raise ValueError("BasicBlock only supports groups=1 and base_width=64")
+ # Both self.conv1 and self.downsample layers downsample the input when stride != 1
+ self.conv1 = conv3x3(inplanes, planes, stride)
+ self.bn1 = nn.BatchNorm2d(planes)
+ self.relu = nn.ReLU(inplace=True)
+ self.conv2 = conv3x3(planes, planes)
+ self.bn2 = nn.BatchNorm2d(planes)
+ self.downsample = downsample
+ self.stride = stride
+
+ def forward(self, x):
+ identity = x
+
+ out = self.conv1(x)
+ out = self.bn1(out)
+ out = self.relu(out)
+
+ out = self.conv2(out)
+ out = self.bn2(out)
+
+ if self.downsample is not None:
+ identity = self.downsample(x)
+
+ out += identity
+ out = self.relu(out)
+
+ return out
+
+
+class Bottleneck(nn.Module):
+ expansion = 4
+
+ def __init__(
+ self, inplanes, planes, stride=1, downsample=None, groups=1, base_width=64
+ ):
+ super(Bottleneck, self).__init__()
+ width = int(planes * (base_width / 64.0)) * groups
+ # Both self.conv2 and self.downsample layers downsample the input when stride != 1
+ self.conv1 = conv1x1(inplanes, width)
+ self.bn1 = nn.BatchNorm2d(width)
+ self.conv2 = conv3x3(width, width, stride, groups)
+ self.bn2 = nn.BatchNorm2d(width)
+ self.conv3 = conv1x1(width, planes * self.expansion)
+ self.bn3 = nn.BatchNorm2d(planes * self.expansion)
+ self.relu = nn.ReLU(inplace=True)
+ self.downsample = downsample
+ self.stride = stride
+
+ def forward(self, x):
+ identity = x
+
+ out = self.conv1(x)
+ out = self.bn1(out)
+ out = self.relu(out)
+
+ out = self.conv2(out)
+ out = self.bn2(out)
+ out = self.relu(out)
+
+ out = self.conv3(out)
+ out = self.bn3(out)
+
+ if self.downsample is not None:
+ identity = self.downsample(x)
+
+ out += identity
+ out = self.relu(out)
+
+ return out
+
+
+class ResNet(nn.Module):
+ def __init__(
+ self,
+ block_name,
+ layers,
+ deep_stem,
+ num_classes,
+ zero_init_residual,
+ groups,
+ width_per_group,
+ ):
+ super(ResNet, self).__init__()
+
+ # planes = [int(width_per_group * groups * 2 ** i) for i in range(4)]
+ if block_name == "BasicBlock":
+ block = BasicBlock
+ elif block_name == "Bottleneck":
+ block = Bottleneck
+ else:
+ raise ValueError("invalid block-name : {:}".format(block_name))
+
+ if not deep_stem:
+ self.conv = nn.Sequential(
+ nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False),
+ nn.BatchNorm2d(64),
+ nn.ReLU(inplace=True),
+ )
+ else:
+ self.conv = nn.Sequential(
+ nn.Conv2d(3, 32, kernel_size=3, stride=2, padding=1, bias=False),
+ nn.BatchNorm2d(32),
+ nn.ReLU(inplace=True),
+ nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1, bias=False),
+ nn.BatchNorm2d(32),
+ nn.ReLU(inplace=True),
+ nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1, bias=False),
+ nn.BatchNorm2d(64),
+ nn.ReLU(inplace=True),
+ )
+ self.inplanes = 64
+ self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
+ self.layer1 = self._make_layer(
+ block, 64, layers[0], stride=1, groups=groups, base_width=width_per_group
+ )
+ self.layer2 = self._make_layer(
+ block, 128, layers[1], stride=2, groups=groups, base_width=width_per_group
+ )
+ self.layer3 = self._make_layer(
+ block, 256, layers[2], stride=2, groups=groups, base_width=width_per_group
+ )
+ self.layer4 = self._make_layer(
+ block, 512, layers[3], stride=2, groups=groups, base_width=width_per_group
+ )
+ self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
+ self.fc = nn.Linear(512 * block.expansion, num_classes)
+ self.message = (
+ "block = {:}, layers = {:}, deep_stem = {:}, num_classes = {:}".format(
+ block, layers, deep_stem, num_classes
+ )
+ )
+
+ self.apply(initialize_resnet)
+
+ # Zero-initialize the last BN in each residual branch,
+ # so that the residual branch starts with zeros, and each residual block behaves like an identity.
+ # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
+ if zero_init_residual:
+ for m in self.modules():
+ if isinstance(m, Bottleneck):
+ nn.init.constant_(m.bn3.weight, 0)
+ elif isinstance(m, BasicBlock):
+ nn.init.constant_(m.bn2.weight, 0)
+
+ def _make_layer(self, block, planes, blocks, stride, groups, base_width):
+ downsample = None
+ if stride != 1 or self.inplanes != planes * block.expansion:
+ if stride == 2:
+ downsample = nn.Sequential(
+ nn.AvgPool2d(kernel_size=2, stride=2, padding=0),
+ conv1x1(self.inplanes, planes * block.expansion, 1),
+ nn.BatchNorm2d(planes * block.expansion),
+ )
+ elif stride == 1:
+ downsample = nn.Sequential(
+ conv1x1(self.inplanes, planes * block.expansion, stride),
+ nn.BatchNorm2d(planes * block.expansion),
+ )
+ else:
+ raise ValueError("invalid stride [{:}] for downsample".format(stride))
+
+ layers = []
+ layers.append(
+ block(self.inplanes, planes, stride, downsample, groups, base_width)
+ )
+ self.inplanes = planes * block.expansion
+ for _ in range(1, blocks):
+ layers.append(block(self.inplanes, planes, 1, None, groups, base_width))
+
+ return nn.Sequential(*layers)
+
+ def get_message(self):
+ return self.message
+
+ def forward(self, x):
+ x = self.conv(x)
+ x = self.maxpool(x)
+
+ x = self.layer1(x)
+ x = self.layer2(x)
+ x = self.layer3(x)
+ x = self.layer4(x)
+
+ features = self.avgpool(x)
+ features = features.view(features.size(0), -1)
+ logits = self.fc(features)
+
+ return features, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/SharedUtils.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/SharedUtils.py
new file mode 100644
index 00000000..adcdf8bc
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/SharedUtils.py
@@ -0,0 +1,37 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+import torch
+import torch.nn as nn
+
+
+def additive_func(A, B):
+ assert A.dim() == B.dim() and A.size(0) == B.size(0), "{:} vs {:}".format(
+ A.size(), B.size()
+ )
+ C = min(A.size(1), B.size(1))
+ if A.size(1) == B.size(1):
+ return A + B
+ elif A.size(1) < B.size(1):
+ out = B.clone()
+ out[:, :C] += A
+ return out
+ else:
+ out = A.clone()
+ out[:, :C] += B
+ return out
+
+
+def change_key(key, value):
+ def func(m):
+ if hasattr(m, key):
+ setattr(m, key, value)
+
+ return func
+
+
+def parse_channel_info(xstring):
+ blocks = xstring.split(" ")
+ blocks = [x.split("-") for x in blocks]
+ blocks = [[int(_) for _ in x] for x in blocks]
+ return blocks
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/__init__.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/__init__.py
new file mode 100644
index 00000000..8da05e39
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/__init__.py
@@ -0,0 +1,335 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+##################################################
+from collections import namedtuple
+from os import path as osp
+from typing import List, Text
+import torch
+
+__all__ = [
+ "change_key",
+ "get_cell_based_tiny_net",
+ "get_search_spaces",
+ "get_cifar_models",
+ "get_imagenet_models",
+ "obtain_model",
+ "obtain_search_model",
+ "load_net_from_checkpoint",
+ "CellStructure",
+ "CellArchitectures",
+]
+
+def dict2config(xdict, logger):
+ assert isinstance(xdict, dict), "invalid type : {:}".format(type(xdict))
+ Arguments = namedtuple("Configure", " ".join(xdict.keys()))
+ content = Arguments(**xdict)
+ if hasattr(logger, "log"):
+ logger.log("{:}".format(content))
+ return content
+
+# useful modules
+# from .config_utils import dict2config
+from .SharedUtils import change_key
+from .cell_searchs import CellStructure, CellArchitectures
+
+
+# Cell-based NAS Models
+def get_cell_based_tiny_net(config):
+ if isinstance(config, dict):
+ config = dict2config(config, None) # to support the argument being a dict
+ super_type = getattr(config, "super_type", "basic")
+ group_names = ["DARTS-V1", "DARTS-V2", "GDAS", "SETN", "ENAS", "RANDOM", "generic"]
+ if super_type == "basic" and config.name in group_names:
+ from .cell_searchs import nas201_super_nets as nas_super_nets
+
+ try:
+ return nas_super_nets[config.name](
+ config.C,
+ config.N,
+ config.max_nodes,
+ config.num_classes,
+ config.space,
+ config.affine,
+ config.track_running_stats,
+ )
+ except:
+ return nas_super_nets[config.name](
+ config.C, config.N, config.max_nodes, config.num_classes, config.space
+ )
+ elif super_type == "search-shape":
+ from .shape_searchs import GenericNAS301Model
+
+ genotype = CellStructure.str2structure(config.genotype)
+ return GenericNAS301Model(
+ config.candidate_Cs,
+ config.max_num_Cs,
+ genotype,
+ config.num_classes,
+ config.affine,
+ config.track_running_stats,
+ )
+ elif super_type == "nasnet-super":
+ from .cell_searchs import nasnet_super_nets as nas_super_nets
+
+ return nas_super_nets[config.name](
+ config.C,
+ config.N,
+ config.steps,
+ config.multiplier,
+ config.stem_multiplier,
+ config.num_classes,
+ config.space,
+ config.affine,
+ config.track_running_stats,
+ )
+ elif config.name == "infer.tiny":
+ from .cell_infers import TinyNetwork
+
+ if hasattr(config, "genotype"):
+ genotype = config.genotype
+ elif hasattr(config, "arch_str"):
+ genotype = CellStructure.str2structure(config.arch_str)
+ else:
+ raise ValueError(
+ "Can not find genotype from this config : {:}".format(config)
+ )
+ return TinyNetwork(config.C, config.N, genotype, config.num_classes)
+ elif config.name == "infer.shape.tiny":
+ from .shape_infers import DynamicShapeTinyNet
+
+ if isinstance(config.channels, str):
+ channels = tuple([int(x) for x in config.channels.split(":")])
+ else:
+ channels = config.channels
+ genotype = CellStructure.str2structure(config.genotype)
+ return DynamicShapeTinyNet(channels, genotype, config.num_classes)
+ elif config.name == "infer.nasnet-cifar":
+ from .cell_infers import NASNetonCIFAR
+
+ raise NotImplementedError
+ else:
+ raise ValueError("invalid network name : {:}".format(config.name))
+
+
+# obtain the search space, i.e., a dict mapping the operation name into a python-function for this op
+def get_search_spaces(xtype, name) -> List[Text]:
+ if xtype == "cell" or xtype == "tss": # The topology search space.
+ from .cell_operations import SearchSpaceNames
+
+ assert name in SearchSpaceNames, "invalid name [{:}] in {:}".format(
+ name, SearchSpaceNames.keys()
+ )
+ return SearchSpaceNames[name]
+ elif xtype == "sss": # The size search space.
+ if name in ["nats-bench", "nats-bench-size"]:
+ return {"candidates": [8, 16, 24, 32, 40, 48, 56, 64], "numbers": 5}
+ else:
+ raise ValueError("Invalid name : {:}".format(name))
+ else:
+ raise ValueError("invalid search-space type is {:}".format(xtype))
+
+
+def get_cifar_models(config, extra_path=None):
+ super_type = getattr(config, "super_type", "basic")
+ if super_type == "basic":
+ from .CifarResNet import CifarResNet
+ from .CifarDenseNet import DenseNet
+ from .CifarWideResNet import CifarWideResNet
+
+ if config.arch == "resnet":
+ return CifarResNet(
+ config.module, config.depth, config.class_num, config.zero_init_residual
+ )
+ elif config.arch == "densenet":
+ return DenseNet(
+ config.growthRate,
+ config.depth,
+ config.reduction,
+ config.class_num,
+ config.bottleneck,
+ )
+ elif config.arch == "wideresnet":
+ return CifarWideResNet(
+ config.depth, config.wide_factor, config.class_num, config.dropout
+ )
+ else:
+ raise ValueError("invalid module type : {:}".format(config.arch))
+ elif super_type.startswith("infer"):
+ from .shape_infers import InferWidthCifarResNet
+ from .shape_infers import InferDepthCifarResNet
+ from .shape_infers import InferCifarResNet
+ from .cell_infers import NASNetonCIFAR
+
+ assert len(super_type.split("-")) == 2, "invalid super_type : {:}".format(
+ super_type
+ )
+ infer_mode = super_type.split("-")[1]
+ if infer_mode == "width":
+ return InferWidthCifarResNet(
+ config.module,
+ config.depth,
+ config.xchannels,
+ config.class_num,
+ config.zero_init_residual,
+ )
+ elif infer_mode == "depth":
+ return InferDepthCifarResNet(
+ config.module,
+ config.depth,
+ config.xblocks,
+ config.class_num,
+ config.zero_init_residual,
+ )
+ elif infer_mode == "shape":
+ return InferCifarResNet(
+ config.module,
+ config.depth,
+ config.xblocks,
+ config.xchannels,
+ config.class_num,
+ config.zero_init_residual,
+ )
+ elif infer_mode == "nasnet.cifar":
+ genotype = config.genotype
+ if extra_path is not None: # reload genotype by extra_path
+ if not osp.isfile(extra_path):
+ raise ValueError("invalid extra_path : {:}".format(extra_path))
+ xdata = torch.load(extra_path)
+ current_epoch = xdata["epoch"]
+ genotype = xdata["genotypes"][current_epoch - 1]
+ C = config.C if hasattr(config, "C") else config.ichannel
+ N = config.N if hasattr(config, "N") else config.layers
+ return NASNetonCIFAR(
+ C, N, config.stem_multi, config.class_num, genotype, config.auxiliary
+ )
+ else:
+ raise ValueError("invalid infer-mode : {:}".format(infer_mode))
+ else:
+ raise ValueError("invalid super-type : {:}".format(super_type))
+
+
+def get_imagenet_models(config):
+ super_type = getattr(config, "super_type", "basic")
+ if super_type == "basic":
+ from .ImageNet_ResNet import ResNet
+ from .ImageNet_MobileNetV2 import MobileNetV2
+
+ if config.arch == "resnet":
+ return ResNet(
+ config.block_name,
+ config.layers,
+ config.deep_stem,
+ config.class_num,
+ config.zero_init_residual,
+ config.groups,
+ config.width_per_group,
+ )
+ elif config.arch == "mobilenet_v2":
+ return MobileNetV2(
+ config.class_num,
+ config.width_multi,
+ config.input_channel,
+ config.last_channel,
+ "InvertedResidual",
+ config.dropout,
+ )
+ else:
+ raise ValueError("invalid arch : {:}".format(config.arch))
+ elif super_type.startswith("infer"): # NAS searched architecture
+ assert len(super_type.split("-")) == 2, "invalid super_type : {:}".format(
+ super_type
+ )
+ infer_mode = super_type.split("-")[1]
+ if infer_mode == "shape":
+ from .shape_infers import InferImagenetResNet
+ from .shape_infers import InferMobileNetV2
+
+ if config.arch == "resnet":
+ return InferImagenetResNet(
+ config.block_name,
+ config.layers,
+ config.xblocks,
+ config.xchannels,
+ config.deep_stem,
+ config.class_num,
+ config.zero_init_residual,
+ )
+ elif config.arch == "MobileNetV2":
+ return InferMobileNetV2(
+ config.class_num, config.xchannels, config.xblocks, config.dropout
+ )
+ else:
+ raise ValueError("invalid arch-mode : {:}".format(config.arch))
+ else:
+ raise ValueError("invalid infer-mode : {:}".format(infer_mode))
+ else:
+ raise ValueError("invalid super-type : {:}".format(super_type))
+
+
+# Try to obtain the network by config.
+def obtain_model(config, extra_path=None):
+ if config.dataset == "cifar":
+ return get_cifar_models(config, extra_path)
+ elif config.dataset == "imagenet":
+ return get_imagenet_models(config)
+ else:
+ raise ValueError("invalid dataset in the model config : {:}".format(config))
+
+
+def obtain_search_model(config):
+ if config.dataset == "cifar":
+ if config.arch == "resnet":
+ from .shape_searchs import SearchWidthCifarResNet
+ from .shape_searchs import SearchDepthCifarResNet
+ from .shape_searchs import SearchShapeCifarResNet
+
+ if config.search_mode == "width":
+ return SearchWidthCifarResNet(
+ config.module, config.depth, config.class_num
+ )
+ elif config.search_mode == "depth":
+ return SearchDepthCifarResNet(
+ config.module, config.depth, config.class_num
+ )
+ elif config.search_mode == "shape":
+ return SearchShapeCifarResNet(
+ config.module, config.depth, config.class_num
+ )
+ else:
+ raise ValueError("invalid search mode : {:}".format(config.search_mode))
+ elif config.arch == "simres":
+ from .shape_searchs import SearchWidthSimResNet
+
+ if config.search_mode == "width":
+ return SearchWidthSimResNet(config.depth, config.class_num)
+ else:
+ raise ValueError("invalid search mode : {:}".format(config.search_mode))
+ else:
+ raise ValueError(
+ "invalid arch : {:} for dataset [{:}]".format(
+ config.arch, config.dataset
+ )
+ )
+ elif config.dataset == "imagenet":
+ from .shape_searchs import SearchShapeImagenetResNet
+
+ assert config.search_mode == "shape", "invalid search-mode : {:}".format(
+ config.search_mode
+ )
+ if config.arch == "resnet":
+ return SearchShapeImagenetResNet(
+ config.block_name, config.layers, config.deep_stem, config.class_num
+ )
+ else:
+ raise ValueError("invalid model config : {:}".format(config))
+ else:
+ raise ValueError("invalid dataset in the model config : {:}".format(config))
+
+
+def load_net_from_checkpoint(checkpoint):
+ assert osp.isfile(checkpoint), "checkpoint {:} does not exist".format(checkpoint)
+ checkpoint = torch.load(checkpoint)
+ model_config = dict2config(checkpoint["model-config"], None)
+ model = obtain_model(model_config)
+ model.load_state_dict(checkpoint["base-model"])
+ return model
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_infers/__init__.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_infers/__init__.py
new file mode 100644
index 00000000..ac1a183d
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_infers/__init__.py
@@ -0,0 +1,5 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+from .tiny_network import TinyNetwork
+from .nasnet_cifar import NASNetonCIFAR
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_infers/cells.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_infers/cells.py
new file mode 100644
index 00000000..40df57b4
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_infers/cells.py
@@ -0,0 +1,155 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+
+import torch
+import torch.nn as nn
+from copy import deepcopy
+
+from models.cell_operations import OPS
+
+
+# Cell for NAS-Bench-201
+class InferCell(nn.Module):
+ def __init__(
+ self, genotype, C_in, C_out, stride, affine=True, track_running_stats=True
+ ):
+ super(InferCell, self).__init__()
+
+ self.layers = nn.ModuleList()
+ self.node_IN = []
+ self.node_IX = []
+ self.genotype = deepcopy(genotype)
+ for i in range(1, len(genotype)):
+ node_info = genotype[i - 1]
+ cur_index = []
+ cur_innod = []
+ for (op_name, op_in) in node_info:
+ if op_in == 0:
+ layer = OPS[op_name](
+ C_in, C_out, stride, affine, track_running_stats
+ )
+ else:
+ layer = OPS[op_name](C_out, C_out, 1, affine, track_running_stats)
+ cur_index.append(len(self.layers))
+ cur_innod.append(op_in)
+ self.layers.append(layer)
+ self.node_IX.append(cur_index)
+ self.node_IN.append(cur_innod)
+ self.nodes = len(genotype)
+ self.in_dim = C_in
+ self.out_dim = C_out
+
+ def extra_repr(self):
+ string = "info :: nodes={nodes}, inC={in_dim}, outC={out_dim}".format(
+ **self.__dict__
+ )
+ laystr = []
+ for i, (node_layers, node_innods) in enumerate(zip(self.node_IX, self.node_IN)):
+ y = [
+ "I{:}-L{:}".format(_ii, _il)
+ for _il, _ii in zip(node_layers, node_innods)
+ ]
+ x = "{:}<-({:})".format(i + 1, ",".join(y))
+ laystr.append(x)
+ return (
+ string
+ + ", [{:}]".format(" | ".join(laystr))
+ + ", {:}".format(self.genotype.tostr())
+ )
+
+ def forward(self, inputs):
+ nodes = [inputs]
+ for i, (node_layers, node_innods) in enumerate(zip(self.node_IX, self.node_IN)):
+ node_feature = sum(
+ self.layers[_il](nodes[_ii])
+ for _il, _ii in zip(node_layers, node_innods)
+ )
+ nodes.append(node_feature)
+ return nodes[-1]
+
+
+# Learning Transferable Architectures for Scalable Image Recognition, CVPR 2018
+class NASNetInferCell(nn.Module):
+ def __init__(
+ self,
+ genotype,
+ C_prev_prev,
+ C_prev,
+ C,
+ reduction,
+ reduction_prev,
+ affine,
+ track_running_stats,
+ ):
+ super(NASNetInferCell, self).__init__()
+ self.reduction = reduction
+ if reduction_prev:
+ self.preprocess0 = OPS["skip_connect"](
+ C_prev_prev, C, 2, affine, track_running_stats
+ )
+ else:
+ self.preprocess0 = OPS["nor_conv_1x1"](
+ C_prev_prev, C, 1, affine, track_running_stats
+ )
+ self.preprocess1 = OPS["nor_conv_1x1"](
+ C_prev, C, 1, affine, track_running_stats
+ )
+
+ if not reduction:
+ nodes, concats = genotype["normal"], genotype["normal_concat"]
+ else:
+ nodes, concats = genotype["reduce"], genotype["reduce_concat"]
+ self._multiplier = len(concats)
+ self._concats = concats
+ self._steps = len(nodes)
+ self._nodes = nodes
+ self.edges = nn.ModuleDict()
+ for i, node in enumerate(nodes):
+ for in_node in node:
+ name, j = in_node[0], in_node[1]
+ stride = 2 if reduction and j < 2 else 1
+ node_str = "{:}<-{:}".format(i + 2, j)
+ self.edges[node_str] = OPS[name](
+ C, C, stride, affine, track_running_stats
+ )
+
+ # [TODO] to support drop_prob in this function..
+ def forward(self, s0, s1, unused_drop_prob):
+ s0 = self.preprocess0(s0)
+ s1 = self.preprocess1(s1)
+
+ states = [s0, s1]
+ for i, node in enumerate(self._nodes):
+ clist = []
+ for in_node in node:
+ name, j = in_node[0], in_node[1]
+ node_str = "{:}<-{:}".format(i + 2, j)
+ op = self.edges[node_str]
+ clist.append(op(states[j]))
+ states.append(sum(clist))
+ return torch.cat([states[x] for x in self._concats], dim=1)
+
+
+class AuxiliaryHeadCIFAR(nn.Module):
+ def __init__(self, C, num_classes):
+ """assuming input size 8x8"""
+ super(AuxiliaryHeadCIFAR, self).__init__()
+ self.features = nn.Sequential(
+ nn.ReLU(inplace=True),
+ nn.AvgPool2d(
+ 5, stride=3, padding=0, count_include_pad=False
+ ), # image size = 2 x 2
+ nn.Conv2d(C, 128, 1, bias=False),
+ nn.BatchNorm2d(128),
+ nn.ReLU(inplace=True),
+ nn.Conv2d(128, 768, 2, bias=False),
+ nn.BatchNorm2d(768),
+ nn.ReLU(inplace=True),
+ )
+ self.classifier = nn.Linear(768, num_classes)
+
+ def forward(self, x):
+ x = self.features(x)
+ x = self.classifier(x.view(x.size(0), -1))
+ return x
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_infers/nasnet_cifar.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_infers/nasnet_cifar.py
new file mode 100644
index 00000000..21094777
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_infers/nasnet_cifar.py
@@ -0,0 +1,118 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+import torch
+import torch.nn as nn
+from copy import deepcopy
+
+from .cells import NASNetInferCell as InferCell, AuxiliaryHeadCIFAR
+
+
+# The macro structure is based on NASNet
+class NASNetonCIFAR(nn.Module):
+ def __init__(
+ self,
+ C,
+ N,
+ stem_multiplier,
+ num_classes,
+ genotype,
+ auxiliary,
+ affine=True,
+ track_running_stats=True,
+ ):
+ super(NASNetonCIFAR, self).__init__()
+ self._C = C
+ self._layerN = N
+ self.stem = nn.Sequential(
+ nn.Conv2d(3, C * stem_multiplier, kernel_size=3, padding=1, bias=False),
+ nn.BatchNorm2d(C * stem_multiplier),
+ )
+
+ # config for each layer
+ layer_channels = (
+ [C] * N + [C * 2] + [C * 2] * (N - 1) + [C * 4] + [C * 4] * (N - 1)
+ )
+ layer_reductions = (
+ [False] * N + [True] + [False] * (N - 1) + [True] + [False] * (N - 1)
+ )
+
+ C_prev_prev, C_prev, C_curr, reduction_prev = (
+ C * stem_multiplier,
+ C * stem_multiplier,
+ C,
+ False,
+ )
+ self.auxiliary_index = None
+ self.auxiliary_head = None
+ self.cells = nn.ModuleList()
+ for index, (C_curr, reduction) in enumerate(
+ zip(layer_channels, layer_reductions)
+ ):
+ cell = InferCell(
+ genotype,
+ C_prev_prev,
+ C_prev,
+ C_curr,
+ reduction,
+ reduction_prev,
+ affine,
+ track_running_stats,
+ )
+ self.cells.append(cell)
+ C_prev_prev, C_prev, reduction_prev = (
+ C_prev,
+ cell._multiplier * C_curr,
+ reduction,
+ )
+ if reduction and C_curr == C * 4 and auxiliary:
+ self.auxiliary_head = AuxiliaryHeadCIFAR(C_prev, num_classes)
+ self.auxiliary_index = index
+ self._Layer = len(self.cells)
+ self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True))
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(C_prev, num_classes)
+ self.drop_path_prob = -1
+
+ def update_drop_path(self, drop_path_prob):
+ self.drop_path_prob = drop_path_prob
+
+ def auxiliary_param(self):
+ if self.auxiliary_head is None:
+ return []
+ else:
+ return list(self.auxiliary_head.parameters())
+
+ def get_message(self):
+ string = self.extra_repr()
+ for i, cell in enumerate(self.cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self.cells), cell.extra_repr()
+ )
+ return string
+
+ def extra_repr(self):
+ return "{name}(C={_C}, N={_layerN}, L={_Layer})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ def forward(self, inputs):
+ stem_feature, logits_aux = self.stem(inputs), None
+ cell_results = [stem_feature, stem_feature]
+ for i, cell in enumerate(self.cells):
+ cell_feature = cell(cell_results[-2], cell_results[-1], self.drop_path_prob)
+ cell_results.append(cell_feature)
+ if (
+ self.auxiliary_index is not None
+ and i == self.auxiliary_index
+ and self.training
+ ):
+ logits_aux = self.auxiliary_head(cell_results[-1])
+ out = self.lastact(cell_results[-1])
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+ if logits_aux is None:
+ return out, logits
+ else:
+ return out, [logits, logits_aux]
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_infers/tiny_network.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_infers/tiny_network.py
new file mode 100644
index 00000000..e8da1e40
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_infers/tiny_network.py
@@ -0,0 +1,63 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+import torch.nn as nn
+from ..cell_operations import ResNetBasicblock
+from .cells import InferCell
+
+
+# The macro structure for architectures in NAS-Bench-201
+class TinyNetwork(nn.Module):
+ def __init__(self, C, N, genotype, num_classes):
+ super(TinyNetwork, self).__init__()
+ self._C = C
+ self._layerN = N
+
+ self.stem = nn.Sequential(
+ nn.Conv2d(3, C, kernel_size=3, padding=1, bias=False), nn.BatchNorm2d(C)
+ )
+
+ layer_channels = [C] * N + [C * 2] + [C * 2] * N + [C * 4] + [C * 4] * N
+ layer_reductions = [False] * N + [True] + [False] * N + [True] + [False] * N
+
+ C_prev = C
+ self.cells = nn.ModuleList()
+ for index, (C_curr, reduction) in enumerate(
+ zip(layer_channels, layer_reductions)
+ ):
+ if reduction:
+ cell = ResNetBasicblock(C_prev, C_curr, 2, True)
+ else:
+ cell = InferCell(genotype, C_prev, C_curr, 1)
+ self.cells.append(cell)
+ C_prev = cell.out_dim
+ self._Layer = len(self.cells)
+
+ self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True))
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(C_prev, num_classes)
+
+ def get_message(self):
+ string = self.extra_repr()
+ for i, cell in enumerate(self.cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self.cells), cell.extra_repr()
+ )
+ return string
+
+ def extra_repr(self):
+ return "{name}(C={_C}, N={_layerN}, L={_Layer})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ def forward(self, inputs):
+ feature = self.stem(inputs)
+ for i, cell in enumerate(self.cells):
+ feature = cell(feature)
+
+ out = self.lastact(feature)
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+
+ return out, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_operations.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_operations.py
new file mode 100644
index 00000000..051539cf
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_operations.py
@@ -0,0 +1,553 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+##################################################
+import torch
+import torch.nn as nn
+
+__all__ = ["OPS", "RAW_OP_CLASSES", "ResNetBasicblock", "SearchSpaceNames"]
+
+OPS = {
+ "none": lambda C_in, C_out, stride, affine, track_running_stats: Zero(
+ C_in, C_out, stride
+ ),
+ "avg_pool_3x3": lambda C_in, C_out, stride, affine, track_running_stats: POOLING(
+ C_in, C_out, stride, "avg", affine, track_running_stats
+ ),
+ "max_pool_3x3": lambda C_in, C_out, stride, affine, track_running_stats: POOLING(
+ C_in, C_out, stride, "max", affine, track_running_stats
+ ),
+ "nor_conv_7x7": lambda C_in, C_out, stride, affine, track_running_stats: ReLUConvBN(
+ C_in,
+ C_out,
+ (7, 7),
+ (stride, stride),
+ (3, 3),
+ (1, 1),
+ affine,
+ track_running_stats,
+ ),
+ "nor_conv_3x3": lambda C_in, C_out, stride, affine, track_running_stats: ReLUConvBN(
+ C_in,
+ C_out,
+ (3, 3),
+ (stride, stride),
+ (1, 1),
+ (1, 1),
+ affine,
+ track_running_stats,
+ ),
+ "nor_conv_1x1": lambda C_in, C_out, stride, affine, track_running_stats: ReLUConvBN(
+ C_in,
+ C_out,
+ (1, 1),
+ (stride, stride),
+ (0, 0),
+ (1, 1),
+ affine,
+ track_running_stats,
+ ),
+ "dua_sepc_3x3": lambda C_in, C_out, stride, affine, track_running_stats: DualSepConv(
+ C_in,
+ C_out,
+ (3, 3),
+ (stride, stride),
+ (1, 1),
+ (1, 1),
+ affine,
+ track_running_stats,
+ ),
+ "dua_sepc_5x5": lambda C_in, C_out, stride, affine, track_running_stats: DualSepConv(
+ C_in,
+ C_out,
+ (5, 5),
+ (stride, stride),
+ (2, 2),
+ (1, 1),
+ affine,
+ track_running_stats,
+ ),
+ "dil_sepc_3x3": lambda C_in, C_out, stride, affine, track_running_stats: SepConv(
+ C_in,
+ C_out,
+ (3, 3),
+ (stride, stride),
+ (2, 2),
+ (2, 2),
+ affine,
+ track_running_stats,
+ ),
+ "dil_sepc_5x5": lambda C_in, C_out, stride, affine, track_running_stats: SepConv(
+ C_in,
+ C_out,
+ (5, 5),
+ (stride, stride),
+ (4, 4),
+ (2, 2),
+ affine,
+ track_running_stats,
+ ),
+ "skip_connect": lambda C_in, C_out, stride, affine, track_running_stats: Identity()
+ if stride == 1 and C_in == C_out
+ else FactorizedReduce(C_in, C_out, stride, affine, track_running_stats),
+}
+
+CONNECT_NAS_BENCHMARK = ["none", "skip_connect", "nor_conv_3x3"]
+NAS_BENCH_201 = ["none", "skip_connect", "nor_conv_1x1", "nor_conv_3x3", "avg_pool_3x3"]
+DARTS_SPACE = [
+ "none",
+ "skip_connect",
+ "dua_sepc_3x3",
+ "dua_sepc_5x5",
+ "dil_sepc_3x3",
+ "dil_sepc_5x5",
+ "avg_pool_3x3",
+ "max_pool_3x3",
+]
+
+SearchSpaceNames = {
+ "connect-nas": CONNECT_NAS_BENCHMARK,
+ "nats-bench": NAS_BENCH_201,
+ "nas-bench-201": NAS_BENCH_201,
+ "darts": DARTS_SPACE,
+}
+
+
+class ReLUConvBN(nn.Module):
+ def __init__(
+ self,
+ C_in,
+ C_out,
+ kernel_size,
+ stride,
+ padding,
+ dilation,
+ affine,
+ track_running_stats=True,
+ ):
+ super(ReLUConvBN, self).__init__()
+ self.op = nn.Sequential(
+ nn.ReLU(inplace=False),
+ nn.Conv2d(
+ C_in,
+ C_out,
+ kernel_size,
+ stride=stride,
+ padding=padding,
+ dilation=dilation,
+ bias=not affine,
+ ),
+ nn.BatchNorm2d(
+ C_out, affine=affine, track_running_stats=track_running_stats
+ ),
+ )
+
+ def forward(self, x):
+ return self.op(x)
+
+
+class SepConv(nn.Module):
+ def __init__(
+ self,
+ C_in,
+ C_out,
+ kernel_size,
+ stride,
+ padding,
+ dilation,
+ affine,
+ track_running_stats=True,
+ ):
+ super(SepConv, self).__init__()
+ self.op = nn.Sequential(
+ nn.ReLU(inplace=False),
+ nn.Conv2d(
+ C_in,
+ C_in,
+ kernel_size=kernel_size,
+ stride=stride,
+ padding=padding,
+ dilation=dilation,
+ groups=C_in,
+ bias=False,
+ ),
+ nn.Conv2d(C_in, C_out, kernel_size=1, padding=0, bias=not affine),
+ nn.BatchNorm2d(
+ C_out, affine=affine, track_running_stats=track_running_stats
+ ),
+ )
+
+ def forward(self, x):
+ return self.op(x)
+
+
+class DualSepConv(nn.Module):
+ def __init__(
+ self,
+ C_in,
+ C_out,
+ kernel_size,
+ stride,
+ padding,
+ dilation,
+ affine,
+ track_running_stats=True,
+ ):
+ super(DualSepConv, self).__init__()
+ self.op_a = SepConv(
+ C_in,
+ C_in,
+ kernel_size,
+ stride,
+ padding,
+ dilation,
+ affine,
+ track_running_stats,
+ )
+ self.op_b = SepConv(
+ C_in, C_out, kernel_size, 1, padding, dilation, affine, track_running_stats
+ )
+
+ def forward(self, x):
+ x = self.op_a(x)
+ x = self.op_b(x)
+ return x
+
+
+class ResNetBasicblock(nn.Module):
+ def __init__(self, inplanes, planes, stride, affine=True, track_running_stats=True):
+ super(ResNetBasicblock, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+ self.conv_a = ReLUConvBN(
+ inplanes, planes, 3, stride, 1, 1, affine, track_running_stats
+ )
+ self.conv_b = ReLUConvBN(
+ planes, planes, 3, 1, 1, 1, affine, track_running_stats
+ )
+ if stride == 2:
+ self.downsample = nn.Sequential(
+ nn.AvgPool2d(kernel_size=2, stride=2, padding=0),
+ nn.Conv2d(
+ inplanes, planes, kernel_size=1, stride=1, padding=0, bias=False
+ ),
+ )
+ elif inplanes != planes:
+ self.downsample = ReLUConvBN(
+ inplanes, planes, 1, 1, 0, 1, affine, track_running_stats
+ )
+ else:
+ self.downsample = None
+ self.in_dim = inplanes
+ self.out_dim = planes
+ self.stride = stride
+ self.num_conv = 2
+
+ def extra_repr(self):
+ string = "{name}(inC={in_dim}, outC={out_dim}, stride={stride})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+ return string
+
+ def forward(self, inputs):
+
+ basicblock = self.conv_a(inputs)
+ basicblock = self.conv_b(basicblock)
+
+ if self.downsample is not None:
+ residual = self.downsample(inputs)
+ else:
+ residual = inputs
+ return residual + basicblock
+
+
+class POOLING(nn.Module):
+ def __init__(
+ self, C_in, C_out, stride, mode, affine=True, track_running_stats=True
+ ):
+ super(POOLING, self).__init__()
+ if C_in == C_out:
+ self.preprocess = None
+ else:
+ self.preprocess = ReLUConvBN(
+ C_in, C_out, 1, 1, 0, 1, affine, track_running_stats
+ )
+ if mode == "avg":
+ self.op = nn.AvgPool2d(3, stride=stride, padding=1, count_include_pad=False)
+ elif mode == "max":
+ self.op = nn.MaxPool2d(3, stride=stride, padding=1)
+ else:
+ raise ValueError("Invalid mode={:} in POOLING".format(mode))
+
+ def forward(self, inputs):
+ if self.preprocess:
+ x = self.preprocess(inputs)
+ else:
+ x = inputs
+ return self.op(x)
+
+
+class Identity(nn.Module):
+ def __init__(self):
+ super(Identity, self).__init__()
+
+ def forward(self, x):
+ return x
+
+
+class Zero(nn.Module):
+ def __init__(self, C_in, C_out, stride):
+ super(Zero, self).__init__()
+ self.C_in = C_in
+ self.C_out = C_out
+ self.stride = stride
+ self.is_zero = True
+
+ def forward(self, x):
+ if self.C_in == self.C_out:
+ if self.stride == 1:
+ return x.mul(0.0)
+ else:
+ return x[:, :, :: self.stride, :: self.stride].mul(0.0)
+ else:
+ shape = list(x.shape)
+ shape[1] = self.C_out
+ zeros = x.new_zeros(shape, dtype=x.dtype, device=x.device)
+ return zeros
+
+ def extra_repr(self):
+ return "C_in={C_in}, C_out={C_out}, stride={stride}".format(**self.__dict__)
+
+
+class FactorizedReduce(nn.Module):
+ def __init__(self, C_in, C_out, stride, affine, track_running_stats):
+ super(FactorizedReduce, self).__init__()
+ self.stride = stride
+ self.C_in = C_in
+ self.C_out = C_out
+ self.relu = nn.ReLU(inplace=False)
+ if stride == 2:
+ # assert C_out % 2 == 0, 'C_out : {:}'.format(C_out)
+ C_outs = [C_out // 2, C_out - C_out // 2]
+ self.convs = nn.ModuleList()
+ for i in range(2):
+ self.convs.append(
+ nn.Conv2d(
+ C_in, C_outs[i], 1, stride=stride, padding=0, bias=not affine
+ )
+ )
+ self.pad = nn.ConstantPad2d((0, 1, 0, 1), 0)
+ elif stride == 1:
+ self.conv = nn.Conv2d(
+ C_in, C_out, 1, stride=stride, padding=0, bias=not affine
+ )
+ else:
+ raise ValueError("Invalid stride : {:}".format(stride))
+ self.bn = nn.BatchNorm2d(
+ C_out, affine=affine, track_running_stats=track_running_stats
+ )
+
+ def forward(self, x):
+ if self.stride == 2:
+ x = self.relu(x)
+ y = self.pad(x)
+ out = torch.cat([self.convs[0](x), self.convs[1](y[:, :, 1:, 1:])], dim=1)
+ else:
+ out = self.conv(x)
+ out = self.bn(out)
+ return out
+
+ def extra_repr(self):
+ return "C_in={C_in}, C_out={C_out}, stride={stride}".format(**self.__dict__)
+
+
+# Auto-ReID: Searching for a Part-Aware ConvNet for Person Re-Identification, ICCV 2019
+class PartAwareOp(nn.Module):
+ def __init__(self, C_in, C_out, stride, part=4):
+ super().__init__()
+ self.part = 4
+ self.hidden = C_in // 3
+ self.avg_pool = nn.AdaptiveAvgPool2d(1)
+ self.local_conv_list = nn.ModuleList()
+ for i in range(self.part):
+ self.local_conv_list.append(
+ nn.Sequential(
+ nn.ReLU(),
+ nn.Conv2d(C_in, self.hidden, 1),
+ nn.BatchNorm2d(self.hidden, affine=True),
+ )
+ )
+ self.W_K = nn.Linear(self.hidden, self.hidden)
+ self.W_Q = nn.Linear(self.hidden, self.hidden)
+
+ if stride == 2:
+ self.last = FactorizedReduce(C_in + self.hidden, C_out, 2)
+ elif stride == 1:
+ self.last = FactorizedReduce(C_in + self.hidden, C_out, 1)
+ else:
+ raise ValueError("Invalid Stride : {:}".format(stride))
+
+ def forward(self, x):
+ batch, C, H, W = x.size()
+ assert H >= self.part, "input size too small : {:} vs {:}".format(
+ x.shape, self.part
+ )
+ IHs = [0]
+ for i in range(self.part):
+ IHs.append(min(H, int((i + 1) * (float(H) / self.part))))
+ local_feat_list = []
+ for i in range(self.part):
+ feature = x[:, :, IHs[i] : IHs[i + 1], :]
+ xfeax = self.avg_pool(feature)
+ xfea = self.local_conv_list[i](xfeax)
+ local_feat_list.append(xfea)
+ part_feature = torch.cat(local_feat_list, dim=2).view(batch, -1, self.part)
+ part_feature = part_feature.transpose(1, 2).contiguous()
+ part_K = self.W_K(part_feature)
+ part_Q = self.W_Q(part_feature).transpose(1, 2).contiguous()
+ weight_att = torch.bmm(part_K, part_Q)
+ attention = torch.softmax(weight_att, dim=2)
+ aggreateF = torch.bmm(attention, part_feature).transpose(1, 2).contiguous()
+ features = []
+ for i in range(self.part):
+ feature = aggreateF[:, :, i : i + 1].expand(
+ batch, self.hidden, IHs[i + 1] - IHs[i]
+ )
+ feature = feature.view(batch, self.hidden, IHs[i + 1] - IHs[i], 1)
+ features.append(feature)
+ features = torch.cat(features, dim=2).expand(batch, self.hidden, H, W)
+ final_fea = torch.cat((x, features), dim=1)
+ outputs = self.last(final_fea)
+ return outputs
+
+
+def drop_path(x, drop_prob):
+ if drop_prob > 0.0:
+ keep_prob = 1.0 - drop_prob
+ mask = x.new_zeros(x.size(0), 1, 1, 1)
+ mask = mask.bernoulli_(keep_prob)
+ x = torch.div(x, keep_prob)
+ x.mul_(mask)
+ return x
+
+
+# Searching for A Robust Neural Architecture in Four GPU Hours
+class GDAS_Reduction_Cell(nn.Module):
+ def __init__(
+ self, C_prev_prev, C_prev, C, reduction_prev, affine, track_running_stats
+ ):
+ super(GDAS_Reduction_Cell, self).__init__()
+ if reduction_prev:
+ self.preprocess0 = FactorizedReduce(
+ C_prev_prev, C, 2, affine, track_running_stats
+ )
+ else:
+ self.preprocess0 = ReLUConvBN(
+ C_prev_prev, C, 1, 1, 0, 1, affine, track_running_stats
+ )
+ self.preprocess1 = ReLUConvBN(
+ C_prev, C, 1, 1, 0, 1, affine, track_running_stats
+ )
+
+ self.reduction = True
+ self.ops1 = nn.ModuleList(
+ [
+ nn.Sequential(
+ nn.ReLU(inplace=False),
+ nn.Conv2d(
+ C,
+ C,
+ (1, 3),
+ stride=(1, 2),
+ padding=(0, 1),
+ groups=8,
+ bias=not affine,
+ ),
+ nn.Conv2d(
+ C,
+ C,
+ (3, 1),
+ stride=(2, 1),
+ padding=(1, 0),
+ groups=8,
+ bias=not affine,
+ ),
+ nn.BatchNorm2d(
+ C, affine=affine, track_running_stats=track_running_stats
+ ),
+ nn.ReLU(inplace=False),
+ nn.Conv2d(C, C, 1, stride=1, padding=0, bias=not affine),
+ nn.BatchNorm2d(
+ C, affine=affine, track_running_stats=track_running_stats
+ ),
+ ),
+ nn.Sequential(
+ nn.ReLU(inplace=False),
+ nn.Conv2d(
+ C,
+ C,
+ (1, 3),
+ stride=(1, 2),
+ padding=(0, 1),
+ groups=8,
+ bias=not affine,
+ ),
+ nn.Conv2d(
+ C,
+ C,
+ (3, 1),
+ stride=(2, 1),
+ padding=(1, 0),
+ groups=8,
+ bias=not affine,
+ ),
+ nn.BatchNorm2d(
+ C, affine=affine, track_running_stats=track_running_stats
+ ),
+ nn.ReLU(inplace=False),
+ nn.Conv2d(C, C, 1, stride=1, padding=0, bias=not affine),
+ nn.BatchNorm2d(
+ C, affine=affine, track_running_stats=track_running_stats
+ ),
+ ),
+ ]
+ )
+
+ self.ops2 = nn.ModuleList(
+ [
+ nn.Sequential(
+ nn.MaxPool2d(3, stride=2, padding=1),
+ nn.BatchNorm2d(
+ C, affine=affine, track_running_stats=track_running_stats
+ ),
+ ),
+ nn.Sequential(
+ nn.MaxPool2d(3, stride=2, padding=1),
+ nn.BatchNorm2d(
+ C, affine=affine, track_running_stats=track_running_stats
+ ),
+ ),
+ ]
+ )
+
+ @property
+ def multiplier(self):
+ return 4
+
+ def forward(self, s0, s1, drop_prob=-1):
+ s0 = self.preprocess0(s0)
+ s1 = self.preprocess1(s1)
+
+ X0 = self.ops1[0](s0)
+ X1 = self.ops1[1](s1)
+ if self.training and drop_prob > 0.0:
+ X0, X1 = drop_path(X0, drop_prob), drop_path(X1, drop_prob)
+
+ # X2 = self.ops2[0] (X0+X1)
+ X2 = self.ops2[0](s0)
+ X3 = self.ops2[1](s1)
+ if self.training and drop_prob > 0.0:
+ X2, X3 = drop_path(X2, drop_prob), drop_path(X3, drop_prob)
+ return torch.cat([X0, X1, X2, X3], dim=1)
+
+
+# To manage the useful classes in this file.
+RAW_OP_CLASSES = {"gdas_reduction": GDAS_Reduction_Cell}
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/__init__.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/__init__.py
new file mode 100644
index 00000000..0d770cb8
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/__init__.py
@@ -0,0 +1,33 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+##################################################
+# The macro structure is defined in NAS-Bench-201
+from .search_model_darts import TinyNetworkDarts
+from .search_model_gdas import TinyNetworkGDAS
+from .search_model_setn import TinyNetworkSETN
+from .search_model_enas import TinyNetworkENAS
+from .search_model_random import TinyNetworkRANDOM
+from .generic_model import GenericNAS201Model
+from .genotypes import Structure as CellStructure, architectures as CellArchitectures
+
+# NASNet-based macro structure
+from .search_model_gdas_nasnet import NASNetworkGDAS
+from .search_model_gdas_frc_nasnet import NASNetworkGDAS_FRC
+from .search_model_darts_nasnet import NASNetworkDARTS
+
+
+nas201_super_nets = {
+ "DARTS-V1": TinyNetworkDarts,
+ "DARTS-V2": TinyNetworkDarts,
+ "GDAS": TinyNetworkGDAS,
+ "SETN": TinyNetworkSETN,
+ "ENAS": TinyNetworkENAS,
+ "RANDOM": TinyNetworkRANDOM,
+ "generic": GenericNAS201Model,
+}
+
+nasnet_super_nets = {
+ "GDAS": NASNetworkGDAS,
+ "GDAS_FRC": NASNetworkGDAS_FRC,
+ "DARTS": NASNetworkDARTS,
+}
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/_test_module.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/_test_module.py
new file mode 100644
index 00000000..cd6fbfb8
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/_test_module.py
@@ -0,0 +1,14 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+##################################################
+import torch
+from search_model_enas_utils import Controller
+
+
+def main():
+ controller = Controller(6, 4)
+ predictions = controller()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/generic_model.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/generic_model.py
new file mode 100644
index 00000000..bbbbb1f0
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/generic_model.py
@@ -0,0 +1,366 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.07 #
+#####################################################
+import torch, random
+import torch.nn as nn
+from copy import deepcopy
+from typing import Text
+from torch.distributions.categorical import Categorical
+
+from ..cell_operations import ResNetBasicblock, drop_path
+from .search_cells import NAS201SearchCell as SearchCell
+from .genotypes import Structure
+
+
+class Controller(nn.Module):
+ # we refer to https://github.com/TDeVries/enas_pytorch/blob/master/models/controller.py
+ def __init__(
+ self,
+ edge2index,
+ op_names,
+ max_nodes,
+ lstm_size=32,
+ lstm_num_layers=2,
+ tanh_constant=2.5,
+ temperature=5.0,
+ ):
+ super(Controller, self).__init__()
+ # assign the attributes
+ self.max_nodes = max_nodes
+ self.num_edge = len(edge2index)
+ self.edge2index = edge2index
+ self.num_ops = len(op_names)
+ self.op_names = op_names
+ self.lstm_size = lstm_size
+ self.lstm_N = lstm_num_layers
+ self.tanh_constant = tanh_constant
+ self.temperature = temperature
+ # create parameters
+ self.register_parameter(
+ "input_vars", nn.Parameter(torch.Tensor(1, 1, lstm_size))
+ )
+ self.w_lstm = nn.LSTM(
+ input_size=self.lstm_size,
+ hidden_size=self.lstm_size,
+ num_layers=self.lstm_N,
+ )
+ self.w_embd = nn.Embedding(self.num_ops, self.lstm_size)
+ self.w_pred = nn.Linear(self.lstm_size, self.num_ops)
+
+ nn.init.uniform_(self.input_vars, -0.1, 0.1)
+ nn.init.uniform_(self.w_lstm.weight_hh_l0, -0.1, 0.1)
+ nn.init.uniform_(self.w_lstm.weight_ih_l0, -0.1, 0.1)
+ nn.init.uniform_(self.w_embd.weight, -0.1, 0.1)
+ nn.init.uniform_(self.w_pred.weight, -0.1, 0.1)
+
+ def convert_structure(self, _arch):
+ genotypes = []
+ for i in range(1, self.max_nodes):
+ xlist = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ op_index = _arch[self.edge2index[node_str]]
+ op_name = self.op_names[op_index]
+ xlist.append((op_name, j))
+ genotypes.append(tuple(xlist))
+ return Structure(genotypes)
+
+ def forward(self):
+
+ inputs, h0 = self.input_vars, None
+ log_probs, entropys, sampled_arch = [], [], []
+ for iedge in range(self.num_edge):
+ outputs, h0 = self.w_lstm(inputs, h0)
+
+ logits = self.w_pred(outputs)
+ logits = logits / self.temperature
+ logits = self.tanh_constant * torch.tanh(logits)
+ # distribution
+ op_distribution = Categorical(logits=logits)
+ op_index = op_distribution.sample()
+ sampled_arch.append(op_index.item())
+
+ op_log_prob = op_distribution.log_prob(op_index)
+ log_probs.append(op_log_prob.view(-1))
+ op_entropy = op_distribution.entropy()
+ entropys.append(op_entropy.view(-1))
+
+ # obtain the input embedding for the next step
+ inputs = self.w_embd(op_index)
+ return (
+ torch.sum(torch.cat(log_probs)),
+ torch.sum(torch.cat(entropys)),
+ self.convert_structure(sampled_arch),
+ )
+
+
+class GenericNAS201Model(nn.Module):
+ def __init__(
+ self, C, N, max_nodes, num_classes, search_space, affine, track_running_stats
+ ):
+ super(GenericNAS201Model, self).__init__()
+ self._C = C
+ self._layerN = N
+ self._max_nodes = max_nodes
+ self._stem = nn.Sequential(
+ nn.Conv2d(3, C, kernel_size=3, padding=1, bias=False), nn.BatchNorm2d(C)
+ )
+ layer_channels = [C] * N + [C * 2] + [C * 2] * N + [C * 4] + [C * 4] * N
+ layer_reductions = [False] * N + [True] + [False] * N + [True] + [False] * N
+ C_prev, num_edge, edge2index = C, None, None
+ self._cells = nn.ModuleList()
+ for index, (C_curr, reduction) in enumerate(
+ zip(layer_channels, layer_reductions)
+ ):
+ if reduction:
+ cell = ResNetBasicblock(C_prev, C_curr, 2)
+ else:
+ cell = SearchCell(
+ C_prev,
+ C_curr,
+ 1,
+ max_nodes,
+ search_space,
+ affine,
+ track_running_stats,
+ )
+ if num_edge is None:
+ num_edge, edge2index = cell.num_edges, cell.edge2index
+ else:
+ assert (
+ num_edge == cell.num_edges and edge2index == cell.edge2index
+ ), "invalid {:} vs. {:}.".format(num_edge, cell.num_edges)
+ self._cells.append(cell)
+ C_prev = cell.out_dim
+ self._op_names = deepcopy(search_space)
+ self._Layer = len(self._cells)
+ self.edge2index = edge2index
+ self.lastact = nn.Sequential(
+ nn.BatchNorm2d(
+ C_prev, affine=affine, track_running_stats=track_running_stats
+ ),
+ nn.ReLU(inplace=True),
+ )
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(C_prev, num_classes)
+ self._num_edge = num_edge
+ # algorithm related
+ self.arch_parameters = nn.Parameter(
+ 1e-3 * torch.randn(num_edge, len(search_space))
+ )
+ self._mode = None
+ self.dynamic_cell = None
+ self._tau = None
+ self._algo = None
+ self._drop_path = None
+ self.verbose = False
+
+ def set_algo(self, algo: Text):
+ # used for searching
+ assert self._algo is None, "This functioin can only be called once."
+ self._algo = algo
+ if algo == "enas":
+ self.controller = Controller(
+ self.edge2index, self._op_names, self._max_nodes
+ )
+ else:
+ self.arch_parameters = nn.Parameter(
+ 1e-3 * torch.randn(self._num_edge, len(self._op_names))
+ )
+ if algo == "gdas":
+ self._tau = 10
+
+ def set_cal_mode(self, mode, dynamic_cell=None):
+ assert mode in ["gdas", "enas", "urs", "joint", "select", "dynamic"]
+ self._mode = mode
+ if mode == "dynamic":
+ self.dynamic_cell = deepcopy(dynamic_cell)
+ else:
+ self.dynamic_cell = None
+
+ def set_drop_path(self, progress, drop_path_rate):
+ if drop_path_rate is None:
+ self._drop_path = None
+ elif progress is None:
+ self._drop_path = drop_path_rate
+ else:
+ self._drop_path = progress * drop_path_rate
+
+ @property
+ def mode(self):
+ return self._mode
+
+ @property
+ def drop_path(self):
+ return self._drop_path
+
+ @property
+ def weights(self):
+ xlist = list(self._stem.parameters())
+ xlist += list(self._cells.parameters())
+ xlist += list(self.lastact.parameters())
+ xlist += list(self.global_pooling.parameters())
+ xlist += list(self.classifier.parameters())
+ return xlist
+
+ def set_tau(self, tau):
+ self._tau = tau
+
+ @property
+ def tau(self):
+ return self._tau
+
+ @property
+ def alphas(self):
+ if self._algo == "enas":
+ return list(self.controller.parameters())
+ else:
+ return [self.arch_parameters]
+
+ @property
+ def message(self):
+ string = self.extra_repr()
+ for i, cell in enumerate(self._cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self._cells), cell.extra_repr()
+ )
+ return string
+
+ def show_alphas(self):
+ with torch.no_grad():
+ if self._algo == "enas":
+ return "w_pred :\n{:}".format(self.controller.w_pred.weight)
+ else:
+ return "arch-parameters :\n{:}".format(
+ nn.functional.softmax(self.arch_parameters, dim=-1).cpu()
+ )
+
+ def extra_repr(self):
+ return "{name}(C={_C}, Max-Nodes={_max_nodes}, N={_layerN}, L={_Layer}, alg={_algo})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ @property
+ def genotype(self):
+ genotypes = []
+ for i in range(1, self._max_nodes):
+ xlist = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ with torch.no_grad():
+ weights = self.arch_parameters[self.edge2index[node_str]]
+ op_name = self._op_names[weights.argmax().item()]
+ xlist.append((op_name, j))
+ genotypes.append(tuple(xlist))
+ return Structure(genotypes)
+
+ def dync_genotype(self, use_random=False):
+ genotypes = []
+ with torch.no_grad():
+ alphas_cpu = nn.functional.softmax(self.arch_parameters, dim=-1)
+ for i in range(1, self._max_nodes):
+ xlist = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ if use_random:
+ op_name = random.choice(self._op_names)
+ else:
+ weights = alphas_cpu[self.edge2index[node_str]]
+ op_index = torch.multinomial(weights, 1).item()
+ op_name = self._op_names[op_index]
+ xlist.append((op_name, j))
+ genotypes.append(tuple(xlist))
+ return Structure(genotypes)
+
+ def get_log_prob(self, arch):
+ with torch.no_grad():
+ logits = nn.functional.log_softmax(self.arch_parameters, dim=-1)
+ select_logits = []
+ for i, node_info in enumerate(arch.nodes):
+ for op, xin in node_info:
+ node_str = "{:}<-{:}".format(i + 1, xin)
+ op_index = self._op_names.index(op)
+ select_logits.append(logits[self.edge2index[node_str], op_index])
+ return sum(select_logits).item()
+
+ def return_topK(self, K, use_random=False):
+ archs = Structure.gen_all(self._op_names, self._max_nodes, False)
+ pairs = [(self.get_log_prob(arch), arch) for arch in archs]
+ if K < 0 or K >= len(archs):
+ K = len(archs)
+ if use_random:
+ return random.sample(archs, K)
+ else:
+ sorted_pairs = sorted(pairs, key=lambda x: -x[0])
+ return_pairs = [sorted_pairs[_][1] for _ in range(K)]
+ return return_pairs
+
+ def normalize_archp(self):
+ if self.mode == "gdas":
+ while True:
+ gumbels = -torch.empty_like(self.arch_parameters).exponential_().log()
+ logits = (self.arch_parameters.log_softmax(dim=1) + gumbels) / self.tau
+ probs = nn.functional.softmax(logits, dim=1)
+ index = probs.max(-1, keepdim=True)[1]
+ one_h = torch.zeros_like(logits).scatter_(-1, index, 1.0)
+ hardwts = one_h - probs.detach() + probs
+ if (
+ (torch.isinf(gumbels).any())
+ or (torch.isinf(probs).any())
+ or (torch.isnan(probs).any())
+ ):
+ continue
+ else:
+ break
+ with torch.no_grad():
+ hardwts_cpu = hardwts.detach().cpu()
+ return hardwts, hardwts_cpu, index, "GUMBEL"
+ else:
+ alphas = nn.functional.softmax(self.arch_parameters, dim=-1)
+ index = alphas.max(-1, keepdim=True)[1]
+ with torch.no_grad():
+ alphas_cpu = alphas.detach().cpu()
+ return alphas, alphas_cpu, index, "SOFTMAX"
+
+ def forward(self, inputs):
+ alphas, alphas_cpu, index, verbose_str = self.normalize_archp()
+ feature = self._stem(inputs)
+ for i, cell in enumerate(self._cells):
+ if isinstance(cell, SearchCell):
+ if self.mode == "urs":
+ feature = cell.forward_urs(feature)
+ if self.verbose:
+ verbose_str += "-forward_urs"
+ elif self.mode == "select":
+ feature = cell.forward_select(feature, alphas_cpu)
+ if self.verbose:
+ verbose_str += "-forward_select"
+ elif self.mode == "joint":
+ feature = cell.forward_joint(feature, alphas)
+ if self.verbose:
+ verbose_str += "-forward_joint"
+ elif self.mode == "dynamic":
+ feature = cell.forward_dynamic(feature, self.dynamic_cell)
+ if self.verbose:
+ verbose_str += "-forward_dynamic"
+ elif self.mode == "gdas":
+ feature = cell.forward_gdas(feature, alphas, index)
+ if self.verbose:
+ verbose_str += "-forward_gdas"
+ elif self.mode == "gdas_v1":
+ feature = cell.forward_gdas_v1(feature, alphas, index)
+ if self.verbose:
+ verbose_str += "-forward_gdas_v1"
+ else:
+ raise ValueError("invalid mode={:}".format(self.mode))
+ else:
+ feature = cell(feature)
+ if self.drop_path is not None:
+ feature = drop_path(feature, self.drop_path)
+ if self.verbose and random.random() < 0.001:
+ print(verbose_str)
+ out = self.lastact(feature)
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+ return out, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/genotypes.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/genotypes.py
new file mode 100644
index 00000000..eaf08547
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/genotypes.py
@@ -0,0 +1,280 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+##################################################
+from copy import deepcopy
+
+
+def get_combination(space, num):
+ combs = []
+ for i in range(num):
+ if i == 0:
+ for func in space:
+ combs.append([(func, i)])
+ else:
+ new_combs = []
+ for string in combs:
+ for func in space:
+ xstring = string + [(func, i)]
+ new_combs.append(xstring)
+ combs = new_combs
+ return combs
+
+
+class Structure:
+ def __init__(self, genotype):
+ assert isinstance(genotype, list) or isinstance(
+ genotype, tuple
+ ), "invalid class of genotype : {:}".format(type(genotype))
+ self.node_num = len(genotype) + 1
+ self.nodes = []
+ self.node_N = []
+ for idx, node_info in enumerate(genotype):
+ assert isinstance(node_info, list) or isinstance(
+ node_info, tuple
+ ), "invalid class of node_info : {:}".format(type(node_info))
+ assert len(node_info) >= 1, "invalid length : {:}".format(len(node_info))
+ for node_in in node_info:
+ assert isinstance(node_in, list) or isinstance(
+ node_in, tuple
+ ), "invalid class of in-node : {:}".format(type(node_in))
+ assert (
+ len(node_in) == 2 and node_in[1] <= idx
+ ), "invalid in-node : {:}".format(node_in)
+ self.node_N.append(len(node_info))
+ self.nodes.append(tuple(deepcopy(node_info)))
+
+ def tolist(self, remove_str):
+ # convert this class to the list, if remove_str is 'none', then remove the 'none' operation.
+ # note that we re-order the input node in this function
+ # return the-genotype-list and success [if unsuccess, it is not a connectivity]
+ genotypes = []
+ for node_info in self.nodes:
+ node_info = list(node_info)
+ node_info = sorted(node_info, key=lambda x: (x[1], x[0]))
+ node_info = tuple(filter(lambda x: x[0] != remove_str, node_info))
+ if len(node_info) == 0:
+ return None, False
+ genotypes.append(node_info)
+ return genotypes, True
+
+ def node(self, index):
+ assert index > 0 and index <= len(self), "invalid index={:} < {:}".format(
+ index, len(self)
+ )
+ return self.nodes[index]
+
+ def tostr(self):
+ strings = []
+ for node_info in self.nodes:
+ string = "|".join([x[0] + "~{:}".format(x[1]) for x in node_info])
+ string = "|{:}|".format(string)
+ strings.append(string)
+ return "+".join(strings)
+
+ def check_valid(self):
+ nodes = {0: True}
+ for i, node_info in enumerate(self.nodes):
+ sums = []
+ for op, xin in node_info:
+ if op == "none" or nodes[xin] is False:
+ x = False
+ else:
+ x = True
+ sums.append(x)
+ nodes[i + 1] = sum(sums) > 0
+ return nodes[len(self.nodes)]
+
+ def to_unique_str(self, consider_zero=False):
+ # this is used to identify the isomorphic cell, which rerquires the prior knowledge of operation
+ # two operations are special, i.e., none and skip_connect
+ nodes = {0: "0"}
+ for i_node, node_info in enumerate(self.nodes):
+ cur_node = []
+ for op, xin in node_info:
+ if consider_zero is None:
+ x = "(" + nodes[xin] + ")" + "@{:}".format(op)
+ elif consider_zero:
+ if op == "none" or nodes[xin] == "#":
+ x = "#" # zero
+ elif op == "skip_connect":
+ x = nodes[xin]
+ else:
+ x = "(" + nodes[xin] + ")" + "@{:}".format(op)
+ else:
+ if op == "skip_connect":
+ x = nodes[xin]
+ else:
+ x = "(" + nodes[xin] + ")" + "@{:}".format(op)
+ cur_node.append(x)
+ nodes[i_node + 1] = "+".join(sorted(cur_node))
+ return nodes[len(self.nodes)]
+
+ def check_valid_op(self, op_names):
+ for node_info in self.nodes:
+ for inode_edge in node_info:
+ # assert inode_edge[0] in op_names, 'invalid op-name : {:}'.format(inode_edge[0])
+ if inode_edge[0] not in op_names:
+ return False
+ return True
+
+ def __repr__(self):
+ return "{name}({node_num} nodes with {node_info})".format(
+ name=self.__class__.__name__, node_info=self.tostr(), **self.__dict__
+ )
+
+ def __len__(self):
+ return len(self.nodes) + 1
+
+ def __getitem__(self, index):
+ return self.nodes[index]
+
+ @staticmethod
+ def str2structure(xstr):
+ if isinstance(xstr, Structure):
+ return xstr
+ assert isinstance(xstr, str), "must take string (not {:}) as input".format(
+ type(xstr)
+ )
+ nodestrs = xstr.split("+")
+ genotypes = []
+ for i, node_str in enumerate(nodestrs):
+ inputs = list(filter(lambda x: x != "", node_str.split("|")))
+ for xinput in inputs:
+ assert len(xinput.split("~")) == 2, "invalid input length : {:}".format(
+ xinput
+ )
+ inputs = (xi.split("~") for xi in inputs)
+ input_infos = tuple((op, int(IDX)) for (op, IDX) in inputs)
+ genotypes.append(input_infos)
+ return Structure(genotypes)
+
+ @staticmethod
+ def str2fullstructure(xstr, default_name="none"):
+ assert isinstance(xstr, str), "must take string (not {:}) as input".format(
+ type(xstr)
+ )
+ nodestrs = xstr.split("+")
+ genotypes = []
+ for i, node_str in enumerate(nodestrs):
+ inputs = list(filter(lambda x: x != "", node_str.split("|")))
+ for xinput in inputs:
+ assert len(xinput.split("~")) == 2, "invalid input length : {:}".format(
+ xinput
+ )
+ inputs = (xi.split("~") for xi in inputs)
+ input_infos = list((op, int(IDX)) for (op, IDX) in inputs)
+ all_in_nodes = list(x[1] for x in input_infos)
+ for j in range(i):
+ if j not in all_in_nodes:
+ input_infos.append((default_name, j))
+ node_info = sorted(input_infos, key=lambda x: (x[1], x[0]))
+ genotypes.append(tuple(node_info))
+ return Structure(genotypes)
+
+ @staticmethod
+ def gen_all(search_space, num, return_ori):
+ assert isinstance(search_space, list) or isinstance(
+ search_space, tuple
+ ), "invalid class of search-space : {:}".format(type(search_space))
+ assert (
+ num >= 2
+ ), "There should be at least two nodes in a neural cell instead of {:}".format(
+ num
+ )
+ all_archs = get_combination(search_space, 1)
+ for i, arch in enumerate(all_archs):
+ all_archs[i] = [tuple(arch)]
+
+ for inode in range(2, num):
+ cur_nodes = get_combination(search_space, inode)
+ new_all_archs = []
+ for previous_arch in all_archs:
+ for cur_node in cur_nodes:
+ new_all_archs.append(previous_arch + [tuple(cur_node)])
+ all_archs = new_all_archs
+ if return_ori:
+ return all_archs
+ else:
+ return [Structure(x) for x in all_archs]
+
+
+class Struct101(Structure):
+ def __init__(self, arch_id, genotype):
+ super().__init__(genotype)
+ self.arch_id = arch_id
+
+
+ResNet_CODE = Structure(
+ [
+ (("nor_conv_3x3", 0),), # node-1
+ (("nor_conv_3x3", 1),), # node-2
+ (("skip_connect", 0), ("skip_connect", 2)),
+ ] # node-3
+)
+
+AllConv3x3_CODE = Structure(
+ [
+ (("nor_conv_3x3", 0),), # node-1
+ (("nor_conv_3x3", 0), ("nor_conv_3x3", 1)), # node-2
+ (("nor_conv_3x3", 0), ("nor_conv_3x3", 1), ("nor_conv_3x3", 2)),
+ ] # node-3
+)
+
+AllFull_CODE = Structure(
+ [
+ (
+ ("skip_connect", 0),
+ ("nor_conv_1x1", 0),
+ ("nor_conv_3x3", 0),
+ ("avg_pool_3x3", 0),
+ ), # node-1
+ (
+ ("skip_connect", 0),
+ ("nor_conv_1x1", 0),
+ ("nor_conv_3x3", 0),
+ ("avg_pool_3x3", 0),
+ ("skip_connect", 1),
+ ("nor_conv_1x1", 1),
+ ("nor_conv_3x3", 1),
+ ("avg_pool_3x3", 1),
+ ), # node-2
+ (
+ ("skip_connect", 0),
+ ("nor_conv_1x1", 0),
+ ("nor_conv_3x3", 0),
+ ("avg_pool_3x3", 0),
+ ("skip_connect", 1),
+ ("nor_conv_1x1", 1),
+ ("nor_conv_3x3", 1),
+ ("avg_pool_3x3", 1),
+ ("skip_connect", 2),
+ ("nor_conv_1x1", 2),
+ ("nor_conv_3x3", 2),
+ ("avg_pool_3x3", 2),
+ ),
+ ] # node-3
+)
+
+AllConv1x1_CODE = Structure(
+ [
+ (("nor_conv_1x1", 0),), # node-1
+ (("nor_conv_1x1", 0), ("nor_conv_1x1", 1)), # node-2
+ (("nor_conv_1x1", 0), ("nor_conv_1x1", 1), ("nor_conv_1x1", 2)),
+ ] # node-3
+)
+
+AllIdentity_CODE = Structure(
+ [
+ (("skip_connect", 0),), # node-1
+ (("skip_connect", 0), ("skip_connect", 1)), # node-2
+ (("skip_connect", 0), ("skip_connect", 1), ("skip_connect", 2)),
+ ] # node-3
+)
+
+architectures = {
+ "resnet": ResNet_CODE,
+ "all_c3x3": AllConv3x3_CODE,
+ "all_c1x1": AllConv1x1_CODE,
+ "all_idnt": AllIdentity_CODE,
+ "all_full": AllFull_CODE,
+}
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_cells.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_cells.py
new file mode 100644
index 00000000..6be7c529
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_cells.py
@@ -0,0 +1,267 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+##################################################
+import math, random, torch
+import warnings
+import torch.nn as nn
+import torch.nn.functional as F
+from copy import deepcopy
+from ..cell_operations import OPS
+
+
+# This module is used for NAS-Bench-201, represents a small search space with a complete DAG
+class NAS201SearchCell(nn.Module):
+ def __init__(
+ self,
+ C_in,
+ C_out,
+ stride,
+ max_nodes,
+ op_names,
+ affine=False,
+ track_running_stats=True,
+ ):
+ super(NAS201SearchCell, self).__init__()
+
+ self.op_names = deepcopy(op_names)
+ self.edges = nn.ModuleDict()
+ self.max_nodes = max_nodes
+ self.in_dim = C_in
+ self.out_dim = C_out
+ for i in range(1, max_nodes):
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ if j == 0:
+ xlists = [
+ OPS[op_name](C_in, C_out, stride, affine, track_running_stats)
+ for op_name in op_names
+ ]
+ else:
+ xlists = [
+ OPS[op_name](C_in, C_out, 1, affine, track_running_stats)
+ for op_name in op_names
+ ]
+ self.edges[node_str] = nn.ModuleList(xlists)
+ self.edge_keys = sorted(list(self.edges.keys()))
+ self.edge2index = {key: i for i, key in enumerate(self.edge_keys)}
+ self.num_edges = len(self.edges)
+
+ def extra_repr(self):
+ string = "info :: {max_nodes} nodes, inC={in_dim}, outC={out_dim}".format(
+ **self.__dict__
+ )
+ return string
+
+ def forward(self, inputs, weightss):
+ nodes = [inputs]
+ for i in range(1, self.max_nodes):
+ inter_nodes = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ weights = weightss[self.edge2index[node_str]]
+ inter_nodes.append(
+ sum(
+ layer(nodes[j]) * w
+ for layer, w in zip(self.edges[node_str], weights)
+ )
+ )
+ nodes.append(sum(inter_nodes))
+ return nodes[-1]
+
+ # GDAS
+ def forward_gdas(self, inputs, hardwts, index):
+ nodes = [inputs]
+ for i in range(1, self.max_nodes):
+ inter_nodes = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ weights = hardwts[self.edge2index[node_str]]
+ argmaxs = index[self.edge2index[node_str]].item()
+ weigsum = sum(
+ weights[_ie] * edge(nodes[j]) if _ie == argmaxs else weights[_ie]
+ for _ie, edge in enumerate(self.edges[node_str])
+ )
+ inter_nodes.append(weigsum)
+ nodes.append(sum(inter_nodes))
+ return nodes[-1]
+
+ # GDAS Variant: https://github.com/D-X-Y/AutoDL-Projects/issues/119
+ def forward_gdas_v1(self, inputs, hardwts, index):
+ nodes = [inputs]
+ for i in range(1, self.max_nodes):
+ inter_nodes = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ weights = hardwts[self.edge2index[node_str]]
+ argmaxs = index[self.edge2index[node_str]].item()
+ weigsum = weights[argmaxs] * self.edges[node_str](nodes[j])
+ inter_nodes.append(weigsum)
+ nodes.append(sum(inter_nodes))
+ return nodes[-1]
+
+ # joint
+ def forward_joint(self, inputs, weightss):
+ nodes = [inputs]
+ for i in range(1, self.max_nodes):
+ inter_nodes = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ weights = weightss[self.edge2index[node_str]]
+ # aggregation = sum( layer(nodes[j]) * w for layer, w in zip(self.edges[node_str], weights) ) / weights.numel()
+ aggregation = sum(
+ layer(nodes[j]) * w
+ for layer, w in zip(self.edges[node_str], weights)
+ )
+ inter_nodes.append(aggregation)
+ nodes.append(sum(inter_nodes))
+ return nodes[-1]
+
+ # uniform random sampling per iteration, SETN
+ def forward_urs(self, inputs):
+ nodes = [inputs]
+ for i in range(1, self.max_nodes):
+ while True: # to avoid select zero for all ops
+ sops, has_non_zero = [], False
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ candidates = self.edges[node_str]
+ select_op = random.choice(candidates)
+ sops.append(select_op)
+ if not hasattr(select_op, "is_zero") or select_op.is_zero is False:
+ has_non_zero = True
+ if has_non_zero:
+ break
+ inter_nodes = []
+ for j, select_op in enumerate(sops):
+ inter_nodes.append(select_op(nodes[j]))
+ nodes.append(sum(inter_nodes))
+ return nodes[-1]
+
+ # select the argmax
+ def forward_select(self, inputs, weightss):
+ nodes = [inputs]
+ for i in range(1, self.max_nodes):
+ inter_nodes = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ weights = weightss[self.edge2index[node_str]]
+ inter_nodes.append(
+ self.edges[node_str][weights.argmax().item()](nodes[j])
+ )
+ # inter_nodes.append( sum( layer(nodes[j]) * w for layer, w in zip(self.edges[node_str], weights) ) )
+ nodes.append(sum(inter_nodes))
+ return nodes[-1]
+
+ # forward with a specific structure
+ def forward_dynamic(self, inputs, structure):
+ nodes = [inputs]
+ for i in range(1, self.max_nodes):
+ cur_op_node = structure.nodes[i - 1]
+ inter_nodes = []
+ for op_name, j in cur_op_node:
+ node_str = "{:}<-{:}".format(i, j)
+ op_index = self.op_names.index(op_name)
+ inter_nodes.append(self.edges[node_str][op_index](nodes[j]))
+ nodes.append(sum(inter_nodes))
+ return nodes[-1]
+
+
+# Learning Transferable Architectures for Scalable Image Recognition, CVPR 2018
+
+
+class MixedOp(nn.Module):
+ def __init__(self, space, C, stride, affine, track_running_stats):
+ super(MixedOp, self).__init__()
+ self._ops = nn.ModuleList()
+ for primitive in space:
+ op = OPS[primitive](C, C, stride, affine, track_running_stats)
+ self._ops.append(op)
+
+ def forward_gdas(self, x, weights, index):
+ return self._ops[index](x) * weights[index]
+
+ def forward_darts(self, x, weights):
+ return sum(w * op(x) for w, op in zip(weights, self._ops))
+
+
+class NASNetSearchCell(nn.Module):
+ def __init__(
+ self,
+ space,
+ steps,
+ multiplier,
+ C_prev_prev,
+ C_prev,
+ C,
+ reduction,
+ reduction_prev,
+ affine,
+ track_running_stats,
+ ):
+ super(NASNetSearchCell, self).__init__()
+ self.reduction = reduction
+ self.op_names = deepcopy(space)
+ if reduction_prev:
+ self.preprocess0 = OPS["skip_connect"](
+ C_prev_prev, C, 2, affine, track_running_stats
+ )
+ else:
+ self.preprocess0 = OPS["nor_conv_1x1"](
+ C_prev_prev, C, 1, affine, track_running_stats
+ )
+ self.preprocess1 = OPS["nor_conv_1x1"](
+ C_prev, C, 1, affine, track_running_stats
+ )
+ self._steps = steps
+ self._multiplier = multiplier
+
+ self._ops = nn.ModuleList()
+ self.edges = nn.ModuleDict()
+ for i in range(self._steps):
+ for j in range(2 + i):
+ node_str = "{:}<-{:}".format(
+ i, j
+ ) # indicate the edge from node-(j) to node-(i+2)
+ stride = 2 if reduction and j < 2 else 1
+ op = MixedOp(space, C, stride, affine, track_running_stats)
+ self.edges[node_str] = op
+ self.edge_keys = sorted(list(self.edges.keys()))
+ self.edge2index = {key: i for i, key in enumerate(self.edge_keys)}
+ self.num_edges = len(self.edges)
+
+ @property
+ def multiplier(self):
+ return self._multiplier
+
+ def forward_gdas(self, s0, s1, weightss, indexs):
+ s0 = self.preprocess0(s0)
+ s1 = self.preprocess1(s1)
+
+ states = [s0, s1]
+ for i in range(self._steps):
+ clist = []
+ for j, h in enumerate(states):
+ node_str = "{:}<-{:}".format(i, j)
+ op = self.edges[node_str]
+ weights = weightss[self.edge2index[node_str]]
+ index = indexs[self.edge2index[node_str]].item()
+ clist.append(op.forward_gdas(h, weights, index))
+ states.append(sum(clist))
+
+ return torch.cat(states[-self._multiplier :], dim=1)
+
+ def forward_darts(self, s0, s1, weightss):
+ s0 = self.preprocess0(s0)
+ s1 = self.preprocess1(s1)
+
+ states = [s0, s1]
+ for i in range(self._steps):
+ clist = []
+ for j, h in enumerate(states):
+ node_str = "{:}<-{:}".format(i, j)
+ op = self.edges[node_str]
+ weights = weightss[self.edge2index[node_str]]
+ clist.append(op.forward_darts(h, weights))
+ states.append(sum(clist))
+
+ return torch.cat(states[-self._multiplier :], dim=1)
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_darts.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_darts.py
new file mode 100644
index 00000000..31041b66
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_darts.py
@@ -0,0 +1,122 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+########################################################
+# DARTS: Differentiable Architecture Search, ICLR 2019 #
+########################################################
+import torch
+import torch.nn as nn
+from copy import deepcopy
+from ..cell_operations import ResNetBasicblock
+from .search_cells import NAS201SearchCell as SearchCell
+from .genotypes import Structure
+
+
+class TinyNetworkDarts(nn.Module):
+ def __init__(
+ self, C, N, max_nodes, num_classes, search_space, affine, track_running_stats
+ ):
+ super(TinyNetworkDarts, self).__init__()
+ self._C = C
+ self._layerN = N
+ self.max_nodes = max_nodes
+ self.stem = nn.Sequential(
+ nn.Conv2d(3, C, kernel_size=3, padding=1, bias=False), nn.BatchNorm2d(C)
+ )
+
+ layer_channels = [C] * N + [C * 2] + [C * 2] * N + [C * 4] + [C * 4] * N
+ layer_reductions = [False] * N + [True] + [False] * N + [True] + [False] * N
+
+ C_prev, num_edge, edge2index = C, None, None
+ self.cells = nn.ModuleList()
+ for index, (C_curr, reduction) in enumerate(
+ zip(layer_channels, layer_reductions)
+ ):
+ if reduction:
+ cell = ResNetBasicblock(C_prev, C_curr, 2)
+ else:
+ cell = SearchCell(
+ C_prev,
+ C_curr,
+ 1,
+ max_nodes,
+ search_space,
+ affine,
+ track_running_stats,
+ )
+ if num_edge is None:
+ num_edge, edge2index = cell.num_edges, cell.edge2index
+ else:
+ assert (
+ num_edge == cell.num_edges and edge2index == cell.edge2index
+ ), "invalid {:} vs. {:}.".format(num_edge, cell.num_edges)
+ self.cells.append(cell)
+ C_prev = cell.out_dim
+ self.op_names = deepcopy(search_space)
+ self._Layer = len(self.cells)
+ self.edge2index = edge2index
+ self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True))
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(C_prev, num_classes)
+ self.arch_parameters = nn.Parameter(
+ 1e-3 * torch.randn(num_edge, len(search_space))
+ )
+
+ def get_weights(self):
+ xlist = list(self.stem.parameters()) + list(self.cells.parameters())
+ xlist += list(self.lastact.parameters()) + list(
+ self.global_pooling.parameters()
+ )
+ xlist += list(self.classifier.parameters())
+ return xlist
+
+ def get_alphas(self):
+ return [self.arch_parameters]
+
+ def show_alphas(self):
+ with torch.no_grad():
+ return "arch-parameters :\n{:}".format(
+ nn.functional.softmax(self.arch_parameters, dim=-1).cpu()
+ )
+
+ def get_message(self):
+ string = self.extra_repr()
+ for i, cell in enumerate(self.cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self.cells), cell.extra_repr()
+ )
+ return string
+
+ def extra_repr(self):
+ return "{name}(C={_C}, Max-Nodes={max_nodes}, N={_layerN}, L={_Layer})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ def genotype(self):
+ genotypes = []
+ for i in range(1, self.max_nodes):
+ xlist = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ with torch.no_grad():
+ weights = self.arch_parameters[self.edge2index[node_str]]
+ op_name = self.op_names[weights.argmax().item()]
+ xlist.append((op_name, j))
+ genotypes.append(tuple(xlist))
+ return Structure(genotypes)
+
+ def forward(self, inputs):
+ alphas = nn.functional.softmax(self.arch_parameters, dim=-1)
+
+ feature = self.stem(inputs)
+ for i, cell in enumerate(self.cells):
+ if isinstance(cell, SearchCell):
+ feature = cell(feature, alphas)
+ else:
+ feature = cell(feature)
+
+ out = self.lastact(feature)
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+
+ return out, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_darts_nasnet.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_darts_nasnet.py
new file mode 100644
index 00000000..7cfdb478
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_darts_nasnet.py
@@ -0,0 +1,178 @@
+####################
+# DARTS, ICLR 2019 #
+####################
+import torch
+import torch.nn as nn
+from copy import deepcopy
+from typing import List, Text, Dict
+from .search_cells import NASNetSearchCell as SearchCell
+
+
+# The macro structure is based on NASNet
+class NASNetworkDARTS(nn.Module):
+ def __init__(
+ self,
+ C: int,
+ N: int,
+ steps: int,
+ multiplier: int,
+ stem_multiplier: int,
+ num_classes: int,
+ search_space: List[Text],
+ affine: bool,
+ track_running_stats: bool,
+ ):
+ super(NASNetworkDARTS, self).__init__()
+ self._C = C
+ self._layerN = N
+ self._steps = steps
+ self._multiplier = multiplier
+ self.stem = nn.Sequential(
+ nn.Conv2d(3, C * stem_multiplier, kernel_size=3, padding=1, bias=False),
+ nn.BatchNorm2d(C * stem_multiplier),
+ )
+
+ # config for each layer
+ layer_channels = (
+ [C] * N + [C * 2] + [C * 2] * (N - 1) + [C * 4] + [C * 4] * (N - 1)
+ )
+ layer_reductions = (
+ [False] * N + [True] + [False] * (N - 1) + [True] + [False] * (N - 1)
+ )
+
+ num_edge, edge2index = None, None
+ C_prev_prev, C_prev, C_curr, reduction_prev = (
+ C * stem_multiplier,
+ C * stem_multiplier,
+ C,
+ False,
+ )
+
+ self.cells = nn.ModuleList()
+ for index, (C_curr, reduction) in enumerate(
+ zip(layer_channels, layer_reductions)
+ ):
+ cell = SearchCell(
+ search_space,
+ steps,
+ multiplier,
+ C_prev_prev,
+ C_prev,
+ C_curr,
+ reduction,
+ reduction_prev,
+ affine,
+ track_running_stats,
+ )
+ if num_edge is None:
+ num_edge, edge2index = cell.num_edges, cell.edge2index
+ else:
+ assert (
+ num_edge == cell.num_edges and edge2index == cell.edge2index
+ ), "invalid {:} vs. {:}.".format(num_edge, cell.num_edges)
+ self.cells.append(cell)
+ C_prev_prev, C_prev, reduction_prev = C_prev, multiplier * C_curr, reduction
+ self.op_names = deepcopy(search_space)
+ self._Layer = len(self.cells)
+ self.edge2index = edge2index
+ self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True))
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(C_prev, num_classes)
+ self.arch_normal_parameters = nn.Parameter(
+ 1e-3 * torch.randn(num_edge, len(search_space))
+ )
+ self.arch_reduce_parameters = nn.Parameter(
+ 1e-3 * torch.randn(num_edge, len(search_space))
+ )
+
+ def get_weights(self) -> List[torch.nn.Parameter]:
+ xlist = list(self.stem.parameters()) + list(self.cells.parameters())
+ xlist += list(self.lastact.parameters()) + list(
+ self.global_pooling.parameters()
+ )
+ xlist += list(self.classifier.parameters())
+ return xlist
+
+ def get_alphas(self) -> List[torch.nn.Parameter]:
+ return [self.arch_normal_parameters, self.arch_reduce_parameters]
+
+ def show_alphas(self) -> Text:
+ with torch.no_grad():
+ A = "arch-normal-parameters :\n{:}".format(
+ nn.functional.softmax(self.arch_normal_parameters, dim=-1).cpu()
+ )
+ B = "arch-reduce-parameters :\n{:}".format(
+ nn.functional.softmax(self.arch_reduce_parameters, dim=-1).cpu()
+ )
+ return "{:}\n{:}".format(A, B)
+
+ def get_message(self) -> Text:
+ string = self.extra_repr()
+ for i, cell in enumerate(self.cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self.cells), cell.extra_repr()
+ )
+ return string
+
+ def extra_repr(self) -> Text:
+ return "{name}(C={_C}, N={_layerN}, steps={_steps}, multiplier={_multiplier}, L={_Layer})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ def genotype(self) -> Dict[Text, List]:
+ def _parse(weights):
+ gene = []
+ for i in range(self._steps):
+ edges = []
+ for j in range(2 + i):
+ node_str = "{:}<-{:}".format(i, j)
+ ws = weights[self.edge2index[node_str]]
+ for k, op_name in enumerate(self.op_names):
+ if op_name == "none":
+ continue
+ edges.append((op_name, j, ws[k]))
+ # (TODO) xuanyidong:
+ # Here the selected two edges might come from the same input node.
+ # And this case could be a problem that two edges will collapse into a single one
+ # due to our assumption -- at most one edge from an input node during evaluation.
+ edges = sorted(edges, key=lambda x: -x[-1])
+ selected_edges = edges[:2]
+ gene.append(tuple(selected_edges))
+ return gene
+
+ with torch.no_grad():
+ gene_normal = _parse(
+ torch.softmax(self.arch_normal_parameters, dim=-1).cpu().numpy()
+ )
+ gene_reduce = _parse(
+ torch.softmax(self.arch_reduce_parameters, dim=-1).cpu().numpy()
+ )
+ return {
+ "normal": gene_normal,
+ "normal_concat": list(
+ range(2 + self._steps - self._multiplier, self._steps + 2)
+ ),
+ "reduce": gene_reduce,
+ "reduce_concat": list(
+ range(2 + self._steps - self._multiplier, self._steps + 2)
+ ),
+ }
+
+ def forward(self, inputs):
+
+ normal_w = nn.functional.softmax(self.arch_normal_parameters, dim=1)
+ reduce_w = nn.functional.softmax(self.arch_reduce_parameters, dim=1)
+
+ s0 = s1 = self.stem(inputs)
+ for i, cell in enumerate(self.cells):
+ if cell.reduction:
+ ww = reduce_w
+ else:
+ ww = normal_w
+ s0, s1 = s1, cell.forward_darts(s0, s1, ww)
+ out = self.lastact(s1)
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+
+ return out, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_enas.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_enas.py
new file mode 100644
index 00000000..7ba91d48
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_enas.py
@@ -0,0 +1,114 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+##########################################################################
+# Efficient Neural Architecture Search via Parameters Sharing, ICML 2018 #
+##########################################################################
+import torch
+import torch.nn as nn
+from copy import deepcopy
+from ..cell_operations import ResNetBasicblock
+from .search_cells import NAS201SearchCell as SearchCell
+from .genotypes import Structure
+from .search_model_enas_utils import Controller
+
+
+class TinyNetworkENAS(nn.Module):
+ def __init__(
+ self, C, N, max_nodes, num_classes, search_space, affine, track_running_stats
+ ):
+ super(TinyNetworkENAS, self).__init__()
+ self._C = C
+ self._layerN = N
+ self.max_nodes = max_nodes
+ self.stem = nn.Sequential(
+ nn.Conv2d(3, C, kernel_size=3, padding=1, bias=False), nn.BatchNorm2d(C)
+ )
+
+ layer_channels = [C] * N + [C * 2] + [C * 2] * N + [C * 4] + [C * 4] * N
+ layer_reductions = [False] * N + [True] + [False] * N + [True] + [False] * N
+
+ C_prev, num_edge, edge2index = C, None, None
+ self.cells = nn.ModuleList()
+ for index, (C_curr, reduction) in enumerate(
+ zip(layer_channels, layer_reductions)
+ ):
+ if reduction:
+ cell = ResNetBasicblock(C_prev, C_curr, 2)
+ else:
+ cell = SearchCell(
+ C_prev,
+ C_curr,
+ 1,
+ max_nodes,
+ search_space,
+ affine,
+ track_running_stats,
+ )
+ if num_edge is None:
+ num_edge, edge2index = cell.num_edges, cell.edge2index
+ else:
+ assert (
+ num_edge == cell.num_edges and edge2index == cell.edge2index
+ ), "invalid {:} vs. {:}.".format(num_edge, cell.num_edges)
+ self.cells.append(cell)
+ C_prev = cell.out_dim
+ self.op_names = deepcopy(search_space)
+ self._Layer = len(self.cells)
+ self.edge2index = edge2index
+ self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True))
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(C_prev, num_classes)
+ # to maintain the sampled architecture
+ self.sampled_arch = None
+
+ def update_arch(self, _arch):
+ if _arch is None:
+ self.sampled_arch = None
+ elif isinstance(_arch, Structure):
+ self.sampled_arch = _arch
+ elif isinstance(_arch, (list, tuple)):
+ genotypes = []
+ for i in range(1, self.max_nodes):
+ xlist = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ op_index = _arch[self.edge2index[node_str]]
+ op_name = self.op_names[op_index]
+ xlist.append((op_name, j))
+ genotypes.append(tuple(xlist))
+ self.sampled_arch = Structure(genotypes)
+ else:
+ raise ValueError("invalid type of input architecture : {:}".format(_arch))
+ return self.sampled_arch
+
+ def create_controller(self):
+ return Controller(len(self.edge2index), len(self.op_names))
+
+ def get_message(self):
+ string = self.extra_repr()
+ for i, cell in enumerate(self.cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self.cells), cell.extra_repr()
+ )
+ return string
+
+ def extra_repr(self):
+ return "{name}(C={_C}, Max-Nodes={max_nodes}, N={_layerN}, L={_Layer})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ def forward(self, inputs):
+
+ feature = self.stem(inputs)
+ for i, cell in enumerate(self.cells):
+ if isinstance(cell, SearchCell):
+ feature = cell.forward_dynamic(feature, self.sampled_arch)
+ else:
+ feature = cell(feature)
+
+ out = self.lastact(feature)
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+
+ return out, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_enas_utils.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_enas_utils.py
new file mode 100644
index 00000000..71d5d0fd
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_enas_utils.py
@@ -0,0 +1,74 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+##########################################################################
+# Efficient Neural Architecture Search via Parameters Sharing, ICML 2018 #
+##########################################################################
+import torch
+import torch.nn as nn
+from torch.distributions.categorical import Categorical
+
+
+class Controller(nn.Module):
+ # we refer to https://github.com/TDeVries/enas_pytorch/blob/master/models/controller.py
+ def __init__(
+ self,
+ num_edge,
+ num_ops,
+ lstm_size=32,
+ lstm_num_layers=2,
+ tanh_constant=2.5,
+ temperature=5.0,
+ ):
+ super(Controller, self).__init__()
+ # assign the attributes
+ self.num_edge = num_edge
+ self.num_ops = num_ops
+ self.lstm_size = lstm_size
+ self.lstm_N = lstm_num_layers
+ self.tanh_constant = tanh_constant
+ self.temperature = temperature
+ # create parameters
+ self.register_parameter(
+ "input_vars", nn.Parameter(torch.Tensor(1, 1, lstm_size))
+ )
+ self.w_lstm = nn.LSTM(
+ input_size=self.lstm_size,
+ hidden_size=self.lstm_size,
+ num_layers=self.lstm_N,
+ )
+ self.w_embd = nn.Embedding(self.num_ops, self.lstm_size)
+ self.w_pred = nn.Linear(self.lstm_size, self.num_ops)
+
+ nn.init.uniform_(self.input_vars, -0.1, 0.1)
+ nn.init.uniform_(self.w_lstm.weight_hh_l0, -0.1, 0.1)
+ nn.init.uniform_(self.w_lstm.weight_ih_l0, -0.1, 0.1)
+ nn.init.uniform_(self.w_embd.weight, -0.1, 0.1)
+ nn.init.uniform_(self.w_pred.weight, -0.1, 0.1)
+
+ def forward(self):
+
+ inputs, h0 = self.input_vars, None
+ log_probs, entropys, sampled_arch = [], [], []
+ for iedge in range(self.num_edge):
+ outputs, h0 = self.w_lstm(inputs, h0)
+
+ logits = self.w_pred(outputs)
+ logits = logits / self.temperature
+ logits = self.tanh_constant * torch.tanh(logits)
+ # distribution
+ op_distribution = Categorical(logits=logits)
+ op_index = op_distribution.sample()
+ sampled_arch.append(op_index.item())
+
+ op_log_prob = op_distribution.log_prob(op_index)
+ log_probs.append(op_log_prob.view(-1))
+ op_entropy = op_distribution.entropy()
+ entropys.append(op_entropy.view(-1))
+
+ # obtain the input embedding for the next step
+ inputs = self.w_embd(op_index)
+ return (
+ torch.sum(torch.cat(log_probs)),
+ torch.sum(torch.cat(entropys)),
+ sampled_arch,
+ )
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_gdas.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_gdas.py
new file mode 100644
index 00000000..82f7b9a1
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_gdas.py
@@ -0,0 +1,142 @@
+###########################################################################
+# Searching for A Robust Neural Architecture in Four GPU Hours, CVPR 2019 #
+###########################################################################
+import torch
+import torch.nn as nn
+from copy import deepcopy
+from ..cell_operations import ResNetBasicblock
+from .search_cells import NAS201SearchCell as SearchCell
+from .genotypes import Structure
+
+
+class TinyNetworkGDAS(nn.Module):
+
+ # def __init__(self, C, N, max_nodes, num_classes, search_space, affine=False, track_running_stats=True):
+ def __init__(
+ self, C, N, max_nodes, num_classes, search_space, affine, track_running_stats
+ ):
+ super(TinyNetworkGDAS, self).__init__()
+ self._C = C
+ self._layerN = N
+ self.max_nodes = max_nodes
+ self.stem = nn.Sequential(
+ nn.Conv2d(3, C, kernel_size=3, padding=1, bias=False), nn.BatchNorm2d(C)
+ )
+
+ layer_channels = [C] * N + [C * 2] + [C * 2] * N + [C * 4] + [C * 4] * N
+ layer_reductions = [False] * N + [True] + [False] * N + [True] + [False] * N
+
+ C_prev, num_edge, edge2index = C, None, None
+ self.cells = nn.ModuleList()
+ for index, (C_curr, reduction) in enumerate(
+ zip(layer_channels, layer_reductions)
+ ):
+ if reduction:
+ cell = ResNetBasicblock(C_prev, C_curr, 2)
+ else:
+ cell = SearchCell(
+ C_prev,
+ C_curr,
+ 1,
+ max_nodes,
+ search_space,
+ affine,
+ track_running_stats,
+ )
+ if num_edge is None:
+ num_edge, edge2index = cell.num_edges, cell.edge2index
+ else:
+ assert (
+ num_edge == cell.num_edges and edge2index == cell.edge2index
+ ), "invalid {:} vs. {:}.".format(num_edge, cell.num_edges)
+ self.cells.append(cell)
+ C_prev = cell.out_dim
+ self.op_names = deepcopy(search_space)
+ self._Layer = len(self.cells)
+ self.edge2index = edge2index
+ self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True))
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(C_prev, num_classes)
+ self.arch_parameters = nn.Parameter(
+ 1e-3 * torch.randn(num_edge, len(search_space))
+ )
+ self.tau = 10
+
+ def get_weights(self):
+ xlist = list(self.stem.parameters()) + list(self.cells.parameters())
+ xlist += list(self.lastact.parameters()) + list(
+ self.global_pooling.parameters()
+ )
+ xlist += list(self.classifier.parameters())
+ return xlist
+
+ def set_tau(self, tau):
+ self.tau = tau
+
+ def get_tau(self):
+ return self.tau
+
+ def get_alphas(self):
+ return [self.arch_parameters]
+
+ def show_alphas(self):
+ with torch.no_grad():
+ return "arch-parameters :\n{:}".format(
+ nn.functional.softmax(self.arch_parameters, dim=-1).cpu()
+ )
+
+ def get_message(self):
+ string = self.extra_repr()
+ for i, cell in enumerate(self.cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self.cells), cell.extra_repr()
+ )
+ return string
+
+ def extra_repr(self):
+ return "{name}(C={_C}, Max-Nodes={max_nodes}, N={_layerN}, L={_Layer})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ def genotype(self):
+ genotypes = []
+ for i in range(1, self.max_nodes):
+ xlist = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ with torch.no_grad():
+ weights = self.arch_parameters[self.edge2index[node_str]]
+ op_name = self.op_names[weights.argmax().item()]
+ xlist.append((op_name, j))
+ genotypes.append(tuple(xlist))
+ return Structure(genotypes)
+
+ def forward(self, inputs):
+ while True:
+ gumbels = -torch.empty_like(self.arch_parameters).exponential_().log()
+ logits = (self.arch_parameters.log_softmax(dim=1) + gumbels) / self.tau
+ probs = nn.functional.softmax(logits, dim=1)
+ index = probs.max(-1, keepdim=True)[1]
+ one_h = torch.zeros_like(logits).scatter_(-1, index, 1.0)
+ hardwts = one_h - probs.detach() + probs
+ if (
+ (torch.isinf(gumbels).any())
+ or (torch.isinf(probs).any())
+ or (torch.isnan(probs).any())
+ ):
+ continue
+ else:
+ break
+
+ feature = self.stem(inputs)
+ for i, cell in enumerate(self.cells):
+ if isinstance(cell, SearchCell):
+ feature = cell.forward_gdas(feature, hardwts, index)
+ else:
+ feature = cell(feature)
+ out = self.lastact(feature)
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+
+ return out, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_gdas_frc_nasnet.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_gdas_frc_nasnet.py
new file mode 100644
index 00000000..9ca5ce75
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_gdas_frc_nasnet.py
@@ -0,0 +1,200 @@
+###########################################################################
+# Searching for A Robust Neural Architecture in Four GPU Hours, CVPR 2019 #
+###########################################################################
+import torch
+import torch.nn as nn
+from copy import deepcopy
+
+from .search_cells import NASNetSearchCell as SearchCell
+from ..cell_operations import RAW_OP_CLASSES
+
+
+# The macro structure is based on NASNet
+class NASNetworkGDAS_FRC(nn.Module):
+ def __init__(
+ self,
+ C,
+ N,
+ steps,
+ multiplier,
+ stem_multiplier,
+ num_classes,
+ search_space,
+ affine,
+ track_running_stats,
+ ):
+ super(NASNetworkGDAS_FRC, self).__init__()
+ self._C = C
+ self._layerN = N
+ self._steps = steps
+ self._multiplier = multiplier
+ self.stem = nn.Sequential(
+ nn.Conv2d(3, C * stem_multiplier, kernel_size=3, padding=1, bias=False),
+ nn.BatchNorm2d(C * stem_multiplier),
+ )
+
+ # config for each layer
+ layer_channels = (
+ [C] * N + [C * 2] + [C * 2] * (N - 1) + [C * 4] + [C * 4] * (N - 1)
+ )
+ layer_reductions = (
+ [False] * N + [True] + [False] * (N - 1) + [True] + [False] * (N - 1)
+ )
+
+ num_edge, edge2index = None, None
+ C_prev_prev, C_prev, C_curr, reduction_prev = (
+ C * stem_multiplier,
+ C * stem_multiplier,
+ C,
+ False,
+ )
+
+ self.cells = nn.ModuleList()
+ for index, (C_curr, reduction) in enumerate(
+ zip(layer_channels, layer_reductions)
+ ):
+ if reduction:
+ cell = RAW_OP_CLASSES["gdas_reduction"](
+ C_prev_prev,
+ C_prev,
+ C_curr,
+ reduction_prev,
+ affine,
+ track_running_stats,
+ )
+ else:
+ cell = SearchCell(
+ search_space,
+ steps,
+ multiplier,
+ C_prev_prev,
+ C_prev,
+ C_curr,
+ reduction,
+ reduction_prev,
+ affine,
+ track_running_stats,
+ )
+ if num_edge is None:
+ num_edge, edge2index = cell.num_edges, cell.edge2index
+ else:
+ assert (
+ reduction
+ or num_edge == cell.num_edges
+ and edge2index == cell.edge2index
+ ), "invalid {:} vs. {:}.".format(num_edge, cell.num_edges)
+ self.cells.append(cell)
+ C_prev_prev, C_prev, reduction_prev = (
+ C_prev,
+ cell.multiplier * C_curr,
+ reduction,
+ )
+ self.op_names = deepcopy(search_space)
+ self._Layer = len(self.cells)
+ self.edge2index = edge2index
+ self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True))
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(C_prev, num_classes)
+ self.arch_parameters = nn.Parameter(
+ 1e-3 * torch.randn(num_edge, len(search_space))
+ )
+ self.tau = 10
+
+ def get_weights(self):
+ xlist = list(self.stem.parameters()) + list(self.cells.parameters())
+ xlist += list(self.lastact.parameters()) + list(
+ self.global_pooling.parameters()
+ )
+ xlist += list(self.classifier.parameters())
+ return xlist
+
+ def set_tau(self, tau):
+ self.tau = tau
+
+ def get_tau(self):
+ return self.tau
+
+ def get_alphas(self):
+ return [self.arch_parameters]
+
+ def show_alphas(self):
+ with torch.no_grad():
+ A = "arch-normal-parameters :\n{:}".format(
+ nn.functional.softmax(self.arch_parameters, dim=-1).cpu()
+ )
+ return "{:}".format(A)
+
+ def get_message(self):
+ string = self.extra_repr()
+ for i, cell in enumerate(self.cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self.cells), cell.extra_repr()
+ )
+ return string
+
+ def extra_repr(self):
+ return "{name}(C={_C}, N={_layerN}, steps={_steps}, multiplier={_multiplier}, L={_Layer})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ def genotype(self):
+ def _parse(weights):
+ gene = []
+ for i in range(self._steps):
+ edges = []
+ for j in range(2 + i):
+ node_str = "{:}<-{:}".format(i, j)
+ ws = weights[self.edge2index[node_str]]
+ for k, op_name in enumerate(self.op_names):
+ if op_name == "none":
+ continue
+ edges.append((op_name, j, ws[k]))
+ edges = sorted(edges, key=lambda x: -x[-1])
+ selected_edges = edges[:2]
+ gene.append(tuple(selected_edges))
+ return gene
+
+ with torch.no_grad():
+ gene_normal = _parse(
+ torch.softmax(self.arch_parameters, dim=-1).cpu().numpy()
+ )
+ return {
+ "normal": gene_normal,
+ "normal_concat": list(
+ range(2 + self._steps - self._multiplier, self._steps + 2)
+ ),
+ }
+
+ def forward(self, inputs):
+ def get_gumbel_prob(xins):
+ while True:
+ gumbels = -torch.empty_like(xins).exponential_().log()
+ logits = (xins.log_softmax(dim=1) + gumbels) / self.tau
+ probs = nn.functional.softmax(logits, dim=1)
+ index = probs.max(-1, keepdim=True)[1]
+ one_h = torch.zeros_like(logits).scatter_(-1, index, 1.0)
+ hardwts = one_h - probs.detach() + probs
+ if (
+ (torch.isinf(gumbels).any())
+ or (torch.isinf(probs).any())
+ or (torch.isnan(probs).any())
+ ):
+ continue
+ else:
+ break
+ return hardwts, index
+
+ hardwts, index = get_gumbel_prob(self.arch_parameters)
+
+ s0 = s1 = self.stem(inputs)
+ for i, cell in enumerate(self.cells):
+ if cell.reduction:
+ s0, s1 = s1, cell(s0, s1)
+ else:
+ s0, s1 = s1, cell.forward_gdas(s0, s1, hardwts, index)
+ out = self.lastact(s1)
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+
+ return out, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_gdas_nasnet.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_gdas_nasnet.py
new file mode 100644
index 00000000..5aff5d35
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_gdas_nasnet.py
@@ -0,0 +1,197 @@
+###########################################################################
+# Searching for A Robust Neural Architecture in Four GPU Hours, CVPR 2019 #
+###########################################################################
+import torch
+import torch.nn as nn
+from copy import deepcopy
+from .search_cells import NASNetSearchCell as SearchCell
+
+
+# The macro structure is based on NASNet
+class NASNetworkGDAS(nn.Module):
+ def __init__(
+ self,
+ C,
+ N,
+ steps,
+ multiplier,
+ stem_multiplier,
+ num_classes,
+ search_space,
+ affine,
+ track_running_stats,
+ ):
+ super(NASNetworkGDAS, self).__init__()
+ self._C = C
+ self._layerN = N
+ self._steps = steps
+ self._multiplier = multiplier
+ self.stem = nn.Sequential(
+ nn.Conv2d(3, C * stem_multiplier, kernel_size=3, padding=1, bias=False),
+ nn.BatchNorm2d(C * stem_multiplier),
+ )
+
+ # config for each layer
+ layer_channels = (
+ [C] * N + [C * 2] + [C * 2] * (N - 1) + [C * 4] + [C * 4] * (N - 1)
+ )
+ layer_reductions = (
+ [False] * N + [True] + [False] * (N - 1) + [True] + [False] * (N - 1)
+ )
+
+ num_edge, edge2index = None, None
+ C_prev_prev, C_prev, C_curr, reduction_prev = (
+ C * stem_multiplier,
+ C * stem_multiplier,
+ C,
+ False,
+ )
+
+ self.cells = nn.ModuleList()
+ for index, (C_curr, reduction) in enumerate(
+ zip(layer_channels, layer_reductions)
+ ):
+ cell = SearchCell(
+ search_space,
+ steps,
+ multiplier,
+ C_prev_prev,
+ C_prev,
+ C_curr,
+ reduction,
+ reduction_prev,
+ affine,
+ track_running_stats,
+ )
+ if num_edge is None:
+ num_edge, edge2index = cell.num_edges, cell.edge2index
+ else:
+ assert (
+ num_edge == cell.num_edges and edge2index == cell.edge2index
+ ), "invalid {:} vs. {:}.".format(num_edge, cell.num_edges)
+ self.cells.append(cell)
+ C_prev_prev, C_prev, reduction_prev = C_prev, multiplier * C_curr, reduction
+ self.op_names = deepcopy(search_space)
+ self._Layer = len(self.cells)
+ self.edge2index = edge2index
+ self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True))
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(C_prev, num_classes)
+ self.arch_normal_parameters = nn.Parameter(
+ 1e-3 * torch.randn(num_edge, len(search_space))
+ )
+ self.arch_reduce_parameters = nn.Parameter(
+ 1e-3 * torch.randn(num_edge, len(search_space))
+ )
+ self.tau = 10
+
+ def get_weights(self):
+ xlist = list(self.stem.parameters()) + list(self.cells.parameters())
+ xlist += list(self.lastact.parameters()) + list(
+ self.global_pooling.parameters()
+ )
+ xlist += list(self.classifier.parameters())
+ return xlist
+
+ def set_tau(self, tau):
+ self.tau = tau
+
+ def get_tau(self):
+ return self.tau
+
+ def get_alphas(self):
+ return [self.arch_normal_parameters, self.arch_reduce_parameters]
+
+ def show_alphas(self):
+ with torch.no_grad():
+ A = "arch-normal-parameters :\n{:}".format(
+ nn.functional.softmax(self.arch_normal_parameters, dim=-1).cpu()
+ )
+ B = "arch-reduce-parameters :\n{:}".format(
+ nn.functional.softmax(self.arch_reduce_parameters, dim=-1).cpu()
+ )
+ return "{:}\n{:}".format(A, B)
+
+ def get_message(self):
+ string = self.extra_repr()
+ for i, cell in enumerate(self.cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self.cells), cell.extra_repr()
+ )
+ return string
+
+ def extra_repr(self):
+ return "{name}(C={_C}, N={_layerN}, steps={_steps}, multiplier={_multiplier}, L={_Layer})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ def genotype(self):
+ def _parse(weights):
+ gene = []
+ for i in range(self._steps):
+ edges = []
+ for j in range(2 + i):
+ node_str = "{:}<-{:}".format(i, j)
+ ws = weights[self.edge2index[node_str]]
+ for k, op_name in enumerate(self.op_names):
+ if op_name == "none":
+ continue
+ edges.append((op_name, j, ws[k]))
+ edges = sorted(edges, key=lambda x: -x[-1])
+ selected_edges = edges[:2]
+ gene.append(tuple(selected_edges))
+ return gene
+
+ with torch.no_grad():
+ gene_normal = _parse(
+ torch.softmax(self.arch_normal_parameters, dim=-1).cpu().numpy()
+ )
+ gene_reduce = _parse(
+ torch.softmax(self.arch_reduce_parameters, dim=-1).cpu().numpy()
+ )
+ return {
+ "normal": gene_normal,
+ "normal_concat": list(
+ range(2 + self._steps - self._multiplier, self._steps + 2)
+ ),
+ "reduce": gene_reduce,
+ "reduce_concat": list(
+ range(2 + self._steps - self._multiplier, self._steps + 2)
+ ),
+ }
+
+ def forward(self, inputs):
+ def get_gumbel_prob(xins):
+ while True:
+ gumbels = -torch.empty_like(xins).exponential_().log()
+ logits = (xins.log_softmax(dim=1) + gumbels) / self.tau
+ probs = nn.functional.softmax(logits, dim=1)
+ index = probs.max(-1, keepdim=True)[1]
+ one_h = torch.zeros_like(logits).scatter_(-1, index, 1.0)
+ hardwts = one_h - probs.detach() + probs
+ if (
+ (torch.isinf(gumbels).any())
+ or (torch.isinf(probs).any())
+ or (torch.isnan(probs).any())
+ ):
+ continue
+ else:
+ break
+ return hardwts, index
+
+ normal_hardwts, normal_index = get_gumbel_prob(self.arch_normal_parameters)
+ reduce_hardwts, reduce_index = get_gumbel_prob(self.arch_reduce_parameters)
+
+ s0 = s1 = self.stem(inputs)
+ for i, cell in enumerate(self.cells):
+ if cell.reduction:
+ hardwts, index = reduce_hardwts, reduce_index
+ else:
+ hardwts, index = normal_hardwts, normal_index
+ s0, s1 = s1, cell.forward_gdas(s0, s1, hardwts, index)
+ out = self.lastact(s1)
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+
+ return out, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_random.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_random.py
new file mode 100644
index 00000000..611dc759
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_random.py
@@ -0,0 +1,102 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+##############################################################################
+# Random Search and Reproducibility for Neural Architecture Search, UAI 2019 #
+##############################################################################
+import torch, random
+import torch.nn as nn
+from copy import deepcopy
+from ..cell_operations import ResNetBasicblock
+from .search_cells import NAS201SearchCell as SearchCell
+from .genotypes import Structure
+
+
+class TinyNetworkRANDOM(nn.Module):
+ def __init__(
+ self, C, N, max_nodes, num_classes, search_space, affine, track_running_stats
+ ):
+ super(TinyNetworkRANDOM, self).__init__()
+ self._C = C
+ self._layerN = N
+ self.max_nodes = max_nodes
+ self.stem = nn.Sequential(
+ nn.Conv2d(3, C, kernel_size=3, padding=1, bias=False), nn.BatchNorm2d(C)
+ )
+
+ layer_channels = [C] * N + [C * 2] + [C * 2] * N + [C * 4] + [C * 4] * N
+ layer_reductions = [False] * N + [True] + [False] * N + [True] + [False] * N
+
+ C_prev, num_edge, edge2index = C, None, None
+ self.cells = nn.ModuleList()
+ for index, (C_curr, reduction) in enumerate(
+ zip(layer_channels, layer_reductions)
+ ):
+ if reduction:
+ cell = ResNetBasicblock(C_prev, C_curr, 2)
+ else:
+ cell = SearchCell(
+ C_prev,
+ C_curr,
+ 1,
+ max_nodes,
+ search_space,
+ affine,
+ track_running_stats,
+ )
+ if num_edge is None:
+ num_edge, edge2index = cell.num_edges, cell.edge2index
+ else:
+ assert (
+ num_edge == cell.num_edges and edge2index == cell.edge2index
+ ), "invalid {:} vs. {:}.".format(num_edge, cell.num_edges)
+ self.cells.append(cell)
+ C_prev = cell.out_dim
+ self.op_names = deepcopy(search_space)
+ self._Layer = len(self.cells)
+ self.edge2index = edge2index
+ self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True))
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(C_prev, num_classes)
+ self.arch_cache = None
+
+ def get_message(self):
+ string = self.extra_repr()
+ for i, cell in enumerate(self.cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self.cells), cell.extra_repr()
+ )
+ return string
+
+ def extra_repr(self):
+ return "{name}(C={_C}, Max-Nodes={max_nodes}, N={_layerN}, L={_Layer})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ def random_genotype(self, set_cache):
+ genotypes = []
+ for i in range(1, self.max_nodes):
+ xlist = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ op_name = random.choice(self.op_names)
+ xlist.append((op_name, j))
+ genotypes.append(tuple(xlist))
+ arch = Structure(genotypes)
+ if set_cache:
+ self.arch_cache = arch
+ return arch
+
+ def forward(self, inputs):
+
+ feature = self.stem(inputs)
+ for i, cell in enumerate(self.cells):
+ if isinstance(cell, SearchCell):
+ feature = cell.forward_dynamic(feature, self.arch_cache)
+ else:
+ feature = cell(feature)
+
+ out = self.lastact(feature)
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+ return out, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_setn.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_setn.py
new file mode 100644
index 00000000..ce38be9a
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_setn.py
@@ -0,0 +1,178 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+######################################################################################
+# One-Shot Neural Architecture Search via Self-Evaluated Template Network, ICCV 2019 #
+######################################################################################
+import torch, random
+import torch.nn as nn
+from copy import deepcopy
+from ..cell_operations import ResNetBasicblock
+from .search_cells import NAS201SearchCell as SearchCell
+from .genotypes import Structure
+
+
+class TinyNetworkSETN(nn.Module):
+ def __init__(
+ self, C, N, max_nodes, num_classes, search_space, affine, track_running_stats
+ ):
+ super(TinyNetworkSETN, self).__init__()
+ self._C = C
+ self._layerN = N
+ self.max_nodes = max_nodes
+ self.stem = nn.Sequential(
+ nn.Conv2d(3, C, kernel_size=3, padding=1, bias=False), nn.BatchNorm2d(C)
+ )
+
+ layer_channels = [C] * N + [C * 2] + [C * 2] * N + [C * 4] + [C * 4] * N
+ layer_reductions = [False] * N + [True] + [False] * N + [True] + [False] * N
+
+ C_prev, num_edge, edge2index = C, None, None
+ self.cells = nn.ModuleList()
+ for index, (C_curr, reduction) in enumerate(
+ zip(layer_channels, layer_reductions)
+ ):
+ if reduction:
+ cell = ResNetBasicblock(C_prev, C_curr, 2)
+ else:
+ cell = SearchCell(
+ C_prev,
+ C_curr,
+ 1,
+ max_nodes,
+ search_space,
+ affine,
+ track_running_stats,
+ )
+ if num_edge is None:
+ num_edge, edge2index = cell.num_edges, cell.edge2index
+ else:
+ assert (
+ num_edge == cell.num_edges and edge2index == cell.edge2index
+ ), "invalid {:} vs. {:}.".format(num_edge, cell.num_edges)
+ self.cells.append(cell)
+ C_prev = cell.out_dim
+ self.op_names = deepcopy(search_space)
+ self._Layer = len(self.cells)
+ self.edge2index = edge2index
+ self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True))
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(C_prev, num_classes)
+ self.arch_parameters = nn.Parameter(
+ 1e-3 * torch.randn(num_edge, len(search_space))
+ )
+ self.mode = "urs"
+ self.dynamic_cell = None
+
+ def set_cal_mode(self, mode, dynamic_cell=None):
+ assert mode in ["urs", "joint", "select", "dynamic"]
+ self.mode = mode
+ if mode == "dynamic":
+ self.dynamic_cell = deepcopy(dynamic_cell)
+ else:
+ self.dynamic_cell = None
+
+ def get_cal_mode(self):
+ return self.mode
+
+ def get_weights(self):
+ xlist = list(self.stem.parameters()) + list(self.cells.parameters())
+ xlist += list(self.lastact.parameters()) + list(
+ self.global_pooling.parameters()
+ )
+ xlist += list(self.classifier.parameters())
+ return xlist
+
+ def get_alphas(self):
+ return [self.arch_parameters]
+
+ def get_message(self):
+ string = self.extra_repr()
+ for i, cell in enumerate(self.cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self.cells), cell.extra_repr()
+ )
+ return string
+
+ def extra_repr(self):
+ return "{name}(C={_C}, Max-Nodes={max_nodes}, N={_layerN}, L={_Layer})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ def genotype(self):
+ genotypes = []
+ for i in range(1, self.max_nodes):
+ xlist = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ with torch.no_grad():
+ weights = self.arch_parameters[self.edge2index[node_str]]
+ op_name = self.op_names[weights.argmax().item()]
+ xlist.append((op_name, j))
+ genotypes.append(tuple(xlist))
+ return Structure(genotypes)
+
+ def dync_genotype(self, use_random=False):
+ genotypes = []
+ with torch.no_grad():
+ alphas_cpu = nn.functional.softmax(self.arch_parameters, dim=-1)
+ for i in range(1, self.max_nodes):
+ xlist = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ if use_random:
+ op_name = random.choice(self.op_names)
+ else:
+ weights = alphas_cpu[self.edge2index[node_str]]
+ op_index = torch.multinomial(weights, 1).item()
+ op_name = self.op_names[op_index]
+ xlist.append((op_name, j))
+ genotypes.append(tuple(xlist))
+ return Structure(genotypes)
+
+ def get_log_prob(self, arch):
+ with torch.no_grad():
+ logits = nn.functional.log_softmax(self.arch_parameters, dim=-1)
+ select_logits = []
+ for i, node_info in enumerate(arch.nodes):
+ for op, xin in node_info:
+ node_str = "{:}<-{:}".format(i + 1, xin)
+ op_index = self.op_names.index(op)
+ select_logits.append(logits[self.edge2index[node_str], op_index])
+ return sum(select_logits).item()
+
+ def return_topK(self, K):
+ archs = Structure.gen_all(self.op_names, self.max_nodes, False)
+ pairs = [(self.get_log_prob(arch), arch) for arch in archs]
+ if K < 0 or K >= len(archs):
+ K = len(archs)
+ sorted_pairs = sorted(pairs, key=lambda x: -x[0])
+ return_pairs = [sorted_pairs[_][1] for _ in range(K)]
+ return return_pairs
+
+ def forward(self, inputs):
+ alphas = nn.functional.softmax(self.arch_parameters, dim=-1)
+ with torch.no_grad():
+ alphas_cpu = alphas.detach().cpu()
+
+ feature = self.stem(inputs)
+ for i, cell in enumerate(self.cells):
+ if isinstance(cell, SearchCell):
+ if self.mode == "urs":
+ feature = cell.forward_urs(feature)
+ elif self.mode == "select":
+ feature = cell.forward_select(feature, alphas_cpu)
+ elif self.mode == "joint":
+ feature = cell.forward_joint(feature, alphas)
+ elif self.mode == "dynamic":
+ feature = cell.forward_dynamic(feature, self.dynamic_cell)
+ else:
+ raise ValueError("invalid mode={:}".format(self.mode))
+ else:
+ feature = cell(feature)
+
+ out = self.lastact(feature)
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+
+ return out, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_setn_nasnet.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_setn_nasnet.py
new file mode 100644
index 00000000..c406fc37
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/cell_searchs/search_model_setn_nasnet.py
@@ -0,0 +1,205 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+######################################################################################
+# One-Shot Neural Architecture Search via Self-Evaluated Template Network, ICCV 2019 #
+######################################################################################
+import torch
+import torch.nn as nn
+from copy import deepcopy
+from typing import List, Text, Dict
+from .search_cells import NASNetSearchCell as SearchCell
+
+
+# The macro structure is based on NASNet
+class NASNetworkSETN(nn.Module):
+ def __init__(
+ self,
+ C: int,
+ N: int,
+ steps: int,
+ multiplier: int,
+ stem_multiplier: int,
+ num_classes: int,
+ search_space: List[Text],
+ affine: bool,
+ track_running_stats: bool,
+ ):
+ super(NASNetworkSETN, self).__init__()
+ self._C = C
+ self._layerN = N
+ self._steps = steps
+ self._multiplier = multiplier
+ self.stem = nn.Sequential(
+ nn.Conv2d(3, C * stem_multiplier, kernel_size=3, padding=1, bias=False),
+ nn.BatchNorm2d(C * stem_multiplier),
+ )
+
+ # config for each layer
+ layer_channels = (
+ [C] * N + [C * 2] + [C * 2] * (N - 1) + [C * 4] + [C * 4] * (N - 1)
+ )
+ layer_reductions = (
+ [False] * N + [True] + [False] * (N - 1) + [True] + [False] * (N - 1)
+ )
+
+ num_edge, edge2index = None, None
+ C_prev_prev, C_prev, C_curr, reduction_prev = (
+ C * stem_multiplier,
+ C * stem_multiplier,
+ C,
+ False,
+ )
+
+ self.cells = nn.ModuleList()
+ for index, (C_curr, reduction) in enumerate(
+ zip(layer_channels, layer_reductions)
+ ):
+ cell = SearchCell(
+ search_space,
+ steps,
+ multiplier,
+ C_prev_prev,
+ C_prev,
+ C_curr,
+ reduction,
+ reduction_prev,
+ affine,
+ track_running_stats,
+ )
+ if num_edge is None:
+ num_edge, edge2index = cell.num_edges, cell.edge2index
+ else:
+ assert (
+ num_edge == cell.num_edges and edge2index == cell.edge2index
+ ), "invalid {:} vs. {:}.".format(num_edge, cell.num_edges)
+ self.cells.append(cell)
+ C_prev_prev, C_prev, reduction_prev = C_prev, multiplier * C_curr, reduction
+ self.op_names = deepcopy(search_space)
+ self._Layer = len(self.cells)
+ self.edge2index = edge2index
+ self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True))
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(C_prev, num_classes)
+ self.arch_normal_parameters = nn.Parameter(
+ 1e-3 * torch.randn(num_edge, len(search_space))
+ )
+ self.arch_reduce_parameters = nn.Parameter(
+ 1e-3 * torch.randn(num_edge, len(search_space))
+ )
+ self.mode = "urs"
+ self.dynamic_cell = None
+
+ def set_cal_mode(self, mode, dynamic_cell=None):
+ assert mode in ["urs", "joint", "select", "dynamic"]
+ self.mode = mode
+ if mode == "dynamic":
+ self.dynamic_cell = deepcopy(dynamic_cell)
+ else:
+ self.dynamic_cell = None
+
+ def get_weights(self):
+ xlist = list(self.stem.parameters()) + list(self.cells.parameters())
+ xlist += list(self.lastact.parameters()) + list(
+ self.global_pooling.parameters()
+ )
+ xlist += list(self.classifier.parameters())
+ return xlist
+
+ def get_alphas(self):
+ return [self.arch_normal_parameters, self.arch_reduce_parameters]
+
+ def show_alphas(self):
+ with torch.no_grad():
+ A = "arch-normal-parameters :\n{:}".format(
+ nn.functional.softmax(self.arch_normal_parameters, dim=-1).cpu()
+ )
+ B = "arch-reduce-parameters :\n{:}".format(
+ nn.functional.softmax(self.arch_reduce_parameters, dim=-1).cpu()
+ )
+ return "{:}\n{:}".format(A, B)
+
+ def get_message(self):
+ string = self.extra_repr()
+ for i, cell in enumerate(self.cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self.cells), cell.extra_repr()
+ )
+ return string
+
+ def extra_repr(self):
+ return "{name}(C={_C}, N={_layerN}, steps={_steps}, multiplier={_multiplier}, L={_Layer})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ def dync_genotype(self, use_random=False):
+ genotypes = []
+ with torch.no_grad():
+ alphas_cpu = nn.functional.softmax(self.arch_parameters, dim=-1)
+ for i in range(1, self.max_nodes):
+ xlist = []
+ for j in range(i):
+ node_str = "{:}<-{:}".format(i, j)
+ if use_random:
+ op_name = random.choice(self.op_names)
+ else:
+ weights = alphas_cpu[self.edge2index[node_str]]
+ op_index = torch.multinomial(weights, 1).item()
+ op_name = self.op_names[op_index]
+ xlist.append((op_name, j))
+ genotypes.append(tuple(xlist))
+ return Structure(genotypes)
+
+ def genotype(self):
+ def _parse(weights):
+ gene = []
+ for i in range(self._steps):
+ edges = []
+ for j in range(2 + i):
+ node_str = "{:}<-{:}".format(i, j)
+ ws = weights[self.edge2index[node_str]]
+ for k, op_name in enumerate(self.op_names):
+ if op_name == "none":
+ continue
+ edges.append((op_name, j, ws[k]))
+ edges = sorted(edges, key=lambda x: -x[-1])
+ selected_edges = edges[:2]
+ gene.append(tuple(selected_edges))
+ return gene
+
+ with torch.no_grad():
+ gene_normal = _parse(
+ torch.softmax(self.arch_normal_parameters, dim=-1).cpu().numpy()
+ )
+ gene_reduce = _parse(
+ torch.softmax(self.arch_reduce_parameters, dim=-1).cpu().numpy()
+ )
+ return {
+ "normal": gene_normal,
+ "normal_concat": list(
+ range(2 + self._steps - self._multiplier, self._steps + 2)
+ ),
+ "reduce": gene_reduce,
+ "reduce_concat": list(
+ range(2 + self._steps - self._multiplier, self._steps + 2)
+ ),
+ }
+
+ def forward(self, inputs):
+ normal_hardwts = nn.functional.softmax(self.arch_normal_parameters, dim=-1)
+ reduce_hardwts = nn.functional.softmax(self.arch_reduce_parameters, dim=-1)
+
+ s0 = s1 = self.stem(inputs)
+ for i, cell in enumerate(self.cells):
+ # [TODO]
+ raise NotImplementedError
+ if cell.reduction:
+ hardwts, index = reduce_hardwts, reduce_index
+ else:
+ hardwts, index = normal_hardwts, normal_index
+ s0, s1 = s1, cell.forward_gdas(s0, s1, hardwts, index)
+ out = self.lastact(s1)
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+
+ return out, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/clone_weights.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/clone_weights.py
new file mode 100644
index 00000000..9e904ac2
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/clone_weights.py
@@ -0,0 +1,74 @@
+import torch
+import torch.nn as nn
+
+
+def copy_conv(module, init):
+ assert isinstance(module, nn.Conv2d), "invalid module : {:}".format(module)
+ assert isinstance(init, nn.Conv2d), "invalid module : {:}".format(init)
+ new_i, new_o = module.in_channels, module.out_channels
+ module.weight.copy_(init.weight.detach()[:new_o, :new_i])
+ if module.bias is not None:
+ module.bias.copy_(init.bias.detach()[:new_o])
+
+
+def copy_bn(module, init):
+ assert isinstance(module, nn.BatchNorm2d), "invalid module : {:}".format(module)
+ assert isinstance(init, nn.BatchNorm2d), "invalid module : {:}".format(init)
+ num_features = module.num_features
+ if module.weight is not None:
+ module.weight.copy_(init.weight.detach()[:num_features])
+ if module.bias is not None:
+ module.bias.copy_(init.bias.detach()[:num_features])
+ if module.running_mean is not None:
+ module.running_mean.copy_(init.running_mean.detach()[:num_features])
+ if module.running_var is not None:
+ module.running_var.copy_(init.running_var.detach()[:num_features])
+
+
+def copy_fc(module, init):
+ assert isinstance(module, nn.Linear), "invalid module : {:}".format(module)
+ assert isinstance(init, nn.Linear), "invalid module : {:}".format(init)
+ new_i, new_o = module.in_features, module.out_features
+ module.weight.copy_(init.weight.detach()[:new_o, :new_i])
+ if module.bias is not None:
+ module.bias.copy_(init.bias.detach()[:new_o])
+
+
+def copy_base(module, init):
+ assert type(module).__name__ in [
+ "ConvBNReLU",
+ "Downsample",
+ ], "invalid module : {:}".format(module)
+ assert type(init).__name__ in [
+ "ConvBNReLU",
+ "Downsample",
+ ], "invalid module : {:}".format(init)
+ if module.conv is not None:
+ copy_conv(module.conv, init.conv)
+ if module.bn is not None:
+ copy_bn(module.bn, init.bn)
+
+
+def copy_basic(module, init):
+ copy_base(module.conv_a, init.conv_a)
+ copy_base(module.conv_b, init.conv_b)
+ if module.downsample is not None:
+ if init.downsample is not None:
+ copy_base(module.downsample, init.downsample)
+ # else:
+ # import pdb; pdb.set_trace()
+
+
+def init_from_model(network, init_model):
+ with torch.no_grad():
+ copy_fc(network.classifier, init_model.classifier)
+ for base, target in zip(init_model.layers, network.layers):
+ assert (
+ type(base).__name__ == type(target).__name__
+ ), "invalid type : {:} vs {:}".format(base, target)
+ if type(base).__name__ == "ConvBNReLU":
+ copy_base(target, base)
+ elif type(base).__name__ == "ResNetBasicblock":
+ copy_basic(target, base)
+ else:
+ raise ValueError("unknown type name : {:}".format(type(base).__name__))
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/initialization.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/initialization.py
new file mode 100644
index 00000000..e82d723a
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/initialization.py
@@ -0,0 +1,16 @@
+import torch
+import torch.nn as nn
+
+
+def initialize_resnet(m):
+ if isinstance(m, nn.Conv2d):
+ nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
+ if m.bias is not None:
+ nn.init.constant_(m.bias, 0)
+ elif isinstance(m, nn.BatchNorm2d):
+ nn.init.constant_(m.weight, 1)
+ if m.bias is not None:
+ nn.init.constant_(m.bias, 0)
+ elif isinstance(m, nn.Linear):
+ nn.init.normal_(m.weight, 0, 0.01)
+ nn.init.constant_(m.bias, 0)
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferCifarResNet.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferCifarResNet.py
new file mode 100644
index 00000000..1731392a
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferCifarResNet.py
@@ -0,0 +1,287 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+import torch.nn as nn
+import torch.nn.functional as F
+
+from ..initialization import initialize_resnet
+
+
+class ConvBNReLU(nn.Module):
+ def __init__(
+ self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu
+ ):
+ super(ConvBNReLU, self).__init__()
+ if has_avg:
+ self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0)
+ else:
+ self.avg = None
+ self.conv = nn.Conv2d(
+ nIn,
+ nOut,
+ kernel_size=kernel,
+ stride=stride,
+ padding=padding,
+ dilation=1,
+ groups=1,
+ bias=bias,
+ )
+ if has_bn:
+ self.bn = nn.BatchNorm2d(nOut)
+ else:
+ self.bn = None
+ if has_relu:
+ self.relu = nn.ReLU(inplace=True)
+ else:
+ self.relu = None
+
+ def forward(self, inputs):
+ if self.avg:
+ out = self.avg(inputs)
+ else:
+ out = inputs
+ conv = self.conv(out)
+ if self.bn:
+ out = self.bn(conv)
+ else:
+ out = conv
+ if self.relu:
+ out = self.relu(out)
+ else:
+ out = out
+
+ return out
+
+
+class ResNetBasicblock(nn.Module):
+ num_conv = 2
+ expansion = 1
+
+ def __init__(self, iCs, stride):
+ super(ResNetBasicblock, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+ assert isinstance(iCs, tuple) or isinstance(
+ iCs, list
+ ), "invalid type of iCs : {:}".format(iCs)
+ assert len(iCs) == 3, "invalid lengths of iCs : {:}".format(iCs)
+
+ self.conv_a = ConvBNReLU(
+ iCs[0],
+ iCs[1],
+ 3,
+ stride,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ )
+ self.conv_b = ConvBNReLU(
+ iCs[1], iCs[2], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False
+ )
+ residual_in = iCs[0]
+ if stride == 2:
+ self.downsample = ConvBNReLU(
+ iCs[0],
+ iCs[2],
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=True,
+ has_bn=False,
+ has_relu=False,
+ )
+ residual_in = iCs[2]
+ elif iCs[0] != iCs[2]:
+ self.downsample = ConvBNReLU(
+ iCs[0],
+ iCs[2],
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=False,
+ )
+ else:
+ self.downsample = None
+ # self.out_dim = max(residual_in, iCs[2])
+ self.out_dim = iCs[2]
+
+ def forward(self, inputs):
+ basicblock = self.conv_a(inputs)
+ basicblock = self.conv_b(basicblock)
+
+ if self.downsample is not None:
+ residual = self.downsample(inputs)
+ else:
+ residual = inputs
+ out = residual + basicblock
+ return F.relu(out, inplace=True)
+
+
+class ResNetBottleneck(nn.Module):
+ expansion = 4
+ num_conv = 3
+
+ def __init__(self, iCs, stride):
+ super(ResNetBottleneck, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+ assert isinstance(iCs, tuple) or isinstance(
+ iCs, list
+ ), "invalid type of iCs : {:}".format(iCs)
+ assert len(iCs) == 4, "invalid lengths of iCs : {:}".format(iCs)
+ self.conv_1x1 = ConvBNReLU(
+ iCs[0], iCs[1], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True
+ )
+ self.conv_3x3 = ConvBNReLU(
+ iCs[1],
+ iCs[2],
+ 3,
+ stride,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ )
+ self.conv_1x4 = ConvBNReLU(
+ iCs[2], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False
+ )
+ residual_in = iCs[0]
+ if stride == 2:
+ self.downsample = ConvBNReLU(
+ iCs[0],
+ iCs[3],
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=True,
+ has_bn=False,
+ has_relu=False,
+ )
+ residual_in = iCs[3]
+ elif iCs[0] != iCs[3]:
+ self.downsample = ConvBNReLU(
+ iCs[0],
+ iCs[3],
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=False,
+ has_bn=False,
+ has_relu=False,
+ )
+ residual_in = iCs[3]
+ else:
+ self.downsample = None
+ # self.out_dim = max(residual_in, iCs[3])
+ self.out_dim = iCs[3]
+
+ def forward(self, inputs):
+
+ bottleneck = self.conv_1x1(inputs)
+ bottleneck = self.conv_3x3(bottleneck)
+ bottleneck = self.conv_1x4(bottleneck)
+
+ if self.downsample is not None:
+ residual = self.downsample(inputs)
+ else:
+ residual = inputs
+ out = residual + bottleneck
+ return F.relu(out, inplace=True)
+
+
+class InferCifarResNet(nn.Module):
+ def __init__(
+ self, block_name, depth, xblocks, xchannels, num_classes, zero_init_residual
+ ):
+ super(InferCifarResNet, self).__init__()
+
+ # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
+ if block_name == "ResNetBasicblock":
+ block = ResNetBasicblock
+ assert (depth - 2) % 6 == 0, "depth should be one of 20, 32, 44, 56, 110"
+ layer_blocks = (depth - 2) // 6
+ elif block_name == "ResNetBottleneck":
+ block = ResNetBottleneck
+ assert (depth - 2) % 9 == 0, "depth should be one of 164"
+ layer_blocks = (depth - 2) // 9
+ else:
+ raise ValueError("invalid block : {:}".format(block_name))
+ assert len(xblocks) == 3, "invalid xblocks : {:}".format(xblocks)
+
+ self.message = (
+ "InferWidthCifarResNet : Depth : {:} , Layers for each block : {:}".format(
+ depth, layer_blocks
+ )
+ )
+ self.num_classes = num_classes
+ self.xchannels = xchannels
+ self.layers = nn.ModuleList(
+ [
+ ConvBNReLU(
+ xchannels[0],
+ xchannels[1],
+ 3,
+ 1,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ )
+ ]
+ )
+ last_channel_idx = 1
+ for stage in range(3):
+ for iL in range(layer_blocks):
+ num_conv = block.num_conv
+ iCs = self.xchannels[last_channel_idx : last_channel_idx + num_conv + 1]
+ stride = 2 if stage > 0 and iL == 0 else 1
+ module = block(iCs, stride)
+ last_channel_idx += num_conv
+ self.xchannels[last_channel_idx] = module.out_dim
+ self.layers.append(module)
+ self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iCs={:}, oC={:3d}, stride={:}".format(
+ stage,
+ iL,
+ layer_blocks,
+ len(self.layers) - 1,
+ iCs,
+ module.out_dim,
+ stride,
+ )
+ if iL + 1 == xblocks[stage]: # reach the maximum depth
+ out_channel = module.out_dim
+ for iiL in range(iL + 1, layer_blocks):
+ last_channel_idx += num_conv
+ self.xchannels[last_channel_idx] = module.out_dim
+ break
+
+ self.avgpool = nn.AvgPool2d(8)
+ self.classifier = nn.Linear(self.xchannels[-1], num_classes)
+
+ self.apply(initialize_resnet)
+ if zero_init_residual:
+ for m in self.modules():
+ if isinstance(m, ResNetBasicblock):
+ nn.init.constant_(m.conv_b.bn.weight, 0)
+ elif isinstance(m, ResNetBottleneck):
+ nn.init.constant_(m.conv_1x4.bn.weight, 0)
+
+ def get_message(self):
+ return self.message
+
+ def forward(self, inputs):
+ x = inputs
+ for i, layer in enumerate(self.layers):
+ x = layer(x)
+ features = self.avgpool(x)
+ features = features.view(features.size(0), -1)
+ logits = self.classifier(features)
+ return features, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferCifarResNet_depth.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferCifarResNet_depth.py
new file mode 100644
index 00000000..c6f9bb32
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferCifarResNet_depth.py
@@ -0,0 +1,263 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+import torch.nn as nn
+import torch.nn.functional as F
+from ..initialization import initialize_resnet
+
+
+class ConvBNReLU(nn.Module):
+ def __init__(
+ self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu
+ ):
+ super(ConvBNReLU, self).__init__()
+ if has_avg:
+ self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0)
+ else:
+ self.avg = None
+ self.conv = nn.Conv2d(
+ nIn,
+ nOut,
+ kernel_size=kernel,
+ stride=stride,
+ padding=padding,
+ dilation=1,
+ groups=1,
+ bias=bias,
+ )
+ if has_bn:
+ self.bn = nn.BatchNorm2d(nOut)
+ else:
+ self.bn = None
+ if has_relu:
+ self.relu = nn.ReLU(inplace=True)
+ else:
+ self.relu = None
+
+ def forward(self, inputs):
+ if self.avg:
+ out = self.avg(inputs)
+ else:
+ out = inputs
+ conv = self.conv(out)
+ if self.bn:
+ out = self.bn(conv)
+ else:
+ out = conv
+ if self.relu:
+ out = self.relu(out)
+ else:
+ out = out
+
+ return out
+
+
+class ResNetBasicblock(nn.Module):
+ num_conv = 2
+ expansion = 1
+
+ def __init__(self, inplanes, planes, stride):
+ super(ResNetBasicblock, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+
+ self.conv_a = ConvBNReLU(
+ inplanes,
+ planes,
+ 3,
+ stride,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ )
+ self.conv_b = ConvBNReLU(
+ planes, planes, 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False
+ )
+ if stride == 2:
+ self.downsample = ConvBNReLU(
+ inplanes,
+ planes,
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=True,
+ has_bn=False,
+ has_relu=False,
+ )
+ elif inplanes != planes:
+ self.downsample = ConvBNReLU(
+ inplanes,
+ planes,
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=False,
+ )
+ else:
+ self.downsample = None
+ self.out_dim = planes
+
+ def forward(self, inputs):
+ basicblock = self.conv_a(inputs)
+ basicblock = self.conv_b(basicblock)
+
+ if self.downsample is not None:
+ residual = self.downsample(inputs)
+ else:
+ residual = inputs
+ out = residual + basicblock
+ return F.relu(out, inplace=True)
+
+
+class ResNetBottleneck(nn.Module):
+ expansion = 4
+ num_conv = 3
+
+ def __init__(self, inplanes, planes, stride):
+ super(ResNetBottleneck, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+ self.conv_1x1 = ConvBNReLU(
+ inplanes, planes, 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True
+ )
+ self.conv_3x3 = ConvBNReLU(
+ planes,
+ planes,
+ 3,
+ stride,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ )
+ self.conv_1x4 = ConvBNReLU(
+ planes,
+ planes * self.expansion,
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=False,
+ )
+ if stride == 2:
+ self.downsample = ConvBNReLU(
+ inplanes,
+ planes * self.expansion,
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=True,
+ has_bn=False,
+ has_relu=False,
+ )
+ elif inplanes != planes * self.expansion:
+ self.downsample = ConvBNReLU(
+ inplanes,
+ planes * self.expansion,
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=False,
+ has_bn=False,
+ has_relu=False,
+ )
+ else:
+ self.downsample = None
+ self.out_dim = planes * self.expansion
+
+ def forward(self, inputs):
+
+ bottleneck = self.conv_1x1(inputs)
+ bottleneck = self.conv_3x3(bottleneck)
+ bottleneck = self.conv_1x4(bottleneck)
+
+ if self.downsample is not None:
+ residual = self.downsample(inputs)
+ else:
+ residual = inputs
+ out = residual + bottleneck
+ return F.relu(out, inplace=True)
+
+
+class InferDepthCifarResNet(nn.Module):
+ def __init__(self, block_name, depth, xblocks, num_classes, zero_init_residual):
+ super(InferDepthCifarResNet, self).__init__()
+
+ # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
+ if block_name == "ResNetBasicblock":
+ block = ResNetBasicblock
+ assert (depth - 2) % 6 == 0, "depth should be one of 20, 32, 44, 56, 110"
+ layer_blocks = (depth - 2) // 6
+ elif block_name == "ResNetBottleneck":
+ block = ResNetBottleneck
+ assert (depth - 2) % 9 == 0, "depth should be one of 164"
+ layer_blocks = (depth - 2) // 9
+ else:
+ raise ValueError("invalid block : {:}".format(block_name))
+ assert len(xblocks) == 3, "invalid xblocks : {:}".format(xblocks)
+
+ self.message = (
+ "InferWidthCifarResNet : Depth : {:} , Layers for each block : {:}".format(
+ depth, layer_blocks
+ )
+ )
+ self.num_classes = num_classes
+ self.layers = nn.ModuleList(
+ [
+ ConvBNReLU(
+ 3, 16, 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=True
+ )
+ ]
+ )
+ self.channels = [16]
+ for stage in range(3):
+ for iL in range(layer_blocks):
+ iC = self.channels[-1]
+ planes = 16 * (2 ** stage)
+ stride = 2 if stage > 0 and iL == 0 else 1
+ module = block(iC, planes, stride)
+ self.channels.append(module.out_dim)
+ self.layers.append(module)
+ self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iC={:}, oC={:3d}, stride={:}".format(
+ stage,
+ iL,
+ layer_blocks,
+ len(self.layers) - 1,
+ planes,
+ module.out_dim,
+ stride,
+ )
+ if iL + 1 == xblocks[stage]: # reach the maximum depth
+ break
+
+ self.avgpool = nn.AvgPool2d(8)
+ self.classifier = nn.Linear(self.channels[-1], num_classes)
+
+ self.apply(initialize_resnet)
+ if zero_init_residual:
+ for m in self.modules():
+ if isinstance(m, ResNetBasicblock):
+ nn.init.constant_(m.conv_b.bn.weight, 0)
+ elif isinstance(m, ResNetBottleneck):
+ nn.init.constant_(m.conv_1x4.bn.weight, 0)
+
+ def get_message(self):
+ return self.message
+
+ def forward(self, inputs):
+ x = inputs
+ for i, layer in enumerate(self.layers):
+ x = layer(x)
+ features = self.avgpool(x)
+ features = features.view(features.size(0), -1)
+ logits = self.classifier(features)
+ return features, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferCifarResNet_width.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferCifarResNet_width.py
new file mode 100644
index 00000000..9400f71a
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferCifarResNet_width.py
@@ -0,0 +1,277 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+import torch.nn as nn
+import torch.nn.functional as F
+from ..initialization import initialize_resnet
+
+
+class ConvBNReLU(nn.Module):
+ def __init__(
+ self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu
+ ):
+ super(ConvBNReLU, self).__init__()
+ if has_avg:
+ self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0)
+ else:
+ self.avg = None
+ self.conv = nn.Conv2d(
+ nIn,
+ nOut,
+ kernel_size=kernel,
+ stride=stride,
+ padding=padding,
+ dilation=1,
+ groups=1,
+ bias=bias,
+ )
+ if has_bn:
+ self.bn = nn.BatchNorm2d(nOut)
+ else:
+ self.bn = None
+ if has_relu:
+ self.relu = nn.ReLU(inplace=True)
+ else:
+ self.relu = None
+
+ def forward(self, inputs):
+ if self.avg:
+ out = self.avg(inputs)
+ else:
+ out = inputs
+ conv = self.conv(out)
+ if self.bn:
+ out = self.bn(conv)
+ else:
+ out = conv
+ if self.relu:
+ out = self.relu(out)
+ else:
+ out = out
+
+ return out
+
+
+class ResNetBasicblock(nn.Module):
+ num_conv = 2
+ expansion = 1
+
+ def __init__(self, iCs, stride):
+ super(ResNetBasicblock, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+ assert isinstance(iCs, tuple) or isinstance(
+ iCs, list
+ ), "invalid type of iCs : {:}".format(iCs)
+ assert len(iCs) == 3, "invalid lengths of iCs : {:}".format(iCs)
+
+ self.conv_a = ConvBNReLU(
+ iCs[0],
+ iCs[1],
+ 3,
+ stride,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ )
+ self.conv_b = ConvBNReLU(
+ iCs[1], iCs[2], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False
+ )
+ residual_in = iCs[0]
+ if stride == 2:
+ self.downsample = ConvBNReLU(
+ iCs[0],
+ iCs[2],
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=True,
+ has_bn=False,
+ has_relu=False,
+ )
+ residual_in = iCs[2]
+ elif iCs[0] != iCs[2]:
+ self.downsample = ConvBNReLU(
+ iCs[0],
+ iCs[2],
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=False,
+ )
+ else:
+ self.downsample = None
+ # self.out_dim = max(residual_in, iCs[2])
+ self.out_dim = iCs[2]
+
+ def forward(self, inputs):
+ basicblock = self.conv_a(inputs)
+ basicblock = self.conv_b(basicblock)
+
+ if self.downsample is not None:
+ residual = self.downsample(inputs)
+ else:
+ residual = inputs
+ out = residual + basicblock
+ return F.relu(out, inplace=True)
+
+
+class ResNetBottleneck(nn.Module):
+ expansion = 4
+ num_conv = 3
+
+ def __init__(self, iCs, stride):
+ super(ResNetBottleneck, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+ assert isinstance(iCs, tuple) or isinstance(
+ iCs, list
+ ), "invalid type of iCs : {:}".format(iCs)
+ assert len(iCs) == 4, "invalid lengths of iCs : {:}".format(iCs)
+ self.conv_1x1 = ConvBNReLU(
+ iCs[0], iCs[1], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True
+ )
+ self.conv_3x3 = ConvBNReLU(
+ iCs[1],
+ iCs[2],
+ 3,
+ stride,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ )
+ self.conv_1x4 = ConvBNReLU(
+ iCs[2], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False
+ )
+ residual_in = iCs[0]
+ if stride == 2:
+ self.downsample = ConvBNReLU(
+ iCs[0],
+ iCs[3],
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=True,
+ has_bn=False,
+ has_relu=False,
+ )
+ residual_in = iCs[3]
+ elif iCs[0] != iCs[3]:
+ self.downsample = ConvBNReLU(
+ iCs[0],
+ iCs[3],
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=False,
+ has_bn=False,
+ has_relu=False,
+ )
+ residual_in = iCs[3]
+ else:
+ self.downsample = None
+ # self.out_dim = max(residual_in, iCs[3])
+ self.out_dim = iCs[3]
+
+ def forward(self, inputs):
+
+ bottleneck = self.conv_1x1(inputs)
+ bottleneck = self.conv_3x3(bottleneck)
+ bottleneck = self.conv_1x4(bottleneck)
+
+ if self.downsample is not None:
+ residual = self.downsample(inputs)
+ else:
+ residual = inputs
+ out = residual + bottleneck
+ return F.relu(out, inplace=True)
+
+
+class InferWidthCifarResNet(nn.Module):
+ def __init__(self, block_name, depth, xchannels, num_classes, zero_init_residual):
+ super(InferWidthCifarResNet, self).__init__()
+
+ # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
+ if block_name == "ResNetBasicblock":
+ block = ResNetBasicblock
+ assert (depth - 2) % 6 == 0, "depth should be one of 20, 32, 44, 56, 110"
+ layer_blocks = (depth - 2) // 6
+ elif block_name == "ResNetBottleneck":
+ block = ResNetBottleneck
+ assert (depth - 2) % 9 == 0, "depth should be one of 164"
+ layer_blocks = (depth - 2) // 9
+ else:
+ raise ValueError("invalid block : {:}".format(block_name))
+
+ self.message = (
+ "InferWidthCifarResNet : Depth : {:} , Layers for each block : {:}".format(
+ depth, layer_blocks
+ )
+ )
+ self.num_classes = num_classes
+ self.xchannels = xchannels
+ self.layers = nn.ModuleList(
+ [
+ ConvBNReLU(
+ xchannels[0],
+ xchannels[1],
+ 3,
+ 1,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ )
+ ]
+ )
+ last_channel_idx = 1
+ for stage in range(3):
+ for iL in range(layer_blocks):
+ num_conv = block.num_conv
+ iCs = self.xchannels[last_channel_idx : last_channel_idx + num_conv + 1]
+ stride = 2 if stage > 0 and iL == 0 else 1
+ module = block(iCs, stride)
+ last_channel_idx += num_conv
+ self.xchannels[last_channel_idx] = module.out_dim
+ self.layers.append(module)
+ self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iCs={:}, oC={:3d}, stride={:}".format(
+ stage,
+ iL,
+ layer_blocks,
+ len(self.layers) - 1,
+ iCs,
+ module.out_dim,
+ stride,
+ )
+
+ self.avgpool = nn.AvgPool2d(8)
+ self.classifier = nn.Linear(self.xchannels[-1], num_classes)
+
+ self.apply(initialize_resnet)
+ if zero_init_residual:
+ for m in self.modules():
+ if isinstance(m, ResNetBasicblock):
+ nn.init.constant_(m.conv_b.bn.weight, 0)
+ elif isinstance(m, ResNetBottleneck):
+ nn.init.constant_(m.conv_1x4.bn.weight, 0)
+
+ def get_message(self):
+ return self.message
+
+ def forward(self, inputs):
+ x = inputs
+ for i, layer in enumerate(self.layers):
+ x = layer(x)
+ features = self.avgpool(x)
+ features = features.view(features.size(0), -1)
+ logits = self.classifier(features)
+ return features, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferImagenetResNet.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferImagenetResNet.py
new file mode 100644
index 00000000..0415e585
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferImagenetResNet.py
@@ -0,0 +1,324 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+import torch.nn as nn
+import torch.nn.functional as F
+from ..initialization import initialize_resnet
+
+
+class ConvBNReLU(nn.Module):
+
+ num_conv = 1
+
+ def __init__(
+ self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu
+ ):
+ super(ConvBNReLU, self).__init__()
+ if has_avg:
+ self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0)
+ else:
+ self.avg = None
+ self.conv = nn.Conv2d(
+ nIn,
+ nOut,
+ kernel_size=kernel,
+ stride=stride,
+ padding=padding,
+ dilation=1,
+ groups=1,
+ bias=bias,
+ )
+ if has_bn:
+ self.bn = nn.BatchNorm2d(nOut)
+ else:
+ self.bn = None
+ if has_relu:
+ self.relu = nn.ReLU(inplace=True)
+ else:
+ self.relu = None
+
+ def forward(self, inputs):
+ if self.avg:
+ out = self.avg(inputs)
+ else:
+ out = inputs
+ conv = self.conv(out)
+ if self.bn:
+ out = self.bn(conv)
+ else:
+ out = conv
+ if self.relu:
+ out = self.relu(out)
+ else:
+ out = out
+
+ return out
+
+
+class ResNetBasicblock(nn.Module):
+ num_conv = 2
+ expansion = 1
+
+ def __init__(self, iCs, stride):
+ super(ResNetBasicblock, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+ assert isinstance(iCs, tuple) or isinstance(
+ iCs, list
+ ), "invalid type of iCs : {:}".format(iCs)
+ assert len(iCs) == 3, "invalid lengths of iCs : {:}".format(iCs)
+
+ self.conv_a = ConvBNReLU(
+ iCs[0],
+ iCs[1],
+ 3,
+ stride,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ )
+ self.conv_b = ConvBNReLU(
+ iCs[1], iCs[2], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False
+ )
+ residual_in = iCs[0]
+ if stride == 2:
+ self.downsample = ConvBNReLU(
+ iCs[0],
+ iCs[2],
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=True,
+ has_bn=True,
+ has_relu=False,
+ )
+ residual_in = iCs[2]
+ elif iCs[0] != iCs[2]:
+ self.downsample = ConvBNReLU(
+ iCs[0],
+ iCs[2],
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=False,
+ )
+ else:
+ self.downsample = None
+ # self.out_dim = max(residual_in, iCs[2])
+ self.out_dim = iCs[2]
+
+ def forward(self, inputs):
+ basicblock = self.conv_a(inputs)
+ basicblock = self.conv_b(basicblock)
+
+ if self.downsample is not None:
+ residual = self.downsample(inputs)
+ else:
+ residual = inputs
+ out = residual + basicblock
+ return F.relu(out, inplace=True)
+
+
+class ResNetBottleneck(nn.Module):
+ expansion = 4
+ num_conv = 3
+
+ def __init__(self, iCs, stride):
+ super(ResNetBottleneck, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+ assert isinstance(iCs, tuple) or isinstance(
+ iCs, list
+ ), "invalid type of iCs : {:}".format(iCs)
+ assert len(iCs) == 4, "invalid lengths of iCs : {:}".format(iCs)
+ self.conv_1x1 = ConvBNReLU(
+ iCs[0], iCs[1], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True
+ )
+ self.conv_3x3 = ConvBNReLU(
+ iCs[1],
+ iCs[2],
+ 3,
+ stride,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ )
+ self.conv_1x4 = ConvBNReLU(
+ iCs[2], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False
+ )
+ residual_in = iCs[0]
+ if stride == 2:
+ self.downsample = ConvBNReLU(
+ iCs[0],
+ iCs[3],
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=True,
+ has_bn=True,
+ has_relu=False,
+ )
+ residual_in = iCs[3]
+ elif iCs[0] != iCs[3]:
+ self.downsample = ConvBNReLU(
+ iCs[0],
+ iCs[3],
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=False,
+ )
+ residual_in = iCs[3]
+ else:
+ self.downsample = None
+ # self.out_dim = max(residual_in, iCs[3])
+ self.out_dim = iCs[3]
+
+ def forward(self, inputs):
+
+ bottleneck = self.conv_1x1(inputs)
+ bottleneck = self.conv_3x3(bottleneck)
+ bottleneck = self.conv_1x4(bottleneck)
+
+ if self.downsample is not None:
+ residual = self.downsample(inputs)
+ else:
+ residual = inputs
+ out = residual + bottleneck
+ return F.relu(out, inplace=True)
+
+
+class InferImagenetResNet(nn.Module):
+ def __init__(
+ self,
+ block_name,
+ layers,
+ xblocks,
+ xchannels,
+ deep_stem,
+ num_classes,
+ zero_init_residual,
+ ):
+ super(InferImagenetResNet, self).__init__()
+
+ # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
+ if block_name == "BasicBlock":
+ block = ResNetBasicblock
+ elif block_name == "Bottleneck":
+ block = ResNetBottleneck
+ else:
+ raise ValueError("invalid block : {:}".format(block_name))
+ assert len(xblocks) == len(
+ layers
+ ), "invalid layers : {:} vs xblocks : {:}".format(layers, xblocks)
+
+ self.message = "InferImagenetResNet : Depth : {:} -> {:}, Layers for each block : {:}".format(
+ sum(layers) * block.num_conv, sum(xblocks) * block.num_conv, xblocks
+ )
+ self.num_classes = num_classes
+ self.xchannels = xchannels
+ if not deep_stem:
+ self.layers = nn.ModuleList(
+ [
+ ConvBNReLU(
+ xchannels[0],
+ xchannels[1],
+ 7,
+ 2,
+ 3,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ )
+ ]
+ )
+ last_channel_idx = 1
+ else:
+ self.layers = nn.ModuleList(
+ [
+ ConvBNReLU(
+ xchannels[0],
+ xchannels[1],
+ 3,
+ 2,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ ),
+ ConvBNReLU(
+ xchannels[1],
+ xchannels[2],
+ 3,
+ 1,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ ),
+ ]
+ )
+ last_channel_idx = 2
+ self.layers.append(nn.MaxPool2d(kernel_size=3, stride=2, padding=1))
+ for stage, layer_blocks in enumerate(layers):
+ for iL in range(layer_blocks):
+ num_conv = block.num_conv
+ iCs = self.xchannels[last_channel_idx : last_channel_idx + num_conv + 1]
+ stride = 2 if stage > 0 and iL == 0 else 1
+ module = block(iCs, stride)
+ last_channel_idx += num_conv
+ self.xchannels[last_channel_idx] = module.out_dim
+ self.layers.append(module)
+ self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iCs={:}, oC={:3d}, stride={:}".format(
+ stage,
+ iL,
+ layer_blocks,
+ len(self.layers) - 1,
+ iCs,
+ module.out_dim,
+ stride,
+ )
+ if iL + 1 == xblocks[stage]: # reach the maximum depth
+ out_channel = module.out_dim
+ for iiL in range(iL + 1, layer_blocks):
+ last_channel_idx += num_conv
+ self.xchannels[last_channel_idx] = module.out_dim
+ break
+ assert last_channel_idx + 1 == len(self.xchannels), "{:} vs {:}".format(
+ last_channel_idx, len(self.xchannels)
+ )
+ self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
+ self.classifier = nn.Linear(self.xchannels[-1], num_classes)
+
+ self.apply(initialize_resnet)
+ if zero_init_residual:
+ for m in self.modules():
+ if isinstance(m, ResNetBasicblock):
+ nn.init.constant_(m.conv_b.bn.weight, 0)
+ elif isinstance(m, ResNetBottleneck):
+ nn.init.constant_(m.conv_1x4.bn.weight, 0)
+
+ def get_message(self):
+ return self.message
+
+ def forward(self, inputs):
+ x = inputs
+ for i, layer in enumerate(self.layers):
+ x = layer(x)
+ features = self.avgpool(x)
+ features = features.view(features.size(0), -1)
+ logits = self.classifier(features)
+ return features, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferMobileNetV2.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferMobileNetV2.py
new file mode 100644
index 00000000..d3db7526
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferMobileNetV2.py
@@ -0,0 +1,176 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+# MobileNetV2: Inverted Residuals and Linear Bottlenecks, CVPR 2018
+#####################################################
+from torch import nn
+
+from ..initialization import initialize_resnet
+from ..SharedUtils import parse_channel_info
+
+
+class ConvBNReLU(nn.Module):
+ def __init__(
+ self,
+ in_planes,
+ out_planes,
+ kernel_size,
+ stride,
+ groups,
+ has_bn=True,
+ has_relu=True,
+ ):
+ super(ConvBNReLU, self).__init__()
+ padding = (kernel_size - 1) // 2
+ self.conv = nn.Conv2d(
+ in_planes,
+ out_planes,
+ kernel_size,
+ stride,
+ padding,
+ groups=groups,
+ bias=False,
+ )
+ if has_bn:
+ self.bn = nn.BatchNorm2d(out_planes)
+ else:
+ self.bn = None
+ if has_relu:
+ self.relu = nn.ReLU6(inplace=True)
+ else:
+ self.relu = None
+
+ def forward(self, x):
+ out = self.conv(x)
+ if self.bn:
+ out = self.bn(out)
+ if self.relu:
+ out = self.relu(out)
+ return out
+
+
+class InvertedResidual(nn.Module):
+ def __init__(self, channels, stride, expand_ratio, additive):
+ super(InvertedResidual, self).__init__()
+ self.stride = stride
+ assert stride in [1, 2], "invalid stride : {:}".format(stride)
+ assert len(channels) in [2, 3], "invalid channels : {:}".format(channels)
+
+ if len(channels) == 2:
+ layers = []
+ else:
+ layers = [ConvBNReLU(channels[0], channels[1], 1, 1, 1)]
+ layers.extend(
+ [
+ # dw
+ ConvBNReLU(channels[-2], channels[-2], 3, stride, channels[-2]),
+ # pw-linear
+ ConvBNReLU(channels[-2], channels[-1], 1, 1, 1, True, False),
+ ]
+ )
+ self.conv = nn.Sequential(*layers)
+ self.additive = additive
+ if self.additive and channels[0] != channels[-1]:
+ self.shortcut = ConvBNReLU(channels[0], channels[-1], 1, 1, 1, True, False)
+ else:
+ self.shortcut = None
+ self.out_dim = channels[-1]
+
+ def forward(self, x):
+ out = self.conv(x)
+ # if self.additive: return additive_func(out, x)
+ if self.shortcut:
+ return out + self.shortcut(x)
+ else:
+ return out
+
+
+class InferMobileNetV2(nn.Module):
+ def __init__(self, num_classes, xchannels, xblocks, dropout):
+ super(InferMobileNetV2, self).__init__()
+ block = InvertedResidual
+ inverted_residual_setting = [
+ # t, c, n, s
+ [1, 16, 1, 1],
+ [6, 24, 2, 2],
+ [6, 32, 3, 2],
+ [6, 64, 4, 2],
+ [6, 96, 3, 1],
+ [6, 160, 3, 2],
+ [6, 320, 1, 1],
+ ]
+ assert len(inverted_residual_setting) == len(
+ xblocks
+ ), "invalid number of layers : {:} vs {:}".format(
+ len(inverted_residual_setting), len(xblocks)
+ )
+ for block_num, ir_setting in zip(xblocks, inverted_residual_setting):
+ assert block_num <= ir_setting[2], "{:} vs {:}".format(
+ block_num, ir_setting
+ )
+ xchannels = parse_channel_info(xchannels)
+ # for i, chs in enumerate(xchannels):
+ # if i > 0: assert chs[0] == xchannels[i-1][-1], 'Layer[{:}] is invalid {:} vs {:}'.format(i, xchannels[i-1], chs)
+ self.xchannels = xchannels
+ self.message = "InferMobileNetV2 : xblocks={:}".format(xblocks)
+ # building first layer
+ features = [ConvBNReLU(xchannels[0][0], xchannels[0][1], 3, 2, 1)]
+ last_channel_idx = 1
+
+ # building inverted residual blocks
+ for stage, (t, c, n, s) in enumerate(inverted_residual_setting):
+ for i in range(n):
+ stride = s if i == 0 else 1
+ additv = True if i > 0 else False
+ module = block(self.xchannels[last_channel_idx], stride, t, additv)
+ features.append(module)
+ self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, Cs={:}, stride={:}, expand={:}, original-C={:}".format(
+ stage,
+ i,
+ n,
+ len(features),
+ self.xchannels[last_channel_idx],
+ stride,
+ t,
+ c,
+ )
+ last_channel_idx += 1
+ if i + 1 == xblocks[stage]:
+ out_channel = module.out_dim
+ for iiL in range(i + 1, n):
+ last_channel_idx += 1
+ self.xchannels[last_channel_idx][0] = module.out_dim
+ break
+ # building last several layers
+ features.append(
+ ConvBNReLU(
+ self.xchannels[last_channel_idx][0],
+ self.xchannels[last_channel_idx][1],
+ 1,
+ 1,
+ 1,
+ )
+ )
+ assert last_channel_idx + 2 == len(self.xchannels), "{:} vs {:}".format(
+ last_channel_idx, len(self.xchannels)
+ )
+ # make it nn.Sequential
+ self.features = nn.Sequential(*features)
+
+ # building classifier
+ self.classifier = nn.Sequential(
+ nn.Dropout(dropout),
+ nn.Linear(self.xchannels[last_channel_idx][1], num_classes),
+ )
+
+ # weight initialization
+ self.apply(initialize_resnet)
+
+ def get_message(self):
+ return self.message
+
+ def forward(self, inputs):
+ features = self.features(inputs)
+ vectors = features.mean([2, 3])
+ predicts = self.classifier(vectors)
+ return features, predicts
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferTinyCellNet.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferTinyCellNet.py
new file mode 100644
index 00000000..d4371793
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/InferTinyCellNet.py
@@ -0,0 +1,65 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+from typing import List, Text, Any
+import torch.nn as nn
+
+from ..cell_operations import ResNetBasicblock
+from ..cell_infers.cells import InferCell
+
+
+class DynamicShapeTinyNet(nn.Module):
+ def __init__(self, channels: List[int], genotype: Any, num_classes: int):
+ super(DynamicShapeTinyNet, self).__init__()
+ self._channels = channels
+ if len(channels) % 3 != 2:
+ raise ValueError("invalid number of layers : {:}".format(len(channels)))
+ self._num_stage = N = len(channels) // 3
+
+ self.stem = nn.Sequential(
+ nn.Conv2d(3, channels[0], kernel_size=3, padding=1, bias=False),
+ nn.BatchNorm2d(channels[0]),
+ )
+
+ # layer_channels = [C ] * N + [C*2 ] + [C*2 ] * N + [C*4 ] + [C*4 ] * N
+ layer_reductions = [False] * N + [True] + [False] * N + [True] + [False] * N
+
+ c_prev = channels[0]
+ self.cells = nn.ModuleList()
+ for index, (c_curr, reduction) in enumerate(zip(channels, layer_reductions)):
+ if reduction:
+ cell = ResNetBasicblock(c_prev, c_curr, 2, True)
+ else:
+ cell = InferCell(genotype, c_prev, c_curr, 1)
+ self.cells.append(cell)
+ c_prev = cell.out_dim
+ self._num_layer = len(self.cells)
+
+ self.lastact = nn.Sequential(nn.BatchNorm2d(c_prev), nn.ReLU(inplace=True))
+ self.global_pooling = nn.AdaptiveAvgPool2d(1)
+ self.classifier = nn.Linear(c_prev, num_classes)
+
+ def get_message(self) -> Text:
+ string = self.extra_repr()
+ for i, cell in enumerate(self.cells):
+ string += "\n {:02d}/{:02d} :: {:}".format(
+ i, len(self.cells), cell.extra_repr()
+ )
+ return string
+
+ def extra_repr(self):
+ return "{name}(C={_channels}, N={_num_stage}, L={_num_layer})".format(
+ name=self.__class__.__name__, **self.__dict__
+ )
+
+ def forward(self, inputs):
+ feature = self.stem(inputs)
+ for i, cell in enumerate(self.cells):
+ feature = cell(feature)
+
+ out = self.lastact(feature)
+ out = self.global_pooling(out)
+ out = out.view(out.size(0), -1)
+ logits = self.classifier(out)
+
+ return out, logits
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/__init__.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/__init__.py
new file mode 100644
index 00000000..9c305ffd
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/__init__.py
@@ -0,0 +1,9 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
+#####################################################
+from .InferCifarResNet_width import InferWidthCifarResNet
+from .InferImagenetResNet import InferImagenetResNet
+from .InferCifarResNet_depth import InferDepthCifarResNet
+from .InferCifarResNet import InferCifarResNet
+from .InferMobileNetV2 import InferMobileNetV2
+from .InferTinyCellNet import DynamicShapeTinyNet
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/shared_utils.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/shared_utils.py
new file mode 100644
index 00000000..86ab9492
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_infers/shared_utils.py
@@ -0,0 +1,5 @@
+def parse_channel_info(xstring):
+ blocks = xstring.split(" ")
+ blocks = [x.split("-") for x in blocks]
+ blocks = [[int(_) for _ in x] for x in blocks]
+ return blocks
diff --git a/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_searchs/SearchCifarResNet.py b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_searchs/SearchCifarResNet.py
new file mode 100644
index 00000000..653051b5
--- /dev/null
+++ b/examples/model_selection/TRAILS-Database-Native-Model-Selection/internal/ml/model_selection/src/third_pkg/models/shape_searchs/SearchCifarResNet.py
@@ -0,0 +1,760 @@
+##################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
+##################################################
+import math, torch
+from collections import OrderedDict
+from bisect import bisect_right
+import torch.nn as nn
+from ..initialization import initialize_resnet
+from ..SharedUtils import additive_func
+from .SoftSelect import select2withP, ChannelWiseInter
+from .SoftSelect import linear_forward
+from .SoftSelect import get_width_choices
+
+
+def get_depth_choices(nDepth, return_num):
+ if nDepth == 2:
+ choices = (1, 2)
+ elif nDepth == 3:
+ choices = (1, 2, 3)
+ elif nDepth > 3:
+ choices = list(range(1, nDepth + 1, 2))
+ if choices[-1] < nDepth:
+ choices.append(nDepth)
+ else:
+ raise ValueError("invalid nDepth : {:}".format(nDepth))
+ if return_num:
+ return len(choices)
+ else:
+ return choices
+
+
+def conv_forward(inputs, conv, choices):
+ iC = conv.in_channels
+ fill_size = list(inputs.size())
+ fill_size[1] = iC - fill_size[1]
+ filled = torch.zeros(fill_size, device=inputs.device)
+ xinputs = torch.cat((inputs, filled), dim=1)
+ outputs = conv(xinputs)
+ selecteds = [outputs[:, :oC] for oC in choices]
+ return selecteds
+
+
+class ConvBNReLU(nn.Module):
+ num_conv = 1
+
+ def __init__(
+ self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu
+ ):
+ super(ConvBNReLU, self).__init__()
+ self.InShape = None
+ self.OutShape = None
+ self.choices = get_width_choices(nOut)
+ self.register_buffer("choices_tensor", torch.Tensor(self.choices))
+
+ if has_avg:
+ self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0)
+ else:
+ self.avg = None
+ self.conv = nn.Conv2d(
+ nIn,
+ nOut,
+ kernel_size=kernel,
+ stride=stride,
+ padding=padding,
+ dilation=1,
+ groups=1,
+ bias=bias,
+ )
+ # if has_bn : self.bn = nn.BatchNorm2d(nOut)
+ # else : self.bn = None
+ self.has_bn = has_bn
+ self.BNs = nn.ModuleList()
+ for i, _out in enumerate(self.choices):
+ self.BNs.append(nn.BatchNorm2d(_out))
+ if has_relu:
+ self.relu = nn.ReLU(inplace=True)
+ else:
+ self.relu = None
+ self.in_dim = nIn
+ self.out_dim = nOut
+ self.search_mode = "basic"
+
+ def get_flops(self, channels, check_range=True, divide=1):
+ iC, oC = channels
+ if check_range:
+ assert (
+ iC <= self.conv.in_channels and oC <= self.conv.out_channels
+ ), "{:} vs {:} | {:} vs {:}".format(
+ iC, self.conv.in_channels, oC, self.conv.out_channels
+ )
+ assert (
+ isinstance(self.InShape, tuple) and len(self.InShape) == 2
+ ), "invalid in-shape : {:}".format(self.InShape)
+ assert (
+ isinstance(self.OutShape, tuple) and len(self.OutShape) == 2
+ ), "invalid out-shape : {:}".format(self.OutShape)
+ # conv_per_position_flops = self.conv.kernel_size[0] * self.conv.kernel_size[1] * iC * oC / self.conv.groups
+ conv_per_position_flops = (
+ self.conv.kernel_size[0] * self.conv.kernel_size[1] * 1.0 / self.conv.groups
+ )
+ all_positions = self.OutShape[0] * self.OutShape[1]
+ flops = (conv_per_position_flops * all_positions / divide) * iC * oC
+ if self.conv.bias is not None:
+ flops += all_positions / divide
+ return flops
+
+ def get_range(self):
+ return [self.choices]
+
+ def forward(self, inputs):
+ if self.search_mode == "basic":
+ return self.basic_forward(inputs)
+ elif self.search_mode == "search":
+ return self.search_forward(inputs)
+ else:
+ raise ValueError("invalid search_mode = {:}".format(self.search_mode))
+
+ def search_forward(self, tuple_inputs):
+ assert (
+ isinstance(tuple_inputs, tuple) and len(tuple_inputs) == 5
+ ), "invalid type input : {:}".format(type(tuple_inputs))
+ inputs, expected_inC, probability, index, prob = tuple_inputs
+ index, prob = torch.squeeze(index).tolist(), torch.squeeze(prob)
+ probability = torch.squeeze(probability)
+ assert len(index) == 2, "invalid length : {:}".format(index)
+ # compute expected flop
+ # coordinates = torch.arange(self.x_range[0], self.x_range[1]+1).type_as(probability)
+ expected_outC = (self.choices_tensor * probability).sum()
+ expected_flop = self.get_flops([expected_inC, expected_outC], False, 1e6)
+ if self.avg:
+ out = self.avg(inputs)
+ else:
+ out = inputs
+ # convolutional layer
+ out_convs = conv_forward(out, self.conv, [self.choices[i] for i in index])
+ out_bns = [self.BNs[idx](out_conv) for idx, out_conv in zip(index, out_convs)]
+ # merge
+ out_channel = max([x.size(1) for x in out_bns])
+ outA = ChannelWiseInter(out_bns[0], out_channel)
+ outB = ChannelWiseInter(out_bns[1], out_channel)
+ out = outA * prob[0] + outB * prob[1]
+ # out = additive_func(out_bns[0]*prob[0], out_bns[1]*prob[1])
+
+ if self.relu:
+ out = self.relu(out)
+ else:
+ out = out
+ return out, expected_outC, expected_flop
+
+ def basic_forward(self, inputs):
+ if self.avg:
+ out = self.avg(inputs)
+ else:
+ out = inputs
+ conv = self.conv(out)
+ if self.has_bn:
+ out = self.BNs[-1](conv)
+ else:
+ out = conv
+ if self.relu:
+ out = self.relu(out)
+ else:
+ out = out
+ if self.InShape is None:
+ self.InShape = (inputs.size(-2), inputs.size(-1))
+ self.OutShape = (out.size(-2), out.size(-1))
+ return out
+
+
+class ResNetBasicblock(nn.Module):
+ expansion = 1
+ num_conv = 2
+
+ def __init__(self, inplanes, planes, stride):
+ super(ResNetBasicblock, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+ self.conv_a = ConvBNReLU(
+ inplanes,
+ planes,
+ 3,
+ stride,
+ 1,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=True,
+ )
+ self.conv_b = ConvBNReLU(
+ planes, planes, 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False
+ )
+ if stride == 2:
+ self.downsample = ConvBNReLU(
+ inplanes,
+ planes,
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=True,
+ has_bn=False,
+ has_relu=False,
+ )
+ elif inplanes != planes:
+ self.downsample = ConvBNReLU(
+ inplanes,
+ planes,
+ 1,
+ 1,
+ 0,
+ False,
+ has_avg=False,
+ has_bn=True,
+ has_relu=False,
+ )
+ else:
+ self.downsample = None
+ self.out_dim = planes
+ self.search_mode = "basic"
+
+ def get_range(self):
+ return self.conv_a.get_range() + self.conv_b.get_range()
+
+ def get_flops(self, channels):
+ assert len(channels) == 3, "invalid channels : {:}".format(channels)
+ flop_A = self.conv_a.get_flops([channels[0], channels[1]])
+ flop_B = self.conv_b.get_flops([channels[1], channels[2]])
+ if hasattr(self.downsample, "get_flops"):
+ flop_C = self.downsample.get_flops([channels[0], channels[-1]])
+ else:
+ flop_C = 0
+ if (
+ channels[0] != channels[-1] and self.downsample is None
+ ): # this short-cut will be added during the infer-train
+ flop_C = (
+ channels[0]
+ * channels[-1]
+ * self.conv_b.OutShape[0]
+ * self.conv_b.OutShape[1]
+ )
+ return flop_A + flop_B + flop_C
+
+ def forward(self, inputs):
+ if self.search_mode == "basic":
+ return self.basic_forward(inputs)
+ elif self.search_mode == "search":
+ return self.search_forward(inputs)
+ else:
+ raise ValueError("invalid search_mode = {:}".format(self.search_mode))
+
+ def search_forward(self, tuple_inputs):
+ assert (
+ isinstance(tuple_inputs, tuple) and len(tuple_inputs) == 5
+ ), "invalid type input : {:}".format(type(tuple_inputs))
+ inputs, expected_inC, probability, indexes, probs = tuple_inputs
+ assert indexes.size(0) == 2 and probs.size(0) == 2 and probability.size(0) == 2
+ out_a, expected_inC_a, expected_flop_a = self.conv_a(
+ (inputs, expected_inC, probability[0], indexes[0], probs[0])
+ )
+ out_b, expected_inC_b, expected_flop_b = self.conv_b(
+ (out_a, expected_inC_a, probability[1], indexes[1], probs[1])
+ )
+ if self.downsample is not None:
+ residual, _, expected_flop_c = self.downsample(
+ (inputs, expected_inC, probability[1], indexes[1], probs[1])
+ )
+ else:
+ residual, expected_flop_c = inputs, 0
+ out = additive_func(residual, out_b)
+ return (
+ nn.functional.relu(out, inplace=True),
+ expected_inC_b,
+ sum([expected_flop_a, expected_flop_b, expected_flop_c]),
+ )
+
+ def basic_forward(self, inputs):
+ basicblock = self.conv_a(inputs)
+ basicblock = self.conv_b(basicblock)
+ if self.downsample is not None:
+ residual = self.downsample(inputs)
+ else:
+ residual = inputs
+ out = additive_func(residual, basicblock)
+ return nn.functional.relu(out, inplace=True)
+
+
+class ResNetBottleneck(nn.Module):
+ expansion = 4
+ num_conv = 3
+
+ def __init__(self, inplanes, planes, stride):
+ super(ResNetBottleneck, self).__init__()
+ assert stride == 1 or stride == 2, "invalid stride {:}".format(stride)
+ self.conv_1x1 = ConvBNReLU(
+ inplanes, planes, 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True
+ )
+ self.conv_3x3 = ConvBNReLU(
+ planes,
... 4841 lines suppressed ...