You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2017/03/10 14:32:40 UTC
[5/6] incubator-metron git commit: METRON-671 Refactor existing
Ansible deployment to use Ambari MPack (dlyle via justinleet) closes
apache/incubator-metron#436
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/base-centos-6.7.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/base-centos-6.7.json b/metron-deployment/packaging/packer-build/base-centos-6.7.json
new file mode 100644
index 0000000..92eda67
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/base-centos-6.7.json
@@ -0,0 +1,123 @@
+{
+ "builders": [
+ {
+ "type": "virtualbox-iso",
+ "boot_command": [
+ "<tab> text ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/{{user `ks_path`}} ksdevice=eth0<enter><wait>"
+ ],
+ "boot_wait": "10s",
+ "disk_size": 40960,
+ "guest_additions_path": "VBoxGuestAdditions_{{.Version}}.iso",
+ "guest_os_type": "RedHat_64",
+ "hard_drive_interface": "sata",
+ "headless": "{{ user `headless` }}",
+ "http_directory": "http",
+ "iso_checksum": "{{user `iso_checksum`}}",
+ "iso_checksum_type": "{{user `iso_checksum_type`}}",
+ "iso_url": "{{user `mirror`}}/{{user `mirror_directory`}}/{{user `iso_name`}}",
+ "output_directory": "packer-{{user `template`}}-virtualbox",
+ "shutdown_command": "echo 'vagrant' | sudo -S /sbin/halt -h -p",
+ "ssh_password": "vagrant",
+ "ssh_port": 22,
+ "ssh_username": "vagrant",
+ "ssh_wait_timeout": "10000s",
+ "vboxmanage": [
+ [
+ "modifyvm",
+ "{{.Name}}",
+ "--memory",
+ "{{ user `memory` }}"
+ ],
+ [
+ "modifyvm",
+ "{{.Name}}",
+ "--nic2",
+ "hostonly"
+ ],
+ [
+ "modifyvm",
+ "{{.Name}}",
+ "--hostonlyadapter2",
+ "vboxnet0"
+ ],
+ [
+ "modifyvm",
+ "{{.Name}}",
+ "--cpus",
+ "{{ user `cpus` }}"
+ ]
+ ],
+ "virtualbox_version_file": ".vbox_version",
+ "vm_name": "{{ user `template` }}"
+ }
+ ],
+ "post-processors": [
+ {
+ "output": "builds/{{user `box_basename`}}.{{.Provider}}.box",
+ "type": "vagrant",
+ "compression_level": 9
+ }
+ ],
+ "provisioners": [
+ {
+ "destination": "/tmp/bento-metadata.json",
+ "source": "{{user `metadata`}}",
+ "type": "file"
+ },
+ {
+ "type": "shell",
+ "scripts": [
+ "scripts/common/metadata.sh",
+ "scripts/common/sshd.sh",
+ "scripts/centos/networking.sh",
+ "scripts/common/vagrant.sh",
+ "scripts/centos/update_all.sh",
+ "scripts/common/reboot.sh"
+ ],
+ "environment_vars": [
+ "HOME_DIR=/home/vagrant",
+ "http_proxy={{user `http_proxy`}}",
+ "https_proxy={{user `https_proxy`}}",
+ "no_proxy={{user `no_proxy`}}"
+ ],
+ "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'"
+ },
+ {
+ "type": "shell",
+ "pause_before" : "2m",
+ "scripts": [
+ "scripts/common/vmtools.sh",
+ "scripts/centos/cleanup.sh",
+ "scripts/common/minimize.sh"
+ ],
+ "environment_vars": [
+ "HOME_DIR=/home/vagrant",
+ "http_proxy={{user `http_proxy`}}",
+ "https_proxy={{user `https_proxy`}}",
+ "no_proxy={{user `no_proxy`}}"
+ ],
+ "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'"
+ }
+ ],
+ "variables": {
+ "box_basename": "base-centos-6.7",
+ "build_timestamp": "{{isotime \"20060102150405\"}}",
+ "cpus": "1",
+ "git_revision": "__unknown_git_revision__",
+ "headless": "",
+ "http_proxy": "{{env `http_proxy`}}",
+ "https_proxy": "{{env `https_proxy`}}",
+ "iso_checksum": "c0c1a05d3d74fb093c6232003da4b22b0680f59d3b2fa2cb7da736bc40b3f2c5",
+ "iso_checksum_type": "sha256",
+ "iso_name": "CentOS-6.7-x86_64-bin-DVD1.iso",
+ "ks_path": "centos-6.7/ks.cfg",
+ "memory": "8192",
+ "metadata": "floppy/dummy_metadata.json",
+ "mirror": "http://archive.kernel.org/centos-vault",
+ "mirror_directory": "6.7/isos/x86_64",
+ "name": "base-centos-6.7",
+ "no_proxy": "{{env `no_proxy`}}",
+ "template": "base-centos-6.7",
+ "version": "2.1.TIMESTAMP"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/bin/bento
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/bin/bento b/metron-deployment/packaging/packer-build/bin/bento
new file mode 100755
index 0000000..5594345
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/bin/bento
@@ -0,0 +1,475 @@
+#!/usr/bin/env ruby
+# -*- encoding: utf-8 -*-
+
+Signal.trap("INT") { exit 1 }
+
+$stdout.sync = true
+$stderr.sync = true
+
+require "benchmark"
+require "digest"
+require "json"
+require "optparse"
+require "ostruct"
+require "tempfile"
+
+class Options
+
+ NAME = File.basename($0).freeze
+
+ def self.parse(args)
+ options = OpenStruct.new
+ options.templates = calculate_templates("*.json")
+
+ ENV['PACKER_CACHE_DIR'] = "packer_cache"
+
+ global = OptionParser.new do |opts|
+ opts.banner = "Usage: #{NAME} [SUBCOMMAND [options]]"
+ opts.separator ""
+ opts.separator <<-COMMANDS.gsub(/^ {8}/, "")
+ build : build one or more templates
+ help : prints this help message
+ list : list all templates in project
+ normalize : normalize one or more templates
+ COMMANDS
+ end
+
+ templates_argv_proc = proc { |options|
+ options.templates = calculate_templates(args) unless args.empty?
+
+ options.templates.each do |t|
+ if !File.exists?("#{t}.json")
+ $stderr.puts "File #{t}.json does not exist for template '#{t}'"
+ exit(1)
+ end
+ end
+ }
+
+ subcommand = {
+ help: {
+ parser: OptionParser.new {},
+ argv: proc { |options|
+ puts global
+ exit(0)
+ }
+ },
+ build: {
+ class: BuildRunner,
+ parser: OptionParser.new { |opts|
+ opts.banner = "Usage: #{NAME} build [options] TEMPLATE[ TEMPLATE ...]"
+
+ opts.on("-n", "--[no-]dry-run", "Dry run (what would happen)") do |opt|
+ options.dry_run = opt
+ end
+
+ opts.on("-d", "--[no-]debug", "Run packer with debug output") do |opt|
+ options.debug = opt
+ end
+
+ opts.on("-a", "--ask", "Run packer with on-error=ask") do |opt|
+ options.ask = opt
+ end
+
+ opts.on("-o BUILDS", "--only BUILDS", "Only build some Packer builds") do |opt|
+ options.builds = opt
+ end
+
+ opts.on("-e BUILDS", "--except BUILDS", "Build all Packer builds except these") do |opt|
+ options.except = opt
+ end
+
+ opts.on("-m MIRROR", "--mirror MIRROR", "Look for isos at MIRROR") do |opt|
+ options.mirror = opt
+ end
+
+ opts.on("-v VERSION", "--version VERSION", "Override the version set in the template") do |opt|
+ options.override_version = opt
+ end
+ },
+ argv: templates_argv_proc
+ },
+ normalize: {
+ class: NormalizeRunner,
+ parser: OptionParser.new { |opts|
+ opts.banner = "Usage: #{NAME} normalize TEMPLATE[ TEMPLATE ...]"
+
+ opts.on("-d", "--[no-]debug", "Run packer with debug output") do |opt|
+ options.debug = opt
+ end
+
+ },
+ argv: templates_argv_proc
+ },
+ list: {
+ class: ListRunner,
+ parser: OptionParser.new { |opts|
+ opts.banner = "Usage: #{NAME} list [TEMPLATE ...]"
+ },
+ argv: templates_argv_proc
+ }
+ }
+
+ global.order!
+ command = args.empty? ? :help : ARGV.shift.to_sym
+ subcommand.fetch(command).fetch(:parser).order!
+ subcommand.fetch(command).fetch(:argv).call(options)
+
+ options.command = command
+ options.klass = subcommand.fetch(command).fetch(:class)
+
+ options
+ end
+
+ def self.calculate_templates(globs)
+ Array(globs).
+ map { |glob| result = Dir.glob(glob); result.empty? ? glob : result }.
+ flatten.
+ sort.
+ delete_if { |file| file =~ /\.variables\./ }.
+ map { |template| template.sub(/\.json$/, '') }
+ end
+end
+
+module Common
+
+ def banner(msg)
+ puts "==> #{msg}"
+ end
+
+ def info(msg)
+ puts " #{msg}"
+ end
+
+ def warn(msg)
+ puts ">>> #{msg}"
+ end
+
+ def duration(total)
+ total = 0 if total.nil?
+ minutes = (total / 60).to_i
+ seconds = (total - (minutes * 60))
+ format("%dm%.2fs", minutes, seconds)
+ end
+end
+
+module PackerExec
+
+ def for_packer_run_with(template)
+ Tempfile.open("#{template}-metadata.json") do |md_file|
+ Tempfile.open("#{template}-metadata-var-file") do |var_file|
+ write_box_metadata(template, md_file)
+ write_var_file(template, md_file, var_file)
+ yield md_file, var_file
+ end
+ end
+ end
+
+ def write_box_metadata(template, io)
+ md = BuildMetadata.new(template, build_timestamp, override_version).read
+
+ io.write(JSON.pretty_generate(md))
+ io.close
+ end
+
+ def write_var_file(template, md_file, io)
+ md = BuildMetadata.new(template, build_timestamp, override_version).read
+
+ io.write(JSON.pretty_generate({
+ box_basename: md[:box_basename],
+ build_timestamp: md[:build_timestamp],
+ git_revision: md[:git_revision],
+ metadata: md_file.path,
+ version: md[:version]
+ }))
+ io.close
+ end
+end
+
+class BuildRunner
+
+ include Common
+ include PackerExec
+
+ attr_reader :templates, :dry_run, :debug, :ask, :builds, :except, :mirror, :override_version, :build_timestamp
+
+ def initialize(opts)
+ @templates = opts.templates
+ @dry_run = opts.dry_run
+ @debug = opts.debug
+ @ask = opts.ask
+ @builds = opts.builds
+ @except = opts.except
+ @mirror = opts.mirror
+ @override_version = opts.override_version
+ @build_timestamp = Time.now.gmtime.strftime("%Y%m%d%H%M%S")
+ end
+
+ def start
+ banner("Starting build for templates: #{templates}")
+ time = Benchmark.measure do
+ templates.each { |template| build(template) }
+ end
+ banner("Build finished in #{duration(time.real)}.")
+ end
+
+ private
+
+ def build(template)
+ for_packer_run_with(template) do |md_file, var_file|
+ cmd = packer_build_cmd(template, var_file.path)
+ banner("[#{template}] Building: '#{cmd.join(' ')}'")
+ time = Benchmark.measure do
+ system(*cmd) or raise "[#{template}] Error building, exited #{$?}"
+ write_final_metadata(template)
+ end
+ banner("[#{template}] Finished building in #{duration(time.real)}.")
+ end
+ end
+
+ def packer_build_cmd(template, var_file)
+ vars = "#{template}.variables.json"
+ headless = !(RUBY_PLATFORM =~ /darwin/)
+ cmd = %W[packer build -var-file=#{var_file} #{template}.json]
+ cmd.insert(2, "-var-file=#{vars}") if File.exist?(vars)
+ cmd.insert(2, "-only=#{builds}") if builds
+ cmd.insert(2, "-except=#{except}") if except
+ # Build the command line in the correct order and without spaces as future input for the splat operator.
+ cmd.insert(2, "mirror=#{mirror}") if mirror
+ cmd.insert(2, "-var") if mirror
+ cmd.insert(2, "headless=true") if headless
+ cmd.insert(2, "-var") if headless
+ cmd.insert(2, "-debug") if debug
+ cmd.insert(2, "-on-error=ask") if ask
+ cmd.insert(0, "echo") if dry_run
+ cmd
+ end
+
+ def write_final_metadata(template)
+ md = BuildMetadata.new(template, build_timestamp, override_version).read
+ path = File.join(File.dirname(__FILE__), "..", "builds")
+ filename = File.join(path, "#{md[:box_basename]}.metadata.json")
+
+ md[:providers] = ProviderMetadata.new(path, md[:box_basename]).read
+
+ if dry_run
+ banner("(Dry run) Metadata file contents would be something similar to:")
+ puts JSON.pretty_generate(md)
+ else
+ File.open(filename, "wb") { |file| file.write(JSON.pretty_generate(md)) }
+ end
+ end
+end
+
+class NormalizeRunner
+
+ include Common
+ include PackerExec
+
+ attr_reader :templates, :build_timestamp, :debug, :override_version
+
+ def initialize(opts)
+ @templates = opts.templates
+ @debug = opts.debug
+ @modified = []
+ @build_timestamp = Time.now.gmtime.strftime("%Y%m%d%H%M%S")
+ end
+
+ def start
+ banner("Normalizing for templates: #{templates}")
+ time = Benchmark.measure do
+ templates.each do |template|
+ validate(template)
+ fix(template)
+ end
+ end
+ if !@modified.empty?
+ info("")
+ info("The following templates were modified:")
+ @modified.sort.each { |template| info(" * #{template}")}
+ end
+ banner("Normalizing finished in #{duration(time.real)}.")
+ end
+
+ private
+
+ def checksum(file)
+ Digest::MD5.file(file).hexdigest
+ end
+
+ def fix(template)
+ file = "#{template}.json"
+
+ banner("[#{template}] Fixing")
+ original_checksum = checksum(file)
+ output = %x{packer fix #{file}}
+ raise "[#{template}] Error fixing, exited #{$?}" if $?.exitstatus != 0
+ # preserve ampersands in shell commands,
+ # see: https://github.com/mitchellh/packer/issues/784
+ output.gsub!("\\u0026", "&")
+ File.open(file, "wb") { |dest| dest.write(output) }
+ fixed_checksum = checksum(file)
+
+ if original_checksum == fixed_checksum
+ puts("No changes made.")
+ else
+ warn("Template #{template} has been modified.")
+ @modified << template
+ end
+ end
+
+ def packer_validate_cmd(template, var_file)
+ vars = "#{template}.variables.json"
+ cmd = %W[packer validate -var-file=#{var_file} #{template}.json]
+ cmd.insert(2, "-var-file=#{vars}") if File.exist?(vars)
+ cmd
+ end
+
+ def validate(template)
+ for_packer_run_with(template) do |md_file, var_file|
+ cmd = packer_validate_cmd(template, var_file.path)
+ banner("[#{template}] Validating: '#{cmd.join(' ')}'")
+ if debug
+ banner("[#{template}] DEBUG: var_file(#{var_file.path}) is:")
+ puts IO.read(var_file.path)
+ banner("[#{template}] DEBUG: md_file(#{md_file.path}) is:")
+ puts IO.read(md_file.path)
+ end
+ system(*cmd) or raise "[#{template}] Error validating, exited #{$?}"
+ end
+ end
+end
+
+class ListRunner
+
+ include Common
+
+ attr_reader :templates
+
+ def initialize(opts)
+ @templates = opts.templates
+ end
+
+ def start
+ templates.each { |template| puts template }
+ end
+end
+
+class Runner
+
+ attr_reader :options
+
+ def initialize(options)
+ @options = options
+ end
+
+ def start
+ options.klass.new(options).start
+ end
+end
+
+class BuildMetadata
+
+ def initialize(template, build_timestamp, override_version)
+ @template = template
+ @build_timestamp = build_timestamp
+ @override_version = override_version
+ end
+
+ def read
+ {
+ name: name,
+ version: version,
+ build_timestamp: build_timestamp,
+ git_revision: git_revision,
+ box_basename: box_basename,
+ template: template_vars.fetch("template", UNKNOWN),
+ }
+ end
+
+ private
+
+ UNKNOWN = "__unknown__".freeze
+
+ attr_reader :template, :build_timestamp, :override_version
+
+ def box_basename
+ "#{name.gsub("/", "__")}-#{version}.git.#{git_revision}"
+ end
+
+ def git_revision
+ sha = %x{git rev-parse HEAD}.strip
+
+ git_clean? ? sha : "#{sha}_dirty"
+ end
+
+ def git_clean?
+ %x{git status --porcelain}.strip.empty?
+ end
+
+ def merged_vars
+ @merged_vars ||= begin
+ if File.exist?("#{template}.variables.json")
+ template_vars.merge(JSON.load(IO.read("#{template}.variables.json")))
+ else
+ template_vars
+ end
+ end
+ end
+
+ def name
+ merged_vars.fetch("name", template)
+ end
+
+ def template_vars
+ @template_vars ||= JSON.load(IO.read("#{template}.json")).fetch("variables")
+ end
+
+ def version
+ if override_version
+ override_version
+ else
+ merged_vars.fetch("version", "#{UNKNOWN}.TIMESTAMP").
+ rpartition(".").first.concat(".#{build_timestamp}")
+ end
+ end
+end
+
+class ProviderMetadata
+
+ def initialize(path, box_basename)
+ @base = File.join(path, box_basename)
+ end
+
+ def read
+ Dir.glob("#{base}.*.box").map do |file|
+ {
+ name: provider_from_file(file),
+ file: "#{File.basename(file)}",
+ checksum_type: "sha256",
+ checksum: shasum(file)
+ }
+ end
+ end
+
+ private
+
+ attr_reader :base
+
+ def provider_from_file(file)
+ case provider = file.sub(/^.*\.([^.]+)\.box$/, '\1')
+ when /vmware/i then "vmware_desktop"
+ else provider
+ end
+ end
+
+ def shasum(file)
+ Digest::SHA256.file(file).hexdigest
+ end
+end
+
+begin
+ Runner.new(Options.parse(ARGV)).start
+rescue => ex
+ $stderr.puts ">>> #{ex.message}"
+ exit(($? && $?.exitstatus) || 99)
+end
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/http/centos-6.7/ks.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/http/centos-6.7/ks.cfg b/metron-deployment/packaging/packer-build/http/centos-6.7/ks.cfg
new file mode 100644
index 0000000..d36102f
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/http/centos-6.7/ks.cfg
@@ -0,0 +1,86 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+install
+cdrom
+lang en_US.UTF-8
+keyboard us
+network --device eth0 --bootproto=dhcp --noipv6
+network --device eth1 --bootproto=static --ip=192.168.66.121 --netmask=255.255.255.0 --noipv6
+rootpw vagrant
+firewall --disabled
+selinux --permissive
+timezone UTC
+unsupported_hardware
+bootloader --location=mbr
+text
+skipx
+zerombr
+clearpart --all --initlabel
+autopart
+auth --enableshadow --passalgo=sha512 --kickstart
+firstboot --disabled
+reboot
+user --name=vagrant --plaintext --password vagrant
+key --skip
+
+%pre
+hostname node1
+%end
+
+%packages --nobase --ignoremissing --excludedocs
+# vagrant needs this to copy initial files via scp
+openssh-clients
+sudo
+wget
+nfs-utils
+-fprintd-pam
+-intltool
+
+# unnecessary firmware
+-aic94xx-firmware
+-atmel-firmware
+-b43-openfwwf
+-bfa-firmware
+-ipw2100-firmware
+-ipw2200-firmware
+-ivtv-firmware
+-iwl100-firmware
+-iwl1000-firmware
+-iwl3945-firmware
+-iwl4965-firmware
+-iwl5000-firmware
+-iwl5150-firmware
+-iwl6000-firmware
+-iwl6000g2a-firmware
+-iwl6050-firmware
+-libertas-usb8388-firmware
+-ql2100-firmware
+-ql2200-firmware
+-ql23xx-firmware
+-ql2400-firmware
+-ql2500-firmware
+-rt61pci-firmware
+-rt73usb-firmware
+-xorg-x11-drv-ati-firmware
+-zd1211-firmware
+
+%post
+# Force to set SELinux to a permissive mode
+sed -i -e 's/\(^SELINUX=\).*$/\1permissive/' /etc/selinux/config
+# sudo
+echo "%vagrant ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/vagrant
+sed -i "s/^[^#].*requiretty/#Defaults requiretty/" /etc/sudoers
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/quick-dev-centos-6.7.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/quick-dev-centos-6.7.json b/metron-deployment/packaging/packer-build/quick-dev-centos-6.7.json
new file mode 100644
index 0000000..96680cb
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/quick-dev-centos-6.7.json
@@ -0,0 +1,132 @@
+{
+ "builders": [
+ {
+ "type": "virtualbox-iso",
+ "boot_command": [
+ "<tab> text ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/{{user `ks_path`}} ksdevice=eth0<enter><wait>"
+ ],
+ "boot_wait": "10s",
+ "disk_size": 40960,
+ "guest_additions_path": "VBoxGuestAdditions_{{.Version}}.iso",
+ "guest_os_type": "RedHat_64",
+ "hard_drive_interface": "sata",
+ "headless": "{{ user `headless` }}",
+ "http_directory": "http",
+ "iso_checksum": "{{user `iso_checksum`}}",
+ "iso_checksum_type": "{{user `iso_checksum_type`}}",
+ "iso_url": "{{user `mirror`}}/{{user `mirror_directory`}}/{{user `iso_name`}}",
+ "output_directory": "packer-{{user `template`}}-virtualbox",
+ "shutdown_command": "echo 'vagrant' | sudo -S /sbin/halt -h -p",
+ "ssh_password": "vagrant",
+ "ssh_port": 22,
+ "ssh_username": "vagrant",
+ "ssh_wait_timeout": "10000s",
+ "vboxmanage": [
+ [
+ "modifyvm",
+ "{{.Name}}",
+ "--memory",
+ "{{ user `memory` }}"
+ ],
+ [
+ "modifyvm",
+ "{{.Name}}",
+ "--nic2",
+ "hostonly"
+ ],
+ [
+ "modifyvm",
+ "{{.Name}}",
+ "--hostonlyadapter2",
+ "vboxnet0"
+ ],
+ [
+ "modifyvm",
+ "{{.Name}}",
+ "--cpus",
+ "{{ user `cpus` }}"
+ ]
+ ],
+ "virtualbox_version_file": ".vbox_version",
+ "vm_name": "{{ user `template` }}"
+ }
+ ],
+ "post-processors": [
+ {
+ "output": "builds/{{user `box_basename`}}.{{.Provider}}.box",
+ "type": "vagrant",
+ "compression_level": 9
+ }
+ ],
+ "provisioners": [
+ {
+ "destination": "/tmp/bento-metadata.json",
+ "source": "{{user `metadata`}}",
+ "type": "file"
+ },
+ {
+ "type": "shell",
+ "scripts": [
+ "scripts/common/metadata.sh",
+ "scripts/common/sshd.sh",
+ "scripts/centos/networking.sh",
+ "scripts/common/vagrant.sh",
+ "scripts/centos/update_all.sh",
+ "scripts/common/reboot.sh"
+ ],
+ "environment_vars": [
+ "HOME_DIR=/home/vagrant",
+ "http_proxy={{user `http_proxy`}}",
+ "https_proxy={{user `https_proxy`}}",
+ "no_proxy={{user `no_proxy`}}"
+ ],
+ "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'"
+ },
+ {
+ "type" : "ansible",
+ "pause_before" : "2m",
+ "playbook_file" : "./ansible/playbook.yml",
+ "sftp_command": "/usr/libexec/openssh/sftp-server",
+ "user" : "vagrant",
+ "host_alias" : "node1",
+ "groups" : ["packer","ambari_master","ambari_slave","metron","search","web","sensors","pcap_server","zeppelin","monit"],
+ "extra_arguments": [ "--extra-vars","metron_rpm_glob=../../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm metron_build_dir=../../..","--skip-tags","report,sensors,quick_dev" ]
+ },
+ {
+ "type": "shell",
+ "scripts": [
+ "scripts/common/vmtools.sh",
+ "scripts/centos/cleanup.sh",
+ "scripts/common/minimize.sh"
+ ],
+ "environment_vars": [
+ "HOME_DIR=/home/vagrant",
+ "http_proxy={{user `http_proxy`}}",
+ "https_proxy={{user `https_proxy`}}",
+ "no_proxy={{user `no_proxy`}}"
+ ],
+ "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'"
+ }
+ ],
+ "variables": {
+ "box_basename": "quick-dev-centos-6.7",
+ "build_timestamp": "{{isotime \"20060102150405\"}}",
+ "cpus": "1",
+ "git_revision": "__unknown_git_revision__",
+ "headless": "",
+ "http_proxy": "{{env `http_proxy`}}",
+ "https_proxy": "{{env `https_proxy`}}",
+ "iso_checksum": "c0c1a05d3d74fb093c6232003da4b22b0680f59d3b2fa2cb7da736bc40b3f2c5",
+ "iso_checksum_type": "sha256",
+ "iso_name": "CentOS-6.7-x86_64-bin-DVD1.iso",
+ "ks_path": "centos-6.7/ks.cfg",
+ "memory": "8192",
+ "metadata": "floppy/dummy_metadata.json",
+ "mirror": "http://archive.kernel.org/centos-vault",
+ "mirror_directory": "6.7/isos/x86_64",
+ "name": "quick-dev-centos-6.7",
+ "no_proxy": "{{env `no_proxy`}}",
+ "template": "quick-dev-centos-6.7",
+ "version": "2.1.TIMESTAMP"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/scripts/centos/cleanup.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/scripts/centos/cleanup.sh b/metron-deployment/packaging/packer-build/scripts/centos/cleanup.sh
new file mode 100644
index 0000000..035ab7e
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/scripts/centos/cleanup.sh
@@ -0,0 +1,25 @@
+#!/bin/sh -eux
+
+# should output one of 'redhat' 'centos' 'oraclelinux'
+distro="`rpm -qf --queryformat '%{NAME}' /etc/redhat-release | cut -f 1 -d '-'`"
+
+if [ "$distro" != 'redhat' ]; then
+ yum -y clean all;
+fi
+
+# Clean up network interface persistence
+rm -f /etc/udev/rules.d/70-persistent-net.rules;
+mkdir -p /etc/udev/rules.d/70-persistent-net.rules;
+rm -f /lib/udev/rules.d/75-persistent-net-generator.rules;
+rm -rf /dev/.udev/;
+
+for ndev in `ls -1 /etc/sysconfig/network-scripts/ifcfg-*`; do
+ if [ "`basename $ndev`" != "ifcfg-lo" ]; then
+ sed -i '/^HWADDR/d' "$ndev";
+ sed -i '/^UUID/d' "$ndev";
+ fi
+done
+
+rm -f VBoxGuestAdditions_*.iso VBoxGuestAdditions_*.iso.?;
+echo "127.0.0.1 localhost" > /etc/hosts
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/scripts/centos/networking.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/scripts/centos/networking.sh b/metron-deployment/packaging/packer-build/scripts/centos/networking.sh
new file mode 100644
index 0000000..0bf05cb
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/scripts/centos/networking.sh
@@ -0,0 +1,20 @@
+#!/bin/sh -eux
+
+case "$PACKER_BUILDER_TYPE" in
+
+virtualbox-iso|virtualbox-ovf)
+ major_version="`sed 's/^.\+ release \([.0-9]\+\).*/\1/' /etc/redhat-release | awk -F. '{print $1}'`";
+
+ if [ "$major_version" -ge 6 ]; then
+ # Fix slow DNS:
+ # Add 'single-request-reopen' so it is included when /etc/resolv.conf is
+ # generated
+ # https://access.redhat.com/site/solutions/58625 (subscription required)
+ echo 'RES_OPTIONS="single-request-reopen"' >>/etc/sysconfig/network;
+ echo "192.168.66.121 node1" >>/etc/hosts;
+ service network restart;
+ echo 'Slow DNS fix applied (single-request-reopen)';
+ fi
+ ;;
+
+esac
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/scripts/centos/update_all.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/scripts/centos/update_all.sh b/metron-deployment/packaging/packer-build/scripts/centos/update_all.sh
new file mode 100644
index 0000000..a864ce2
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/scripts/centos/update_all.sh
@@ -0,0 +1,20 @@
+#!/bin/sh -eux
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+yum update -y
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/scripts/common/metadata.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/scripts/common/metadata.sh b/metron-deployment/packaging/packer-build/scripts/common/metadata.sh
new file mode 100755
index 0000000..42386ac
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/scripts/common/metadata.sh
@@ -0,0 +1,6 @@
+#!/bin/sh -eux
+
+mkdir -p /etc;
+cp /tmp/bento-metadata.json /etc/bento-metadata.json;
+chmod 0444 /etc/bento-metadata.json;
+rm -f /tmp/bento-metadata.json;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/scripts/common/minimize.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/scripts/common/minimize.sh b/metron-deployment/packaging/packer-build/scripts/common/minimize.sh
new file mode 100644
index 0000000..1ca88be
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/scripts/common/minimize.sh
@@ -0,0 +1,28 @@
+#!/bin/sh -eux
+
+case "$PACKER_BUILDER_TYPE" in
+ qemu) exit 0 ;;
+esac
+
+set +e
+swapuuid="`/sbin/blkid -o value -l -s UUID -t TYPE=swap`";
+case "$?" in
+ 2|0) ;;
+ *) exit 1 ;;
+esac
+set -e
+
+if [ "x${swapuuid}" != "x" ]; then
+ # Whiteout the swap partition to reduce box size
+ # Swap is disabled till reboot
+ swappart="`readlink -f /dev/disk/by-uuid/$swapuuid`";
+ /sbin/swapoff "$swappart";
+ dd if=/dev/zero of="$swappart" bs=1M || echo "dd exit code $? is suppressed";
+ /sbin/mkswap -U "$swapuuid" "$swappart";
+fi
+
+dd if=/dev/zero of=/EMPTY bs=1M || echo "dd exit code $? is suppressed";
+rm -f /EMPTY;
+# Block until the empty file has been removed, otherwise, Packer
+# will try to kill the box while the disk is still full and that's bad
+sync;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/scripts/common/reboot.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/scripts/common/reboot.sh b/metron-deployment/packaging/packer-build/scripts/common/reboot.sh
new file mode 100644
index 0000000..1365474
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/scripts/common/reboot.sh
@@ -0,0 +1,20 @@
+#!/bin/sh -eux
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+reboot
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/scripts/common/sshd.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/scripts/common/sshd.sh b/metron-deployment/packaging/packer-build/scripts/common/sshd.sh
new file mode 100644
index 0000000..1bb16f5
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/scripts/common/sshd.sh
@@ -0,0 +1,20 @@
+#!/bin/sh -eux
+
+SSHD_CONFIG="/etc/ssh/sshd_config"
+
+# ensure that there is a trailing newline before attempting to concatenate
+sed -i -e '$a\' "$SSHD_CONFIG"
+
+USEDNS="UseDNS no"
+if grep -q -E "^[[:space:]]*UseDNS" "$SSHD_CONFIG"; then
+ sed -i "s/^\s*UseDNS.*/${USEDNS}/" "$SSHD_CONFIG"
+else
+ echo "$USEDNS" >>"$SSHD_CONFIG"
+fi
+
+GSSAPI="GSSAPIAuthentication no"
+if grep -q -E "^[[:space:]]*GSSAPIAuthentication" "$SSHD_CONFIG"; then
+ sed -i "s/^\s*GSSAPIAuthentication.*/${GSSAPI}/" "$SSHD_CONFIG"
+else
+ echo "$GSSAPI" >>"$SSHD_CONFIG"
+fi
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/scripts/common/sudoers.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/scripts/common/sudoers.sh b/metron-deployment/packaging/packer-build/scripts/common/sudoers.sh
new file mode 100644
index 0000000..5fce3dd
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/scripts/common/sudoers.sh
@@ -0,0 +1,4 @@
+#!/bin/bash -eux
+
+sed -i -e '/Defaults\s\+env_reset/a Defaults\texempt_group=sudo' /etc/sudoers
+sed -i -e 's/%sudo ALL=(ALL:ALL) ALL/%sudo ALL=NOPASSWD:ALL/g' /etc/sudoers
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/scripts/common/vagrant.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/scripts/common/vagrant.sh b/metron-deployment/packaging/packer-build/scripts/common/vagrant.sh
new file mode 100644
index 0000000..b69c3a5
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/scripts/common/vagrant.sh
@@ -0,0 +1,19 @@
+#!/bin/sh -eux
+
+# set a default HOME_DIR environment variable if not set
+HOME_DIR="${HOME_DIR:-/home/vagrant}";
+
+pubkey_url="https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant.pub";
+mkdir -p $HOME_DIR/.ssh;
+if command -v wget >/dev/null 2>&1; then
+ wget --no-check-certificate "$pubkey_url" -O $HOME_DIR/.ssh/authorized_keys;
+elif command -v curl >/dev/null 2>&1; then
+ curl --insecure --location "$pubkey_url" > $HOME_DIR/.ssh/authorized_keys;
+elif command -v fetch >/dev/null 2>&1; then
+ fetch -am -o $HOME_DIR/.ssh/authorized_keys "$pubkey_url";
+else
+ echo "Cannot download vagrant public key";
+ exit 1;
+fi
+chown -R vagrant $HOME_DIR/.ssh;
+chmod -R go-rwsx $HOME_DIR/.ssh;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/scripts/common/vmtools.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/scripts/common/vmtools.sh b/metron-deployment/packaging/packer-build/scripts/common/vmtools.sh
new file mode 100644
index 0000000..3656cf7
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/scripts/common/vmtools.sh
@@ -0,0 +1,61 @@
+#!/bin/sh -eux
+
+# set a default HOME_DIR environment variable if not set
+HOME_DIR="${HOME_DIR:-/home/vagrant}";
+
+case "$PACKER_BUILDER_TYPE" in
+
+virtualbox-iso|virtualbox-ovf)
+ yum install -y kernel-devel-`uname -r`
+ yum install -y gcc
+ yum install -y binutils
+ yum install -y make
+ yum install -y perl
+ yum install -y bzip2
+ mkdir -p /tmp/vbox;
+ ver="`cat /home/vagrant/.vbox_version`";
+ mount -o loop $HOME_DIR/VBoxGuestAdditions_${ver}.iso /tmp/vbox;
+ sh /tmp/vbox/VBoxLinuxAdditions.run \
+ || echo "VBoxLinuxAdditions.run exited $? and is suppressed." \
+ "For more read https://www.virtualbox.org/ticket/12479";
+ umount /tmp/vbox;
+ rm -rf /tmp/vbox;
+ #rm -f $HOME_DIR/*.iso;
+ ;;
+
+vmware-iso|vmware-vmx)
+ mkdir -p /tmp/vmfusion;
+ mkdir -p /tmp/vmfusion-archive;
+ mount -o loop $HOME_DIR/linux.iso /tmp/vmfusion;
+ tar xzf /tmp/vmfusion/VMwareTools-*.tar.gz -C /tmp/vmfusion-archive;
+ /tmp/vmfusion-archive/vmware-tools-distrib/vmware-install.pl --force-install;
+ umount /tmp/vmfusion;
+ rm -rf /tmp/vmfusion;
+ rm -rf /tmp/vmfusion-archive;
+ rm -f $HOME_DIR/*.iso;
+ ;;
+
+parallels-iso|parallels-pvm)
+ mkdir -p /tmp/parallels;
+ mount -o loop $HOME_DIR/prl-tools-lin.iso /tmp/parallels;
+ /tmp/parallels/install --install-unattended-with-deps \
+ || (code="$?"; \
+ echo "Parallels tools installation exited $code, attempting" \
+ "to output /var/log/parallels-tools-install.log"; \
+ cat /var/log/parallels-tools-install.log; \
+ exit $code);
+ umount /tmp/parallels;
+ rm -rf /tmp/parallels;
+ rm -f $HOME_DIR/*.iso;
+ ;;
+
+qemu)
+ echo "Don't need anything for this one"
+ ;;
+
+*)
+ echo "Unknown Packer Builder Type >>$PACKER_BUILDER_TYPE<< selected.";
+ echo "Known are virtualbox-iso|virtualbox-ovf|vmware-iso|vmware-vmx|parallels-iso|parallels-pvm.";
+ ;;
+
+esac
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/.gitignore b/metron-deployment/packer-build/.gitignore
deleted file mode 100644
index 68bf87b..0000000
--- a/metron-deployment/packer-build/.gitignore
+++ /dev/null
@@ -1,12 +0,0 @@
-.bundle/
-iso
-*.box
-virtualfloppy.vfd
-packer_cache
-packer.log
-.DS_Store
-/packer-*/
-*.variables.json
-/builds/
-.kitchen
-.kitchen.*.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/README.md b/metron-deployment/packer-build/README.md
deleted file mode 100644
index 1100d5f..0000000
--- a/metron-deployment/packer-build/README.md
+++ /dev/null
@@ -1,52 +0,0 @@
-Build Metron Images
-=========================
-
-Based on the fantastic [Bento](https://github.com/chef/bento) project developed by Chef.
-
-Images Provided
----------------------
-- hdp-centos-6.7: Centos 6.7 + HDP. Used in the quick-dev-platform Vagrant image
-- metron-centos-6.7: Centos 6.7 + HDP + Metron. Used for the codelab-platform Vagrant image.
-
-Prerequisites
----------------------
-- [Packer](https://www.packer.io/) 0.10.1
-- [Virtualbox](https://www.virtualbox.org/) 5.0.16
-- Be sure to build Metron prior to building the images (cd *your-project-directory*/metron-platform && mvn clean package -DskipTests)
-
-Build Both Images
-----------------------
- Navigate to \<your-project-directory\>/metron-deployment/packer-build
- Execute bin/bento build
-
- Packer will build both images and export .box files to the ./builds directory.
-
-Build Single Images
-----------------------
- Navigate to *your-project-directory*/metron-deployment/packer-build
- * HDP Centos
- ```
-bin/bento build hdp-centos-6.7.json
-```
- * Full Metron
- ```
-bin/bento build metron-centos-6.7.json
-```
-
-Using Your New Box File
-----------------------
-Modify the relevant Vagrantfile (codelab-platform or quick-dev-platform) replacing the lines:
-```
-<pre><code>config.vm.box = "<i>box_name</i>"
-config.ssh.insert_key = true</code></pre>
-```
-with
-```
-<pre></code>config.vm.box = "<i>test_box_name</i>"
-config.vm.box = "<i>PathToBoxfile/Boxfilename</i>"
-config.ssh.insert_key = true</code></pre>
-```
-Launch the image as usual.
-
-Node: Vagrant will cache boxes, you can force Vagrant to reload your box by running <code>vagrant box remove <i>test_box_name</i></code> before launching your new image.
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/ansible.cfg b/metron-deployment/packer-build/ansible.cfg
deleted file mode 100644
index e329517..0000000
--- a/metron-deployment/packer-build/ansible.cfg
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-[defaults]
-host_key_checking = false
-library = ../extra_modules
-roles_path = ../roles
-pipelining = True
-remote_user = vagrant
-log_path = ./ansible.log
-
-# fix for "ssh throws 'unix domain socket too long' " problem
-[ssh_connection]
-control_path = ~/.ssh/ansible-ssh-%%C
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/ansible/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/ansible/playbook.yml b/metron-deployment/packer-build/ansible/playbook.yml
deleted file mode 100644
index 8f922ba..0000000
--- a/metron-deployment/packer-build/ansible/playbook.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: ../../playbooks/metron_full_install.yml
-
-- hosts: packer
- become: true
- roles:
- - role: ambari_gather_facts
- tasks:
- - include: tasks/cleanup.yml
- tags:
- - packer-cleanup
-
-- hosts: packer
- become: true
- roles:
- - role: ambari_gather_facts
- - role: metron_hbase_tables
- - role: metron_kafka_topics
- tags:
- - packer-cleanup
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/ansible/tasks/cleanup.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/ansible/tasks/cleanup.yml b/metron-deployment/packer-build/ansible/tasks/cleanup.yml
deleted file mode 100644
index 4c321c9..0000000
--- a/metron-deployment/packer-build/ansible/tasks/cleanup.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: ../../roles/ambari_config/tasks/start_hdp.yml
-
-- name: Stop Monit
- service:
- name: monit
- state: stopped
- register: command_result
- failed_when: "command_result|failed and 'no service or tool found for: monit' not in command_result.msg"
-
-- include: stop_storm.yml
-
-- include: clear_topics.yml
-
-- name: Delete ES Indices
- uri:
- method: DELETE
- url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/b*,s*,y*"
-
-- name: Stop the ambari cluster
- ambari_cluster_state:
- host: "{{ groups.ambari_master[0] }}"
- port: "{{ ambari_port }}"
- username: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- cluster_name: "{{ cluster_name }}"
- cluster_state: stopped
- wait_for_complete: true
-
-- include: remove_services.yml
-
-- include: ../../roles/ambari_config/tasks/start_hdp.yml
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/ansible/tasks/clear_topics.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/ansible/tasks/clear_topics.yml b/metron-deployment/packer-build/ansible/tasks/clear_topics.yml
deleted file mode 100644
index cc39419..0000000
--- a/metron-deployment/packer-build/ansible/tasks/clear_topics.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: "Delete topics"
- shell: "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper {{ zookeeper_url }} --delete --topic {{ item }}"
- register: result
- with_items:
- - bro
- - yaf
- - snort
- - enrichments
- - pcap
- failed_when: result.rc != 0 and ("does not exist" not in result.stderr)
- changed_when: result.rc == 0
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/ansible/tasks/remove_services.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/ansible/tasks/remove_services.yml b/metron-deployment/packer-build/ansible/tasks/remove_services.yml
deleted file mode 100644
index 9d2477c..0000000
--- a/metron-deployment/packer-build/ansible/tasks/remove_services.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Remove Secondary Namenode
- uri:
- url: "http://{{ambari_host}}:{{ambari_port}}/api/v1/clusters/metron_cluster/hosts/node1/host_components/SECONDARY_NAMENODE"
- method: DELETE
- HEADER_X-Requested-By: "{{ ambari_user }}"
- user: "{{ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- status_code: 200,201,404
-
-- name: Remove MapReduce2 Service
- uri:
- url: "http://{{ambari_host}}:{{ambari_port}}/api/v1/clusters/metron_cluster/services/MAPREDUCE2"
- method: DELETE
- HEADER_X-Requested-By: "{{ ambari_user }}"
- user: "{{ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- status_code: 200,201,404
-
-- name: Remove YARN Service
- uri:
- url: "http://{{ambari_host}}:{{ambari_port}}/api/v1/clusters/metron_cluster/services/YARN"
- method: DELETE
- HEADER_X-Requested-By: "{{ ambari_user }}"
- user: "{{ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- status_code: 200,201,404
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/ansible/tasks/stop_storm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/ansible/tasks/stop_storm.yml b/metron-deployment/packer-build/ansible/tasks/stop_storm.yml
deleted file mode 100644
index 1c1b647..0000000
--- a/metron-deployment/packer-build/ansible/tasks/stop_storm.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: "List Storm Topologies"
- shell: storm list | grep ACTIVE | awk '{print $1}'
- register: topologies
-
-- name: "Kill Storm Topologies"
- shell: "storm kill {{ item }}"
- register: result
- with_items:
- - "{{ topologies.stdout_lines }}"
- failed_when: result.rc != 0 and ("NotAliveException" not in result.stdout)
- changed_when: result.rc == 0
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/bin/bento
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/bin/bento b/metron-deployment/packer-build/bin/bento
deleted file mode 100755
index 982fc8d..0000000
--- a/metron-deployment/packer-build/bin/bento
+++ /dev/null
@@ -1,469 +0,0 @@
-#!/usr/bin/env ruby
-# -*- encoding: utf-8 -*-
-
-Signal.trap("INT") { exit 1 }
-
-$stdout.sync = true
-$stderr.sync = true
-
-require "benchmark"
-require "digest"
-require "json"
-require "optparse"
-require "ostruct"
-require "tempfile"
-
-class Options
-
- NAME = File.basename($0).freeze
-
- def self.parse(args)
- options = OpenStruct.new
- options.templates = calculate_templates("*.json")
-
- ENV['PACKER_CACHE_DIR'] = "packer_cache"
-
- global = OptionParser.new do |opts|
- opts.banner = "Usage: #{NAME} [SUBCOMMAND [options]]"
- opts.separator ""
- opts.separator <<-COMMANDS.gsub(/^ {8}/, "")
- build : build one or more templates
- help : prints this help message
- list : list all templates in project
- normalize : normalize one or more templates
- COMMANDS
- end
-
- templates_argv_proc = proc { |options|
- options.templates = calculate_templates(args) unless args.empty?
-
- options.templates.each do |t|
- if !File.exists?("#{t}.json")
- $stderr.puts "File #{t}.json does not exist for template '#{t}'"
- exit(1)
- end
- end
- }
-
- subcommand = {
- help: {
- parser: OptionParser.new {},
- argv: proc { |options|
- puts global
- exit(0)
- }
- },
- build: {
- class: BuildRunner,
- parser: OptionParser.new { |opts|
- opts.banner = "Usage: #{NAME} build [options] TEMPLATE[ TEMPLATE ...]"
-
- opts.on("-n", "--[no-]dry-run", "Dry run (what would happen)") do |opt|
- options.dry_run = opt
- end
-
- opts.on("-d", "--[no-]debug", "Run packer with debug output") do |opt|
- options.debug = opt
- end
-
- opts.on("-o BUILDS", "--only BUILDS", "Only build some Packer builds") do |opt|
- options.builds = opt
- end
-
- opts.on("-e BUILDS", "--except BUILDS", "Build all Packer builds except these") do |opt|
- options.except = opt
- end
-
- opts.on("-m MIRROR", "--mirror MIRROR", "Look for isos at MIRROR") do |opt|
- options.mirror = opt
- end
-
- opts.on("-v VERSION", "--version VERSION", "Override the version set in the template") do |opt|
- options.override_version = opt
- end
- },
- argv: templates_argv_proc
- },
- normalize: {
- class: NormalizeRunner,
- parser: OptionParser.new { |opts|
- opts.banner = "Usage: #{NAME} normalize TEMPLATE[ TEMPLATE ...]"
-
- opts.on("-d", "--[no-]debug", "Run packer with debug output") do |opt|
- options.debug = opt
- end
-
- },
- argv: templates_argv_proc
- },
- list: {
- class: ListRunner,
- parser: OptionParser.new { |opts|
- opts.banner = "Usage: #{NAME} list [TEMPLATE ...]"
- },
- argv: templates_argv_proc
- }
- }
-
- global.order!
- command = args.empty? ? :help : ARGV.shift.to_sym
- subcommand.fetch(command).fetch(:parser).order!
- subcommand.fetch(command).fetch(:argv).call(options)
-
- options.command = command
- options.klass = subcommand.fetch(command).fetch(:class)
-
- options
- end
-
- def self.calculate_templates(globs)
- Array(globs).
- map { |glob| result = Dir.glob(glob); result.empty? ? glob : result }.
- flatten.
- sort.
- delete_if { |file| file =~ /\.variables\./ }.
- map { |template| template.sub(/\.json$/, '') }
- end
-end
-
-module Common
-
- def banner(msg)
- puts "==> #{msg}"
- end
-
- def info(msg)
- puts " #{msg}"
- end
-
- def warn(msg)
- puts ">>> #{msg}"
- end
-
- def duration(total)
- total = 0 if total.nil?
- minutes = (total / 60).to_i
- seconds = (total - (minutes * 60))
- format("%dm%.2fs", minutes, seconds)
- end
-end
-
-module PackerExec
-
- def for_packer_run_with(template)
- Tempfile.open("#{template}-metadata.json") do |md_file|
- Tempfile.open("#{template}-metadata-var-file") do |var_file|
- write_box_metadata(template, md_file)
- write_var_file(template, md_file, var_file)
- yield md_file, var_file
- end
- end
- end
-
- def write_box_metadata(template, io)
- md = BuildMetadata.new(template, build_timestamp, override_version).read
-
- io.write(JSON.pretty_generate(md))
- io.close
- end
-
- def write_var_file(template, md_file, io)
- md = BuildMetadata.new(template, build_timestamp, override_version).read
-
- io.write(JSON.pretty_generate({
- box_basename: md[:box_basename],
- build_timestamp: md[:build_timestamp],
- git_revision: md[:git_revision],
- metadata: md_file.path,
- version: md[:version]
- }))
- io.close
- end
-end
-
-class BuildRunner
-
- include Common
- include PackerExec
-
- attr_reader :templates, :dry_run, :debug, :builds, :except, :mirror, :override_version, :build_timestamp
-
- def initialize(opts)
- @templates = opts.templates
- @dry_run = opts.dry_run
- @debug = opts.debug
- @builds = opts.builds
- @except = opts.except
- @mirror = opts.mirror
- @override_version = opts.override_version
- @build_timestamp = Time.now.gmtime.strftime("%Y%m%d%H%M%S")
- end
-
- def start
- banner("Starting build for templates: #{templates}")
- time = Benchmark.measure do
- templates.each { |template| build(template) }
- end
- banner("Build finished in #{duration(time.real)}.")
- end
-
- private
-
- def build(template)
- for_packer_run_with(template) do |md_file, var_file|
- cmd = packer_build_cmd(template, var_file.path)
- banner("[#{template}] Building: '#{cmd.join(' ')}'")
- time = Benchmark.measure do
- system(*cmd) or raise "[#{template}] Error building, exited #{$?}"
- write_final_metadata(template)
- end
- banner("[#{template}] Finished building in #{duration(time.real)}.")
- end
- end
-
- def packer_build_cmd(template, var_file)
- vars = "#{template}.variables.json"
- headless = !(RUBY_PLATFORM =~ /darwin/)
- cmd = %W[packer build -var-file=#{var_file} #{template}.json]
- cmd.insert(2, "-var-file=#{vars}") if File.exist?(vars)
- cmd.insert(2, "-only=#{builds}") if builds
- cmd.insert(2, "-except=#{except}") if except
- # Build the command line in the correct order and without spaces as future input for the splat operator.
- cmd.insert(2, "mirror=#{mirror}") if mirror
- cmd.insert(2, "-var") if mirror
- cmd.insert(2, "headless=true") if headless
- cmd.insert(2, "-var") if headless
- cmd.insert(2, "-debug") if debug
- cmd.insert(0, "echo") if dry_run
- cmd
- end
-
- def write_final_metadata(template)
- md = BuildMetadata.new(template, build_timestamp, override_version).read
- path = File.join(File.dirname(__FILE__), "..", "builds")
- filename = File.join(path, "#{md[:box_basename]}.metadata.json")
-
- md[:providers] = ProviderMetadata.new(path, md[:box_basename]).read
-
- if dry_run
- banner("(Dry run) Metadata file contents would be something similar to:")
- puts JSON.pretty_generate(md)
- else
- File.open(filename, "wb") { |file| file.write(JSON.pretty_generate(md)) }
- end
- end
-end
-
-class NormalizeRunner
-
- include Common
- include PackerExec
-
- attr_reader :templates, :build_timestamp, :debug, :override_version
-
- def initialize(opts)
- @templates = opts.templates
- @debug = opts.debug
- @modified = []
- @build_timestamp = Time.now.gmtime.strftime("%Y%m%d%H%M%S")
- end
-
- def start
- banner("Normalizing for templates: #{templates}")
- time = Benchmark.measure do
- templates.each do |template|
- validate(template)
- fix(template)
- end
- end
- if !@modified.empty?
- info("")
- info("The following templates were modified:")
- @modified.sort.each { |template| info(" * #{template}")}
- end
- banner("Normalizing finished in #{duration(time.real)}.")
- end
-
- private
-
- def checksum(file)
- Digest::MD5.file(file).hexdigest
- end
-
- def fix(template)
- file = "#{template}.json"
-
- banner("[#{template}] Fixing")
- original_checksum = checksum(file)
- output = %x{packer fix #{file}}
- raise "[#{template}] Error fixing, exited #{$?}" if $?.exitstatus != 0
- # preserve ampersands in shell commands,
- # see: https://github.com/mitchellh/packer/issues/784
- output.gsub!("\\u0026", "&")
- File.open(file, "wb") { |dest| dest.write(output) }
- fixed_checksum = checksum(file)
-
- if original_checksum == fixed_checksum
- puts("No changes made.")
- else
- warn("Template #{template} has been modified.")
- @modified << template
- end
- end
-
- def packer_validate_cmd(template, var_file)
- vars = "#{template}.variables.json"
- cmd = %W[packer validate -var-file=#{var_file} #{template}.json]
- cmd.insert(2, "-var-file=#{vars}") if File.exist?(vars)
- cmd
- end
-
- def validate(template)
- for_packer_run_with(template) do |md_file, var_file|
- cmd = packer_validate_cmd(template, var_file.path)
- banner("[#{template}] Validating: '#{cmd.join(' ')}'")
- if debug
- banner("[#{template}] DEBUG: var_file(#{var_file.path}) is:")
- puts IO.read(var_file.path)
- banner("[#{template}] DEBUG: md_file(#{md_file.path}) is:")
- puts IO.read(md_file.path)
- end
- system(*cmd) or raise "[#{template}] Error validating, exited #{$?}"
- end
- end
-end
-
-class ListRunner
-
- include Common
-
- attr_reader :templates
-
- def initialize(opts)
- @templates = opts.templates
- end
-
- def start
- templates.each { |template| puts template }
- end
-end
-
-class Runner
-
- attr_reader :options
-
- def initialize(options)
- @options = options
- end
-
- def start
- options.klass.new(options).start
- end
-end
-
-class BuildMetadata
-
- def initialize(template, build_timestamp, override_version)
- @template = template
- @build_timestamp = build_timestamp
- @override_version = override_version
- end
-
- def read
- {
- name: name,
- version: version,
- build_timestamp: build_timestamp,
- git_revision: git_revision,
- box_basename: box_basename,
- template: template_vars.fetch("template", UNKNOWN),
- }
- end
-
- private
-
- UNKNOWN = "__unknown__".freeze
-
- attr_reader :template, :build_timestamp, :override_version
-
- def box_basename
- "#{name.gsub("/", "__")}-#{version}.git.#{git_revision}"
- end
-
- def git_revision
- sha = %x{git rev-parse HEAD}.strip
-
- git_clean? ? sha : "#{sha}_dirty"
- end
-
- def git_clean?
- %x{git status --porcelain}.strip.empty?
- end
-
- def merged_vars
- @merged_vars ||= begin
- if File.exist?("#{template}.variables.json")
- template_vars.merge(JSON.load(IO.read("#{template}.variables.json")))
- else
- template_vars
- end
- end
- end
-
- def name
- merged_vars.fetch("name", template)
- end
-
- def template_vars
- @template_vars ||= JSON.load(IO.read("#{template}.json")).fetch("variables")
- end
-
- def version
- if override_version
- override_version
- else
- merged_vars.fetch("version", "#{UNKNOWN}.TIMESTAMP").
- rpartition(".").first.concat(".#{build_timestamp}")
- end
- end
-end
-
-class ProviderMetadata
-
- def initialize(path, box_basename)
- @base = File.join(path, box_basename)
- end
-
- def read
- Dir.glob("#{base}.*.box").map do |file|
- {
- name: provider_from_file(file),
- file: "#{File.basename(file)}",
- checksum_type: "sha256",
- checksum: shasum(file)
- }
- end
- end
-
- private
-
- attr_reader :base
-
- def provider_from_file(file)
- case provider = file.sub(/^.*\.([^.]+)\.box$/, '\1')
- when /vmware/i then "vmware_desktop"
- else provider
- end
- end
-
- def shasum(file)
- Digest::SHA256.file(file).hexdigest
- end
-end
-
-begin
- Runner.new(Options.parse(ARGV)).start
-rescue => ex
- $stderr.puts ">>> #{ex.message}"
- exit(($? && $?.exitstatus) || 99)
-end
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/hdp-centos-6.7.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/hdp-centos-6.7.json b/metron-deployment/packer-build/hdp-centos-6.7.json
deleted file mode 100644
index 76efc8e..0000000
--- a/metron-deployment/packer-build/hdp-centos-6.7.json
+++ /dev/null
@@ -1,132 +0,0 @@
-{
- "builders": [
- {
- "type": "virtualbox-iso",
- "boot_command": [
- "<tab> text ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/{{user `ks_path`}} ksdevice=eth0<enter><wait>"
- ],
- "boot_wait": "10s",
- "disk_size": 40960,
- "guest_additions_path": "VBoxGuestAdditions_{{.Version}}.iso",
- "guest_os_type": "RedHat_64",
- "hard_drive_interface": "sata",
- "headless": "{{ user `headless` }}",
- "http_directory": "http",
- "iso_checksum": "{{user `iso_checksum`}}",
- "iso_checksum_type": "{{user `iso_checksum_type`}}",
- "iso_url": "{{user `mirror`}}/{{user `mirror_directory`}}/{{user `iso_name`}}",
- "output_directory": "packer-{{user `template`}}-virtualbox",
- "shutdown_command": "echo 'vagrant' | sudo -S /sbin/halt -h -p",
- "ssh_password": "vagrant",
- "ssh_port": 22,
- "ssh_username": "vagrant",
- "ssh_wait_timeout": "10000s",
- "vboxmanage": [
- [
- "modifyvm",
- "{{.Name}}",
- "--memory",
- "{{ user `memory` }}"
- ],
- [
- "modifyvm",
- "{{.Name}}",
- "--nic2",
- "hostonly"
- ],
- [
- "modifyvm",
- "{{.Name}}",
- "--hostonlyadapter2",
- "vboxnet0"
- ],
- [
- "modifyvm",
- "{{.Name}}",
- "--cpus",
- "{{ user `cpus` }}"
- ]
- ],
- "virtualbox_version_file": ".vbox_version",
- "vm_name": "{{ user `template` }}"
- }
- ],
- "post-processors": [
- {
- "output": "builds/{{user `box_basename`}}.{{.Provider}}.box",
- "type": "vagrant",
- "compression_level": 9
- }
- ],
- "provisioners": [
- {
- "destination": "/tmp/bento-metadata.json",
- "source": "{{user `metadata`}}",
- "type": "file"
- },
- {
- "type": "shell",
- "scripts": [
- "scripts/common/metadata.sh",
- "scripts/common/sshd.sh",
- "scripts/centos/networking.sh",
- "scripts/common/vagrant.sh",
- "scripts/centos/update_all.sh",
- "scripts/common/reboot.sh"
- ],
- "environment_vars": [
- "HOME_DIR=/home/vagrant",
- "http_proxy={{user `http_proxy`}}",
- "https_proxy={{user `https_proxy`}}",
- "no_proxy={{user `no_proxy`}}"
- ],
- "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'"
- },
- {
- "type" : "ansible",
- "pause_before" : "2m",
- "playbook_file" : "./ansible/playbook.yml",
- "sftp_command": "/usr/libexec/openssh/sftp-server",
- "user" : "vagrant",
- "host_alias" : "node1",
- "groups" : ["packer","ambari_master","ambari_slave"],
- "extra_arguments": [ "--tags", "ambari", "--skip-tags","packer-cleanup" ]
- },
- {
- "type": "shell",
- "scripts": [
- "scripts/common/vmtools.sh",
- "scripts/centos/cleanup.sh",
- "scripts/common/minimize.sh"
- ],
- "environment_vars": [
- "HOME_DIR=/home/vagrant",
- "http_proxy={{user `http_proxy`}}",
- "https_proxy={{user `https_proxy`}}",
- "no_proxy={{user `no_proxy`}}"
- ],
- "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'"
- }
- ],
- "variables": {
- "box_basename": "hdp-centos-6.7",
- "build_timestamp": "{{isotime \"20060102150405\"}}",
- "cpus": "1",
- "git_revision": "__unknown_git_revision__",
- "headless": "",
- "http_proxy": "{{env `http_proxy`}}",
- "https_proxy": "{{env `https_proxy`}}",
- "iso_checksum": "c0c1a05d3d74fb093c6232003da4b22b0680f59d3b2fa2cb7da736bc40b3f2c5",
- "iso_checksum_type": "sha256",
- "iso_name": "CentOS-6.7-x86_64-bin-DVD1.iso",
- "ks_path": "centos-6.7/ks.cfg",
- "memory": "8192",
- "metadata": "floppy/dummy_metadata.json",
- "mirror": "http://mirrors.kernel.org/centos",
- "mirror_directory": "6.7/isos/x86_64",
- "name": "hdp-centos-6.7",
- "no_proxy": "{{env `no_proxy`}}",
- "template": "hdp-centos-6.7",
- "version": "2.1.TIMESTAMP"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/http/centos-6.7/ks.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/http/centos-6.7/ks.cfg b/metron-deployment/packer-build/http/centos-6.7/ks.cfg
deleted file mode 100644
index d36102f..0000000
--- a/metron-deployment/packer-build/http/centos-6.7/ks.cfg
+++ /dev/null
@@ -1,86 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-install
-cdrom
-lang en_US.UTF-8
-keyboard us
-network --device eth0 --bootproto=dhcp --noipv6
-network --device eth1 --bootproto=static --ip=192.168.66.121 --netmask=255.255.255.0 --noipv6
-rootpw vagrant
-firewall --disabled
-selinux --permissive
-timezone UTC
-unsupported_hardware
-bootloader --location=mbr
-text
-skipx
-zerombr
-clearpart --all --initlabel
-autopart
-auth --enableshadow --passalgo=sha512 --kickstart
-firstboot --disabled
-reboot
-user --name=vagrant --plaintext --password vagrant
-key --skip
-
-%pre
-hostname node1
-%end
-
-%packages --nobase --ignoremissing --excludedocs
-# vagrant needs this to copy initial files via scp
-openssh-clients
-sudo
-wget
-nfs-utils
--fprintd-pam
--intltool
-
-# unnecessary firmware
--aic94xx-firmware
--atmel-firmware
--b43-openfwwf
--bfa-firmware
--ipw2100-firmware
--ipw2200-firmware
--ivtv-firmware
--iwl100-firmware
--iwl1000-firmware
--iwl3945-firmware
--iwl4965-firmware
--iwl5000-firmware
--iwl5150-firmware
--iwl6000-firmware
--iwl6000g2a-firmware
--iwl6050-firmware
--libertas-usb8388-firmware
--ql2100-firmware
--ql2200-firmware
--ql23xx-firmware
--ql2400-firmware
--ql2500-firmware
--rt61pci-firmware
--rt73usb-firmware
--xorg-x11-drv-ati-firmware
--zd1211-firmware
-
-%post
-# Force to set SELinux to a permissive mode
-sed -i -e 's/\(^SELINUX=\).*$/\1permissive/' /etc/selinux/config
-# sudo
-echo "%vagrant ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/vagrant
-sed -i "s/^[^#].*requiretty/#Defaults requiretty/" /etc/sudoers
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/metron-centos-6.7.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/metron-centos-6.7.json b/metron-deployment/packer-build/metron-centos-6.7.json
deleted file mode 100644
index a4cf625..0000000
--- a/metron-deployment/packer-build/metron-centos-6.7.json
+++ /dev/null
@@ -1,131 +0,0 @@
-{
- "builders": [
- {
- "type": "virtualbox-iso",
- "boot_command": [
- "<tab> text ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/{{user `ks_path`}} ksdevice=eth0<enter><wait>"
- ],
- "boot_wait": "10s",
- "disk_size": 40960,
- "guest_additions_path": "VBoxGuestAdditions_{{.Version}}.iso",
- "guest_os_type": "RedHat_64",
- "hard_drive_interface": "sata",
- "headless": "{{ user `headless` }}",
- "http_directory": "http",
- "iso_checksum": "{{user `iso_checksum`}}",
- "iso_checksum_type": "{{user `iso_checksum_type`}}",
- "iso_url": "{{user `mirror`}}/{{user `mirror_directory`}}/{{user `iso_name`}}",
- "output_directory": "packer-{{user `template`}}-virtualbox",
- "shutdown_command": "echo 'vagrant' | sudo -S /sbin/halt -h -p",
- "ssh_password": "vagrant",
- "ssh_port": 22,
- "ssh_username": "vagrant",
- "ssh_wait_timeout": "10000s",
- "vboxmanage": [
- [
- "modifyvm",
- "{{.Name}}",
- "--memory",
- "{{ user `memory` }}"
- ],
- [
- "modifyvm",
- "{{.Name}}",
- "--nic2",
- "hostonly"
- ],
- [
- "modifyvm",
- "{{.Name}}",
- "--hostonlyadapter2",
- "vboxnet0"
- ],
- [
- "modifyvm",
- "{{.Name}}",
- "--cpus",
- "{{ user `cpus` }}"
- ]
- ],
- "virtualbox_version_file": ".vbox_version",
- "vm_name": "{{ user `template` }}"
- }
- ],
- "post-processors": [
- {
- "output": "builds/{{user `box_basename`}}.{{.Provider}}.box",
- "type": "vagrant",
- "compression_level": 9
- }
- ],
- "provisioners": [
- {
- "destination": "/tmp/bento-metadata.json",
- "source": "{{user `metadata`}}",
- "type": "file"
- },
- {
- "type": "shell",
- "scripts": [
- "scripts/common/metadata.sh",
- "scripts/common/sshd.sh",
- "scripts/centos/networking.sh",
- "scripts/common/vagrant.sh",
- "scripts/centos/update_all.sh",
- "scripts/common/reboot.sh"
- ],
- "environment_vars": [
- "HOME_DIR=/home/vagrant",
- "http_proxy={{user `http_proxy`}}",
- "https_proxy={{user `https_proxy`}}",
- "no_proxy={{user `no_proxy`}}"
- ],
- "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'"
- },
- {
- "type" : "ansible",
- "pause_before" : "2m",
- "playbook_file" : "./ansible/playbook.yml",
- "sftp_command": "/usr/libexec/openssh/sftp-server",
- "user" : "vagrant",
- "host_alias" : "node1",
- "groups" : ["packer","ambari_master","ambari_slave","metron_kafka_topics","metron_hbase_tables","enrichment","search","web","sensors","pcap_server","metron"]
- },
- {
- "type": "shell",
- "scripts": [
- "scripts/common/vmtools.sh",
- "scripts/centos/cleanup.sh",
- "scripts/common/minimize.sh"
- ],
- "environment_vars": [
- "HOME_DIR=/home/vagrant",
- "http_proxy={{user `http_proxy`}}",
- "https_proxy={{user `https_proxy`}}",
- "no_proxy={{user `no_proxy`}}"
- ],
- "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'"
- }
- ],
- "variables": {
- "box_basename": "metron-centos-6.7",
- "build_timestamp": "{{isotime \"20060102150405\"}}",
- "cpus": "1",
- "git_revision": "__unknown_git_revision__",
- "headless": "",
- "http_proxy": "{{env `http_proxy`}}",
- "https_proxy": "{{env `https_proxy`}}",
- "iso_checksum": "c0c1a05d3d74fb093c6232003da4b22b0680f59d3b2fa2cb7da736bc40b3f2c5",
- "iso_checksum_type": "sha256",
- "iso_name": "CentOS-6.7-x86_64-bin-DVD1.iso",
- "ks_path": "centos-6.7/ks.cfg",
- "memory": "8192",
- "metadata": "floppy/dummy_metadata.json",
- "mirror": "http://mirrors.kernel.org/centos",
- "mirror_directory": "6.7/isos/x86_64",
- "name": "metron-centos-6.7",
- "no_proxy": "{{env `no_proxy`}}",
- "template": "metron-centos-6.7",
- "version": "2.1.TIMESTAMP"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/scripts/centos/cleanup.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/scripts/centos/cleanup.sh b/metron-deployment/packer-build/scripts/centos/cleanup.sh
deleted file mode 100644
index 035ab7e..0000000
--- a/metron-deployment/packer-build/scripts/centos/cleanup.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/sh -eux
-
-# should output one of 'redhat' 'centos' 'oraclelinux'
-distro="`rpm -qf --queryformat '%{NAME}' /etc/redhat-release | cut -f 1 -d '-'`"
-
-if [ "$distro" != 'redhat' ]; then
- yum -y clean all;
-fi
-
-# Clean up network interface persistence
-rm -f /etc/udev/rules.d/70-persistent-net.rules;
-mkdir -p /etc/udev/rules.d/70-persistent-net.rules;
-rm -f /lib/udev/rules.d/75-persistent-net-generator.rules;
-rm -rf /dev/.udev/;
-
-for ndev in `ls -1 /etc/sysconfig/network-scripts/ifcfg-*`; do
- if [ "`basename $ndev`" != "ifcfg-lo" ]; then
- sed -i '/^HWADDR/d' "$ndev";
- sed -i '/^UUID/d' "$ndev";
- fi
-done
-
-rm -f VBoxGuestAdditions_*.iso VBoxGuestAdditions_*.iso.?;
-echo "127.0.0.1 localhost" > /etc/hosts
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/scripts/centos/networking.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/scripts/centos/networking.sh b/metron-deployment/packer-build/scripts/centos/networking.sh
deleted file mode 100644
index 0bf05cb..0000000
--- a/metron-deployment/packer-build/scripts/centos/networking.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh -eux
-
-case "$PACKER_BUILDER_TYPE" in
-
-virtualbox-iso|virtualbox-ovf)
- major_version="`sed 's/^.\+ release \([.0-9]\+\).*/\1/' /etc/redhat-release | awk -F. '{print $1}'`";
-
- if [ "$major_version" -ge 6 ]; then
- # Fix slow DNS:
- # Add 'single-request-reopen' so it is included when /etc/resolv.conf is
- # generated
- # https://access.redhat.com/site/solutions/58625 (subscription required)
- echo 'RES_OPTIONS="single-request-reopen"' >>/etc/sysconfig/network;
- echo "192.168.66.121 node1" >>/etc/hosts;
- service network restart;
- echo 'Slow DNS fix applied (single-request-reopen)';
- fi
- ;;
-
-esac
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/scripts/centos/update_all.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/scripts/centos/update_all.sh b/metron-deployment/packer-build/scripts/centos/update_all.sh
deleted file mode 100644
index a864ce2..0000000
--- a/metron-deployment/packer-build/scripts/centos/update_all.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh -eux
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-yum update -y
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/scripts/common/metadata.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/scripts/common/metadata.sh b/metron-deployment/packer-build/scripts/common/metadata.sh
deleted file mode 100755
index 42386ac..0000000
--- a/metron-deployment/packer-build/scripts/common/metadata.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh -eux
-
-mkdir -p /etc;
-cp /tmp/bento-metadata.json /etc/bento-metadata.json;
-chmod 0444 /etc/bento-metadata.json;
-rm -f /tmp/bento-metadata.json;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/scripts/common/minimize.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/scripts/common/minimize.sh b/metron-deployment/packer-build/scripts/common/minimize.sh
deleted file mode 100644
index 1ca88be..0000000
--- a/metron-deployment/packer-build/scripts/common/minimize.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/sh -eux
-
-case "$PACKER_BUILDER_TYPE" in
- qemu) exit 0 ;;
-esac
-
-set +e
-swapuuid="`/sbin/blkid -o value -l -s UUID -t TYPE=swap`";
-case "$?" in
- 2|0) ;;
- *) exit 1 ;;
-esac
-set -e
-
-if [ "x${swapuuid}" != "x" ]; then
- # Whiteout the swap partition to reduce box size
- # Swap is disabled till reboot
- swappart="`readlink -f /dev/disk/by-uuid/$swapuuid`";
- /sbin/swapoff "$swappart";
- dd if=/dev/zero of="$swappart" bs=1M || echo "dd exit code $? is suppressed";
- /sbin/mkswap -U "$swapuuid" "$swappart";
-fi
-
-dd if=/dev/zero of=/EMPTY bs=1M || echo "dd exit code $? is suppressed";
-rm -f /EMPTY;
-# Block until the empty file has been removed, otherwise, Packer
-# will try to kill the box while the disk is still full and that's bad
-sync;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/scripts/common/reboot.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/scripts/common/reboot.sh b/metron-deployment/packer-build/scripts/common/reboot.sh
deleted file mode 100644
index 1365474..0000000
--- a/metron-deployment/packer-build/scripts/common/reboot.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh -eux
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-reboot
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/scripts/common/sshd.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/scripts/common/sshd.sh b/metron-deployment/packer-build/scripts/common/sshd.sh
deleted file mode 100644
index 1bb16f5..0000000
--- a/metron-deployment/packer-build/scripts/common/sshd.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh -eux
-
-SSHD_CONFIG="/etc/ssh/sshd_config"
-
-# ensure that there is a trailing newline before attempting to concatenate
-sed -i -e '$a\' "$SSHD_CONFIG"
-
-USEDNS="UseDNS no"
-if grep -q -E "^[[:space:]]*UseDNS" "$SSHD_CONFIG"; then
- sed -i "s/^\s*UseDNS.*/${USEDNS}/" "$SSHD_CONFIG"
-else
- echo "$USEDNS" >>"$SSHD_CONFIG"
-fi
-
-GSSAPI="GSSAPIAuthentication no"
-if grep -q -E "^[[:space:]]*GSSAPIAuthentication" "$SSHD_CONFIG"; then
- sed -i "s/^\s*GSSAPIAuthentication.*/${GSSAPI}/" "$SSHD_CONFIG"
-else
- echo "$GSSAPI" >>"$SSHD_CONFIG"
-fi
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/scripts/common/sudoers.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/scripts/common/sudoers.sh b/metron-deployment/packer-build/scripts/common/sudoers.sh
deleted file mode 100644
index 5fce3dd..0000000
--- a/metron-deployment/packer-build/scripts/common/sudoers.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash -eux
-
-sed -i -e '/Defaults\s\+env_reset/a Defaults\texempt_group=sudo' /etc/sudoers
-sed -i -e 's/%sudo ALL=(ALL:ALL) ALL/%sudo ALL=NOPASSWD:ALL/g' /etc/sudoers
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/scripts/common/vagrant.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/scripts/common/vagrant.sh b/metron-deployment/packer-build/scripts/common/vagrant.sh
deleted file mode 100644
index b69c3a5..0000000
--- a/metron-deployment/packer-build/scripts/common/vagrant.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh -eux
-
-# set a default HOME_DIR environment variable if not set
-HOME_DIR="${HOME_DIR:-/home/vagrant}";
-
-pubkey_url="https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant.pub";
-mkdir -p $HOME_DIR/.ssh;
-if command -v wget >/dev/null 2>&1; then
- wget --no-check-certificate "$pubkey_url" -O $HOME_DIR/.ssh/authorized_keys;
-elif command -v curl >/dev/null 2>&1; then
- curl --insecure --location "$pubkey_url" > $HOME_DIR/.ssh/authorized_keys;
-elif command -v fetch >/dev/null 2>&1; then
- fetch -am -o $HOME_DIR/.ssh/authorized_keys "$pubkey_url";
-else
- echo "Cannot download vagrant public key";
- exit 1;
-fi
-chown -R vagrant $HOME_DIR/.ssh;
-chmod -R go-rwsx $HOME_DIR/.ssh;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packer-build/scripts/common/vmtools.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/scripts/common/vmtools.sh b/metron-deployment/packer-build/scripts/common/vmtools.sh
deleted file mode 100644
index 3656cf7..0000000
--- a/metron-deployment/packer-build/scripts/common/vmtools.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/bin/sh -eux
-
-# set a default HOME_DIR environment variable if not set
-HOME_DIR="${HOME_DIR:-/home/vagrant}";
-
-case "$PACKER_BUILDER_TYPE" in
-
-virtualbox-iso|virtualbox-ovf)
- yum install -y kernel-devel-`uname -r`
- yum install -y gcc
- yum install -y binutils
- yum install -y make
- yum install -y perl
- yum install -y bzip2
- mkdir -p /tmp/vbox;
- ver="`cat /home/vagrant/.vbox_version`";
- mount -o loop $HOME_DIR/VBoxGuestAdditions_${ver}.iso /tmp/vbox;
- sh /tmp/vbox/VBoxLinuxAdditions.run \
- || echo "VBoxLinuxAdditions.run exited $? and is suppressed." \
- "For more read https://www.virtualbox.org/ticket/12479";
- umount /tmp/vbox;
- rm -rf /tmp/vbox;
- #rm -f $HOME_DIR/*.iso;
- ;;
-
-vmware-iso|vmware-vmx)
- mkdir -p /tmp/vmfusion;
- mkdir -p /tmp/vmfusion-archive;
- mount -o loop $HOME_DIR/linux.iso /tmp/vmfusion;
- tar xzf /tmp/vmfusion/VMwareTools-*.tar.gz -C /tmp/vmfusion-archive;
- /tmp/vmfusion-archive/vmware-tools-distrib/vmware-install.pl --force-install;
- umount /tmp/vmfusion;
- rm -rf /tmp/vmfusion;
- rm -rf /tmp/vmfusion-archive;
- rm -f $HOME_DIR/*.iso;
- ;;
-
-parallels-iso|parallels-pvm)
- mkdir -p /tmp/parallels;
- mount -o loop $HOME_DIR/prl-tools-lin.iso /tmp/parallels;
- /tmp/parallels/install --install-unattended-with-deps \
- || (code="$?"; \
- echo "Parallels tools installation exited $code, attempting" \
- "to output /var/log/parallels-tools-install.log"; \
- cat /var/log/parallels-tools-install.log; \
- exit $code);
- umount /tmp/parallels;
- rm -rf /tmp/parallels;
- rm -f $HOME_DIR/*.iso;
- ;;
-
-qemu)
- echo "Don't need anything for this one"
- ;;
-
-*)
- echo "Unknown Packer Builder Type >>$PACKER_BUILDER_TYPE<< selected.";
- echo "Known are virtualbox-iso|virtualbox-ovf|vmware-iso|vmware-vmx|parallels-iso|parallels-pvm.";
- ;;
-
-esac
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/playbooks/ambari_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/ambari_install.yml b/metron-deployment/playbooks/ambari_install.yml
index a8f3f7e..51aa907 100644
--- a/metron-deployment/playbooks/ambari_install.yml
+++ b/metron-deployment/playbooks/ambari_install.yml
@@ -54,6 +54,13 @@
- ambari-agent
- hdp-install
+- hosts: metron
+ become: true
+ roles:
+ - role: metron-rpms
+ tags:
+ - metron-deploy
+
- hosts: ambari_master
become: true
roles:
@@ -61,3 +68,10 @@
tags:
- hdp-install
- hdp-deploy
+
+- hosts: ambari_master
+ become: true
+ roles:
+ - role: load_web_templates
+ tags:
+ - load_templates