You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
382 lines
12 KiB
382 lines
12 KiB
#!/usr/bin/ruby |
|
|
|
require 'optparse' |
|
require 'yaml' |
|
require 'fileutils' |
|
require 'pathname' |
|
|
|
@options = {:num_procs => 2, :memory => 2000} |
|
|
|
@bitness = { |
|
'i386' => 32, |
|
'amd64' => 64, |
|
} |
|
|
|
@arches = { |
|
'i386' => 'i386', |
|
'amd64' => 'x86_64', |
|
} |
|
|
|
def system!(cmd) |
|
system(cmd) or raise "failed to run #{cmd}" |
|
end |
|
|
|
def sanitize(str, where) |
|
raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/) |
|
str |
|
end |
|
|
|
def sanitize_path(str, where) |
|
raise "unsanitary string in #{where}" if (str =~ /[^@\w\/.:+-]/) |
|
str |
|
end |
|
|
|
def info(str) |
|
puts str unless @options[:quiet] |
|
end |
|
|
|
def build_one_configuration(suite, arch, build_desc) |
|
FileUtils.rm_f("var/install.log") |
|
FileUtils.rm_f("var/build.log") |
|
|
|
bits = @bitness[arch] or raise "unknown architecture ${arch}" |
|
|
|
if ENV["USE_LXC"] |
|
ENV["LXC_ARCH"] = arch |
|
ENV["LXC_SUITE"] = suite |
|
end |
|
|
|
suitearch = "#{suite}-#{arch}" |
|
|
|
info "Stopping target if it is up" |
|
system "stop-target" |
|
|
|
sleep 1 |
|
|
|
unless @options[:skip_image] |
|
info "Making a new image copy" |
|
system! "make-clean-vm --suite #{suite} --arch #{arch}" |
|
end |
|
|
|
info "Starting target" |
|
system! "start-target #{bits} #{suitearch}&" |
|
|
|
$stdout.write "Checking if target is up" |
|
|
|
(1..30).each do |
|
system "on-target true 2> /dev/null" and break |
|
sleep 2 |
|
$stdout.write '.' |
|
end |
|
|
|
info '' |
|
|
|
system! "on-target true" |
|
|
|
system! "on-target -u root tee -a /etc/sudoers.d/#{ENV['DISTRO'] || 'ubuntu'} > /dev/null << EOF |
|
%#{ENV['DISTRO'] || 'ubuntu'} ALL=(ALL) NOPASSWD: ALL |
|
EOF" if build_desc["sudo"] and @options[:allow_sudo] |
|
|
|
info "Preparing build environment" |
|
system! "on-target setarch #{@arches[arch]} bash < target-bin/init-build.sh" |
|
|
|
build_desc["files"].each do |filename| |
|
filename = sanitize(filename, "files section") |
|
system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/" |
|
end |
|
|
|
if build_desc["enable_cache"] |
|
if File.directory?("cache/#{build_desc["name"]}") |
|
system! "copy-to-target #{@quiet_flag} cache/#{build_desc["name"]}/ cache/" |
|
end |
|
|
|
if File.directory?("cache/common") |
|
system! "copy-to-target #{@quiet_flag} cache/common/ cache/" |
|
end |
|
end |
|
|
|
if build_desc["multiarch"] |
|
info "Adding multiarch support (log in var/install.log)" |
|
for a in build_desc["multiarch"] |
|
system! "on-target -u root dpkg --add-architecture #{a} >> var/install.log 2>&1" |
|
end |
|
end |
|
|
|
info "Updating apt-get repository (log in var/install.log)" |
|
system! "on-target -u root apt-get update >> var/install.log 2>&1" |
|
|
|
info "Installing additional packages (log in var/install.log)" |
|
system! "on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install #{build_desc["packages"].join(" ")} >> var/install.log 2>&1" |
|
|
|
if build_desc["alternatives"] |
|
info "Set alternatives (log in var/install.log)" |
|
for a in build_desc["alternatives"] |
|
system! "on-target -u root update-alternatives --set #{a["package"]} #{a["path"]} >> var/install.log 2>&1" |
|
end |
|
end |
|
|
|
if @options[:upgrade] || system("on-target -u root '[ ! -e /var/cache/gitian/initial-upgrade ]'") |
|
info "Upgrading system, may take a while (log in var/install.log)" |
|
system! "on-target -u root bash < target-bin/upgrade-system.sh >> var/install.log 2>&1" |
|
end |
|
info "Creating package manifest" |
|
system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest" |
|
|
|
info "Creating build script (var/build-script)" |
|
|
|
File.open("var/build-script", "w") do |script| |
|
script.puts "#!/bin/bash" |
|
script.puts "set -e" |
|
script.puts "export LANG='en_US.UTF-8'" |
|
script.puts "export LC_ALL='en_US.UTF-8'" |
|
script.puts "umask 002" |
|
script.puts "export OUTDIR=$HOME/out" |
|
script.puts "GBUILD_BITS=#{bits}" |
|
if build_desc["enable_cache"] |
|
script.puts "GBUILD_CACHE_ENABLED=1" |
|
script.puts "GBUILD_PACKAGE_CACHE=$HOME/cache/#{build_desc["name"]}" |
|
script.puts "GBUILD_COMMON_CACHE=$HOME/cache/common" |
|
end |
|
script.puts "MAKEOPTS=(-j#{@options[:num_procs]})" |
|
script.puts |
|
author_date = nil |
|
build_desc["remotes"].each do |remote| |
|
dir = sanitize(remote["dir"], remote["dir"]) |
|
|
|
author_date = `cd inputs/#{dir} && git log --format=@%at -1 | date +"%F %T" -u -f -`.strip |
|
raise "error looking up author date in #{dir}" unless $?.exitstatus == 0 |
|
|
|
system! "copy-to-target #{@quiet_flag} inputs/#{dir} build/" |
|
script.puts "(cd build/#{dir} && git reset -q --hard && git clean -q -f -d)" |
|
end |
|
script.puts |
|
ref_datetime = build_desc["reference_datetime"] || author_date |
|
(ref_date, ref_time) = ref_datetime.split |
|
script.puts "REFERENCE_DATETIME='#{ref_datetime}'" |
|
script.puts "REFERENCE_DATE='#{ref_date}'" |
|
script.puts "REFERENCE_TIME='#{ref_time}'" |
|
script.puts |
|
script.puts "cd build" |
|
script.puts build_desc["script"] |
|
end |
|
|
|
info "Running build script (log in var/build.log)" |
|
system! "on-target setarch #{@arches[arch]} bash -x < var/build-script > var/build.log 2>&1" |
|
end |
|
|
|
################################ |
|
|
|
OptionParser.new do |opts| |
|
opts.banner = "Usage: build [options] <build-description>.yml" |
|
|
|
opts.on("--allow-sudo", "override SECURITY on the target VM and allow the use of sudo with no password for the default user") do |v| |
|
@options[:allow_sudo] = v |
|
end |
|
opts.on("-i", "--skip-image", "reuse current target image") do |v| |
|
@options[:skip_image] = v |
|
end |
|
opts.on("--upgrade", "upgrade guest with latest packages") do |v| |
|
@options[:upgrade] = v |
|
end |
|
opts.on("-q", "--quiet", "be quiet") do |v| |
|
@options[:quiet] = v |
|
end |
|
opts.on("-j PROCS", "--num-make PROCS", "number of processes to use") do |v| |
|
@options[:num_procs] = v |
|
end |
|
opts.on("-m MEM", "--memory MEM", "memory to allocate in MiB") do |v| |
|
@options[:memory] = v |
|
end |
|
opts.on("-c PAIRS", "--commit PAIRS", "comma separated list of DIRECTORY=COMMIT pairs") do |v| |
|
@options[:commit] = v |
|
end |
|
opts.on("-u PAIRS", "--url PAIRS", "comma separated list of DIRECTORY=URL pairs") do |v| |
|
@options[:url] = v |
|
end |
|
opts.on("-o", "--cache-read-only", "only use existing cache files, do not update them") do |v| |
|
@options[:cache_ro] = v |
|
end |
|
end.parse! |
|
|
|
if !ENV["USE_LXC"] and !ENV["USE_DOCKER"] and !ENV["USE_VBOX"] and !File.exist?("/dev/kvm") |
|
$stderr.puts "\n************* WARNING: kvm not loaded, this will probably not work out\n\n" |
|
end |
|
|
|
base_dir = Pathname.new(__FILE__).expand_path.dirname.parent |
|
libexec_dir = base_dir + 'libexec' |
|
|
|
ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH'] |
|
ENV['GITIAN_BASE'] = base_dir.to_s |
|
ENV['NPROCS'] = @options[:num_procs].to_s |
|
ENV['VMEM'] = @options[:memory].to_s |
|
@quiet_flag = @options[:quiet] ? "-q" : "" |
|
|
|
build_desc_file = ARGV.shift or raise "must supply YAML build description file" |
|
|
|
build_desc = YAML.load_file(build_desc_file) |
|
|
|
in_sums = [] |
|
|
|
build_dir = 'build' |
|
result_dir = 'result' |
|
cache_dir = 'cache' |
|
enable_cache = build_desc["enable_cache"] |
|
|
|
FileUtils.rm_rf(build_dir) |
|
FileUtils.mkdir(build_dir) |
|
FileUtils.mkdir_p(result_dir) |
|
|
|
package_name = build_desc["name"] or raise "must supply name" |
|
package_name = sanitize(package_name, "package name") |
|
|
|
if enable_cache |
|
FileUtils.mkdir_p(File.join(cache_dir, "common")) |
|
FileUtils.mkdir_p(File.join(cache_dir, package_name)) |
|
end |
|
|
|
distro = build_desc["distro"] || "ubuntu" |
|
suites = build_desc["suites"] or raise "must supply suites" |
|
archs = build_desc["architectures"] or raise "must supply architectures" |
|
build_desc["reference_datetime"] or build_desc["remotes"].size > 0 or raise "must supply `reference_datetime` or `remotes`" |
|
docker_image_digests = build_desc["docker_image_digests"] || [] |
|
|
|
# if docker_image_digests are supplied, it must be the same length as suites |
|
if docker_image_digests.size > 0 and suites.size != docker_image_digests.size |
|
raise "`suites` and `docker_image_digests` must both be the same size if both are supplied" |
|
elsif ENV["USE_DOCKER"] and docker_image_digests.size > 0 and suites.size == docker_image_digests.size |
|
suites = docker_image_digests |
|
end |
|
|
|
ENV['DISTRO'] = distro |
|
|
|
desc_sum = `sha256sum #{build_desc_file}` |
|
desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml") |
|
in_sums << desc_sum |
|
|
|
build_desc["files"].each do |filename| |
|
filename = sanitize(filename, "files section") |
|
in_sums << `cd inputs && sha256sum #{filename}` |
|
end |
|
|
|
commits = {} |
|
|
|
if @options[:commit] |
|
@options[:commit].split(',').each do |pair| |
|
(dir, commit) = pair.split('=') |
|
commits[dir] = commit |
|
end |
|
end |
|
|
|
urls = {} |
|
|
|
if @options[:url] |
|
@options[:url].split(',').each do |pair| |
|
(dir, url) = pair.split('=') |
|
urls[dir] = url |
|
end |
|
end |
|
|
|
build_desc["remotes"].each do |remote| |
|
if !remote["commit"] |
|
remote["commit"] = commits[remote["dir"]] |
|
raise "must specify a commit for directory #{remote["dir"]}" unless remote["commit"] |
|
end |
|
if urls[remote["dir"]] |
|
remote["url"] = urls[remote["dir"]] |
|
end |
|
dir = sanitize(remote["dir"], remote["dir"]) |
|
commit = sanitize(remote["commit"], remote["commit"]) |
|
unless File.exist?("inputs/#{dir}") |
|
system!("git init inputs/#{dir}") |
|
system!("cd inputs/#{dir} && git fetch --update-head-ok #{sanitize_path(remote["url"], remote["url"])} +refs/tags/*:refs/tags/* +refs/heads/*:refs/heads/*") |
|
system!("cd inputs/#{dir} && git checkout -q #{commit}") |
|
system!("cd inputs/#{dir} && git submodule update --init --recursive --force") |
|
end |
|
commit = `cd inputs/#{dir} && git log --format=%H -1 #{commit}`.strip |
|
raise "error looking up commit for tag #{remote["commit"]}" unless $?.exitstatus == 0 |
|
in_sums << "git:#{commit} #{dir}" |
|
end |
|
|
|
base_manifests = YAML::Omap.new |
|
|
|
suites.each do |suite| |
|
suite = sanitize(suite, "suite") |
|
archs.each do |arch| |
|
info "--- Building for #{suite} #{arch} ---" |
|
arch = sanitize(arch, "architecture") |
|
|
|
# Build! |
|
build_one_configuration(suite, arch, build_desc) |
|
|
|
info "Grabbing results from target" |
|
system! "copy-from-target #{@quiet_flag} out #{build_dir}" |
|
|
|
if enable_cache && !@options[:cache_ro] |
|
info "Grabbing cache from target" |
|
system! "copy-from-target #{@quiet_flag} cache/#{package_name}/ #{cache_dir}" |
|
system! "copy-from-target #{@quiet_flag} cache/common/ #{cache_dir}" |
|
end |
|
base_manifest = File.read("var/base-#{suite}-#{arch}.manifest") |
|
base_manifests["#{suite}-#{arch}"] = base_manifest |
|
end |
|
end |
|
|
|
out_dir = File.join(build_dir, "out") |
|
out_sums = {} |
|
cache_common_dir = File.join(cache_dir, "common") |
|
cache_package_dir = File.join(cache_dir, "#{package_name}") |
|
cache_common_sums = {} |
|
cache_package_sums = {} |
|
|
|
info "Generating report" |
|
Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out| |
|
next if File.directory?(file_in_out) |
|
file = file_in_out.sub(out_dir + File::SEPARATOR, '') |
|
file = sanitize_path(file, file_in_out) |
|
out_sums[file] = `cd #{out_dir} && sha256sum #{file}` |
|
raise "failed to sum #{file}" unless $? == 0 |
|
puts out_sums[file] unless @options[:quiet] |
|
end |
|
|
|
if enable_cache |
|
Dir.glob(File.join(cache_common_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out| |
|
next if File.directory?(file_in_out) |
|
file = file_in_out.sub(cache_common_dir + File::SEPARATOR, '') |
|
file = sanitize_path(file, file_in_out) |
|
cache_common_sums[file] = `cd #{cache_common_dir} && sha256sum #{file}` |
|
raise "failed to sum #{file}" unless $? == 0 |
|
end |
|
|
|
Dir.glob(File.join(cache_package_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out| |
|
next if File.directory?(file_in_out) |
|
file = file_in_out.sub(cache_package_dir + File::SEPARATOR, '') |
|
file = sanitize_path(file, file_in_out) |
|
cache_package_sums[file] = `cd #{cache_package_dir} && sha256sum #{file}` |
|
raise "failed to sum #{file}" unless $? == 0 |
|
end |
|
end |
|
|
|
out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('') |
|
|
|
in_manifest = in_sums.join('') |
|
|
|
cache_common_manifest = cache_common_sums.keys.sort.map { |key| cache_common_sums[key] }.join('') |
|
cache_package_manifest = cache_package_sums.keys.sort.map { |key| cache_package_sums[key] }.join('') |
|
|
|
# Use Omap to keep result deterministic |
|
report = YAML::Omap[ |
|
'out_manifest', out_manifest, |
|
'in_manifest', in_manifest, |
|
'base_manifests', base_manifests, |
|
'cache_common_manifest', cache_common_manifest, |
|
'cache_package_manifest', cache_package_manifest, |
|
] |
|
|
|
result_file = "#{package_name}-res.yml" |
|
File.open(File.join(result_dir, result_file), "w") do |io| |
|
io.write report.to_yaml |
|
end |
|
|
|
system!("cd #{result_dir} && sha256sum #{result_file}") unless @options[:quiet] |
|
|
|
info "Done."
|
|
|