You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

274 lines
7.7 KiB

#!/usr/bin/ruby
require 'optparse'
require 'yaml'
require 'fileutils'
require 'pathname'
@options = {:num_procs => 2, :memory => 2000}
@bitness = {
'i386' => 32,
'amd64' => 64,
}
@arches = {
'i386' => 'i386',
'amd64' => 'x86_64',
}
def system!(cmd)
system(cmd) or raise "failed to run #{cmd}"
end
def sanitize(str, where)
raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/)
str
end
def sanitize_path(str, where)
raise "unsanitary string in #{where}" if (str =~ /[^\w\/.:+-]/)
str
end
def info(str)
puts str unless @options[:quiet]
end
def build_one_configuration(suite, arch, build_desc, reference_datetime)
FileUtils.rm_f("var/build.log")
bits = @bitness[arch] or raise "unknown architecture ${arch}"
if ENV["USE_LXC"]
ENV["LXC_ARCH"] = arch
ENV["LXC_SUITE"] = suite
end
suitearch = "#{suite}-#{arch}"
info "Stopping target if it is up"
system "stop-target"
sleep 1
unless @options[:skip_image]
info "Making a new image copy"
system! "make-clean-vm --suite #{suite} --arch #{arch}"
end
info "Starting target"
system! "start-target #{bits} #{suitearch}&"
$stdout.write "Checking if target is up"
(1..10).each do
system "on-target true 2> /dev/null" and break
sleep 2
$stdout.write '.'
end
info ''
system! "on-target true"
info "Preparing build environment"
system! "on-target setarch #{@arches[arch]} bash < target-bin/init-build.sh"
build_desc["files"].each do |filename|
filename = sanitize(filename, "files section")
system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/"
end
info "Updating apt-get repository (log in var/install.log)"
system! "on-target -u root apt-get update > var/install.log 2>&1"
info "Installing additional packages (log in var/install.log)"
system! "on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"
info "Grabbing package manifest"
system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest"
info "Creating build script (var/build-script)"
File.open("var/build-script", "w") do |script|
script.puts "#!/bin/bash"
script.puts "set -e"
script.puts "umask 002"
script.puts "export OUTDIR=$HOME/out"
script.puts "GBUILD_BITS=#{bits}"
script.puts "MAKEOPTS=(-j#{@options[:num_procs]})"
(ref_date, ref_time) = reference_datetime.split
script.puts "REFERENCE_DATETIME='#{reference_datetime}'"
script.puts "REFERENCE_DATE='#{ref_date}'"
script.puts "REFERENCE_TIME='#{ref_time}'"
script.puts
build_desc["remotes"].each do |remote|
dir = sanitize(remote["dir"], remote["dir"])
system! "copy-to-target #{@quiet_flag} inputs/#{dir} build/"
script.puts "(cd build/#{dir} && git reset -q --hard && git clean -q -f -d)"
end
script.puts "cd build"
script.puts build_desc["script"]
end
info "Running build script (log in var/build.log)"
system! "on-target setarch #{@arches[arch]} bash -x < var/build-script > var/build.log 2>&1"
end
################################
OptionParser.new do |opts|
opts.banner = "Usage: build [options] <build-description>.yml"
opts.on("-i", "--skip-image", "reuse current target image") do |v|
@options[:skip_image] = v
end
opts.on("-q", "--quiet", "be quiet") do |v|
@options[:quiet] = v
end
opts.on("-j PROCS", "--num-make PROCS", "number of processes to use") do |v|
@options[:num_procs] = v
end
opts.on("-m MEM", "--memory MEM", "memory to allocate in MiB") do |v|
@options[:memory] = v
end
opts.on("-c PAIRS", "--commit PAIRS", "comma separated list of DIRECTORY=COMMIT pairs") do |v|
@options[:commit] = v
end
opts.on("-u PAIRS", "--url PAIRS", "comma separated list of DIRECTORY=URL pairs") do |v|
@options[:url] = v
end
end.parse!
if !ENV["USE_LXC"] and !File.exist?("/dev/kvm")
$stderr.puts "\n************* WARNING: kvm not loaded, this will probably not work out\n\n"
end
base_dir = Pathname.new(__FILE__).expand_path.dirname.parent
libexec_dir = base_dir + 'libexec'
ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH']
ENV['GITIAN_BASE'] = base_dir.to_s
ENV['NPROCS'] = @options[:num_procs].to_s
ENV['VMEM'] = @options[:memory].to_s
@quiet_flag = @options[:quiet] ? "-q" : ""
build_desc_file = ARGV.shift or raise "must supply YAML build description file"
build_desc = YAML.load_file(build_desc_file)
in_sums = []
build_dir = 'build'
result_dir = 'result'
FileUtils.rm_rf(build_dir)
FileUtils.mkdir(build_dir)
FileUtils.mkdir_p(result_dir)
package_name = build_desc["name"] or raise "must supply name"
package_name = sanitize(package_name, "package name")
suites = build_desc["suites"] or raise "must supply suites"
archs = build_desc["architectures"] or raise "must supply architectures"
reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime"
desc_sum = `sha256sum #{build_desc_file}`
desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
in_sums << desc_sum
build_desc["files"].each do |filename|
filename = sanitize(filename, "files section")
in_sums << `cd inputs && sha256sum #{filename}`
end
commits = {}
if @options[:commit]
@options[:commit].split(',').each do |pair|
(dir, commit) = pair.split('=')
commits[dir] = commit
end
end
urls = {}
if @options[:url]
@options[:url].split(',').each do |pair|
(dir, url) = pair.split('=')
urls[dir] = url
end
end
build_desc["remotes"].each do |remote|
if !remote["commit"]
remote["commit"] = commits[remote["dir"]]
raise "must specify a commit for directory #{remote["dir"]}" unless remote["commit"]
end
if urls[remote["dir"]]
remote["url"] = urls[remote["dir"]]
end
dir = sanitize(remote["dir"], remote["dir"])
unless File.exist?("inputs/#{dir}")
system!("git init inputs/#{dir}")
end
system!("cd inputs/#{dir} && git fetch --update-head-ok #{sanitize_path(remote["url"], remote["url"])} +refs/tags/*:refs/tags/* +refs/heads/*:refs/heads/*")
commit = sanitize(remote["commit"], remote["commit"])
commit = `cd inputs/#{dir} && git log --format=%H -1 #{commit}`.strip
raise "error looking up commit for tag #{remote["commit"]}" unless $?.exitstatus == 0
system!("cd inputs/#{dir} && git checkout -q #{commit}")
in_sums << "git:#{commit} #{dir}"
end
base_manifests = YAML::Omap.new
suites.each do |suite|
suite = sanitize(suite, "suite")
archs.each do |arch|
info "--- Building for #{suite} #{arch} ---"
arch = sanitize(arch, "architecture")
# Build!
build_one_configuration(suite, arch, build_desc, reference_datetime)
info "Grabbing results"
system! "copy-from-target #{@quiet_flag} out #{build_dir}"
base_manifest = File.read("var/base-#{suite}-#{arch}.manifest")
base_manifests["#{suite}-#{arch}"] = base_manifest
end
end
out_dir = File.join(build_dir, "out")
out_sums = {}
info "Generating report"
Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
next if File.directory?(file_in_out)
file = file_in_out.sub(out_dir + File::SEPARATOR, '')
file = sanitize_path(file, file_in_out)
out_sums[file] = `cd #{out_dir} && sha256sum #{file}`
raise "failed to sum #{file}" unless $? == 0
puts out_sums[file] unless @options[:quiet]
end
out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
in_manifest = in_sums.join('')
# Use Omap to keep result deterministic
report = YAML::Omap[
'out_manifest', out_manifest,
'in_manifest', in_manifest,
'base_manifests', base_manifests,
]
result_file = "#{package_name}-res.yml"
File.open(File.join(result_dir, result_file), "w") do |io|
io.write report.to_yaml
end
system!("cd #{result_dir} && sha256sum #{result_file}") unless @options[:quiet]
info "Done."