gitian-builder/bin/gbuild

#!/usr/bin/ruby

require 'optparse'
require 'yaml'
require 'fileutils'
require 'pathname'

@options = {:num_procs => 2, :memory => 2000}

@bitness = {
  'i386' => 32,
  'amd64' => 64,
}

@arches = {
  'i386' => 'i386',
  'amd64' => 'x86_64',
}

def system!(cmd)
  system(cmd) or raise "failed to run #{cmd}"
end

def sanitize(str, where)
  raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/)
  str
end

def sanitize_path(str, where)
  raise "unsanitary string in #{where}" if (str =~ /[^@\w\/.:+-]/)
  str
end

def info(str)
  puts str unless @options[:quiet]
end

def build_one_configuration(suite, arch, build_desc)
  FileUtils.rm_f("var/install.log")
  FileUtils.rm_f("var/build.log")

  bits = @bitness[arch] or raise "unknown architecture ${arch}"

  if ENV["USE_LXC"]
      ENV["LXC_ARCH"] = arch
      ENV["LXC_SUITE"] = suite
  end

  suitearch = "#{suite}-#{arch}"

  info "Stopping target if it is up"
  system "stop-target"

  sleep 1

  unless @options[:skip_image]
    info "Making a new image copy"
    system! "make-clean-vm --suite #{suite} --arch #{arch}"
  end

  info "Starting target"
  system! "start-target #{bits} #{suitearch}&"

  $stdout.write "Checking if target is up"

  (1..30).each do
    system "on-target true 2> /dev/null" and break
    sleep 2
    $stdout.write '.'
  end

  info ''

  system! "on-target true"

  system! "on-target -u root tee -a /etc/sudoers.d/#{ENV['DISTRO'] || 'ubuntu'} > /dev/null << EOF
%#{ENV['DISTRO'] || 'ubuntu'} ALL=(ALL) NOPASSWD: ALL
EOF" if build_desc["sudo"] and @options[:allow_sudo]

  info "Preparing build environment"
  system! "on-target setarch #{@arches[arch]} bash < target-bin/init-build.sh"

  build_desc["files"].each do |filename|
    filename = sanitize(filename, "files section")
    system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/"
  end

  if build_desc["enable_cache"]
    if File.directory?("cache/#{build_desc["name"]}")
      system! "copy-to-target #{@quiet_flag} cache/#{build_desc["name"]}/ cache/"
    end

    if File.directory?("cache/common")
      system! "copy-to-target #{@quiet_flag} cache/common/ cache/"
    end
  end

  if build_desc["multiarch"]
    info "Adding multiarch support (log in var/install.log)"
    for a in build_desc["multiarch"]
      system! "on-target -u root dpkg --add-architecture #{a} >> var/install.log 2>&1"
    end
  end

  info "Updating apt-get repository (log in var/install.log)"
  system! "on-target -u root apt-get update >> var/install.log 2>&1"

  info "Installing additional packages (log in var/install.log)"
  system! "on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install #{build_desc["packages"].join(" ")} >> var/install.log 2>&1"

  if build_desc["alternatives"]
    info "Set alternatives (log in var/install.log)"
    for a in build_desc["alternatives"]
      system! "on-target -u root update-alternatives --set #{a["package"]} #{a["path"]} >> var/install.log 2>&1"
    end
  end

  if @options[:upgrade] || system("on-target -u root '[ ! -e /var/cache/gitian/initial-upgrade ]'")
      info "Upgrading system, may take a while (log in var/install.log)"
      system! "on-target -u root bash < target-bin/upgrade-system.sh >> var/install.log 2>&1"
  end
  info "Creating package manifest"
  system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest"

  info "Creating build script (var/build-script)"

  File.open("var/build-script", "w") do |script|
    script.puts "#!/bin/bash"
    script.puts "set -e"
    script.puts "export LANG='en_US.UTF-8'"
    script.puts "export LC_ALL='en_US.UTF-8'"
    script.puts "umask 002"
    script.puts "export OUTDIR=$HOME/out"
    script.puts "GBUILD_BITS=#{bits}"
    if build_desc["enable_cache"]
      script.puts "GBUILD_CACHE_ENABLED=1"
      script.puts "GBUILD_PACKAGE_CACHE=$HOME/cache/#{build_desc["name"]}"
      script.puts "GBUILD_COMMON_CACHE=$HOME/cache/common"
    end
    script.puts "MAKEOPTS=(-j#{@options[:num_procs]})"
    script.puts
    author_date = nil
    build_desc["remotes"].each do |remote|
      dir = sanitize(remote["dir"], remote["dir"])

      author_date = `cd inputs/#{dir} && git log --format=@%at -1 | date +"%F %T" -u -f -`.strip
      raise "error looking up author date in #{dir}" unless $?.exitstatus == 0

      system! "copy-to-target #{@quiet_flag} inputs/#{dir} build/"
      script.puts "(cd build/#{dir} && git reset -q --hard && git clean -q -f -d)"
    end
    script.puts
    ref_datetime = build_desc["reference_datetime"] || author_date
    (ref_date, ref_time) = ref_datetime.split
    script.puts "REFERENCE_DATETIME='#{ref_datetime}'"
    script.puts "REFERENCE_DATE='#{ref_date}'"
    script.puts "REFERENCE_TIME='#{ref_time}'"
    script.puts
    script.puts "cd build"
    script.puts build_desc["script"]
  end

  info "Running build script (log in var/build.log)"
  system! "on-target setarch #{@arches[arch]} bash -x < var/build-script > var/build.log 2>&1"
end

################################

OptionParser.new do |opts|
  opts.banner = "Usage: build [options] <build-description>.yml"

  opts.on("--allow-sudo", "override SECURITY on the target VM and allow the use of sudo with no password for the default user") do |v|
    @options[:allow_sudo] = v
  end
  opts.on("-i", "--skip-image", "reuse current target image") do |v|
    @options[:skip_image] = v
  end
  opts.on("--upgrade", "upgrade guest with latest packages") do |v|
    @options[:upgrade] = v
  end
  opts.on("-q", "--quiet", "be quiet") do |v|
    @options[:quiet] = v
  end
  opts.on("-j PROCS", "--num-make PROCS", "number of processes to use") do |v|
    @options[:num_procs] = v
  end
  opts.on("-m MEM", "--memory MEM", "memory to allocate in MiB") do |v|
    @options[:memory] = v
  end
  opts.on("-c PAIRS", "--commit PAIRS", "comma separated list of DIRECTORY=COMMIT pairs") do |v|
    @options[:commit] = v
  end
  opts.on("-u PAIRS", "--url PAIRS", "comma separated list of DIRECTORY=URL pairs") do |v|
    @options[:url] = v
  end
  opts.on("-o", "--cache-read-only", "only use existing cache files, do not update them") do |v|
    @options[:cache_ro] = v
  end
end.parse!

if !ENV["USE_LXC"] and !ENV["USE_DOCKER"] and !ENV["USE_VBOX"] and !File.exist?("/dev/kvm")
    $stderr.puts "\n************* WARNING: kvm not loaded, this will probably not work out\n\n"
end

base_dir = Pathname.new(__FILE__).expand_path.dirname.parent
libexec_dir = base_dir + 'libexec'

ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH']
ENV['GITIAN_BASE'] = base_dir.to_s
ENV['NPROCS'] = @options[:num_procs].to_s
ENV['VMEM'] = @options[:memory].to_s
@quiet_flag = @options[:quiet] ? "-q" : ""

build_desc_file = ARGV.shift or raise "must supply YAML build description file"

build_desc = YAML.load_file(build_desc_file)

in_sums = []

build_dir = 'build'
result_dir = 'result'
cache_dir = 'cache'
enable_cache = build_desc["enable_cache"]

FileUtils.rm_rf(build_dir)
FileUtils.mkdir(build_dir)
FileUtils.mkdir_p(result_dir)

package_name = build_desc["name"] or raise "must supply name"
package_name = sanitize(package_name, "package name")

if enable_cache
  FileUtils.mkdir_p(File.join(cache_dir, "common"))
  FileUtils.mkdir_p(File.join(cache_dir, package_name))
end

distro = build_desc["distro"] || "ubuntu"
suites = build_desc["suites"] or raise "must supply suites"
archs = build_desc["architectures"] or raise "must supply architectures"
build_desc["reference_datetime"] or build_desc["remotes"].size > 0 or raise "must supply `reference_datetime` or `remotes`"
docker_image_digests = build_desc["docker_image_digests"] || []

# if docker_image_digests are supplied, it must be the same length as suites
if docker_image_digests.size > 0 and suites.size != docker_image_digests.size
  raise "`suites` and `docker_image_digests` must both be the same size if both are supplied"
elsif ENV["USE_DOCKER"] and docker_image_digests.size > 0 and suites.size == docker_image_digests.size
  suites = docker_image_digests
end

ENV['DISTRO'] = distro

desc_sum = `sha256sum #{build_desc_file}`
desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
in_sums << desc_sum

build_desc["files"].each do |filename|
  filename = sanitize(filename, "files section")
  in_sums << `cd inputs && sha256sum #{filename}`
end

commits = {}

if @options[:commit]
  @options[:commit].split(',').each do |pair|
    (dir, commit) = pair.split('=')
    commits[dir] = commit
  end
end

urls = {}

if @options[:url]
  @options[:url].split(',').each do |pair|
    (dir, url) = pair.split('=')
    urls[dir] = url
  end
end

build_desc["remotes"].each do |remote|
  if !remote["commit"]
    remote["commit"] = commits[remote["dir"]]
    raise "must specify a commit for directory #{remote["dir"]}" unless remote["commit"]
  end
  if urls[remote["dir"]]
    remote["url"] = urls[remote["dir"]]
  end
  dir = sanitize(remote["dir"], remote["dir"])
  commit = sanitize(remote["commit"], remote["commit"])
  unless File.exist?("inputs/#{dir}")
    system!("git init inputs/#{dir}")
    system!("cd inputs/#{dir} && git fetch --update-head-ok #{sanitize_path(remote["url"], remote["url"])} +refs/tags/*:refs/tags/* +refs/heads/*:refs/heads/*")
    system!("cd inputs/#{dir} && git checkout -q #{commit}")
    system!("cd inputs/#{dir} && git submodule update --init --recursive --force")
  end
  commit = `cd inputs/#{dir} && git log --format=%H -1 #{commit}`.strip
  raise "error looking up commit for tag #{remote["commit"]}" unless $?.exitstatus == 0
  in_sums << "git:#{commit} #{dir}"
end

base_manifests = YAML::Omap.new

suites.each do |suite|
  suite = sanitize(suite, "suite")
  archs.each do |arch|
    info "--- Building for #{suite} #{arch} ---"
    arch = sanitize(arch, "architecture")

    # Build!
    build_one_configuration(suite, arch, build_desc)

    info "Grabbing results from target"
    system! "copy-from-target #{@quiet_flag} out #{build_dir}"

    if enable_cache && !@options[:cache_ro]
      info "Grabbing cache from target"
      system! "copy-from-target #{@quiet_flag} cache/#{package_name}/ #{cache_dir}"
      system! "copy-from-target #{@quiet_flag} cache/common/ #{cache_dir}"
    end
    base_manifest = File.read("var/base-#{suite}-#{arch}.manifest")
    base_manifests["#{suite}-#{arch}"] = base_manifest
  end
end

out_dir = File.join(build_dir, "out")
out_sums = {}
cache_common_dir = File.join(cache_dir, "common")
cache_package_dir = File.join(cache_dir, "#{package_name}")
cache_common_sums = {}
cache_package_sums = {}

info "Generating report"
Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  next if File.directory?(file_in_out)
  file = file_in_out.sub(out_dir + File::SEPARATOR, '')
  file = sanitize_path(file, file_in_out)
  out_sums[file] = `cd #{out_dir} && sha256sum #{file}`
  raise "failed to sum #{file}" unless $? == 0
  puts out_sums[file] unless @options[:quiet]
end

if enable_cache
  Dir.glob(File.join(cache_common_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
    next if File.directory?(file_in_out)
    file = file_in_out.sub(cache_common_dir + File::SEPARATOR, '')
    file = sanitize_path(file, file_in_out)
    cache_common_sums[file] = `cd #{cache_common_dir} && sha256sum #{file}`
    raise "failed to sum #{file}" unless $? == 0
  end

  Dir.glob(File.join(cache_package_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
    next if File.directory?(file_in_out)
    file = file_in_out.sub(cache_package_dir + File::SEPARATOR, '')
    file = sanitize_path(file, file_in_out)
    cache_package_sums[file] = `cd #{cache_package_dir} && sha256sum #{file}`
    raise "failed to sum #{file}" unless $? == 0
  end
end

out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')

in_manifest = in_sums.join('')

cache_common_manifest = cache_common_sums.keys.sort.map { |key| cache_common_sums[key] }.join('')
cache_package_manifest = cache_package_sums.keys.sort.map { |key| cache_package_sums[key] }.join('')

# Use Omap to keep result deterministic
report = YAML::Omap[
  'out_manifest', out_manifest,
  'in_manifest', in_manifest,
  'base_manifests', base_manifests,
  'cache_common_manifest', cache_common_manifest,
  'cache_package_manifest', cache_package_manifest,
]

result_file = "#{package_name}-res.yml"
File.open(File.join(result_dir, result_file), "w") do |io|
  io.write report.to_yaml
end

system!("cd #{result_dir} && sha256sum #{result_file}") unless @options[:quiet]

info "Done."