multi-architecture builds

README.md

@@ -26,3 +26,16 @@ Then execute the build using a YAML description file (can be run as non-root):
     bin/gbuild <package>-desc.yml
 
 The resulting report will appear in result/\<package\>-res.yml
+
+## Poking around
+
+* Log files are captured to the _var_ directory
+* You can run the utilities in libexec by running `PATH="libexec:$PATH"`
+* To start the target VM run `start-target`
+* To ssh into the target run `on-target` or `on-target -u root`
+* On the target, the _build_ directory contains the code as it is compiled and _install_ contains intermediate libraries
+* By convention, the script in \<package\>-desc.yml starts with any environment setup you would need to manually compile things on the target
+
+TODO:
+- disable sudo in target, just in case of a hypervisor exploit
+- tar and other archive timestamp setter

bin/gbuild

@@ -7,6 +7,11 @@ require 'pathname'
 
 @options = {}
 
+@bitness = {
+  'i386' => 32,
+  'amd64' => 64,
+}
+
 def system!(cmd)
   system(cmd) or raise "failed to run #{cmd}"
 end
@@ -16,10 +21,85 @@ def sanitize(str, where)
   str
 end
 
+def sanitize_path(str, where)
+  raise "unsanitary string in #{where}" if (str =~ /[^\w\/.-]/)
+  str
+end
+
 def info(str)
   puts str unless @options[:quiet]
 end
 
+def build_one_configuration(suite, arch, build_desc, reference_datetime)
+  bits = @bitness[arch] or raise "unknown architecture ${arch}"
+  suitearch = "#{suite}-#{arch}"
+
+  info "Stopping target if it is up"
+  system "stop-target"
+
+  sleep 1
+
+  unless @options[:skip_image]
+    info "Making a new image copy"
+    system! "cp base-#{suitearch}.qcow2 target-#{suitearch}.qcow2"
+  end
+
+  info "Starting target"
+  system! "start-target #{bits} #{suitearch}&"
+
+  $stdout.write "Checking if target is up"
+
+  (1..10).each do
+    system "on-target true 2> /dev/null" and break
+    sleep 2
+    $stdout.write '.'
+  end
+
+  info ''
+
+  system! "on-target true"
+
+  info "Preparing build environment"
+  system! "on-target bash < target-bin/init-build.sh"
+
+  build_desc["files"].each do |filename|
+    filename = sanitize(filename, "files section")
+    system! "cd inputs && copy-to-target #{filename} build/"
+  end
+
+  info "Installing additional packages (log in var/install.log)"
+  system! "on-target -u root apt-get -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"
+
+  info "Grabbing package manifest"
+  system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest"
+
+  info "Creating build script (var/build-script)"
+
+  File.open("var/build-script", "w") do |script|
+    script.puts "#!/bin/bash"
+    script.puts "set -e"
+    script.puts "export OUTDIR=$HOME/out"
+    script.puts "GBUILD_BITS=#{bits}"
+    script.puts "MAKEOPTS=(-j2)"
+    (ref_date, ref_time) = reference_datetime.split
+    script.puts "REFERENCE_DATETIME='#{reference_datetime}'"
+    script.puts "REFERENCE_DATE='#{ref_date}'"
+    script.puts "REFERENCE_TIME='#{ref_time}'"
+    script.puts
+    build_desc["remotes"].each do |remote|
+      script.puts "git clone -q #{remote["url"]} build/#{remote["dir"]}"
+      script.puts "(cd build/#{remote["dir"]} && git checkout -q #{remote["commit"]})"
+    end
+    script.puts "cd build"
+    script.puts build_desc["script"]
+  end
+
+  info "Running build script (log in var/build.log)"
+  system! "on-target bash < var/build-script > var/build.log 2>&1"
+end
+
+################################
+
 OptionParser.new do |opts|
   opts.banner = "Usage: build [options] <build-description>.yml"
 
@@ -39,18 +119,9 @@ ENV['GITIAN_BASE'] = base_dir.to_s
 
 build_desc_file = ARGV.shift or raise "must supply YAML build description file"
 
-in_sums = []
-
 build_desc = YAML.load_file(build_desc_file)
 
-package_name = build_desc["name"] or raise "must supply name"
-package_name = sanitize(package_name, "package name")
-
-desc_sum = `sha256sum #{build_desc_file}`
-desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
-in_sums << desc_sum
-
-reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime"
+in_sums = []
 
 build_dir = 'build'
 result_dir = 'result'
@@ -59,79 +130,49 @@ FileUtils.rm_rf(build_dir)
 FileUtils.mkdir(build_dir)
 FileUtils.mkdir_p(result_dir)
 
-info "Stopping target if it is up"
-system "stop-target"
-
-sleep 1
-
-unless @options[:skip_image]
-  info "Making a new image copy"
-  system! "cp base.qcow2 target.qcow2"
-end
+package_name = build_desc["name"] or raise "must supply name"
+package_name = sanitize(package_name, "package name")
 
-info "Starting target"
-system! "start-target &"
+suites = build_desc["suites"] or raise "must supply suites"
+archs = build_desc["architectures"] or raise "must supply architectures"
+reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime"
 
-$stdout.write "Checking if target is up"
+desc_sum = `sha256sum #{build_desc_file}`
+desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
+in_sums << desc_sum
 
-(1..10).each do
-  system "on-target true 2> /dev/null" and break
-  sleep 2
-  $stdout.write '.'
+build_desc["files"].each do |filename|
+  filename = sanitize(filename, "files section")
+  in_sums << `cd inputs && sha256sum #{filename}`
 end
 
-info ''
+base_manifests = YAML::Omap.new
 
-system! "on-target true"
+suites.each do |suite|
+  suite = sanitize(suite, "suite")
+  archs.each do |arch|
+    info "--- Building for #{suite} #{arch} ---"
+    arch = sanitize(arch, "architecture")
 
-info "Preparing build environment"
-system! "on-target bash < target-bin/init-build.sh"
+    # Build!
+    build_one_configuration(suite, arch, build_desc, reference_datetime)
 
-build_desc["files"].each do |filename|
-  filename = sanitize(filename, "files section")
-  system! "cd inputs && copy-to-target #{filename} build/"
-  in_sums << `cd inputs && sha256sum #{filename}`
-end
+    info "Grabbing results"
+    system! "copy-from-target out #{build_dir}"
 
-info "Installing additional packages (log in var/install.log)"
-system! "on-target -u root apt-get -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"
-
-info "Grabbing package manifest"
-system! "on-target -u root bash < target-bin/grab-packages.sh > var/base.manifest"
-
-info "Creating build script (var/build-script)"
-
-File.open("var/build-script", "w") do |script|
-  script.puts "#!/bin/bash"
-  script.puts "set -e"
-  script.puts "export OUTDIR=$HOME/out"
-  script.puts "MAKEOPTS=(-j2)"
-  (ref_date, ref_time) = reference_datetime.split
-  script.puts "REFERENCE_DATETIME='#{reference_datetime}'"
-  script.puts "REFERENCE_DATE='#{ref_date}'"
-  script.puts "REFERENCE_TIME='#{ref_time}'"
-  script.puts
-  build_desc["remotes"].each do |remote|
-    script.puts "git clone -q #{remote["url"]} build/#{remote["dir"]}"
-    script.puts "(cd build/#{remote["dir"]} && git checkout -q #{remote["commit"]})"
+    base_manifest = File.read("var/base-#{suite}-#{arch}.manifest")
+    base_manifests["#{suite}-#{arch}"] = base_manifest
   end
-  script.puts "cd build"
-  script.puts build_desc["script"]
 end
 
-info "Running build script (log in var/build.log)"
-system! "on-target bash < var/build-script > var/build.log 2>&1"
-
-info "Grabbing results"
-system! "copy-from-target out #{build_dir}"
-
 out_dir = File.join(build_dir, "out")
 out_sums = {}
 
 info "Generating report"
-Dir.new(out_dir).each do |file|
-  next if file.start_with?(".")
-  file = sanitize(file, out_dir)
+Dir.glob(File.join(out_dir, '**', '*')).sort.each do |file_in_out|
+  next if File.directory?(file_in_out)
+  file = file_in_out.sub(out_dir + File::SEPARATOR, '')
+  file = sanitize_path(file, file_in_out)
   out_sums[file] = `cd #{out_dir} && sha256sum #{file}`
   raise "failed to sum #{file}" unless $? == 0
   puts out_sums[file] unless @options[:quiet]
@@ -139,15 +180,13 @@ end
 
 out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
 
-base_manifest = File.read('var/base.manifest')
-
 in_manifest = in_sums.join('')
 
 # Use Omap to keep result deterministic
 report = YAML::Omap[
   'out_manifest', out_manifest,
   'in_manifest', in_manifest,
-  'base_manifest', base_manifest,
+  'base_manifests', base_manifests,
 ]
 
 result_file = "#{package_name}-res.yml"

bin/gitian

@@ -1,50 +0,0 @@
-#!/usr/bin/python
-
-import subprocess
-import sys
-import os
-sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), "../lib"))
-sys.path.append('/usr/lib/gitian')
-from gitian_util import *
-
-def check_command(command):
-    if commands.get(command) is None:
-        print>>sys.stderr, "usage: %s CMD\ntry: %s help"%(prog, prog)
-        exit(1)
-    return find_command(command)
-
-args = sys.argv[:]
-prog = args.pop(0)
-
-if len(args) < 1:
-    print>>sys.stderr, "usage: %s CMD\n\ntry:\n  %s help\nor:\n  %s help CMD"%(prog, prog, prog)
-    exit(1)
-
-commands = {
-    "release-build": "Build all packages into the 'dist' directory",
-    "package-build": "Build a single package into the 'dist' directory",
-    "package-new": "Insert a new package into the distribution",
-    "release-upload": "Upload a release to a web server",
-}
-if args[0] == "help":
-    if len(args) == 1:
-        for command in commands.keys():
-            print command, " - ", commands[command]
-    else:
-        command = args[1]
-        command_path = find_command(command)
-        ret = subprocess.call([command_path, "-h"])
-elif args[0] == 'shell-complete':
-    if len(args) == 1 or args[1] == "help":
-        for command in commands.keys():
-            print "%s:%s"%(command, commands[command])
-    else:
-        command = args[1]
-        command_path = find_command(command)
-        ret = subprocess.call([command_path, "--shell-complete"])
-else:
-    command = args.pop(0)
-    command_path = find_command(command)
-    args.insert(0, command_path)
-    os.execv(command_path, args)
-

bin/make-base-vm

@@ -5,10 +5,63 @@ SUITE=lucid
 ARCH=amd64
 MIRROR=http://${MIRROR_HOST:-`hostname`}:3142/archive.ubuntu.com/ubuntu
 
+usage() {
+  echo "Usage: ${0##*/} [OPTION]... <command>"
+  echo "Make a base client."
+  echo
+  cat << EOF
+  --help     display this help and exit
+  --suite=U  build suite U instead of lucid
+  --arch=A   build architecture A (e.g. i386) instead of amd64
+
+  The MIRROR_HOST environment variable can be used to change the
+  apt-cacher host.  It should be something that the target VM can
+  resolve (not localhost).
+EOF
+}
+
+if [ $# != 0 ] ; then
+  while true ; do
+    case "$1" in
+      --help|-h)
+        usage
+        exit 0
+        ;;
+      --suite|-s)
+        SUITE="$2"
+        shift 2
+        ;;
+      --arch|-a)
+        ARCH="$2"
+        shift 2
+        ;;
+      --*)
+        echo "unrecognized option $1"
+        exit 1
+        ;;
+      *)
+        break
+        ;;
+    esac
+  done
+fi
+
 mkdir -p var
 
 if [ ! -e var/id_dsa ]; then
   ssh-keygen -t dsa -f var/id_dsa -N ""
 fi
-sudo vmbuilder kvm ubuntu --arch=$ARCH --suite=$SUITE --addpkg=openssh-server,pciutils,build-essential,git-core,subversion --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=$MIRROR --dest=base --flavour=virtual --overwrite
-mv base/*.qcow2 base.qcow2
+
+OUT=base-$SUITE-$ARCH
+if [ -e $OUT.qcow2 ]; then
+  echo $OUT.qcow2 already exists, please remove it first
+  exit 1
+fi
+
+rm -rf $OUT
+
+sudo vmbuilder kvm ubuntu --arch=$ARCH --suite=$SUITE --addpkg=openssh-server,pciutils,build-essential,git-core,subversion --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=$MIRROR --dest=$OUT --flavour=virtual --overwrite
+
+mv $OUT/*.qcow2 $OUT.qcow2
+
+rm -rf $OUT

libexec/start-target

@@ -2,7 +2,10 @@
 
 . gconfig
 
-kvm -m 2000 -smp 2 -drive file=target.qcow2 -net nic,model=virtio -net user,hostfwd=tcp::$VM_SSH_PORT-:22 -vnc :16 > var/target.log 2>&1 &
+ARCH=qemu$1
+SUFFIX=$2
+
+kvm -cpu $ARCH -m 2000 -smp 2 -drive file=target-$SUFFIX.qcow2 -net nic,model=virtio -net user,hostfwd=tcp::$VM_SSH_PORT-:22 -vnc :16 > var/target.log 2>&1 &
 
 echo $! > var/target.pid
 wait

target-bin/init-build.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-rm -rf inst out build
+rm -rf install out build
 mkdir build
 mkdir out
-mkdir inst
+mkdir install