Преглед изворни кода

multi-architecture builds

tags/0.1
devrandom пре 10 година
родитељ
комит
928a5f0880
6 измењених фајлова са 183 додато и 125 уклоњено
  1. 13
    0
      README.md
  2. 109
    70
      bin/gbuild
  3. 0
    50
      bin/gitian
  4. 55
    2
      bin/make-base-vm
  5. 4
    1
      libexec/start-target
  6. 2
    2
      target-bin/init-build.sh

+ 13
- 0
README.md Прегледај датотеку

@@ -26,3 +26,16 @@ Then execute the build using a YAML description file (can be run as non-root):
bin/gbuild <package>-desc.yml

The resulting report will appear in result/\<package\>-res.yml

## Poking around

* Log files are captured to the _var_ directory
* You can run the utilities in libexec by running `PATH="libexec:$PATH"`
* To start the target VM run `start-target`
* To ssh into the target run `on-target` or `on-target -u root`
* On the target, the _build_ directory contains the code as it is compiled and _install_ contains intermediate libraries
* By convention, the script in \<package\>-desc.yml starts with any environment setup you would need to manually compile things on the target

TODO:
- disable sudo in target, just in case of a hypervisor exploit
- tar and other archive timestamp setter

+ 109
- 70
bin/gbuild Прегледај датотеку

@@ -7,6 +7,11 @@ require 'pathname'

@options = {}

@bitness = {
'i386' => 32,
'amd64' => 64,
}

def system!(cmd)
system(cmd) or raise "failed to run #{cmd}"
end
@@ -16,10 +21,85 @@ def sanitize(str, where)
str
end

def sanitize_path(str, where)
raise "unsanitary string in #{where}" if (str =~ /[^\w\/.-]/)
str
end

def info(str)
puts str unless @options[:quiet]
end

def build_one_configuration(suite, arch, build_desc, reference_datetime)
bits = @bitness[arch] or raise "unknown architecture ${arch}"
suitearch = "#{suite}-#{arch}"

info "Stopping target if it is up"
system "stop-target"

sleep 1

unless @options[:skip_image]
info "Making a new image copy"
system! "cp base-#{suitearch}.qcow2 target-#{suitearch}.qcow2"
end

info "Starting target"
system! "start-target #{bits} #{suitearch}&"

$stdout.write "Checking if target is up"

(1..10).each do
system "on-target true 2> /dev/null" and break
sleep 2
$stdout.write '.'
end

info ''

system! "on-target true"

info "Preparing build environment"
system! "on-target bash < target-bin/init-build.sh"

build_desc["files"].each do |filename|
filename = sanitize(filename, "files section")
system! "cd inputs && copy-to-target #{filename} build/"
end

info "Installing additional packages (log in var/install.log)"
system! "on-target -u root apt-get -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"

info "Grabbing package manifest"
system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest"

info "Creating build script (var/build-script)"

File.open("var/build-script", "w") do |script|
script.puts "#!/bin/bash"
script.puts "set -e"
script.puts "export OUTDIR=$HOME/out"
script.puts "GBUILD_BITS=#{bits}"
script.puts "MAKEOPTS=(-j2)"
(ref_date, ref_time) = reference_datetime.split
script.puts "REFERENCE_DATETIME='#{reference_datetime}'"
script.puts "REFERENCE_DATE='#{ref_date}'"
script.puts "REFERENCE_TIME='#{ref_time}'"
script.puts
build_desc["remotes"].each do |remote|
script.puts "git clone -q #{remote["url"]} build/#{remote["dir"]}"
script.puts "(cd build/#{remote["dir"]} && git checkout -q #{remote["commit"]})"
end
script.puts "cd build"
script.puts build_desc["script"]
end

info "Running build script (log in var/build.log)"
system! "on-target bash < var/build-script > var/build.log 2>&1"
end

################################

OptionParser.new do |opts|
opts.banner = "Usage: build [options] <build-description>.yml"

@@ -39,18 +119,9 @@ ENV['GITIAN_BASE'] = base_dir.to_s

build_desc_file = ARGV.shift or raise "must supply YAML build description file"

in_sums = []

build_desc = YAML.load_file(build_desc_file)

package_name = build_desc["name"] or raise "must supply name"
package_name = sanitize(package_name, "package name")

desc_sum = `sha256sum #{build_desc_file}`
desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
in_sums << desc_sum

reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime"
in_sums = []

build_dir = 'build'
result_dir = 'result'
@@ -59,79 +130,49 @@ FileUtils.rm_rf(build_dir)
FileUtils.mkdir(build_dir)
FileUtils.mkdir_p(result_dir)

info "Stopping target if it is up"
system "stop-target"

sleep 1

unless @options[:skip_image]
info "Making a new image copy"
system! "cp base.qcow2 target.qcow2"
end
package_name = build_desc["name"] or raise "must supply name"
package_name = sanitize(package_name, "package name")

info "Starting target"
system! "start-target &"
suites = build_desc["suites"] or raise "must supply suites"
archs = build_desc["architectures"] or raise "must supply architectures"
reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime"

$stdout.write "Checking if target is up"
desc_sum = `sha256sum #{build_desc_file}`
desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
in_sums << desc_sum

(1..10).each do
system "on-target true 2> /dev/null" and break
sleep 2
$stdout.write '.'
build_desc["files"].each do |filename|
filename = sanitize(filename, "files section")
in_sums << `cd inputs && sha256sum #{filename}`
end

info ''
base_manifests = YAML::Omap.new

system! "on-target true"
suites.each do |suite|
suite = sanitize(suite, "suite")
archs.each do |arch|
info "--- Building for #{suite} #{arch} ---"
arch = sanitize(arch, "architecture")

info "Preparing build environment"
system! "on-target bash < target-bin/init-build.sh"
# Build!
build_one_configuration(suite, arch, build_desc, reference_datetime)

build_desc["files"].each do |filename|
filename = sanitize(filename, "files section")
system! "cd inputs && copy-to-target #{filename} build/"
in_sums << `cd inputs && sha256sum #{filename}`
end
info "Grabbing results"
system! "copy-from-target out #{build_dir}"

info "Installing additional packages (log in var/install.log)"
system! "on-target -u root apt-get -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"

info "Grabbing package manifest"
system! "on-target -u root bash < target-bin/grab-packages.sh > var/base.manifest"

info "Creating build script (var/build-script)"

File.open("var/build-script", "w") do |script|
script.puts "#!/bin/bash"
script.puts "set -e"
script.puts "export OUTDIR=$HOME/out"
script.puts "MAKEOPTS=(-j2)"
(ref_date, ref_time) = reference_datetime.split
script.puts "REFERENCE_DATETIME='#{reference_datetime}'"
script.puts "REFERENCE_DATE='#{ref_date}'"
script.puts "REFERENCE_TIME='#{ref_time}'"
script.puts
build_desc["remotes"].each do |remote|
script.puts "git clone -q #{remote["url"]} build/#{remote["dir"]}"
script.puts "(cd build/#{remote["dir"]} && git checkout -q #{remote["commit"]})"
base_manifest = File.read("var/base-#{suite}-#{arch}.manifest")
base_manifests["#{suite}-#{arch}"] = base_manifest
end
script.puts "cd build"
script.puts build_desc["script"]
end

info "Running build script (log in var/build.log)"
system! "on-target bash < var/build-script > var/build.log 2>&1"

info "Grabbing results"
system! "copy-from-target out #{build_dir}"

out_dir = File.join(build_dir, "out")
out_sums = {}

info "Generating report"
Dir.new(out_dir).each do |file|
next if file.start_with?(".")
file = sanitize(file, out_dir)
Dir.glob(File.join(out_dir, '**', '*')).sort.each do |file_in_out|
next if File.directory?(file_in_out)
file = file_in_out.sub(out_dir + File::SEPARATOR, '')
file = sanitize_path(file, file_in_out)
out_sums[file] = `cd #{out_dir} && sha256sum #{file}`
raise "failed to sum #{file}" unless $? == 0
puts out_sums[file] unless @options[:quiet]
@@ -139,15 +180,13 @@ end

out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')

base_manifest = File.read('var/base.manifest')

in_manifest = in_sums.join('')

# Use Omap to keep result deterministic
report = YAML::Omap[
'out_manifest', out_manifest,
'in_manifest', in_manifest,
'base_manifest', base_manifest,
'base_manifests', base_manifests,
]

result_file = "#{package_name}-res.yml"

+ 0
- 50
bin/gitian Прегледај датотеку

@@ -1,50 +0,0 @@
#!/usr/bin/python

import subprocess
import sys
import os
sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), "../lib"))
sys.path.append('/usr/lib/gitian')
from gitian_util import *

def check_command(command):
if commands.get(command) is None:
print>>sys.stderr, "usage: %s CMD\ntry: %s help"%(prog, prog)
exit(1)
return find_command(command)

args = sys.argv[:]
prog = args.pop(0)

if len(args) < 1:
print>>sys.stderr, "usage: %s CMD\n\ntry:\n %s help\nor:\n %s help CMD"%(prog, prog, prog)
exit(1)

commands = {
"release-build": "Build all packages into the 'dist' directory",
"package-build": "Build a single package into the 'dist' directory",
"package-new": "Insert a new package into the distribution",
"release-upload": "Upload a release to a web server",
}
if args[0] == "help":
if len(args) == 1:
for command in commands.keys():
print command, " - ", commands[command]
else:
command = args[1]
command_path = find_command(command)
ret = subprocess.call([command_path, "-h"])
elif args[0] == 'shell-complete':
if len(args) == 1 or args[1] == "help":
for command in commands.keys():
print "%s:%s"%(command, commands[command])
else:
command = args[1]
command_path = find_command(command)
ret = subprocess.call([command_path, "--shell-complete"])
else:
command = args.pop(0)
command_path = find_command(command)
args.insert(0, command_path)
os.execv(command_path, args)


+ 55
- 2
bin/make-base-vm Прегледај датотеку

@@ -5,10 +5,63 @@ SUITE=lucid
ARCH=amd64
MIRROR=http://${MIRROR_HOST:-`hostname`}:3142/archive.ubuntu.com/ubuntu

usage() {
echo "Usage: ${0##*/} [OPTION]... <command>"
echo "Make a base client."
echo
cat << EOF
--help display this help and exit
--suite=U build suite U instead of lucid
--arch=A build architecture A (e.g. i386) instead of amd64

The MIRROR_HOST environment variable can be used to change the
apt-cacher host. It should be something that the target VM can
resolve (not localhost).
EOF
}

if [ $# != 0 ] ; then
while true ; do
case "$1" in
--help|-h)
usage
exit 0
;;
--suite|-s)
SUITE="$2"
shift 2
;;
--arch|-a)
ARCH="$2"
shift 2
;;
--*)
echo "unrecognized option $1"
exit 1
;;
*)
break
;;
esac
done
fi

mkdir -p var

if [ ! -e var/id_dsa ]; then
ssh-keygen -t dsa -f var/id_dsa -N ""
fi
sudo vmbuilder kvm ubuntu --arch=$ARCH --suite=$SUITE --addpkg=openssh-server,pciutils,build-essential,git-core,subversion --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=$MIRROR --dest=base --flavour=virtual --overwrite
mv base/*.qcow2 base.qcow2

OUT=base-$SUITE-$ARCH
if [ -e $OUT.qcow2 ]; then
echo $OUT.qcow2 already exists, please remove it first
exit 1
fi

rm -rf $OUT

sudo vmbuilder kvm ubuntu --arch=$ARCH --suite=$SUITE --addpkg=openssh-server,pciutils,build-essential,git-core,subversion --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=$MIRROR --dest=$OUT --flavour=virtual --overwrite

mv $OUT/*.qcow2 $OUT.qcow2

rm -rf $OUT

+ 4
- 1
libexec/start-target Прегледај датотеку

@@ -2,7 +2,10 @@

. gconfig

kvm -m 2000 -smp 2 -drive file=target.qcow2 -net nic,model=virtio -net user,hostfwd=tcp::$VM_SSH_PORT-:22 -vnc :16 > var/target.log 2>&1 &
ARCH=qemu$1
SUFFIX=$2

kvm -cpu $ARCH -m 2000 -smp 2 -drive file=target-$SUFFIX.qcow2 -net nic,model=virtio -net user,hostfwd=tcp::$VM_SSH_PORT-:22 -vnc :16 > var/target.log 2>&1 &

echo $! > var/target.pid
wait

+ 2
- 2
target-bin/init-build.sh Прегледај датотеку

@@ -1,6 +1,6 @@
#!/bin/sh

rm -rf inst out build
rm -rf install out build
mkdir build
mkdir out
mkdir inst
mkdir install

Loading…
Откажи
Сачувај