commit
4db2a9dfa0
13 changed files with 428 additions and 0 deletions
@ -0,0 +1,8 @@
@@ -0,0 +1,8 @@
|
||||
*-stamp |
||||
*.substvars |
||||
*.log |
||||
*.pyc |
||||
build |
||||
var |
||||
result |
||||
inputs |
@ -0,0 +1,25 @@
@@ -0,0 +1,25 @@
|
||||
# Gitian |
||||
|
||||
Read about the project goals at the "project home page":https://gitian.org/ . |
||||
|
||||
This package can do a deterministic build of a package inside a VM. |
||||
|
||||
## Deterministic build inside a VM |
||||
|
||||
This performs a build inside a VM, with deterministic inputs and outputs. If the build script takes care of all sources of non-determinism (mostly caused by timestamps), the result will always be the same. This allows multiple independent verifiers to sign a binary with the assurance that it really came from the source they reviewed. |
||||
|
||||
Synopsis: |
||||
|
||||
* Install prereqs: |
||||
|
||||
sudo apt-get install python-vm-builder qemu-kvm |
||||
|
||||
* This will create the base VM for use in further builds (requires sudo): |
||||
|
||||
bin/make-base-vm |
||||
|
||||
* This will build using a YAML description file (can be run as non-root): |
||||
|
||||
bin/gbuild _package_-desc.yml |
||||
|
||||
The resulting report will appear in result/_package_-res.yml |
@ -0,0 +1,160 @@
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/ruby |
||||
|
||||
require 'optparse' |
||||
require 'yaml' |
||||
require 'fileutils' |
||||
require 'pathname' |
||||
|
||||
@options = {} |
||||
|
||||
def system!(cmd) |
||||
system(cmd) or raise "failed to run #{cmd}" |
||||
end |
||||
|
||||
def sanitize(str, where) |
||||
raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/) |
||||
str |
||||
end |
||||
|
||||
def info(str) |
||||
puts str unless @options[:quiet] |
||||
end |
||||
|
||||
OptionParser.new do |opts| |
||||
opts.banner = "Usage: build [options] <build-description>.yml" |
||||
|
||||
opts.on("-i", "--skip-image", "reuse current target image") do |v| |
||||
@options[:skip_image] = v |
||||
end |
||||
opts.on("-q", "--quiet", "be quiet") do |v| |
||||
@options[:skip_image] = v |
||||
end |
||||
end.parse! |
||||
|
||||
base_dir = Pathname.new(__FILE__).expand_path.dirname.parent |
||||
libexec_dir = base_dir + 'libexec' |
||||
|
||||
ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH'] |
||||
ENV['GITIAN_BASE'] = base_dir.to_s |
||||
|
||||
build_desc_file = ARGV.shift or raise "must supply YAML build description file" |
||||
|
||||
in_sums = [] |
||||
|
||||
build_desc = YAML.load_file(build_desc_file) |
||||
|
||||
package_name = build_desc["name"] or raise "must supply name" |
||||
package_name = sanitize(package_name, "package name") |
||||
|
||||
desc_sum = `sha256sum #{build_desc_file}` |
||||
desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml") |
||||
in_sums << desc_sum |
||||
|
||||
reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime" |
||||
|
||||
build_dir = 'build' |
||||
result_dir = 'result' |
||||
|
||||
FileUtils.rm_rf(build_dir) |
||||
FileUtils.mkdir(build_dir) |
||||
FileUtils.mkdir_p(result_dir) |
||||
|
||||
info "Stopping target if it is up" |
||||
system "stop-target" |
||||
|
||||
sleep 1 |
||||
|
||||
unless @options[:skip_image] |
||||
info "Making a new image copy" |
||||
system! "cp base.qcow2 target.qcow2" |
||||
end |
||||
|
||||
info "Starting target" |
||||
system! "start-target &" |
||||
|
||||
$stdout.write "Checking if target is up" |
||||
|
||||
(1..10).each do |
||||
system "on-target true 2> /dev/null" and break |
||||
sleep 2 |
||||
$stdout.write '.' |
||||
end |
||||
|
||||
info '' |
||||
|
||||
system! "on-target true" |
||||
|
||||
info "Installing additional packages (log in var/install.log)" |
||||
system! "on-target -u root apt-get -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1" |
||||
|
||||
info "Grabbing package manifest" |
||||
system! "on-target -u root bash < target-bin/grab-packages.sh > var/base.manifest" |
||||
|
||||
info "Preparing build environment" |
||||
system! "on-target bash < target-bin/init-build.sh" |
||||
|
||||
build_desc["files"].each do |filename| |
||||
filename = sanitize(filename, "files section") |
||||
system! "cd inputs ; copy-to-target #{filename} build/" |
||||
in_sums << `cd inputs ; sha256sum #{filename}` |
||||
end |
||||
|
||||
info "Creating build script (var/build-script)" |
||||
|
||||
File.open("var/build-script", "w") do |script| |
||||
script.puts "#!/bin/bash" |
||||
script.puts "set -e" |
||||
script.puts "export OUTDIR=$HOME/out" |
||||
script.puts "MAKEOPTS=(-j2)" |
||||
(ref_date, ref_time) = reference_datetime.split |
||||
script.puts "REFERENCE_DATETIME='#{reference_datetime}'" |
||||
script.puts "REFERENCE_DATE='#{ref_date}'" |
||||
script.puts "REFERENCE_TIME='#{ref_time}'" |
||||
script.puts |
||||
build_desc["remotes"].each do |remote| |
||||
script.puts "git clone -q #{remote["url"]} build/#{remote["dir"]}" |
||||
script.puts "(cd build/#{remote["dir"]} ; git checkout -q #{remote["commit"]})" |
||||
end |
||||
script.puts "cd build" |
||||
script.puts build_desc["script"] |
||||
end |
||||
|
||||
info "Running build script (log in var/build.log)" |
||||
system! "on-target bash < var/build-script > var/build.log 2>&1" |
||||
|
||||
info "Grabbing results" |
||||
system! "copy-from-target out #{build_dir}" |
||||
|
||||
out_dir = File.join(build_dir, "out") |
||||
out_sums = {} |
||||
|
||||
info "Generating report" |
||||
Dir.new(out_dir).each do |file| |
||||
next if file.start_with?(".") |
||||
file = sanitize(file, out_dir) |
||||
out_sums[file] = `cd #{out_dir} ; sha256sum #{file}` |
||||
raise "failed to sum #{file}" unless $? == 0 |
||||
puts out_sums[file] unless @options[:quiet] |
||||
end |
||||
|
||||
out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('') |
||||
|
||||
base_manifest = File.read('var/base.manifest') |
||||
|
||||
in_manifest = in_sums.join('') |
||||
|
||||
# Use Omap to keep result deterministic |
||||
report = YAML::Omap[ |
||||
'out_manifest', out_manifest, |
||||
'in_manifest', in_manifest, |
||||
'base_manifest', base_manifest, |
||||
] |
||||
|
||||
result_file = "#{package_name}-res.yml" |
||||
File.open(File.join(result_dir, result_file), "w") do |io| |
||||
io.write report.to_yaml |
||||
end |
||||
|
||||
system!("cd #{result_dir} ; sha256sum #{result_file}") unless @options[:quiet] |
||||
|
||||
info "Done." |
@ -0,0 +1,50 @@
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/python |
||||
|
||||
import subprocess |
||||
import sys |
||||
import os |
||||
sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), "../lib")) |
||||
sys.path.append('/usr/lib/gitian') |
||||
from gitian_util import * |
||||
|
||||
def check_command(command): |
||||
if commands.get(command) is None: |
||||
print>>sys.stderr, "usage: %s CMD\ntry: %s help"%(prog, prog) |
||||
exit(1) |
||||
return find_command(command) |
||||
|
||||
args = sys.argv[:] |
||||
prog = args.pop(0) |
||||
|
||||
if len(args) < 1: |
||||
print>>sys.stderr, "usage: %s CMD\n\ntry:\n %s help\nor:\n %s help CMD"%(prog, prog, prog) |
||||
exit(1) |
||||
|
||||
commands = { |
||||
"release-build": "Build all packages into the 'dist' directory", |
||||
"package-build": "Build a single package into the 'dist' directory", |
||||
"package-new": "Insert a new package into the distribution", |
||||
"release-upload": "Upload a release to a web server", |
||||
} |
||||
if args[0] == "help": |
||||
if len(args) == 1: |
||||
for command in commands.keys(): |
||||
print command, " - ", commands[command] |
||||
else: |
||||
command = args[1] |
||||
command_path = find_command(command) |
||||
ret = subprocess.call([command_path, "-h"]) |
||||
elif args[0] == 'shell-complete': |
||||
if len(args) == 1 or args[1] == "help": |
||||
for command in commands.keys(): |
||||
print "%s:%s"%(command, commands[command]) |
||||
else: |
||||
command = args[1] |
||||
command_path = find_command(command) |
||||
ret = subprocess.call([command_path, "--shell-complete"]) |
||||
else: |
||||
command = args.pop(0) |
||||
command_path = find_command(command) |
||||
args.insert(0, command_path) |
||||
os.execv(command_path, args) |
||||
|
@ -0,0 +1,11 @@
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh |
||||
set -e |
||||
|
||||
SUITE=lucid |
||||
ARCH=amd64 |
||||
|
||||
if [ ! -e var/id_dsa ]; then |
||||
ssh-keygen -t dsa -f var/id_dsa -N "" |
||||
fi |
||||
sudo vmbuilder kvm ubuntu --arch=$ARCH --suite=$SUITE --addpkg=openssh-server,pciutils,build-essential,git-core,mercurial,subversion --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=http://localhost:3142/ubuntu --dest=base --flavour=virtual --overwrite |
||||
mv base/*.qcow2 base.qcow2 |
@ -0,0 +1,44 @@
@@ -0,0 +1,44 @@
|
||||
#!/bin/sh |
||||
|
||||
. gconfig |
||||
|
||||
TUSER=ubuntu |
||||
|
||||
usage() { |
||||
echo "Usage: ${0##*/} [OPTION]... <command>" |
||||
echo "Run command on build target." |
||||
echo |
||||
cat << EOF |
||||
--help display this help and exit |
||||
--user=U run as U instead of ubuntu |
||||
EOF |
||||
} |
||||
|
||||
if [ $# != 0 ] ; then |
||||
while true ; do |
||||
case "$1" in |
||||
--help|-h) |
||||
usage |
||||
exit 0 |
||||
;; |
||||
--user|-u) |
||||
TUSER="$2" |
||||
shift 2 |
||||
;; |
||||
--*) |
||||
echo "unrecognized option $1" |
||||
exit 1 |
||||
;; |
||||
*) |
||||
break |
||||
;; |
||||
esac |
||||
done |
||||
fi |
||||
|
||||
if [ $# = 0 ] ; then |
||||
usage |
||||
exit 1 |
||||
fi |
||||
|
||||
scp -oConnectTimeout=2 -i ${GITIAN_BASE:-.}/var/id_dsa -P $VM_SSH_PORT -r $TUSER@localhost:$1 $2 |
@ -0,0 +1,44 @@
@@ -0,0 +1,44 @@
|
||||
#!/bin/sh |
||||
|
||||
. gconfig |
||||
|
||||
TUSER=ubuntu |
||||
|
||||
usage() { |
||||
echo "Usage: ${0##*/} [OPTION]... <command>" |
||||
echo "Run command on build target." |
||||
echo |
||||
cat << EOF |
||||
--help display this help and exit |
||||
--user=U run as U instead of ubuntu |
||||
EOF |
||||
} |
||||
|
||||
if [ $# != 0 ] ; then |
||||
while true ; do |
||||
case "$1" in |
||||
--help|-h) |
||||
usage |
||||
exit 0 |
||||
;; |
||||
--user|-u) |
||||
TUSER="$2" |
||||
shift 2 |
||||
;; |
||||
--*) |
||||
echo "unrecognized option $1" |
||||
exit 1 |
||||
;; |
||||
*) |
||||
break |
||||
;; |
||||
esac |
||||
done |
||||
fi |
||||
|
||||
if [ $# = 0 ] ; then |
||||
usage |
||||
exit 1 |
||||
fi |
||||
|
||||
scp -oConnectTimeout=2 -i ${GITIAN_BASE:-.}/var/id_dsa -P $VM_SSH_PORT $1 $TUSER@localhost:$2 |
@ -0,0 +1,44 @@
@@ -0,0 +1,44 @@
|
||||
#!/bin/sh |
||||
|
||||
. gconfig |
||||
|
||||
TUSER=ubuntu |
||||
|
||||
usage() { |
||||
echo "Usage: ${0##*/} [OPTION]... <command>" |
||||
echo "Run command on build target." |
||||
echo |
||||
cat << EOF |
||||
--help display this help and exit |
||||
--user=U run as U instead of ubuntu |
||||
EOF |
||||
} |
||||
|
||||
if [ $# != 0 ] ; then |
||||
while true ; do |
||||
case "$1" in |
||||
--help|-h) |
||||
usage |
||||
exit 0 |
||||
;; |
||||
--user|-u) |
||||
TUSER="$2" |
||||
shift 2 |
||||
;; |
||||
--*) |
||||
echo "unrecognized option $1" |
||||
exit 1 |
||||
;; |
||||
*) |
||||
break |
||||
;; |
||||
esac |
||||
done |
||||
fi |
||||
|
||||
#if [ $# = 0 ] ; then |
||||
# usage |
||||
# exit 1 |
||||
#fi |
||||
|
||||
ssh -oConnectTimeout=2 -i ${GITIAN_BASE:-.}/var/id_dsa -p $VM_SSH_PORT $TUSER@localhost $* |
@ -0,0 +1,9 @@
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh |
||||
|
||||
. gconfig |
||||
|
||||
kvm -m 2000 -smp 2 -drive file=target.qcow2 -net nic,model=virtio -net user,hostfwd=tcp::$VM_SSH_PORT-:22 -vnc :16 > var/target.log 2>&1 & |
||||
|
||||
echo $! > var/target.pid |
||||
wait |
||||
rm var/target.pid |
@ -0,0 +1,14 @@
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh |
||||
|
||||
if [ ! -e var/target.pid ]; then exit; fi |
||||
|
||||
on-target -u root halt |
||||
sleep 5 |
||||
|
||||
if [ ! -e var/target.pid ]; then exit; fi |
||||
sleep 5 |
||||
|
||||
if [ ! -e var/target.pid ]; then exit; fi |
||||
|
||||
echo Killing target since it did not shutdown within 10 seconds |
||||
kill `cat var/target.pid` |
@ -0,0 +1,12 @@
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh |
||||
|
||||
# Get an installed package manifest |
||||
|
||||
set -e |
||||
|
||||
cd /var/cache/apt/archives |
||||
|
||||
#apt-get clean |
||||
|
||||
dpkg-query -W -f '${Package}\n' | xargs -n 50 apt-get install --reinstall -y -d > /dev/null |
||||
sha256sum *.deb | sort --key 2 |
Loading…
Reference in new issue