Browse Source

initial

pull/34/head
devrandom 11 years ago
commit
4db2a9dfa0
  1. 8
      .gitignore
  2. 25
      README.md
  3. 160
      bin/gbuild
  4. 50
      bin/gitian
  5. 11
      bin/make-base-vm
  6. 44
      libexec/copy-from-target
  7. 44
      libexec/copy-to-target
  8. 1
      libexec/gconfig
  9. 44
      libexec/on-target
  10. 9
      libexec/start-target
  11. 14
      libexec/stop-target
  12. 12
      target-bin/grab-packages.sh
  13. 6
      target-bin/init-build.sh

8
.gitignore vendored

@ -0,0 +1,8 @@ @@ -0,0 +1,8 @@
*-stamp
*.substvars
*.log
*.pyc
build
var
result
inputs

25
README.md

@ -0,0 +1,25 @@ @@ -0,0 +1,25 @@
# Gitian
Read about the project goals at the "project home page":https://gitian.org/ .
This package can do a deterministic build of a package inside a VM.
## Deterministic build inside a VM
This performs a build inside a VM, with deterministic inputs and outputs. If the build script takes care of all sources of non-determinism (mostly caused by timestamps), the result will always be the same. This allows multiple independent verifiers to sign a binary with the assurance that it really came from the source they reviewed.
Synopsis:
* Install prereqs:
sudo apt-get install python-vm-builder qemu-kvm
* This will create the base VM for use in further builds (requires sudo):
bin/make-base-vm
* This will build using a YAML description file (can be run as non-root):
bin/gbuild _package_-desc.yml
The resulting report will appear in result/_package_-res.yml

160
bin/gbuild

@ -0,0 +1,160 @@ @@ -0,0 +1,160 @@
#!/usr/bin/ruby
require 'optparse'
require 'yaml'
require 'fileutils'
require 'pathname'
@options = {}
def system!(cmd)
system(cmd) or raise "failed to run #{cmd}"
end
def sanitize(str, where)
raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/)
str
end
def info(str)
puts str unless @options[:quiet]
end
OptionParser.new do |opts|
opts.banner = "Usage: build [options] <build-description>.yml"
opts.on("-i", "--skip-image", "reuse current target image") do |v|
@options[:skip_image] = v
end
opts.on("-q", "--quiet", "be quiet") do |v|
@options[:skip_image] = v
end
end.parse!
base_dir = Pathname.new(__FILE__).expand_path.dirname.parent
libexec_dir = base_dir + 'libexec'
ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH']
ENV['GITIAN_BASE'] = base_dir.to_s
build_desc_file = ARGV.shift or raise "must supply YAML build description file"
in_sums = []
build_desc = YAML.load_file(build_desc_file)
package_name = build_desc["name"] or raise "must supply name"
package_name = sanitize(package_name, "package name")
desc_sum = `sha256sum #{build_desc_file}`
desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
in_sums << desc_sum
reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime"
build_dir = 'build'
result_dir = 'result'
FileUtils.rm_rf(build_dir)
FileUtils.mkdir(build_dir)
FileUtils.mkdir_p(result_dir)
info "Stopping target if it is up"
system "stop-target"
sleep 1
unless @options[:skip_image]
info "Making a new image copy"
system! "cp base.qcow2 target.qcow2"
end
info "Starting target"
system! "start-target &"
$stdout.write "Checking if target is up"
(1..10).each do
system "on-target true 2> /dev/null" and break
sleep 2
$stdout.write '.'
end
info ''
system! "on-target true"
info "Installing additional packages (log in var/install.log)"
system! "on-target -u root apt-get -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"
info "Grabbing package manifest"
system! "on-target -u root bash < target-bin/grab-packages.sh > var/base.manifest"
info "Preparing build environment"
system! "on-target bash < target-bin/init-build.sh"
build_desc["files"].each do |filename|
filename = sanitize(filename, "files section")
system! "cd inputs ; copy-to-target #{filename} build/"
in_sums << `cd inputs ; sha256sum #{filename}`
end
info "Creating build script (var/build-script)"
File.open("var/build-script", "w") do |script|
script.puts "#!/bin/bash"
script.puts "set -e"
script.puts "export OUTDIR=$HOME/out"
script.puts "MAKEOPTS=(-j2)"
(ref_date, ref_time) = reference_datetime.split
script.puts "REFERENCE_DATETIME='#{reference_datetime}'"
script.puts "REFERENCE_DATE='#{ref_date}'"
script.puts "REFERENCE_TIME='#{ref_time}'"
script.puts
build_desc["remotes"].each do |remote|
script.puts "git clone -q #{remote["url"]} build/#{remote["dir"]}"
script.puts "(cd build/#{remote["dir"]} ; git checkout -q #{remote["commit"]})"
end
script.puts "cd build"
script.puts build_desc["script"]
end
info "Running build script (log in var/build.log)"
system! "on-target bash < var/build-script > var/build.log 2>&1"
info "Grabbing results"
system! "copy-from-target out #{build_dir}"
out_dir = File.join(build_dir, "out")
out_sums = {}
info "Generating report"
Dir.new(out_dir).each do |file|
next if file.start_with?(".")
file = sanitize(file, out_dir)
out_sums[file] = `cd #{out_dir} ; sha256sum #{file}`
raise "failed to sum #{file}" unless $? == 0
puts out_sums[file] unless @options[:quiet]
end
out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
base_manifest = File.read('var/base.manifest')
in_manifest = in_sums.join('')
# Use Omap to keep result deterministic
report = YAML::Omap[
'out_manifest', out_manifest,
'in_manifest', in_manifest,
'base_manifest', base_manifest,
]
result_file = "#{package_name}-res.yml"
File.open(File.join(result_dir, result_file), "w") do |io|
io.write report.to_yaml
end
system!("cd #{result_dir} ; sha256sum #{result_file}") unless @options[:quiet]
info "Done."

50
bin/gitian

@ -0,0 +1,50 @@ @@ -0,0 +1,50 @@
#!/usr/bin/python
import subprocess
import sys
import os
sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), "../lib"))
sys.path.append('/usr/lib/gitian')
from gitian_util import *
def check_command(command):
if commands.get(command) is None:
print>>sys.stderr, "usage: %s CMD\ntry: %s help"%(prog, prog)
exit(1)
return find_command(command)
args = sys.argv[:]
prog = args.pop(0)
if len(args) < 1:
print>>sys.stderr, "usage: %s CMD\n\ntry:\n %s help\nor:\n %s help CMD"%(prog, prog, prog)
exit(1)
commands = {
"release-build": "Build all packages into the 'dist' directory",
"package-build": "Build a single package into the 'dist' directory",
"package-new": "Insert a new package into the distribution",
"release-upload": "Upload a release to a web server",
}
if args[0] == "help":
if len(args) == 1:
for command in commands.keys():
print command, " - ", commands[command]
else:
command = args[1]
command_path = find_command(command)
ret = subprocess.call([command_path, "-h"])
elif args[0] == 'shell-complete':
if len(args) == 1 or args[1] == "help":
for command in commands.keys():
print "%s:%s"%(command, commands[command])
else:
command = args[1]
command_path = find_command(command)
ret = subprocess.call([command_path, "--shell-complete"])
else:
command = args.pop(0)
command_path = find_command(command)
args.insert(0, command_path)
os.execv(command_path, args)

11
bin/make-base-vm

@ -0,0 +1,11 @@ @@ -0,0 +1,11 @@
#!/bin/sh
set -e
SUITE=lucid
ARCH=amd64
if [ ! -e var/id_dsa ]; then
ssh-keygen -t dsa -f var/id_dsa -N ""
fi
sudo vmbuilder kvm ubuntu --arch=$ARCH --suite=$SUITE --addpkg=openssh-server,pciutils,build-essential,git-core,mercurial,subversion --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=http://localhost:3142/ubuntu --dest=base --flavour=virtual --overwrite
mv base/*.qcow2 base.qcow2

44
libexec/copy-from-target

@ -0,0 +1,44 @@ @@ -0,0 +1,44 @@
#!/bin/sh
. gconfig
TUSER=ubuntu
usage() {
echo "Usage: ${0##*/} [OPTION]... <command>"
echo "Run command on build target."
echo
cat << EOF
--help display this help and exit
--user=U run as U instead of ubuntu
EOF
}
if [ $# != 0 ] ; then
while true ; do
case "$1" in
--help|-h)
usage
exit 0
;;
--user|-u)
TUSER="$2"
shift 2
;;
--*)
echo "unrecognized option $1"
exit 1
;;
*)
break
;;
esac
done
fi
if [ $# = 0 ] ; then
usage
exit 1
fi
scp -oConnectTimeout=2 -i ${GITIAN_BASE:-.}/var/id_dsa -P $VM_SSH_PORT -r $TUSER@localhost:$1 $2

44
libexec/copy-to-target

@ -0,0 +1,44 @@ @@ -0,0 +1,44 @@
#!/bin/sh
. gconfig
TUSER=ubuntu
usage() {
echo "Usage: ${0##*/} [OPTION]... <command>"
echo "Run command on build target."
echo
cat << EOF
--help display this help and exit
--user=U run as U instead of ubuntu
EOF
}
if [ $# != 0 ] ; then
while true ; do
case "$1" in
--help|-h)
usage
exit 0
;;
--user|-u)
TUSER="$2"
shift 2
;;
--*)
echo "unrecognized option $1"
exit 1
;;
*)
break
;;
esac
done
fi
if [ $# = 0 ] ; then
usage
exit 1
fi
scp -oConnectTimeout=2 -i ${GITIAN_BASE:-.}/var/id_dsa -P $VM_SSH_PORT $1 $TUSER@localhost:$2

1
libexec/gconfig

@ -0,0 +1 @@ @@ -0,0 +1 @@
VM_SSH_PORT=2223

44
libexec/on-target

@ -0,0 +1,44 @@ @@ -0,0 +1,44 @@
#!/bin/sh
. gconfig
TUSER=ubuntu
usage() {
echo "Usage: ${0##*/} [OPTION]... <command>"
echo "Run command on build target."
echo
cat << EOF
--help display this help and exit
--user=U run as U instead of ubuntu
EOF
}
if [ $# != 0 ] ; then
while true ; do
case "$1" in
--help|-h)
usage
exit 0
;;
--user|-u)
TUSER="$2"
shift 2
;;
--*)
echo "unrecognized option $1"
exit 1
;;
*)
break
;;
esac
done
fi
#if [ $# = 0 ] ; then
# usage
# exit 1
#fi
ssh -oConnectTimeout=2 -i ${GITIAN_BASE:-.}/var/id_dsa -p $VM_SSH_PORT $TUSER@localhost $*

9
libexec/start-target

@ -0,0 +1,9 @@ @@ -0,0 +1,9 @@
#!/bin/sh
. gconfig
kvm -m 2000 -smp 2 -drive file=target.qcow2 -net nic,model=virtio -net user,hostfwd=tcp::$VM_SSH_PORT-:22 -vnc :16 > var/target.log 2>&1 &
echo $! > var/target.pid
wait
rm var/target.pid

14
libexec/stop-target

@ -0,0 +1,14 @@ @@ -0,0 +1,14 @@
#!/bin/sh
if [ ! -e var/target.pid ]; then exit; fi
on-target -u root halt
sleep 5
if [ ! -e var/target.pid ]; then exit; fi
sleep 5
if [ ! -e var/target.pid ]; then exit; fi
echo Killing target since it did not shutdown within 10 seconds
kill `cat var/target.pid`

12
target-bin/grab-packages.sh

@ -0,0 +1,12 @@ @@ -0,0 +1,12 @@
#!/bin/sh
# Get an installed package manifest
set -e
cd /var/cache/apt/archives
#apt-get clean
dpkg-query -W -f '${Package}\n' | xargs -n 50 apt-get install --reinstall -y -d > /dev/null
sha256sum *.deb | sort --key 2

6
target-bin/init-build.sh

@ -0,0 +1,6 @@ @@ -0,0 +1,6 @@
#!/bin/sh
rm -rf inst out build
mkdir build
mkdir out
mkdir inst
Loading…
Cancel
Save