initial

.gitignore

@@ -0,0 +1,8 @@
+*-stamp
+*.substvars
+*.log
+*.pyc
+build
+var
+result
+inputs

README.md

@@ -0,0 +1,25 @@
+# Gitian
+
+Read about the project goals at the "project home page":https://gitian.org/ .
+
+This package can do a deterministic build of a package inside a VM.
+
+## Deterministic build inside a VM
+
+This performs a build inside a VM, with deterministic inputs and outputs.  If the build script takes care of all sources of non-determinism (mostly caused by timestamps), the result will always be the same.  This allows multiple independent verifiers to sign a binary with the assurance that it really came from the source they reviewed.
+
+Synopsis:
+
+* Install prereqs:
+
+    sudo apt-get install python-vm-builder qemu-kvm
+
+* This will create the base VM for use in further builds (requires sudo):
+
+    bin/make-base-vm
+
+* This will build using a YAML description file (can be run as non-root):
+
+    bin/gbuild _package_-desc.yml
+
+The resulting report will appear in result/_package_-res.yml

bin/gbuild

@@ -0,0 +1,160 @@
+#!/usr/bin/ruby
+
+require 'optparse'
+require 'yaml'
+require 'fileutils'
+require 'pathname'
+
+@options = {}
+
+def system!(cmd)
+  system(cmd) or raise "failed to run #{cmd}"
+end
+
+def sanitize(str, where)
+  raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/)
+  str
+end
+
+def info(str)
+  puts str unless @options[:quiet]
+end
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: build [options] <build-description>.yml"
+
+  opts.on("-i", "--skip-image", "reuse current target image") do |v|
+    @options[:skip_image] = v
+  end
+  opts.on("-q", "--quiet", "be quiet") do |v|
+    @options[:skip_image] = v
+  end
+end.parse!
+
+base_dir = Pathname.new(__FILE__).expand_path.dirname.parent
+libexec_dir = base_dir + 'libexec'
+
+ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH']
+ENV['GITIAN_BASE'] = base_dir.to_s
+
+build_desc_file = ARGV.shift or raise "must supply YAML build description file"
+
+in_sums = []
+
+build_desc = YAML.load_file(build_desc_file)
+
+package_name = build_desc["name"] or raise "must supply name"
+package_name = sanitize(package_name, "package name")
+
+desc_sum = `sha256sum #{build_desc_file}`
+desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
+in_sums << desc_sum
+
+reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime"
+
+build_dir = 'build'
+result_dir = 'result'
+
+FileUtils.rm_rf(build_dir)
+FileUtils.mkdir(build_dir)
+FileUtils.mkdir_p(result_dir)
+
+info "Stopping target if it is up"
+system "stop-target"
+
+sleep 1
+
+unless @options[:skip_image]
+  info "Making a new image copy"
+  system! "cp base.qcow2 target.qcow2"
+end
+
+info "Starting target"
+system! "start-target &"
+
+$stdout.write "Checking if target is up"
+
+(1..10).each do
+  system "on-target true 2> /dev/null" and break
+  sleep 2
+  $stdout.write '.'
+end
+
+info ''
+
+system! "on-target true"
+
+info "Installing additional packages (log in var/install.log)"
+system! "on-target -u root apt-get -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"
+
+info "Grabbing package manifest"
+system! "on-target -u root bash < target-bin/grab-packages.sh > var/base.manifest"
+
+info "Preparing build environment"
+system! "on-target bash < target-bin/init-build.sh"
+
+build_desc["files"].each do |filename|
+  filename = sanitize(filename, "files section")
+  system! "cd inputs ; copy-to-target #{filename} build/"
+  in_sums << `cd inputs ; sha256sum #{filename}`
+end
+
+info "Creating build script (var/build-script)"
+
+File.open("var/build-script", "w") do |script|
+  script.puts "#!/bin/bash"
+  script.puts "set -e"
+  script.puts "export OUTDIR=$HOME/out"
+  script.puts "MAKEOPTS=(-j2)"
+  (ref_date, ref_time) = reference_datetime.split
+  script.puts "REFERENCE_DATETIME='#{reference_datetime}'"
+  script.puts "REFERENCE_DATE='#{ref_date}'"
+  script.puts "REFERENCE_TIME='#{ref_time}'"
+  script.puts
+  build_desc["remotes"].each do |remote|
+    script.puts "git clone -q #{remote["url"]} build/#{remote["dir"]}"
+    script.puts "(cd build/#{remote["dir"]} ; git checkout -q #{remote["commit"]})"
+  end
+  script.puts "cd build"
+  script.puts build_desc["script"]
+end
+
+info "Running build script (log in var/build.log)"
+system! "on-target bash < var/build-script > var/build.log 2>&1"
+
+info "Grabbing results"
+system! "copy-from-target out #{build_dir}"
+
+out_dir = File.join(build_dir, "out")
+out_sums = {}
+
+info "Generating report"
+Dir.new(out_dir).each do |file|
+  next if file.start_with?(".")
+  file = sanitize(file, out_dir)
+  out_sums[file] = `cd #{out_dir} ; sha256sum #{file}`
+  raise "failed to sum #{file}" unless $? == 0
+  puts out_sums[file] unless @options[:quiet]
+end
+
+out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
+
+base_manifest = File.read('var/base.manifest')
+
+in_manifest = in_sums.join('')
+
+# Use Omap to keep result deterministic
+report = YAML::Omap[
+  'out_manifest', out_manifest,
+  'in_manifest', in_manifest,
+  'base_manifest', base_manifest,
+]
+
+result_file = "#{package_name}-res.yml"
+File.open(File.join(result_dir, result_file), "w") do |io|
+  io.write report.to_yaml
+end
+
+system!("cd #{result_dir} ; sha256sum #{result_file}") unless @options[:quiet]
+
+info "Done."

bin/gitian

@@ -0,0 +1,50 @@
+#!/usr/bin/python
+
+import subprocess
+import sys
+import os
+sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), "../lib"))
+sys.path.append('/usr/lib/gitian')
+from gitian_util import *
+
+def check_command(command):
+    if commands.get(command) is None:
+        print>>sys.stderr, "usage: %s CMD\ntry: %s help"%(prog, prog)
+        exit(1)
+    return find_command(command)
+
+args = sys.argv[:]
+prog = args.pop(0)
+
+if len(args) < 1:
+    print>>sys.stderr, "usage: %s CMD\n\ntry:\n  %s help\nor:\n  %s help CMD"%(prog, prog, prog)
+    exit(1)
+
+commands = {
+    "release-build": "Build all packages into the 'dist' directory",
+    "package-build": "Build a single package into the 'dist' directory",
+    "package-new": "Insert a new package into the distribution",
+    "release-upload": "Upload a release to a web server",
+}
+if args[0] == "help":
+    if len(args) == 1:
+        for command in commands.keys():
+            print command, " - ", commands[command]
+    else:
+        command = args[1]
+        command_path = find_command(command)
+        ret = subprocess.call([command_path, "-h"])
+elif args[0] == 'shell-complete':
+    if len(args) == 1 or args[1] == "help":
+        for command in commands.keys():
+            print "%s:%s"%(command, commands[command])
+    else:
+        command = args[1]
+        command_path = find_command(command)
+        ret = subprocess.call([command_path, "--shell-complete"])
+else:
+    command = args.pop(0)
+    command_path = find_command(command)
+    args.insert(0, command_path)
+    os.execv(command_path, args)
+

bin/make-base-vm

@@ -0,0 +1,11 @@
+#!/bin/sh
+set -e
+
+SUITE=lucid
+ARCH=amd64
+
+if [ ! -e var/id_dsa ]; then
+  ssh-keygen -t dsa -f var/id_dsa -N ""
+fi
+sudo vmbuilder kvm ubuntu --arch=$ARCH --suite=$SUITE --addpkg=openssh-server,pciutils,build-essential,git-core,mercurial,subversion --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=http://localhost:3142/ubuntu --dest=base --flavour=virtual --overwrite
+mv base/*.qcow2 base.qcow2

libexec/copy-from-target

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+. gconfig
+
+TUSER=ubuntu
+
+usage() {
+  echo "Usage: ${0##*/} [OPTION]... <command>"
+  echo "Run command on build target."
+  echo
+  cat << EOF
+  --help    display this help and exit
+  --user=U  run as U instead of ubuntu
+EOF
+}
+
+if [ $# != 0 ] ; then
+  while true ; do
+    case "$1" in
+      --help|-h)
+        usage
+        exit 0
+        ;;
+      --user|-u)
+        TUSER="$2"
+        shift 2
+        ;;
+      --*)
+        echo "unrecognized option $1"
+        exit 1
+        ;;
+      *)
+        break
+        ;;
+    esac
+  done
+fi
+
+if [ $# = 0 ] ; then
+  usage
+  exit 1
+fi
+
+scp -oConnectTimeout=2 -i ${GITIAN_BASE:-.}/var/id_dsa -P $VM_SSH_PORT -r $TUSER@localhost:$1 $2

libexec/copy-to-target

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+. gconfig
+
+TUSER=ubuntu
+
+usage() {
+  echo "Usage: ${0##*/} [OPTION]... <command>"
+  echo "Run command on build target."
+  echo
+  cat << EOF
+  --help    display this help and exit
+  --user=U  run as U instead of ubuntu
+EOF
+}
+
+if [ $# != 0 ] ; then
+  while true ; do
+    case "$1" in
+      --help|-h)
+        usage
+        exit 0
+        ;;
+      --user|-u)
+        TUSER="$2"
+        shift 2
+        ;;
+      --*)
+        echo "unrecognized option $1"
+        exit 1
+        ;;
+      *)
+        break
+        ;;
+    esac
+  done
+fi
+
+if [ $# = 0 ] ; then
+  usage
+  exit 1
+fi
+
+scp -oConnectTimeout=2 -i ${GITIAN_BASE:-.}/var/id_dsa -P $VM_SSH_PORT $1 $TUSER@localhost:$2

libexec/gconfig

@@ -0,0 +1 @@
+VM_SSH_PORT=2223

libexec/on-target

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+. gconfig
+
+TUSER=ubuntu
+
+usage() {
+  echo "Usage: ${0##*/} [OPTION]... <command>"
+  echo "Run command on build target."
+  echo
+  cat << EOF
+  --help    display this help and exit
+  --user=U  run as U instead of ubuntu
+EOF
+}
+
+if [ $# != 0 ] ; then
+  while true ; do
+    case "$1" in
+      --help|-h)
+        usage
+        exit 0
+        ;;
+      --user|-u)
+        TUSER="$2"
+        shift 2
+        ;;
+      --*)
+        echo "unrecognized option $1"
+        exit 1
+        ;;
+      *)
+        break
+        ;;
+    esac
+  done
+fi
+
+#if [ $# = 0 ] ; then
+#  usage
+#  exit 1
+#fi
+
+ssh -oConnectTimeout=2 -i ${GITIAN_BASE:-.}/var/id_dsa -p $VM_SSH_PORT $TUSER@localhost $*

libexec/start-target

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+. gconfig
+
+kvm -m 2000 -smp 2 -drive file=target.qcow2 -net nic,model=virtio -net user,hostfwd=tcp::$VM_SSH_PORT-:22 -vnc :16 > var/target.log 2>&1 &
+
+echo $! > var/target.pid
+wait
+rm var/target.pid

libexec/stop-target

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+if [ ! -e var/target.pid ]; then exit; fi
+
+on-target -u root halt
+sleep 5
+
+if [ ! -e var/target.pid ]; then exit; fi
+sleep 5
+
+if [ ! -e var/target.pid ]; then exit; fi
+
+echo Killing target since it did not shutdown within 10 seconds
+kill `cat var/target.pid`

target-bin/grab-packages.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Get an installed package manifest
+
+set -e
+
+cd /var/cache/apt/archives
+
+#apt-get clean
+
+dpkg-query -W -f '${Package}\n' | xargs -n 50 apt-get install --reinstall -y -d  > /dev/null
+sha256sum *.deb | sort --key 2

target-bin/init-build.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+rm -rf inst out build
+mkdir build
+mkdir out
+mkdir inst