You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

gbuild 12KB

10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381
  1. #!/usr/bin/ruby
  2. require 'optparse'
  3. require 'yaml'
  4. require 'fileutils'
  5. require 'pathname'
  6. @options = {:num_procs => 2, :memory => 2000}
  7. @bitness = {
  8. 'i386' => 32,
  9. 'amd64' => 64,
  10. }
  11. @arches = {
  12. 'i386' => 'i386',
  13. 'amd64' => 'x86_64',
  14. }
  15. def system!(cmd)
  16. system(cmd) or raise "failed to run #{cmd}"
  17. end
  18. def sanitize(str, where)
  19. raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/)
  20. str
  21. end
  22. def sanitize_path(str, where)
  23. raise "unsanitary string in #{where}" if (str =~ /[^@\w\/.:+-]/)
  24. str
  25. end
  26. def info(str)
  27. puts str unless @options[:quiet]
  28. end
  29. def build_one_configuration(suite, arch, build_desc)
  30. FileUtils.rm_f("var/build.log")
  31. bits = @bitness[arch] or raise "unknown architecture ${arch}"
  32. if ENV["USE_LXC"]
  33. ENV["LXC_ARCH"] = arch
  34. ENV["LXC_SUITE"] = suite
  35. end
  36. suitearch = "#{suite}-#{arch}"
  37. info "Stopping target if it is up"
  38. system "stop-target"
  39. sleep 1
  40. unless @options[:skip_image]
  41. info "Making a new image copy"
  42. system! "make-clean-vm --suite #{suite} --arch #{arch}"
  43. end
  44. info "Starting target"
  45. system! "start-target #{bits} #{suitearch}&"
  46. $stdout.write "Checking if target is up"
  47. (1..30).each do
  48. system "on-target true 2> /dev/null" and break
  49. sleep 2
  50. $stdout.write '.'
  51. end
  52. info ''
  53. system! "on-target true"
  54. system! "on-target -u root tee -a /etc/sudoers.d/#{ENV['DISTRO'] || 'ubuntu'} > /dev/null << EOF
  55. %#{ENV['DISTRO'] || 'ubuntu'} ALL=(ALL) NOPASSWD: ALL
  56. EOF" if build_desc["sudo"] and @options[:allow_sudo]
  57. info "Preparing build environment"
  58. system! "on-target setarch #{@arches[arch]} bash < target-bin/init-build.sh"
  59. build_desc["files"].each do |filename|
  60. filename = sanitize(filename, "files section")
  61. system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/"
  62. end
  63. if build_desc["enable_cache"]
  64. if File.directory?("cache/#{build_desc["name"]}")
  65. system! "copy-to-target #{@quiet_flag} cache/#{build_desc["name"]}/ cache/"
  66. end
  67. if File.directory?("cache/common")
  68. system! "copy-to-target #{@quiet_flag} cache/common/ cache/"
  69. end
  70. end
  71. if build_desc["multiarch"]
  72. info "Adding multiarch support (log in var/install.log)"
  73. for a in build_desc["multiarch"]
  74. system! "on-target -u root dpkg --add-architecture #{a} > var/install.log 2>&1"
  75. end
  76. end
  77. info "Updating apt-get repository (log in var/install.log)"
  78. system! "on-target -u root apt-get update > var/install.log 2>&1"
  79. info "Installing additional packages (log in var/install.log)"
  80. system! "on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"
  81. if build_desc["alternatives"]
  82. info "Set alternatives (log in var/install.log)"
  83. for a in build_desc["alternatives"]
  84. system! "on-target -u root update-alternatives --set #{a["package"]} #{a["path"]} > var/install.log 2>&1"
  85. end
  86. end
  87. if @options[:upgrade] || system("on-target -u root '[ ! -e /var/cache/gitian/initial-upgrade ]'")
  88. info "Upgrading system, may take a while"
  89. system! "on-target -u root bash < target-bin/upgrade-system.sh > var/install.log 2>&1"
  90. end
  91. info "Creating package manifest"
  92. system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest"
  93. info "Creating build script (var/build-script)"
  94. File.open("var/build-script", "w") do |script|
  95. script.puts "#!/bin/bash"
  96. script.puts "set -e"
  97. script.puts "export LANG='en_US.UTF-8'"
  98. script.puts "export LC_ALL='en_US.UTF-8'"
  99. script.puts "umask 002"
  100. script.puts "export OUTDIR=$HOME/out"
  101. script.puts "GBUILD_BITS=#{bits}"
  102. if build_desc["enable_cache"]
  103. script.puts "GBUILD_CACHE_ENABLED=1"
  104. script.puts "GBUILD_PACKAGE_CACHE=$HOME/cache/#{build_desc["name"]}"
  105. script.puts "GBUILD_COMMON_CACHE=$HOME/cache/common"
  106. end
  107. script.puts "MAKEOPTS=(-j#{@options[:num_procs]})"
  108. script.puts
  109. author_date = nil
  110. build_desc["remotes"].each do |remote|
  111. dir = sanitize(remote["dir"], remote["dir"])
  112. author_date = `cd inputs/#{dir} && git log --format=@%at -1 | date +"%F %T" -u -f -`.strip
  113. raise "error looking up author date in #{dir}" unless $?.exitstatus == 0
  114. system! "copy-to-target #{@quiet_flag} inputs/#{dir} build/"
  115. script.puts "(cd build/#{dir} && git reset -q --hard && git clean -q -f -d)"
  116. end
  117. script.puts
  118. ref_datetime = build_desc["reference_datetime"] || author_date
  119. (ref_date, ref_time) = ref_datetime.split
  120. script.puts "REFERENCE_DATETIME='#{ref_datetime}'"
  121. script.puts "REFERENCE_DATE='#{ref_date}'"
  122. script.puts "REFERENCE_TIME='#{ref_time}'"
  123. script.puts
  124. script.puts "cd build"
  125. script.puts build_desc["script"]
  126. end
  127. info "Running build script (log in var/build.log)"
  128. system! "on-target setarch #{@arches[arch]} bash -x < var/build-script > var/build.log 2>&1"
  129. end
  130. ################################
  131. OptionParser.new do |opts|
  132. opts.banner = "Usage: build [options] <build-description>.yml"
  133. opts.on("--allow-sudo", "override SECURITY on the target VM and allow the use of sudo with no password for the default user") do |v|
  134. @options[:allow_sudo] = v
  135. end
  136. opts.on("-i", "--skip-image", "reuse current target image") do |v|
  137. @options[:skip_image] = v
  138. end
  139. opts.on("--upgrade", "upgrade guest with latest packages") do |v|
  140. @options[:upgrade] = v
  141. end
  142. opts.on("-q", "--quiet", "be quiet") do |v|
  143. @options[:quiet] = v
  144. end
  145. opts.on("-j PROCS", "--num-make PROCS", "number of processes to use") do |v|
  146. @options[:num_procs] = v
  147. end
  148. opts.on("-m MEM", "--memory MEM", "memory to allocate in MiB") do |v|
  149. @options[:memory] = v
  150. end
  151. opts.on("-c PAIRS", "--commit PAIRS", "comma separated list of DIRECTORY=COMMIT pairs") do |v|
  152. @options[:commit] = v
  153. end
  154. opts.on("-u PAIRS", "--url PAIRS", "comma separated list of DIRECTORY=URL pairs") do |v|
  155. @options[:url] = v
  156. end
  157. opts.on("-o", "--cache-read-only", "only use existing cache files, do not update them") do |v|
  158. @options[:cache_ro] = v
  159. end
  160. end.parse!
  161. if !ENV["USE_LXC"] and !ENV["USE_VBOX"] and !File.exist?("/dev/kvm")
  162. $stderr.puts "\n************* WARNING: kvm not loaded, this will probably not work out\n\n"
  163. end
  164. base_dir = Pathname.new(__FILE__).expand_path.dirname.parent
  165. libexec_dir = base_dir + 'libexec'
  166. ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH']
  167. ENV['GITIAN_BASE'] = base_dir.to_s
  168. ENV['NPROCS'] = @options[:num_procs].to_s
  169. ENV['VMEM'] = @options[:memory].to_s
  170. @quiet_flag = @options[:quiet] ? "-q" : ""
  171. build_desc_file = ARGV.shift or raise "must supply YAML build description file"
  172. build_desc = YAML.load_file(build_desc_file)
  173. in_sums = []
  174. build_dir = 'build'
  175. result_dir = 'result'
  176. cache_dir = 'cache'
  177. enable_cache = build_desc["enable_cache"]
  178. FileUtils.rm_rf(build_dir)
  179. FileUtils.mkdir(build_dir)
  180. FileUtils.mkdir_p(result_dir)
  181. package_name = build_desc["name"] or raise "must supply name"
  182. package_name = sanitize(package_name, "package name")
  183. if enable_cache
  184. FileUtils.mkdir_p(File.join(cache_dir, "common"))
  185. FileUtils.mkdir_p(File.join(cache_dir, package_name))
  186. end
  187. distro = build_desc["distro"] || "ubuntu"
  188. suites = build_desc["suites"] or raise "must supply suites"
  189. archs = build_desc["architectures"] or raise "must supply architectures"
  190. build_desc["reference_datetime"] or build_desc["remotes"].size > 0 or raise "must supply `reference_datetime` or `remotes`"
  191. docker_image_digests = build_desc["docker_image_digests"] || []
  192. # if docker_image_digests are supplied, it must be the same length as suites
  193. if docker_image_digests.size > 0 and suites.size != docker_image_digests.size
  194. raise "`suites` and `docker_image_digests` must both be the same size if both are supplied"
  195. elsif ENV["USE_DOCKER"] and docker_image_digests.size > 0 and suites.size == docker_image_digests.size
  196. suites = docker_image_digests
  197. end
  198. ENV['DISTRO'] = distro
  199. desc_sum = `sha256sum #{build_desc_file}`
  200. desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
  201. in_sums << desc_sum
  202. build_desc["files"].each do |filename|
  203. filename = sanitize(filename, "files section")
  204. in_sums << `cd inputs && sha256sum #{filename}`
  205. end
  206. commits = {}
  207. if @options[:commit]
  208. @options[:commit].split(',').each do |pair|
  209. (dir, commit) = pair.split('=')
  210. commits[dir] = commit
  211. end
  212. end
  213. urls = {}
  214. if @options[:url]
  215. @options[:url].split(',').each do |pair|
  216. (dir, url) = pair.split('=')
  217. urls[dir] = url
  218. end
  219. end
  220. build_desc["remotes"].each do |remote|
  221. if !remote["commit"]
  222. remote["commit"] = commits[remote["dir"]]
  223. raise "must specify a commit for directory #{remote["dir"]}" unless remote["commit"]
  224. end
  225. if urls[remote["dir"]]
  226. remote["url"] = urls[remote["dir"]]
  227. end
  228. dir = sanitize(remote["dir"], remote["dir"])
  229. unless File.exist?("inputs/#{dir}")
  230. system!("git init inputs/#{dir}")
  231. end
  232. system!("cd inputs/#{dir} && git fetch --update-head-ok #{sanitize_path(remote["url"], remote["url"])} +refs/tags/*:refs/tags/* +refs/heads/*:refs/heads/*")
  233. commit = sanitize(remote["commit"], remote["commit"])
  234. commit = `cd inputs/#{dir} && git log --format=%H -1 #{commit}`.strip
  235. raise "error looking up commit for tag #{remote["commit"]}" unless $?.exitstatus == 0
  236. system!("cd inputs/#{dir} && git checkout -q #{commit}")
  237. system!("cd inputs/#{dir} && git submodule update --init --recursive --force")
  238. in_sums << "git:#{commit} #{dir}"
  239. end
  240. base_manifests = YAML::Omap.new
  241. suites.each do |suite|
  242. suite = sanitize(suite, "suite")
  243. archs.each do |arch|
  244. info "--- Building for #{suite} #{arch} ---"
  245. arch = sanitize(arch, "architecture")
  246. # Build!
  247. build_one_configuration(suite, arch, build_desc)
  248. info "Grabbing results from target"
  249. system! "copy-from-target #{@quiet_flag} out #{build_dir}"
  250. if enable_cache && !@options[:cache_ro]
  251. info "Grabbing cache from target"
  252. system! "copy-from-target #{@quiet_flag} cache/#{package_name}/ #{cache_dir}"
  253. system! "copy-from-target #{@quiet_flag} cache/common/ #{cache_dir}"
  254. end
  255. base_manifest = File.read("var/base-#{suite}-#{arch}.manifest")
  256. base_manifests["#{suite}-#{arch}"] = base_manifest
  257. end
  258. end
  259. out_dir = File.join(build_dir, "out")
  260. out_sums = {}
  261. cache_common_dir = File.join(cache_dir, "common")
  262. cache_package_dir = File.join(cache_dir, "#{package_name}")
  263. cache_common_sums = {}
  264. cache_package_sums = {}
  265. info "Generating report"
  266. Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  267. next if File.directory?(file_in_out)
  268. file = file_in_out.sub(out_dir + File::SEPARATOR, '')
  269. file = sanitize_path(file, file_in_out)
  270. out_sums[file] = `cd #{out_dir} && sha256sum #{file}`
  271. raise "failed to sum #{file}" unless $? == 0
  272. puts out_sums[file] unless @options[:quiet]
  273. end
  274. if enable_cache
  275. Dir.glob(File.join(cache_common_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  276. next if File.directory?(file_in_out)
  277. file = file_in_out.sub(cache_common_dir + File::SEPARATOR, '')
  278. file = sanitize_path(file, file_in_out)
  279. cache_common_sums[file] = `cd #{cache_common_dir} && sha256sum #{file}`
  280. raise "failed to sum #{file}" unless $? == 0
  281. end
  282. Dir.glob(File.join(cache_package_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  283. next if File.directory?(file_in_out)
  284. file = file_in_out.sub(cache_package_dir + File::SEPARATOR, '')
  285. file = sanitize_path(file, file_in_out)
  286. cache_package_sums[file] = `cd #{cache_package_dir} && sha256sum #{file}`
  287. raise "failed to sum #{file}" unless $? == 0
  288. end
  289. end
  290. out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
  291. in_manifest = in_sums.join('')
  292. cache_common_manifest = cache_common_sums.keys.sort.map { |key| cache_common_sums[key] }.join('')
  293. cache_package_manifest = cache_package_sums.keys.sort.map { |key| cache_package_sums[key] }.join('')
  294. # Use Omap to keep result deterministic
  295. report = YAML::Omap[
  296. 'out_manifest', out_manifest,
  297. 'in_manifest', in_manifest,
  298. 'base_manifests', base_manifests,
  299. 'cache_common_manifest', cache_common_manifest,
  300. 'cache_package_manifest', cache_package_manifest,
  301. ]
  302. result_file = "#{package_name}-res.yml"
  303. File.open(File.join(result_dir, result_file), "w") do |io|
  304. io.write report.to_yaml
  305. end
  306. system!("cd #{result_dir} && sha256sum #{result_file}") unless @options[:quiet]
  307. info "Done."