You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

gbuild 6.5KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234
  1. #!/usr/bin/ruby
  2. require 'optparse'
  3. require 'yaml'
  4. require 'fileutils'
  5. require 'pathname'
  6. @options = {:num_procs => 2, :memory => 2000}
  7. @bitness = {
  8. 'i386' => 32,
  9. 'amd64' => 64,
  10. }
  11. def system!(cmd)
  12. system(cmd) or raise "failed to run #{cmd}"
  13. end
  14. def sanitize(str, where)
  15. raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/)
  16. str
  17. end
  18. def sanitize_path(str, where)
  19. raise "unsanitary string in #{where}" if (str =~ /[^\w\/.-]/)
  20. str
  21. end
  22. def info(str)
  23. puts str unless @options[:quiet]
  24. end
  25. def build_one_configuration(suite, arch, build_desc, reference_datetime)
  26. bits = @bitness[arch] or raise "unknown architecture ${arch}"
  27. suitearch = "#{suite}-#{arch}"
  28. info "Stopping target if it is up"
  29. system "stop-target"
  30. sleep 1
  31. unless @options[:skip_image]
  32. info "Making a new image copy"
  33. system! "cp base-#{suitearch}.qcow2 target-#{suitearch}.qcow2"
  34. end
  35. info "Starting target"
  36. system! "start-target #{bits} #{suitearch}&"
  37. $stdout.write "Checking if target is up"
  38. (1..10).each do
  39. system "on-target true 2> /dev/null" and break
  40. sleep 2
  41. $stdout.write '.'
  42. end
  43. info ''
  44. system! "on-target true"
  45. info "Preparing build environment"
  46. system! "on-target bash < target-bin/init-build.sh"
  47. build_desc["files"].each do |filename|
  48. filename = sanitize(filename, "files section")
  49. system! "cd inputs && copy-to-target #{@quiet_flag} #{filename} build/"
  50. end
  51. info "Updating apt-get repository (log in var/install.log)"
  52. system! "on-target -u root apt-get update > var/install.log 2>&1"
  53. info "Installing additional packages (log in var/install.log)"
  54. system! "on-target -u root apt-get -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"
  55. info "Grabbing package manifest"
  56. system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest"
  57. info "Creating build script (var/build-script)"
  58. File.open("var/build-script", "w") do |script|
  59. script.puts "#!/bin/bash"
  60. script.puts "set -e"
  61. script.puts "export OUTDIR=$HOME/out"
  62. script.puts "GBUILD_BITS=#{bits}"
  63. script.puts "MAKEOPTS=(-j#{@options[:num_procs]})"
  64. (ref_date, ref_time) = reference_datetime.split
  65. script.puts "REFERENCE_DATETIME='#{reference_datetime}'"
  66. script.puts "REFERENCE_DATE='#{ref_date}'"
  67. script.puts "REFERENCE_TIME='#{ref_time}'"
  68. script.puts
  69. build_desc["remotes"].each do |remote|
  70. script.puts "git clone -q #{remote["url"]} build/#{remote["dir"]}"
  71. script.puts "(cd build/#{remote["dir"]} && git checkout -q #{remote["commit"]})"
  72. end
  73. script.puts "cd build"
  74. script.puts build_desc["script"]
  75. end
  76. info "Running build script (log in var/build.log)"
  77. system! "on-target bash < var/build-script > var/build.log 2>&1"
  78. end
  79. ################################
  80. OptionParser.new do |opts|
  81. opts.banner = "Usage: build [options] <build-description>.yml"
  82. opts.on("-i", "--skip-image", "reuse current target image") do |v|
  83. @options[:skip_image] = v
  84. end
  85. opts.on("-q", "--quiet", "be quiet") do |v|
  86. @options[:quiet] = v
  87. end
  88. opts.on("-j PROCS", "--num-make PROCS", "number of processes to use") do |v|
  89. @options[:num_procs] = v
  90. end
  91. opts.on("-m MEM", "--memory MEM", "memory to allocate in MiB") do |v|
  92. @options[:memory] = v
  93. end
  94. opts.on("-c PAIRS", "--commit PAIRS", "comma separated list of DIRECTORY=COMMIT pairs") do |v|
  95. @options[:commit] = v
  96. end
  97. end.parse!
  98. if !File.exist?("/dev/kvm")
  99. $stderr.puts "\n************* WARNING: kvm not loaded, this will probably not work out\n\n"
  100. end
  101. base_dir = Pathname.new(__FILE__).expand_path.dirname.parent
  102. libexec_dir = base_dir + 'libexec'
  103. ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH']
  104. ENV['GITIAN_BASE'] = base_dir.to_s
  105. ENV['NPROCS'] = @options[:num_procs].to_s
  106. ENV['VMEM'] = @options[:memory].to_s
  107. @quiet_flag = @options[:quiet] ? "-q" : ""
  108. build_desc_file = ARGV.shift or raise "must supply YAML build description file"
  109. build_desc = YAML.load_file(build_desc_file)
  110. in_sums = []
  111. build_dir = 'build'
  112. result_dir = 'result'
  113. FileUtils.rm_rf(build_dir)
  114. FileUtils.mkdir(build_dir)
  115. FileUtils.mkdir_p(result_dir)
  116. package_name = build_desc["name"] or raise "must supply name"
  117. package_name = sanitize(package_name, "package name")
  118. suites = build_desc["suites"] or raise "must supply suites"
  119. archs = build_desc["architectures"] or raise "must supply architectures"
  120. reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime"
  121. desc_sum = `sha256sum #{build_desc_file}`
  122. desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
  123. in_sums << desc_sum
  124. build_desc["files"].each do |filename|
  125. filename = sanitize(filename, "files section")
  126. in_sums << `cd inputs && sha256sum #{filename}`
  127. end
  128. commits = {}
  129. if @options[:commit]
  130. @options[:commit].split(',').each do |pair|
  131. (dir, commit) = pair.split('=')
  132. commits[dir] = commit
  133. end
  134. end
  135. build_desc["remotes"].each do |remote|
  136. if !remote["commit"]
  137. remote["commit"] = commits[remote["dir"]]
  138. raise "must specify a commit for directory #{remote["dir"]}" unless remote["commit"]
  139. end
  140. end
  141. base_manifests = YAML::Omap.new
  142. suites.each do |suite|
  143. suite = sanitize(suite, "suite")
  144. archs.each do |arch|
  145. info "--- Building for #{suite} #{arch} ---"
  146. arch = sanitize(arch, "architecture")
  147. # Build!
  148. build_one_configuration(suite, arch, build_desc, reference_datetime)
  149. info "Grabbing results"
  150. system! "copy-from-target #{@quiet_flag} out #{build_dir}"
  151. base_manifest = File.read("var/base-#{suite}-#{arch}.manifest")
  152. base_manifests["#{suite}-#{arch}"] = base_manifest
  153. end
  154. end
  155. out_dir = File.join(build_dir, "out")
  156. out_sums = {}
  157. info "Generating report"
  158. Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  159. next if File.directory?(file_in_out)
  160. file = file_in_out.sub(out_dir + File::SEPARATOR, '')
  161. file = sanitize_path(file, file_in_out)
  162. out_sums[file] = `cd #{out_dir} && sha256sum #{file}`
  163. raise "failed to sum #{file}" unless $? == 0
  164. puts out_sums[file] unless @options[:quiet]
  165. end
  166. out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
  167. in_manifest = in_sums.join('')
  168. # Use Omap to keep result deterministic
  169. report = YAML::Omap[
  170. 'out_manifest', out_manifest,
  171. 'in_manifest', in_manifest,
  172. 'base_manifests', base_manifests,
  173. ]
  174. result_file = "#{package_name}-res.yml"
  175. File.open(File.join(result_dir, result_file), "w") do |io|
  176. io.write report.to_yaml
  177. end
  178. system!("cd #{result_dir} && sha256sum #{result_file}") unless @options[:quiet]
  179. info "Done."