You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
10 年之前
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. #!/usr/bin/ruby
  2. require 'optparse'
  3. require 'yaml'
  4. require 'fileutils'
  5. require 'pathname'
  6. @options = {:num_procs => 2, :memory => 2000}
  7. @bitness = {
  8. 'i386' => 32,
  9. 'amd64' => 64,
  10. }
  11. def system!(cmd)
  12. system(cmd) or raise "failed to run #{cmd}"
  13. end
  14. def sanitize(str, where)
  15. raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/)
  16. str
  17. end
  18. def sanitize_path(str, where)
  19. raise "unsanitary string in #{where}" if (str =~ /[^\w\/.:-]/)
  20. str
  21. end
  22. def info(str)
  23. puts str unless @options[:quiet]
  24. end
  25. def build_one_configuration(suite, arch, build_desc, reference_datetime)
  26. FileUtils.rm_f("var/build.log")
  27. bits = @bitness[arch] or raise "unknown architecture ${arch}"
  28. if ENV["USE_LXC"]
  29. ENV["LXC_ARCH"] = arch
  30. ENV["LXC_SUITE"] = suite
  31. end
  32. suitearch = "#{suite}-#{arch}"
  33. info "Stopping target if it is up"
  34. system "stop-target"
  35. sleep 1
  36. unless @options[:skip_image]
  37. info "Making a new image copy"
  38. system! "make-clean-vm --suite #{suite} --arch #{arch}"
  39. end
  40. info "Starting target"
  41. system! "start-target #{bits} #{suitearch}&"
  42. $stdout.write "Checking if target is up"
  43. (1..10).each do
  44. system "on-target true 2> /dev/null" and break
  45. sleep 2
  46. $stdout.write '.'
  47. end
  48. info ''
  49. system! "on-target true"
  50. info "Preparing build environment"
  51. system! "on-target bash < target-bin/init-build.sh"
  52. build_desc["files"].each do |filename|
  53. filename = sanitize(filename, "files section")
  54. system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/"
  55. end
  56. info "Updating apt-get repository (log in var/install.log)"
  57. system! "on-target -u root apt-get update > var/install.log 2>&1"
  58. info "Installing additional packages (log in var/install.log)"
  59. system! "on-target -u root apt-get -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"
  60. info "Grabbing package manifest"
  61. system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest"
  62. info "Creating build script (var/build-script)"
  63. File.open("var/build-script", "w") do |script|
  64. script.puts "#!/bin/bash"
  65. script.puts "set -e"
  66. script.puts "export OUTDIR=$HOME/out"
  67. script.puts "GBUILD_BITS=#{bits}"
  68. script.puts "MAKEOPTS=(-j#{@options[:num_procs]})"
  69. (ref_date, ref_time) = reference_datetime.split
  70. script.puts "REFERENCE_DATETIME='#{reference_datetime}'"
  71. script.puts "REFERENCE_DATE='#{ref_date}'"
  72. script.puts "REFERENCE_TIME='#{ref_time}'"
  73. script.puts
  74. build_desc["remotes"].each do |remote|
  75. dir = sanitize(remote["dir"], remote["dir"])
  76. if File.exist?("inputs/#{dir}")
  77. system!("cd inputs/#{dir} && git fetch")
  78. else
  79. system!("git clone -q #{sanitize_path(remote["url"], remote["url"])} inputs/#{dir}")
  80. end
  81. system! "copy-to-target #{@quiet_flag} inputs/#{dir} build/"
  82. script.puts "(cd build/#{dir} && git checkout -q #{remote["commit"]})"
  83. end
  84. script.puts "cd build"
  85. script.puts build_desc["script"]
  86. end
  87. info "Running build script (log in var/build.log)"
  88. system! "on-target bash < var/build-script > var/build.log 2>&1"
  89. end
  90. ################################
  91. OptionParser.new do |opts|
  92. opts.banner = "Usage: build [options] <build-description>.yml"
  93. opts.on("-i", "--skip-image", "reuse current target image") do |v|
  94. @options[:skip_image] = v
  95. end
  96. opts.on("-q", "--quiet", "be quiet") do |v|
  97. @options[:quiet] = v
  98. end
  99. opts.on("-j PROCS", "--num-make PROCS", "number of processes to use") do |v|
  100. @options[:num_procs] = v
  101. end
  102. opts.on("-m MEM", "--memory MEM", "memory to allocate in MiB") do |v|
  103. @options[:memory] = v
  104. end
  105. opts.on("-c PAIRS", "--commit PAIRS", "comma separated list of DIRECTORY=COMMIT pairs") do |v|
  106. @options[:commit] = v
  107. end
  108. end.parse!
  109. if !File.exist?("/dev/kvm")
  110. $stderr.puts "\n************* WARNING: kvm not loaded, this will probably not work out\n\n"
  111. end
  112. base_dir = Pathname.new(__FILE__).expand_path.dirname.parent
  113. libexec_dir = base_dir + 'libexec'
  114. ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH']
  115. ENV['GITIAN_BASE'] = base_dir.to_s
  116. ENV['NPROCS'] = @options[:num_procs].to_s
  117. ENV['VMEM'] = @options[:memory].to_s
  118. @quiet_flag = @options[:quiet] ? "-q" : ""
  119. build_desc_file = ARGV.shift or raise "must supply YAML build description file"
  120. build_desc = YAML.load_file(build_desc_file)
  121. in_sums = []
  122. build_dir = 'build'
  123. result_dir = 'result'
  124. FileUtils.rm_rf(build_dir)
  125. FileUtils.mkdir(build_dir)
  126. FileUtils.mkdir_p(result_dir)
  127. package_name = build_desc["name"] or raise "must supply name"
  128. package_name = sanitize(package_name, "package name")
  129. suites = build_desc["suites"] or raise "must supply suites"
  130. archs = build_desc["architectures"] or raise "must supply architectures"
  131. reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime"
  132. desc_sum = `sha256sum #{build_desc_file}`
  133. desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
  134. in_sums << desc_sum
  135. build_desc["files"].each do |filename|
  136. filename = sanitize(filename, "files section")
  137. in_sums << `cd inputs && sha256sum #{filename}`
  138. end
  139. commits = {}
  140. if @options[:commit]
  141. @options[:commit].split(',').each do |pair|
  142. (dir, commit) = pair.split('=')
  143. commits[dir] = commit
  144. end
  145. end
  146. build_desc["remotes"].each do |remote|
  147. if !remote["commit"]
  148. remote["commit"] = commits[remote["dir"]]
  149. raise "must specify a commit for directory #{remote["dir"]}" unless remote["commit"]
  150. end
  151. end
  152. base_manifests = YAML::Omap.new
  153. suites.each do |suite|
  154. suite = sanitize(suite, "suite")
  155. archs.each do |arch|
  156. info "--- Building for #{suite} #{arch} ---"
  157. arch = sanitize(arch, "architecture")
  158. # Build!
  159. build_one_configuration(suite, arch, build_desc, reference_datetime)
  160. info "Grabbing results"
  161. system! "copy-from-target #{@quiet_flag} out #{build_dir}"
  162. base_manifest = File.read("var/base-#{suite}-#{arch}.manifest")
  163. base_manifests["#{suite}-#{arch}"] = base_manifest
  164. end
  165. end
  166. out_dir = File.join(build_dir, "out")
  167. out_sums = {}
  168. info "Generating report"
  169. Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  170. next if File.directory?(file_in_out)
  171. file = file_in_out.sub(out_dir + File::SEPARATOR, '')
  172. file = sanitize_path(file, file_in_out)
  173. out_sums[file] = `cd #{out_dir} && sha256sum #{file}`
  174. raise "failed to sum #{file}" unless $? == 0
  175. puts out_sums[file] unless @options[:quiet]
  176. end
  177. out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
  178. in_manifest = in_sums.join('')
  179. # Use Omap to keep result deterministic
  180. report = YAML::Omap[
  181. 'out_manifest', out_manifest,
  182. 'in_manifest', in_manifest,
  183. 'base_manifests', base_manifests,
  184. ]
  185. result_file = "#{package_name}-res.yml"
  186. File.open(File.join(result_dir, result_file), "w") do |io|
  187. io.write report.to_yaml
  188. end
  189. system!("cd #{result_dir} && sha256sum #{result_file}") unless @options[:quiet]
  190. info "Done."