You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

gbuild 11KB

10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365
  1. #!/usr/bin/ruby
  2. require 'optparse'
  3. require 'yaml'
  4. require 'fileutils'
  5. require 'pathname'
  6. @options = {:num_procs => 2, :memory => 2000}
  7. @bitness = {
  8. 'i386' => 32,
  9. 'amd64' => 64,
  10. }
  11. @arches = {
  12. 'i386' => 'i386',
  13. 'amd64' => 'x86_64',
  14. }
  15. def system!(cmd)
  16. system(cmd) or raise "failed to run #{cmd}"
  17. end
  18. def sanitize(str, where)
  19. raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/)
  20. str
  21. end
  22. def sanitize_path(str, where)
  23. raise "unsanitary string in #{where}" if (str =~ /[^@\w\/.:+-]/)
  24. str
  25. end
  26. def info(str)
  27. puts str unless @options[:quiet]
  28. end
  29. def build_one_configuration(suite, arch, build_desc)
  30. FileUtils.rm_f("var/build.log")
  31. bits = @bitness[arch] or raise "unknown architecture ${arch}"
  32. if ENV["USE_LXC"]
  33. ENV["LXC_ARCH"] = arch
  34. ENV["LXC_SUITE"] = suite
  35. end
  36. suitearch = "#{suite}-#{arch}"
  37. info "Stopping target if it is up"
  38. system "stop-target"
  39. sleep 1
  40. unless @options[:skip_image]
  41. info "Making a new image copy"
  42. system! "make-clean-vm --suite #{suite} --arch #{arch}"
  43. end
  44. info "Starting target"
  45. system! "start-target #{bits} #{suitearch}&"
  46. $stdout.write "Checking if target is up"
  47. (1..30).each do
  48. system "on-target true 2> /dev/null" and break
  49. sleep 2
  50. $stdout.write '.'
  51. end
  52. info ''
  53. system! "on-target true"
  54. system! "on-target -u root tee -a /etc/sudoers.d/#{ENV['DISTRO'] || 'ubuntu'} > /dev/null << EOF
  55. %#{ENV['DISTRO'] || 'ubuntu'} ALL=(ALL) NOPASSWD: ALL
  56. EOF" if build_desc["sudo"] and @options[:allow_sudo]
  57. info "Preparing build environment"
  58. system! "on-target setarch #{@arches[arch]} bash < target-bin/init-build.sh"
  59. build_desc["files"].each do |filename|
  60. filename = sanitize(filename, "files section")
  61. system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/"
  62. end
  63. if build_desc["enable_cache"]
  64. if File.directory?("cache/#{build_desc["name"]}")
  65. system! "copy-to-target #{@quiet_flag} cache/#{build_desc["name"]}/ cache/"
  66. end
  67. if File.directory?("cache/common")
  68. system! "copy-to-target #{@quiet_flag} cache/common/ cache/"
  69. end
  70. end
  71. info "Updating apt-get repository (log in var/install.log)"
  72. system! "on-target -u root apt-get update > var/install.log 2>&1"
  73. info "Installing additional packages (log in var/install.log)"
  74. system! "on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"
  75. if build_desc["alternatives"]
  76. info "Set alternatives (log in var/install.log)"
  77. for a in build_desc["alternatives"]
  78. system! "on-target -u root update-alternatives --set #{a["package"]} #{a["path"]} > var/install.log 2>&1"
  79. end
  80. end
  81. if @options[:upgrade] || system("on-target -u root '[ ! -e /var/cache/gitian/initial-upgrade ]'")
  82. info "Upgrading system, may take a while"
  83. system! "on-target -u root bash < target-bin/upgrade-system.sh > var/install.log 2>&1"
  84. end
  85. info "Creating package manifest"
  86. system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest"
  87. info "Creating build script (var/build-script)"
  88. File.open("var/build-script", "w") do |script|
  89. script.puts "#!/bin/bash"
  90. script.puts "set -e"
  91. script.puts "export LANG='en_US.UTF-8'"
  92. script.puts "export LC_ALL='en_US.UTF-8'"
  93. script.puts "umask 002"
  94. script.puts "export OUTDIR=$HOME/out"
  95. script.puts "GBUILD_BITS=#{bits}"
  96. if build_desc["enable_cache"]
  97. script.puts "GBUILD_CACHE_ENABLED=1"
  98. script.puts "GBUILD_PACKAGE_CACHE=$HOME/cache/#{build_desc["name"]}"
  99. script.puts "GBUILD_COMMON_CACHE=$HOME/cache/common"
  100. end
  101. script.puts "MAKEOPTS=(-j#{@options[:num_procs]})"
  102. script.puts
  103. author_date = nil
  104. build_desc["remotes"].each do |remote|
  105. dir = sanitize(remote["dir"], remote["dir"])
  106. author_date = `cd inputs/#{dir} && git log --format=@%at -1 | date +"%F %T" -u -f -`.strip
  107. raise "error looking up author date in #{dir}" unless $?.exitstatus == 0
  108. system! "copy-to-target #{@quiet_flag} inputs/#{dir} build/"
  109. script.puts "(cd build/#{dir} && git reset -q --hard && git clean -q -f -d)"
  110. end
  111. script.puts
  112. ref_datetime = build_desc["reference_datetime"] || author_date
  113. (ref_date, ref_time) = ref_datetime.split
  114. script.puts "REFERENCE_DATETIME='#{ref_datetime}'"
  115. script.puts "REFERENCE_DATE='#{ref_date}'"
  116. script.puts "REFERENCE_TIME='#{ref_time}'"
  117. script.puts
  118. script.puts "cd build"
  119. script.puts build_desc["script"]
  120. end
  121. info "Running build script (log in var/build.log)"
  122. system! "on-target setarch #{@arches[arch]} bash -x < var/build-script > var/build.log 2>&1"
  123. end
  124. ################################
  125. OptionParser.new do |opts|
  126. opts.banner = "Usage: build [options] <build-description>.yml"
  127. opts.on("--allow-sudo", "override SECURITY on the target VM and allow the use of sudo with no password for the default user") do |v|
  128. @options[:allow_sudo] = v
  129. end
  130. opts.on("-i", "--skip-image", "reuse current target image") do |v|
  131. @options[:skip_image] = v
  132. end
  133. opts.on("--upgrade", "upgrade guest with latest packages") do |v|
  134. @options[:upgrade] = v
  135. end
  136. opts.on("-q", "--quiet", "be quiet") do |v|
  137. @options[:quiet] = v
  138. end
  139. opts.on("-j PROCS", "--num-make PROCS", "number of processes to use") do |v|
  140. @options[:num_procs] = v
  141. end
  142. opts.on("-m MEM", "--memory MEM", "memory to allocate in MiB") do |v|
  143. @options[:memory] = v
  144. end
  145. opts.on("-c PAIRS", "--commit PAIRS", "comma separated list of DIRECTORY=COMMIT pairs") do |v|
  146. @options[:commit] = v
  147. end
  148. opts.on("-u PAIRS", "--url PAIRS", "comma separated list of DIRECTORY=URL pairs") do |v|
  149. @options[:url] = v
  150. end
  151. opts.on("-o", "--cache-read-only", "only use existing cache files, do not update them") do |v|
  152. @options[:cache_ro] = v
  153. end
  154. end.parse!
  155. if !ENV["USE_LXC"] and !File.exist?("/dev/kvm")
  156. $stderr.puts "\n************* WARNING: kvm not loaded, this will probably not work out\n\n"
  157. end
  158. base_dir = Pathname.new(__FILE__).expand_path.dirname.parent
  159. libexec_dir = base_dir + 'libexec'
  160. ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH']
  161. ENV['GITIAN_BASE'] = base_dir.to_s
  162. ENV['NPROCS'] = @options[:num_procs].to_s
  163. ENV['VMEM'] = @options[:memory].to_s
  164. @quiet_flag = @options[:quiet] ? "-q" : ""
  165. build_desc_file = ARGV.shift or raise "must supply YAML build description file"
  166. build_desc = YAML.load_file(build_desc_file)
  167. in_sums = []
  168. build_dir = 'build'
  169. result_dir = 'result'
  170. cache_dir = 'cache'
  171. enable_cache = build_desc["enable_cache"]
  172. FileUtils.rm_rf(build_dir)
  173. FileUtils.mkdir(build_dir)
  174. FileUtils.mkdir_p(result_dir)
  175. package_name = build_desc["name"] or raise "must supply name"
  176. package_name = sanitize(package_name, "package name")
  177. if enable_cache
  178. FileUtils.mkdir_p(File.join(cache_dir, "common"))
  179. FileUtils.mkdir_p(File.join(cache_dir, package_name))
  180. end
  181. distro = build_desc["distro"] || "ubuntu"
  182. suites = build_desc["suites"] or raise "must supply suites"
  183. archs = build_desc["architectures"] or raise "must supply architectures"
  184. build_desc["reference_datetime"] or build_desc["remotes"].size > 0 or raise "must supply `reference_datetime` or `remotes`"
  185. ENV['DISTRO'] = distro
  186. desc_sum = `sha256sum #{build_desc_file}`
  187. desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
  188. in_sums << desc_sum
  189. build_desc["files"].each do |filename|
  190. filename = sanitize(filename, "files section")
  191. in_sums << `cd inputs && sha256sum #{filename}`
  192. end
  193. commits = {}
  194. if @options[:commit]
  195. @options[:commit].split(',').each do |pair|
  196. (dir, commit) = pair.split('=')
  197. commits[dir] = commit
  198. end
  199. end
  200. urls = {}
  201. if @options[:url]
  202. @options[:url].split(',').each do |pair|
  203. (dir, url) = pair.split('=')
  204. urls[dir] = url
  205. end
  206. end
  207. build_desc["remotes"].each do |remote|
  208. if !remote["commit"]
  209. remote["commit"] = commits[remote["dir"]]
  210. raise "must specify a commit for directory #{remote["dir"]}" unless remote["commit"]
  211. end
  212. if urls[remote["dir"]]
  213. remote["url"] = urls[remote["dir"]]
  214. end
  215. dir = sanitize(remote["dir"], remote["dir"])
  216. unless File.exist?("inputs/#{dir}")
  217. system!("git init inputs/#{dir}")
  218. end
  219. system!("cd inputs/#{dir} && git fetch --update-head-ok #{sanitize_path(remote["url"], remote["url"])} +refs/tags/*:refs/tags/* +refs/heads/*:refs/heads/*")
  220. commit = sanitize(remote["commit"], remote["commit"])
  221. commit = `cd inputs/#{dir} && git log --format=%H -1 #{commit}`.strip
  222. raise "error looking up commit for tag #{remote["commit"]}" unless $?.exitstatus == 0
  223. system!("cd inputs/#{dir} && git checkout -q #{commit}")
  224. in_sums << "git:#{commit} #{dir}"
  225. end
  226. base_manifests = YAML::Omap.new
  227. suites.each do |suite|
  228. suite = sanitize(suite, "suite")
  229. archs.each do |arch|
  230. info "--- Building for #{suite} #{arch} ---"
  231. arch = sanitize(arch, "architecture")
  232. # Build!
  233. build_one_configuration(suite, arch, build_desc)
  234. info "Grabbing results from target"
  235. system! "copy-from-target #{@quiet_flag} out #{build_dir}"
  236. if enable_cache && !@options[:cache_ro]
  237. info "Grabbing cache from target"
  238. system! "copy-from-target #{@quiet_flag} cache/#{package_name}/ #{cache_dir}"
  239. system! "copy-from-target #{@quiet_flag} cache/common/ #{cache_dir}"
  240. end
  241. base_manifest = File.read("var/base-#{suite}-#{arch}.manifest")
  242. base_manifests["#{suite}-#{arch}"] = base_manifest
  243. end
  244. end
  245. out_dir = File.join(build_dir, "out")
  246. out_sums = {}
  247. cache_common_dir = File.join(cache_dir, "common")
  248. cache_package_dir = File.join(cache_dir, "#{package_name}")
  249. cache_common_sums = {}
  250. cache_package_sums = {}
  251. info "Generating report"
  252. Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  253. next if File.directory?(file_in_out)
  254. file = file_in_out.sub(out_dir + File::SEPARATOR, '')
  255. file = sanitize_path(file, file_in_out)
  256. out_sums[file] = `cd #{out_dir} && sha256sum #{file}`
  257. raise "failed to sum #{file}" unless $? == 0
  258. puts out_sums[file] unless @options[:quiet]
  259. end
  260. if enable_cache
  261. Dir.glob(File.join(cache_common_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  262. next if File.directory?(file_in_out)
  263. file = file_in_out.sub(cache_common_dir + File::SEPARATOR, '')
  264. file = sanitize_path(file, file_in_out)
  265. cache_common_sums[file] = `cd #{cache_common_dir} && sha256sum #{file}`
  266. raise "failed to sum #{file}" unless $? == 0
  267. end
  268. Dir.glob(File.join(cache_package_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  269. next if File.directory?(file_in_out)
  270. file = file_in_out.sub(cache_package_dir + File::SEPARATOR, '')
  271. file = sanitize_path(file, file_in_out)
  272. cache_package_sums[file] = `cd #{cache_package_dir} && sha256sum #{file}`
  273. raise "failed to sum #{file}" unless $? == 0
  274. end
  275. end
  276. out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
  277. in_manifest = in_sums.join('')
  278. cache_common_manifest = cache_common_sums.keys.sort.map { |key| cache_common_sums[key] }.join('')
  279. cache_package_manifest = cache_package_sums.keys.sort.map { |key| cache_package_sums[key] }.join('')
  280. # Use Omap to keep result deterministic
  281. report = YAML::Omap[
  282. 'out_manifest', out_manifest,
  283. 'in_manifest', in_manifest,
  284. 'base_manifests', base_manifests,
  285. 'cache_common_manifest', cache_common_manifest,
  286. 'cache_package_manifest', cache_package_manifest,
  287. ]
  288. result_file = "#{package_name}-res.yml"
  289. File.open(File.join(result_dir, result_file), "w") do |io|
  290. io.write report.to_yaml
  291. end
  292. system!("cd #{result_dir} && sha256sum #{result_file}") unless @options[:quiet]
  293. info "Done."