You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

gbuild 12KB

10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382
  1. #!/usr/bin/ruby
  2. require 'optparse'
  3. require 'yaml'
  4. require 'fileutils'
  5. require 'pathname'
  6. @options = {:num_procs => 2, :memory => 2000}
  7. @bitness = {
  8. 'i386' => 32,
  9. 'amd64' => 64,
  10. }
  11. @arches = {
  12. 'i386' => 'i386',
  13. 'amd64' => 'x86_64',
  14. }
  15. def system!(cmd)
  16. system(cmd) or raise "failed to run #{cmd}"
  17. end
  18. def sanitize(str, where)
  19. raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/)
  20. str
  21. end
  22. def sanitize_path(str, where)
  23. raise "unsanitary string in #{where}" if (str =~ /[^@\w\/.:+-]/)
  24. str
  25. end
  26. def info(str)
  27. puts str unless @options[:quiet]
  28. end
  29. def build_one_configuration(suite, arch, build_desc)
  30. FileUtils.rm_f("var/install.log")
  31. FileUtils.rm_f("var/build.log")
  32. bits = @bitness[arch] or raise "unknown architecture ${arch}"
  33. if ENV["USE_LXC"]
  34. ENV["LXC_ARCH"] = arch
  35. ENV["LXC_SUITE"] = suite
  36. end
  37. suitearch = "#{suite}-#{arch}"
  38. info "Stopping target if it is up"
  39. system "stop-target"
  40. sleep 1
  41. unless @options[:skip_image]
  42. info "Making a new image copy"
  43. system! "make-clean-vm --suite #{suite} --arch #{arch}"
  44. end
  45. info "Starting target"
  46. system! "start-target #{bits} #{suitearch}&"
  47. $stdout.write "Checking if target is up"
  48. (1..30).each do
  49. system "on-target true 2> /dev/null" and break
  50. sleep 2
  51. $stdout.write '.'
  52. end
  53. info ''
  54. system! "on-target true"
  55. system! "on-target -u root tee -a /etc/sudoers.d/#{ENV['DISTRO'] || 'ubuntu'} > /dev/null << EOF
  56. %#{ENV['DISTRO'] || 'ubuntu'} ALL=(ALL) NOPASSWD: ALL
  57. EOF" if build_desc["sudo"] and @options[:allow_sudo]
  58. info "Preparing build environment"
  59. system! "on-target setarch #{@arches[arch]} bash < target-bin/init-build.sh"
  60. build_desc["files"].each do |filename|
  61. filename = sanitize(filename, "files section")
  62. system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/"
  63. end
  64. if build_desc["enable_cache"]
  65. if File.directory?("cache/#{build_desc["name"]}")
  66. system! "copy-to-target #{@quiet_flag} cache/#{build_desc["name"]}/ cache/"
  67. end
  68. if File.directory?("cache/common")
  69. system! "copy-to-target #{@quiet_flag} cache/common/ cache/"
  70. end
  71. end
  72. if build_desc["multiarch"]
  73. info "Adding multiarch support (log in var/install.log)"
  74. for a in build_desc["multiarch"]
  75. system! "on-target -u root dpkg --add-architecture #{a} >> var/install.log 2>&1"
  76. end
  77. end
  78. info "Updating apt-get repository (log in var/install.log)"
  79. system! "on-target -u root apt-get update >> var/install.log 2>&1"
  80. info "Installing additional packages (log in var/install.log)"
  81. system! "on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install #{build_desc["packages"].join(" ")} >> var/install.log 2>&1"
  82. if build_desc["alternatives"]
  83. info "Set alternatives (log in var/install.log)"
  84. for a in build_desc["alternatives"]
  85. system! "on-target -u root update-alternatives --set #{a["package"]} #{a["path"]} >> var/install.log 2>&1"
  86. end
  87. end
  88. if @options[:upgrade] || system("on-target -u root '[ ! -e /var/cache/gitian/initial-upgrade ]'")
  89. info "Upgrading system, may take a while (log in var/install.log)"
  90. system! "on-target -u root bash < target-bin/upgrade-system.sh >> var/install.log 2>&1"
  91. end
  92. info "Creating package manifest"
  93. system! "on-target -u root bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest"
  94. info "Creating build script (var/build-script)"
  95. File.open("var/build-script", "w") do |script|
  96. script.puts "#!/bin/bash"
  97. script.puts "set -e"
  98. script.puts "export LANG='en_US.UTF-8'"
  99. script.puts "export LC_ALL='en_US.UTF-8'"
  100. script.puts "umask 002"
  101. script.puts "export OUTDIR=$HOME/out"
  102. script.puts "GBUILD_BITS=#{bits}"
  103. if build_desc["enable_cache"]
  104. script.puts "GBUILD_CACHE_ENABLED=1"
  105. script.puts "GBUILD_PACKAGE_CACHE=$HOME/cache/#{build_desc["name"]}"
  106. script.puts "GBUILD_COMMON_CACHE=$HOME/cache/common"
  107. end
  108. script.puts "MAKEOPTS=(-j#{@options[:num_procs]})"
  109. script.puts
  110. author_date = nil
  111. build_desc["remotes"].each do |remote|
  112. dir = sanitize(remote["dir"], remote["dir"])
  113. author_date = `cd inputs/#{dir} && git log --format=@%at -1 | date +"%F %T" -u -f -`.strip
  114. raise "error looking up author date in #{dir}" unless $?.exitstatus == 0
  115. system! "copy-to-target #{@quiet_flag} inputs/#{dir} build/"
  116. script.puts "(cd build/#{dir} && git reset -q --hard && git clean -q -f -d)"
  117. end
  118. script.puts
  119. ref_datetime = build_desc["reference_datetime"] || author_date
  120. (ref_date, ref_time) = ref_datetime.split
  121. script.puts "REFERENCE_DATETIME='#{ref_datetime}'"
  122. script.puts "REFERENCE_DATE='#{ref_date}'"
  123. script.puts "REFERENCE_TIME='#{ref_time}'"
  124. script.puts
  125. script.puts "cd build"
  126. script.puts build_desc["script"]
  127. end
  128. info "Running build script (log in var/build.log)"
  129. system! "on-target setarch #{@arches[arch]} bash -x < var/build-script > var/build.log 2>&1"
  130. end
  131. ################################
  132. OptionParser.new do |opts|
  133. opts.banner = "Usage: build [options] <build-description>.yml"
  134. opts.on("--allow-sudo", "override SECURITY on the target VM and allow the use of sudo with no password for the default user") do |v|
  135. @options[:allow_sudo] = v
  136. end
  137. opts.on("-i", "--skip-image", "reuse current target image") do |v|
  138. @options[:skip_image] = v
  139. end
  140. opts.on("--upgrade", "upgrade guest with latest packages") do |v|
  141. @options[:upgrade] = v
  142. end
  143. opts.on("-q", "--quiet", "be quiet") do |v|
  144. @options[:quiet] = v
  145. end
  146. opts.on("-j PROCS", "--num-make PROCS", "number of processes to use") do |v|
  147. @options[:num_procs] = v
  148. end
  149. opts.on("-m MEM", "--memory MEM", "memory to allocate in MiB") do |v|
  150. @options[:memory] = v
  151. end
  152. opts.on("-c PAIRS", "--commit PAIRS", "comma separated list of DIRECTORY=COMMIT pairs") do |v|
  153. @options[:commit] = v
  154. end
  155. opts.on("-u PAIRS", "--url PAIRS", "comma separated list of DIRECTORY=URL pairs") do |v|
  156. @options[:url] = v
  157. end
  158. opts.on("-o", "--cache-read-only", "only use existing cache files, do not update them") do |v|
  159. @options[:cache_ro] = v
  160. end
  161. end.parse!
  162. if !ENV["USE_LXC"] and !ENV["USE_DOCKER"] and !ENV["USE_VBOX"] and !File.exist?("/dev/kvm")
  163. $stderr.puts "\n************* WARNING: kvm not loaded, this will probably not work out\n\n"
  164. end
  165. base_dir = Pathname.new(__FILE__).expand_path.dirname.parent
  166. libexec_dir = base_dir + 'libexec'
  167. ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH']
  168. ENV['GITIAN_BASE'] = base_dir.to_s
  169. ENV['NPROCS'] = @options[:num_procs].to_s
  170. ENV['VMEM'] = @options[:memory].to_s
  171. @quiet_flag = @options[:quiet] ? "-q" : ""
  172. build_desc_file = ARGV.shift or raise "must supply YAML build description file"
  173. build_desc = YAML.load_file(build_desc_file)
  174. in_sums = []
  175. build_dir = 'build'
  176. result_dir = 'result'
  177. cache_dir = 'cache'
  178. enable_cache = build_desc["enable_cache"]
  179. FileUtils.rm_rf(build_dir)
  180. FileUtils.mkdir(build_dir)
  181. FileUtils.mkdir_p(result_dir)
  182. package_name = build_desc["name"] or raise "must supply name"
  183. package_name = sanitize(package_name, "package name")
  184. if enable_cache
  185. FileUtils.mkdir_p(File.join(cache_dir, "common"))
  186. FileUtils.mkdir_p(File.join(cache_dir, package_name))
  187. end
  188. distro = build_desc["distro"] || "ubuntu"
  189. suites = build_desc["suites"] or raise "must supply suites"
  190. archs = build_desc["architectures"] or raise "must supply architectures"
  191. build_desc["reference_datetime"] or build_desc["remotes"].size > 0 or raise "must supply `reference_datetime` or `remotes`"
  192. docker_image_digests = build_desc["docker_image_digests"] || []
  193. # if docker_image_digests are supplied, it must be the same length as suites
  194. if docker_image_digests.size > 0 and suites.size != docker_image_digests.size
  195. raise "`suites` and `docker_image_digests` must both be the same size if both are supplied"
  196. elsif ENV["USE_DOCKER"] and docker_image_digests.size > 0 and suites.size == docker_image_digests.size
  197. suites = docker_image_digests
  198. end
  199. ENV['DISTRO'] = distro
  200. desc_sum = `sha256sum #{build_desc_file}`
  201. desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
  202. in_sums << desc_sum
  203. build_desc["files"].each do |filename|
  204. filename = sanitize(filename, "files section")
  205. in_sums << `cd inputs && sha256sum #{filename}`
  206. end
  207. commits = {}
  208. if @options[:commit]
  209. @options[:commit].split(',').each do |pair|
  210. (dir, commit) = pair.split('=')
  211. commits[dir] = commit
  212. end
  213. end
  214. urls = {}
  215. if @options[:url]
  216. @options[:url].split(',').each do |pair|
  217. (dir, url) = pair.split('=')
  218. urls[dir] = url
  219. end
  220. end
  221. build_desc["remotes"].each do |remote|
  222. if !remote["commit"]
  223. remote["commit"] = commits[remote["dir"]]
  224. raise "must specify a commit for directory #{remote["dir"]}" unless remote["commit"]
  225. end
  226. if urls[remote["dir"]]
  227. remote["url"] = urls[remote["dir"]]
  228. end
  229. dir = sanitize(remote["dir"], remote["dir"])
  230. unless File.exist?("inputs/#{dir}")
  231. system!("git init inputs/#{dir}")
  232. end
  233. system!("cd inputs/#{dir} && git fetch --update-head-ok #{sanitize_path(remote["url"], remote["url"])} +refs/tags/*:refs/tags/* +refs/heads/*:refs/heads/*")
  234. commit = sanitize(remote["commit"], remote["commit"])
  235. commit = `cd inputs/#{dir} && git log --format=%H -1 #{commit}`.strip
  236. raise "error looking up commit for tag #{remote["commit"]}" unless $?.exitstatus == 0
  237. system!("cd inputs/#{dir} && git checkout -q #{commit}")
  238. system!("cd inputs/#{dir} && git submodule update --init --recursive --force")
  239. in_sums << "git:#{commit} #{dir}"
  240. end
  241. base_manifests = YAML::Omap.new
  242. suites.each do |suite|
  243. suite = sanitize(suite, "suite")
  244. archs.each do |arch|
  245. info "--- Building for #{suite} #{arch} ---"
  246. arch = sanitize(arch, "architecture")
  247. # Build!
  248. build_one_configuration(suite, arch, build_desc)
  249. info "Grabbing results from target"
  250. system! "copy-from-target #{@quiet_flag} out #{build_dir}"
  251. if enable_cache && !@options[:cache_ro]
  252. info "Grabbing cache from target"
  253. system! "copy-from-target #{@quiet_flag} cache/#{package_name}/ #{cache_dir}"
  254. system! "copy-from-target #{@quiet_flag} cache/common/ #{cache_dir}"
  255. end
  256. base_manifest = File.read("var/base-#{suite}-#{arch}.manifest")
  257. base_manifests["#{suite}-#{arch}"] = base_manifest
  258. end
  259. end
  260. out_dir = File.join(build_dir, "out")
  261. out_sums = {}
  262. cache_common_dir = File.join(cache_dir, "common")
  263. cache_package_dir = File.join(cache_dir, "#{package_name}")
  264. cache_common_sums = {}
  265. cache_package_sums = {}
  266. info "Generating report"
  267. Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  268. next if File.directory?(file_in_out)
  269. file = file_in_out.sub(out_dir + File::SEPARATOR, '')
  270. file = sanitize_path(file, file_in_out)
  271. out_sums[file] = `cd #{out_dir} && sha256sum #{file}`
  272. raise "failed to sum #{file}" unless $? == 0
  273. puts out_sums[file] unless @options[:quiet]
  274. end
  275. if enable_cache
  276. Dir.glob(File.join(cache_common_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  277. next if File.directory?(file_in_out)
  278. file = file_in_out.sub(cache_common_dir + File::SEPARATOR, '')
  279. file = sanitize_path(file, file_in_out)
  280. cache_common_sums[file] = `cd #{cache_common_dir} && sha256sum #{file}`
  281. raise "failed to sum #{file}" unless $? == 0
  282. end
  283. Dir.glob(File.join(cache_package_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  284. next if File.directory?(file_in_out)
  285. file = file_in_out.sub(cache_package_dir + File::SEPARATOR, '')
  286. file = sanitize_path(file, file_in_out)
  287. cache_package_sums[file] = `cd #{cache_package_dir} && sha256sum #{file}`
  288. raise "failed to sum #{file}" unless $? == 0
  289. end
  290. end
  291. out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
  292. in_manifest = in_sums.join('')
  293. cache_common_manifest = cache_common_sums.keys.sort.map { |key| cache_common_sums[key] }.join('')
  294. cache_package_manifest = cache_package_sums.keys.sort.map { |key| cache_package_sums[key] }.join('')
  295. # Use Omap to keep result deterministic
  296. report = YAML::Omap[
  297. 'out_manifest', out_manifest,
  298. 'in_manifest', in_manifest,
  299. 'base_manifests', base_manifests,
  300. 'cache_common_manifest', cache_common_manifest,
  301. 'cache_package_manifest', cache_package_manifest,
  302. ]
  303. result_file = "#{package_name}-res.yml"
  304. File.open(File.join(result_dir, result_file), "w") do |io|
  305. io.write report.to_yaml
  306. end
  307. system!("cd #{result_dir} && sha256sum #{result_file}") unless @options[:quiet]
  308. info "Done."