You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

gbuild 14KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418
  1. #!/usr/bin/ruby
  2. require 'optparse'
  3. require 'yaml'
  4. require 'fileutils'
  5. require 'pathname'
  6. @options = {:num_procs => 2, :memory => 2000}
  7. @bitness = {
  8. 'i386' => 32,
  9. 'amd64' => 64,
  10. }
  11. @arches = {
  12. 'i386' => 'i386',
  13. 'amd64' => 'x86_64',
  14. }
  15. def system!(cmd)
  16. system(cmd) or raise "failed to run #{cmd}"
  17. end
  18. def sanitize(str, where)
  19. raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/)
  20. str
  21. end
  22. def sanitize_path(str, where)
  23. raise "unsanitary string in #{where}" if (str =~ /[^@\w\/.:+-]/)
  24. str
  25. end
  26. def info(str)
  27. puts str unless @options[:quiet]
  28. end
  29. def build_one_configuration(suite, arch, build_desc)
  30. FileUtils.rm_f("var/install.log")
  31. FileUtils.rm_f("var/build.log")
  32. bits = @bitness[arch] or raise "unknown architecture ${arch}"
  33. if ENV["USE_LXC"]
  34. ENV["LXC_ARCH"] = arch
  35. ENV["LXC_SUITE"] = suite
  36. end
  37. if ENV["USE_DOCKER"] and build_desc["distro"].eql? "centos"
  38. ontarget_root_extra = "-w /root"
  39. else
  40. ontarget_root_extra = ""
  41. end
  42. suitearch = "#{suite}-#{arch}"
  43. info "Stopping target if it is up"
  44. system "stop-target"
  45. sleep 1
  46. unless @options[:skip_image]
  47. info "Making a new image copy"
  48. system! "make-clean-vm --suite #{suite} --arch #{arch}"
  49. end
  50. info "Starting target"
  51. system! "start-target #{bits} #{suitearch}&"
  52. $stdout.write "Checking if target is up"
  53. (1..30).each do
  54. system "on-target true 2> /dev/null" and break
  55. sleep 2
  56. $stdout.write '.'
  57. end
  58. info ''
  59. system! "on-target true"
  60. system! "on-target -u root #{ontarget_root_extra} tee -a /etc/sudoers.d/#{ENV['DISTRO'] || 'ubuntu'} > /dev/null << EOF
  61. %#{ENV['DISTRO'] || 'ubuntu'} ALL=(ALL) NOPASSWD: ALL
  62. EOF" if build_desc["sudo"] and @options[:allow_sudo]
  63. info "Preparing build environment"
  64. system! "on-target setarch #{@arches[arch]} bash < target-bin/init-build.sh"
  65. build_desc["files"].each do |filename|
  66. filename = sanitize(filename, "files section")
  67. system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/"
  68. end
  69. if build_desc["enable_cache"]
  70. if File.directory?("cache/#{build_desc["name"]}")
  71. system! "copy-to-target #{@quiet_flag} cache/#{build_desc["name"]}/ cache/"
  72. end
  73. if File.directory?("cache/common")
  74. system! "copy-to-target #{@quiet_flag} cache/common/ cache/"
  75. end
  76. end
  77. if build_desc["multiarch"]
  78. info "Adding multiarch support (log in var/install.log)"
  79. for a in build_desc["multiarch"]
  80. system! "on-target -u root #{ontarget_root_extra} dpkg --add-architecture #{a} >> var/install.log 2>&1"
  81. end
  82. end
  83. case build_desc["distro"]
  84. when "centos"
  85. info "Updating yum repository (log in var/install.log)"
  86. system! "on-target -u root #{ontarget_root_extra} yum -y makecache fast >> var/install.log 2>&1"
  87. else
  88. info "Updating apt-get repository (log in var/install.log)"
  89. system! "on-target -u root #{ontarget_root_extra} apt-get update >> var/install.log 2>&1"
  90. end
  91. info "Installing additional packages (log in var/install.log)"
  92. case build_desc["distro"]
  93. when "centos"
  94. system! "on-target -u root #{ontarget_root_extra} yum -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"
  95. else
  96. system! "on-target -u root #{ontarget_root_extra} -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install #{build_desc["packages"].join(" ")} >> var/install.log 2>&1"
  97. end
  98. if build_desc["alternatives"]
  99. info "Set alternatives (log in var/install.log)"
  100. for a in build_desc["alternatives"]
  101. system! "on-target -u root #{ontarget_root_extra} update-alternatives --set #{a["package"]} #{a["path"]} >> var/install.log 2>&1"
  102. end
  103. end
  104. if @options[:upgrade] || system("on-target -u root #{ontarget_root_extra} '[ ! -e /var/cache/gitian/initial-upgrade ]'")
  105. info "Upgrading system, may take a while (log in var/install.log)"
  106. case build_desc["distro"]
  107. when "centos"
  108. system! "on-target -u root #{ontarget_root_extra} mkdir -p /var/cache/gitian"
  109. system! "on-target -u root #{ontarget_root_extra} yum -y update > var/upgrade.log 2>&1"
  110. system! "copy-to-target #{@quiet_flag} var/upgrade.log /var/cache/gitian/upgrade.log"
  111. system! "on-target -u root #{ontarget_root_extra} touch /var/cache/gitian/initial-upgrade"
  112. else
  113. system! "on-target -u root #{ontarget_root_extra} bash < target-bin/upgrade-system.sh >> var/install.log 2>&1"
  114. end
  115. end
  116. info "Creating package manifest"
  117. case build_desc["distro"]
  118. when "centos"
  119. system! "on-target -u root #{ontarget_root_extra} yumdb get checksum_data | awk '/checksum_data =/ { print $3, package; next } { package=$1 }' | sort --key 2 > var/base-#{suitearch}.manifest"
  120. else
  121. system! "on-target -u root #{ontarget_root_extra} bash < target-bin/grab-packages.sh > var/base-#{suitearch}.manifest"
  122. end
  123. info "Creating build script (var/build-script)"
  124. File.open("var/build-script", "w") do |script|
  125. script.puts "#!/bin/bash"
  126. script.puts "set -e"
  127. script.puts "export LANG='en_US.UTF-8'"
  128. script.puts "export LC_ALL='en_US.UTF-8'"
  129. script.puts "umask 002"
  130. script.puts "export OUTDIR=$HOME/out"
  131. script.puts "GBUILD_BITS=#{bits}"
  132. if build_desc["enable_cache"]
  133. script.puts "GBUILD_CACHE_ENABLED=1"
  134. script.puts "GBUILD_PACKAGE_CACHE=$HOME/cache/#{build_desc["name"]}"
  135. script.puts "GBUILD_COMMON_CACHE=$HOME/cache/common"
  136. end
  137. script.puts "MAKEOPTS=(-j#{@options[:num_procs]})"
  138. script.puts
  139. author_date = nil
  140. build_desc["remotes"].each do |remote|
  141. dir = sanitize(remote["dir"], remote["dir"])
  142. author_date = `cd inputs/#{dir} > /dev/null && git log --format=@%at -1 | date +"%F %T" -u -f -`.strip
  143. raise "error looking up author date in #{dir}" unless $?.exitstatus == 0
  144. system! "copy-to-target #{@quiet_flag} inputs/#{dir} build/"
  145. script.puts "(cd build/#{dir} && git reset -q --hard && git clean -q -f -d)"
  146. end
  147. script.puts
  148. ref_datetime = build_desc["reference_datetime"] || author_date
  149. (ref_date, ref_time) = ref_datetime.split
  150. script.puts "REFERENCE_DATETIME='#{ref_datetime}'"
  151. script.puts "REFERENCE_DATE='#{ref_date}'"
  152. script.puts "REFERENCE_TIME='#{ref_time}'"
  153. script.puts
  154. script.puts "cd build"
  155. script.puts build_desc["script"]
  156. end
  157. info "Running build script (log in var/build.log)"
  158. system! "on-target setarch #{@arches[arch]} bash -x < var/build-script > var/build.log 2>&1"
  159. end
  160. ################################
  161. OptionParser.new do |opts|
  162. opts.banner = "Usage: build [options] <build-description>.yml"
  163. opts.on("--allow-sudo", "override SECURITY on the target VM and allow the use of sudo with no password for the default user") do |v|
  164. @options[:allow_sudo] = v
  165. end
  166. opts.on("-i", "--skip-image", "reuse current target image") do |v|
  167. @options[:skip_image] = v
  168. end
  169. opts.on("--upgrade", "upgrade guest with latest packages") do |v|
  170. @options[:upgrade] = v
  171. end
  172. opts.on("-q", "--quiet", "be quiet") do |v|
  173. @options[:quiet] = v
  174. end
  175. opts.on("-j PROCS", "--num-make PROCS", "number of processes to use") do |v|
  176. @options[:num_procs] = v
  177. end
  178. opts.on("-m MEM", "--memory MEM", "memory to allocate in MiB") do |v|
  179. @options[:memory] = v
  180. end
  181. opts.on("-c PAIRS", "--commit PAIRS", "comma separated list of DIRECTORY=COMMIT pairs") do |v|
  182. @options[:commit] = v
  183. end
  184. opts.on("-u PAIRS", "--url PAIRS", "comma separated list of DIRECTORY=URL pairs") do |v|
  185. @options[:url] = v
  186. end
  187. opts.on("-o", "--cache-read-only", "only use existing cache files, do not update them") do |v|
  188. @options[:cache_ro] = v
  189. end
  190. end.parse!
  191. if !ENV["USE_LXC"] and !ENV["USE_DOCKER"] and !ENV["USE_VBOX"] and !File.exist?("/dev/kvm")
  192. $stderr.puts "\n************* WARNING: kvm not loaded, this will probably not work out\n\n"
  193. end
  194. base_dir = Pathname.new(__FILE__).expand_path.dirname.parent
  195. libexec_dir = base_dir + 'libexec'
  196. ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH']
  197. ENV['GITIAN_BASE'] = base_dir.to_s
  198. ENV['NPROCS'] = @options[:num_procs].to_s
  199. ENV['VMEM'] = @options[:memory].to_s
  200. @quiet_flag = @options[:quiet] ? "-q" : ""
  201. build_desc_file = ARGV.shift or raise "must supply YAML build description file"
  202. build_desc = YAML.load_file(build_desc_file)
  203. in_sums = []
  204. build_dir = 'build'
  205. result_dir = 'result'
  206. cache_dir = 'cache'
  207. work_dir = 'var'
  208. enable_cache = build_desc["enable_cache"]
  209. FileUtils.rm_rf(build_dir)
  210. FileUtils.mkdir(build_dir)
  211. FileUtils.mkdir_p(result_dir)
  212. FileUtils.mkdir_p(work_dir)
  213. package_name = build_desc["name"] or raise "must supply name"
  214. package_name = sanitize(package_name, "package name")
  215. if enable_cache
  216. FileUtils.mkdir_p(File.join(cache_dir, "common"))
  217. FileUtils.mkdir_p(File.join(cache_dir, package_name))
  218. end
  219. distro = build_desc["distro"] || "ubuntu"
  220. suites = build_desc["suites"] or raise "must supply suites"
  221. archs = build_desc["architectures"] or raise "must supply architectures"
  222. build_desc["reference_datetime"] or build_desc["remotes"].size > 0 or raise "must supply `reference_datetime` or `remotes`"
  223. docker_image_digests = build_desc["docker_image_digests"] || []
  224. # if docker_image_digests are supplied, it must be the same length as suites
  225. if docker_image_digests.size > 0 and suites.size != docker_image_digests.size
  226. raise "`suites` and `docker_image_digests` must both be the same size if both are supplied"
  227. elsif ENV["USE_DOCKER"] and docker_image_digests.size > 0 and suites.size == docker_image_digests.size
  228. suites = docker_image_digests
  229. end
  230. ENV['DISTRO'] = distro
  231. desc_sum = `sha256sum #{build_desc_file}`
  232. desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
  233. in_sums << desc_sum
  234. build_desc["files"].each do |filename|
  235. filename = sanitize(filename, "files section")
  236. in_sums << `cd inputs && sha256sum #{filename}`
  237. end
  238. commits = {}
  239. if @options[:commit]
  240. @options[:commit].split(',').each do |pair|
  241. (dir, commit) = pair.split('=')
  242. commits[dir] = commit
  243. end
  244. end
  245. urls = {}
  246. if @options[:url]
  247. @options[:url].split(',').each do |pair|
  248. (dir, url) = pair.split('=')
  249. urls[dir] = url
  250. end
  251. end
  252. build_desc["remotes"].each do |remote|
  253. if !remote["commit"]
  254. remote["commit"] = commits[remote["dir"]]
  255. raise "must specify a commit for directory #{remote["dir"]}" unless remote["commit"]
  256. end
  257. if urls[remote["dir"]]
  258. remote["url"] = urls[remote["dir"]]
  259. end
  260. dir = sanitize(remote["dir"], remote["dir"])
  261. unless File.exist?("inputs/#{dir}")
  262. system!("git init inputs/#{dir}")
  263. end
  264. system!("cd inputs/#{dir} && git fetch --update-head-ok #{sanitize_path(remote["url"], remote["url"])} +refs/tags/*:refs/tags/* +refs/heads/*:refs/heads/*")
  265. commit = sanitize(remote["commit"], remote["commit"])
  266. commit = `cd inputs/#{dir} > /dev/null && git log --format=%H -1 #{commit}`.strip
  267. raise "error looking up commit for tag #{remote["commit"]}" unless $?.exitstatus == 0
  268. info("commit is #{commit}")
  269. system!("cd inputs/#{dir} && git checkout -q #{commit}")
  270. system!("cd inputs/#{dir} && git submodule update --init --recursive --force")
  271. in_sums << "git:#{commit} #{dir}"
  272. end
  273. base_manifests = YAML::Omap.new
  274. suites.each do |suite|
  275. suite = sanitize(suite, "suite")
  276. archs.each do |arch|
  277. info "--- Building for #{suite} #{arch} ---"
  278. arch = sanitize(arch, "architecture")
  279. # Build!
  280. build_one_configuration(suite, arch, build_desc)
  281. info "Grabbing results from target"
  282. system! "copy-from-target #{@quiet_flag} out #{build_dir}"
  283. if enable_cache && !@options[:cache_ro]
  284. info "Grabbing cache from target"
  285. system! "copy-from-target #{@quiet_flag} cache/#{package_name}/ #{cache_dir}"
  286. system! "copy-from-target #{@quiet_flag} cache/common/ #{cache_dir}"
  287. end
  288. base_manifest = File.read("var/base-#{suite}-#{arch}.manifest")
  289. base_manifests["#{suite}-#{arch}"] = base_manifest
  290. end
  291. end
  292. out_dir = File.join(build_dir, "out")
  293. out_sums = {}
  294. cache_common_dir = File.join(cache_dir, "common")
  295. cache_package_dir = File.join(cache_dir, "#{package_name}")
  296. cache_common_sums = {}
  297. cache_package_sums = {}
  298. info "Generating report"
  299. Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  300. next if File.directory?(file_in_out)
  301. file = file_in_out.sub(out_dir + File::SEPARATOR, '')
  302. file = sanitize_path(file, file_in_out)
  303. out_sums[file] = `cd #{out_dir} > /dev/null && sha256sum #{file}`
  304. raise "failed to sum #{file}" unless $? == 0
  305. puts out_sums[file] unless @options[:quiet]
  306. end
  307. if enable_cache
  308. Dir.glob(File.join(cache_common_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  309. next if File.directory?(file_in_out)
  310. file = file_in_out.sub(cache_common_dir + File::SEPARATOR, '')
  311. file = sanitize_path(file, file_in_out)
  312. cache_common_sums[file] = `cd #{cache_common_dir} > /dev/null && sha256sum #{file}`
  313. raise "failed to sum #{file}" unless $? == 0
  314. end
  315. Dir.glob(File.join(cache_package_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
  316. next if File.directory?(file_in_out)
  317. file = file_in_out.sub(cache_package_dir + File::SEPARATOR, '')
  318. file = sanitize_path(file, file_in_out)
  319. cache_package_sums[file] = `cd #{cache_package_dir} > /dev/null && sha256sum #{file}`
  320. raise "failed to sum #{file}" unless $? == 0
  321. end
  322. end
  323. out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
  324. in_manifest = in_sums.join('')
  325. cache_common_manifest = cache_common_sums.keys.sort.map { |key| cache_common_sums[key] }.join('')
  326. cache_package_manifest = cache_package_sums.keys.sort.map { |key| cache_package_sums[key] }.join('')
  327. # Use Omap to keep result deterministic
  328. report = YAML::Omap[
  329. 'out_manifest', out_manifest,
  330. 'in_manifest', in_manifest,
  331. 'base_manifests', base_manifests,
  332. 'cache_common_manifest', cache_common_manifest,
  333. 'cache_package_manifest', cache_package_manifest,
  334. ]
  335. result_file = "#{package_name}-res.yml"
  336. File.open(File.join(result_dir, result_file), "w") do |io|
  337. io.write report.to_yaml
  338. end
  339. system!("cd #{result_dir} && sha256sum #{result_file}") unless @options[:quiet]
  340. info "Done."