You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

gitian_updater.py 18KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496
  1. #!/usr/bin/python
  2. # Gitian Downloader - download/update and verify a gitian distribution package
  3. # This library is free software; you can redistribute it and/or
  4. # modify it under the terms of the GNU Library General Public
  5. # License as published by the Free Software Foundation; either
  6. # version 2 of the License, or (at your option) any later version.
  7. #
  8. # This library is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. # Library General Public License for more details.
  12. #
  13. # You should have received a copy of the GNU Library General Public
  14. # License along with this library; if not, write to the Free Software
  15. # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  16. import sys, os, subprocess
  17. from os import path
  18. import shutil
  19. import re
  20. import tempfile
  21. import atexit
  22. import urllib2
  23. import argparse
  24. import yaml
  25. import time
  26. from hashlib import sha256
  27. from zipfile import ZipFile
  28. from distutils.version import LooseVersion
  29. """downloader config sample:
  30. ---
  31. name: foo
  32. waiting_period: 24
  33. urls:
  34. - url: https://foo.org/gitian/foo.zip
  35. version_url: https://foo.org/gitian/foo.ver
  36. rss:
  37. - url: https://foo.org/gitian/foo.rss
  38. xpath: //item/link/text()
  39. pattern: foo-(\d+.\d+.\d+)-linux.zip
  40. signers:
  41. 0A82509767C7D4A5D14DA2301AE1D35043E08E54:
  42. weight: 40
  43. name: BlueMatt
  44. key: bluematt
  45. """
  46. inject_config_string = "INJECT" + "CONFIG"
  47. injected_config = """INJECTCONFIG"""
  48. have_injected_config = injected_config != inject_config_string
  49. quiet = 0
  50. def check_name_and_version(out_manifest, old_manifest):
  51. if out_manifest['name'] != old_manifest['name']:
  52. print>>sys.stderr, "The old directory has a manifest for a different package"
  53. sys.exit(3)
  54. if LooseVersion(out_manifest['release']) < LooseVersion(old_manifest['release']):
  55. if quiet <= 1:
  56. print>>sys.stderr, "This is a downgrade from version %s to %s"%(old_manifest['release'],out_manifest['release'])
  57. if not args.force:
  58. print>>sys.stderr, "Use --force if you really want to downgrade"
  59. sys.exit(4)
  60. elif LooseVersion(out_manifest['release']) == LooseVersion(old_manifest['release']):
  61. if quiet <= 1:
  62. print>>sys.stderr, "This is a reinstall of version %s"%(old_manifest['release'])
  63. else:
  64. if quiet == 0:
  65. print>>sys.stderr, "Upgrading from version %s to %s"%(old_manifest['release'],out_manifest['release'])
  66. def copy_to_destination(from_path, to_path, out_manifest, old_manifest):
  67. for root, dirs, files in os.walk(from_path, topdown = True):
  68. rel = path.relpath(root, from_path)
  69. if not path.exists(path.join(to_path, rel)):
  70. os.mkdir(path.normpath(path.join(to_path, rel)))
  71. for f in files:
  72. shutil.copy2(path.join(root, f), path.join(to_path, rel, f))
  73. if old_manifest:
  74. removed = set(old_manifest['sums'].keys()).difference(out_manifest['sums'].keys())
  75. for f in removed:
  76. if path.exists(path.join(to_path, f)):
  77. os.unlink(path.join(to_path, f))
  78. f = file(path.join(to_path, '.gitian-manifest'), 'w')
  79. yaml.dump(out_manifest, f)
  80. f.close()
  81. def sha256sum(path):
  82. h = sha256()
  83. f = open(path)
  84. while True:
  85. buf = f.read(10240)
  86. if buf == "":
  87. break
  88. h.update(buf)
  89. return h.hexdigest()
  90. def sanitize_path(dir_name, name, where):
  91. if re.search(r'[^/\w.-]', name):
  92. raise ValueError, "unsanitary path in %s"%(where)
  93. full_name = path.normpath(path.join(dir_name, name))
  94. if full_name.find(dir_name + os.sep) != 0:
  95. raise ValueError, "unsafe path in %s"%(where)
  96. def remove_temp(tdir):
  97. shutil.rmtree(tdir)
  98. def download(url, dest):
  99. if quiet == 0:
  100. print "Downloading from %s"%(url)
  101. file_name = url.split('/')[-1]
  102. u = urllib2.urlopen(url)
  103. f = open(dest, 'wb')
  104. meta = u.info()
  105. file_size = int(meta.getheaders("Content-Length")[0])
  106. if quiet == 0:
  107. print "Downloading: %s Bytes: %s"%(file_name, file_size)
  108. file_size_dl = 0
  109. block_sz = 65536
  110. while True:
  111. buffer = u.read(block_sz)
  112. if not buffer:
  113. break
  114. file_size_dl += len(buffer)
  115. f.write(buffer)
  116. status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
  117. status = status + chr(8)*(len(status)+1)
  118. if quiet == 0:
  119. print status,
  120. if quiet == 0:
  121. print
  122. f.close()
  123. def extract(dir_name, zip_path):
  124. zipfile = ZipFile(zip_path, 'r')
  125. files = []
  126. for name in zipfile.namelist():
  127. sanitize_path(dir_name, name, "zip file")
  128. zipfile.extractall(dir_name)
  129. zipfile.close()
  130. for name in zipfile.namelist():
  131. if path.isfile(path.join(dir_name, name)):
  132. files.append(path.normpath(name))
  133. return files
  134. def get_assertions(gpg_path, temp_dir, unpack_dir, file_names):
  135. assertions = {"build" : {}}
  136. sums = {}
  137. name = None
  138. release = None
  139. to_check = {}
  140. for file_name in file_names:
  141. sums[file_name] = sha256sum(os.path.join(unpack_dir, file_name))
  142. to_check[file_name] = 1
  143. out_manifest = False
  144. error = False
  145. optionals = None
  146. for file_name in file_names:
  147. if file_name.startswith("gitian"):
  148. del to_check[file_name]
  149. if file_name.endswith(".assert"):
  150. popen = subprocess.Popen([gpg_path, '--status-fd', '1', '--homedir', path.join(temp_dir, 'gpg'), '--verify', os.path.join(unpack_dir, file_name + '.sig'), os.path.join(unpack_dir, file_name)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  151. gpgout = popen.communicate()[0]
  152. retcode = popen.wait()
  153. if retcode != 0:
  154. if quiet <= 1:
  155. print>>sys.stderr, 'PGP verify failed for %s' %(file_name)
  156. error = True
  157. continue
  158. match = re.search(r'^\[GNUPG:\] VALIDSIG ([A-F0-9]+)', gpgout, re.M)
  159. assertions['build'][match.group(1)] = 1
  160. f = file(os.path.join(unpack_dir, file_name), 'r')
  161. assertion = yaml.load(f, OrderedDictYAMLLoader)
  162. f.close()
  163. if assertion['out_manifest']:
  164. if out_manifest:
  165. if out_manifest != assertion['out_manifest'] or release != assertion['release'] or name != assertion['name'] or optionals != assertion.get('optionals', []):
  166. print>>sys.stderr, 'not all out manifests/releases/names/optionals are identical'
  167. error = True
  168. continue
  169. else:
  170. out_manifest = assertion['out_manifest']
  171. release = assertion['release']
  172. name = assertion['name']
  173. optionals = assertion.get('optionals', [])
  174. if out_manifest:
  175. for line in out_manifest.split("\n"):
  176. if line != "":
  177. shasum = line[0:64]
  178. summed_file = line[66:]
  179. if not sums.has_key(summed_file):
  180. if not summed_file in optionals:
  181. print>>sys.stderr, "missing file %s" %(summed_file)
  182. error = True
  183. elif sums[summed_file] != shasum:
  184. print>>sys.stderr, "sha256sum mismatch on %s" %(summed_file)
  185. error = True
  186. else:
  187. del to_check[summed_file]
  188. if len(to_check) > 0 and quiet == 0:
  189. print>>sys.stderr, "Some of the files were not checksummed:"
  190. for key in to_check:
  191. print>>sys.stderr, " ", key
  192. else:
  193. print>>sys.stderr, 'No build assertions found'
  194. manifest = { 'sums' : sums, 'release' : release, 'name': name, 'optionals': optionals }
  195. return (not error, assertions, manifest)
  196. def import_keys(gpg_path, temp_dir, config):
  197. gpg_dir = path.join(temp_dir, 'gpg')
  198. os.mkdir(gpg_dir, 0700)
  199. signers = config['signers']
  200. for keyid in signers:
  201. key_path = path.join('gitian', signers[keyid]['key'] + '-key.pgp')
  202. popen = subprocess.Popen([gpg_path, '--status-fd', '1', '--homedir', gpg_dir, '--import', path.join(temp_dir, 'unpack', key_path)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  203. gpgout = popen.communicate(signers[keyid]['key'])[0]
  204. if popen.wait() != 0:
  205. print>>sys.stderr, 'Key %s failed to import'%(keyid)
  206. continue
  207. expected_keyid = keyid
  208. if signers[keyid].has_key('keyid'):
  209. expected_keyid = signers[keyid]['keyid']
  210. if gpgout.count(expected_keyid) == 0:
  211. print>>sys.stderr, 'Key file %s did not contain the key %s'%(key_path, keyid)
  212. if gpgout.count('IMPORT_OK') != 1 and quiet <= 1:
  213. print>>sys.stderr, 'Key file %s contained more than one key'%(key_path)
  214. def check_assertions(config, assertions):
  215. total_weight = 0
  216. signers = config['signers']
  217. if quiet == 0:
  218. print 'Signatures from:'
  219. for key in assertions['build']:
  220. if not signers.has_key(key):
  221. if quiet <= 1:
  222. print>>sys.stderr, 'key %s is not in config, skipping'%(key)
  223. continue
  224. if quiet == 0:
  225. print ' %s : weight %d'%(signers[key]['name'], signers[key]['weight'])
  226. total_weight += signers[key]['weight']
  227. if total_weight < config['minimum_weight']:
  228. print>>sys.stderr, "The total weight of signatures is %d, which is less than the minimum required %d"%(total_weight, config['minimum_weight'])
  229. return None
  230. return total_weight
  231. class OrderedDictYAMLLoader(yaml.Loader):
  232. """
  233. A YAML loader that loads ordered yaml maps into a dictionary.
  234. """
  235. def __init__(self, *args, **kwargs):
  236. yaml.Loader.__init__(self, *args, **kwargs)
  237. self.add_constructor(u'!omap', type(self).construct_yaml_map)
  238. def construct_yaml_map(self, node):
  239. data = dict()
  240. yield data
  241. for mapping in node.value:
  242. for key, value in mapping.value:
  243. key = self.construct_object(key)
  244. value = self.construct_object(value)
  245. data[key] = value
  246. def run():
  247. full_prog = sys.argv[0]
  248. prog = os.path.basename(full_prog)
  249. parser = argparse.ArgumentParser(description='Download a verify a gitian package')
  250. parser.add_argument('-u', '--url', metavar='URL', type=str, nargs='+', required=False,
  251. help='one or more URLs where the package can be found')
  252. parser.add_argument('-c', '--config', metavar='CONF', type=str, required=not have_injected_config,
  253. help='a configuration file')
  254. parser.add_argument('-d', '--dest', metavar='DEST', type=str, required=False,
  255. help='the destination directory for unpacking')
  256. parser.add_argument('-q', '--quiet', action='append_const', const=1, default=[], help='be quiet')
  257. parser.add_argument('-f', '--force', action='store_true', help='force downgrades and such')
  258. parser.add_argument('-n', '--dryrun', action='store_true', help='do not actually copy to destination')
  259. parser.add_argument('-m', '--customize', metavar='OUTPUT', type=str, help='generate a customized version of the script with the given config')
  260. parser.add_argument('-w', '--wait', type=float, metavar='HOURS', help='observe a waiting period or use zero for no waiting')
  261. parser.add_argument('-g', '--gpg', metavar='GPG', type=str, help='path to GnuPG')
  262. parser.add_argument('-p', '--post', metavar='COMMAND', type=str, help='Run after a successful install')
  263. args = parser.parse_args()
  264. quiet = len(args.quiet)
  265. if args.config:
  266. f = file(args.config, 'r')
  267. if args.customize:
  268. s = file(full_prog, 'r')
  269. script = s.read()
  270. s.close()
  271. config = f.read()
  272. script = script.replace(inject_config_string, config)
  273. s = file(args.customize, 'w')
  274. s.write(script)
  275. s.close()
  276. os.chmod(args.customize, 0750)
  277. sys.exit(0)
  278. config = yaml.safe_load(f)
  279. f.close()
  280. else:
  281. config = yaml.safe_load(injected_config)
  282. dest_path = args.dest
  283. if not dest_path:
  284. parser.error('argument -d/--dest is required unless -m is specified')
  285. if args.wait is not None:
  286. config['waiting_period'] = args.wait
  287. gpg_path = args.gpg
  288. if not gpg_path:
  289. gpg_path = 'gpg'
  290. rsses = []
  291. if args.url:
  292. urls = [{ 'url' : url, 'version_url' : None} for url in args.url]
  293. else:
  294. urls = config.get('urls')
  295. if not urls:
  296. parser.error('argument -u/--url is required since config does not specify it')
  297. if config.has_key('rss'):
  298. rsses = config['rss']
  299. # TODO: rss, atom, etc.
  300. old_manifest = None
  301. if path.exists(dest_path):
  302. files = os.listdir(dest_path)
  303. if path.dirname(full_prog) == dest_path:
  304. files.remove(prog)
  305. if not files.count('.gitian-manifest') and len(files) > 0:
  306. print>>sys.stderr, "destination already exists, no .gitian-manifest and directory not empty. Please empty destination."
  307. sys.exit(1)
  308. f = file(os.path.join(dest_path,'.gitian-manifest'), 'r')
  309. old_manifest = yaml.load(f, OrderedDictYAMLLoader)
  310. f.close()
  311. if config.get('waiting_period', 0) > 0:
  312. waiting_file = path.join(dest_path, '.gitian-waiting')
  313. if path.exists(waiting_file):
  314. f = file(waiting_file, 'r')
  315. waiting = yaml.load(f)
  316. f.close()
  317. wait_start = waiting['time']
  318. out_manifest = waiting['out_manifest']
  319. waiting_path = waiting['waiting_path']
  320. wait_time = wait_start + config['waiting_period'] * 3600 - time.time()
  321. if wait_time > 0:
  322. print>>sys.stderr, "Waiting another %.2f hours before applying update in %s"%(wait_time / 3600, waiting_path)
  323. sys.exit(100)
  324. os.remove(waiting_file)
  325. if args.dryrun:
  326. print>>sys.stderr, "Dry run, not copying"
  327. else:
  328. copy_to_destination(path.join(waiting_path, 'unpack'), dest_path, out_manifest, old_manifest)
  329. if args.post:
  330. os.system(args.post)
  331. if quiet == 0:
  332. print>>sys.stderr, "Copied from waiting area to destination"
  333. shutil.rmtree(waiting_path)
  334. sys.exit(0)
  335. temp_dir = tempfile.mkdtemp('', prog)
  336. atexit.register(remove_temp, temp_dir)
  337. package_file = path.join(temp_dir, 'package')
  338. downloaded = False
  339. checked = False
  340. if rsses:
  341. import libxml2
  342. for rss in rsses:
  343. try:
  344. feed = libxml2.parseDoc(urllib2.urlopen(rss['url']).read())
  345. url = None
  346. release = None
  347. # Find the first matching node
  348. for node in feed.xpathEval(rss['xpath']):
  349. m = re.search(rss['pattern'], str(node))
  350. if m:
  351. if len(m.groups()) > 0:
  352. release = m.group(1)
  353. url = str(node)
  354. break
  355. # Make sure it's a new release
  356. if old_manifest and release == old_manifest['release'] and not args.force:
  357. checked = True
  358. else:
  359. try:
  360. download(url, package_file)
  361. downloaded = True
  362. break
  363. except:
  364. print>>sys.stderr, "could not download from %s, trying next rss"%(url)
  365. pass
  366. except:
  367. print>>sys.stderr, "could read not from rss %s"%(rss)
  368. pass
  369. if not downloaded:
  370. for url in urls:
  371. try:
  372. release = None
  373. if url['version_url']:
  374. f = urllib2.urlopen(url['version_url'])
  375. release = f.read(100).strip()
  376. f.close()
  377. if old_manifest and release == old_manifest['release'] and not args.force:
  378. checked = True
  379. else:
  380. download(url['url'], package_file)
  381. downloaded = True
  382. except:
  383. print>>sys.stderr, "could not download from %s, trying next url"%(url)
  384. raise
  385. if not downloaded:
  386. if checked:
  387. if quiet == 0:
  388. print>>sys.stderr, "same release, not downloading"
  389. else:
  390. print>>sys.stderr, "out of places to try downloading from, try later"
  391. sys.exit(2)
  392. unpack_dir = path.join(temp_dir, 'unpack')
  393. files = extract(unpack_dir, package_file)
  394. import_keys(gpg_path, temp_dir, config)
  395. (success, assertions, out_manifest) = get_assertions(gpg_path, temp_dir, unpack_dir, files)
  396. if old_manifest:
  397. check_name_and_version(out_manifest, old_manifest)
  398. if not success and quiet <= 1:
  399. print>>sys.stderr, "There were errors getting assertions"
  400. total_weight = check_assertions(config, assertions)
  401. if total_weight is None:
  402. print>>sys.stderr, "There were errors checking assertions, build is untrusted, aborting"
  403. sys.exit(5)
  404. if quiet == 0:
  405. print>>sys.stderr, "Successful with signature weight %d"%(total_weight)
  406. if config.get('waiting_period', 0) > 0 and path.exists(dest_path):
  407. waiting_path = tempfile.mkdtemp('', prog)
  408. shutil.copytree(unpack_dir, path.join(waiting_path, 'unpack'))
  409. f = file(path.join(dest_path, '.gitian-waiting'), 'w')
  410. yaml.dump({'time': time.time(), 'out_manifest': out_manifest, 'waiting_path': waiting_path}, f)
  411. f.close()
  412. if quiet == 0:
  413. print>>sys.stderr, "Started waiting period"
  414. else:
  415. if args.dryrun:
  416. print>>sys.stderr, "Dry run, not copying"
  417. else:
  418. copy_to_destination(unpack_dir, dest_path, out_manifest, old_manifest)
  419. if args.post:
  420. os.system(args.post)
  421. if __name__ == '__main__':
  422. run()