You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

gitian-updater 11KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323
  1. #!/usr/bin/python
  2. # Gitian Downloader - download/update and verify a gitian distribution package
  3. # This library is free software; you can redistribute it and/or
  4. # modify it under the terms of the GNU Library General Public
  5. # License as published by the Free Software Foundation; either
  6. # version 2 of the License, or (at your option) any later version.
  7. #
  8. # This library is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. # Library General Public License for more details.
  12. #
  13. # You should have received a copy of the GNU Library General Public
  14. # License along with this library; if not, write to the Free Software
  15. # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  16. import sys, os, subprocess
  17. from os import path
  18. import shutil
  19. import re
  20. import tempfile
  21. import atexit
  22. import urllib2
  23. import libxml2
  24. import argparse
  25. import yaml
  26. from zipfile import ZipFile
  27. inject_config_string = "INJECT" + "CONFIG"
  28. injected_config = """INJECTCONFIG"""
  29. have_injected_config = injected_config != inject_config_string
  30. quiet = 0
  31. def sanitize_path(dir_name, name, where):
  32. if re.search(r'[^/\w.-]', name):
  33. raise ValueError, "unsanitary path in %s"%(where)
  34. full_name = path.normpath(path.join(dir_name, name))
  35. if full_name.find(dir_name + os.sep) != 0:
  36. raise ValueError, "unsafe path in %s"%(where)
  37. def remove_temp(tdir):
  38. shutil.rmtree(tdir)
  39. def download(url, dest):
  40. if quiet == 0:
  41. print "Downloading from %s"%(url)
  42. file_name = url.split('/')[-1]
  43. u = urllib2.urlopen(url)
  44. f = open(dest, 'w')
  45. meta = u.info()
  46. file_size = int(meta.getheaders("Content-Length")[0])
  47. if quiet == 0:
  48. print "Downloading: %s Bytes: %s"%(file_name, file_size)
  49. file_size_dl = 0
  50. block_sz = 65536
  51. while True:
  52. buffer = u.read(block_sz)
  53. if not buffer:
  54. break
  55. file_size_dl += len(buffer)
  56. f.write(buffer)
  57. status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
  58. status = status + chr(8)*(len(status)+1)
  59. if quiet == 0:
  60. print status,
  61. if quiet == 0:
  62. print
  63. f.close()
  64. def extract(dir_name, zip_path):
  65. zipfile = ZipFile(zip_path, 'r')
  66. files = []
  67. for name in zipfile.namelist():
  68. sanitize_path(dir_name, name, "zip file")
  69. zipfile.extractall(dir_name)
  70. zipfile.close()
  71. for name in zipfile.namelist():
  72. if path.isfile(path.join(dir_name, name)):
  73. files.append(path.normpath(name))
  74. return files
  75. def get_assertions(temp_dir, unpack_dir, file_names):
  76. assertions = {"build" : {}}
  77. sums = {}
  78. to_check = {}
  79. for file_name in file_names:
  80. shasum = subprocess.Popen(["sha256sum", '-b', os.path.join(unpack_dir, file_name)], stdout=subprocess.PIPE).communicate()[0][0:64]
  81. sums[file_name] = shasum
  82. to_check[file_name] = 1
  83. out_manifest = False
  84. error = False
  85. for file_name in file_names:
  86. if file_name.startswith("gitian"):
  87. del to_check[file_name]
  88. if file_name.endswith(".assert"):
  89. popen = subprocess.Popen(["gpg", '--status-fd', '1', '--homedir', path.join(temp_dir, 'gpg'), '--verify', os.path.join(unpack_dir, file_name + '.pgp'), os.path.join(unpack_dir, file_name)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  90. gpgout = popen.communicate()[0]
  91. retcode = popen.wait()
  92. if retcode != 0:
  93. if quiet <= 1:
  94. print>>sys.stderr, 'PGP verify failed for %s' %(file_name)
  95. error = True
  96. continue
  97. match = re.search(r'^\[GNUPG:\] VALIDSIG ([A-F0-9]+)', gpgout, re.M)
  98. assertions['build'][match.group(1)] = 1
  99. f = file(os.path.join(unpack_dir, file_name), 'r')
  100. assertion = yaml.load(f, OrderedDictYAMLLoader)
  101. f.close()
  102. if assertion['out_manifest']:
  103. if out_manifest:
  104. if out_manifest != assertion['out_manifest']:
  105. print>>sys.stderr, 'not all out manifests are identical'
  106. error = True
  107. continue
  108. else:
  109. out_manifest = assertion['out_manifest']
  110. if out_manifest:
  111. for line in out_manifest.split("\n"):
  112. if line != "":
  113. shasum = line[0:64]
  114. summed_file = line[66:]
  115. if sums[summed_file] != shasum:
  116. print>>sys.stderr, "sha256sum mismatch on %s" %(summed_file)
  117. error = True
  118. del to_check[summed_file]
  119. if len(to_check) > 0 and quiet == 0:
  120. print>>sys.stderr, "Some of the files were not checksummed:"
  121. for key in to_check:
  122. print>>sys.stderr, " ", key
  123. else:
  124. print>>sys.stderr, 'No build assertions found'
  125. error = True
  126. return (not error, assertions, sums)
  127. def import_keys(temp_dir, config):
  128. gpg_dir = path.join(temp_dir, 'gpg')
  129. os.mkdir(gpg_dir, 0700)
  130. signers = config['signers']
  131. for keyid in signers:
  132. key_path = path.join('gitian', signers[keyid]['key'] + '-key.pgp')
  133. popen = subprocess.Popen(['gpg', '--status-fd', '1', '--homedir', gpg_dir, '--import', path.join(temp_dir, 'unpack', key_path)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  134. gpgout = popen.communicate(signers[keyid]['key'])[0]
  135. if popen.wait() != 0:
  136. print>>sys.stderr, 'Key %s failed to import'%(keyid)
  137. continue
  138. expected_keyid = keyid
  139. if signers[keyid].has_key('keyid'):
  140. expected_keyid = signers[keyid]['keyid']
  141. if gpgout.count(expected_keyid) == 0:
  142. print>>sys.stderr, 'Key file %s did not contain the key %s'%(key_path, keyid)
  143. if gpgout.count('IMPORT_OK') != 1 and quiet <= 1:
  144. print>>sys.stderr, 'Key file %s contained more than one key'%(key_path)
  145. def check_assertions(config, assertions):
  146. total_weight = 0
  147. signers = config['signers']
  148. if quiet == 0:
  149. print 'Signatures from:'
  150. for key in assertions['build']:
  151. if not signers.has_key(key):
  152. if quiet <= 1:
  153. print>>sys.stderr, 'key %s is not in config, skipping'%(key)
  154. continue
  155. if quiet == 0:
  156. print ' %s : weight %d'%(signers[key]['name'], signers[key]['weight'])
  157. total_weight += signers[key]['weight']
  158. if total_weight < config['minimum_weight']:
  159. print>>sys.stderr, "The total weight of signatures is %d, which is less than the minimum required %d"%(total_weight, config['minimum_weight'])
  160. return False
  161. return total_weight
  162. class OrderedDictYAMLLoader(yaml.Loader):
  163. """
  164. A YAML loader that loads ordered yaml maps into a dictionary.
  165. """
  166. def __init__(self, *args, **kwargs):
  167. yaml.Loader.__init__(self, *args, **kwargs)
  168. self.add_constructor(u'!omap', type(self).construct_yaml_map)
  169. def construct_yaml_map(self, node):
  170. data = dict()
  171. yield data
  172. for mapping in node.value:
  173. for key, value in mapping.value:
  174. key = self.construct_object(key)
  175. value = self.construct_object(value)
  176. data[key] = value
  177. full_prog = sys.argv[0]
  178. prog = os.path.basename(full_prog)
  179. parser = argparse.ArgumentParser(description='Download a verify a gitian package')
  180. parser.add_argument('-u', '--url', metavar='URL', type=str, nargs='+', required=False,
  181. help='one or more URLs where the package can be found')
  182. parser.add_argument('-c', '--config', metavar='CONF', type=str, required=not have_injected_config,
  183. help='a configuration file')
  184. parser.add_argument('-d', '--dest', metavar='DEST', type=str, required=False,
  185. help='the destination directory for unpacking')
  186. parser.add_argument('-q', '--quiet', action='append_const', const=1, default=[], help='be quiet')
  187. parser.add_argument('-m', '--customize', metavar='OUTPUT', type=str, help='generate a customized version of the script with the given config')
  188. args = parser.parse_args()
  189. quiet = len(args.quiet)
  190. if args.config:
  191. f = file(args.config, 'r')
  192. if args.customize:
  193. s = file(full_prog, 'r')
  194. script = s.read()
  195. s.close()
  196. config = f.read()
  197. script = script.replace(inject_config_string, config)
  198. s = file(args.customize, 'w')
  199. s.write(script)
  200. s.close()
  201. os.chmod(args.customize, 0750)
  202. exit(0)
  203. config = yaml.safe_load(f)
  204. f.close()
  205. else:
  206. config = yaml.safe_load(injected_config)
  207. if not args.dest:
  208. parser.error('argument -d/--dest is required unless -m is specified')
  209. rsses = []
  210. if args.url:
  211. urls = args.url
  212. else:
  213. urls = config['urls']
  214. if config.has_key('rss'):
  215. rsses = config['rss']
  216. if not urls:
  217. parser.error('argument -u/--url is required since config does not specify it')
  218. # TODO: rss, atom, etc.
  219. if path.exists(args.dest):
  220. print>>sys.stderr, "destination already exists, please remove it first"
  221. exit(1)
  222. temp_dir = tempfile.mkdtemp('', prog)
  223. atexit.register(remove_temp, temp_dir)
  224. package_file = path.join(temp_dir, 'package')
  225. downloaded = False
  226. for rss in rsses:
  227. try:
  228. feed = libxml2.parseDoc(urllib2.urlopen(rss['url']).read())
  229. url = None
  230. for node in feed.xpathEval(rss['xpath']):
  231. if re.search(rss['pattern'], str(node)):
  232. url = str(node)
  233. break
  234. try:
  235. download(url, package_file)
  236. downloaded = True
  237. break
  238. except:
  239. print>>sys.stderr, "could not download from %s, trying next rss"%(url)
  240. pass
  241. except:
  242. print>>sys.stderr, "could read not from rss %s"%(rss)
  243. pass
  244. if not downloaded:
  245. for url in urls:
  246. try:
  247. download(url, package_file)
  248. downloaded = True
  249. break
  250. except:
  251. print>>sys.stderr, "could not download from %s, trying next url"%(url)
  252. pass
  253. if not downloaded:
  254. print>>sys.stderr, "out of places to download from, try later"
  255. exit(1)
  256. unpack_dir = path.join(temp_dir, 'unpack')
  257. files = extract(unpack_dir, package_file)
  258. import_keys(temp_dir, config)
  259. (success, assertions, out_manifest) = get_assertions(temp_dir, unpack_dir, files)
  260. if not success and quiet <= 1:
  261. print>>sys.stderr, "There were errors getting assertions"
  262. total_weight = check_assertions(config, assertions)
  263. if not total_weight:
  264. print>>sys.stderr, "There were errors checking assertions, build is untrusted, aborting"
  265. exit(1)
  266. if quiet == 0:
  267. print>>sys.stderr, "Successful with signature weight %d"%(total_weight)
  268. shutil.copytree(unpack_dir, args.dest)
  269. f = file(path.join(args.dest, '.manifest'), 'w')
  270. yaml.dump(out_manifest, f)
  271. f.close()
  272. #os.system("cd %s ; /bin/bash"%(temp_dir))