@ -34,6 +34,7 @@ from distutils.version import LooseVersion
@@ -34,6 +34,7 @@ from distutils.version import LooseVersion
- - -
name : foo
waiting_period : 24
urls :
- url : https : / / foo . org / gitian / foo . zip
version_url : https : / / foo . org / gitian / foo . ver
@ -275,215 +276,220 @@ class OrderedDictYAMLLoader(yaml.Loader):
@@ -275,215 +276,220 @@ class OrderedDictYAMLLoader(yaml.Loader):
value = self . construct_object ( value )
data [ key ] = value
full_prog = sys . argv [ 0 ]
prog = os . path . basename ( full_prog )
parser = argparse . ArgumentParser ( description = ' Download a verify a gitian package ' )
parser . add_argument ( ' -u ' , ' --url ' , metavar = ' URL ' , type = str , nargs = ' + ' , required = False ,
help = ' one or more URLs where the package can be found ' )
parser . add_argument ( ' -c ' , ' --config ' , metavar = ' CONF ' , type = str , required = not have_injected_config ,
help = ' a configuration file ' )
parser . add_argument ( ' -d ' , ' --dest ' , metavar = ' DEST ' , type = str , required = False ,
help = ' the destination directory for unpacking ' )
parser . add_argument ( ' -q ' , ' --quiet ' , action = ' append_const ' , const = 1 , default = [ ] , help = ' be quiet ' )
parser . add_argument ( ' -f ' , ' --force ' , action = ' store_true ' , help = ' force downgrades and such ' )
parser . add_argument ( ' -n ' , ' --dryrun ' , action = ' store_true ' , help = ' do not actually copy to destination ' )
parser . add_argument ( ' -m ' , ' --customize ' , metavar = ' OUTPUT ' , type = str , help = ' generate a customized version of the script with the given config ' )
parser . add_argument ( ' -w ' , ' --wait ' , type = float , metavar = ' HOURS ' , help = ' observe a waiting period or use zero for no waiting ' )
parser . add_argument ( ' -g ' , ' --gpg ' , metavar = ' GPG ' , type = str , help = ' path to GnuPG ' )
parser . add_argument ( ' -p ' , ' --post ' , metavar = ' COMMAND ' , type = str , help = ' Run after a successful install ' )
args = parser . parse_args ( )
quiet = len ( args . quiet )
if args . config :
f = file ( args . config , ' r ' )
if args . customize :
s = file ( full_prog , ' r ' )
script = s . read ( )
s . close ( )
config = f . read ( )
script = script . replace ( inject_config_string , config )
s = file ( args . customize , ' w ' )
s . write ( script )
s . close ( )
os . chmod ( args . customize , 0750 )
sys . exit ( 0 )
config = yaml . safe_load ( f )
f . close ( )
else :
config = yaml . safe_load ( injected_config )
dest_path = args . dest
if not dest_path :
parser . error ( ' argument -d/--dest is required unless -m is specified ' )
if args . wait is not None :
config [ ' waiting_period ' ] = args . wait
gpg_path = args . gpg
def run ( ) :
full_prog = sys . argv [ 0 ]
prog = os . path . basename ( full_prog )
parser = argparse . ArgumentParser ( description = ' Download a verify a gitian package ' )
parser . add_argument ( ' -u ' , ' --url ' , metavar = ' URL ' , type = str , nargs = ' + ' , required = False ,
help = ' one or more URLs where the package can be found ' )
parser . add_argument ( ' -c ' , ' --config ' , metavar = ' CONF ' , type = str , required = not have_injected_config ,
help = ' a configuration file ' )
parser . add_argument ( ' -d ' , ' --dest ' , metavar = ' DEST ' , type = str , required = False ,
help = ' the destination directory for unpacking ' )
parser . add_argument ( ' -q ' , ' --quiet ' , action = ' append_const ' , const = 1 , default = [ ] , help = ' be quiet ' )
parser . add_argument ( ' -f ' , ' --force ' , action = ' store_true ' , help = ' force downgrades and such ' )
parser . add_argument ( ' -n ' , ' --dryrun ' , action = ' store_true ' , help = ' do not actually copy to destination ' )
parser . add_argument ( ' -m ' , ' --customize ' , metavar = ' OUTPUT ' , type = str , help = ' generate a customized version of the script with the given config ' )
parser . add_argument ( ' -w ' , ' --wait ' , type = float , metavar = ' HOURS ' , help = ' observe a waiting period or use zero for no waiting ' )
parser . add_argument ( ' -g ' , ' --gpg ' , metavar = ' GPG ' , type = str , help = ' path to GnuPG ' )
parser . add_argument ( ' -p ' , ' --post ' , metavar = ' COMMAND ' , type = str , help = ' Run after a successful install ' )
args = parser . parse_args ( )
quiet = len ( args . quiet )
if args . config :
f = file ( args . config , ' r ' )
if args . customize :
s = file ( full_prog , ' r ' )
script = s . read ( )
s . close ( )
config = f . read ( )
script = script . replace ( inject_config_string , config )
s = file ( args . customize , ' w ' )
s . write ( script )
s . close ( )
os . chmod ( args . customize , 0750 )
sys . exit ( 0 )
config = yaml . safe_load ( f )
f . close ( )
else :
config = yaml . safe_load ( injected_config )
if not gpg_path :
gpg_path = ' gpg '
dest_path = args . dest
rsses = [ ]
if not dest_path :
parser . error ( ' argument -d/--dest is required unless -m is specified ' )
if args . url :
urls = [ { ' url ' : url , ' version_url ' : None } for url in args . url ]
else :
urls = config . get ( ' urls ' )
if not urls :
parser . error ( ' argument -u/--url is required since config does not specify it ' )
if config . has_key ( ' rss ' ) :
rsses = config [ ' rss ' ]
if args . wait is not None :
config [ ' waiting_period ' ] = args . wait
# TODO: rss, atom, etc.
old_manifest = None
gpg_path = args . gpg
if path . exists ( dest_path ) :
files = os . listdir ( dest_path )
if path . dirname ( full_prog ) == dest_path :
files . remove ( prog )
if not gpg_path :
gpg_path = ' gpg '
if not files . count ( ' .gitian-manifest ' ) and len ( files ) > 0 :
print >> sys . stderr , " destination already exists, no .gitian-manifest and directory not empty. Please empty destination. "
sys . exit ( 1 )
f = file ( os . path . join ( dest_path , ' .gitian-manifest ' ) , ' r ' )
old_manifest = yaml . load ( f , OrderedDictYAMLLoader )
f . close ( )
rsses = [ ]
if config . get ( ' waiting_period ' , 0 ) > 0 :
waiting_file = path . join ( dest_path , ' .gitian-waiting ' )
if path . exists ( waiting_file ) :
f = file ( waiting_file , ' r ' )
waiting = yaml . load ( f )
if args . url :
urls = [ { ' url ' : url , ' version_url ' : None } for url in args . url ]
else :
urls = config . get ( ' urls ' )
if not urls :
parser . error ( ' argument -u/--url is required since config does not specify it ' )
if config . has_key ( ' rss ' ) :
rsses = config [ ' rss ' ]
# TODO: rss, atom, etc.
old_manifest = None
if path . exists ( dest_path ) :
files = os . listdir ( dest_path )
if path . dirname ( full_prog ) == dest_path :
files . remove ( prog )
if not files . count ( ' .gitian-manifest ' ) and len ( files ) > 0 :
print >> sys . stderr , " destination already exists, no .gitian-manifest and directory not empty. Please empty destination. "
sys . exit ( 1 )
f = file ( os . path . join ( dest_path , ' .gitian-manifest ' ) , ' r ' )
old_manifest = yaml . load ( f , OrderedDictYAMLLoader )
f . close ( )
wait_start = waiting [ ' time ' ]
out_manifest = waiting [ ' out_manifest ' ]
waiting_path = waiting [ ' waiting_path ' ]
wait_time = wait_start + config [ ' waiting_period ' ] * 3600 - time . time ( )
if wait_time > 0 :
print >> sys . stderr , " Waiting another %.2f hours before applying update in %s " % ( wait_time / 3600 , waiting_path )
sys . exit ( 100 )
os . remove ( waiting_file )
if args . dryrun :
print >> sys . stderr , " Dry run, not copying "
else :
copy_to_destination ( path . join ( waiting_path , ' unpack ' ) , dest_path , out_manifest , old_manifest )
if args . post :
os . system ( args . post )
if quiet == 0 :
print >> sys . stderr , " Copied from waiting area to destination "
shutil . rmtree ( waiting_path )
sys . exit ( 0 )
temp_dir = tempfile . mkdtemp ( ' ' , prog )
atexit . register ( remove_temp , temp_dir )
package_file = path . join ( temp_dir , ' package ' )
downloaded = False
checked = False
if rsses :
import libxml2
for rss in rsses :
try :
feed = libxml2 . parseDoc ( urllib2 . urlopen ( rss [ ' url ' ] ) . read ( ) )
url = None
release = None
# Find the first matching node
for node in feed . xpathEval ( rss [ ' xpath ' ] ) :
m = re . search ( rss [ ' pattern ' ] , str ( node ) )
if m :
if len ( m . groups ( ) ) > 0 :
release = m . group ( 1 )
url = str ( node )
break
# Make sure it's a new release
if old_manifest and release == old_manifest [ ' release ' ] and not args . force :
checked = True
if config . get ( ' waiting_period ' , 0 ) > 0 :
waiting_file = path . join ( dest_path , ' .gitian-waiting ' )
if path . exists ( waiting_file ) :
f = file ( waiting_file , ' r ' )
waiting = yaml . load ( f )
f . close ( )
wait_start = waiting [ ' time ' ]
out_manifest = waiting [ ' out_manifest ' ]
waiting_path = waiting [ ' waiting_path ' ]
wait_time = wait_start + config [ ' waiting_period ' ] * 3600 - time . time ( )
if wait_time > 0 :
print >> sys . stderr , " Waiting another %.2f hours before applying update in %s " % ( wait_time / 3600 , waiting_path )
sys . exit ( 100 )
os . remove ( waiting_file )
if args . dryrun :
print >> sys . stderr , " Dry run, not copying "
else :
try :
download ( url , package_file )
copy_to_destination ( path . join ( waiting_path , ' unpack ' ) , dest_path , out_manifest , old_manifest )
if args . post :
os . system ( args . post )
if quiet == 0 :
print >> sys . stderr , " Copied from waiting area to destination "
shutil . rmtree ( waiting_path )
sys . exit ( 0 )
temp_dir = tempfile . mkdtemp ( ' ' , prog )
atexit . register ( remove_temp , temp_dir )
package_file = path . join ( temp_dir , ' package ' )
downloaded = False
checked = False
if rsses :
import libxml2
for rss in rsses :
try :
feed = libxml2 . parseDoc ( urllib2 . urlopen ( rss [ ' url ' ] ) . read ( ) )
url = None
release = None
# Find the first matching node
for node in feed . xpathEval ( rss [ ' xpath ' ] ) :
m = re . search ( rss [ ' pattern ' ] , str ( node ) )
if m :
if len ( m . groups ( ) ) > 0 :
release = m . group ( 1 )
url = str ( node )
break
# Make sure it's a new release
if old_manifest and release == old_manifest [ ' release ' ] and not args . force :
checked = True
else :
try :
download ( url , package_file )
downloaded = True
break
except :
print >> sys . stderr , " could not download from %s , trying next rss " % ( url )
pass
except :
print >> sys . stderr , " could read not from rss %s " % ( rss )
pass
if not downloaded :
for url in urls :
try :
release = None
if url [ ' version_url ' ] :
f = urllib2 . urlopen ( url [ ' version_url ' ] )
release = f . read ( 100 ) . strip ( )
f . close ( )
if old_manifest and release == old_manifest [ ' release ' ] and not args . force :
checked = True
else :
download ( url [ ' url ' ] , package_file )
downloaded = True
break
except :
print >> sys . stderr , " could not download from %s , trying next rss " % ( url )
pass
except :
print >> sys . stderr , " could read not from rss %s " % ( rss )
pass
if not downloaded :
for url in urls :
try :
release = None
if url [ ' version_url ' ] :
f = urllib2 . urlopen ( url [ ' version_url ' ] )
release = f . read ( 100 ) . strip ( )
f . close ( )
if old_manifest and release == old_manifest [ ' release ' ] and not args . force :
checked = True
else :
download ( url [ ' url ' ] , package_file )
downloaded = True
except :
print >> sys . stderr , " could not download from %s , trying next url " % ( url )
raise
if not downloaded :
if checked :
if quiet == 0 :
print >> sys . stderr , " same release, not downloading "
else :
print >> sys . stderr , " out of places to try downloading from, try later "
sys . exit ( 2 )
except :
print >> sys . stderr , " could not download from %s , trying next url " % ( url )
raise
unpack_dir = path . join ( temp_dir , ' unpack ' )
files = extract ( unpack_dir , package_file )
if not downloaded :
if checked :
if quiet == 0 :
print >> sys . stderr , " same release, not downloading "
else :
print >> sys . stderr , " out of places to try downloading from, try later "
sys . exit ( 2 )
import_keys ( gpg_path , temp_dir , config )
unpack_dir = path . join ( temp_dir , ' unpack ' )
files = extract ( unpack_dir , package_file )
( success , assertions , out_manifest ) = get_assertions ( gpg_path , temp_dir , unpack_dir , files )
import_keys ( gpg_path , temp_dir , config )
if old_manifest :
check_name_and_version ( out_manifest , old_manifest )
( success , assertions , out_manifest ) = get_assertions ( gpg_path , temp_dir , unpack_dir , files )
if not success and quiet < = 1 :
print >> sys . stderr , " There were errors getting assertions "
if old_manifest :
check_name_and_version ( out_manifest , old_manifest )
total_weight = check_assertions ( config , assertions )
if total_weight is None :
print >> sys . stderr , " There were errors checking assertions, build is untrusted, aborting "
sys . exit ( 5 )
if not success and quiet < = 1 :
print >> sys . stderr , " There were errors getting assertions "
if quiet == 0 :
print >> sys . stderr , " Successful with signature weight %d " % ( total_weight )
total_weight = check_assertions ( config , assertions )
if total_weight is None :
print >> sys . stderr , " There were errors checking assertions, build is untrusted, aborting "
sys . exit ( 5 )
if config . get ( ' waiting_period ' , 0 ) > 0 and path . exists ( dest_path ) :
waiting_path = tempfile . mkdtemp ( ' ' , prog )
shutil . copytree ( unpack_dir , path . join ( waiting_path , ' unpack ' ) )
f = file ( path . join ( dest_path , ' .gitian-waiting ' ) , ' w ' )
yaml . dump ( { ' time ' : time . time ( ) , ' out_manifest ' : out_manifest , ' waiting_path ' : waiting_path } , f )
f . close ( )
if quiet == 0 :
print >> sys . stderr , " Started waiting period "
else :
if args . dryrun :
print >> sys . stderr , " Dry run, not copying "
print >> sys . stderr , " Successful with signature weight %d " % ( total_weight )
if config . get ( ' waiting_period ' , 0 ) > 0 and path . exists ( dest_path ) :
waiting_path = tempfile . mkdtemp ( ' ' , prog )
shutil . copytree ( unpack_dir , path . join ( waiting_path , ' unpack ' ) )
f = file ( path . join ( dest_path , ' .gitian-waiting ' ) , ' w ' )
yaml . dump ( { ' time ' : time . time ( ) , ' out_manifest ' : out_manifest , ' waiting_path ' : waiting_path } , f )
f . close ( )
if quiet == 0 :
print >> sys . stderr , " Started waiting period "
else :
copy_to_destination ( unpack_dir , dest_path , out_manifest , old_manifest )
if args . dryrun :
print >> sys . stderr , " Dry run, not copying "
else :
copy_to_destination ( unpack_dir , dest_path , out_manifest , old_manifest )
if args . post :
os . system ( args . post )
if args . post :
os . system ( args . post )
if __name__ == ' __main__ ' :
run ( )