From 1b20f3a7708fa07666b5d96c38e0c494cee8a146 Mon Sep 17 00:00:00 2001 From: Josef Skladanka Date: May 21 2019 11:14:59 +0000 Subject: [PATCH 1/2] make the code python23 compatible --- diff --git a/get_latest_image/get_latest_image.py b/get_latest_image/get_latest_image.py index 97503a5..3d9acc6 100755 --- a/get_latest_image/get_latest_image.py +++ b/get_latest_image/get_latest_image.py @@ -6,11 +6,21 @@ import argparse import requests import datetime import json -import ConfigParser from pprint import pprint from lxml import etree -from urllib2 import urlopen, URLError, HTTPError + +try: + import configparser +except ImportError: + import ConfigParser as configparser + +try: + from urllib.request import urlopen + from urllib.error import HTTPError, URLError +except ImportError: + from urllib2 import urlopen, URLError, HTTPError + CONFIGFILE = '/etc/taskotron_base_images/config.ini' @@ -21,7 +31,7 @@ def read_config(configfile): global IMGFAC_BASE_URL if os.path.isfile(configfile): - config = ConfigParser.SafeConfigParser() + config = configparser.SafeConfigParser() config.read(configfile) config = dict(config.items('default')) if 'imgfac_base_url' in config: @@ -39,7 +49,7 @@ def download_image(image, outdir, processing_prefix=None): url = "%s/raw_image" % image['href'] filename = os.path.join(outdir, filename) - print "Downloading %s to %s" % (url, filename) + print("Downloading %s to %s" % (url, filename)) # Save to local file response = urlopen(url) @@ -145,10 +155,10 @@ def check_update(filename, get_missing=False, processing_prefix=None): try: timestamp, os_name, release, flavor, arch = filename.split('-') except ValueError: - print "Skipping %r; Unknown filename format" % f + print("Skipping %r; Unknown filename format" % f) continue - print "Seen file: %s" % f + print("Seen file: %s" % f) # Store image types that were seen, so we can later check for the # missing types and download the files rfa = (release, flavor, arch) @@ -165,7 +175,7 @@ def check_update(filename, get_missing=False, processing_prefix=None): try: latest = latest_images[rfa] except KeyError: - print "Skipping %r; Type %r not found in ImageFactory" % (f, rfa) + print("Skipping %r; Type %r not found in ImageFactory" % (f, rfa)) continue if latest['timestamp'] > timestamp: @@ -265,7 +275,7 @@ if __name__ == "__main__": images = get_latest_image(args.release, args.flavor, args.arch) if not images: - print "Nothing to do." + print("Nothing to do.") sys.exit() if args.dryrun: diff --git a/process_for_taskotron.py b/process_for_taskotron.py index e141143..1eb3a48 100755 --- a/process_for_taskotron.py +++ b/process_for_taskotron.py @@ -4,13 +4,22 @@ import os import sys import argparse import shutil -import ConfigParser -import urllib2 import gzip from pprint import pprint from get_latest_image import get_latest_image as gli +try: + import configparser +except ImportError: + import ConfigParser as configparser + +try: + from urllib.request import urlopen +except ImportError: + from urllib2 import urlopen + + CONFIGFILE = '/etc/taskotron_base_images/config.ini' # Defaults @@ -20,7 +29,7 @@ def read_config(configfile): global IMGFAC_BASE_URL if os.path.isfile(configfile): - config = ConfigParser.SafeConfigParser() + config = configparser.SafeConfigParser() config.read(configfile) config = dict(config.items('default')) if 'imgfac_base_url' in config: @@ -60,13 +69,13 @@ def parse_args(): def get_repourl_from_mirrorlist(release, arch): url = "https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-%s&arch=%s" % (release, arch) - response = urllib2.urlopen(url) + response = urlopen(url) data = response.read() # second line of the repodata is url return data.split('\n')[1].strip() def convert_to_qcow2(filename): - print "Converting %s to qcow2" % filename + print("Converting %s to qcow2" % filename) new_filename = "%s.qcow2" % ".".join(filename.split('.')[:-1]) os.system("qemu-img convert -f raw %s -O qcow2 %s" % (filename, new_filename)) os.unlink(filename) @@ -74,7 +83,7 @@ def convert_to_qcow2(filename): def gzip_and_publish(src, dst): """This is needed in order to keep the compressed file sparse""" - print "Publishing %s as %s" % (src, dst) + print("Publishing %s as %s" % (src, dst)) os.system("gzip -k -c --no-name %s > %s" % (src, dst)) if __name__ == "__main__": @@ -85,11 +94,11 @@ if __name__ == "__main__": images = gli.check_update(args.taskotrondir, get_missing=True, processing_prefix="PROCESSING-") if not images: - print "Nothing to do." + print("Nothing to do.") sys.exit() if args.dryrun: - print "Would get and process these images:" + print("Would get and process these images:") pprint(images) sys.exit() @@ -100,5 +109,5 @@ if __name__ == "__main__": image['filename'] = "%s.qcow2" % ".".join(image['filename'].split('.')[:-1]) if args.publicdir: gzip_and_publish(filename, os.path.join(args.publicdir, "%s.gz" % image['filename'])) - print "Finishing up" + print("Finishing up") shutil.move(filename, os.path.join(args.taskotrondir, image['filename'])) diff --git a/prune_images/prune_images.py b/prune_images/prune_images.py index 8acd878..7fc6b72 100755 --- a/prune_images/prune_images.py +++ b/prune_images/prune_images.py @@ -4,13 +4,17 @@ import os import re import sys import argparse -import ConfigParser import requests import json from lxml import etree from pprint import pprint +try: + import configparser +except ImportError: + import ConfigParser as configparser + CONFIGFILE = '/etc/taskotron_base_images/config.ini' # Defaults IMGFAC_BASE_URL = "http://127.0.0.1:8075/imagefactory" @@ -19,7 +23,7 @@ def read_config(configfile): global IMGFAC_BASE_URL if os.path.isfile(configfile): - config = ConfigParser.SafeConfigParser() + config = configparser.SafeConfigParser() config.read(configfile) config = dict(config.items('default')) @@ -46,11 +50,11 @@ def prune_failed(dryrun=False): if data['status'] != "FAILED": continue - print "Removing: %s" % img['base_image']['href'] + print("Removing: %s" % img['base_image']['href']) if not dryrun: r = requests.delete(img['base_image']['href']) if not r.ok: - print "Failed to delete: %s" % img['base_image']['href'] + print("Failed to delete: %s" % img['base_image']['href']) continue pruned_images.append(img['base_image']['href']) @@ -114,11 +118,11 @@ def prune_old(keep=1, dryrun=False): else: to_prune = value for image in to_prune: - print "Removing: %s" % image['href'] + print("Removing: %s" % image['href']) if not dryrun: r = requests.delete(image['href']) if not r.ok: - print "Failed to delete: %s" % image['href'] + print("Failed to delete: %s" % image['href']) continue pruned_images.append(image['href']) @@ -135,14 +139,14 @@ def prune_files(dirname, keep, verbose=False, dryrun=False): for f in filenames: filename = os.path.split(f)[-1] if not re_filename.match(filename): - print "Skipping %r; Unknown filename format" % f + print("Skipping %r; Unknown filename format" % f) continue without_extension = '.'.join(filename.split('.')[:-1]) try: timestamp, os_name, release, flavor, arch = without_extension.split('-') except ValueError: - print "Skipping %r; Unknown filename format" % f + print("Skipping %r; Unknown filename format" % f) continue relevant_files.setdefault((os_name, release, flavor, arch), []) @@ -154,10 +158,10 @@ def prune_files(dirname, keep, verbose=False, dryrun=False): to_remove[key] = relevant_files[key][keep:] if verbose or dryrun: - print "Relevant files:" + print("Relevant files:") pprint(relevant_files) print - print "Files to be removed:" + print("Files to be removed:") pprint(to_remove) print @@ -165,10 +169,10 @@ def prune_files(dirname, keep, verbose=False, dryrun=False): for files in to_remove.itervalues(): for fname in files: try: - print "Removing: %s" % os.path.join(dirname, fname) + print("Removing: %s" % os.path.join(dirname, fname)) os.remove(os.path.join(dirname, fname)) except OSError: - print "Could not remove %s" % os.path.join(dirname, fname) + print("Could not remove %s" % os.path.join(dirname, fname)) diff --git a/report_failures/report_failures.py b/report_failures/report_failures.py index 7569b70..6f9497b 100755 --- a/report_failures/report_failures.py +++ b/report_failures/report_failures.py @@ -5,7 +5,6 @@ import sys import argparse import requests import json -import ConfigParser import smtplib @@ -13,6 +12,10 @@ from email.mime.text import MIMEText from pprint import pformat from lxml import etree +try: + import configparser +except ImportError: + import ConfigParser as configparser CONFIGFILE = '/etc/taskotron_base_images/config.ini' # Defaults @@ -29,7 +32,7 @@ def read_config(configfile): global SMTP_SERVER if os.path.isfile(configfile): - config = ConfigParser.SafeConfigParser() + config = configparser.SafeConfigParser() config.read(configfile) config = dict(config.items('default')) if 'imgfac_base_url' in config: @@ -167,13 +170,13 @@ if __name__ == "__main__": data = get_failed_images_metadata() if not data: - print "No failed builds, nothing to do" + print("No failed builds, nothing to do") sys.exit() if args.subcommand == 'email': if args.dryrun: - print "Sever: %s\nFrom: %s\nTo: %s\nSubject: %s\nMessage:\n%s" % ( + print("Sever: %s\nFrom: %s\nTo: %s\nSubject: %s\nMessage:\n%s" % ( args.smtpserver, args.sender, args.to, args.subject, pformat(data) - ) + )) sys.exit() send_email(args.sender, args.to, args.subject, pformat(data), args.smtpserver) diff --git a/trigger_build/trigger_build.py b/trigger_build/trigger_build.py index b57a912..836b7b5 100755 --- a/trigger_build/trigger_build.py +++ b/trigger_build/trigger_build.py @@ -5,12 +5,22 @@ import sys import argparse import requests import datetime -import urllib2 import json -import ConfigParser from lxml import etree from pprint import pprint +try: + import configparser +except ImportError: + import ConfigParser as configparser + +try: + from urllib.request import urlopen +except ImportError: + from urllib2 import urlopen + + + CONFIGFILE = '/etc/taskotron_base_images/config.ini' # set to the scriptdir by default TEMPLATES_BASEDIR = os.path.abspath(os.path.dirname(sys.argv[0])) @@ -26,7 +36,7 @@ def read_config(configfile): global DEPLOYMENT_TYPE if os.path.isfile(configfile): - config = ConfigParser.SafeConfigParser() + config = configparser.SafeConfigParser() config.read(configfile) config = dict(config.items('default')) @@ -39,7 +49,7 @@ def read_config(configfile): def get_repourl_from_mirrorlist(release, arch): url = "https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-%s&arch=%s" % (release, arch) - response = urllib2.urlopen(url) + response = urlopen(url) data = response.read() # second line of the repodata is url # FIXME: nasty hack, as f23 needs 'Server' instead of 'Everything' (f24+ is ok) @@ -89,7 +99,7 @@ def trigger_build(release, arch="x86_64", flavor="minimal", disk_size="10G", dry data["filename"] = data["filename"] % data if dryrun: - print "Would trigger build with:" + print("Would trigger build with:") pprint(data) sys.exit() @@ -114,14 +124,14 @@ def trigger_build(release, arch="x86_64", flavor="minimal", disk_size="10G", dry # pretty-print what's returned from the request if r.ok: - print r.json()['base_image']['href'] + print(r.json()['base_image']['href']) # print json.dumps( # r.json(), # sort_keys=True, # indent=4, # separators=(',', ': ')) else: - print r.text + print(r.text) return r From fa78d6925aa94effd05ae89e361097b9d1bde050 Mon Sep 17 00:00:00 2001 From: Josef Skladanka Date: May 21 2019 11:14:59 +0000 Subject: [PATCH 2/2] add requirements.txt --- diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a3596c0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +lxml +requests