diff --git a/packaging/version_status.py b/packaging/version_status.py new file mode 100755 index 0000000..376a84a --- /dev/null +++ b/packaging/version_status.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import sys +reload(sys) +sys.setdefaultencoding('utf8') +import datetime +import xml.etree.ElementTree as ET +import ftplib +import gzip +import json +import re +from StringIO import StringIO +import urllib2 + + +# Returns true if `value` is an integer represented as a string. +def is_int(value): + # type: (str) -> bool + try: + value = int(value) + except ValueError: + return False + return True + + +# Returns a new string with all instances of multiple whitespace +# replaced with a single space. +def collapse_multiple_spaces(line): + # type: (str) -> str + return " ".join(line.split()) + + +# Extracts the file name from a line of an FTP listing. +# The input must be a valid directory entry (starting with "-" or "d"). +def ftp_file_name_from_listing(line): + # type: (str) -> str + line = collapse_multiple_spaces(line) + return line.split(" ", 8)[-1] + + +# Extracts a list of the directories and a list of the files +# from an FTP listing. +def ftp_list_dir_process_listing(lines): + # type: (List[str]) -> List[str], List[str] + dirs = [] + files = [] + for line in lines: + if line.startswith("d"): + dirs.append(ftp_file_name_from_listing(line)) + elif line.startswith("-"): + files.append(ftp_file_name_from_listing(line)) + return dirs, files + + +# Lists the remote FTP directory located at `path`. +# Returns a list of the directories and a list of the files. +def ftp_list_dir(ftp, path): + # type: (ftplib.FTP, str) -> List[str], List[str] + ftp.cwd(path) + lines = [] + ftp.retrlines("LIST", lines.append) + return ftp_list_dir_process_listing(lines) + + +# Downloads a binary file to a string. +# Returns the string. +def ftp_download(ftp, path): + # type: (ftplib.FTP, str) -> str + blocks = [] + ftp.retrbinary("RETR {0}".format(path), blocks.append) + return "".join(blocks) + + +# Extracts the list of links from an HTML string. +def http_links_from_listing(html): + # type: (str) -> List[str] + pattern = re.compile(r"""href=['"]+([^'"]+)['"]+""") + return re.findall(pattern, html) + + +# Extracts the list of paths (relative links, except to ../*) from an HTML string. +def http_paths_from_listing(html): + # type: (str) -> List[str] + paths = [] + for link in http_links_from_listing(html): + if link.startswith(".."): + continue + if "://" in link: + continue + paths.append(link) + return paths + + +# Downloads a file as string from an URL. Decodes correctly. +def http_download_txt(url): + # type: (str) -> str + r = urllib2.urlopen(url) + encoding = r.headers.getparam("charset") + if not encoding: + encoding = "utf-8" + return r.read().decode(encoding) + + +# Extracts the list of paths (relative links, except to ../*) from the HTML code +# located at `url`. +def http_list_dir(url): + # type: (str) -> List[str] + try: + html = http_download_txt(url) + except: + return [] + return http_paths_from_listing(html) + + +# Extracts the version and maintainer info for a package, from a Debian repository Packages file. +def deb_packages_extract_version(packages, name): + # type: (str, str) -> str, str + inside = False + version = None + maintainer = None + for line in packages.split("\n"): + if line == "Package: " + name: + inside = True + elif not line: + if inside: + break + else: + if inside: + if line.startswith("Version:"): + version = line.split(":", 1)[-1].strip() + elif line.startswith("Maintainer:"): + maintainer = line.split(":", 1)[-1].strip() + return version, maintainer + +# Extracts the version and maintainer info for a package, from an Arch PKGBUILD file. +def arch_pkgbuild_extract_version(pkgbuild): + # type: (str) -> str, str + version = None + maintainer = None + for line in pkgbuild.split("\n"): + if line.startswith("# Maintainer:"): + maintainer = line.split(":", 1)[-1].strip() + elif line.startswith("pkgver="): + version = line.split("=", 1)[-1].strip() + return version, maintainer + + +# Debian + +def get_debian_release_version(release): + data = http_download_txt("http://metadata.ftp-master.debian.org/changelogs/main/t/tint2/{0}_changelog".format(release)) + version = data.split("\n", 1)[0].split("(", 1)[-1].split(")", 1)[0].strip() + maintainer = [line.split("--", 1)[-1] for line in data.split("\n") if line.startswith(" --")][0].split(" ")[0].strip() + return release, version, maintainer + + +def get_debian_versions(): + return "Debian", "debian", [get_debian_release_version(release) for release in ["stable", "testing", "unstable", "experimental"]] + + +# Ubuntu + +def get_ubuntu_versions(): + data = http_download_txt("https://api.launchpad.net/1.0/ubuntu/+archive/primary?ws.op=getPublishedSources&source_name=tint2&exact_match=true") + data = json.loads(data)["entries"] + data.reverse() + versions = [] + for package in data: + if package["status"] == "Published": + version = package["source_package_version"] + release = package["distro_series_link"].split("/")[-1] + maintainer = package["package_maintainer_link"] + versions.append((release, version, maintainer)) + return "Ubuntu", "ubuntu", versions + + +# BunsenLabs + +def get_bunsenlabs_versions(): + dirs = http_list_dir("https://eu.pkg.bunsenlabs.org/debian/dists/") + versions = [] + for d in dirs: + if d.endswith("/") and "/" not in d[:-2]: + release = d.replace("/", "") + packages = http_download_txt("https://eu.pkg.bunsenlabs.org/debian/dists/{0}/main/binary-amd64/Packages".format(release)) + version, maintainer = deb_packages_extract_version(packages, "tint2") + if version: + versions.append((release, version, maintainer)) + return "BunsenLabs", "bunsenlabs", versions + + +# Arch + +def get_arch_versions(): + pkgbuild = http_download_txt("https://git.archlinux.org/svntogit/community.git/plain/trunk/PKGBUILD?h=packages/tint2") + version, maintainer = arch_pkgbuild_extract_version(pkgbuild) + return "Arch Linux", "archlinux", [("Community", version, maintainer)] + + +# Fedora + +def get_fedora_versions(): + dirs = http_list_dir("http://mirror.switch.ch/ftp/mirror/fedora/linux/development/") + versions = [] + for d in dirs: + if d.endswith("/") and "/" not in d[:-1]: + release = d.replace("/", "") + packages = http_list_dir("http://mirror.switch.ch/ftp/mirror/fedora/linux/development/{0}/Everything/source/tree/Packages/t/".format(release)) + for p in packages: + if p.startswith("tint2-"): + version = p.split("-", 1)[-1].split(".fc")[0] + v = (release, version, "") + if v not in versions: + versions.append(v) + return "Fedora", "fedora", versions + + +# Red Hat (EPEL) + +def get_redhat_epel_versions(): + dirs = http_list_dir("http://mirror.switch.ch/ftp/mirror/epel/") + versions = [] + for d in dirs: + if d.endswith("/") and "/" not in d[:-1] and is_int(d[:-1]): + release = d.replace("/", "") + packages = http_list_dir("http://mirror.switch.ch/ftp/mirror/epel/{0}/SRPMS/t/".format(release)) + for p in packages: + if p.startswith("tint2-"): + version = p.split("-", 1)[-1].split(".el")[0] + v = (release, version, "") + if v not in versions: + versions.append(v) + return "RedHat (EPEL)", "rhel", versions + + +# SUSE + +def get_suse_versions(): + ftp = ftplib.FTP("mirror.switch.ch") + ftp.login() + releases, _ = ftp_list_dir(ftp, "/mirror/opensuse/opensuse/distribution/leap/") + versions = [] + for release in releases: + root = "/mirror/opensuse/opensuse/distribution/leap/{0}/repo/oss/suse/repodata/".format(release) + _, files = ftp_list_dir(ftp, root) + for fname in files: + if fname.endswith("-primary.xml.gz"): + data = ftp_download(ftp, "{0}/{1}".format(root, fname)) + xml = gzip.GzipFile(fileobj=StringIO(data)).read() + root = ET.fromstring(xml) + for package in root.findall("{http://linux.duke.edu/metadata/common}package"): + name = package.find("{http://linux.duke.edu/metadata/common}name").text + if name == "tint2": + version = package.find("{http://linux.duke.edu/metadata/common}version").get("ver") + versions.append((release, version, "")) + ftp.quit() + return "OpenSUSE", "opensuse", versions + + +# Gentoo + +def get_gentoo_versions(): + files = http_list_dir("https://gitweb.gentoo.org/repo/gentoo.git/tree/x11-misc/tint2") + versions = [] + for f in files: + if "tint2" in f and f.endswith(".ebuild"): + version = f.split("tint2-")[-1].split(".ebuild")[0] + v = ("", version, "") + if v not in versions: + versions.append(v) + return "Gentoo", "gentoo", versions + + +# Void + +def get_void_versions(): + template = http_download_txt("https://raw.githubusercontent.com/voidlinux/void-packages/master/srcpkgs/tint2/template") + versions = [] + version = None + maintainer = None + for line in template.split("\n"): + if line.startswith("version="): + version = line.split("=", 1)[-1].replace('"', "").strip() + elif line.startswith("maintainer="): + maintainer = line.split("=", 1)[-1].replace('"', "").strip() + if version: + versions.append(("", version, maintainer)) + return "Void Linux", "void", versions + + +# FreeBSD + +def get_freebsd_versions(): + makefile = http_download_txt("https://svnweb.freebsd.org/ports/head/x11/tint/Makefile?view=co") + versions = [] + version = None + maintainer = None + for line in makefile.split("\n"): + if line.startswith("PORTVERSION="): + version = line.split("=", 1)[-1].strip() + elif line.startswith("MAINTAINER="): + maintainer = line.split("=", 1)[-1].strip() + if version: + versions.append(("", version, maintainer)) + return "FreeBSD", "freebsd", versions + + +# OpenBSD + +def get_openbsd_versions(): + makefile = http_download_txt("http://cvsweb.openbsd.org/cgi-bin/cvsweb/~checkout~/ports/x11/tint2/Makefile?rev=1.5&content-type=text/plain") + versions = [] + version = None + for line in makefile.split("\n"): + if line.startswith("V="): + version = line.split("=", 1)[-1].strip() + if version: + versions.append(("", version, "")) + return "OpenBSD", "openbsd", versions + + +def get_tint2_version(): + readme = http_download_txt("https://gitlab.com/o9000/tint2/raw/master/README.md") + version = readme.split("\n", 1)[0].split(":", 1)[-1].strip() + return version + + +def main(): + latest = get_tint2_version() + distros = [] + distros.append(get_debian_versions()) + distros.append(get_ubuntu_versions()) + distros.append(get_bunsenlabs_versions()) + distros.append(get_arch_versions()) + distros.append(get_fedora_versions()) + distros.append(get_redhat_epel_versions()) + distros.append(get_suse_versions()) + distros.append(get_gentoo_versions()) + distros.append(get_void_versions()) + distros.append(get_freebsd_versions()) + distros.append(get_openbsd_versions()) + print "| Distribution | Release | Version | Status |" + print "| ------------ | ------- | ------- | ------ |" + for dist, dcode, releases in distros: + icon = "![](numix-icons/distributor-logo-{0}.svg.png)".format(dcode) + for r in releases: + if r[1] == latest: + status = ":white_check_mark: Latest" + else: + status = ":warning: Out of date" + print "| {0} {1} | {2} | {3} | {4} |".format(icon, dist, r[0], r[1], status) + utc_datetime = datetime.datetime.utcnow() + utc_datetime.strftime("%Y-%m-%d %H:%M UTC") + print "" + print "Last updated:", utc_datetime + + +if __name__ == "__main__": + main() diff --git a/packaging/version_status_test.py b/packaging/version_status_test.py new file mode 100755 index 0000000..dcde4ba --- /dev/null +++ b/packaging/version_status_test.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import unittest +from version_status import * + + +class TestStringFunctions(unittest.TestCase): + def test_collapse_multiple_spaces(self): + self.assertEqual(collapse_multiple_spaces("asdf"), "asdf") + self.assertEqual(collapse_multiple_spaces("as df"), "as df") + self.assertEqual(collapse_multiple_spaces("as df"), "as df") + self.assertEqual(collapse_multiple_spaces("a s d f"), "a s d f") + + +class TestFtpFunctions(unittest.TestCase): + def test_ftp_file_name_from_listing(self): + self.assertEqual(ftp_file_name_from_listing("-rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18 README"), "README") + self.assertEqual(ftp_file_name_from_listing("-rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18:12 README"), "README") + self.assertEqual(ftp_file_name_from_listing("-rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18 READ ME"), "READ ME") + self.assertEqual(ftp_file_name_from_listing("-rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18:12 READ ME"), "READ ME") + self.assertEqual(ftp_file_name_from_listing("-rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18 README"), "README") + self.assertEqual(ftp_file_name_from_listing("-rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18:12 README.txt"), "README.txt") + self.assertEqual(ftp_file_name_from_listing("-rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18:12 READ ME.txt"), "READ ME.txt") + + def test_ftp_list_dir_process_listing(self): + lines = [ "-rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18 README", + "-rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18:11 READ ME.txt", + "drwxr-sr-x 5 1176 1176 4096 Dec 19 2000 pool", + "drwxr-sr-x 4 1176 1176 4096 Nov 17 2008 project", + "drwxr-xr-x 3 1176 1176 4096 Oct 10 2012 tools"] + dirs_check = ["pool", "project", "tools"] + files_check = ["README", "READ ME.txt"] + dirs, files = ftp_list_dir_process_listing(lines) + dirs.sort() + dirs_check.sort() + files.sort() + files_check.sort() + self.assertEqual(dirs, dirs_check) + self.assertEqual(files, files_check) + + +class TestHttpFunctions(unittest.TestCase): + def test_http_links_from_listing(self): + html = """ +
../ + bunsen-hydrogen/ 08-May-2017 20:31 - + jessie-backports/ 01-Jul-2017 15:58 - + unstable/ 12-Aug-2017 19:32 - +
../ + bunsen-hydrogen/ 08-May-2017 20:31 - + jessie-backports/ 01-Jul-2017 15:58 - + unstable/ 12-Aug-2017 19:32 - +