Add flatpak nightly repository

1 files changed, 766 insertions, 0 deletions

diff --git a/scripts/flat-manager-client b/scripts/flat-manager-client
new file mode 100755
index 00000000..183ebf81
--- /dev/null
+++ b/scripts/flat-manager-client
@@ -0,0 +1,766 @@
+#!/usr/bin/python3
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import asyncio
+import base64
+import binascii
+import errno
+import fnmatch
+import gzip
+import json
+import logging
+import os
+import sys
+import time
+import traceback
+from argparse import ArgumentParser
+from functools import reduce
+from urllib.parse import urljoin, urlparse, urlsplit, urlunparse, urlunsplit
+
+import aiohttp
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
+
+import gi
+gi.require_version('OSTree', '1.0')
+from gi.repository import Gio, GLib, OSTree
+
+UPLOAD_CHUNK_LIMIT = 4 * 1024 * 1024
+DEFAULT_LIMIT = 2 ** 16
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+class UsageException(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+    def __str__(self):
+        return self.msg
+
+class ApiError(Exception):
+    def __init__(self, response, body):
+        self.url = response.url
+        self.status = response.status
+
+        try:
+            self.body = json.loads(response);
+        except:
+            self.body = {"status": self.status, "error-type": "no-error", "message": "No json error details from server"}
+
+    def repr(self):
+        return {
+            "type": "api",
+            "url": self.url,
+            "status_code": self.status,
+            "details": self.body
+        }
+
+    def __str__(self):
+        return "Api call to %s failed with status %d, details: %s" % (self.url, self.status, self.body)
+        return json.dumps(self.repr(), indent=4)
+
+# This is similar to the regular payload, but opens the file lazily
+class AsyncNamedFilePart(aiohttp.payload.Payload):
+    def __init__(self,
+                 value,
+                 disposition='attachment',
+                 *args,
+                 **kwargs):
+        self._file = None
+        if 'filename' not in kwargs:
+            kwargs['filename'] = os.path.basename(value)
+
+        super().__init__(value, *args, **kwargs)
+
+        if self._filename is not None and disposition is not None:
+            self.set_content_disposition(disposition, filename=self._filename, quote_fields=False)
+
+        self._size = os.stat(value).st_size
+
+    async def write(self, writer):
+        if self._file is None or self._file.closed:
+            self._file = open(self._value, 'rb')
+        try:
+            chunk = self._file.read(DEFAULT_LIMIT)
+            while chunk:
+                await writer.write(chunk)
+                chunk = self._file.read(DEFAULT_LIMIT)
+        finally:
+            self._file.close()
+
+    @property
+    def size(self):
+        return self._size
+
+def ostree_object_path(repo, obj):
+    repodir = repo.get_path().get_path()
+    return os.path.join(repodir, 'objects', obj[0:2], obj[2:])
+
+def ostree_get_dir_files(repo, objects, dirtree):
+    if dirtree.endswith(".dirtree"):
+        dirtree = dirtree[:-8]
+    dirtreev = repo.load_variant(OSTree.ObjectType.DIR_TREE, dirtree)[1]
+    iter = OSTree.RepoCommitTraverseIter()
+    iter.init_dirtree(repo, dirtreev, 0)
+    while True:
+        type = iter.next()
+        if type == OSTree.RepoCommitIterResult.END:
+            break
+        if type == OSTree.RepoCommitIterResult.ERROR:
+            break
+        if type == OSTree.RepoCommitIterResult.FILE:
+            d = iter.get_file()
+            objects.add(d.out_checksum + ".filez")
+        if type == OSTree.RepoCommitIterResult.DIR:
+            pass
+
+def local_needed_files(repo, metadata_objects):
+    objects = set()
+    for c in metadata_objects:
+        if c.endswith(".dirtree"):
+            ostree_get_dir_files(repo, objects, c)
+    return objects
+
+def local_needed_metadata_dirtree(repo, objects, dirtree_content, dirtree_meta):
+    objects.add(dirtree_meta + ".dirmeta")
+    dirtree_content_name = dirtree_content + ".dirtree"
+    if dirtree_content_name in objects:
+        return
+    objects.add(dirtree_content_name)
+
+    dirtreev = repo.load_variant(OSTree.ObjectType.DIR_TREE, dirtree_content)[1]
+    iter = OSTree.RepoCommitTraverseIter()
+    iter.init_dirtree(repo, dirtreev, 0)
+    while True:
+        type = iter.next()
+        if type == OSTree.RepoCommitIterResult.END:
+            break
+        if type == OSTree.RepoCommitIterResult.ERROR:
+            break
+        if type == OSTree.RepoCommitIterResult.FILE:
+            pass
+        if type == OSTree.RepoCommitIterResult.DIR:
+            d = iter.get_dir()
+            local_needed_metadata_dirtree(repo, objects, d.out_content_checksum, d.out_meta_checksum)
+
+def local_needed_metadata(repo, commits):
+    objects = set()
+    for rev in commits:
+        objects.add(rev + ".commit")
+        commitv = repo.load_variant(OSTree.ObjectType.COMMIT, rev)[1]
+        iter = OSTree.RepoCommitTraverseIter()
+        iter.init_commit(repo, commitv, 0)
+        while True:
+            type = iter.next()
+            if type == OSTree.RepoCommitIterResult.END:
+                break
+            if type == OSTree.RepoCommitIterResult.ERROR:
+                break
+            if type == OSTree.RepoCommitIterResult.FILE:
+                pass
+            if type == OSTree.RepoCommitIterResult.DIR:
+                d = iter.get_dir()
+                local_needed_metadata_dirtree(repo, objects, d.out_content_checksum, d.out_meta_checksum)
+    return objects
+
+
+def chunks(l, n):
+    """Yield successive n-sized chunks from l."""
+    for i in range(0, len(l), n):
+        yield l[i:i + n]
+
+async def missing_objects(session, build_url, token, wanted):
+    missing=[]
+    for chunk in chunks(wanted, 2000):
+        wanted_json=json.dumps({'wanted': chunk}).encode('utf-8')
+        data=gzip.compress(wanted_json)
+        headers = {
+            'Authorization': 'Bearer ' + token,
+            'Content-Encoding': 'gzip',
+            'Content-Type': 'application/json'
+        }
+        resp = await session.get(build_url + "/missing_objects", data=data, headers=headers)
+        async with resp:
+            if resp.status != 200:
+                raise ApiError(resp, await resp.text())
+            data = await resp.json()
+            missing.extend(data["missing"])
+    return missing
+
+@retry(
+    stop=stop_after_attempt(6),
+    wait=wait_fixed(20),
+    retry=retry_if_exception_type(ApiError),
+    reraise=True,
+)
+async def upload_files(session, build_url, token, files):
+    if len(files) == 0:
+        return
+    print("Uploading %d files (%d bytes)" % (len(files), reduce(lambda x, y: x + y, map(lambda f: f.size, files))))
+    with aiohttp.MultipartWriter() as writer:
+        for f in files:
+            writer.append(f)
+    writer.headers['Authorization'] = 'Bearer ' + token
+    resp = await session.request("post", build_url + '/upload', data=writer, headers=writer.headers)
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+
+async def upload_deltas(session, repo_path, build_url, token, deltas, refs, ignore_delta):
+    if not len(deltas):
+        return
+
+    req = []
+    for ref, commit in refs.items():
+        # Skip screenshots here
+        parts = ref.split("/")
+        if len(parts) == 4 and (parts[0] == "app" or parts[0] =="runtime") and not should_skip_delta(parts[1], ignore_delta):
+            for delta in deltas:
+                # Only upload from-scratch deltas, as these are the only reused ones
+                if delta == commit:
+                    print(" %s: %s" % (ref, delta))
+                    delta_name = delta_name_encode (delta)
+                    delta_dir = repo_path + "/deltas/" + delta_name[:2] + "/" + delta_name[2:]
+                    parts = os.listdir(delta_dir)
+                    for part in parts:
+                        req.append(AsyncNamedFilePart(delta_dir + "/" + part, filename = delta_name + "." + part + ".delta"))
+
+    if len(req):
+        await upload_files(session, build_url, token, req)
+
+
+async def upload_objects(session, repo_path, build_url, token, objects):
+    req = []
+    total_size = 0
+    for file_obj in objects:
+        named = get_object_multipart(repo_path, file_obj)
+        file_size = named.size
+        if total_size + file_size > UPLOAD_CHUNK_LIMIT: # The new object would bring us over the chunk limit
+            if len(req) > 0: # We already have some objects, upload those first
+                next_req = [named]
+                total_size = file_size
+            else:
+                next_req  =  []
+                req.append(named)
+                total_size = 0
+            await upload_files(session, build_url, token, req)
+            req = next_req
+        else:
+            total_size = total_size + file_size
+            req.append(named)
+
+    # Upload any remainder
+    await upload_files(session, build_url, token, req)
+
+async def create_ref(session, build_url, token, ref, commit):
+    print("Creating ref %s with commit %s" % (ref, commit))
+    resp = await session.post(build_url + "/build_ref", headers={'Authorization': 'Bearer ' + token}, json= { "ref": ref, "commit": commit} )
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+
+        data = await resp.json()
+        return data
+
+async def add_extra_ids(session, build_url, token, extra_ids):
+    print("Adding extra ids %s" % (extra_ids))
+    resp = await session.post(build_url + "/add_extra_ids", headers={'Authorization': 'Bearer ' + token}, json= { "ids": extra_ids} )
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+
+        data = await resp.json()
+        return data
+
+async def get_build(session, build_url, token):
+    resp = await session.get(build_url, headers={'Authorization': 'Bearer ' + token})
+    if resp.status != 200:
+        raise ApiError(resp, await resp.text())
+    data = await resp.json()
+    return data
+
+# For stupid reasons this is a string with json, lets expand it
+def reparse_job_results(job):
+    job["results"] = json.loads(job.get("results", "{}"))
+    return job
+
+async def get_job(session, job_url, token):
+    resp = await session.get(job_url, headers={'Authorization': 'Bearer ' + token}, json={})
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+        data = await resp.json()
+        return data
+
+async def wait_for_job(session, job_url, token):
+    reported_delay = False
+    old_job_status  = 0
+    printed_len = 0
+    iterations_since_change=0
+    error_iterations = 0
+    while True:
+        try:
+            resp = await session.get(job_url, headers={'Authorization': 'Bearer ' + token}, json={'log-offset': printed_len})
+            async with resp:
+                if resp.status == 200:
+                    error_iterations = 0
+                    job = await resp.json()
+                    job_status = job['status']
+                    if job_status == 0 and not reported_delay:
+                        reported_delay = True
+                        start_after_struct = job.get("start_after", None)
+                        if start_after_struct:
+                            start_after = start_after_struct.get("secs_since_epoch", None)
+                            now = time.time()
+                            if start_after and start_after > now:
+                                print("Waiting %d seconds before starting job" % (int(start_after - now)))
+                    if job_status > 0 and old_job_status == 0:
+                        print("/ Job was started");
+                    old_job_status = job_status
+                    log = job['log']
+                    if len(log) > 0:
+                        iterations_since_change=0
+                        for line in log.splitlines(True):
+                            print("| %s" % line, end="")
+                        printed_len = printed_len + len(log)
+                    else:
+                        iterations_since_change=iterations_since_change+1
+                    if job_status > 1:
+                        if job_status == 2:
+                            print("\ Job completed successfully")
+                        else:
+                            print("\ Job failed")
+                        return job
+                else:
+                    iterations_since_change=4 # Start at 4 so we ramp up the delay faster
+                    error_iterations=error_iterations + 1
+                    if error_iterations <= 5:
+                        print("Unexpected response %s getting job log, ignoring" % resp.status)
+                    else:
+                        raise ApiError(resp, await resp.text())
+        except OSError as e:
+            if e.args[0] == errno.ECONNRESET:
+                # Client disconnected, retry
+                # Not sure exactly why, but i got a lot of ConnectionResetErrors here
+                # in tests. I guess the server stops reusing a http2 session after a bit
+                # Should be fine to retry with the backof
+                pass
+            else:
+                raise
+        # Some polling backoff to avoid loading the server
+        if iterations_since_change <= 1:
+            sleep_time=1
+        elif iterations_since_change < 5:
+            sleep_time=3
+        elif iterations_since_change < 15:
+            sleep_time=5
+        elif iterations_since_change < 30:
+            sleep_time=10
+        else:
+            sleep_time=60
+        time.sleep(sleep_time)
+
+async def commit_build(session, build_url, eol, eol_rebase, wait, token):
+    print("Committing build %s" % (build_url))
+    resp = await session.post(build_url + "/commit", headers={'Authorization': 'Bearer ' + token}, json= {
+        "endoflife": eol, "endoflife_rebase": eol_rebase
+    })
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+
+        job = await resp.json()
+        job_url = resp.headers['location'];
+
+        if wait:
+            print("Waiting for commit job")
+            job = await wait_for_job(session, job_url, token);
+
+        reparse_job_results(job)
+        job["location"] = job_url
+        return job
+
+async def publish_build(session, build_url, wait, token):
+    print("Publishing build %s" % (build_url))
+    resp = await session.post(build_url + "/publish", headers={'Authorization': 'Bearer ' + token}, json= { } )
+    async with resp:
+        if resp.status == 400:
+            body = await resp.text()
+            try:
+                msg = json.loads(body)
+                if msg.get("current-state", "") == "published":
+                    print("the build has been already published")
+                    return {}
+            except:
+                pass
+
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+
+        job = await resp.json()
+        job_url = resp.headers['location'];
+
+        if wait:
+            print("Waiting for publish job")
+            job = await wait_for_job(session, job_url, token);
+
+        reparse_job_results(job)
+        job["location"] = job_url
+        return job
+
+async def purge_build(session, build_url, token):
+    print("Purging build %s" % (build_url))
+    resp = await session.post(build_url + "/purge", headers={'Authorization': 'Bearer ' + token}, json= {} )
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+        return await resp.json()
+
+async def create_token(session, manager_url, token, name, subject, scope, duration):
+    token_url = urljoin(manager_url, "/api/v1/token_subset")
+    resp = await session.post(token_url, headers={'Authorization': 'Bearer ' + token}, json = {
+        "name": name,
+        "sub": subject,
+        "scope": scope,
+        "duration": duration,
+    })
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+        return await resp.json()
+
+def get_object_multipart(repo_path, object):
+    return AsyncNamedFilePart(repo_path + "/objects/" + object[:2] + "/" + object[2:], filename=object)
+
+async def create_command(session, args):
+    build_url = urljoin(args.manager_url, "/api/v1/build")
+    resp = await session.post(build_url, headers={'Authorization': 'Bearer ' + args.token}, json={
+        "repo": args.repo
+    })
+    async with resp:
+        if resp.status != 200:
+            raise ApiError(resp, await resp.text())
+        data = await resp.json()
+        data["location"] = resp.headers['location']
+        if not args.print_output:
+            print(resp.headers['location'])
+        return data
+
+def delta_name_part_encode(commit):
+    return base64.b64encode(binascii.unhexlify(commit), b"+_")[:-1].decode("utf-8")
+
+def delta_name_encode (delta):
+    return "-".join(map(delta_name_part_encode, delta.split("-")))
+
+def should_skip_delta(id, globs):
+    if globs:
+        for glob in globs:
+            if fnmatch.fnmatch(id, glob):
+                return True
+    return False
+
+def build_url_to_api(build_url):
+    parts = urlparse(build_url)
+    path = os.path.dirname(os.path.dirname(parts.path))
+    return urlunparse((parts.scheme, parts.netloc, path, None, None, None))
+
+async def push_command(session, args):
+    local_repo = OSTree.Repo.new(Gio.File.new_for_path(args.repo_path))
+    try:
+        local_repo.open(None)
+    except GLib.Error as err:
+        raise UsageException("Can't open repo %s: %s" % (args.repo_path, err.message)) from err
+
+    refs = {}
+    if len(args.branches) == 0:
+        _, all_refs = local_repo.list_refs(None, None)
+        for ref in all_refs:
+            if ref.startswith("app/") or ref.startswith("runtime/") or ref.startswith("screenshots/"):
+                refs[ref] = all_refs[ref]
+    else:
+        for branch in args.branches:
+            _, rev = local_repo.resolve_rev(branch, False)
+            refs[branch] = rev
+
+    if (args.minimal_token):
+        id = os.path.basename(urlparse(args.build_url).path)
+        token = create_token(args.build_url, args.token, "minimal-upload", "build/%s" % (id), ["upload"], 60*60)["token"]
+    else:
+        token = args.token
+
+    print("Uploading refs to %s: %s"% (args.build_url, list(refs)))
+
+    metadata_objects = local_needed_metadata(local_repo, refs.values())
+
+    print("Refs contain %d metadata objects" % (len(metadata_objects)))
+
+    missing_metadata_objects = await missing_objects(session, args.build_url, token, list(metadata_objects))
+
+    print("Remote missing %d of those" % (len(missing_metadata_objects)))
+
+    file_objects = local_needed_files(local_repo, missing_metadata_objects)
+    print("Has %d file objects for those" % (len(file_objects)))
+
+    missing_file_objects = await missing_objects(session, args.build_url, token, list(file_objects))
+    print("Remote missing %d of those" % (len(missing_file_objects)))
+
+    # First upload all missing file objects
+    print("Uploading file objects")
+    await upload_objects(session, args.repo_path, args.build_url, token, missing_file_objects)
+
+    # Then all the metadata
+    print("Uploading metadata objects")
+    await upload_objects(session, args.repo_path, args.build_url, token, missing_metadata_objects)
+
+    _, deltas = local_repo.list_static_delta_names()
+    print("Uploading deltas")
+    await upload_deltas(session, args.repo_path, args.build_url, token, deltas, refs, args.ignore_delta)
+
+    # Then the refs
+    for ref, commit in refs.items():
+        await create_ref(session, args.build_url, token, ref, commit)
+
+    # Then any extra ids
+    if args.extra_id:
+        await add_extra_ids(session, args.build_url, token, args.extra_id)
+
+    commit_job = None
+    publish_job = None
+    update_job = None
+
+    # Note, this always uses the full token, as the minimal one only has upload permissions
+    if args.commit or args.publish:
+        commit_job = await commit_build(session, args.build_url, args.end_of_life, args.end_of_life_rebase, args.publish or args.wait, args.token)
+
+    if args.publish:
+        publish_job = await publish_build(session, args.build_url, args.wait or args.wait_update, args.token)
+        update_job_id = publish_job.get("results", {}).get("update-repo-job", None)
+        if update_job_id:
+            print("Queued repo update job %d" %(update_job_id))
+            update_job_url = build_url_to_api(args.build_url) + "/job/" + str(update_job_id)
+            if args.wait_update:
+                print("Waiting for repo update job")
+                update_job = await wait_for_job (session, update_job_url, token);
+            else:
+                update_job = await get_job(session, update_job_url, token)
+            reparse_job_results(update_job)
+            update_job["location"] = update_job_url
+
+    data = await get_build(session, args.build_url, args.token)
+    if commit_job:
+        data["commit_job"] = commit_job
+    if publish_job:
+        data["publish_job"] = publish_job
+    if update_job:
+        data["update_job"] = update_job
+    return data
+
+async def commit_command(session, args):
+    job = await commit_build(session, args.build_url, args.end_of_life, args.end_of_life_rebase, args.wait, args.token)
+    return job
+
+async def publish_command(session, args):
+    job = await publish_build(session, args.build_url, args.wait or args.wait_update, args.token)
+    update_job_id = job.get("results", {}).get("update-repo-job", None)
+    if update_job_id:
+        print("Queued repo update job %d" %(update_job_id))
+        update_job_url = build_url_to_api(args.build_url) + "/job/" + str(update_job_id)
+        if args.wait_update:
+            print("Waiting for repo update job")
+            update_job = await wait_for_job(session, update_job_url, args.token);
+        else:
+            update_job = await get_job(session, update_job_url, args.token)
+        reparse_job_results(update_job)
+        update_job["location"] = update_job_url
+    return job
+
+async def purge_command(session, args):
+    job = await purge_build(session, args.build_url, args.token)
+    return job
+
+async def create_token_command(session, args):
+    data = await create_token(session, args.manager_url, args.token, args.name, args.subject, args.scope, args.duration)
+    if not args.print_output:
+        print(data['token'])
+    return data
+
+async def follow_job_command(session, args):
+    job = await wait_for_job(session, args.job_url, args.token)
+    return job
+
+async def run_with_session(args):
+    timeout = aiohttp.ClientTimeout(total=90*60)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        result = await args.func(session, args)
+    return result
+
+if __name__ == '__main__':
+    progname = os.path.basename(sys.argv[0])
+
+    parser = ArgumentParser(prog=progname)
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='enable verbose output')
+    parser.add_argument('--debug', action='store_true',
+                        help='enable debugging output')
+    parser.add_argument('--output', help='Write output json to file')
+    parser.add_argument('--print-output', action='store_true', help='Print output json')
+    parser.add_argument('--token', help='use this token')
+    parser.add_argument('--token-file', help='use token from file')
+    subparsers = parser.add_subparsers(title='subcommands',
+                                       dest='subparser_name',
+                                       description='valid subcommands',
+                                       help='additional help')
+
+    create_parser = subparsers.add_parser('create', help='Create new build')
+    create_parser.add_argument('manager_url', help='remote repo manager url')
+    create_parser.add_argument('repo', help='repo name')
+    create_parser.set_defaults(func=create_command)
+
+    push_parser = subparsers.add_parser('push', help='Push to repo manager')
+    push_parser.add_argument('build_url', help='remote build url')
+    push_parser.add_argument('repo_path', help='local repository')
+    push_parser.add_argument('branches', nargs='*', help='branches to push')
+    push_parser.add_argument('--commit', action='store_true',
+                             help='commit build after pushing')
+    push_parser.add_argument('--publish', action='store_true',
+                             help='publish build after committing')
+    push_parser.add_argument('--extra-id', action='append', help='add extra collection-id')
+    push_parser.add_argument('--ignore-delta', action='append', help='don\'t upload deltas matching this glob')
+    push_parser.add_argument('--wait', action='store_true',
+                             help='wait for commit/publish to finish')
+    push_parser.add_argument('--wait-update', action='store_true',
+                             help='wait for update-repo to finish')
+    push_parser.add_argument('--minimal-token', action='store_true',
+                             help='Create minimal token for the upload')
+    push_parser.add_argument('--end-of-life', help='Set end of life')
+    push_parser.add_argument('--end-of-life-rebase', help='Set new ID which will supercede the current one')
+    push_parser.set_defaults(func=push_command)
+
+    commit_parser = subparsers.add_parser('commit', help='Commit build')
+    commit_parser.add_argument('--wait', action='store_true',
+                             help='wait for commit to finish')
+    commit_parser.add_argument('--end-of-life', help='Set end of life')
+    commit_parser.add_argument('--end-of-life-rebase', help='Set new ID which will supercede the current one')
+    commit_parser.add_argument('build_url', help='remote build url')
+    commit_parser.set_defaults(func=commit_command)
+
+    publish_parser = subparsers.add_parser('publish', help='Publish build')
+    publish_parser.add_argument('--wait', action='store_true',
+                             help='wait for publish to finish')
+    publish_parser.add_argument('--wait-update', action='store_true',
+                             help='wait for update-repo to finish')
+    publish_parser.add_argument('build_url', help='remote build url')
+    publish_parser.set_defaults(func=publish_command)
+
+    purge_parser = subparsers.add_parser('purge', help='Purge build')
+    purge_parser.add_argument('build_url', help='remote build url')
+    purge_parser.set_defaults(func=purge_command)
+
+    create_token_parser = subparsers.add_parser('create-token', help='Create subset token')
+    create_token_parser.add_argument('manager_url', help='remote repo manager url')
+    create_token_parser.add_argument('name', help='Name')
+    create_token_parser.add_argument('subject', help='Subject')
+    create_token_parser.add_argument('scope', nargs='*', help='Scope')
+    create_token_parser.add_argument('--duration', help='Duration until expires, in seconds',
+                                     default=60*60*24, # Default duration is one day
+                                     type=int)
+    create_token_parser.set_defaults(func=create_token_command)
+
+    follow_job_parser = subparsers.add_parser('follow-job', help='Follow existing job log')
+    follow_job_parser.add_argument('job_url', help='url of job')
+    follow_job_parser.set_defaults(func=follow_job_command)
+
+    args = parser.parse_args()
+
+    loglevel = logging.WARNING
+    if args.verbose:
+        loglevel = logging.INFO
+    if args.debug:
+        loglevel = logging.DEBUG
+
+    logging.basicConfig(format='%(module)s: %(levelname)s: %(message)s',
+                        level=loglevel, stream=sys.stderr)
+
+    if not args.subparser_name:
+        print("No subcommand specified, see --help for usage")
+        exit(1)
+
+    if not args.token:
+        if args.token_file:
+            file = open(args.token_file, 'rb')
+            args.token = file.read().splitlines()[0].decode("utf-8").strip()
+        elif "REPO_TOKEN" in os.environ:
+            args.token = os.environ["REPO_TOKEN"]
+        else:
+            print("No token available, pass with --token, --token-file or $REPO_TOKEN")
+            exit(1)
+
+
+    res = 1
+    output = None
+    try:
+        loop = asyncio.get_event_loop()
+        result = loop.run_until_complete(run_with_session(args))
+
+        output = {
+            "command": args.subparser_name,
+            "result": result,
+        }
+        res = 0
+    except SystemExit:
+        # Something called sys.exit(), lets just exit
+        res = 1
+        raise # Pass on regular exit callse
+    except ApiError as e:
+        eprint(str(e))
+        output = {
+            "command": args.subparser_name,
+            "error": e.repr(),
+        }
+    except UsageException as e:
+        eprint(str(e))
+        output = {
+            "error": {
+                "type": "usage",
+                "details": {
+                    "message": str(e),
+                }
+            }
+        }
+    except:
+        ei = sys.exc_info()
+        eprint("Unexpected %s exception in %s: %s" % (ei[0].__name__, args.subparser_name, ei[1]))
+        eprint(traceback.format_exc())
+        output = {
+            "command": args.subparser_name,
+            "error": {
+                "type": "exception",
+                "details": {
+                    "error-type": ei[0].__name__,
+                    "message": str(ei[1]),
+                }
+            }
+        }
+        res = 1
+
+    if output:
+        if args.print_output:
+            print(json.dumps(output, indent=4))
+        if args.output:
+            f = open(args.output,"w+")
+            f.write(json.dumps(output, indent=4))
+            f.write("\n")
+            f.close()
+    exit(res)