# -*- org-adapt-indentation: nil; org-edit-src-content-indentation: 0; -*-
#+TITLE: Python release workflow
#+OPTIONS: num:nil

* Using this file :noexport:
You can either [[https://orgmode.org/manual/Extracting-Source-Code.html][tangle]]
this file in emacs using =C-c C-v t= or you can tangle
the whole file from the command line using with the following.
#+begin_src bash :var THIS_FILE=(buffer-file-name) :results none
: ${THIS_FILE:="./release.org"}
emacs --batch \
      --load org \
      --load ob-shell \
      --eval "(org-babel-tangle-file \"${THIS_FILE}\")"
#+end_src

The core functionality is tangled to [[file:./../bin/python-release-functions.sh]].
It can be sourced in a shell or from a script using =source path/to/bin/python-release-functions.sh=
to make the functions defined in this file available for use.
* Python release process
Release workflow.

Don't tag a release on GitHub until all the tests pass,
the package contents are what we want and expect, and
you have run =push-release= to pypi testing and checked it.
Once they do, tag it with the version that you set below
so that everything is on the same page. If there are multiple
packages per repo the tag is usually prefixed with the module name.

Note that if you use the =--local ~/path/to/my/working/repo= option as the source repo
then =git pull= is called with =--force= since the assumption is that =git commit --amend=
may be used in certain cases.

*NEVER USE THESE FUNCTIONS ON YOUR WORKING REPO, YOU WILL LOOSE ANY STASHED WORK OR UNTRACKED FILES*

*WHEN YOU PUSH TO TEST*
Inspect _everything_ at https://test.pypi.org/project/${packagename}.
MAKE SURE THE HASHES MATCH (tail hashes vs curl output)
You can also check https://test.pypi.org/project/ontquery/#files
** Example release
#+NAME: release-examples
#+CAPTION: examples, this is horrible and dangerous, never do this this way run the 3 commands separately
#+BEGIN_SRC bash :eval never :noweb yes
source ~/git/pyontutils/bin/python-release-functions.sh
SOMEVAR=some-value \
build-release org repo folder packagename version --some-arg
PYTHONPATH=~/git/pyontutils: SCICRUNCH_API_KEY=$(cat ~/ni/dev/secrets.yaml | grep tgbugs-travis | awk '{ print $2 }') \
build-release tgbugs ontquery ontquery ontquery 0.1.0 --release
exit  # if try to copy paste this block terminate here to prevent dumbs
push-release ontquery ~/nas/software-releases ontquery 0.1.0
# DO GitHub RELEASE HERE
are-you-sure && \
final-release ~/nas/software-releases ontquery 0.1.0
#+END_SRC


This is a reasonable time to tag the release on GitHub.
* Config files
#+CAPTION: [[file:${HOME}/.pypirc]] on the release host (only need to create once)
#+BEGIN_SRC toml
[distutils]
index-servers =
    pypi
    test

[pypi]
repository: https://upload.pypi.org/legacy/
username: your-username

[test]
repository: https://test.pypi.org/legacy/
username: your-username
password: set-this-one-for-simplicity
#+END_SRC
* Code
** Python release functions
Tangle this block so you can source [[file:./../bin/python-release-functions.sh]]
#+NAME: all-blocks
#+CAPTION: run this to export all the things
#+HEADER: :tangle ../bin/python-release-functions.sh :comments noweb
#+BEGIN_SRC bash :eval never :noweb yes
<<build-release>>
<<push-release>>
# TODO github-release
<<final-release>>
#+END_SRC
** Build release
#+NAME: build-release
#+begin_src bash :eval never :exports code
build-release () {
    # example
    # build-release org    repo     folder   packagename version
    # build-release tgbugs ontquery ontquery ontquery    0.0.8

    local POSITIONAL=()
    local INTEGRATION_PACKAGES=()
    while [[ $# -gt 0 ]]
    do
    key="$1"

    # argh (((((((( darned mismatched parens :/
    case $key in
        -l|--local)
        local CLONEFROM="$2"
        shift # past argument
        shift # past value
        ;;
        -f|--artifact-folder)
        local ARTIFACT_FOLDER="$2"
        shift # past argument
        shift # past value
        ;;
        -p|--base-path)
        local BASE_PATH="$2"
        shift # past argument
        shift # past value
        ;;
        -b|--branch)
        local BRANCH="$2"
        shift # past argument
        shift # past value
        ;;
        -i|--install-package)
        INTEGRATION_PACKAGES+=("$2")
        shift
        shift
        ;;
        --tag-no-rename)
        local TAG_NO_RENAME=YES
        shift # past argument
        ;;
        --keep-artifacts)
        local KEEP_ARTIFACTS=YES
        shift # past argument
        ;;
        ,*)    # unknown option
        POSITIONAL+=("$1") # save it in an array for later
        shift # past argument
        ;;
    esac
    done

    local org=${POSITIONAL[0]}
    local repo=${POSITIONAL[1]}
    local folder=${POSITIONAL[2]}
    local packagename=${POSITIONAL[3]}
    local version=${POSITIONAL[4]}
    local REST=${POSITIONAL[@]:5}  # remaining position passed along
    echo $REST

    if [[ ${folder} == ${packagename} || -n ${TAG_NO_RENAME} ]]; then
        local tag=${version}
    else
        local tag=${packagename}-${version}
    fi

    # TODO make sure no vars are null

    : ${BASE_PATH:=/tmp}  # allow override for cases where /tmp causes test failure

    echo $org $repo $folder $packagename $version $tag $CLONEFROM $ARTIFACT_FOLDER $BASE_PATH ${INTEGRATION_PACKAGES[@]}

    cd ${BASE_PATH}  # ensure we are always working in tmp for the rest of the time

    TEST_PATH=${BASE_PATH}/release-testing/${packagename}  # allow multiple builds at the same time

    if [ -d ${repo} ]; then
        rm -r ${TEST_PATH}
    fi
    mkdir -p ${TEST_PATH}

    if [ -d ${repo} ]; then
        pushd ${repo}
        rurl="$(git remote get-url origin)"
        if [[ -z ${CLONEFROM} && ! $rurl =~ "https://" && ! $rurl =~ "git@" ]]; then
            git remote set-url origin https://github.com/${org}/${repo}.git
        elif [[ -n ${CLONEFROM} && "$rurl" != "${CLONEFROM}" ]]; then
            git remote set-url origin "${CLONEFROM}"
        fi
        git fetch || return $?  # fail on bad clone to prevent testing against stale code
        git reset --hard origin/master
        git clean -dfx
        popd
    else
        if [[ -n ${CLONEFROM} ]]; then
            git clone ${CLONEFROM} ${repo}
        else
            git clone https://github.com/${org}/${repo}.git
        fi
    fi
    # TODO __version__ check against ${version}

    pushd ${folder} || return $?  # or subfolder

    if [[ $(git tag -l ${tag}) ]]; then
        echo "${tag} has already been released for this repo!"
        return 1
    fi

    if [[ -n ${BRANCH} ]]; then
        git checkout ${BRANCH}
        git pull  # in the event that a local branch already exists
    else
        git checkout -f master  # just like clean -dfx this should wipe changes just in case
    fi
    #git checkout ${version}  # only if all tests are go and release is tagged
    PYTHONPATH=${PYTHONPATH}$(realpath .) python setup.py sdist $REST  # pass $REST along eg for --release
    if [ $? -ne 0 ]; then
        echo "setup.py failed"
        popd > /dev/null
        return 1
    fi

    unset PYTHONPATH
    cp dist/${packagename}-${version}* ${TEST_PATH}

    pushd ${TEST_PATH}
    tar xvzf ${packagename}-${version}.tar.gz
    if [ $? -ne 0 ]; then
        echo "tar failed, probably due to a version mismatch"
        popd > /dev/null
        popd > /dev/null
        return 1
    fi
    pushd ${packagename}-${version}

    # pipenv --rm swears no venv exists, if no Pipfile
    # exists even if adding a Pipfile will magically
    # reveal that there was in fact a venv and thus that
    # every other pipenv command knows about it but
    # naieve little rm is kept in the dark, so we yell
    # into the 'void' just to make sure
    touch Pipfile
    pipenv --rm  # clean any existing env
    pipenv --python 3.7  # for some reason 3.6 lingers in some envs
    if [[ -n ${INTEGRATION_PACKAGES} ]]; then
        echo $(color yellow)installing integration packages$(color off) ${INTEGRATION_PACKAGES[@]}
        pipenv run pip install ${INTEGRATION_PACKAGES[@]} || return $?
    fi
    echo $(color yellow)installing$(color off) ${packagename}
    pipenv run pip install -e .[test] || return $?  # .[services] for ontquery full install
    pipenv run python setup.py test || local FAILURE=$?
    # FIXME popd on failure ... can't && because we loose the next popd instead of exiting
    # everything should pass if not, keep going until it does
    popd
    popd
    # build the wheel from the sdist NOT from the repo
    pushd dist/
    tar xvzf ${packagename}-${version}.tar.gz
    pushd ./${packagename}-${version}/
    python setup.py bdist_wheel $@
    mv dist/*.whl ../
    popd
    rm ./${packagename}-${version}/ -r
    popd
    # background here to twine?
    popd
    if [[ -n ${FAILURE} ]]; then
        echo "$(color red)TESTS FAILED$(color off)";
    fi

    if [[ -n ${ARTIFACT_FOLDER} ]]; then
        if [ ! -d "${ARTIFACT_FOLDER}" ]; then
            mkdir -p "${ARTIFACT_FOLDER}"
        fi
        cp ${folder}/dist/${packagename}-${version}* "${ARTIFACT_FOLDER}"
        echo "build artifacts have been copied to ${ARTIFACT_FOLDER}"
    fi

    # FIXME need multiple repos when packages share a repo
    # basically a test for if [[ package == repo ]] or something
    if [[ -n ${KEEP_ARTIFACTS} ]]; then
        echo "$(color yellow)keeping artifacts$(color off)"
    elif [[ -n ${CLONEFROM} || ${BRANCH} ]]; then
        rm ${folder}/dist/${packagename}-${version}*
        if [[ -n ${CLONEFROM} ]]; then
            echo "$(color yellow)release build was cloned from a local source$(color off) ${CLONEFROM}"
        else
            echo "$(color yellow)release build was cloned from a specific branch$(color off) ${BRANCH}"
        fi
        echo "$(color ltyellow)therefore removing the build artifacts to prevent$(color off)"
        echo "$(color ltyellow)accidental releases built from a private source$(color off)"
    fi
}
#+end_src

** Push release
#+NAME: push-release
#+BEGIN_SRC bash :eval never :exports code
function push-release () {
    # example
    # push-release folder   software_releases_path    packagename version
    # push-release ontquery ~/nas/software-releases   ontquery    0.0.8
    local folder=$1
    shift
    local software_releases_path=$1
    shift
    local packagename=$1
    shift
    local version=$1
    shift

    rsync -a -v --ignore-existing ${folder}/dist/${packagename/-/*}-${version}{-,.tar}* ${software_releases_path}/ || return $?
    pushd ${software_releases_path}
    sha256sum ${packagename/-/*}-${version}{-,.tar}* >> hashes
    twine upload --repository test ${packagename/-/*}-${version}{-,.tar}* || return $?
    sleep 1
    echo "test pypi hashes"
    curl https://test.pypi.org/pypi/${packagename}/json | python -m json.tool | grep "\(sha256\|filename\)" | grep -B1 "${version}" | awk '{ gsub(/"/, "", $2); printf("%s ", $2) }' | sed 's/,\ /\n/g'
    echo "local hashes"
    tail -n2 hashes
    echo go inspect https://test.pypi.org/project/${packagename}
    echo and go do the github release
    popd
}
#+END_SRC
** TODO GitHub release
#+NAME: github-release
#+BEGIN_SRC python :eval never
import requests
from sparcur.utils
#from sparcur.utils import mimetype  # FIXME or something like that
# TODO api token

suffix_to_mime = {
    '.whl': 'application/octet-stream',  # technically zip ...
    '.gz': 'application/gzip',
    '.zip': 'application/zip',
}


class BadAssetSuffixError(Exception):
    """ u wot m8 !? """


def upload_assets(upload_base, version, *asset_paths):
    for asset in asset_paths:
        name = asset.name
        requests.post()


def github_release(org, repo, version, hashes, *assets, branch='master'):
    """ hashes should be the output of sha256sum {packagename}-{version} """
    # FIXME pyontutils violates some assumptions about 1:1 ness here

    asset_paths = tuple(Path(a).resolve() for a in assets)
    bads = [p.suffix  for p in asset_paths if p.suffix not in suffix_to_mime]
    if bads:
        raise BadAssetSuffixError(' '.join(bads))

    base = 'https://api.github.com'
    path = f'/repos/{org}/{repo}/releases'
    headers = {'Accept': 'application/vnd.github.v3+json'}
    json_data = {'tag_name': version,
                 'target_commitish': branch,
                 'name': version,
                 'body': hashes,
                 'draft': False,  # ok because we can add assets later
                 'prerelease': False}

    url = base + path
    resp = requests.post(url, headers=headers, json=json_data)
    rel_J = resp.json()
    uu = rel_j['upload_url']

    upload_base = uu.replace('{?name,label}', '')

    upload_assets(upload_base, *asset_paths)
#+END_SRC

** Final release
#+NAME: final-release
#+CAPTION: on the release host final upload from previous block
#+CAPTION: you will need to enter your password
#+BEGIN_SRC bash :eval never :exports code
function final-release () {
    # example
    # final-release software_releases_path    packagename version
    # final-release ~/nas/software-releases   ontquery    0.0.8
    local software_releases_path=$1
    shift
    local packagename=$1
    shift
    local version=$1
    shift

    pushd ${software_releases_path}

    twine upload --repository pypi ${packagename/-/*}-${version}{-,.tar}* || return $?  # enter password here

    sleep 1
    echo "pypi hashes"
    curl https://pypi.org/pypi/${packagename}/json | python -m json.tool | grep "\(sha256\|filename\)" | grep -B1 "${version}" | awk '{ gsub(/"/, "", $2); printf("%s ", $2) }' | sed 's/,\ /\n/g'
    echo "local hashes"
    tail -n2 hashes
    echo go inspect https://pypi.org/project/${packagename}

    popd
}
#+END_SRC
** TODO Next version                                               :noexport:
#+NAME: release-next
#+HEADER: :shebang "#!/usr/bin/env python3.7"
#+begin_src python :tangle ./../bin/release-next :tangle-mode (identity #o755)
import setuptools
from distutils.version import LooseVersion
import importlib.util
from urllib.parse import urlparse
import requests
import augpathlib as aug


last_output = [None]
def fake_setup(*args, **kwargs):
    last_output[0] = args, kwargs


setuptools.setup = fake_setup


class Version(LooseVersion):
    def next(self, increment='current'):
        nv = [_ for _ in self.version]
        if increment == 'current':
            nv[-1] = nv[-1] + 1

        elif increment == 'minor':
            if 'dev' in nv:
                nv = nv[:3]
            else:
                if 'post' in nv:
                   nv = nv[:3]

                nv[-1] = nv[-1] + 1

        elif increment == 'dev':
            if 'post' in nv:
                nv = nv[:3]

            nv[-1] = nv[-1] + 1
            if 'dev' not in nv:
                nv.extend(['dev', 0])

        elif increment == 'post':
            raise NotImplementedError('TODO')

        s = ''
        oe = None
        for e in nv:
            if isinstance(oe, int):
                s += '.'

            oe = e
            if isinstance(e, int):
                e = str(e)

            s += e

        return self.__class__(s)

    def _cmp(self, other):
        if isinstance(other, str):
            other = LooseVersion(other)

        sversion = [_ for _ in self.version]
        oversion = [_ for _ in other.version]
        ls = len(sversion)
        lo = len(oversion)
        # FIXME .post0 vs .1
        if lo < ls:
            oversion.extend([0] * (ls - lo))
        elif ls < lo:
            sversion.extend([0] * (lo - ls))

        # 0.0.1.1 comes ... after 0.0.1.post1 ???
        for v in [sversion, oversion]:
            if 'dev' in v:
                v[v.index('dev')] = -4
            elif 'post' in v:
                v[v.index('post')] = 1

        if sversion == oversion:
            return 0
        if sversion < oversion:
            return -1
        if sversion > oversion:
            return 1


class SetupPath(aug.RepoPath):
    # TODO get latest release info from github and pypi

    @property
    def setupfu(self):
        with self.folder:
            spec = importlib.util.spec_from_file_location('setup', self.setup_file)
            setup = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(setup)
            args, kwargs = last_output[0]
            return setup, args, kwargs

    @property
    def setup_kwargs(self):
        if not hasattr(self, '_setup_kwargs'):
            mod, args, kwargs = self.setupfu
            self._setup_kwargs = kwargs

        return self._setup_kwargs

    @property
    def version_latest_pypi(self):
        return Version(self.pypi_json['info']['version'])

    @property
    def version_latest_released(self):
        # git, pypi, tag??
        vers = sorted(Version(_) for _ in self.pypi_json['releases'])
        return vers[-1]

    def version_next(self, increment='current'):
        # major middle minor dev post  # or something
        # FIXME
        return self.version_latest_pypi.next(increment=increment)

    @property
    def version_repo(self):
        return Version(self.setup_kwargs['version'])

    @property
    def version_new(self):
        # TODO cases dev normal
        # want dev release but repo is at an unreleased normal
        # want normal, already released this one
        # want dev, already released this one
        # want *, repo skips a version
        return self.version_repo
        raise NotImplementedError('TODO')

    @property
    def module_init_file(self):
        return self.module / '__init__.py'

    @property
    def module(self):
        kwargs = self.setup_kwargs
        name = kwargs['name']
        packages = kwargs['packages']
        for package in packages:
            if package == name:
                return self.folder / name

        raise NotImplementedError(f'Don\'t know how to release packages whose name does not match a package name. {name} {packages}')

    @property
    def setup_file(self):
        return self.folder / 'setup.py'

    @property
    def folder(self):
        if not self.is_absolute() or '..' in self.parts:
            return self.resolve().folder

        if self.is_dir():
            for f in self.glob('setup.py'):
                return self

        return self.parent.folder

    @property
    def pypi_json(self):
        if not hasattr(self, '_pypi_json'):
            self._pypi_request = requests.get(f'https://pypi.org/pypi/{self.arg_packagename}/json')
            self._pypi_json = self._pypi_request.json()

        return self._pypi_json

    @property
    def arg_org(self):
        u = urlparse(self.remote_uri_human())
        _, org, repo, *_ = u.path.split('/')
        return org

    @property
    def arg_repo(self):
        u = urlparse(self.remote_uri_human())
        _, org, repo, *_ = u.path.split('/')
        return repo

    @property
    def arg_folder(self):
        return self.folder.relative_to(self.working_dir.parent)

    @property
    def arg_packagename(self):
        return self.setup_kwargs['name']

    @property
    def arg_rest(self):
        # TODO
        return ''

    @property
    def command(self):
        rest = self.arg_rest
        rest = ' ' + self.rest if rest else ''
        return f'build-release {self.arg_org} {self.arg_repo} {self.arg_folder} {self.arg_packagename} {self.version_new}{rest}'


SetupPath._bind_flavours()


def main():
   sp = SetupPath('.')  # options.path
   print(sp.version_next())
   print(sp.command)
   spn = SetupPath('~/git/rdflib').expanduser()
   asdf = sorted([Version(_) for _ in spn.pypi_json['releases'].keys()])
   print(asdf)
   print([v.next('current') for v in asdf])
   print([v.next('dev') for v in asdf])
   print([v.next('minor') for v in asdf])
   #breakpoint()


if __name__ == '__main__':
    main()
#+end_src

#+NAME: release-next-old
#+BEGIN_SRC bash :eval never :exports neither
release-next () {
    # example
    # release-next path/to/folder/module/__init__.py
    # vs
    # release-next path/to/folder/module
    # vs
    # release-next path/to/folder

    # behavior should probably be to search recursively up until we find a setup.py file ...
    WORKING_DIR=$(git rev-parse --show-toplevel)
    MODULE_PATH=$(dirname INIT_PATH)
    FOLDER=$(dirname MODULE_PATH)
    SETUP_PATH="${FOLDER}/setup.py"
    ORG=
    # get folder package name
    # get version
    # find setup.py
}
#+END_SRC

** Utils
#+name: &are-you-sure
#+caption: also defined in [[file:../nifstd/scigraph/README.org::&are-you-sure][&are-you-sure]]
#+begin_src bash :eval never
function are-you-sure () {
    read -p "Are you sure you want to push the final release? yes/N " -n 1 choice
    # ((((
    case "${choice}" in
        yes|YES) echo ;;
        n|N) echo; echo "Not pushing final release."; return 1;;
        '?') echo; echo "$(set -o posix; set | grep -v '^_')"; return 1;;
        ,*)   echo; echo "Not pushing final release."; return 1;;
    esac
    echo "Pushing final release ..."
}
#+end_src
* Examples
These are examples. They may be out of date and already finished.
#+CAPTION: pyontutils examples
#+BEGIN_SRC bash :eval never
build-release tgbugs pyontutils pyontutils/librdflib librdflib 0.0.1
push-release pyontutils/librdflib ~/nas/software-releases librdflib 0.0.1
final-release ~/nas/software-releases librdflib 0.0.1

build-release tgbugs pyontutils pyontutils/htmlfn htmlfn 0.0.1
push-release pyontutils/htmlfn ~/nas/software-releases htmlfn 0.0.1
final-release ~/nas/software-releases htmlfn 0.0.1

build-release tgbugs pyontutils pyontutils/ttlser ttlser 1.0.0
push-release pyontutils/ttlser ~/nas/software-releases ttlser 1.0.0
final-release ~/nas/software-releases ttlser 1.0.0

build-release tgbugs pyontutils pyontutils pyontutils 0.1.2
push-release pyontutils ~/nas/software-releases pyontutils 0.1.2
final-release ~/nas/software-releases pyontutils 0.1.2

NIFSTD_CHECKOUT_OK=1 build-release tgbugs pyontutils pyontutils/neurondm neurondm 0.1.0
push-release pyontutils/neurondm ~/nas/software-releases neurondm 0.1.0
final-release ~/nas/software-releases neurondm 0.1.0

build-release tgbugs pyontutils pyontutils/nifstd nifstd-tools 0.0.1
#+END_SRC

* pyontutils full repo release testing
NOTE if you reuse a repo run =git clean -dfx= to clear all untracked files.
#+BEGIN_SRC bash :eval never
pushd /tmp
git clone https://github.com/tgbugs/pyontutils.git
pushd pyontutils
python setup.py sdist; cp dist/pyontutils* /tmp/release-testing
for f in {librdflib,htmlfn,ttlser,neurondm,nifstd}; do pushd $f; python setup.py sdist; cp dist/$f* /tmp/release-testing/; popd; done
pushd /tmp/release-testing
find -name "*.tar.gz" -exec tar xvzf {} \;
for f in {librdflib,htmlfn,ttlser,pyontutils,neurondm,nifstd}; do pushd $f*/; pip install -e .[test]; python setup.py test; popd; done
#+END_SRC

From inside /tmp/${repo}
#+NAME: bdist_wheel-from-sdist
#+CAPTION: build wheels from sdist never from repo directly
#+BEGIN_SRC bash :eval never
pushd dist/
tar xvzf pyontutils*.tar.gz
pushd pyontutils*/
python setup.py bdist_wheel
mv dist/*.whl ../
popd
rm ./pyontutils*/ -r
popd

for f in {librdflib,htmlfn,ttlser,neurondm,nifstd}; do
pushd $f/dist
tar xvzf $f*.tar.gz
pushd $f*/
python setup.py bdist_wheel
mv dist/*.whl ../
popd
rm ./$f*/ -r
popd
done
#+END_SRC
