# -*- coding: utf-8 -*-
from setuptools import setup

packages = \
['cloudsend', 'cloudsend.resources.remote_files']

package_data = \
{'': ['*'],
 'cloudsend': ['resources/instancetypes-aws.csv',
               'resources/instancetypes-aws.csv',
               'resources/requirements.txt',
               'resources/requirements.txt']}

install_requires = \
['asyncssh>=2.12.0,<2.13.0',
 'boto3>=1,<2',
 'jcs>=0.2.1,<0.3.0',
 'ortools>=9,<10',
 'psutil>=5.9.4,<6.0.0',
 'pyyaml>=6.0,<7.0']

entry_points = \
{'console_scripts': ['cli = cloudsend.cli:main',
                     'client = cloudsend.maestroclient:main',
                     'demo = cloudsend.demo:main']}

setup_kwargs = {
    'name': 'cloudsend',
    'version': '0.4.2',
    'description': 'CloudSend is a Python package that allows you to run any script on a cloud service (for now AWS only).',
    'long_description': '# Description\n\nCloudSend is a Python package that allows you to run any script on a cloud service (for now AWS only).\n\n# Features\n\n- Easily run scripts on AWS by writing a simple configuration file\n- Handles Python and Julia scripts, or any command\n- Handles PyPi , Conda/Mamba, Apt-get and Julia environments\n- Concurrent instance support\n- Handles disconnections from instances, including stopped or terminated instances\n- Handles interruption of CloudSend, with state recovery\n- Runs locally or on a remote instance, with \'watcher\' functionality \n\n# Pre-requisites\n\nIn order to use the python AWS client (Boto3), you need to have an existing AWS account and to setup your computer for AWS.\n\n## with AWS CLI\n\n1. Go to [the AWS Signup page](https://portal.aws.amazon.com/billing/signup#/start/email) and create an account\n2. Download [the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)\n3. In the AWS web console, [create a user with administrator privilege](https://docs.aws.amazon.com/streams/latest/dev/setting-up.html)\n4. In the AWS web console, under the AMI section, click on the new user and make sure you create an access key under the tab "Security Credentials". Make sure "Console Password" is Enabled as well\n5. In ther Terminal, use the AWS CLI to setup your configuration:\n```\naws configure\n```\nSee [https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html](here)\n\n6. To run in `remote` mode, you also need to [add the following credentials to your user](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) (maybe):\n- iam:PassRole\n- iam:CreateRole\n- ec2:AssociateIamInstanceProfile\n- ec2:ReplaceIamInstanceProfileAssociation\n\n## manually\n\n1. Go to [the AWS Signup page](https://portal.aws.amazon.com/billing/signup#/start/email) and create an account\n2. In the AWS web console, [create a user with administrator privilege](https://docs.aws.amazon.com/streams/latest/dev/setting-up.html)\n3. In the AWS web console, under the IAM section, click on the new user and make sure you create an access key under the tab "Security Credentials". Make sure "Console Password" is Enabled as well\n4. Add your new user credentials manually, [in the credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html)\n\n##### \'~/.aws/config\' example (\'C:\\Users\\USERNAME\\\\.aws\\config\' on Windows)\n\n```\n[default]\nregion = eu-west-3\noutput = json\n```\n\n##### \'~/.aws/credentials\' example (\'C:\\Users\\USERNAME\\\\.aws\\credentials\' on Windows)\n\n```\n[default]\naws_access_key_id = YOUR_ACCESS_KEY_ID\naws_secret_access_key = YOUR_SECRET_ACCESS_KEY\n```\n\n## Setting up a separate user with least permissions (manually) \n\n1. In the AWS web console, in the IAM service, create a group \'cloudsend-users\' with `AmazonEC2FullAccess` and `IAMFullAccess` permissions\n2. In the AWS web console, in the IAM service, create a user USERNAME attached to the \'cloudsend-users\' group:\n### Step 1\n![add user 1](./images/adduser1.jpg)\n### Step 2\n![add user 2](./images/adduser2.jpg)\n### ... Step 5\n![add user 3](./images/adduser3.jpg)\n\n  **COPY the Access Key info !**\n\n3. Add your new user profile manually, [in the credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html)\n\n##### \'~/.aws/config\' example (\'C:\\Users\\USERNAME\\\\.aws\\config\' on Windows)\n\n```\n[default]\nregion = eu-west-3\noutput = json\n\n[profile cloudsend]\nregion = eu-west-3\noutput = json\n```\n\n##### \'~/.aws/credentials\' example (\'C:\\Users\\USERNAME\\\\.aws\\credentials\' on Windows)\n\n```\n[default]\naws_access_key_id = YOUR_ACCESS_KEY_ID\naws_secret_access_key = YOUR_SECRET_ACCESS_KEY\n\n[cloudsend]\naws_access_key_id = YOU_PROFILE_ACCESS_KEY_ID\naws_secret_access_key = YOUR_PROFILE_SECRET_ACCESS_KEY\n```\n\n4. add the \'profile\' : \'cloudsend_USERNAME\' to the configuration\n\n```python\nconfig = {\n\n    ################################################################################\n    # GLOBALS\n    ################################################################################\n\n    \'project\'      : \'test\' ,                             # this will be concatenated with the instance hashes (if not None) \n    \'profile\'      : \'cloudsend\' ,\n    ...\n```\n\n\n# Installation\n\n## On MacOS / Linux\n\n### with pip\n\n```bash\npython3 -m venv .venv\nsource ./.venv/bin/activate\npython3 -m ensurepip --default-pip\npython3 -m pip install -r requirements.txt\n```\n\n### with poetry\n\n```bash\ncurl -sSL https://install.python-poetry.org | python3.8 -\npoetry install\n```\n\n## On Windows (powershell)\n\n### with pip\n\n```bat\nC:\\> python3 -m venv .venv\nC:\\> .venv\\\\Scripts\\\\activate.bat\nC:\\> python -m pip install -r requirements.txt\n```\n\n### with poetry\n\n```bat\nC:\\> (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | py -\nC:\\> poetry install\n```\n\n# Usage / Test runs\n\n```bash\n# copy the example file\ncp example/config.example.py config.py\n#\n# EDIT THE FILE\n#\n\n# to run with pip\npython3 -m cloudsend.demo config\n# to run with pip with reset (maestro and instances)\npython3 -m cloudsend.demo config reset\n# to run with poetry\npoetry run demo config\n# to run with poetry with reset (maestro and the instances)\npoetry run demo config reset\n```\n\n# Configuration example\n\n```python\nconfig = {\n\n    ################################################################################\n    # GLOBALS\n    ################################################################################\n\n    \'project\'      : \'test\' ,                             # this will be concatenated with the instance hashes (if not None) \n    \'profile\'      : None ,                               # if you want to use a specific profile (user/region), specify its name here\n    \'dev\'          : False ,                              # When True, this will ensure the same instance and dev environement are being used (while working on building up the project) \n    \'debug\'        : 1 ,                                  # debug level (0...3)\n    \'maestro\'      : \'local\' ,                            # where the \'maestro\' resides: local\' | \'remote\' (micro instance)\n    \'auto_stop\'    : True ,                               # will automatically stop the instances and the maestro, once the jobs are done\n    \'provider\'     : \'aws\' ,                              # the provider name (\'aws\' | \'azure\' | ...)\n    \'job_assign\'   : None ,                               # algorithm used for job assignation / task scheduling (\'random\' | \'multi_knapsack\')\n    \'recover\'      : True ,                               # if True, CloudSend will always save the state and try to recover this state on the next execution\n    \'print_deploy\' : False ,                              # if True, this will cause the deploy stage to print more (and lock)\n    \'mutualize_uploads\' : True ,                          # adjusts the directory structure of the uploads ... (False = per job or True = global/mutualized)\n\n\n    ################################################################################\n    # INSTANCES / HARDWARE\n    ################################################################################\n\n    \'instances\' : [\n        { \n            \'region\'       : None ,                       # can be None or has to be valid. Overrides AWS user region configuration.\n            \'cloud_id\'     : None ,                       # can be None, or even wrong/non-existing - then the default one is used\n            \'img_id\'       : \'ami-077fd75cd229c811b\' ,    # OS image: has to be valid and available for the profile (user/region)\n            \'img_username\' : \'ubuntu\' ,                   # the SSH user for the image\n            \'type\'         : \'t2.micro\' ,                 # proprietary size spec (has to be valid)\n            \'cpus\'         : None ,                       # number of CPU cores\n            \'gpu\'          : None ,                       # the proprietary type of the GPU \n            \'disk_size\'    : None ,                       # the disk size of this instance type (in GB)\n            \'disk_type\'    : None ,                       # the proprietary disk type of this instance type: \'standard\', \'io1\', \'io2\', \'st1\', etc\n            \'eco\'          : True ,                       # eco == True >> SPOT e.g.\n            \'eco_life\'     : None ,                       # lifecycle of the machine in ECO mode (datetime.timedelta object) (can be None with eco = True)\n            \'max_bid\'      : None ,                       # max bid ($/hour) (can be None with eco = True)\n            \'number\'       : 1 ,                          # multiplicity: the number of instance(s) to create\n            \'explode\'      : True                         # multiplicity: can this instance type be distributed accross multiple instances, to split CPUs\n        }\n\n    ] ,\n\n    ################################################################################\n    # ENVIRONMENTS / SOFTWARE\n    ################################################################################\n\n    \'environments\' : [\n        {\n            \'name\'         : None ,                       # name of the environment - should be unique if not \'None\'. \'None\' only when len(environments)==1\n\n            # env_conda + env_pypi  : mamba is used to setup the env (pip dependencies included)\n            # env_conda (only)      : mamba is used to setup the env\n            # env_pypi  (only)      : venv + pip is used to setup the env \n\n            \'command\'      : \'example/install_julia.sh\' ,      # None, or a string: path to a bash file to execute when deploying\n            \'env_aptget\'   : [ "openssh-client"] ,        # None, an array of librarires/binaries for apt-get\n            \'env_conda\'    : "example/environment.yml",   # None, an array of libraries, a path to environment.yml  file, or a path to the root of a conda environment\n            \'env_conda_channels\' : None ,                 # None, an array of channels. If None (or absent), defaults and conda-forge will be used\n            \'env_pypi\'     : "example/requirements.txt" , # None, an array of libraries, a path to requirements.txt file, or a path to the root of a venv environment \n            \'env_julia\'    : [ "Wavelets" ] ,             # None, a string or an array of Julia packages to install (requires julia)\n        }\n    ] ,\n\n    ################################################################################\n    # JOBS / SCRIPTS\n    ################################################################################\n\n    \'jobs\' : [\n        {\n            \'env_name\'     : None ,                       # the environment to use (can be \'None\' if solely one environment is provided above)\n            \'cpus_req\'     : None ,                       # the CPU(s) requirements for the process (can be None)\n            \'run_script\'   : \'example/run_remote.py 1 10\',# the script to run (Python (.py) or Julia (.jl) for now) (prioritised vs \'run_command\')\n            \'run_command\'  : None ,                       # the command to run\n            \'upload_files\' : [ "uploaded.txt"] ,          # any file to upload (array or string) - will be put in the same directory\n            \'input_file\'   : \'input.dat\' ,                # the input file name (used by the script)\n            \'output_file\'  : \'output.dat\' ,               # the output file name (used by the script)\n            \'repeat\'       : 2 ,                          # the number of times this job is repeated\n        } ,\n        {\n            \'env_name\'     : None ,                       # the environment to use (can be \'None\' if solely one environment is provided above)\n            \'cpus_req\'     : None ,                       # the CPU(s) requirements for the process (can be None)\n            \'run_script\'   : \'example/run_remote.py 2 12\',# the script to run (Python (.py) or Julia (.jl) for now) (prioritised vs \'run_command\')\n            \'run_command\'  : None ,                       # the command to run\n            \'upload_files\' : [ "uploaded.txt"] ,          # any file to upload (array or string) - will be put in the same directory\n            \'input_file\'   : \'input.dat\' ,                # the input file name (used by the script)\n            \'output_file\'  : \'output.dat\' ,               # the output file name (used by the script)\n        }\n    ]\n}\n```\n\n# Minimum configuration example\n\n```python\nconfig = {\n    \'debug\'        : 1 ,                                  # debug level (0...3)\n    \'maestro\'      : \'local\' ,                            # where the \'maestro\' resides: local\' | \'remote\' (nano instance) | \'lambda\'\n    \'provider\'     : \'aws\' ,                              # the provider name (\'aws\' | \'azure\' | ...)\n\n    \'instances\' : [\n        { \n            \'type\'         : \'t2.micro\' ,                 # proprietary size spec (has to be valid)\n        }\n\n    ] ,\n\n    \'environments\' : [\n        {\n            \'name\'         : None ,                       # name of the environment - should be unique if not \'None\'. \'None\' only when len(environments)==1\n            \'env_conda\'    : "example/environment.yml",   # None, an array of libraries, a path to environment.yml  file, or a path to the root of a conda environment\n            \'env_julia\'    : ["Wavelets"] ,                       # None, a string or an array of Julia packages to install (requires julia)\n        }\n    ] ,\n\n    \'jobs\' : [\n        {\n            \'env_name\'     : None ,                       # the environment to use (can be \'None\' if solely one environment is provided above)\n            \'cpus_req\'     : None ,                       # the CPU(s) requirements for the process (can be None)\n            \'run_script\'   : \'example/run_remote.py 1 10\',# the script to run (Python (.py) or Julia (.jl) for now) (prioritised vs \'run_command\')\n            \'upload_files\' : [ "uploaded.txt"] ,          # any file to upload (array or string) - will be put in the same directory\n            \'input_file\'   : \'input.dat\' ,                # the input file name (used by the script)\n            \'output_file\'  : \'output.dat\' ,               # the output file name (used by the script)\n        } ,\n        {\n            \'env_name\'     : None ,                       # the environment to use (can be \'None\' if solely one environment is provided above)\n            \'cpus_req\'     : None ,                       # the CPU(s) requirements for the process (can be None)\n            \'run_script\'   : \'example/run_remote.py 2 12\',# the script to run (Python (.py) or Julia (.jl) for now) (prioritised vs \'run_command\')\n            \'upload_files\' : [ "uploaded.txt"] ,          # any file to upload (array or string) - will be put in the same directory\n            \'input_file\'   : \'input.dat\' ,                # the input file name (used by the script)\n            \'output_file\'  : \'output.dat\' ,               # the output file name (used by the script)\n        }\n    ]\n}\n```\n\n\n# Python API\n\n```python\nclass CloudSendProcessState(IntFlag):\n    UNKNOWN   = 0\n    WAIT      = 1  # waiting for bootstraping\n    QUEUE     = 2  # queued (for sequential scripts)\n    IDLE      = 4  # script about to start\n    RUNNING   = 8  # script running\n    DONE      = 16 # script has completed\n    ABORTED   = 32 # script has been aborted\n    ANY       = 32 + 16 + 8 + 4 + 2 + 1 \n\nclass CloudSendLightProvider(ABC):\nclass CloudSendFatProvider(ABC):\n\n    def debug(self,level,*args,**kwargs):\n\n    # start the provider: creates the instances\n    # if reset = True, CloudSend forces a process cleanup as well as more re-uploads\n    def start(self,reset):\n\n    # deploy all materials (environments, files, scripts etc.)\n    def deploy(self):\n\n    # run the jobs\n    # returns a CloudSendRunSession\n    def run(self,wait=False):\n\n    # wait for the processes to reach a state\n    def wait(self,job_state,run_session=None):\n\n    # get the states of the processes\n    def get_jobs_states(self,run_session=None):\n\n    def print_jobs_summary(self,run_session=None,instance=None):\n\n    def print_aborted_logs(self,run_session=None,instance=None):\n\n    def fetch_results(self,out_directory,run_session=None):\n\n    # wakeup = start + assign + deploy + run + watch\n    def wakeup(self)\n\n    @abstractmethod\n    def get_region(self):\n\n    @abstractmethod\n    def get_recommended_cpus(self,inst_cfg):\n\n    @abstractmethod\n    def create_instance_objects(self,config,for_maestro):\n\n    @abstractmethod\n    def find_instance(self,config):\n\n    @abstractmethod\n    def start_instance(self,instance):\n\n    @abstractmethod\n    def stop_instance(self,instance):\n\n    @abstractmethod\n    def terminate_instance(self,instance):\n\n    @abstractmethod\n    def update_instance_info(self,instance):    \n\n# GLOBAL methods \n\ndef get_client(provider=\'aws\',maestro=\'local\'):\n\ndef init_instance_name(instance_config):\n\ndef debug(level,*args,**kwargs):\n```\n\n# CloudSend usage\n\n## Python programmatic use\n\nNote: this demo works the same way, whether CloudSend runs locally or remotely\n\n```python\n\nfrom cloudsend      import provider as cloudsend\nfrom cloudsend.core import CloudSendProcessState\nimport asyncio \n\n# load config\nconfig = __import__(config).config\n\n# create provider: this loads the config\nprovider = cloudsend.get_client(config)\n\n# start the provider: this attempts to create the instances\nawait provider.start()\n\n# deploy the necessary stuff onto the instances\nawait provider.deploy()\n\n# run the jobs and get active processes objects back\nrun_session = await provider.run()\n\n# wait for the active proccesses to be done or aborted:\nawait provider.wait(CloudSendProcessState.DONE|CloudSendProcessState.ABORTED)\n\n# you can get the state of all jobs this way:\nawait provider.get_jobs_states()\n# or get the state for a specific run session:\nawait provider.get_jobs_states(run_session)\n\n# you can print processes summary with:\nawait provider.print_jobs_summary()\n\n# get the results file locally\nawait provider.fetch_results(\'./tmp\')\n\n```\n\n## CLI use\n\nNote: the commands below work the same way, whether CloudSend runs locally or remotely\n\n```bash\nCLI commands go here\n```\n\n# Contributing\nPull requests are welcome. For major changes, please open an issue first to discuss what you would like to change.\n\nPlease make sure to update tests as appropriate.\n\n# License\n[MIT](https://choosealicense.com/licenses/mit/)',
    'author': 'Your Name',
    'author_email': 'you@example.com',
    'maintainer': 'None',
    'maintainer_email': 'None',
    'url': 'https://github.com/benbenz/cloudsend',
    'packages': packages,
    'package_data': package_data,
    'install_requires': install_requires,
    'entry_points': entry_points,
    'python_requires': '>=3.9,<4.0',
}


setup(**setup_kwargs)
