def buildImage = "icr.io/streamsets_private/sdk-test-env:latest"
def failedStages = []
def integrationTestResults = ''
def integrationTestSummary = [:]
def unitTestResults_310 = ''
def unitTestResults_311 = ''
def unitTestResults_312 = ''
def unitTestSummary = [:]
def unstableStages = []


pipeline {
    options { timestamps () }
    agent {
        label 'testframework-builder-podman'
    }
    triggers {
        cron(env.BRANCH_NAME == 'master' ? '0 5 * * 1-5' : '')
    }
    parameters {
        booleanParam 'run_integration_test'
        booleanParam 'run_documentation_test'
        choice choices: ['dev', 'preprod'], name: 'environment'
        string name: 'STF_BRANCH', defaultValue: '', description: 'STF branch to use for downstream builds'
        booleanParam 'build_stf'
    }
    stages {
        stage('Build & Test') {
            parallel {
                stage('Build') {
                    stages {
                        stage('Configure') {
                            steps {
                                script {
                                    env.BUILD_SDK = (hasChangesIn('python/streamsets/sdk') || hasChangesIn('python/Dockerfile'))
                                    env.BUILD_STF = (params.STF_BRANCH || params.build_stf)
                                    env.TARGET_BRANCH = env.BRANCH_NAME
                                }
                            }
                        }
                        stage('Sonarqube scan') {
                            when {
                                expression {
                                    env.BRANCH_NAME == 'master' && !env.CHANGE_ID
                                }
                            }
                            steps {
                                withCredentials([string(credentialsId: '36abdb24-82b3-44f7-9472-14bd93afd70c', variable: 'SONAR_TOKEN')]) {
                                    withEnv (["SONARQUBE_URL=https://sonar.streamsets.net", "REPO_PATH=${WORKSPACE}/python"]){
                                        script {

                                            sh '''
                                                # Our internal SonarQube instance is version 7.9.5 which is EOL, so we have to grab an older version of the scanner image (4.6)
                                                podman run \
                                                    --rm \
                                                    -e SONAR_HOST_URL="${SONARQUBE_URL}"  \
                                                    -e SONAR_TOKEN=${SONAR_TOKEN} \
                                                    -v "${REPO_PATH}:/usr/src" \
                                                    sonarsource/sonar-scanner-cli:4.6 -X
                                            '''
                                        }
                                    }
                                }
                            }
                        }
                        stage('SDK Image build') {
                            when {
                                anyOf {
                                    expression {
                                        return env.BUILD_SDK.toBoolean()
                                    }
                                    expression {
                                        return env.BUILD_STF.toBoolean()
                                    }
                                }
                            }
                            steps {
                                script {
                                    env.BUILD_TAG = getBranchTag()
                                    env.SDK_IMAGE_NAME = "icr.io/streamsets_private/sdk-python:${BUILD_TAG}"
                                }
                                script {
                                    sh '''
                                        echo "Building image ${SDK_IMAGE_NAME} ..."
                                        podman build ${DOCKER_BUILD_ARGS} -t ${SDK_IMAGE_NAME} python \
                                            && podman push ${SDK_IMAGE_NAME}
                                    '''
                                }
                            }
                            post {
                                success {
                                    echo "SDK image build success!"
                                    script{
                                        currentBuild.description = "SDK image name: ${SDK_IMAGE_NAME}"
                                    }
                                }
                                failure {
                                    echo "SDK image build failed."
                                }
                            }
                        }
                        stage('STF Podman build') {
                            when {
                                expression {
                                    return env.BUILD_STF.toBoolean()
                                }
                            }
                            steps {
                                script {
                                    def buildParams = [
                                        string(name: "SDK_BRANCH", value: getBranchTag())
                                    ]

                                    // Determine STF job based on target branch
                                    if (params.STF_BRANCH) {
                                        stfJob = "testframework/${params.STF_BRANCH}-podman"
                                    } else if (env.TARGET_BRANCH.startsWith("3.")) {
                                        stfJob = "testframework/3.x-podman/"
                                    } else {
                                        stfJob = "testframework/master-podman/"
                                    }

                                    stfBuild = [:]
                                    stfBuild = build job: stfJob,
                                                    parameters: buildParams,
                                                    propagate: false
                                    env.STF_BUILD_URL = stfBuild.getAbsoluteUrl()
                                }
                            }
                            post {
                                success {
                                    echo "STF podman build success! ${STF_BUILD_URL}"
                                }
                                unstable {
                                    echo "STF podman build unstable: ${STF_BUILD_URL}"
                                }
                                failure {
                                    echo "STF podman build failed: ${STF_BUILD_URL}"
                                }
                            }
                        }
                        stage('STF Docker build') {
                            when {
                                expression {
                                    return env.BUILD_STF.toBoolean()
                                }
                            }
                            steps {
                                script {
                                    def buildParams = [
                                        string(name: "SDK_BRANCH", value: getBranchTag())
                                    ]

                                    // Determine STF job based on target branch
                                    if (params.STF_BRANCH) {
                                        stfJob = "testframework/${params.STF_BRANCH}"
                                    } else if (env.TARGET_BRANCH.startsWith("3.")) {
                                        stfJob = "testframework/3.x/"
                                    } else {
                                        stfJob = "testframework/master/"
                                    }

                                    stfBuild = [:]
                                    stfBuild = build job: stfJob,
                                                    parameters: buildParams,
                                                    propagate: false
                                    env.STF_BUILD_URL = stfBuild.getAbsoluteUrl()
                                }
                            }
                            post {
                                success {
                                    echo "STF docker build success! ${STF_BUILD_URL}"
                                }
                                unstable {
                                    echo "STF docker build unstable: ${STF_BUILD_URL}"
                                }
                                failure {
                                    echo "STF docker build failed: ${STF_BUILD_URL}"
                                }
                            }
                        }
                    }
                }
                stage('Test') {
                    agent {
                        label 'testframework-builder'
                    }
                    stages {
                        stage('Build Test Image') {
                            when {
                                expression {
                                    return hasChangesIn('python/evergreen/Dockerfile')
                                }
                            }
                            steps {
                                catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
                                    script {
                                        def buildTag = getBranchTag(true)
                                        buildImage = "icr.io/streamsets_private/sdk-test-env:${buildTag}"
                                        def image = docker.build("${buildImage}", 'python/evergreen/')

                                        docker.withRegistry('https://icr.io', 'sx-automation-icr-api-key') {
                                            image.push()
                                        }

                                    }
                                }
                            }
                            post {
                                unstable {
                                    script {
                                        unstableStages.add('Build Test Image')
                                    }
                                }
                            }
                        }
                        stage('Lint & Format') {
                            agent {
                                docker {
                                    image 'python:3.10'
                                    reuseNode true
                                }
                            }
                            steps {
                                catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
                                    withEnv(["HOME=${env.WORKSPACE}"]) {
                                        sh 'git config --global --add safe.directory ${WORKSPACE}'
                                        sh 'pip install pre-commit --user'
                                        sh '.local/bin/pre-commit run --config python/evergreen/.pre-commit-config.yaml --all-files --show-diff-on-failure --verbose'
                                    }
                                }
                            }
                            post {
                                unstable {
                                    script {
                                        unstableStages.add('Lint & Format')
                                    }
                                }
                            }
                        }
                        stage('Unit Tests') {
                            agent {
                                docker {
                                    args '-v /var/run/docker.sock:/var/run/docker.sock -u root:root'
                                    image "${buildImage}"
                                    reuseNode true
                                    registryCredentialsId 'sx-automation-icr-api-key'
                                    registryUrl 'https://icr.io'
                                }
                            }
                            steps {
                                catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                                    withEnv(["HOME=${env.WORKSPACE}"]) {
                                        script {
                                            sh 'git config --global --add safe.directory ${WORKSPACE}'
                                            sh 'tox -c python/tox.ini -e unit-310,unit-311,unit-312'
                                        }
                                    }
                                }
                            }
                            post {
                                always {
                                    script {
                                        unitTestResults_310 = junit testResults: 'python/results_unit_310.xml'
                                        unitTestResults_311 = junit testResults: 'python/results_unit_311.xml'
                                        unitTestResults_312 = junit testResults: 'python/results_unit_312.xml'
                                    }
                                }
                                success {
                                    recordCoverage(
                                        tools: [[parser: 'COBERTURA', pattern: '**/coverage_unit_310.xml']],
                                        id: 'unit_310',
                                        name: 'Python3.10 Unit Tests'
                                    )
                                    recordCoverage(
                                        tools: [[parser: 'COBERTURA', pattern: '**/coverage_unit_311.xml']],
                                        id: 'unit_311',
                                        name: 'Python3.11 Unit Tests'
                                    )
                                    recordCoverage(
                                        tools: [[parser: 'COBERTURA', pattern: '**/coverage_unit_312.xml']],
                                        id: 'unit_312',
                                        name: 'Python3.12 Unit Tests'
                                    )
                                }
                                failure {
                                    script {
                                        if (unitTestResults_310.failCount || unitTestResults_311.failCount || unitTestResults_312.failCount) {
                                            failedStages.add('Unit Tests')
                                            unitTestSummary.put('Python 3.10', ['total': unitTestResults_310.totalCount,
                                                                            'passing': unitTestResults_310.passCount,
                                                                            'failing': unitTestResults_310.failCount,
                                                                            'skipped': unitTestResults_310.skipCount])
                                            unitTestSummary.put('Python 3.11', ['total': unitTestResults_311.totalCount,
                                                                                'passing': unitTestResults_311.passCount,
                                                                                'failing': unitTestResults_311.failCount,
                                                                                'skipped': unitTestResults_311.skipCount])
                                            unitTestSummary.put('Python 3.12', ['total': unitTestResults_312.totalCount,
                                                                                'passing': unitTestResults_312.passCount,
                                                                                'failing': unitTestResults_312.failCount,
                                                                                'skipped': unitTestResults_312.skipCount])
                                        }
                                    }
                                }
                            }
                        }
                        stage('Integration Tests') {
                            when {
                                anyOf {
                                    allOf {
                                        expression {
                                            params.run_integration_test
                                        }
                                        triggeredBy cause: "UserIdCause"
                                    }
                                    branch 'master'
                                }
                                beforeAgent true
                            }
                            agent {
                                docker {
                                    args '-v /var/run/docker.sock:/var/run/docker.sock -u root:root'
                                    image "${buildImage}"
                                    label 'DataOps_SDK'
                                    registryCredentialsId 'sx-automation-icr-api-key'
                                    registryUrl 'https://icr.io'
                                }
                            }
                            steps {
                                script {
                                    if (params.environment == 'dev') {
                                        env.ASTER_URL = "https://dev.login.streamsets.com"
                                        env.TRANSFORMER_VERSION = "4.2.0"
                                        env.CRED_ID = "ebf3adfe-5bed-4de2-8137-677723792dd1"
                                        env.SYS_ADMIN = "86ce4d9f-b17d-408b-a932-2835b3213c29"
                                    } else {
                                        env.ASTER_URL = "https://cascade.login.streamsets.com"
                                        env.TRANSFORMER_VERSION = "5.2.0"
                                        env.CRED_ID = "08c98114-44f0-43cf-8bd2-09a3798b90ba"
                                        env.SYS_ADMIN = "4489142f-e036-4671-958b-9b6798f34350"
                                    }
                                    env.FIREBASE_ID = "ASTER_DEV_FIREBASE_API_KEY"
                                    env.ORG_ADMIN = "jenkins_org_admin_test_account"
                                    catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {

                                        withEnv(["HOME=${env.WORKSPACE}", "ASTER_URL=${env.ASTER_URL}",
                                                "SDC_VERSION=5.2.0", "TRANSFORMER_VERSION=${env.TRANSFORMER_VERSION}", "SUBNET_IDS=[subnet-319c5468]",
                                                "VPC_ID=vpc-12279b77", "SECURITY_GROUP_ID=sg-03434536f7156364d", "REGION=us-west-2",
                                                "INSTANCE_PROFILE='arn:aws:iam::316386816690:instance-profile/csp-test-role'"]) {
                                            withCredentials([usernamePassword(credentialsId: env.ORG_ADMIN,
                                                                            passwordVariable: 'ORG_ADMIN_PASSWORD',
                                                                            usernameVariable: 'ORG_ADMIN_EMAIL'),
                                                            usernamePassword(credentialsId: env.SYS_ADMIN, passwordVariable: 'SYS_ADMIN_PASSWORD', usernameVariable: 'SYS_ADMIN_EMAIL'),
                                                            string(credentialsId: env.FIREBASE_ID, variable: 'FIREBASE_API_KEY'),
                                                            [$class: 'AmazonWebServicesCredentialsBinding',
                                                            credentialsId: '50cddd55-b7a4-4fc1-a0e3-064c17af537c',
                                                            accessKeyVariable: 'AWS_ACCESS_KEY_ID',
                                                            secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'],
                                                            usernamePassword(credentialsId: '1f48db52-3ea4-4c95-9b47-194c5a3c9798',
                                                            passwordVariable: 'SNOWFLAKE_PASSWORD',
                                                            usernameVariable: 'SNOWFLAKE_USERNAME')
                                                            ]) {
                                                sh 'git config --global --add safe.directory ${WORKSPACE}'
                                                sh 'tox -c python/tox.ini -e integration-312'
                                            }
                                        }
                                    }
                                }
                            }
                            post {
                                always {
                                    script {
                                        integrationTestResults = junit testResults: 'python/results_integration_312.xml'
                                    }
                                }
                                success {
                                    recordCoverage(
                                        tools: [[parser: 'COBERTURA', pattern: '**/coverage_integration_312.xml']],
                                        id: 'integration_312',
                                        name: 'Integration Tests'
                                    )
                                }
                                failure {
                                    script {
                                        failedStages.add('Integration Tests')
                                        integrationTestSummary['total'] = integrationTestResults.totalCount
                                        integrationTestSummary['passing'] = integrationTestResults.passCount
                                        integrationTestSummary['failing'] = integrationTestResults.failCount
                                        integrationTestSummary['skipped'] = integrationTestResults.skipCount
                                    }
                                }
                            }
                        }
                        stage('Documentation Build') {
                            when {
                                anyOf {
                                    allOf {
                                        expression {
                                            params.run_documentation_test
                                        }
                                        triggeredBy cause: "UserIdCause"
                                    }
                                    branch 'master'
                                }
                                beforeAgent true
                            }
                            agent {
                                docker {
                                    args '-v /var/run/docker.sock:/var/run/docker.sock -u root:root'
                                    image "${buildImage}"
                                    reuseNode true
                                    registryCredentialsId 'sx-automation-icr-api-key'
                                    registryUrl 'https://icr.io'
                                }
                            }
                            steps {
                                catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
                                    withEnv(["HOME=${env.WORKSPACE}"]) {
                                        sh 'git config --global --add safe.directory ${WORKSPACE}'
                                        sh 'tox -c python/tox.ini -e docs'
                                    }
                                }
                            }
                            post{
                                unstable {
                                    script {
                                        unstableStages.add('Documentation Build')
                                    }
                                }
                            }
                        }
                    }
                    post {
                        success {
                            script {
                                if (env.BRANCH_NAME == 'master') {
                                    slackSend(
                                        channel: "#ep-controlplane",
                                        color: '#007D00',
                                        message:"Evergreen Master Status: PASSING \n Build: ${env.BUILD_NUMBER} (<${env.BUILD_URL}|Open>)"
                                    )
                                }
                            }
                        }
                        unstable {
                            script {
                                if (env.BRANCH_NAME == 'master') {
                                    def unstableMessage = "Evergreen Master Status: UNSTABLE \n Unstable Stages:"
                                    unstableStages.each { stage ->
                                        unstableMessage += "\n - ${stage}"
                                    }
                                    unstableMessage += "\n Build: ${env.BUILD_NUMBER} (<${env.BUILD_URL}|Open>)"
                                    slackSend(
                                        channel: "#ep-controlplane",
                                        color: '#FFEF66',
                                        message: unstableMessage
                                    )
                                }
                            }
                        }
                        failure {
                            script {
                                if (env.BRANCH_NAME == 'master') {
                                    def failureMessage = "Evergreen Master Status: FAILED \n\n"

                                    if ('Unit Tests' in failedStages) {
                                        unitTestSummary.each{ key, value ->
                                            failureMessage += """${key} Unit Tests:\n\n
                                                                \tTotal: ${value['total']}\n
                                                                \tPassing: ${value['passing']}\n
                                                                \tFailing: ${value['failing']}\n
                                                                \tSkipped: ${value['skipped']}\n\n"""}
                                    }
                                    if ('Integration Tests' in failedStages) {
                                        failureMessage += "Integration Test Results:\n"
                                        failureMessage += "\t Total: ${integrationTestSummary['total']}\n"
                                        failureMessage += "\t Passing: ${integrationTestSummary['passing']}\n"
                                        failureMessage += "\t Failing: ${integrationTestSummary['failing']}\n"
                                        failureMessage += "\t Skipped: ${integrationTestSummary['skipped']}\n\n"
                                    }

                                    failureMessage += "Build #${env.BUILD_NUMBER} (<${env.BUILD_URL}|Open>)"
                                    slackSend(
                                        channel: "#ep-controlplane",
                                        color: '#FF0000',
                                        message: failureMessage
                                    )
                                }
                            }
                        }
                        cleanup {
                            cleanWs()
                        }
                    }
                }
            }
        }
    }
}

def boolean hasChangesIn(String module) {
  // For PRs, compare against target branch
  // For branch builds, compare against previous commit
  def gitCommand
  if (env.CHANGE_ID) {
    // This is a PR - compare against target branch
    // Use CHANGE_TARGET which Jenkins provides for PRs
    gitCommand = "git diff --name-only origin/PR-${env.CHANGE_ID}^1 HEAD -- '${module}' 2>/dev/null || git diff --name-only HEAD~1 HEAD -- '${module}' 2>/dev/null || echo '${module}'"
  } else {
    // This is a branch build - compare against previous commit
    // If this is the first commit or comparison fails, assume changes exist
    gitCommand = "git diff-tree --no-commit-id --name-only -r ${env.GIT_COMMIT} -- '${module}' 2>/dev/null || echo '${module}'"
  }

  def output = sh(script: gitCommand, returnStdout: true).trim()

  if (output) {
    echo "Changes detected in module: '${module}'"
    return true
  } else {
    echo "No changes detected in module: '${module}'"
    return false
  }
}


def String getBranchTag(boolean useLatestForMaster = false) {
    if (env.CHANGE_ID) {
        return "PR-${env.CHANGE_ID}"
    } else if (useLatestForMaster && env.BRANCH_NAME == 'master') {
        return 'latest'
    } else {
        return env.BRANCH_NAME
    }
}
