Release/docker 2.2.2

Jenkinsfile

@@ -20,7 +20,6 @@ LINT_OUTPUT = ''
 SCAN_OUTPUT = ''
 IMAGE_SIZE = 0
 RPMversion = ''
-TIMESTAMP = new Date().format('yyyyMMdd')
 
 // Define local funtions
 
@@ -198,7 +197,7 @@ void copyRPMs() {
     else if (marklogicVersion == "12") {
         RPMsuffix = ".nightly-rhel"
         RPMbranch = "b12"
-        RPMversion = "12.0"
+        RPMversion = "12.1"
     }
     else {
         error "Invalid value in marklogicVersion parameter."
@@ -235,6 +234,8 @@ void buildDockerImage() {
     publishImage="marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}"
     mlVerShort=marklogicVersion.split("\\.")[0]
     latestTag="marklogic/marklogic-server-${dockerImageType}:latest-${mlVerShort}"
+    timeStamp = new Date().format('yyyyMMdd')
+    timestamptedTag = builtImage.replace('nightly', timeStamp)
     sh "make build docker_image_type=${dockerImageType} dockerTag=${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}"
     currentBuild.displayName = "#${BUILD_NUMBER}: ${marklogicVersion}-${env.dockerImageType} (${env.dockerVersion})"
 }
@@ -280,7 +281,7 @@ void structureTests() {
  * Runs Docker functional tests using the 'make docker-tests' target.
  */
 void dockerTests() {
-    sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} upgrade_image=${upgradeDockerImage} marklogicVersion=${marklogicVersion} build_branch=${env.BRANCH_NAME} dockerVersion=${env.dockerVersion} docker_image_type=${dockerImageType}"
+    sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} upgrade_image=${upgradeDockerImage} marklogicVersion=${marklogicVersion} build_branch=${env.BRANCH_NAME} dockerVersion=${env.dockerVersion} docker_image_type=${dockerImageType} DOCKER_TEST_LIST=\"${params.DOCKER_TEST_LIST}\""
 }
 
 /**
@@ -335,31 +336,32 @@ void publishToInternalRegistry() {
             docker tag ${builtImage} ${dockerRegistry}/${builtImage}
             docker tag ${builtImage} ${dockerRegistry}/${publishImage}
             docker tag ${builtImage} ${dockerRegistry}/${latestTag}
-            docker tag ${builtImage} ${dockerRegistry}/${builtImage}-${TIMESTAMP}
+            docker tag ${builtImage} ${dockerRegistry}/${timestamptedTag}
             docker push ${dockerRegistry}/${builtImage}
             docker push ${dockerRegistry}/${publishImage}
             docker push ${dockerRegistry}/${latestTag}
-            docker push ${dockerRegistry}/${builtImage}-${TIMESTAMP}
+            docker push ${dockerRegistry}/${timestamptedTag}
         """
 
     }
     // Publish to private ECR repository that is used by the performance team. (only ML11)
-    if ( params.marklogicVersion == "11" ) {
-        withCredentials( [[
-            $class: 'AmazonWebServicesCredentialsBinding',
-            credentialsId: "aws-engineering-ct-ecr",
-            accessKeyVariable: 'AWS_ACCESS_KEY_ID',
-            secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
-            ]]) {
-                sh """
-                    aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 713759029616.dkr.ecr.us-west-2.amazonaws.com
-                    docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion}
-                    docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}
-	                docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion}
-                    docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}
-                """
-            }
-    }
+    // (disabled since it's not needed)
+    // if ( params.marklogicVersion == "11" ) {
+    //     withCredentials( [[
+    //         $class: 'AmazonWebServicesCredentialsBinding',
+    //         credentialsId: "aws-engineering-ct-ecr",
+    //         accessKeyVariable: 'AWS_ACCESS_KEY_ID',
+    //         secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
+    //         ]]) {
+    //             sh """
+    //                 aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 713759029616.dkr.ecr.us-west-2.amazonaws.com
+    //                 docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion}
+    //                 docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}
+	//                 docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion}
+    //                 docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}
+    //             """
+    //         }
+    // }
 
     // Publish to private ACR Sandbox repository that is used by PDC. (only ML12)
     if ( params.marklogicVersion == "12" ) {
@@ -381,7 +383,7 @@ void publishToInternalRegistry() {
  * Runs asynchronously (wait: false).
  */
 void scanWithBlackDuck() {
-    build job: 'securityscans/Blackduck/KubeNinjas/docker', wait: false, parameters: [ string(name: 'branch', value: "${env.BRANCH_NAME}"), string(name: 'CONTAINER_IMAGES', value: "${dockerRegistry}/${publishImage}") ]
+    build job: 'securityscans/Blackduck/KubeNinjas/docker', wait: false, parameters: [ string(name: 'BRANCH', value: "${env.BRANCH_NAME}"), string(name: 'CONTAINER_IMAGES', value: "${dockerRegistry}/${publishImage}"), string(name: 'ML_VER', value: "${params.marklogicVersion}") ]
 }
 
 /**
@@ -436,32 +438,36 @@ pipeline {
         // Trigger nightly builds on the develop branch for every supported version of MarkLogic
         // and for every supported image type.
         // Include SCAP scan for rootless images
-        parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 02 * * * % marklogicVersion=10;dockerImageType=ubi
-                                                             00 02 * * * % marklogicVersion=10;dockerImageType=ubi-rootless;SCAP_SCAN=true
-                                                             00 02 * * * % marklogicVersion=11;dockerImageType=ubi
-                                                             30 02 * * * % marklogicVersion=11;dockerImageType=ubi-rootless;SCAP_SCAN=true
-                                                             30 02 * * * % marklogicVersion=12;dockerImageType=ubi
-                                                             30 02 * * * % marklogicVersion=12;dockerImageType=ubi-rootless;SCAP_SCAN=true
+        parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 04 * * * % marklogicVersion=10;dockerImageType=ubi
+                                                             00 04 * * * % marklogicVersion=10;dockerImageType=ubi-rootless;SCAP_SCAN=true
+                                                             00 03 * * * % marklogicVersion=11;dockerImageType=ubi
+                                                             00 03 * * * % marklogicVersion=11;dockerImageType=ubi-rootless;SCAP_SCAN=true
                                                              00 03 * * * % marklogicVersion=11;dockerImageType=ubi9
                                                              00 03 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless;SCAP_SCAN=true
-                                                             00 03 * * * % marklogicVersion=12;dockerImageType=ubi9
-                                                             30 03 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless;SCAP_SCAN=true''' : '')
+                                                             00 02 * * * % marklogicVersion=12;dockerImageType=ubi
+                                                             00 02 * * * % marklogicVersion=12;dockerImageType=ubi-rootless;SCAP_SCAN=true
+                                                             00 02 * * * % marklogicVersion=12;dockerImageType=ubi9
+                                                             00 02 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless;SCAP_SCAN=true
+                                                             00 05 * * 7 % marklogicVersion=10;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency
+                                                             30 05 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency
+                                                             00 06 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency''' : '')
     }
     environment {
         QA_LICENSE_KEY = credentials('QA_LICENSE_KEY')
     }
 
     parameters {
         string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true)
-        string(name: 'dockerVersion', defaultValue: '2.2.1', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true)
+        string(name: 'dockerVersion', defaultValue: '2.2.2', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true)
         choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\nubi9-rootless\nubi9', description: 'Platform type for Docker image. Will be made part of the docker image tag')
         string(name: 'upgradeDockerImage', defaultValue: '', description: 'Docker image for testing upgrades. Defaults to ubi image if left blank.\n Currently upgrading to ubi-rotless is not supported hence the test is skipped when ubi-rootless image is provided.', trim: true)
-        choice(name: 'marklogicVersion', choices: '11\n12\n10', description: 'MarkLogic Server Branch. used to pick appropriate rpm')
+        choice(name: 'marklogicVersion', choices: '12\n11\n10', description: 'MarkLogic Server Branch. used to pick appropriate rpm')
         string(name: 'ML_RPM', defaultValue: '', description: 'URL for RPM to be used for Image creation. \n If left blank nightly ML rpm will be used.\n Please provide Jenkins accessible path e.g. /project/engineering or /project/qa', trim: true)
         string(name: 'ML_CONVERTERS', defaultValue: '', description: 'URL for the converters RPM to be included in the image creation \n If left blank the nightly ML Converters Package will be used.', trim: true)
         booleanParam(name: 'PUBLISH_IMAGE', defaultValue: false, description: 'Publish image to internal registry')
         booleanParam(name: 'TEST_STRUCTURE', defaultValue: true, description: 'Run container structure tests')
-        booleanParam(name: 'DOCKER_TESTS', defaultValue: true, description: 'Run docker tests')
+    booleanParam(name: 'DOCKER_TESTS', defaultValue: true, description: 'Run docker tests')
+    string(name: 'DOCKER_TEST_LIST', defaultValue: '', description: 'Comma separated list of test names to run (e.g Test one, Test two). Leave empty to run all tests.', trim: true)
         booleanParam(name: 'SCAP_SCAN', defaultValue: false, description: 'Run Open SCAP scan on the image.')
     }
 

Makefile

@@ -71,7 +71,20 @@ docker-tests:
 	python3 -m venv python_env; \
 	source ./python_env/bin/activate; \
 	pip3 install -r requirements.txt; \
-	robot -x docker-tests.xml --outputdir test_results --randomize all --variable TEST_IMAGE:${current_image} --variable UPGRADE_TEST_IMAGE:${upgrade_image} --variable MARKLOGIC_VERSION:${marklogicVersion} --variable BUILD_BRANCH:${build_branch} --variable MARKLOGIC_DOCKER_VERSION:${dockerVersion} --variable IMAGE_TYPE:${docker_image_type} --maxerrorlines 9999 ./docker-tests.robot; \
+	TEST_ARGS=""; \
+	if [ -n "$(DOCKER_TEST_LIST)" ]; then \
+		echo "$(DOCKER_TEST_LIST)" | sed 's/,/\n/g' | while IFS= read -r ITEM; do \
+			ITEM=$$(echo "$$ITEM" | sed -e 's/^ *//' -e 's/ *$$//' -e 's/^"//' -e 's/"$$//'); \
+			[ -n "$$ITEM" ] && echo "--test \"$$ITEM\""; \
+		done | tr '\n' ' ' > /tmp/test_args; \
+		TEST_ARGS=$$(cat /tmp/test_args); \
+		rm -f /tmp/test_args; \
+		echo "Running selected tests: $(DOCKER_TEST_LIST)"; \
+	else \
+		TEST_ARGS="--exclude long_running"; \
+        echo "Running all tests except those tagged with 'long_running'"; \
+	fi; \
+	eval "robot --consolewidth 120 -x docker-tests.xml --outputdir test_results --randomize all --variable TEST_IMAGE:${current_image} --variable UPGRADE_TEST_IMAGE:${upgrade_image} --variable MARKLOGIC_VERSION:${marklogicVersion} --variable BUILD_BRANCH:${build_branch} --variable MARKLOGIC_DOCKER_VERSION:${dockerVersion} --variable IMAGE_TYPE:${docker_image_type} --maxerrorlines 9999 $$TEST_ARGS ./docker-tests.robot"; \
 	deactivate; \
 	rm -rf python_env
 

dockerFiles/marklogic-deps-ubi9:base

@@ -4,7 +4,7 @@
 #
 ###############################################################
 
-FROM registry.access.redhat.com/ubi9/ubi-minimal:9.6-1754584681
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.6-1758184547
 LABEL "com.marklogic.maintainer"="docker@marklogic.com"
 
 ###############################################################

dockerFiles/marklogic-deps-ubi:base

@@ -4,7 +4,7 @@
 #
 ###############################################################
 
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1753978370
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1756195339
 LABEL "com.marklogic.maintainer"="docker@marklogic.com"
 
 # MarkLogic version passed from build to enable conditional deps

src/scripts/start-marklogic-rootless.sh

@@ -100,8 +100,12 @@ else
     error "INSTALL_CONVERTERS must be true or false." exit
 fi
 
-N_RETRY=5
+# N_RETRY: Number of retries for failed operations (default: 5)  
+N_RETRY=${N_RETRY:-15}
+# RETRY_INTERVAL: Interval in seconds between retries (default: 10)
 RETRY_INTERVAL=10
+# CURL_TIMEOUT: Timeout in seconds for curl commands (default: 300)
+CURL_TIMEOUT=${CURL_TIMEOUT:-300}
 
 ################################################################
 # restart_check(hostname, baseline_timestamp)
@@ -117,11 +121,11 @@ RETRY_INTERVAL=10
 function restart_check {
     info "Waiting for MarkLogic to restart."
     local retry_count LAST_START
-    LAST_START=$(curl -s --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}" "http://$1:8001/admin/v1/timestamp")
+    LAST_START=$(curl -s -m "${CURL_TIMEOUT}" --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}" "http://$1:8001/admin/v1/timestamp")
     for ((retry_count = 0; retry_count < N_RETRY; retry_count = retry_count + 1)); do
         if [[ "$2" == "${LAST_START}" ]] || [[ -z "${LAST_START}" ]]; then
             sleep ${RETRY_INTERVAL}
-            LAST_START=$(curl -s --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}" "http://$1:8001/admin/v1/timestamp")
+            LAST_START=$(curl -s -m "${CURL_TIMEOUT}" --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}" "http://$1:8001/admin/v1/timestamp")
         else
             info "MarkLogic has restarted."
             return 0
@@ -178,7 +182,7 @@ function validate_cert {
     local cacertfile=$1
     local return_code
     local curl_output
-    curl_output=$(curl -s -S -L --cacert "${cacertfile}" --ssl "${ML_BOOTSTRAP_PROTOCOL}"://"${MARKLOGIC_BOOTSTRAP_HOST}":8001 --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}")
+    curl_output=$(curl -s -S -L -m "${CURL_TIMEOUT}" --cacert "${cacertfile}" --ssl "${ML_BOOTSTRAP_PROTOCOL}"://"${MARKLOGIC_BOOTSTRAP_HOST}":8001 --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}")
     return_code=$?
     if [[ $return_code != 0 ]]; then
         info "$curl_output"
@@ -193,6 +197,8 @@ function validate_cert {
 # Use RETRY_INTERVAL to tune the test length.
 # Validate that response code is the same as expected response
 # code or exit with an error.
+# For instance-admin endpoint, implement special retry logic with exponential backoff
+# due to network latency issues.
 #
 #   $1 :  Flag indicating if the script should exit if the given response code is not received ("true" to exit, "false" to return the response code")
 #   $2 :  The target url to test against
@@ -206,9 +212,26 @@ function curl_retry_validate {
     local expected_response_code=$1; shift
     local curl_options=("$@")
 
+    # Special case: instance-admin must only be invoked once (non-idempotent)
+    if [[ "${endpoint}" == *"/admin/v1/instance-admin"* ]]; then
+        response=$(curl -s -m "${CURL_TIMEOUT}" -w '%{http_code}' "${curl_options[@]}" "$endpoint")
+        response_code=$(tail -n1 <<< "$response")
+        response_content=$(sed '$ d' <<< "$response")
+
+        if [[ ${response_code} -eq ${expected_response_code} ]]; then
+            return "${response_code}"
+        fi
+
+        echo "${response_content}" > start-marklogic_curl_retry_validate.log
+        if [[ "${return_error}" == "false" ]] ; then
+            return "${response_code}"
+        fi
+        [[ -f "start-marklogic_curl_retry_validate.log" ]] && cat start-marklogic_curl_retry_validate.log
+        error "Expected response code ${expected_response_code}, got ${response_code} from ${endpoint}." exit
+    fi
+
     for ((retry_count = 0; retry_count < N_RETRY; retry_count = retry_count + 1)); do
-
-        response=$(curl -s -m 30 -w '%{http_code}' "${curl_options[@]}" "$endpoint")
+        response=$(curl -s -m "${CURL_TIMEOUT}" -w '%{http_code}' "${curl_options[@]}" "$endpoint")
         response_code=$(tail -n1 <<< "$response")
         response_content=$(sed '$ d' <<< "$response")
 
@@ -373,7 +396,7 @@ elif [[ "${MARKLOGIC_INIT}" == "true" ]]; then
     fi
 
     info "Initializing MarkLogic on ${HOSTNAME}"
-    TIMESTAMP=$(curl --anyauth -m 30 -s --retry 5 \
+    TIMESTAMP=$(curl --anyauth -m "${CURL_TIMEOUT}" -s --retry 5 \
         -i -X POST -H "Content-type:application/json" \
         -d "${LICENSE_PAYLOAD}" \
         http://"${HOSTNAME}":8001/admin/v1/init |