Add Branch and RC Awareness to Version Lint & Fix Semver Regex (feast-dev#998)

mrzzy · web-flow · commit 7fa3b077fc61 · 2020-09-11T15:54:00.000+08:00
* Update version lint script to detect maven, docker image and stable version .

* Allow users to lint versions against their merge branch of choice by branch TARGET_MERGE_BRANCH

* Update development image tag version from 'dev' to 'develop' to make it easier to lint version.

* Update lint version script to only echo file contents when version is wrong.

* Update lint script to flag all failures instead of just one.

* Remove file contents output from lint version script.

* Update lint version script to make output more concise and readable.

* Use develop images instead of stable images in master deployments.

* Update outdated documentation in serving readme.

* Fix outdated version in datatypes java README.md and add version lint check.

* Add version lint checks to documentation.

* Fix semver regex in github actions workflow not matching suffixless version tags.

* Add version lint check for latest stable version in changelog.

* Update lint-version script use tool agnostic variable names.

* Fix typo.
diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml
@@ -31,21 +31,21 @@ jobs:
         run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
       - name: Push image
         run: make push-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
-      - name: Push image to feast dev
+      - name: Push development Docker image
         run: |
           if [ ${GITHUB_REF#refs/*/} == "master" ]; then
-            docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:dev
-            docker push gcr.io/kf-feast/feast-${{ matrix.component }}:dev
+            docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:develop
+            docker push gcr.io/kf-feast/feast-${{ matrix.component }}:develop
           fi
       - name: Get version
         run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/}
-      - name: Push versioned release
+      - name: Push versioned Docker image
         run: |
           # Build and push semver tagged commits
           # Regular expression should match MAJOR.MINOR.PATCH[-PRERELEASE[.IDENTIFIER]]
           # eg. v0.7.1 v0.7.2-alpha v0.7.2-rc.1
-          rx='^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))$ '
-          if echo "${RELEASE_VERSION}" | grep -P "$rx" &>/dev/null ; then
+          SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
+          if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
             VERSION_WITHOUT_PREFIX=${RELEASE_VERSION:1}
 
             docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
diff --git a/datatypes/java/README.md b/datatypes/java/README.md
@@ -16,7 +16,7 @@ Dependency Coordinates
 <dependency>
   <groupId>dev.feast</groupId>
   <artifactId>datatypes-java</artifactId>
-  <version>0.4.0-SNAPSHOT</version>
+  <version>0.7-SNAPSHOT</version>
 </dependency>
 ```
 
diff --git a/docs/contributing/development-guide.md b/docs/contributing/development-guide.md
@@ -210,7 +210,7 @@ grpc_cli call localhost:6566 GetFeastServingInfo ''
 
 ```text
 connecting to localhost:6566
-version: "0.6.2-SNAPSHOT"
+version: "0.7-SNAPSHOT"
 type: FEAST_SERVING_TYPE_ONLINE
 
 Rpc succeeded with OK status
diff --git a/infra/charts/feast/charts/feast-core/README.md b/infra/charts/feast/charts/feast-core/README.md
@@ -23,7 +23,7 @@ Current chart version is `0.7-SNAPSHOT`
 | gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account |
 | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
 | image.repository | string | `"gcr.io/kf-feast/feast-core"` | Docker image repository |
-| image.tag | string | `"0.6.2"` | Image tag |
+| image.tag | string | `"develop"` | Image tag |
 | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress |
 | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth |
 | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use |
diff --git a/infra/charts/feast/charts/feast-core/values.yaml b/infra/charts/feast/charts/feast-core/values.yaml
@@ -5,7 +5,7 @@ image:
   # image.repository -- Docker image repository
   repository: gcr.io/kf-feast/feast-core
   # image.tag -- Image tag
-  tag: 0.6.2
+  tag: develop
   # image.pullPolicy -- Image pull policy
   pullPolicy: IfNotPresent
 
diff --git a/infra/charts/feast/charts/feast-jobcontroller/README.md b/infra/charts/feast/charts/feast-jobcontroller/README.md
@@ -23,7 +23,7 @@ Current chart version is `0.7-SNAPSHOT`
 | gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account |
 | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
 | image.repository | string | `"gcr.io/kf-feast/feast-jobcontroller"` | Docker image repository |
-| image.tag | string | `"0.6.2"` | Image tag |
+| image.tag | string | `"develop"` | Image tag |
 | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress |
 | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth |
 | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use |
diff --git a/infra/charts/feast/charts/feast-jobcontroller/values.yaml b/infra/charts/feast/charts/feast-jobcontroller/values.yaml
@@ -5,7 +5,7 @@ image:
   # image.repository -- Docker image repository
   repository: gcr.io/kf-feast/feast-jobcontroller
   # image.tag -- Image tag
-  tag: 0.6.2
+  tag: develop
   # image.pullPolicy -- Image pull policy
   pullPolicy: IfNotPresent
 
diff --git a/infra/charts/feast/charts/feast-jupyter/README.md b/infra/charts/feast/charts/feast-jupyter/README.md
@@ -17,5 +17,5 @@ Current chart version is `0.7-SNAPSHOT`
 | gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account |
 | image.pullPolicy | string | `"Always"` | Image pull policy |
 | image.repository | string | `"gcr.io/kf-feast/feast-jupyter"` | Docker image repository |
-| image.tag | string | `"0.6.2"` | Image tag |
+| image.tag | string | `"develop"` | Image tag |
 | replicaCount | int | `1` | Number of pods that will be created |
diff --git a/infra/charts/feast/charts/feast-jupyter/values.yaml b/infra/charts/feast/charts/feast-jupyter/values.yaml
@@ -5,7 +5,7 @@ image:
   # image.repository -- Docker image repository
   repository: gcr.io/kf-feast/feast-jupyter
   # image.tag -- Image tag
-  tag: 0.6.2
+  tag: develop
   # image.pullPolicy -- Image pull policy
   pullPolicy: Always
 
@@ -16,4 +16,4 @@ gcpServiceAccount:
     # gcpServiceAccount.existingSecret.name -- Name of the existing secret containing the service account
     name: feast-gcp-service-account
     # gcpServiceAccount.existingSecret.key -- Key in the secret data (file name of the service account)
-    key: credentials.json
+    key: credentials.json
diff --git a/infra/charts/feast/charts/feast-serving/README.md b/infra/charts/feast/charts/feast-serving/README.md
@@ -23,7 +23,7 @@ Current chart version is `0.7-SNAPSHOT`
 | gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account |
 | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
 | image.repository | string | `"gcr.io/kf-feast/feast-serving"` | Docker image repository |
-| image.tag | string | `"0.6.2"` | Image tag |
+| image.tag | string | `"develop"` | Image tag |
 | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress |
 | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth |
 | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use |
diff --git a/infra/charts/feast/charts/feast-serving/values.yaml b/infra/charts/feast/charts/feast-serving/values.yaml
@@ -5,7 +5,7 @@ image:
   # image.repository -- Docker image repository
   repository: gcr.io/kf-feast/feast-serving
   # image.tag -- Image tag
-  tag: 0.6.2
+  tag: develop
   # image.pullPolicy -- Image pull policy
   pullPolicy: IfNotPresent
 
diff --git a/infra/charts/feast/requirements.lock b/infra/charts/feast/requirements.lock
@@ -1,16 +1,16 @@
 dependencies:
 - name: feast-core
   repository: ""
-  version: 0.6.2
+  version: 0.7-SNAPSHOT
 - name: feast-serving
   repository: ""
-  version: 0.6.2
+  version: 0.7-SNAPSHOT
 - name: feast-serving
   repository: ""
-  version: 0.6.2
+  version: 0.7-SNAPSHOT
 - name: feast-jupyter
   repository: ""
-  version: 0.6.2
+  version: 0.7-SNAPSHOT
 - name: postgresql
   repository: https://kubernetes-charts.storage.googleapis.com/
   version: 8.6.1
diff --git a/infra/docker-compose/.env.sample b/infra/docker-compose/.env.sample
@@ -1,8 +1,8 @@
 COMPOSE_PROJECT_NAME=feast
-FEAST_VERSION=0.6.2
+FEAST_VERSION=develop
 GCP_SERVICE_ACCOUNT=./gcp-service-accounts/key.json
 FEAST_CORE_CONFIG=./core/core.yml
 FEAST_JOB_CONTROLLER_CONFIG=./jobcontroller/jobcontroller.yml
 FEAST_HISTORICAL_SERVING_CONFIG=./serving/historical-serving.yml
 FEAST_HISTORICAL_SERVING_ENABLED=false
-FEAST_ONLINE_SERVING_CONFIG=./serving/online-serving.yml
+FEAST_ONLINE_SERVING_CONFIG=./serving/online-serving.yml
diff --git a/infra/scripts/validate-version-consistency.sh b/infra/scripts/validate-version-consistency.sh
@@ -1,80 +1,131 @@
 #!/usr/bin/env bash
 
 # This script will scan through a list of files to validate that all versions are consistent with
-# - Master version (could be snapshot)
-# - Highest stable commit (latest tag)
+# - Master version: version set in maven (could be snapshot)
+# - Release version: 'dev' on master, Lastest tag on release branches.
+# - Stable Version: Highest stable tag. release candidates not included.
+# Usage: ./validate-version-consistency.sh 
+# Optionaly set TARGET_MERGE_BRANCH var to the target merge branch to lint 
+#   versions against the given merge branch.
 set -e
 
+BRANCH_NAME=${TARGET_MERGE_BRANCH-$(git rev-parse --abbrev-ref HEAD)}
+# Matches (ie vMAJOR.MINOR-branch) release branch names
+RELEASE_BRANCH_REGEX="^v[0-9]+\.[0-9]+-branch$"
+
 # Determine the current Feast version from Maven (pom.xml)
 export FEAST_MASTER_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
 [[ -z "$FEAST_MASTER_VERSION" ]] && {
   echo "$FEAST_MASTER_VERSION is missing, please check pom.xml and maven"
   exit 1
 }
+echo "Linting Master Version: $FEAST_MASTER_VERSION"
 
-# Determine the highest released version from Git history
-git fetch --prune --unshallow --tags || true
-FEAST_RELEASE_VERSION_WITH_V=$(git tag -l --sort -version:refname | head -n 1)
-echo $FEAST_RELEASE_VERSION_WITH_V
-
-export FEAST_RELEASE_VERSION=${FEAST_RELEASE_VERSION_WITH_V#"v"}
-echo $FEAST_RELEASE_VERSION
-
+# Determine Last release tag relative to current branch
+if [ $BRANCH_NAME = "master" ]
+then
+    # Use development version
+    FEAST_RELEASE_VERSION="develop"
+elif echo "$BRANCH_NAME" | grep -P $RELEASE_BRANCH_REGEX &>/dev/null
+then
+    # Use last release tag tagged on the release branch
+    LAST_MERGED_TAG=$(git tag -l --sort -version:refname --merged | head -n 1)
+    FEAST_MASTER_VERSION=${LAST_MERGED_TAG#"v"}
+else
+    # Do not enforce version linting as we don't know if the target merge branch FEAST_RELEASE_VERSION="_ANY"
+    FEAST_RELEASE_VERSION="_ANY"
+    echo "WARNING: Skipping docker version lint"
+fi
 [[ -z "$FEAST_RELEASE_VERSION" ]] && {
   echo "FEAST_RELEASE_VERSION is missing"
   exit 1
 }
+export FEAST_RELEASE_VERSION
+echo "Linting Release Version: $FEAST_RELEASE_VERSION"
+
+# Determine highest stable version (no release candidates) relative to current branch.
+# Regular expression for matching stable tags in the format vMAJOR.MINOR.PATCH
+STABLE_TAG_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$"
+if [ $BRANCH_NAME = "master" ]
+then
+    # Use last stable tag repo wide
+    LAST_STABLE_TAG=$(git tag --sort -version:refname | grep -P "$STABLE_TAG_REGEX" | head -n 1)
+    FEAST_STABLE_VERSION=${LAST_STABLE_TAG#"v"}
+elif echo "$BRANCH_NAME" | grep -P $RELEASE_BRANCH_REGEX &>/dev/null
+then
+    # Use last stable tag tagged on the release branch
+    LAST_STABLE_MERGE_TAG=$(git tag --sort -version:refname --merged | grep -P "$STABLE_TAG_REGEX" | head -n 1)
+    FEAST_STABLE_VERSION=${LAST_STABLE_MERGE_TAG#"v"}
+else
+    # Do not enforce version linting as we don't know if the target merge branch
+    FEAST_STABLE_VERSION="_ANY"
+    echo "WARNING: Skipping stable version lint"
+fi
+[[ -z "$FEAST_STABLE_VERSION" ]] && {
+  echo "FEAST_STABLE_VERSION is missing"
+  exit 1
+}
+export FEAST_STABLE_VERSION
+echo "Linting Stable Version: $FEAST_STABLE_VERSION"
 
 # List of files to validate with master version (from pom.xml)
 # Structure is a comma separated list of structure
 # <File to validate>, <Amount of occurrences of specific version to look for>, <version to look for>
-#
 
 declare -a files_to_validate_version=(
   "infra/charts/feast/Chart.yaml,1,${FEAST_MASTER_VERSION}"
   "infra/charts/feast/charts/feast-core/Chart.yaml,1,${FEAST_MASTER_VERSION}"
   "infra/charts/feast/charts/feast-core/values.yaml,1,${FEAST_RELEASE_VERSION}"
+  "infra/charts/feast/charts/feast-core/README.md,1,${FEAST_MASTER_VERSION}"
   "infra/charts/feast/charts/feast-core/README.md,1,${FEAST_RELEASE_VERSION}"
   "infra/charts/feast/charts/feast-serving/Chart.yaml,1,${FEAST_MASTER_VERSION}"
   "infra/charts/feast/charts/feast-jupyter/values.yaml,1,${FEAST_RELEASE_VERSION}"
   "infra/charts/feast/charts/feast-jupyter/README.md,1,${FEAST_RELEASE_VERSION}"
+  "infra/charts/feast/charts/feast-jupyter/README.md,1,${FEAST_MASTER_VERSION}"
   "infra/charts/feast/charts/feast-jupyter/Chart.yaml,1,${FEAST_MASTER_VERSION}"
   "infra/charts/feast/charts/feast-serving/values.yaml,1,${FEAST_RELEASE_VERSION}"
   "infra/charts/feast/charts/feast-serving/README.md,1,${FEAST_RELEASE_VERSION}"
+  "infra/charts/feast/charts/feast-serving/README.md,1,${FEAST_MASTER_VERSION}"
   "infra/charts/feast/charts/feast-jobcontroller/Chart.yaml,1,${FEAST_MASTER_VERSION}"
   "infra/charts/feast/charts/feast-jobcontroller/values.yaml,1,${FEAST_RELEASE_VERSION}"
+  "infra/charts/feast/charts/feast-jobcontroller/README.md,1,${FEAST_MASTER_VERSION}"
   "infra/charts/feast/charts/feast-jobcontroller/README.md,1,${FEAST_RELEASE_VERSION}"
   "infra/charts/feast/requirements.yaml,4,${FEAST_MASTER_VERSION}"
-  "infra/charts/feast/requirements.lock,4,${FEAST_RELEASE_VERSION}"
+  "infra/charts/feast/requirements.lock,4,${FEAST_MASTER_VERSION}"
   "infra/docker-compose/.env.sample,1,${FEAST_RELEASE_VERSION}"
+  "datatypes/java/README.md,1,${FEAST_MASTER_VERSION}"
+  "docs/contributing/development-guide.md,4,${FEAST_MASTER_VERSION}"
+  "docs/administration/audit-logging.md,1,${FEAST_STABLE_VERSION}"
+  "docs/getting-started/deploying-feast/docker-compose.md,1,${FEAST_STABLE_VERSION}"
+  "docs/getting-started/deploying-feast/kubernetes.md,1,${FEAST_STABLE_VERSION}"
+  "README.md,1,${FEAST_STABLE_VERSION}"
+  "CHANGELOG.md,2,${FEAST_STABLE_VERSION}"
 )
 
 echo
 echo "Testing list of files to ensure they have the correct version"
 echo
 
+
+IS_LINT_SUCCESS=true
 for i in "${files_to_validate_version[@]}"; do
   IFS=',' read -r FILE_PATH EXPECTED_OCCURRENCES VERSION <<<"${i}"
-  echo
-  echo
-  echo "Testing whether versions are correctly set within file: $FILE_PATH"
-  echo
-  echo "File contents:"
-  echo "========================================================="
-  cat "$FILE_PATH"
-  echo
+  # Disable version lint if '_ANY' specified as version.
+  if [ "$VERSION" = "_ANY" ]
+  then
+      continue
+  fi
+  
   echo "========================================================="
-  ACTUAL_OCCURRENCES=$(grep -c "$VERSION" "$FILE_PATH" || true)
+  echo "Testing whether versions are correctly set within file: $FILE_PATH"
+  ACTUAL_OCCURRENCES=$(grep -c -P "\bv?$VERSION\b" "$FILE_PATH" || true)
 
   if [ "${ACTUAL_OCCURRENCES}" -eq "${EXPECTED_OCCURRENCES}" ]; then
-    echo "SUCCESS"
-    echo
-    echo "Expecting $EXPECTED_OCCURRENCES occurrences of $VERSION in $FILE_PATH, and found $ACTUAL_OCCURRENCES"
+    echo "OK: Expecting $EXPECTED_OCCURRENCES occurrences of '$VERSION' in $FILE_PATH, and found $ACTUAL_OCCURRENCES"
   else
-    echo "FAILURE"
-    echo
-    echo "Expecting $EXPECTED_OCCURRENCES occurrences of $VERSION in $FILE_PATH, but found $ACTUAL_OCCURRENCES"
-    exit 1
+    echo "FAIL: Expecting $EXPECTED_OCCURRENCES occurrences of '$VERSION' in $FILE_PATH, but found $ACTUAL_OCCURRENCES"
+    IS_LINT_SUCCESS=false
   fi
-  echo "========================================================="
 done
+
+if $IS_LINT_SUCCESS; then exit 0; else exit 1; fi
diff --git a/serving/README.md b/serving/README.md
@@ -11,8 +11,8 @@ From the Feast project root directory, run the following Maven command to start
 ```bash
 # Assumptions: 
 # - Local Feast Core is running on localhost:6565
+# Uses configuration from serving/src/main/resources/application.yml
 mvn -pl serving spring-boot:run -Dspring-boot.run.arguments=\
---feast.store.config-path=./sample_redis_config.yml,\
 --feast.core-host=localhost,\
 --feast.core-port=6565
 ```
