Merge pull request #28 from sql-machine-learning/last_day_pr

polish start.bash and make it more user-friendly

Author: lhw362950217

provision.bash

@@ -13,7 +13,7 @@ apt-get update
 
 echo "Installing Docker ..."
 # c.f. https://dockr.ly/3cExcay
-if ! which docker > /dev/null; then
+if which docker > /dev/null; then
     echo "Docker had been installed. Skip."
 else
     best_install_url=$(find_fastest_docker_url)

sqlflow

@@ -12,12 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-echo "Docker pull SQLFlow images ..."
+cat <<EOF
+This script is safe to re-run, feel free to retry when it exits abnormally.
+Especially when we are waiting for a pod in Kubernetes cluster, it may
+pull image from registry and take a lot of time to startup.
+
+EOF
+
+if [[ "$(whoami)" != "root" ]]; then
+    echo "Please change to root user and retry."
+    exit 1
+fi
+
+echo "Docker pull dependency images, you can comment this if already have them ..."
 # c.f. https://github.com/sql-machine-learning/sqlflow/blob/develop/.travis.yml
 docker pull sqlflow/sqlflow:jupyter
 docker pull sqlflow/sqlflow:mysql
 docker pull sqlflow/sqlflow:server
 docker pull sqlflow/sqlflow:step
+docker pull argoproj/argoexec:v2.7.7
+docker pull argoproj/argocli:v2.7.7
+docker pull argoproj/workflow-controller:v2.7.7
 echo "Done."
 
 # NOTE: According to https://stackoverflow.com/a/16619261/724872,
@@ -29,6 +44,26 @@ source $(dirname $0)/sqlflow/docker/dev/find_fastest_resources.sh
 # Find a way that we do not need to use 'set -e'
 set +e
 
+# Execute cmd until given output is present
+# or exit when timeout (50*3s)
+# "$1" is user message
+# "$2" is cmd
+# "$3" is expected output
+function wait_or_exit() {
+    echo -n "Waiting for $1 "
+    for i in {1..50}; do
+        $2 | grep -o -q "$3"
+        if [[ $? -eq 0 ]]; then
+            echo "Done"
+            return
+        fi
+        echo -n "."
+        sleep 3
+    done
+    echo "Fail"
+    exit
+}
+
 # Use a faster kube image and docker registry
 echo "Start minikube cluster ..."
 minikube_status=$(minikube status | grep "apiserver: Running")
@@ -49,68 +84,64 @@ else
     fi
 fi
 
-# Test if a Kubernetes resource is alive
+wait_or_exit "minikube" "minikube status" "apiserver: Running"
+
+# Test if a Kubernetes pod is ready
 # "$1" shoulde be namespace id e.g. argo
-# "$2" should be resource id e.g. pod/argo-server
-function is_resource_alive() {
-    local type=$(echo "$2" | cut -d / -f1)
-    local name=$(echo "$2" | cut -d / -f2)
-    if kubectl get -n "$1" "$2" | grep -q -o "$name" >/dev/null; then
-        # make sure relative pod is alive
-        if kubectl get pod -n "$1" | grep "$name" | grep "Running" >/dev/null; then
-            echo "yes"
-        else
-            echo "no"
-        fi
+# "$2" should be pod selector e.g. k8s-app=kubernetes-dashboard
+function is_pod_ready() {
+    pod=$(kubectl get pod -n "$1" -l "$2" -o name | tail -1)
+    if [[ -z "$pod" ]]; then
+        echo "no"
+        return
+    fi
+    ready=$(kubectl get -n "$1" "$pod" -o jsonpath='{.status.containerStatuses[0].ready}')
+    if [[ "$ready" == "true" ]]; then
+        echo "yes"
     else
         echo "no"
     fi
 }
 
 echo "Start argo ..."
-argo_server_alive=$(is_resource_alive "argo" "service/argo-server")
+argo_server_alive=$(is_pod_ready "argo" "app=argo-server")
 if [[ "$argo_server_alive" == "yes" ]]; then
     echo "Already in running."
 else
     $(dirname $0)/sqlflow/scripts/travis/start_argo.sh
 fi
+wait_or_exit "argo" "is_pod_ready argo app=argo-server" "yes"
 
-echo "Strat Kubernetes Dashboard..."
-dashboard_alive=$(is_resource_alive "kubernetes-dashboard" "service/kubernetes-dashboard")
+echo "Strat Kubernetes Dashboard ..."
+dashboard_alive=$(is_pod_ready "kubernetes-dashboard" "k8s-app=kubernetes-dashboard")
 if [[ "$dashboard_alive" == "yes" ]]; then
     echo "Already in running."
 else
-    nohup minikube dashboard &
+    nohup minikube dashboard >/dev/null 2>&1 &
 fi
+wait_or_exit "Kubernetes Dashboard" "is_pod_ready kubernetes-dashboard k8s-app=kubernetes-dashboard" "yes"
 
 echo "Strat SQLFlow ..."
-sqlflow_alive=$(is_resource_alive "default" "pod/sqlflow-server")
+sqlflow_alive=$(is_pod_ready "default" "app=sqlflow-server")
 if [[ "$sqlflow_alive" == "yes" ]]; then
     echo "Already in running."
 else
     kubectl apply -f sqlflow/doc/run/k8s/install-sqlflow.yaml
 fi
+wait_or_exit "SQLFlow" "is_pod_ready default app=sqlflow-server" "yes"
 
 # Kill port exposing if it already exist
 function stop_expose() {
-    ps -elf | grep "kubectl port-forward" | grep "$1" | grep "$2" | awk '{print $4}' | xargs kill  >/dev/null
+    ps -elf | grep "kubectl port-forward" | grep "$1" | grep "$2" | awk '{print $4}' | xargs kill  >/dev/null 2>&1
 }
 
 # Kubernetes port-forwarding
 # "$1" should be namespace
 # "$2" should be resource, e.g. service/argo-server
 # "$3" should be port mapping, e.g. 8000:80
 function expose() {
-    echo "Stop exposing $3..."
     stop_expose "$2" "$3"
-    echo "Detecting service and exposing port ..."
-    while [[ true ]]; do
-        local alive=$(is_resource_alive "$1" "$2")
-        if [[ "$alive" == "yes" ]]; then
-            break
-        fi
-        sleep 1
-    done
+    echo "Exposing port for $2 at $3 ..."
     nohup kubectl port-forward -n $1 --address='0.0.0.0' $2 $3 >>port-forward-log 2>&1 &
 }
 
@@ -119,18 +150,25 @@ expose kubernetes-dashboard service/kubernetes-dashboard 9000:80
 expose argo service/argo-server 9001:2746
 expose default pod/sqlflow-server 8888:8888
 expose default pod/sqlflow-server 3306:3306
+expose default pod/sqlflow-server 50051:50051
 
 jupyter_addr=$(kubectl logs pod/sqlflow-server notebook | grep -o -E "http://127.0.0.1[^?]+\?token=.*" | head -1)
+mysql_addr="mysql://root:root@tcp($(kubectl get -o jsonpath='{.status.podIP}' pod/sqlflow-server))/?maxAllowedPacket=0"
 
-cat <<EOF
+echo -e "
+\033[32m
 Congratulations, SQLFlow playground is up!
 
 Access Jupyter Notebook at: $jupyter_addr
 Access Kubernetes Dashboard at: http://localhost:9000
 Access Argo Dashboard at: http://localhost:9001
-Access SQLFlow with cli: refer to https://github.com/sql-machine-learning/sqlflow/blob/develop/doc/run/cli.md
+Access SQLFlow with cli: ./sqlflow --datasource="\"$mysql_addr\""
 
 Stop minikube with: minikube stop
-Stop vagrant with: vagrant halt
-EOF
+Stop vagrant vm with: vagrant halt
 
+[Dangerous]
+Destroy minikube with: minikube delete && rm -rf ~/.minikube
+Destroy vagrant vm with: vagrant destroy
+\033[0m
+"