This is a simble GKE cluster provisioning module for testing purposes. For a more customizable module, use the official hashicorp one ( https://github.com/terraform-google-modules/terraform-google-kubernetes-engine )
gcloud auth application-default login
export GCLOUD_PROJECT=operations-237013 GCLOUD_REGION=europe-west1
cd terraform
terraform init
terraform plan
terraform apply
gcloud config set project $GCLOUD_PROJECT
gcloud container clusters get-credentials $(terraform output kubernetes_cluster_name | sed 's/"//g') --region $(terraform output kubernetes_cluster_region | sed 's/"//g')
cat << EOF > ../k8s/gcs/config-connector.yml
---
apiVersion: core.cnrm.cloud.google.com/v1beta1
kind: ConfigConnector
metadata:
name: configconnector.core.cnrm.cloud.google.com
spec:
mode: cluster
googleServiceAccount: "$(terraform output kubernetes_cluster_gke_sa_email | sed 's/"//g')"
EOF
kubectl apply -f ../k8s/gcs/config-connector.yml
cat << EOF > ../k8s/gcs/pubsub.yml
---
apiVersion: pubsub.cnrm.cloud.google.com/v1beta1
kind: PubSubTopic
metadata:
annotations:
cnrm.cloud.google.com/project-id: $( terraform output project_id )
labels:
environment: test
name: pubsubtopic-sample
EOF
kubectl apply -f ../k8s/gcs/pubsub.yml
Install chart ( https://github.com/GoogleCloudPlatform/spark-on-k8s-operator )
helm repo add spark-operator https://googlecloudplatform.github.io/spark-on-k8s-operator
helm install my-release spark-operator/spark-operator
kubectl apply -f k8s/spark-serviceaccount.yml
k apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/spark-on-k8s-operator/master/examples/spark-pi-custom-resource.yaml