SQL_SCRIPT_DIR="tools/migrations" DOCKER_SERVICE_NAME="repodiff-service" DOCKER_CONTAINER_REGISTRY="gcr.io" DOCKER_IMAGE_NAME="repodiff-image" DOCKER_TAG_NAME="latest" DOCKER_DOCKERFILE_DIR="." DOCKER_TARGET_PORT=$(shell cat config.json | jq ".port") DOCKER_CLUSTER_NAME="repodiff-default-cluster" DOCKER_DEPLOYMENT_NAME="repodiff-deployment" DOCKER_REPLICA_COUNT="1" DOCKER_CANONICAL_ID=$(DOCKER_CONTAINER_REGISTRY)/$(GOOGLE_PROJECT_ID)/$(DOCKER_IMAGE_NAME):$(DOCKER_TAG_NAME) PORT_HTTP="80" GCE_ZONE="us-west1-b" GCE_IMAGE_PROJECT="ubuntu-os-cloud" GCE_IMAGE_FAMILY="ubuntu-1604-lts" TMP_CREDENTIAL_FNAME=service_account_credentials.json # https://cloud.google.com/compute/docs/machine-types GCE_MACHINE_TYPE="n1-standard-64" PROJECT_NAME="auto-diff-android-branches" REMOTE_MACHINE_NAME=mithalop5 FIREWALL_NAME=public-http-access DISK_SIZE=500GB RUN_COMMAND_REMOTE=gcloud compute --project $(PROJECT_NAME) ssh --zone $(GCE_ZONE) "$(REMOTE_MACHINE_NAME)" --command SCP_TO_HOST=gcloud compute --project $(PROJECT_NAME) scp --zone $(GCE_ZONE) SERVICE_ACCOUNT_ID=repo-reader SERVICE_ACCOUNT=$(SERVICE_ACCOUNT_ID)@$(PROJECT_NAME).iam.gserviceaccount.com ifeq ($(ROLE),prod) GCP_DB_USER=$(GCP_DB_USER_PROD) GCP_DB_INSTANCE_CONNECTION_NAME=$(GCP_DB_INSTANCE_CONNECTION_NAME_PROD) GCP_DB_PASSWORD=$(GCP_DB_PASSWORD_PROD) GCP_DB_NAME=$(GCP_DB_NAME_PROD) GCP_DB_PROXY_PORT=$(GCP_DB_PROXY_PORT_PROD) else GCP_DB_USER=$(GCP_DB_USER_DEV) GCP_DB_INSTANCE_CONNECTION_NAME=$(GCP_DB_INSTANCE_CONNECTION_NAME_DEV) GCP_DB_PASSWORD=$(GCP_DB_PASSWORD_DEV) GCP_DB_NAME=$(GCP_DB_NAME_DEV) GCP_DB_PROXY_PORT=$(GCP_DB_PROXY_PORT_DEV) endif bootstrap: mkdir -p $(GOPATH)/src ./tools/setup_go_path_symlink.sh # include $GOPATH/bin as part of system path grep -q -F 'export PATH=$$PATH:$$GOPATH/bin' ~/.bashrc || echo 'export PATH=$$PATH:$$GOPATH/bin' >> ~/.bashrc source ~/.bashrc cd $(GOPATH)/src/repodiff; go get github.com/GoogleCloudPlatform/cloudsql-proxy/cmd/cloud_sql_proxy; \ go get github.com/golang/dep/cmd/dep; \ dep ensure; \ go build run: go build; ROLE="dev" ./repodiff run_prod: go build; ROLE="prod" ./repodiff reformat: go fmt . db_shell: mysql -u $(GCP_DB_USER) -h 127.0.0.1 -P $(GCP_DB_PROXY_PORT) -p$(GCP_DB_PASSWORD) $(GCP_DB_NAME) ${EXTRA} db_proxy: $(GOPATH)/bin/cloud_sql_proxy -instances=$(GCP_DB_INSTANCE_CONNECTION_NAME_DEV)=tcp:$(GCP_DB_PROXY_PORT_DEV) & $(GOPATH)/bin/cloud_sql_proxy -instances=$(GCP_DB_INSTANCE_CONNECTION_NAME_PROD)=tcp:$(GCP_DB_PROXY_PORT_PROD) & db_proxy_ignore_err: make db_proxy; true start_sql: make db_proxy_ignore_err & make db_shell < $(SQL_SCRIPT_DIR)/required_meta.sql db_upgrade: make start_sql python tools/upgrade_db.py upgrade $(SQL_SCRIPT_DIR) db_downgrade: make start_sql python tools/upgrade_db.py downgrade $(SQL_SCRIPT_DIR) example: make db_shell EXTRA="-e 'DESCRIBE project_differential;'" test: rm -rf build/ ROLE="dev" go test ./... -v | grep -v PASS | grep -v RUN sql_script: python tools/create_sql_script.py ssh: gcloud compute --project $(PROJECT_NAME) ssh --zone $(GCE_ZONE) $(REMOTE_MACHINE_NAME) deploy: gcloud config set project $(PROJECT_NAME) @echo "Starting docker image build" make build_container_image @echo "Creating machine if it doesn't already exist" gcloud compute instances create $(REMOTE_MACHINE_NAME) \ --machine-type $(GCE_MACHINE_TYPE) \ --boot-disk-size $(DISK_SIZE) \ --scopes https://www.googleapis.com/auth/source.read_only,https://www.googleapis.com/auth/compute \ --zone $(GCE_ZONE) \ --local-ssd interface=nvme \ --metadata-from-file startup-script=remote_scripts/gce_startup.sh \ --metadata AUTHOR=$(USER),SERVICE_ACCOUNT=$(SERVICE_ACCOUNT),GOOGLE_PROJECT_ID=$(GOOGLE_PROJECT_ID) \ --image-project $(GCE_IMAGE_PROJECT) \ --image-family $(GCE_IMAGE_FAMILY) \ --min-cpu-platform skylake \ --service-account $(SERVICE_ACCOUNT) \ 2>/dev/null || true @echo "Hackily waiting a bit for instance to start up" # TODO(slobdell) need to add a mechanism to block until startup script has completed @sleep 60 ./tools/clear_service_account_keys.py $(SERVICE_ACCOUNT) 2>/dev/null || true gcloud iam service-accounts keys create $(TMP_CREDENTIAL_FNAME) --iam-account $(SERVICE_ACCOUNT) $(RUN_COMMAND_REMOTE) 'mkdir -p /tmp/scripts' $(SCP_TO_HOST) remote_scripts/* "$(REMOTE_MACHINE_NAME)":/tmp/scripts/ $(SCP_TO_HOST) $(TMP_CREDENTIAL_FNAME) "$(REMOTE_MACHINE_NAME)":/tmp/ rm $(TMP_CREDENTIAL_FNAME) @echo "Stopping all existing docker images" $(RUN_COMMAND_REMOTE) 'docker stop $$(docker ps -a -q)' 2>/dev/null || true docker image save $(DOCKER_CANONICAL_ID) -o transferrable_docker_image.tar \ && $(SCP_TO_HOST) transferrable_docker_image.tar "$(REMOTE_MACHINE_NAME)":~/transferred_docker_image.tar \ && $(RUN_COMMAND_REMOTE) 'docker load -i transferred_docker_image.tar' \ && $(RUN_COMMAND_REMOTE) 'docker run -d --rm -p $(DOCKER_TARGET_PORT):$(DOCKER_TARGET_PORT) $(DOCKER_CANONICAL_ID)' \ && gcloud compute firewall-rules create $(FIREWALL_NAME) --allow tcp:$(DOCKER_TARGET_PORT) 2>/dev/null || true \ && gcloud compute firewall-rules update $(FIREWALL_NAME) --allow tcp:$(DOCKER_TARGET_PORT) --source-tags="$(REMOTE_MACHINE_NAME)" --source-ranges=0.0.0.0/0 --description="Allow requests over HTTP" @make output_instance_url --no-print-directory @rm transferrable_docker_image.tar output_instance_url: @echo "Monitor progress at http://"$(shell (gcloud compute instances list | grep $(REMOTE_MACHINE_NAME) | awk -F ' ' '{print $$5}')):$(DOCKER_TARGET_PORT)/health destroy: gcloud compute instances delete $(REMOTE_MACHINE_NAME) --zone $(GCE_ZONE) --quiet ############## DOCKER DEPLOYMENT build_container_image: mkdir -p ./build # move contents of entire directory into build find . -type f -not -path 'build' -exec cp --parents '{}' 'build/' \; # copy python scripts repo uses, TODO re-structure codebase so the dependencies align with file structure cp ../../*.{txt,py} build/ # Application credentials must be downloaded from https://pantheon.corp.google.com; set this environment variable to the path of the downloaded file cp $(GOOGLE_APPLICATION_CREDENTIALS) build/ # copy local environment variables into Dockerfile cat Dockerfile | envsubst > build/Dockerfile # copy permissions from local gitcookies into Dockerfile (container will sync repo as you) cp static/docker_git_config ./build/.gitconfig cp ~/.gitcookies ./build/.gitcookies docker build -t $(DOCKER_CANONICAL_ID) ./build rm -rf ./build docker_shell: docker run -it --rm $(DOCKER_CANONICAL_ID) bash upload_container_image: gcloud config set project $(GOOGLE_PROJECT_ID) gcloud docker -- push $(DOCKER_CANONICAL_ID) run_container_local: docker run --rm -p $(DOCKER_TARGET_PORT):$(DOCKER_TARGET_PORT) $(DOCKER_CANONICAL_ID) create_container_cluster: gcloud container clusters create $(DOCKER_CLUSTER_NAME) \ --num-nodes=3 \ --machine-type $(GCE_MACHINE_TYPE) \ --zone $(GCE_ZONE) create_container_cluster_verify: gcloud compute instances list expose_to_internets: kubectl expose deployment $(DOCKER_DEPLOYMENT_NAME) --type=LoadBalancer --port $(PORT_HTTP) --target-port $(DOCKER_TARGET_PORT) expose_to_internets_verify: kubectl get service scale: kubectl scale deployment $(DOCKER_DEPLOYMENT_NAME) --replicas=$(DOCKER_REPLICA_COUNT) cleanup: kubectl delete service ############## END DOCKER DEPLOYMENT