diff --git a/Makefile b/Makefile index ca03ce30..252adbba 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,7 @@ $(error IMAGE_NAME is not set; invoke make with something like IMAGE_NAME=google endif .PHONY: all -all: - cloud-test +all: cloud-test ## Files that must be refreshed every build diff --git a/README.md b/README.md index ab7fc243..4a35d047 100644 --- a/README.md +++ b/README.md @@ -77,10 +77,11 @@ shell. You need to be authenticated to a Google Cloud Project to invoke the Google Container Builder service, and also to run the system tests. ```shell -$ export GCLOUD_PROJECT=YOUR-PROJECT-NAME +$ export GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME $ DOCKER_NAMESPACE=gcr.io/${GCLOUD_PROJECT} $ CANDIDATE_NAME=`date +%Y-%m-%d_%H_%M` $ export IMAGE_NAME=${DOCKER_NAMESPACE}/python:${CANDIDATE_NAME} +$ gcloud config set project ${GOOGLE_CLOUD_PROJECT} ``` To rebuild the image using the Google Container Builder service, do the @@ -98,6 +99,47 @@ $ make local-build $ make local-test ``` +To open an interactive shell session to this image after building it, do the following: + +``` shell +docker run --it --entrypoint /bin/bash ${IMAGE_NAME} +``` + +## Running the system tests + +To run the system tests, you need a Google Cloud Project with a service account. +From the [Google Cloud Console](https://console.cloud.google.com/), either +create a new project or switch to an existing one. Next, +[create a service account]( +https://cloud.google.com/iam/docs/creating-managing-service-accounts) that will +be used to run the system tests. Once you have a service account, +[create and download a service account key](https://cloud.google.com/iam/docs/managing-service-account-keys). + +In the +[IAM & Admin](https://console.cloud.google.com/permissions/projectpermissions) +section, grant the `Owner` role to the service account you created above. Also +grant the `Editor` role to the `cloud-logs@google.com` service account. + + +Then, follow the +[system test setup instructions](https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/CONTRIBUTING.rst#running-system-tests). It +describes various steps, including running some scripts to populate and/or +delete datastore example data and indexes (populate_datastore.py, +clear_datastore.py, and `gcloud preview datastore create-indexes +system_tests/data/index.yaml`). + +From the cloud console, you will need to enable the following APIs for your project: + +- Bigquery API +- Cloud Bigtable Admin API +- Google Cloud Natural Language API +- Google Cloud Pub/Sub API +- Google Cloud Storage JSON API +- Google Cloud Vision API +- Google Translate API +- Stackdriver Logging API +- Stackdriver Monitoring API + ## Contributing changes * See [CONTRIBUTING.md](CONTRIBUTING.md) diff --git a/cloudbuild.yaml.in b/cloudbuild.yaml.in index 776bc6a2..b8ed6baa 100644 --- a/cloudbuild.yaml.in +++ b/cloudbuild.yaml.in @@ -1,4 +1,4 @@ -timeout: 3600s +timeout: 7200s steps: - name: gcr.io/cloud-builders/docker args: ['build', '--tag=interpreter', '--no-cache', 'python-interpreter-builder'] diff --git a/jenkins_build.sh b/jenkins_build.sh index b081255f..f28196a7 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -11,8 +11,31 @@ if [ -z "${DOCKER_NAMESPACE+set}" ] ; then echo "Error: DOCKER_NAMESPACE is not set; invoke with something like DOCKER_NAMESPACE=gcr.io/YOUR-PROJECT-NAME" >&2 exit 1 fi -IMAGE_NAME="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:${CANDIDATE_NAME}" +export IMAGE_NAME="${DOCKER_NAMESPACE}/${RUNTIME_NAME}:${CANDIDATE_NAME}" + +if [ -z "${GOOGLE_CLOUD_PROJECT+set}" ] ; then + echo "Error: GOOGLE_CLOUD_PROJECT is not set; invoke with something like GOOGLE_CLOUD_PROJECT=YOUR-PROJECT-NAME" >&2 + exit 1 +fi -export IMAGE_NAME export FORCE_REBUILD -make cloud-test + +echo "==================================================================" +echo "Building image ${IMAGE_NAME} and pushing to ${DOCKER_NAMESPACE} using ${GOOGLE_CLOUD_PROJECT}" +echo "==================================================================" + +make cloud-build + +# We explicitly pull the image using 'gcloud', instead of letting +# Docker do it, so that we have the right credentials. +echo "==================================================================" +gcloud info +echo gcloud docker -- pull "${IMAGE_NAME}" +gcloud docker -- pull "${IMAGE_NAME}" +echo "==================================================================" + +# Note that system test failures might be caused environment factors +# outside our control. Also, the images will be pushed to GCR by the +# previous build step regardless of system test failures. +make integration-tests || \ + echo "ERROR: System test failure, please examine the logs" diff --git a/system_tests/Dockerfile.in b/system_tests/Dockerfile.in index 6cbf6db3..97f8e004 100644 --- a/system_tests/Dockerfile.in +++ b/system_tests/Dockerfile.in @@ -1,22 +1,15 @@ FROM ${IMAGE_NAME} -# Inject secrets -ADD data/ data/ -ENV GOOGLE_APPLICATION_CREDENTIALS /app/data/cloud-python-runtime-qa-credentials.json -ENV GCLOUD_PROJECT cloud-python-runtime-qa +# Secrets injected at runtime +ENV GOOGLE_APPLICATION_CREDENTIALS=/app/credentials/credentials.json +ENV GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} # Get the source. -RUN git clone https://github.com/GoogleCloudPlatform/google-cloud-python.git +RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/google-cloud-python.git WORKDIR google-cloud-python # Install tox for running the system tests RUN pip install --upgrade tox -# List of system tests to run. Ideally would be all of them. But -# some of them get permission or other errors that are probably -# related to incomplete test setup. -ENV MODULES="datastore storage speech bigquery pubsub language translate monitoring bigtable" - -# Run Python 2.7, 3.4 system tests -RUN GOOGLE_CLOUD_TESTS_API_KEY=$(cat /app/data/cloud-python-runtime-qa-api-key.txt) \ - tox -e system-tests,system-tests3 -- ${MODULES} +# Run Python 2.7, 3.5 system tests +ENTRYPOINT ["tox", "-e", "system-tests,system-tests3"] diff --git a/system_tests/Makefile b/system_tests/Makefile index ecfba4ac..cec219ca 100644 --- a/system_tests/Makefile +++ b/system_tests/Makefile @@ -2,9 +2,20 @@ # a `git clone` which can not be cached. DOCKER_FLAGS ?= --no-cache +ifndef GOOGLE_APPLICATION_CREDENTIALS +$(error GOOGLE_APPLICATION_CREDENTIALS is not set; download service account credentials in JSON format from the Google Cloud Console and invoke make with something like GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json) +endif + +ifndef GOOGLE_CLOUD_PROJECT +$(error GOOGLE_CLOUD_PROJECT is not set; invoke make with something like GOOGLE_CLOUD_PROJECT=my-project-name) +endif + + .PHONY: all all: Dockerfile - docker build $(DOCKER_FLAGS) . + @echo "Running system tests in project ${GOOGLE_CLOUD_PROJECT} using service account credentials from ${GOOGLE_APPLICATION_CREDENTIALS}" + docker build --tag google-cloud-python-system-tests $(DOCKER_FLAGS) . + docker run --rm -v $(GOOGLE_APPLICATION_CREDENTIALS):/app/credentials/credentials.json google-cloud-python-system-tests .PHONY: Dockerfile Dockerfile: Dockerfile.in