diff --git a/events/next18/README.md b/events/next18/README.md new file mode 100644 index 00000000..de90147b --- /dev/null +++ b/events/next18/README.md @@ -0,0 +1,48 @@ +# Quarterly business review demo + +This sample was created for a talk for Google Cloud NEXT'18 entitled "Building +on the Docs Editors: APIs and Apps Script". It is the implementation of a +commandline tool that: + +* Extracts template variables out of a Google Slides template presentation +* Writes those variables to a Google Sheets spreadsheet +* Adds data to the spreadsheet based on those variables from a stub data service +* Generates new Google Slides presentations using the template and the + spreadsheet data + +## Getting started + +* Follow the [Sheets API python quickstart](https://developers.google.com/sheets/api/quickstart/python) + * Make sure to save the `credentials.json` file in your working directory +* Enable the Google Slides API, Google Drive API and Google Sheets API in your + developer project +* Run the tool with no arguments to complete the OAuth consent flow: + +```bash +$ python qbr_tool.py +``` + +* Run the tool: + +```bash +# Create the spreadsheet from the Google Slides template. +# For example, 13My9SxkotWssCc2F5yaXp2fzGrzoYV6maytr3qAT9GQ +$ python qbr_tool.py create_sheet --template_id ; + +# Outputs: +# Spreadsheet URL: https://docs.google.com/spreadsheets/d/ + +# Add data from the stub customer service +$ python qbr_tool.py add_customers \ + --spreadsheet_id \ + --customer_id jupiter + +# Generate the filled in presentation +$ python qbr_tool.py create_presentations + --spreadsheet_id \ + --customer_id jupiter + +# Outputs: +# jupiter: https://docs.google.com/presentations/d/ +``` + diff --git a/events/next18/customer_data_service.py b/events/next18/customer_data_service.py new file mode 100644 index 00000000..d7e2e143 --- /dev/null +++ b/events/next18/customer_data_service.py @@ -0,0 +1,127 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=E1102 +# python3 +"""Retrieves customer data from our companies "internal" customer data serivce. + +This data service is meant to be illustrative for demo purposes, as its just +hard coded data. This can be any internal or on-premise data store. +""" + + +class CustomerDataService(object): + _CUSTOMER_DATA = { + 'mars': { + 'customer_name': 'Mars Inc.', + 'customer_logo': + 'https://upload.wikimedia.org/wikipedia/commons/thumb/0/02/' + + 'OSIRIS_Mars_true_color.jpg/550px-OSIRIS_Mars_true_color.jpg', + 'curr_q': 'Q2', + 'curr_q_total_sales': '$2,532,124', + 'curr_q_qoq': '0.054', + 'prev_q': 'Q1', + 'prev_q_total_sales': '$2,413,584', + 'next_q': 'Q3', + 'next_q_total_sales_proj': '$2,634,765', + 'next_q_qoq_proj': '0.041', + 'top1_sku': 'Phobos', + 'top1_sales': '$334,384', + 'top2_sku': 'Deimos', + 'top2_sales': '$315,718', + 'top3_sku': 'Charon', + 'top3_sales': '$285,727', + 'top4_sku': 'Nix', + 'top4_sales': '$264,023', + 'top5_sku': 'Hydra', + 'top5_sales': '$212,361', + }, + 'jupiter': { + 'customer_name': 'Jupiter LLC', + 'customer_logo': + 'https://upload.wikimedia.org/wikipedia/commons/thumb/2/2b/' + + 'Jupiter_and_its_shrunken_Great_Red_Spot.jpg/660px-Jupiter_' + + 'and_its_shrunken_Great_Red_Spot.jpg', + 'curr_q': 'Q2', + 'curr_q_total_sales': '$1,532,124', + 'curr_q_qoq': '0.031', + 'prev_q': 'Q1', + 'prev_q_total_sales': '$1,413,584', + 'next_q': 'Q3', + 'next_q_total_sales_proj': '$1,634,765', + 'next_q_qoq_proj': '0.021', + 'top1_sku': 'Io', + 'top1_sales': '$234,384', + 'top2_sku': 'Europa', + 'top2_sales': '$215,718', + 'top3_sku': 'Ganymede', + 'top3_sales': '$185,727', + 'top4_sku': 'Callisto', + 'top4_sales': '$164,023', + 'top5_sku': 'Amalthea', + 'top5_sales': '$112,361', + }, + 'saturn': { + 'customer_name': 'Saturn', + 'customer_logo': + 'https://upload.wikimedia.org/wikipedia/commons/thumb/c/c7/' + + 'Saturn_during_Equinox.jpg/800px-Saturn_during_Equinox.jpg', + 'curr_q': 'Q2', + 'curr_q_total_sales': '$2,532,124', + 'curr_q_qoq': '0.032', + 'prev_q': 'Q1', + 'prev_q_total_sales': '$2,413,584', + 'next_q': 'Q3', + 'next_q_total_sales_proj': '$2,634,765', + 'next_q_qoq_proj': '0.029', + 'top1_sku': 'Mimas', + 'top1_sales': '$334,384', + 'top2_sku': 'Enceladus', + 'top2_sales': '$315,718', + 'top3_sku': 'Tethys', + 'top3_sales': '$285,727', + 'top4_sku': 'Dione', + 'top4_sales': '$264,023', + 'top5_sku': 'Rhea', + 'top5_sales': '$212,361', + }, + 'neptune': { + 'customer_name': 'Neptune', + 'customer_logo': + 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/56/' + + 'Neptune_Full.jpg/600px-Neptune_Full.jpg', + 'curr_q': 'Q2', + 'curr_q_total_sales': '$2,532,124', + 'curr_q_qoq': '0.027', + 'prev_q': 'Q1', + 'prev_q_total_sales': '$2,413,584', + 'next_q': 'Q3', + 'next_q_total_sales_proj': '$2,634,765', + 'next_q_qoq_proj': '0.039', + 'top1_sku': 'Triton', + 'top1_sales': '$334,384', + 'top2_sku': 'Nereid', + 'top2_sales': '$315,718', + 'top3_sku': 'Naiad', + 'top3_sales': '$285,727', + 'top4_sku': 'Thalassa', + 'top4_sales': '$264,023', + 'top5_sku': 'Despina', + 'top5_sales': '$212,361', + }, + } + + def GetCustomerData(self, customer_id, properties): + customer_data = self._CUSTOMER_DATA[customer_id] + return [customer_data[p.lower()] for p in properties] diff --git a/events/next18/customer_spreadsheet_reader.py b/events/next18/customer_spreadsheet_reader.py new file mode 100644 index 00000000..e0c6167d --- /dev/null +++ b/events/next18/customer_spreadsheet_reader.py @@ -0,0 +1,74 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=E1102 +# python3 +"""Reads the customer data from the template spreadsheet.""" + +import collections + + +class CustomerSpreadsheetReader(object): + + def __init__(self, sheets_service, spreadsheet_id): + self._sheets_service = sheets_service + self._spreadsheet_id = spreadsheet_id + self._data_filters = collections.OrderedDict() + + def ReadColumnData(self, column_id): + data_filter = { + 'developerMetadataLookup': { + 'metadataKey': 'column_id', + 'metadataValue': column_id, + } + } + self._data_filters[column_id] = data_filter + + def ExecuteRead(self): + filters = list(self._data_filters.values()) + get_body = {'dataFilters': filters} + read_fields = ','.join([ + 'sheets.properties.sheetId', + 'sheets.data.rowData.values.formattedValue', + 'developerMetadata.metadataValue']) + spreadsheet = self._sheets_service.spreadsheets().getByDataFilter( + spreadsheetId=self._spreadsheet_id, body=get_body, + fields=read_fields).execute() + customer_spreadsheet = CustomerSpreadsheet( + spreadsheet, self._data_filters) + self._data_filters = collections.OrderedDict() + return customer_spreadsheet + + +class CustomerSpreadsheet(object): + + def __init__(self, spreadsheet, data_filters): + self._spreadsheet = spreadsheet + self._data_filters = data_filters + + def GetSheetId(self): + sheet = self._spreadsheet.get('sheets')[0] + return sheet.get('properties').get('sheetId') + + def GetTemplateId(self): + metadata = self._spreadsheet.get('developerMetadata')[0] + return metadata.get('metadataValue') + + def GetColumnData(self, column_id): + index = list(self._data_filters.keys()).index(column_id) + data = self._spreadsheet.get('sheets')[0].get('data')[index] + values = [row.get('values')[0].get('formattedValue') + for row in data.get('rowData')] + # Remove the first value which is just the label + return values[1:] diff --git a/events/next18/presentation_reader.py b/events/next18/presentation_reader.py new file mode 100644 index 00000000..edc82861 --- /dev/null +++ b/events/next18/presentation_reader.py @@ -0,0 +1,77 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=E1102 +# python3 +"""Reads presentation data. + +Retrieves a presentation and extracts the placeholders and the presentation's +title. +""" +import re + + +class PresentationReader(object): + + def __init__(self, slides_service, presentation_id): + self._slides_service = slides_service + self._presentation_id = presentation_id + self._presentation = None + + def _InitPresentation(self): + if not self._presentation: + self._presentation = self._slides_service.presentations().get( + presentationId=self._presentation_id).execute() + + def GetTitle(self): + self._InitPresentation() + return self._presentation.get('title') + + def GetAllPlaceholders(self): + self._InitPresentation() + slides = self._presentation.get('slides') + placeholders = [] + for slide in slides: + elements = slide.get('pageElements') + for element in elements: + shape = element.get('shape') + table = element.get('table') + # Skip page elements that aren't shapes or tables since they're + # the only types that support text. + if not shape and not table: + continue + if shape: + placeholders += self._GetPlaceholdersFromText( + shape.get('text')) + elif table: + rows = table.get('tableRows') + for row in rows: + cells = row.get('tableCells') + for cell in cells: + placeholders += self._GetPlaceholdersFromText( + cell.get('text')) + # Return the unique placeholders + seen = set() + return [p for p in placeholders if not (p in seen or seen.add(p))] + + def _GetPlaceholdersFromText(self, text): + if not text: + return [] + placeholders = [] + elements = text.get('textElements') + for element in elements: + if element.get('textRun'): + content = element.get('textRun').get('content') + placeholders += re.findall('{.*?}', content) + return placeholders diff --git a/events/next18/presentation_writer.py b/events/next18/presentation_writer.py new file mode 100644 index 00000000..16cfb568 --- /dev/null +++ b/events/next18/presentation_writer.py @@ -0,0 +1,60 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=E1102 +# python3 +"""Functionality for writing to a presentation. +""" + + +class PresentationWriter(object): + """Queues writes for modifying a presentation. + + Call ExecuteBatchUpdate to flush pending writes. + """ + + def __init__(self, slides_service, presentation_id): + self._slides_service = slides_service + self._presentation_id = presentation_id + self._requests = [] + + def ReplaceAllText(self, find_text, replace_text): + request = { + 'replaceAllText': { + 'replaceText': replace_text, + 'containsText': { + 'text': find_text, + 'matchCase': True + } + } + } + self._requests.append(request) + + def ReplaceAllShapesWithImage(self, find_text, image_url): + request = { + 'replaceAllShapesWithImage': { + 'imageUrl': image_url, + 'containsText': { + 'text': find_text, + 'matchCase': True + } + } + } + self._requests.append(request) + + def ExecuteBatchUpdate(self): + body = {'requests': self._requests} + self._requests = [] + self._slides_service.presentations().batchUpdate( + presentationId=self._presentation_id, body=body).execute() diff --git a/events/next18/qbr_tool.py b/events/next18/qbr_tool.py new file mode 100644 index 00000000..0af5bf00 --- /dev/null +++ b/events/next18/qbr_tool.py @@ -0,0 +1,183 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=E1102,W0141 +# python3 +"""Tool for generating quarterly business reviews. + +Reads from an internal data source, pushes that to Google Sheets, then finally +pushes the data to Google Slides +""" + +from __future__ import print_function + +import argparse +import re + +from apiclient.discovery import build +import customer_data_service +import customer_spreadsheet_reader +from httplib2 import Http +from oauth2client import file as oauth_file, client, tools +import presentation_reader +import presentation_writer +import spreadsheet_writer + + +SCOPES = ['https://www.googleapis.com/auth/drive'] +store = oauth_file.Storage('token.json') +creds = store.get() +if not creds or creds.invalid: + flow = client.flow_from_clientsecrets('credentials.json', SCOPES) + creds = tools.run_flow(flow, store) + +slides_service = build('slides', 'v1', http=creds.authorize(Http())) +sheets_service = build('sheets', 'v4', http=creds.authorize(Http())) +drive_service = build('drive', 'v3', http=creds.authorize(Http())) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + 'command', + help='The command to run', + choices=['create_sheet', 'create_presentations', 'add_customers']) + parser.add_argument('--spreadsheet_id', help='The spreadsheet to use') + parser.add_argument( + '--template_id', help='The presentation to use as a template') + parser.add_argument( + '--customer_ids', nargs='+', help='The customers to use') + args = parser.parse_args() + + if args.command == 'create_sheet': + create_sheet(args.template_id) + elif args.command == 'create_presentations': + create_presentations(args.spreadsheet_id, args.customer_ids) + elif args.command == 'add_customers': + add_customers(args.spreadsheet_id, args.customer_ids) + + +def create_sheet(template_id): + pres_reader = presentation_reader.PresentationReader( + slides_service, template_id) + placeholders = pres_reader.GetAllPlaceholders() + presentation_title = pres_reader.GetTitle() + + # Create the data manager spreadsheet + spreadsheet_title = 'Data Sheet - ' + presentation_title + spreadsheet = spreadsheet_writer.CreateSpreadsheet( + sheets_service=sheets_service, + title=spreadsheet_title, + sheet_titles=['Customer Data']) + + # Get the spreadsheet ID and sheet IDs from the created spreadsheet. + spreadsheet_id = spreadsheet.get('spreadsheetId') + sheet_id = spreadsheet.get('sheets')[0].get('properties').get('sheetId') + + # Write the placeholders and metadata to the spreadsheet. + writer = spreadsheet_writer.SpreadsheetWriter( + sheets_service, spreadsheet_id) + writer.PopulateColumn( + sheet_id=sheet_id, + column_index=0, + column_id='placeholders', + values=placeholders) + writer.AddTemplateIdToSpreadsheetMetadata(template_id) + writer.ExecuteBatchUpdate() + + print('Spreadsheet URL: https://docs.google.com/spreadsheets/d/' + + spreadsheet_id) + + +def add_customers(spreadsheet_id, customer_ids): + # Read the placeholders by querying for the developer metadata we added + # while creating the spreadsheet + spreadsheet_reader = customer_spreadsheet_reader.CustomerSpreadsheetReader( + sheets_service, spreadsheet_id) + spreadsheet_reader.ReadColumnData('placeholders') + customer_spreadsheet = spreadsheet_reader.ExecuteRead() + + sheet_id = customer_spreadsheet.GetSheetId() + placeholders = customer_spreadsheet.GetColumnData('placeholders') + + # Process the placeholders into our query properties + properties = [] + for p in placeholders: + # Remove any suffix from the property name + m = re.search(r'{(\w+)(\.\w+)*}', p) + properties.append(m.group(1)) + + data_service = customer_data_service.CustomerDataService() + writer = spreadsheet_writer.SpreadsheetWriter( + sheets_service, spreadsheet_id) + + for customer_id in customer_ids: + # Get the customer data from the internal customer data service + customer_data = data_service.GetCustomerData(customer_id, properties) + + # Write the customer data to the spreadsheet + writer.InsertColumn(sheet_id=sheet_id, column_index=1) + writer.PopulateColumn( + sheet_id=sheet_id, + column_index=1, + column_id=customer_id, + values=customer_data) + + writer.ExecuteBatchUpdate() + + +def create_presentations(spreadsheet_id, customer_ids): + spreadsheet_reader = customer_spreadsheet_reader.CustomerSpreadsheetReader( + sheets_service, spreadsheet_id) + + spreadsheet_reader.ReadColumnData('placeholders') + for customer_id in customer_ids: + spreadsheet_reader.ReadColumnData(customer_id) + + customer_spreadsheet = spreadsheet_reader.ExecuteRead() + placeholders = customer_spreadsheet.GetColumnData('placeholders') + + # Get the template presentation ID and its title + template_id = customer_spreadsheet.GetTemplateId() + pres_reader = presentation_reader.PresentationReader( + slides_service, template_id) + title = pres_reader.GetTitle() + + # Generate a presentation for each customer + for customer_id in customer_ids: + # Create a copy of the presentation + new_title = customer_id + ' - ' + title + presentation_id = drive_service.files().copy( + fileId=template_id, body={ + 'name': new_title + }).execute().get('id') + + # Replace the placeholders with the customer data in the copy + data = customer_spreadsheet.GetColumnData(customer_id) + data_dict = dict(zip(placeholders, data)) + writer = presentation_writer.PresentationWriter(slides_service, + presentation_id) + for placeholder, value in data_dict.items(): + if re.findall(r'{(\w+).image}', placeholder): + writer.ReplaceAllShapesWithImage(placeholder, value) + else: + writer.ReplaceAllText(placeholder, value) + writer.ExecuteBatchUpdate() + + print(customer_id + + ': https://docs.google.com/presentation/d/' + presentation_id) + + +if __name__ == '__main__': + main() diff --git a/events/next18/spreadsheet_writer.py b/events/next18/spreadsheet_writer.py new file mode 100644 index 00000000..e8295489 --- /dev/null +++ b/events/next18/spreadsheet_writer.py @@ -0,0 +1,139 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=E1102 +# python3 +"""Functionality for creating and writing to a spreadsheet.""" + + +def CreateSpreadsheet(sheets_service, title, sheet_titles): + """Creates an empty spreadsheet. + + It creates a spreadsheet with the provided title, and creates a sheet for + each entry in the sheet_titles list with the corresponding sheet title. + """ + sheets = [] + for sheet_title in sheet_titles: + sheet = { + 'properties': { + 'title': sheet_title, + }, + } + sheets.append(sheet) + + spreadsheet = { + 'properties': { + 'title': title, + }, + 'sheets': sheets, + } + return sheets_service.spreadsheets().create(body=spreadsheet).execute() + + +class SpreadsheetWriter(object): + """Queues writes for modifying a spreadsheet. + + Call ExecuteBatchUpdate to flush pending writes. + """ + + def __init__(self, sheets_service, spreadsheet_id): + self._sheets_service = sheets_service + self._spreadsheet_id = spreadsheet_id + self._requests = [] + + def InsertColumn(self, sheet_id, column_index): + request = { + 'insertDimension': { + 'range': { + 'sheetId': sheet_id, + 'dimension': 'COLUMNS', + 'startIndex': column_index, + 'endIndex': column_index + 1, + }, + } + } + self._requests.append(request) + + def PopulateColumn(self, sheet_id, column_index, column_id, values): + # Include the column ID in the column values + values = [column_id] + values + + # Populate the column with the values + rows = [] + for value in values: + row_data = { + 'values': [ + { + 'userEnteredValue': { + 'stringValue': value + } + } + ] + } + rows.append(row_data) + + update_request = { + 'updateCells': { + 'rows': rows, + 'fields': 'userEnteredValue', + 'start': { + 'sheetId': sheet_id, + 'rowIndex': 0, + 'columnIndex': column_index + } + } + } + self._requests.append(update_request) + + # Add developer metadata to the column to make it easier to read later + # by being able to just query it by the column ID + metadata_request = { + 'createDeveloperMetadata': { + 'developerMetadata': { + 'metadataKey': 'column_id', + 'metadataValue': column_id, + 'location': { + 'dimensionRange': { + 'sheetId': sheet_id, + 'dimension': 'COLUMNS', + 'startIndex': column_index, + 'endIndex': column_index + 1, + } + }, + 'visibility': 'DOCUMENT', + } + } + } + self._requests.append(metadata_request) + + def AddTemplateIdToSpreadsheetMetadata(self, template_id): + request = { + 'createDeveloperMetadata': { + 'developerMetadata': { + 'metadataKey': 'template_id', + 'metadataValue': template_id, + 'location': { + 'spreadsheet': True + }, + 'visibility': 'DOCUMENT', + } + } + } + self._requests.append(request) + + def ExecuteBatchUpdate(self): + body = {'requests': self._requests} + self._requests = [] + return self._sheets_service.spreadsheets().batchUpdate( + spreadsheetId=self._spreadsheet_id, body=body).execute()