diff --git a/.gemini/common/constraints.md b/.gemini/common/constraints.md deleted file mode 100644 index 1b563eab3b8..00000000000 --- a/.gemini/common/constraints.md +++ /dev/null @@ -1,8 +0,0 @@ -## Constraints - -- Only add git commits. Do not change git history. -- Follow the spec file for development. - - Check off items in the "Acceptance - criteria" and "Detailed steps" sections with `[x]`. - - Please do this as they are completed. - - Refer back to the spec after each step. diff --git a/.gemini/common/docs.md b/.gemini/common/docs.md deleted file mode 100644 index 1a718005ad7..00000000000 --- a/.gemini/common/docs.md +++ /dev/null @@ -1,9 +0,0 @@ -## Documentation - -If a method or property is implementing the same interface as a third-party -package such as pandas or scikit-learn, place the relevant docstring in the -corresponding `third_party/bigframes_vendored/package_name` directory, not in -the `bigframes` directory. Implementations may be placed in the `bigframes` -directory, though. - -@../tools/test_docs.md diff --git a/.gemini/tools/style_nox.md b/.gemini/tools/style_nox.md deleted file mode 100644 index 894fd102363..00000000000 --- a/.gemini/tools/style_nox.md +++ /dev/null @@ -1,18 +0,0 @@ -## Code Style with nox - -- We use the automatic code formatter `black`. You can run it using - the nox session `format`. This will eliminate many lint errors. Run via: - - ```bash - nox -r -s format - ``` - -- PEP8 compliance is required, with exceptions defined in the linter configuration. - If you have ``nox`` installed, you can test that you have not introduced - any non-compliant code via: - - ``` - nox -r -s lint - ``` - -- When writing tests, use the idiomatic "pytest" style. diff --git a/.gemini/tools/test_docs.md b/.gemini/tools/test_docs.md deleted file mode 100644 index 5cb988186c7..00000000000 --- a/.gemini/tools/test_docs.md +++ /dev/null @@ -1,10 +0,0 @@ -## Testing code samples - -Code samples are very important for accurate documentation. We use the "doctest" -framework to ensure the samples are functioning as expected. After adding a code -sample, please ensure it is correct by running doctest. To run the samples -doctests for just a single method, refer to the following example: - -```bash -pytest --doctest-modules bigframes/pandas/__init__.py::bigframes.pandas.cut -``` diff --git a/.gemini/tools/test_nox.md b/.gemini/tools/test_nox.md deleted file mode 100644 index 023ada1b61f..00000000000 --- a/.gemini/tools/test_nox.md +++ /dev/null @@ -1,28 +0,0 @@ -## Testing with nox - -Use `nox` to instrument our tests. - -- To test your changes, run unit tests with `nox`: - - ```bash - nox -r -s unit - ``` - -- To run a single unit test: - - ```bash - nox -r -s unit-3.14 -- -k - ``` - -- Ignore this step if you lack access to Google Cloud resources. To run system - tests, you can execute:: - - # Run all system tests - $ nox -r -s system - - # Run a single system test - $ nox -r -s system-3.14 -- -k - -- The codebase must have better coverage than it had previously after each - change. You can test coverage via `nox -s unit system cover` (takes a long - time). Omit `system` if you lack access to cloud resources. diff --git a/.gemini/tools/test_pytest.md b/.gemini/tools/test_pytest.md deleted file mode 100644 index 5228ae06ba8..00000000000 --- a/.gemini/tools/test_pytest.md +++ /dev/null @@ -1,9 +0,0 @@ -## Testing with pytest - -Use `pytest` to instrument our tests. - -- To test your changes, run `pytest`: - - ```bash - pytest :: - ``` diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000..7686a50da62 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,12 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax +# Note: This file is autogenerated. To make changes to the codeowner team, please update .repo-metadata.json. + +# @googleapis/yoshi-python @googleapis/api-bigquery-dataframe are the default owners for changes in this repo +* @googleapis/yoshi-python @googleapis/api-bigquery-dataframe + +# @googleapis/python-samples-reviewers @googleapis/api-bigquery-dataframe are the default owners for samples changes +/samples/ @googleapis/python-samples-reviewers @googleapis/api-bigquery-dataframe diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 00000000000..939e5341e74 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,28 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows [Google's Open Source Community +Guidelines](https://opensource.google.com/conduct/). diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000000..0745497ddf2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,58 @@ +--- +name: Bug report +about: Create a report to help us improve + +--- + +Thanks for stopping by to let us know something could be better! + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. + +Please run down the following list and make sure you've tried the usual "quick fixes": + + - Search the issues already opened: https://github.com/googleapis/python-bigquery-dataframes/issues + - Search StackOverflow: https://stackoverflow.com/questions/tagged/google-cloud-platform+python + +If you are still having issues, please be sure to include as much information as possible: + +#### Environment details + + - OS type and version: + - Python version: `python --version` + - pip version: `pip --version` + - `bigframes` version: `pip show bigframes` + + +```python +import sys +import bigframes +import google.cloud.bigquery +import pandas +import pyarrow + +print(f"Python: {sys.version}") +print(f"bigframes=={bigframes.__version__}") +print(f"google-cloud-bigquery=={google.cloud.bigquery.__version__}") +print(f"pandas=={pandas.__version__}") +print(f"pyarrow=={pyarrow.__version__}") +``` + +#### Steps to reproduce + + 1. ? + 2. ? + +#### Code example + +```python +# example +``` + +#### Stack trace +``` +# example +``` + +Making sure to follow these steps will guarantee the quickest resolution possible. + +Thanks! diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000000..6365857f33c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,18 @@ +--- +name: Feature request +about: Suggest an idea for this library + +--- + +Thanks for stopping by to let us know something could be better! + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. + + **Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + **Describe the solution you'd like** +A clear and concise description of what you want to happen. + **Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + **Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/support_request.md b/.github/ISSUE_TEMPLATE/support_request.md new file mode 100644 index 00000000000..99586903212 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/support_request.md @@ -0,0 +1,7 @@ +--- +name: Support request +about: If you have a support contract with Google, please create an issue in the Google Cloud Support console. + +--- + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000000..3e59d9a70d1 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,7 @@ +Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: +- [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery-dataframes/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea +- [ ] Ensure the tests and linter pass +- [ ] Code coverage does not decrease (if any source code was changed) +- [ ] Appropriate docs were updated (if necessary) + +Fixes # 🦕 diff --git a/bigframes/extensions/__init__.py b/.github/auto-label.yaml similarity index 82% rename from bigframes/extensions/__init__.py rename to .github/auto-label.yaml index 58d482ea386..21786a4eb08 100644 --- a/bigframes/extensions/__init__.py +++ b/.github/auto-label.yaml @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,3 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +requestsize: + enabled: true + +path: + pullrequest: true + paths: + samples: "samples" diff --git a/.github/blunderbuss.yml b/.github/blunderbuss.yml new file mode 100644 index 00000000000..8d9cb1008e7 --- /dev/null +++ b/.github/blunderbuss.yml @@ -0,0 +1,17 @@ +# Blunderbuss config +# +# This file controls who is assigned for pull requests and issues. +# Note: This file is autogenerated. To make changes to the assignee +# team, please update `codeowner_team` in `.repo-metadata.json`. +assign_issues: + - googleapis/api-bigquery-dataframe + +assign_issues_by: + - labels: + - "samples" + to: + - googleapis/python-samples-reviewers + - googleapis/api-bigquery-dataframe + +assign_prs: + - googleapis/api-bigquery-dataframe diff --git a/.github/header-checker-lint.yml b/.github/header-checker-lint.yml new file mode 100644 index 00000000000..6fe78aa7987 --- /dev/null +++ b/.github/header-checker-lint.yml @@ -0,0 +1,15 @@ +{"allowedCopyrightHolders": ["Google LLC"], + "allowedLicenses": ["Apache-2.0", "MIT", "BSD-3"], + "ignoreFiles": ["**/requirements.txt", "**/requirements-test.txt", "**/__init__.py", "samples/**/constraints.txt", "samples/**/constraints-test.txt"], + "sourceFileExtensions": [ + "ts", + "js", + "java", + "sh", + "Dockerfile", + "yaml", + "py", + "html", + "txt" + ] +} \ No newline at end of file diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml new file mode 100644 index 00000000000..13d4d87263b --- /dev/null +++ b/.github/workflows/docs-deploy.yml @@ -0,0 +1,57 @@ +name: Deploy docs to GitHub Pages + +on: + # Runs on pushes targeting the default branch + # TODO(tswast): Update this to only be releases once we confirm it's working. + push: + branches: ["main"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + # Build job + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run docs + run: | + nox -s docs + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/_build/html/ + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000000..6773aef7c2a --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,41 @@ +on: + pull_request: + branches: + - main + push: + branches: + - main +name: docs +jobs: + docs: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run docs + run: | + nox -s docs + docfx: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run docfx + run: | + nox -s docfx diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml new file mode 100644 index 00000000000..588aa854f32 --- /dev/null +++ b/.github/workflows/js-tests.yml @@ -0,0 +1,20 @@ +name: js-tests +on: + pull_request: + branches: + - main + push: + branches: + - main +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install modules + working-directory: ./tests/js + run: npm install + - name: Run tests + working-directory: ./tests/js + run: npm test diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000..7914b72651e --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,28 @@ +on: + pull_request: + branches: + - main + push: + branches: + - main +name: lint +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run lint + run: | + nox -s lint + - name: Run lint_setup_py + run: | + nox -s lint_setup_py diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml new file mode 100644 index 00000000000..fc9e9709461 --- /dev/null +++ b/.github/workflows/mypy.yml @@ -0,0 +1,25 @@ +on: + pull_request: + branches: + - main + push: + branches: + - main +name: mypy +jobs: + mypy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run mypy + run: | + nox -s mypy diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml new file mode 100644 index 00000000000..518cec63125 --- /dev/null +++ b/.github/workflows/unittest.yml @@ -0,0 +1,64 @@ +on: + pull_request: + branches: + - main + push: + branches: + - main +name: unittest +jobs: + unit: + # TODO(https://github.com/googleapis/gapic-generator-python/issues/2303): use `ubuntu-latest` once this bug is fixed. + # Use ubuntu-22.04 until Python 3.7 is removed from the test matrix + # https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories + runs-on: ubuntu-22.04 + strategy: + matrix: + python: ['3.9', '3.10', '3.11', '3.12', '3.13'] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run unit tests + env: + COVERAGE_FILE: .coverage-${{ matrix.python }} + run: | + nox -s unit-${{ matrix.python }} + - name: Upload coverage results + uses: actions/upload-artifact@v4 + with: + name: coverage-artifact-${{ matrix.python }} + path: .coverage-${{ matrix.python }} + include-hidden-files: true + + cover: + runs-on: ubuntu-latest + needs: + - unit + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install coverage + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install coverage + - name: Download coverage results + uses: actions/download-artifact@v4 + with: + path: .coverage-results/ + - name: Report coverage results + run: | + find .coverage-results -type f -name '*.zip' -exec unzip {} \; + coverage combine .coverage-results/**/.coverage* + coverage report --show-missing --fail-under=35 diff --git a/.gitignore b/.gitignore index d92dcc6b474..52dcccd33d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,223 +1,67 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[codz] -*$py.class +*.py[cod] +*.sw[op] # C extensions *.so -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg +# Packages *.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec +*.egg-info +dist +build +eggs +.eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +__pycache__ # Installer logs pip-log.txt -pip-delete-this-directory.txt # Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ .coverage -.coverage.* +.nox .cache -nosetests.xml -coverage.xml -*.cover -*.py.cover -*.lcov -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -# Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -# poetry.lock -# poetry.toml - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. -# https://pdm-project.org/en/latest/usage/project/#working-with-version-control -# pdm.lock -# pdm.toml -.pdm-python -.pdm-build/ +.pytest_cache -# pixi -# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. -# pixi.lock -# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one -# in the .venv directory. It is recommended not to include this directory in version control. -.pixi/* -!.pixi/config.toml -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ +# Mac +.DS_Store -# Celery stuff -celerybeat-schedule* -celerybeat.pid +# JetBrains +.idea -# Redis -*.rdb -*.aof -*.pid +# VS Code +.vscode -# RabbitMQ -mnesia/ -rabbitmq/ -rabbitmq-data/ +# emacs +*~ -# ActiveMQ -activemq-data/ +# Built documentation +docs/_build +bigquery/docs/generated +docs.metadata -# SageMath parsed files -*.sage.py - -# Environments -.env -.envrc -.venv +# Virtual environment env/ venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject -# Rope project settings -.ropeproject +# Test logs +coverage.xml +*sponge_log.xml -# mkdocs documentation -/site +# System test environment variables. +system_tests/local_test_setup +tests/js/node_modules/ -# mypy +# Make sure a generated file isn't accidentally committed. +pylintrc +pylintrc.test +dummy.pkl .mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -# .idea/ - -# Abstra -# Abstra is an AI-powered process automation framework. -# Ignore directories containing user credentials, local state, and settings. -# Learn more at https://abstra.io/docs -.abstra/ - -# Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore -# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, -# you could uncomment the following to ignore the entire vscode folder -# .vscode/ -# Temporary file for partial code execution -tempCodeRunnerFile.py - -# Ruff stuff: -.ruff_cache/ - -# PyPI configuration file -.pypirc - -# Marimo -marimo/_static/ -marimo/_lsp/ -__marimo__/ - -# Streamlit -.streamlit/secrets.toml - -# Tests -*sponge_log.xml diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 01b0af912ee..6cc03455da4 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -1,11 +1,11 @@ #!/bin/bash -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,24 +13,43 @@ # See the License for the specific language governing permissions and # limitations under the License. -# `-e` enables the script to automatically fail when a command fails -# `-o pipefail` sets the exit code to non-zero if any command fails, -# or zero if all commands in the pipeline exit successfully. set -eo pipefail -cd "${KOKORO_ARTIFACTS_DIR}/git/bigframes" -pwd +PROJECT_SCM="github/python-bigquery-dataframes" -# If NOX_SESSION is set, it only runs the specified session, -# otherwise run all the sessions. -NOX_SESSION_ARG="" +if [[ -z "${PROJECT_ROOT:-}" ]]; then + PROJECT_ROOT="${KOKORO_ARTIFACTS_DIR}/${PROJECT_SCM}" +fi + +cd "${PROJECT_ROOT}" + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Workaround https://github.com/pytest-dev/pytest/issues/9567 +export PY_IGNORE_IMPORTMISMATCH=1 -# IF NOX_FILE is set, it runs the specific nox file, -# otherwise it runs noxfile.py in the package directory. -NOX_FILE_ARG="" +# Debug: show build environment +env | grep KOKORO -[[ -z "${NOX_SESSION}" ]] || NOX_SESSION_ARG="-s ${NOX_SESSION}" +# Install pip +python3 -m pip install --upgrade --quiet pip +python3 -m pip --version -[[ -z "${NOX_FILE}" ]] || NOX_FILE_ARG="-f ${NOX_FILE}" +# Remove old nox +python3 -m pip uninstall --yes --quiet nox-automation + +# Install nox +python3 -m pip install --upgrade --quiet nox +python3 -m nox --version + +# If NOX_SESSION is set, it only runs the specified session, +# otherwise run all the sessions. +if [[ -n "${NOX_SESSION:-}" ]]; then + python3 -m nox --stop-on-first-error -s ${NOX_SESSION:-} +else + python3 -m nox --stop-on-first-error +fi -python3 -m nox ${NOX_SESSION_ARG} $NOX_FILE_ARG +# Prevent kokoro from trying to collect many mb of artifacts, wasting several minutes +sudo rm -rf "${KOKORO_ARTIFACTS_DIR?}"/* diff --git a/.kokoro/buildwheel.cfg b/.kokoro/buildwheel.cfg deleted file mode 100644 index af89c99e419..00000000000 --- a/.kokoro/buildwheel.cfg +++ /dev/null @@ -1,22 +0,0 @@ -# -*- protobuffer -*- -# proto-file: google3/devtools/kokoro/config/proto/build.proto -# proto-message: BuildConfig - -build_file: "bigframes-internal/bigframes/.kokoro/buildwheel.sh" -container_properties { - docker_image: "us-docker.pkg.dev/artifact-foundry-prod/docker-3p-trusted/python@sha256:0b3498e251759df85a00474be7d3b791d6abe1600ce3531a649e42964749655f" -} - -fileset_artifacts { - name: "artifacts" - artifact_globs: "artifacts/*" - error_if_missing: true - destinations { - store_attestation: true - gcs { - gcs_root_path: "oss-exit-gate-prod-projects-bucket/bigframes/pypi/attestations" - } - } - generate_sbom_from_fileset: true - generate_attestation: true -} diff --git a/.kokoro/buildwheel.sh b/.kokoro/buildwheel.sh deleted file mode 100755 index b64b6412b02..00000000000 --- a/.kokoro/buildwheel.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -set -euo pipefail - -cd "${KOKORO_ARTIFACTS_DIR}/git/bigframes-internal/bigframes" - -### Set up Airlock for Bookworm -rm -f /etc/apt/sources.list.d/* /etc/apt/sources.list -echo 'deb https://us-apt.pkg.dev/remote/artifact-foundry-prod/debian-3p-remote-bookworm bookworm main' | \ - tee -a /etc/apt/sources.list.d/artifact-registry.list - -# Set up Airlock for Python -cat > "$HOME/.pypirc" < "$HOME/.pip/pip.conf" <&2 ;} +function println { printf '%s\n' "$(now) $*" ;} + + +# Populates requested secrets set in SECRET_MANAGER_KEYS from service account: +# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com +SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" +msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" +mkdir -p ${SECRET_LOCATION} +for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") +do + msg "Retrieving secret ${key}" + docker run --entrypoint=gcloud \ + --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \ + gcr.io/google.com/cloudsdktool/cloud-sdk \ + secrets versions access latest \ + --project cloud-devrel-kokoro-resources \ + --secret ${key} > \ + "${SECRET_LOCATION}/${key}" + if [[ $? == 0 ]]; then + msg "Secret written to ${SECRET_LOCATION}/${key}" + else + msg "Error retrieving secret ${key}" + fi +done diff --git a/.kokoro/presubmit/common.cfg b/.kokoro/presubmit/common.cfg index 5d40578ac79..97e0651aa92 100644 --- a/.kokoro/presubmit/common.cfg +++ b/.kokoro/presubmit/common.cfg @@ -7,4 +7,4 @@ action { } } -build_file: "bigframes/.kokoro/build.sh" +build_file: "python-bigquery-dataframes/.kokoro/build.sh" diff --git a/.kokoro/presubmit/doctest.cfg b/.kokoro/presubmit/doctest.cfg new file mode 100644 index 00000000000..2aad95beed4 --- /dev/null +++ b/.kokoro/presubmit/doctest.cfg @@ -0,0 +1,12 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "cleanup doctest" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "bigframes-testing" +} diff --git a/.kokoro/continuous/continuous-docs-linux.cfg b/.kokoro/presubmit/e2e-gerrit.cfg similarity index 56% rename from .kokoro/continuous/continuous-docs-linux.cfg rename to .kokoro/presubmit/e2e-gerrit.cfg index 93832f9e5aa..19913344b6c 100644 --- a/.kokoro/continuous/continuous-docs-linux.cfg +++ b/.kokoro/presubmit/e2e-gerrit.cfg @@ -1,6 +1,7 @@ # Format: //devtools/kokoro/config/proto/build.proto +# Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "docs docfx" + value: "system_noextras e2e notebook" } diff --git a/.kokoro/presubmit/e2e.cfg b/.kokoro/presubmit/e2e.cfg new file mode 100644 index 00000000000..e049dd30b3a --- /dev/null +++ b/.kokoro/presubmit/e2e.cfg @@ -0,0 +1,17 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "e2e unit_prerelease system_prerelease system_noextras" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "bigframes-load-testing" +} + +env_vars: { + key: "BIGFRAMES_TEST_MODEL_VERTEX_ENDPOINT" + value: "https://us-central1-aiplatform.googleapis.com/v1/projects/272725758477/locations/us-central1/endpoints/590545496255234048" +} diff --git a/.kokoro/presubmit/notebook.cfg b/.kokoro/presubmit/notebook.cfg new file mode 100644 index 00000000000..cc73c3bea40 --- /dev/null +++ b/.kokoro/presubmit/notebook.cfg @@ -0,0 +1,12 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "notebook" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "bigframes-testing" +} diff --git a/.kokoro/continuous/continuous-lint-linux.cfg b/.kokoro/presubmit/prerelease-deps.cfg similarity index 61% rename from .kokoro/continuous/continuous-lint-linux.cfg rename to .kokoro/presubmit/prerelease-deps.cfg index 490cafab878..3595fb43f5c 100644 --- a/.kokoro/continuous/continuous-lint-linux.cfg +++ b/.kokoro/presubmit/prerelease-deps.cfg @@ -1,6 +1,7 @@ # Format: //devtools/kokoro/config/proto/build.proto +# Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "lint mypy" + value: "prerelease_deps" } diff --git a/.kokoro/presubmit/presubmit-docs-linux.cfg b/.kokoro/presubmit/presubmit-docs-linux.cfg deleted file mode 100644 index 93832f9e5aa..00000000000 --- a/.kokoro/presubmit/presubmit-docs-linux.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "NOX_SESSION" - value: "docs docfx" -} diff --git a/.kokoro/presubmit/presubmit-doctest-linux.cfg b/.kokoro/presubmit/presubmit-doctest-linux.cfg deleted file mode 100644 index af74ca0fbcd..00000000000 --- a/.kokoro/presubmit/presubmit-doctest-linux.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "NOX_SESSION" - value: "doctest" -} diff --git a/.kokoro/presubmit/presubmit-e2e-linux.cfg b/.kokoro/presubmit/presubmit-e2e-linux.cfg deleted file mode 100644 index ed73a083ee0..00000000000 --- a/.kokoro/presubmit/presubmit-e2e-linux.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "NOX_SESSION" - value: "e2e" -} diff --git a/.kokoro/presubmit/presubmit-gerrit.cfg b/.kokoro/presubmit/presubmit-gerrit.cfg new file mode 100644 index 00000000000..18a4c35325b --- /dev/null +++ b/.kokoro/presubmit/presubmit-gerrit.cfg @@ -0,0 +1 @@ +# Format: //devtools/kokoro/config/proto/build.proto diff --git a/.kokoro/presubmit/presubmit-lint-linux.cfg b/.kokoro/presubmit/presubmit-lint-linux.cfg deleted file mode 100644 index 490cafab878..00000000000 --- a/.kokoro/presubmit/presubmit-lint-linux.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "NOX_SESSION" - value: "lint mypy" -} diff --git a/.kokoro/presubmit/presubmit-prerelease-linux.cfg b/.kokoro/presubmit/presubmit-prerelease-linux.cfg deleted file mode 100644 index bfc635fbf30..00000000000 --- a/.kokoro/presubmit/presubmit-prerelease-linux.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "NOX_SESSION" - value: "unit_prerelease system_prerelease" -} diff --git a/.kokoro/presubmit/presubmit-system-linux.cfg b/.kokoro/presubmit/presubmit-system-linux.cfg deleted file mode 100644 index 9fa4e1b73b2..00000000000 --- a/.kokoro/presubmit/presubmit-system-linux.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "NOX_SESSION" - value: "system" -} diff --git a/.kokoro/presubmit/presubmit-unit-linux.cfg b/.kokoro/presubmit/presubmit-unit-linux.cfg deleted file mode 100644 index 09f4f4b8311..00000000000 --- a/.kokoro/presubmit/presubmit-unit-linux.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "NOX_SESSION" - value: "unit" -} diff --git a/.kokoro/presubmit/presubmit.cfg b/.kokoro/presubmit/presubmit.cfg new file mode 100644 index 00000000000..8f43917d92f --- /dev/null +++ b/.kokoro/presubmit/presubmit.cfg @@ -0,0 +1 @@ +# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file diff --git a/.kokoro/presubmit/windows.cfg b/.kokoro/presubmit/windows.cfg new file mode 100644 index 00000000000..806986138dd --- /dev/null +++ b/.kokoro/presubmit/windows.cfg @@ -0,0 +1,3 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +build_file: "python-bigquery-dataframes/scripts/windows/build.bat" diff --git a/.kokoro/release-nightly.sh b/.kokoro/release-nightly.sh new file mode 100755 index 00000000000..124e4b8b486 --- /dev/null +++ b/.kokoro/release-nightly.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Based loosely on +# https://github.com/googleapis/python-bigquery/blob/main/.kokoro/release.sh + +set -eo pipefail +set -x + +# Parse command line arguments +DRY_RUN= +while [ $# -gt 0 ] ; do + case "$1" in + -d | --dry-run ) + DRY_RUN=true + ;; + -h | --help ) + echo -e "USAGE: `basename $0` [ -d | --dry-run ]" + exit + ;; + esac + shift 1; +done + +if [[ -z "${KOKORO_GOB_COMMIT}" ]]; then + PROJECT_SCM="github/python-bigquery-dataframes" +else + PROJECT_SCM="git/bigframes" +fi + +if [ -z "${PROJECT_ROOT:-}" ]; then + PROJECT_ROOT="${KOKORO_ARTIFACTS_DIR}/${PROJECT_SCM}" +fi + +# Move into the package, build the distribution and upload to shared bucket. +# See internal bug 274624240 for details. + +cd "${PROJECT_ROOT}" +rm -rf build dist + +# Workaround the fact that the repository that has been fetched before the +# build script. See: go/kokoro-native-docker-migration#known-issues and +# internal issue b/261050975. +git config --global --add safe.directory "${PROJECT_ROOT}" + +# Workaround for older pip not able to resolve dependencies. See internal +# issue 316909553. +python3.10 -m pip install pip==25.0.1 + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Install dependencies, as the following steps depend on it +python3.10 -m pip install -e .[all] + +# Update version string to include git hash and date +CURRENT_DATE=$(date '+%Y%m%d') +GIT_HASH=$(git rev-parse --short HEAD) +BIGFRAMES_VERSION=$(python3.10 -c "import bigframes; print(bigframes.__version__)") +RELEASE_VERSION=${BIGFRAMES_VERSION}dev${CURRENT_DATE}+${GIT_HASH} +sed -i -e "s/$BIGFRAMES_VERSION/$RELEASE_VERSION/g" bigframes/version.py + +# Generate the package wheel +python3.10 setup.py sdist bdist_wheel + +# Make sure that the wheel file is generated +VERSION_WHEEL=`ls dist/bigframes-*.whl` +num_wheel_files=`echo $VERSION_WHEEL | wc -w` +if [ $num_wheel_files -ne 1 ] ; then + echo "Exactly one wheel file should have been generated, found $num_wheel_files: $VERSION_WHEEL" + exit -1 +fi + +# Create a copy of the wheel with a well known, version agnostic name +LATEST_WHEEL=dist/bigframes-latest-py2.py3-none-any.whl +cp $VERSION_WHEEL $LATEST_WHEEL +cp dist/bigframes-*.tar.gz dist/bigframes-latest.tar.gz + +if ! [ ${DRY_RUN} ]; then +for gcs_path in gs://vertex_sdk_private_releases/bigframe/ \ + gs://dl-platform-colab/bigframes/ \ + gs://bigframes-wheels/; + do + gsutil cp -v dist/* ${gcs_path} + gsutil cp -v LICENSE ${gcs_path} + gsutil -m cp -r -v "notebooks/" ${gcs_path}notebooks/ + + done + + # publish API coverage information to BigQuery + # Note: only the kokoro service account has permission to write to this + # table, if you want to test this step, point it to a table you have + # write access to + COVERAGE_TABLE=bigframes-metrics.coverage_report.bigframes_coverage_nightly + python3.10 scripts/publish_api_coverage.py \ + bigquery \ + --bigframes_version=$BIGFRAMES_VERSION \ + --release_version=$RELEASE_VERSION \ + --bigquery_table=$COVERAGE_TABLE +fi + +# Undo the file changes, in case this script is running on a +# non-temporary instance of the bigframes repo +# TODO: This doesn't work with (set -eo pipefail) if the failure happened after +# the changes were made but before this cleanup, because the script would +# terminate with the failure itself. See if we can ensure the cleanup. +sed -i -e "s/$RELEASE_VERSION/$BIGFRAMES_VERSION/g" bigframes/version.py + +if ! [ ${DRY_RUN} ]; then + # Copy docs and wheels to Google Drive + python3.10 scripts/upload_to_google_drive.py +fi diff --git a/.kokoro/release.cfg b/.kokoro/release.cfg deleted file mode 100644 index 18c85a69d63..00000000000 --- a/.kokoro/release.cfg +++ /dev/null @@ -1,23 +0,0 @@ -# -*- protobuffer -*- -# proto-file: google3/devtools/kokoro/config/proto/build.proto -# proto-message: BuildConfig - -build_file: "bigframes-internal/bigframes/.kokoro/release.sh" -container_properties { - docker_image: "us-docker.pkg.dev/artifact-foundry-prod/docker-3p-trusted/ubuntu:22.04" -} - -fileset_artifacts { - name: "manifest" - artifact_globs: "manifest.json" - error_if_missing: true - destinations { - store_attestation: false - gcs { - gcs_root_path: "oss-exit-gate-prod-projects-bucket/bigframes/pypi/manifests" - populate_content_type: true - } - } - generate_sbom_from_fileset: false - generate_attestation: false -} diff --git a/.kokoro/release.sh b/.kokoro/release.sh deleted file mode 100755 index 58b865a6f54..00000000000 --- a/.kokoro/release.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -euo pipefail - -cd "${KOKORO_ARTIFACTS_DIR}" - -cat > manifest.json <<'EOF' -{ - "publish_all": true -} -EOF diff --git a/.kokoro/requirements/build.in b/.kokoro/requirements/build.in deleted file mode 100644 index bd63670492d..00000000000 --- a/.kokoro/requirements/build.in +++ /dev/null @@ -1,6 +0,0 @@ ---only-binary :all: -twine>=6.2.0 -build>=1.3.0 -wheel>=0.46.2 -keyring>=25.7.0 -keyrings.google-artifactregistry-auth>=1.1.2 diff --git a/.kokoro/requirements/build.txt b/.kokoro/requirements/build.txt deleted file mode 100644 index ed4ce023d35..00000000000 --- a/.kokoro/requirements/build.txt +++ /dev/null @@ -1,447 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.13 -# by the following command: -# -# pip-compile --generate-hashes --output-file=.kokoro/requirements/build.txt .kokoro/requirements/build.in -# ---only-binary :all: - -build==1.3.0 \ - --hash=sha256:7145f0b5061ba90a1500d60bd1b13ca0a8a4cebdd0cc16ed8adf1c0e739f43b4 - # via -r build.in -cachetools==6.2.2 \ - --hash=sha256:6c09c98183bf58560c97b2abfcedcbaf6a896a490f534b031b661d3723b45ace - # via google-auth -certifi==2025.11.12 \ - --hash=sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b - # via requests -cffi==2.0.0 \ - --hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \ - --hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \ - --hash=sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f \ - --hash=sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9 \ - --hash=sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44 \ - --hash=sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2 \ - --hash=sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c \ - --hash=sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75 \ - --hash=sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65 \ - --hash=sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e \ - --hash=sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a \ - --hash=sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e \ - --hash=sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25 \ - --hash=sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a \ - --hash=sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe \ - --hash=sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b \ - --hash=sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91 \ - --hash=sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592 \ - --hash=sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187 \ - --hash=sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c \ - --hash=sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1 \ - --hash=sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94 \ - --hash=sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba \ - --hash=sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb \ - --hash=sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165 \ - --hash=sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca \ - --hash=sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c \ - --hash=sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6 \ - --hash=sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c \ - --hash=sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0 \ - --hash=sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743 \ - --hash=sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63 \ - --hash=sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5 \ - --hash=sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5 \ - --hash=sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4 \ - --hash=sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d \ - --hash=sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b \ - --hash=sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93 \ - --hash=sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205 \ - --hash=sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27 \ - --hash=sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512 \ - --hash=sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d \ - --hash=sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c \ - --hash=sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037 \ - --hash=sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26 \ - --hash=sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322 \ - --hash=sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb \ - --hash=sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c \ - --hash=sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8 \ - --hash=sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4 \ - --hash=sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414 \ - --hash=sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9 \ - --hash=sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664 \ - --hash=sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9 \ - --hash=sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775 \ - --hash=sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739 \ - --hash=sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc \ - --hash=sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062 \ - --hash=sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe \ - --hash=sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9 \ - --hash=sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92 \ - --hash=sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5 \ - --hash=sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13 \ - --hash=sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d \ - --hash=sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26 \ - --hash=sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f \ - --hash=sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495 \ - --hash=sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b \ - --hash=sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6 \ - --hash=sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c \ - --hash=sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef \ - --hash=sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5 \ - --hash=sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18 \ - --hash=sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad \ - --hash=sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3 \ - --hash=sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7 \ - --hash=sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5 \ - --hash=sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534 \ - --hash=sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49 \ - --hash=sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2 \ - --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \ - --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \ - --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf - # via cryptography -charset-normalizer==3.4.4 \ - --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ - --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ - --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ - --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ - --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ - --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ - --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ - --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ - --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ - --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ - --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ - --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ - --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ - --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ - --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ - --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ - --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ - --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ - --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ - --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ - --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ - --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ - --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ - --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ - --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ - --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ - --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ - --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ - --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ - --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ - --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ - --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ - --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ - --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ - --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ - --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ - --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ - --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ - --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ - --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ - --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ - --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ - --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ - --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ - --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ - --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ - --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ - --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ - --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ - --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ - --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ - --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ - --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ - --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ - --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ - --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ - --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ - --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ - --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ - --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ - --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ - --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ - --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ - --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ - --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ - --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ - --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ - --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ - --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ - --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ - --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ - --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ - --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ - --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ - --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ - --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ - --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ - --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ - --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ - --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ - --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ - --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ - --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ - --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ - --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ - --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ - --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ - --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ - --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ - --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ - --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ - --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ - --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ - --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ - --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ - --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ - --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ - --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ - --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ - --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ - --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ - --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ - --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ - --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ - --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ - --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ - --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ - --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ - --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ - --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ - --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ - --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 - # via requests -cryptography==50.0.0 \ - --hash=sha256:031e2d5dd4bb9caa3ca9c82e5a197fd8ae680232cee62603d1a813f3f07e3d03 \ - --hash=sha256:031e2d5dd4bb9caa3ca9c82e5a197fd8ae680232cee62603d1a813f3f07e3d03 \ - --hash=sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7 \ - --hash=sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7 \ - --hash=sha256:07479a1cb08219ab719147e742e76090c9c773321959bb94946fffdd397a6437 \ - --hash=sha256:07479a1cb08219ab719147e742e76090c9c773321959bb94946fffdd397a6437 \ - --hash=sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987 \ - --hash=sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987 \ - --hash=sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025 \ - --hash=sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025 \ - --hash=sha256:11b74db56cdbe3cdee6e3f6982ecb70334fa10dce99ed58bf7894aaaa3b2a037 \ - --hash=sha256:11b74db56cdbe3cdee6e3f6982ecb70334fa10dce99ed58bf7894aaaa3b2a037 \ - --hash=sha256:12b9c6996425c76ea6c457ace4f3073e715b8c545add07cd1a8f3a4f90691269 \ - --hash=sha256:12b9c6996425c76ea6c457ace4f3073e715b8c545add07cd1a8f3a4f90691269 \ - --hash=sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105 \ - --hash=sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105 \ - --hash=sha256:19736989797678c6af1e55cd49055cdbcb55d8f6b5583ac5335f933aba9101dc \ - --hash=sha256:19736989797678c6af1e55cd49055cdbcb55d8f6b5583ac5335f933aba9101dc \ - --hash=sha256:1b4a266766514614f8aa60416e71f2fc6e575d36e7bdc90f644fadb2f4b75b95 \ - --hash=sha256:1b4a266766514614f8aa60416e71f2fc6e575d36e7bdc90f644fadb2f4b75b95 \ - --hash=sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b \ - --hash=sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b \ - --hash=sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47 \ - --hash=sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47 \ - --hash=sha256:3f5735ffe4996d28b809371756219f5354864902a3b9e7c0b9ee87041209fc9c \ - --hash=sha256:3f5735ffe4996d28b809371756219f5354864902a3b9e7c0b9ee87041209fc9c \ - --hash=sha256:49e7d93abdbd2990caced757e5fade25302f719c3c8fb6e6fff2dde98999fc41 \ - --hash=sha256:49e7d93abdbd2990caced757e5fade25302f719c3c8fb6e6fff2dde98999fc41 \ - --hash=sha256:5e34edd123674534acd70147f0ca331eaa2c74e6325fb2028c886aa26ba0b68c \ - --hash=sha256:5e34edd123674534acd70147f0ca331eaa2c74e6325fb2028c886aa26ba0b68c \ - --hash=sha256:62598a8a57f815db4c6259a4e97d857dab56697e7de8e8ab02352ab74da1995d \ - --hash=sha256:62598a8a57f815db4c6259a4e97d857dab56697e7de8e8ab02352ab74da1995d \ - --hash=sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7 \ - --hash=sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7 \ - --hash=sha256:6ba6a53445bd3cfa809ef3ef5f1589aa6ba08784a1d962bf47d0940e871dab1c \ - --hash=sha256:6ba6a53445bd3cfa809ef3ef5f1589aa6ba08784a1d962bf47d0940e871dab1c \ - --hash=sha256:6e7d61120573a7f2cd94cc095f9e81f6967c61ccdf194285aa143ecec8e0b708 \ - --hash=sha256:6e7d61120573a7f2cd94cc095f9e81f6967c61ccdf194285aa143ecec8e0b708 \ - --hash=sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef \ - --hash=sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef \ - --hash=sha256:80b63928fa35083b33966ce1efb70e5b9607181e49dcd1c22c8c005e319f667f \ - --hash=sha256:80b63928fa35083b33966ce1efb70e5b9607181e49dcd1c22c8c005e319f667f \ - --hash=sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f \ - --hash=sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f \ - --hash=sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a \ - --hash=sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a \ - --hash=sha256:8d89f3976b10b4ce31118de72329025f70d2c6ead14a8217c5514dd2c6d5a78f \ - --hash=sha256:8d89f3976b10b4ce31118de72329025f70d2c6ead14a8217c5514dd2c6d5a78f \ - --hash=sha256:8eb5e1172eb569ea8a872796576e6a67c276351728b6455d5beb01242b027c6a \ - --hash=sha256:8eb5e1172eb569ea8a872796576e6a67c276351728b6455d5beb01242b027c6a \ - --hash=sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a \ - --hash=sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a \ - --hash=sha256:910d11e1a385c654bf738bf3e6b8e6ed5de0f5610fcae2be9e5b398d8081d20e \ - --hash=sha256:910d11e1a385c654bf738bf3e6b8e6ed5de0f5610fcae2be9e5b398d8081d20e \ - --hash=sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3 \ - --hash=sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3 \ - --hash=sha256:9aa87839c383bdbab6ef865787a1fb877af8dd03464c4400322726feaaadfc6d \ - --hash=sha256:9aa87839c383bdbab6ef865787a1fb877af8dd03464c4400322726feaaadfc6d \ - --hash=sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3 \ - --hash=sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3 \ - --hash=sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f \ - --hash=sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f \ - --hash=sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae \ - --hash=sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae \ - --hash=sha256:bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30 \ - --hash=sha256:bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30 \ - --hash=sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9 \ - --hash=sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9 \ - --hash=sha256:c99c003e088647b8a5b7c145d6f78c335f6348332b62e142d411c4b63d1460b9 \ - --hash=sha256:c99c003e088647b8a5b7c145d6f78c335f6348332b62e142d411c4b63d1460b9 \ - --hash=sha256:ccdc4a71a4dabae05de219404f9f4abc38e3b58422177ff93d0da05967dafa07 \ - --hash=sha256:ccdc4a71a4dabae05de219404f9f4abc38e3b58422177ff93d0da05967dafa07 \ - --hash=sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba \ - --hash=sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba \ - --hash=sha256:d58c3db7cd6eed54e6c06744db55456b65ebd7492ddeae9c1e93cfca7aa857d3 \ - --hash=sha256:d58c3db7cd6eed54e6c06744db55456b65ebd7492ddeae9c1e93cfca7aa857d3 \ - --hash=sha256:d764dcf130c428ef66786f866dd750f53182bc608813489915e9fc106bb0c82f \ - --hash=sha256:d764dcf130c428ef66786f866dd750f53182bc608813489915e9fc106bb0c82f \ - --hash=sha256:df2a58a472f332225671c35b0a830208b86d004f82baa8530fa3782c85646533 \ - --hash=sha256:df2a58a472f332225671c35b0a830208b86d004f82baa8530fa3782c85646533 \ - --hash=sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5 \ - --hash=sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5 \ - --hash=sha256:ecfed7367f965a0328cfbdd70da860f15441f002f613185668c6e6ebf5a0ac11 \ - --hash=sha256:ecfed7367f965a0328cfbdd70da860f15441f002f613185668c6e6ebf5a0ac11 \ - --hash=sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9 \ - --hash=sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9 \ - --hash=sha256:f59e38625469987d7ef6d495323c55e7db6c212eaf6112267e0d3b565a2e9c9f \ - --hash=sha256:f59e38625469987d7ef6d495323c55e7db6c212eaf6112267e0d3b565a2e9c9f \ - --hash=sha256:f89831ef99dd7dd169ab06d63a831adb9e20a87aac6d380266bbda5823349169 \ - --hash=sha256:f89831ef99dd7dd169ab06d63a831adb9e20a87aac6d380266bbda5823349169 \ - --hash=sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645 \ - --hash=sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645 - # via secretstorage -docutils==0.22.3 \ - --hash=sha256:bd772e4aca73aff037958d44f2be5229ded4c09927fcf8690c577b66234d6ceb - # via readme-renderer -google-auth==2.43.0 \ - --hash=sha256:af628ba6fa493f75c7e9dbe9373d148ca9f4399b5ea29976519e0a3848eddd16 - # via keyrings-google-artifactregistry-auth -id==1.5.0 \ - --hash=sha256:f1434e1cef91f2cbb8a4ec64663d5a23b9ed43ef44c4c957d02583d61714c658 - # via twine -idna==3.15 \ - --hash=sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8 - # via requests -jaraco-classes==3.4.0 \ - --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 - # via keyring -jaraco-context==6.0.1 \ - --hash=sha256:f797fc481b490edb305122c9181830a3a5b76d84ef6d1aef2fb9b47ab956f9e4 - # via keyring -jaraco-functools==4.3.0 \ - --hash=sha256:227ff8ed6f7b8f62c56deff101545fa7543cf2c8e7b82a7c2116e672f29c26e8 - # via keyring -jeepney==0.9.0 \ - --hash=sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683 - # via - # keyring - # secretstorage -keyring==25.7.0 \ - --hash=sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f - # via - # -r build.in - # keyrings-google-artifactregistry-auth - # twine -keyrings-google-artifactregistry-auth==1.1.2 \ - --hash=sha256:e3f18b50fa945c786593014dc225810d191671d4f5f8e12d9259e39bad3605a3 - # via -r build.in -markdown-it-py==4.0.0 \ - --hash=sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147 - # via rich -mdurl==0.1.2 \ - --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 - # via markdown-it-py -more-itertools==10.8.0 \ - --hash=sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b - # via - # jaraco-classes - # jaraco-functools -nh3==0.3.2 \ - --hash=sha256:019ecbd007536b67fdf76fab411b648fb64e2257ca3262ec80c3425c24028c80 \ - --hash=sha256:03d617e5c8aa7331bd2659c654e021caf9bba704b109e7b2b28b039a00949fe5 \ - --hash=sha256:0dca4365db62b2d71ff1620ee4f800c4729849906c5dd504ee1a7b2389558e31 \ - --hash=sha256:0fe7ee035dd7b2290715baf29cb27167dddd2ff70ea7d052c958dbd80d323c99 \ - --hash=sha256:13398e676a14d6233f372c75f52d5ae74f98210172991f7a3142a736bd92b131 \ - --hash=sha256:169db03df90da63286e0560ea0efa9b6f3b59844a9735514a1d47e6bb2c8c61b \ - --hash=sha256:1710f3901cd6440ca92494ba2eb6dc260f829fa8d9196b659fa10de825610ce0 \ - --hash=sha256:1f9ba555a797dbdcd844b89523f29cdc90973d8bd2e836ea6b962cf567cadd93 \ - --hash=sha256:2ab70e8c6c7d2ce953d2a58102eefa90c2d0a5ed7aa40c7e29a487bc5e613131 \ - --hash=sha256:2c9850041b77a9147d6bbd6dbbf13eeec7009eb60b44e83f07fcb2910075bf9b \ - --hash=sha256:403c11563e50b915d0efdb622866d1d9e4506bce590ef7da57789bf71dd148b5 \ - --hash=sha256:45c953e57028c31d473d6b648552d9cab1efe20a42ad139d78e11d8f42a36130 \ - --hash=sha256:562da3dca7a17f9077593214a9781a94b8d76de4f158f8c895e62f09573945fe \ - --hash=sha256:6d66f41672eb4060cf87c037f760bdbc6847852ca9ef8e9c5a5da18f090abf87 \ - --hash=sha256:7064ccf5ace75825bd7bf57859daaaf16ed28660c1c6b306b649a9eda4b54b1e \ - --hash=sha256:72d67c25a84579f4a432c065e8b4274e53b7cf1df8f792cf846abfe2c3090866 \ - --hash=sha256:7bb18403f02b655a1bbe4e3a4696c2ae1d6ae8f5991f7cacb684b1ae27e6c9f7 \ - --hash=sha256:91e9b001101fb4500a2aafe3e7c92928d85242d38bf5ac0aba0b7480da0a4cd6 \ - --hash=sha256:a40202fd58e49129764f025bbaae77028e420f1d5b3c8e6f6fd3a6490d513868 \ - --hash=sha256:c8745454cdd28bbbc90861b80a0111a195b0e3961b9fa2e672be89eb199fa5d8 \ - --hash=sha256:cf5964d54edd405e68583114a7cba929468bcd7db5e676ae38ee954de1cfc104 \ - --hash=sha256:d18957a90806d943d141cc5e4a0fefa1d77cf0d7a156878bf9a66eed52c9cc7d \ - --hash=sha256:dce4248edc427c9b79261f3e6e2b3ecbdd9b88c267012168b4a7b3fc6fd41d13 \ - --hash=sha256:f2f55c4d2d5a207e74eefe4d828067bbb01300e06e2a7436142f915c5928de07 \ - --hash=sha256:f97f8b25cb2681d25e2338148159447e4d689aafdccfcf19e61ff7db3905768a - # via readme-renderer -packaging==25.0 \ - --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 - # via - # build - # twine - # wheel -pluggy==1.6.0 \ - --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 - # via keyrings-google-artifactregistry-auth -pyasn1==0.6.4 \ - --hash=sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.2 \ - --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a - # via google-auth -pycparser==2.23 \ - --hash=sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934 - # via cffi -pygments==2.19.2 \ - --hash=sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b - # via - # readme-renderer - # rich -pyproject-hooks==1.2.0 \ - --hash=sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913 - # via build -readme-renderer==44.0 \ - --hash=sha256:2fbca89b81a08526aadf1357a8c2ae889ec05fb03f5da67f9769c9a592166151 - # via twine -requests==2.33.0 \ - --hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b - # via - # id - # keyrings-google-artifactregistry-auth - # requests-toolbelt - # twine -requests-toolbelt==1.0.0 \ - --hash=sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06 - # via twine -rfc3986==2.0.0 \ - --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd - # via twine -rich==14.2.0 \ - --hash=sha256:76bc51fe2e57d2b1be1f96c524b890b816e334ab4c1e45888799bfaab0021edd - # via twine -rsa==4.9.1 \ - --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 - # via google-auth -secretstorage==3.5.0 \ - --hash=sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137 - # via keyring -twine==6.2.0 \ - --hash=sha256:418ebf08ccda9a8caaebe414433b0ba5e25eb5e4a927667122fbe8f829f985d8 - # via -r build.in -urllib3==2.7.0 \ - --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 - # via - # requests - # twine -wheel==0.46.2 \ - --hash=sha256:33ae60725d69eaa249bc1982e739943c23b34b58d51f1cb6253453773aca6e65 - # via -r build.in diff --git a/.kokoro/samples/lint/common.cfg b/.kokoro/samples/lint/common.cfg new file mode 100644 index 00000000000..b4d26c1f982 --- /dev/null +++ b/.kokoro/samples/lint/common.cfg @@ -0,0 +1,34 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "lint" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-dataframes/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/lint/continuous.cfg b/.kokoro/samples/lint/continuous.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/lint/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/lint/periodic.cfg b/.kokoro/samples/lint/periodic.cfg new file mode 100644 index 00000000000..50fec964973 --- /dev/null +++ b/.kokoro/samples/lint/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/.kokoro/samples/lint/presubmit.cfg b/.kokoro/samples/lint/presubmit.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/lint/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.10/common.cfg b/.kokoro/samples/python3.10/common.cfg new file mode 100644 index 00000000000..8f9c66c571b --- /dev/null +++ b/.kokoro/samples/python3.10/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.10" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-310" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-dataframes/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.10/continuous.cfg b/.kokoro/samples/python3.10/continuous.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.10/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.10/periodic-head.cfg b/.kokoro/samples/python3.10/periodic-head.cfg new file mode 100644 index 00000000000..123a35fbd3d --- /dev/null +++ b/.kokoro/samples/python3.10/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/continuous/continuous-doctest-linux.cfg b/.kokoro/samples/python3.10/periodic.cfg similarity index 53% rename from .kokoro/continuous/continuous-doctest-linux.cfg rename to .kokoro/samples/python3.10/periodic.cfg index af74ca0fbcd..71cd1e597e3 100644 --- a/.kokoro/continuous/continuous-doctest-linux.cfg +++ b/.kokoro/samples/python3.10/periodic.cfg @@ -1,6 +1,6 @@ # Format: //devtools/kokoro/config/proto/build.proto env_vars: { - key: "NOX_SESSION" - value: "doctest" + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" } diff --git a/.kokoro/samples/python3.10/presubmit.cfg b/.kokoro/samples/python3.10/presubmit.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.10/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.11/common.cfg b/.kokoro/samples/python3.11/common.cfg new file mode 100644 index 00000000000..1bba39114aa --- /dev/null +++ b/.kokoro/samples/python3.11/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.11" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-311" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-dataframes/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.11/continuous.cfg b/.kokoro/samples/python3.11/continuous.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.11/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.11/periodic-head.cfg b/.kokoro/samples/python3.11/periodic-head.cfg new file mode 100644 index 00000000000..123a35fbd3d --- /dev/null +++ b/.kokoro/samples/python3.11/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.11/periodic.cfg b/.kokoro/samples/python3.11/periodic.cfg new file mode 100644 index 00000000000..71cd1e597e3 --- /dev/null +++ b/.kokoro/samples/python3.11/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/.kokoro/samples/python3.11/presubmit.cfg b/.kokoro/samples/python3.11/presubmit.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.11/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.12/common.cfg b/.kokoro/samples/python3.12/common.cfg new file mode 100644 index 00000000000..abf83e196db --- /dev/null +++ b/.kokoro/samples/python3.12/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.12" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-312" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-dataframes/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.12/continuous.cfg b/.kokoro/samples/python3.12/continuous.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.12/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.12/periodic-head.cfg b/.kokoro/samples/python3.12/periodic-head.cfg new file mode 100644 index 00000000000..123a35fbd3d --- /dev/null +++ b/.kokoro/samples/python3.12/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.12/periodic.cfg b/.kokoro/samples/python3.12/periodic.cfg new file mode 100644 index 00000000000..71cd1e597e3 --- /dev/null +++ b/.kokoro/samples/python3.12/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/.kokoro/samples/python3.12/presubmit.cfg b/.kokoro/samples/python3.12/presubmit.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.12/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.13/common.cfg b/.kokoro/samples/python3.13/common.cfg new file mode 100644 index 00000000000..6a5d9a20800 --- /dev/null +++ b/.kokoro/samples/python3.13/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.13" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-313" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-dataframes/.kokoro/trampoline_v2.sh" diff --git a/.kokoro/samples/python3.13/continuous.cfg b/.kokoro/samples/python3.13/continuous.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.13/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.13/periodic-head.cfg b/.kokoro/samples/python3.13/periodic-head.cfg new file mode 100644 index 00000000000..123a35fbd3d --- /dev/null +++ b/.kokoro/samples/python3.13/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.13/periodic.cfg b/.kokoro/samples/python3.13/periodic.cfg new file mode 100644 index 00000000000..71cd1e597e3 --- /dev/null +++ b/.kokoro/samples/python3.13/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/.kokoro/samples/python3.13/presubmit.cfg b/.kokoro/samples/python3.13/presubmit.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.13/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.7/common.cfg b/.kokoro/samples/python3.7/common.cfg new file mode 100644 index 00000000000..09d7af02ba9 --- /dev/null +++ b/.kokoro/samples/python3.7/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.7" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-py37" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-dataframes/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.7/continuous.cfg b/.kokoro/samples/python3.7/continuous.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.7/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.7/periodic-head.cfg b/.kokoro/samples/python3.7/periodic-head.cfg new file mode 100644 index 00000000000..123a35fbd3d --- /dev/null +++ b/.kokoro/samples/python3.7/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.7/periodic.cfg b/.kokoro/samples/python3.7/periodic.cfg new file mode 100644 index 00000000000..71cd1e597e3 --- /dev/null +++ b/.kokoro/samples/python3.7/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/.kokoro/samples/python3.7/presubmit.cfg b/.kokoro/samples/python3.7/presubmit.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.7/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.8/common.cfg b/.kokoro/samples/python3.8/common.cfg new file mode 100644 index 00000000000..976d9ce8c5c --- /dev/null +++ b/.kokoro/samples/python3.8/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.8" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-py38" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-dataframes/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.8/continuous.cfg b/.kokoro/samples/python3.8/continuous.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.8/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.8/periodic-head.cfg b/.kokoro/samples/python3.8/periodic-head.cfg new file mode 100644 index 00000000000..123a35fbd3d --- /dev/null +++ b/.kokoro/samples/python3.8/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.8/periodic.cfg b/.kokoro/samples/python3.8/periodic.cfg new file mode 100644 index 00000000000..71cd1e597e3 --- /dev/null +++ b/.kokoro/samples/python3.8/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/.kokoro/samples/python3.8/presubmit.cfg b/.kokoro/samples/python3.8/presubmit.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.8/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.9/common.cfg b/.kokoro/samples/python3.9/common.cfg new file mode 100644 index 00000000000..603cfffa280 --- /dev/null +++ b/.kokoro/samples/python3.9/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.9" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-py39" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-dataframes/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.9/continuous.cfg b/.kokoro/samples/python3.9/continuous.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.9/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.9/periodic-head.cfg b/.kokoro/samples/python3.9/periodic-head.cfg new file mode 100644 index 00000000000..123a35fbd3d --- /dev/null +++ b/.kokoro/samples/python3.9/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-dataframes/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.9/periodic.cfg b/.kokoro/samples/python3.9/periodic.cfg new file mode 100644 index 00000000000..71cd1e597e3 --- /dev/null +++ b/.kokoro/samples/python3.9/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/.kokoro/samples/python3.9/presubmit.cfg b/.kokoro/samples/python3.9/presubmit.cfg new file mode 100644 index 00000000000..a1c8d9759c8 --- /dev/null +++ b/.kokoro/samples/python3.9/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/test-samples-against-head.sh b/.kokoro/test-samples-against-head.sh new file mode 100755 index 00000000000..e9d8bd79a64 --- /dev/null +++ b/.kokoro/test-samples-against-head.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A customized test runner for samples. +# +# For periodic builds, you can specify this file for testing against head. + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh new file mode 100755 index 00000000000..53e365bc4e7 --- /dev/null +++ b/.kokoro/test-samples-impl.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +# Exit early if samples don't exist +if ! find samples -name 'requirements.txt' | grep -q .; then + echo "No tests run. './samples/**/requirements.txt' not found" + exit 0 +fi + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Install nox +# `virtualenv==20.26.6` is added for Python 3.7 compatibility +python3.9 -m pip install --upgrade --quiet nox virtualenv==20.26.6 + +# Use secrets acessor service account to get secrets +if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then + gcloud auth activate-service-account \ + --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ + --project="cloud-devrel-kokoro-resources" +fi + +# This script will create 3 files: +# - testing/test-env.sh +# - testing/service-account.json +# - testing/client-secrets.json +./scripts/decrypt-secrets.sh + +source ./testing/test-env.sh +export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json + +# For cloud-run session, we activate the service account for gcloud sdk. +gcloud auth activate-service-account \ + --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" + +export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json + +echo -e "\n******************** TESTING PROJECTS ********************" + +# Switch to 'fail at end' to allow all tests to complete before exiting. +set +e +# Use RTN to return a non-zero value if the test fails. +RTN=0 +ROOT=$(pwd) +# Find all requirements.txt in the samples directory (may break on whitespace). +for file in samples/**/requirements.txt; do + cd "$ROOT" + # Navigate to the project folder. + file=$(dirname "$file") + cd "$file" + + echo "------------------------------------------------------------" + echo "- testing $file" + echo "------------------------------------------------------------" + + # Use nox to execute the tests for the project. + python3.9 -m nox -s "$RUN_TESTS_SESSION" + EXIT=$? + + # If this is a periodic build, send the test log to the FlakyBot. + # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. + if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + fi + + if [[ $EXIT -ne 0 ]]; then + RTN=1 + echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" + else + echo -e "\n Testing completed.\n" + fi + +done +cd "$ROOT" + +# Workaround for Kokoro permissions issue: delete secrets +rm testing/{test-env.sh,client-secrets.json,service-account.json} + +exit "$RTN" diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh new file mode 100755 index 00000000000..7933d820149 --- /dev/null +++ b/.kokoro/test-samples.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The default test runner for samples. +# +# For periodic builds, we rewinds the repo to the latest release, and +# run test-samples-impl.sh. + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +# Run periodic samples tests at latest release +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + # preserving the test runner implementation. + cp .kokoro/test-samples-impl.sh "${TMPDIR}/test-samples-impl.sh" + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + echo "Now we rewind the repo back to the latest release..." + LATEST_RELEASE=$(git describe --abbrev=0 --tags) + git checkout $LATEST_RELEASE + echo "The current head is: " + echo $(git rev-parse --verify HEAD) + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + # move back the test runner implementation if there's no file. + if [ ! -f .kokoro/test-samples-impl.sh ]; then + cp "${TMPDIR}/test-samples-impl.sh" .kokoro/test-samples-impl.sh + fi +fi + +exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh new file mode 100755 index 00000000000..48f79699706 --- /dev/null +++ b/.kokoro/trampoline.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Always run the cleanup script, regardless of the success of bouncing into +# the container. +function cleanup() { + chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + echo "cleanup"; +} +trap cleanup EXIT + +$(dirname $0)/populate-secrets.sh # Secret Manager secrets. +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" \ No newline at end of file diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh new file mode 100755 index 00000000000..35fa529231d --- /dev/null +++ b/.kokoro/trampoline_v2.sh @@ -0,0 +1,487 @@ +#!/usr/bin/env bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# To run this script, first download few files from gcs to /dev/shm. +# (/dev/shm is passed into the container as KOKORO_GFILE_DIR). +# +# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm +# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm +# +# Then run the script. +# .kokoro/trampoline_v2.sh +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. + + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.5" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For FlakyBot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" +else + PROJECT_ROOT="$(repo_root $(pwd))" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}:latest" | grep "${TRAMPOLINE_IMAGE}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" diff --git a/.librarian/state.yaml b/.librarian/state.yaml new file mode 100644 index 00000000000..e37895f78d5 --- /dev/null +++ b/.librarian/state.yaml @@ -0,0 +1,11 @@ +image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:c8612d3fffb3f6a32353b2d1abd16b61e87811866f7ec9d65b59b02eb452a620 +libraries: + - id: bigframes + version: 2.32.0 + last_generated_commit: "" + apis: [] + source_roots: + - . + preserve_regex: [] + remove_regex: [] + tag_format: v{version} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000..096bdeb2a78 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +default_install_hook_types: +- pre-commit +- commit-msg + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + exclude: "^tests/unit/core/compile/sqlglot/.*snapshots" + - id: check-yaml +- repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + name: isort (python) +- repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black +- repo: https://github.com/pycqa/flake8 + rev: 7.1.2 + hooks: + - id: flake8 +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.15.0 + hooks: + - id: mypy + additional_dependencies: [types-requests, types-tabulate, types-PyYAML, pandas-stubs<=2.2.3.241126] + exclude: "^third_party" + args: ["--check-untyped-defs", "--explicit-package-bases", "--ignore-missing-imports"] +- repo: https://github.com/biomejs/pre-commit + rev: v2.2.4 + hooks: + - id: biome-check + files: '\.(js|css)$' +- repo: https://github.com/compilerla/conventional-pre-commit + rev: fdde5f0251edbfc554795afdd6df71826d6602f3 + hooks: + - id: conventional-pre-commit + stages: [commit-msg] + args: [] diff --git a/.repo-metadata.json b/.repo-metadata.json index b988476c181..0efaa967d2c 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -1,9 +1,16 @@ { - "client_documentation": "https://googleapis.dev/python/bigframes/latest", - "distribution_name": "bigframes", + "name": "bigframes", + "name_pretty": "A unified Python API in BigQuery", + "product_documentation": "https://cloud.google.com/bigquery", + "client_documentation": "https://cloud.google.com/python/docs/reference/bigframes/latest", + "issue_tracker": "https://github.com/googleapis/python-bigquery-dataframes/issues", + "release_level": "preview", "language": "python", "library_type": "INTEGRATION", - "name": "bigframes", - "release_level": "stable", - "repo": "googleapis/google-cloud-python" -} \ No newline at end of file + "repo": "googleapis/python-bigquery-dataframes", + "distribution_name": "bigframes", + "api_id": "bigquery.googleapis.com", + "default_version": "", + "codeowner_team": "@googleapis/api-bigquery-dataframe", + "api_shortname": "bigquery" +} diff --git a/.trampolinerc b/.trampolinerc new file mode 100644 index 00000000000..0080152373d --- /dev/null +++ b/.trampolinerc @@ -0,0 +1,61 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add required env vars here. +required_envvars+=( +) + +# Add env vars which are passed down into the container here. +pass_down_envvars+=( + "NOX_SESSION" + ############### + # Docs builds + ############### + "STAGING_BUCKET" + "V2_STAGING_BUCKET" + ################## + # Samples builds + ################## + "INSTALL_LIBRARY_FROM_SOURCE" + "RUN_TESTS_SESSION" + "BUILD_SPECIFIC_GCLOUD_PROJECT" + # Target directories. + "RUN_TESTS_DIRS" + # The nox session to run. + "RUN_TESTS_SESSION" +) + +# Prevent unintentional override on the default image. +if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \ + [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." + exit 1 +fi + +# Define the default value if it makes sense. +if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then + TRAMPOLINE_IMAGE_UPLOAD="" +fi + +if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + TRAMPOLINE_IMAGE="" +fi + +if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then + TRAMPOLINE_DOCKERFILE="" +fi + +if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then + TRAMPOLINE_BUILD_FILE="" +fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 6584f20bb87..b49afe535c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,365 +4,6 @@ [1]: https://pypi.org/project/bigframes/#history -## [2.48.0](https://github.com/googleapis/google-cloud-python/compare/bigframes-v2.47.0...bigframes-v2.48.0) (2026-08-12) - - -### Features - -* **bigframes:** Transpiler supports more string ops ([#17693](https://github.com/googleapis/google-cloud-python/issues/17693)) ([7d2bc21](https://github.com/googleapis/google-cloud-python/commit/7d2bc213caabb23eef04e8242e1a05351013d217)) - - -### Bug Fixes - -* **bigframes:** fix field name typos for ai.generate* functions ([#17983](https://github.com/googleapis/google-cloud-python/issues/17983)) ([1b5c48b](https://github.com/googleapis/google-cloud-python/commit/1b5c48b851d92d77f9fb4ce5beea57a22281445d)) -* **bigframes:** resolve session-scoped API method logging ([#18076](https://github.com/googleapis/google-cloud-python/issues/18076)) ([ace618b](https://github.com/googleapis/google-cloud-python/commit/ace618b66ca74199d7269f95ef94d5e5a1effac8)) -* **bigframes:** update GeminiTextGenerator default model to gemini-2.5-flash ([#18060](https://github.com/googleapis/google-cloud-python/issues/18060)) ([5770ff6](https://github.com/googleapis/google-cloud-python/commit/5770ff6bb4bd5601e8596a1100de85fec891e02a)) -* bump @angular/compiler, @angular/common, @angular/core, @angular/forms, @angular/platform-browser, @angular/router and @angular/compiler-cli in /packages/bigframes/bigframes/display/table_widget_angular ([#17992](https://github.com/googleapis/google-cloud-python/issues/17992)) ([346aaab](https://github.com/googleapis/google-cloud-python/commit/346aaab68b04fe5d280a5b49aee376d67b4233ad)) -* bump fast-uri from 3.1.4 to 3.1.5 in /packages/bigframes/bigframes/display/table_widget_angular ([#17989](https://github.com/googleapis/google-cloud-python/issues/17989)) ([3b3f3f4](https://github.com/googleapis/google-cloud-python/commit/3b3f3f4743b64e9ce483392930aa91efa6f022cb)) -* bump hono from 4.12.31 to 4.13.1 in /packages/bigframes/bigframes/display/table_widget_angular ([#18032](https://github.com/googleapis/google-cloud-python/issues/18032)) ([cddf35b](https://github.com/googleapis/google-cloud-python/commit/cddf35b4df9f4d3466a91a82cd703be727b37b3b)) -* bump ip-address and express-rate-limit in /packages/bigframes/bigframes/display/table_widget_angular ([#17985](https://github.com/googleapis/google-cloud-python/issues/17985)) ([02ed656](https://github.com/googleapis/google-cloud-python/commit/02ed656410a9682d308eaf9d3086fe76cf7e27d0)) -* bump undici and @angular/build in /packages/bigframes/bigframes/display/table_widget_angular ([#17986](https://github.com/googleapis/google-cloud-python/issues/17986)) ([6938061](https://github.com/googleapis/google-cloud-python/commit/69380617aa68f565e0885911c87520abe216c358)) -* bump undici from 7.25.0 to 7.29.0 in /packages/bigframes/tests/js ([#17987](https://github.com/googleapis/google-cloud-python/issues/17987)) ([65a3571](https://github.com/googleapis/google-cloud-python/commit/65a3571325be9497c17d1c640b3cd56d8a7766e2)) - -## [2.47.0](https://github.com/googleapis/google-cloud-python/compare/bigframes-v2.46.0...bigframes-v2.47.0) (2026-08-03) - - -### Features - -* **bigframes:** add ai.embed and ai.similarity to bigquery accessor ([#17927](https://github.com/googleapis/google-cloud-python/issues/17927)) ([2d1372b](https://github.com/googleapis/google-cloud-python/commit/2d1372b25b871aaebba0296f516dbbdd9ef43a37)) - - -### Bug Fixes - -* **bigframes:** fix mypy errors in _magics.py related to get_ipython ([#17979](https://github.com/googleapis/google-cloud-python/issues/17979)) ([24d955e](https://github.com/googleapis/google-cloud-python/commit/24d955e7573acf5e2941f9d10b342b171d9def24)) -* bump brace-expansion from 5.0.6 to 5.0.7 in /packages/bigframes/bigframes/display/table_widget_angular ([#17794](https://github.com/googleapis/google-cloud-python/issues/17794)) ([2df1bb5](https://github.com/googleapis/google-cloud-python/commit/2df1bb5cb53906bfc41de92c10fdb5a8920ce36f)) -* bump fast-uri from 3.1.1 to 3.1.4 in /packages/bigframes/bigframes/display/table_widget_angular ([#17828](https://github.com/googleapis/google-cloud-python/issues/17828)) ([8ce495a](https://github.com/googleapis/google-cloud-python/commit/8ce495ab51d35cc56332f27589ce33b035215abf)) -* bump hono from 4.12.16 to 4.12.31 in /packages/bigframes/bigframes/display/table_widget_angular ([#17829](https://github.com/googleapis/google-cloud-python/issues/17829)) ([6d6fa39](https://github.com/googleapis/google-cloud-python/commit/6d6fa399b1578dd6c0ed7b2284b35f6bb0922da8)) -* bump immutable from 5.1.7 to 5.1.9 in /packages/bigframes/bigframes/display/table_widget_angular ([#17830](https://github.com/googleapis/google-cloud-python/issues/17830)) ([6abb1da](https://github.com/googleapis/google-cloud-python/commit/6abb1da519e7d4acdbdd531982530047fdafe854)) -* bump postcss from 8.5.14 to 8.5.23 in /packages/bigframes/bigframes/display/table_widget_angular ([#17908](https://github.com/googleapis/google-cloud-python/issues/17908)) ([e68eb8f](https://github.com/googleapis/google-cloud-python/commit/e68eb8ff03948b4a694aca465b69543ae0be6c27)) -* bump tar from 7.5.16 to 7.5.20 in /packages/bigframes/bigframes/display/table_widget_angular ([#17793](https://github.com/googleapis/google-cloud-python/issues/17793)) ([3502d41](https://github.com/googleapis/google-cloud-python/commit/3502d4183d9864e4310b3daf063ec90be9969e25)) -* require Protobuf 6.33.5+ ([#17743](https://github.com/googleapis/google-cloud-python/issues/17743)) ([d267342](https://github.com/googleapis/google-cloud-python/commit/d26734293c23f06ccce048f7d9b0fa365e813410)) - -## [2.46.0](https://github.com/googleapis/google-cloud-python/compare/bigframes-v2.45.0...bigframes-v2.46.0) (2026-07-16) - - -### Features - -* **bigframes:** Support groupby.agg/transform with udf transpiler ([#17613](https://github.com/googleapis/google-cloud-python/issues/17613)) ([cae94f9](https://github.com/googleapis/google-cloud-python/commit/cae94f99121d7708671a35acf82616cfe378cccb)) -* **bigframes:** support offset-based column access via iloc ([#17367](https://github.com/googleapis/google-cloud-python/issues/17367)) ([4253fab](https://github.com/googleapis/google-cloud-python/commit/4253fab07ccdb2b94e247f8dade793828754b88b)) - - -### Bug Fixes - -* **bigframes:** Fix sqlglot backend regressions ([#17655](https://github.com/googleapis/google-cloud-python/issues/17655)) ([91f93bc](https://github.com/googleapis/google-cloud-python/commit/91f93bcd7b71b6cea62f506ed684500cec1eb6bb)) -* bump gradio from 6.15.0 to 6.15.1 in /packages/bigframes ([#17712](https://github.com/googleapis/google-cloud-python/issues/17712)) ([a85d59f](https://github.com/googleapis/google-cloud-python/commit/a85d59f39998d94cbac8d5e98547f18b3cc5e5be)) -* bump mistune from 3.2.1 to 3.3.0 in /packages/bigframes ([#17694](https://github.com/googleapis/google-cloud-python/issues/17694)) ([e5f7fef](https://github.com/googleapis/google-cloud-python/commit/e5f7fef31c2bbe5f559f4c79fdaf4ebcf6e1bd3f)) -* bump soupsieve from 2.7 to 2.8.4 in /packages/bigframes ([#17695](https://github.com/googleapis/google-cloud-python/issues/17695)) ([635da34](https://github.com/googleapis/google-cloud-python/commit/635da3453b2ba78b8abea43c554a055257f33aa1)) -* bump transformers from 5.3.0 to 5.5.0 in /packages/bigframes ([#17700](https://github.com/googleapis/google-cloud-python/issues/17700)) ([4b049c4](https://github.com/googleapis/google-cloud-python/commit/4b049c4eb8dc1ec91320b55fe515c339cd448af3)) -* emit bracketed inline array syntax for scalar subquery expressions ([#17716](https://github.com/googleapis/google-cloud-python/issues/17716)) ([ce5fd50](https://github.com/googleapis/google-cloud-python/commit/ce5fd500b68c16f56ea8066d8a6fa4b0b8d92081)) - - -### Documentation - -* make landing page quickstart runnable ([fc423c8](https://github.com/googleapis/google-cloud-python/commit/fc423c809cc80168f45fee795d5db5dc7a571fb1)) -* make landing page quickstart runnable ([#17687](https://github.com/googleapis/google-cloud-python/issues/17687)) ([fc423c8](https://github.com/googleapis/google-cloud-python/commit/fc423c809cc80168f45fee795d5db5dc7a571fb1)) - -## [2.45.0](https://github.com/googleapis/google-cloud-python/compare/bigframes-v2.44.0...bigframes-v2.45.0) (2026-07-08) - - -### Features - -* **bigframes:** add ai.classify, ai.score, ai.if_ to the df bq accessor ([#17569](https://github.com/googleapis/google-cloud-python/issues/17569)) ([4f94be8](https://github.com/googleapis/google-cloud-python/commit/4f94be8f01971380f0fb5b433ab33d7b4cb7176d)) -* **bigframes:** Enable local udf execution ([#17588](https://github.com/googleapis/google-cloud-python/issues/17588)) ([b8ed34c](https://github.com/googleapis/google-cloud-python/commit/b8ed34cc05101c58ef285822d86298cd0f56613c)) -* **bigframes:** UDF transpiler handles some control flow ([#17558](https://github.com/googleapis/google-cloud-python/issues/17558)) ([a8cbde3](https://github.com/googleapis/google-cloud-python/commit/a8cbde39199f838a43ebc8b938ad722595655abd)) -* support gemini-3.x models ([#17615](https://github.com/googleapis/google-cloud-python/issues/17615)) ([5d0efa3](https://github.com/googleapis/google-cloud-python/commit/5d0efa3cb86568a33a5b3097f30733d39fcbef66)) - - -### Bug Fixes - -* bump gdal from 3.13.0 to 3.13.1 in /packages/bigframes ([#17609](https://github.com/googleapis/google-cloud-python/issues/17609)) ([0f4bfed](https://github.com/googleapis/google-cloud-python/commit/0f4bfed4685a362f6487cd4cb02ead3c0dde85c9)) -* bump gradio from 5.39.0 to 6.15.0 in /packages/bigframes ([#17619](https://github.com/googleapis/google-cloud-python/issues/17619)) ([bddda6a](https://github.com/googleapis/google-cloud-python/commit/bddda6a11a9c9bcce2d9e8b665b63d47f49f894f)) -* bump transformers from 4.54.1 to 5.3.0 in /packages/bigframes ([#17610](https://github.com/googleapis/google-cloud-python/issues/17610)) ([10eca3f](https://github.com/googleapis/google-cloud-python/commit/10eca3f4b6578c9451b06cdb2889561563fa8d0d)) - -## [2.44.0](https://github.com/googleapis/google-cloud-python/compare/bigframes-v2.43.0...bigframes-v2.44.0) (2026-06-25) - - -### Features - -* add date functions to `bigframes.bigquery` module ([#17514](https://github.com/googleapis/google-cloud-python/issues/17514)) ([e5d2e35](https://github.com/googleapis/google-cloud-python/commit/e5d2e35db94373ca395976fd755c2bc7e0a060bd)) -* **bigframes:** add AI TVFs to the pandas bq accessor ([#17402](https://github.com/googleapis/google-cloud-python/issues/17402)) ([ee74e31](https://github.com/googleapis/google-cloud-python/commit/ee74e3140a2e11936c36714a27393c3072bed6c7)) -* Experimental transpilation of unannotated python callables ([#17419](https://github.com/googleapis/google-cloud-python/issues/17419)) ([ea9aad9](https://github.com/googleapis/google-cloud-python/commit/ea9aad9a43c306ab109054183b257e6c41a1b2e6)) -* support gemini-3.x models in loader and update default model to gemini-3.5-flash ([#17557](https://github.com/googleapis/google-cloud-python/issues/17557)) ([3619b29](https://github.com/googleapis/google-cloud-python/commit/3619b29e10ae04623d101808cb98be5edbb483b4)) -* support interactive execution of deferred DataFrames in TableWidget ([#17486](https://github.com/googleapis/google-cloud-python/issues/17486)) ([421eebd](https://github.com/googleapis/google-cloud-python/commit/421eebdb31d526a6d5ba27c433cf2803d7619be3)) - - -### Bug Fixes - -* avoid invalid CAST(NULL AS NULL) in SQLGlot compiler ([#17487](https://github.com/googleapis/google-cloud-python/issues/17487)) ([3b79caa](https://github.com/googleapis/google-cloud-python/commit/3b79caa8f40f61ccd7c655542e9f242f34e068e2)) -* **bigframes:** world-readable temp zip in create_cloud_function ([#17522](https://github.com/googleapis/google-cloud-python/issues/17522)) ([e726878](https://github.com/googleapis/google-cloud-python/commit/e7268785c6736c10c1337160b4d8606975062637)) -* bump @angular/common, @angular/forms, @angular/platform-browser and @angular/router in /packages/bigframes/bigframes/display/table_widget_angular ([#17525](https://github.com/googleapis/google-cloud-python/issues/17525)) ([2f893b1](https://github.com/googleapis/google-cloud-python/commit/2f893b1b53e7394655fd204d1f8a138212ad8227)) -* bump langsmith from 0.8.0 to 0.8.18 in /packages/bigframes ([#17518](https://github.com/googleapis/google-cloud-python/issues/17518)) ([f23063f](https://github.com/googleapis/google-cloud-python/commit/f23063f9182cdec868c16afb80304892850fbe88)) -* bump msgpack from 1.1.1 to 1.2.1 in /packages/bigframes ([#17520](https://github.com/googleapis/google-cloud-python/issues/17520)) ([36b5b7e](https://github.com/googleapis/google-cloud-python/commit/36b5b7ebb01030a2d0f10d49fe4827ddc79dde9a)) -* bump undici and @angular/build in /packages/bigframes/bigframes/display/table_widget_angular ([#17519](https://github.com/googleapis/google-cloud-python/issues/17519)) ([6fc45e3](https://github.com/googleapis/google-cloud-python/commit/6fc45e3790c5a248dcec4b74799834c7b9219ef0)) -* handle empty endpoints during cloud function reuse ([#17501](https://github.com/googleapis/google-cloud-python/issues/17501)) ([4f5593a](https://github.com/googleapis/google-cloud-python/commit/4f5593a520b5afdeb02cc28f19a9596dbc35a90f)) - - -### Documentation - -* ensure that PlotAccessor is included in the API reference ([#17513](https://github.com/googleapis/google-cloud-python/issues/17513)) ([6febabf](https://github.com/googleapis/google-cloud-python/commit/6febabf795106a0c336dc905fc23da88d8cc94a0)) - -## [2.43.0](https://github.com/googleapis/google-cloud-python/compare/bigframes-v2.42.0...bigframes-v2.43.0) (2026-06-12) - - -### Documentation - -* add a notebook explaining bqsql magics cell chaining (#17216) ([1a0de4a7701b7fdf4c2593b1960f1194ebc49793](https://github.com/googleapis/google-cloud-python/commit/1a0de4a7701b7fdf4c2593b1960f1194ebc49793)) - - -### Features - -* add `bigframes.bigquery.bit_count` and conversion scalar function (#17433) ([7f29823fadb3cff42dbe666f8c7aa33bab3c7021](https://github.com/googleapis/google-cloud-python/commit/7f29823fadb3cff42dbe666f8c7aa33bab3c7021)) - - -### Bug Fixes - -* preserve aliases on cast columns and fix star selection in sqlglot (#17394) (#17455) ([145034a345eb3e14ea3f23dfcafa3d2409a09067](https://github.com/googleapis/google-cloud-python/commit/145034a345eb3e14ea3f23dfcafa3d2409a09067)) -* bump pyarrow from 15.0.2 to 23.0.1 in /packages/bigframes (#17386) ([f59c2b2aa61316cf04b650933036ef50f6a1f08c](https://github.com/googleapis/google-cloud-python/commit/f59c2b2aa61316cf04b650933036ef50f6a1f08c)) -* improve error message when unescaped `{` are found in SQL cells (#17346) ([3a90cc8e867c8a2d2f8060858fde9eda94f80a54](https://github.com/googleapis/google-cloud-python/commit/3a90cc8e867c8a2d2f8060858fde9eda94f80a54)) - -## [2.42.0](https://github.com/googleapis/google-cloud-python/compare/bigframes-v2.41.0...bigframes-v2.42.0) (2026-06-08) - - -### Features - -* create `Series.bigquery.function_name` accessors for array and AEAD functions (#17279) ([d01a4ba30040cfcb6498d0e9ef3ed3a54d56239d](https://github.com/googleapis/google-cloud-python/commit/d01a4ba30040cfcb6498d0e9ef3ed3a54d56239d)) -* support automatic per-cell execution history filtering and isolated callbacks (#17144) ([7d440111d836b94f0ce22f6b08c7ce0e7bf4a38a](https://github.com/googleapis/google-cloud-python/commit/7d440111d836b94f0ce22f6b08c7ce0e7bf4a38a)) -* Add ai_generate functions to the dataframe bq accessor (#17302) ([6b62cb6fb3de94326b8944ae08a400c12529cad2](https://github.com/googleapis/google-cloud-python/commit/6b62cb6fb3de94326b8944ae08a400c12529cad2)) - - -### Bug Fixes - -* nameless column to_frame bug for pandas 3.0 (#17371) ([b23bfa4ceb819bca8201a7fe8b64a9bed56733f0](https://github.com/googleapis/google-cloud-python/commit/b23bfa4ceb819bca8201a7fe8b64a9bed56733f0)) -* include pyopenssl as a dependency (#17362) ([1f6205ee5a370249ece2c2cc7131a47830ef00ea](https://github.com/googleapis/google-cloud-python/commit/1f6205ee5a370249ece2c2cc7131a47830ef00ea)) -* Fix IsInOp literal bug with sqlglot (#17356) ([a3d93afe74dd2b5ec8a2ae92f91c95962764debe](https://github.com/googleapis/google-cloud-python/commit/a3d93afe74dd2b5ec8a2ae92f91c95962764debe)) - -## [2.41.0](https://github.com/googleapis/google-cloud-python/compare/bigframes-v2.40.0...bigframes-v2.41.0) (2026-05-28) - - -### Documentation - -* modernize multimodal tutorials and migrate legacy blob APIs (#16918) ([05d80c3cccc237480dc5f589b7768b57a147cb0e](https://github.com/googleapis/google-cloud-python/commit/05d80c3cccc237480dc5f589b7768b57a147cb0e)) - - -### Features - -* Defer unnamed @udf deployment until needed (#17217) ([ad3b8fa9693b7d23859c417f2f5954ea946f2bb8](https://github.com/googleapis/google-cloud-python/commit/ad3b8fa9693b7d23859c417f2f5954ea946f2bb8)) -* set up Angular infrastructure for TableWidget (#16934) ([4d20bab8ce15c31e5832789e6a5d306983b8584a](https://github.com/googleapis/google-cloud-python/commit/4d20bab8ce15c31e5832789e6a5d306983b8584a)) -* support pandas inputs in more bigframes.bigquery functions (#17224) ([d4d885547f99c08caabad5e715aecd8c6f1fb4d6](https://github.com/googleapis/google-cloud-python/commit/d4d885547f99c08caabad5e715aecd8c6f1fb4d6)) -* add more scalar array functions to `bigframes.bigquery` (#17213) ([4f8a6c81797204f4334c7251c244bc0b6bd568e2](https://github.com/googleapis/google-cloud-python/commit/4f8a6c81797204f4334c7251c244bc0b6bd568e2)) -* add `bigframes.bigquery.deterministic_decrypt*` and `bigframes.bigquery.deterministic_encrypt` functions (#17212) ([85f36725802f3e7ad16156ca9e957e61d57d3112](https://github.com/googleapis/google-cloud-python/commit/85f36725802f3e7ad16156ca9e957e61d57d3112)) -* add `bigframes.bigquery.aead.*` scalar functions (#17168) ([a7e4d048e254cdb723df0a47ccbd8d09aed00c7a](https://github.com/googleapis/google-cloud-python/commit/a7e4d048e254cdb723df0a47ccbd8d09aed00c7a)) -* complete deprecation and cleanup of multimodal blob APIs (#16618) ([3624f3bb102e7d599097975db5cdaee508c9549a](https://github.com/googleapis/google-cloud-python/commit/3624f3bb102e7d599097975db5cdaee508c9549a)) -* support output_mode for ai.classify (#17097) ([098c35c5a8383d1585848e10806f9914b2ef4f97](https://github.com/googleapis/google-cloud-python/commit/098c35c5a8383d1585848e10806f9914b2ef4f97)) - - -### Bug Fixes - -* cast JSON and nested struct columns to string for anywidget rendering (#17189) ([994a22d64856b436d196743d16c3fd1967b20784](https://github.com/googleapis/google-cloud-python/commit/994a22d64856b436d196743d16c3fd1967b20784)) -* Respect display.progress_bar=None in background threads (#16715) ([07dd3315447d2feb6de8d53e0915798da9c04151](https://github.com/googleapis/google-cloud-python/commit/07dd3315447d2feb6de8d53e0915798da9c04151)) - - -### Dependencies - -* bump mistune from 3.1.3 to 3.2.1 in /packages/bigframes (#17202) ([52f21788f76575036624c9163b63620a1bb92a83](https://github.com/googleapis/google-cloud-python/commit/52f21788f76575036624c9163b63620a1bb92a83)) -* bump langsmith from 0.4.10 to 0.8.0 in /packages/bigframes (#17210) ([9dd0c02c585f7fda34d6e2199ab2bc7c0b5a246a](https://github.com/googleapis/google-cloud-python/commit/9dd0c02c585f7fda34d6e2199ab2bc7c0b5a246a)) -* bump gdal from 3.8.4 to 3.13.0 in /packages/bigframes (#17204) ([900007bab07feb7580cb7e8a36a5d4ee4cce14ab](https://github.com/googleapis/google-cloud-python/commit/900007bab07feb7580cb7e8a36a5d4ee4cce14ab)) - -## [2.40.0](https://github.com/googleapis/google-cloud-python/compare/bigframes-v2.39.0...bigframes-v2.40.0) (2026-05-13) - - -### Documentation - -* Add docs to the to_csv methods of dataframe and series (#16570) ([a8fccefd868e3474d3a2cfbabc03364891e05824](https://github.com/googleapis/google-cloud-python/commit/a8fccefd868e3474d3a2cfbabc03364891e05824)) - - -### Features - -* add more params to ai.classify (#16990) ([e9c52b12c02f8b15e43b62e6f3fb7617ac3bdfd9](https://github.com/googleapis/google-cloud-python/commit/e9c52b12c02f8b15e43b62e6f3fb7617ac3bdfd9)) -* add support for `hparam_range` and `hparam_candidates` to `bigframes.bigquery.create_model` (#16640) ([ca47835ce0e381c0833545ca1cf7734c3c34ceb5](https://github.com/googleapis/google-cloud-python/commit/ca47835ce0e381c0833545ca1cf7734c3c34ceb5)) -* update ai.score to match its SQL version (#16919) ([9f42fe1436df61ca0abad77bb4b51ed983a85a48](https://github.com/googleapis/google-cloud-python/commit/9f42fe1436df61ca0abad77bb4b51ed983a85a48)) -* update ai.if_() params to match the SQL version (#16857) ([f3cb4ad04a15a58a931d4feb43b172805209cf58](https://github.com/googleapis/google-cloud-python/commit/f3cb4ad04a15a58a931d4feb43b172805209cf58)) -* Support unstable sort_values, sort_index (#16665) ([bbdeb70fff766dc51bcac32b5312c13ce16764d4](https://github.com/googleapis/google-cloud-python/commit/bbdeb70fff766dc51bcac32b5312c13ce16764d4)) -* Support Expression objects in create_model options (#16606) ([cf12ffd858bdba0a95dba8fd591ed9adcf8c0e8a](https://github.com/googleapis/google-cloud-python/commit/cf12ffd858bdba0a95dba8fd591ed9adcf8c0e8a)) -* implement ai.similarity (#16771) ([d4afa2c835d53983ecd22e2f9835107791cde65f](https://github.com/googleapis/google-cloud-python/commit/d4afa2c835d53983ecd22e2f9835107791cde65f)) -* implement ai.embed (#16759) ([fcb4579b9e273c3ad43ed150f4ef0fbb7daeef2c](https://github.com/googleapis/google-cloud-python/commit/fcb4579b9e273c3ad43ed150f4ef0fbb7daeef2c)) -* Add bigframes.execution_history API to track BigQuery jobs (#16588) ([fa20a740b15accf2b1ae18a9ac20b75f006dbcad](https://github.com/googleapis/google-cloud-python/commit/fa20a740b15accf2b1ae18a9ac20b75f006dbcad)) -* Support loading avro, orc data (#16555) ([6d46cba3777c1b2adf6f1f86f6d3db3ea30c55d2](https://github.com/googleapis/google-cloud-python/commit/6d46cba3777c1b2adf6f1f86f6d3db3ea30c55d2)) -* Add numpy ufunc support to col expressions (#16554) ([2f792abd5d48ec680305e1e4ec9136360e16c9a5](https://github.com/googleapis/google-cloud-python/commit/2f792abd5d48ec680305e1e4ec9136360e16c9a5)) - - -### Bug Fixes - -* avoid `copy` argument warning in `to_pandas` (#16917) ([fe5245b8f20dd94231e72e2572609e029ee137c7](https://github.com/googleapis/google-cloud-python/commit/fe5245b8f20dd94231e72e2572609e029ee137c7)) -* BigFrames respects bq default region (#16933) ([ef9945a5d6296e6bbf00b6ef980462f5a0b91b20](https://github.com/googleapis/google-cloud-python/commit/ef9945a5d6296e6bbf00b6ef980462f5a0b91b20)) -* Fix bugs compiling ambiguous ids and in subqueries (#16617) ([479e44ddb8ba7515797f062064c4ebf2db5d09f2](https://github.com/googleapis/google-cloud-python/commit/479e44ddb8ba7515797f062064c4ebf2db5d09f2)) -* avoid views when querying BigLake tables from SQL cells (#16562) ([fdd3e0de66377d75ec235e4fc071e4ecc33a35c7](https://github.com/googleapis/google-cloud-python/commit/fdd3e0de66377d75ec235e4fc071e4ecc33a35c7)) - -## [2.39.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.38.0...v2.39.0) (2026-03-31) - - -### Documentation - -* Rename Blob column references to ObjectRef column (#2535) ([44e0ffd947e9db66ab612f92de6e31f1085e7968](https://github.com/googleapis/python-bigquery-dataframes/commit/44e0ffd947e9db66ab612f92de6e31f1085e7968)) -* gemini retouch of the index page for seo (#2514) ([2e5311e2242b039da4c8e37b7b48942fa8ed34c2](https://github.com/googleapis/python-bigquery-dataframes/commit/2e5311e2242b039da4c8e37b7b48942fa8ed34c2)) - - -### Features - -* expose DataFrame.bigquery in both pandas and bigframes DataFrames (#2533) ([69fe317612a69aa92f06f0c418c67aa1f9488bd2](https://github.com/googleapis/python-bigquery-dataframes/commit/69fe317612a69aa92f06f0c418c67aa1f9488bd2)) -* support full round-trip persistence for multimodal reference cols (#2511) ([494a0a113b1ba6dcdc9f9b85a4f750d093f5652f](https://github.com/googleapis/python-bigquery-dataframes/commit/494a0a113b1ba6dcdc9f9b85a4f750d093f5652f)) -* add `df.bigquery.ai.forecast` method to pandas dataframe accessor (#2518) ([1126cec9cdfcc1ec1062c60e5affbe1b60223767](https://github.com/googleapis/python-bigquery-dataframes/commit/1126cec9cdfcc1ec1062c60e5affbe1b60223767)) - - -### Bug Fixes - -* handle aggregate operations on empty selections (#2510) ([34fb5daa93726d0d3ff364912a3c1de0fc535fb2](https://github.com/googleapis/python-bigquery-dataframes/commit/34fb5daa93726d0d3ff364912a3c1de0fc535fb2)) -* Localize BigQuery log suppression for gbq.py (#2541) ([af49ca29399aa2c63753d9045fd382e30334d134](https://github.com/googleapis/python-bigquery-dataframes/commit/af49ca29399aa2c63753d9045fd382e30334d134)) -* to_gbq may swap data columns when replace table (#2532) ([17ecc65e1c0397ef349fca4afcf5a77af72aa798](https://github.com/googleapis/python-bigquery-dataframes/commit/17ecc65e1c0397ef349fca4afcf5a77af72aa798)) -* Respect remote function config changes even if logic unchanged (#2512) ([b9524284ad3b457b15598f546bac04c76b3e27b8](https://github.com/googleapis/python-bigquery-dataframes/commit/b9524284ad3b457b15598f546bac04c76b3e27b8)) -* support melting empty DataFrames without crashing (#2509) ([e8c46032154e186042314d97aa813301413d8a13](https://github.com/googleapis/python-bigquery-dataframes/commit/e8c46032154e186042314d97aa813301413d8a13)) - -## [2.38.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.37.0...v2.38.0) (2026-03-16) - - -### Documentation - -* add notebooks to user guide page (#2505) ([5cf37888bc0b4b1b0993dadd1e0fe5ee08341ef4](https://github.com/googleapis/python-bigquery-dataframes/commit/5cf37888bc0b4b1b0993dadd1e0fe5ee08341ef4)) -* Fix typo in ExperimentOptions class docstring (#2498) ([077cb2ebe515fc5e07bcbb5dc663edd28d3eaf00](https://github.com/googleapis/python-bigquery-dataframes/commit/077cb2ebe515fc5e07bcbb5dc663edd28d3eaf00)) - - -### Features - -* add `df.bigquery` pandas accessor (#2513) ([91b6c245521218bb78b543885e1b9424278ce2ab](https://github.com/googleapis/python-bigquery-dataframes/commit/91b6c245521218bb78b543885e1b9424278ce2ab)) -* use EUC for AI IF, CLASSIFY, and SCORE when connection is not provided (#2507) ([fe94910abff28e244dd79e1540a6c2184a12eb44](https://github.com/googleapis/python-bigquery-dataframes/commit/fe94910abff28e244dd79e1540a6c2184a12eb44)) -* Add `bigframes.bigquery.rand()` function (#2501) ([5c43efb745118f506ecc30196da68e9d6f4346dc](https://github.com/googleapis/python-bigquery-dataframes/commit/5c43efb745118f506ecc30196da68e9d6f4346dc)) -* add bigquery.ml.get_insights function (#2493) ([d29a60953ac989bb2c95e6eec3010620ac776a3c](https://github.com/googleapis/python-bigquery-dataframes/commit/d29a60953ac989bb2c95e6eec3010620ac776a3c)) -* Add str, dt accessors to pd.col Expression objects (#2488) ([ce5de57019449ca77d308946df72f04289343b51](https://github.com/googleapis/python-bigquery-dataframes/commit/ce5de57019449ca77d308946df72f04289343b51)) - - -### Bug Fixes - -* handle unsupported types and empty results in describe (#2506) ([2326ad6aec15c20a66756eff093b50be484b3ba8](https://github.com/googleapis/python-bigquery-dataframes/commit/2326ad6aec15c20a66756eff093b50be484b3ba8)) -* no longer automatically use anywidget in the `%%bqsql` magics (#2504) ([43353e2bc9ffbc38b7383c24ecaac80d3b8bab32](https://github.com/googleapis/python-bigquery-dataframes/commit/43353e2bc9ffbc38b7383c24ecaac80d3b8bab32)) - -## [2.37.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.36.0...v2.37.0) (2026-03-03) - - -### Documentation - -* Fix recall_score doc example (#2477) ([a6f499c1e225a962b53621158f9d4a19ca220ccd](https://github.com/googleapis/python-bigquery-dataframes/commit/a6f499c1e225a962b53621158f9d4a19ca220ccd)) -* add code sample and docstring for bpd.options.experiments.sql_compiler (#2474) ([867951bcabcff12e2fce88143b45d929d3237088](https://github.com/googleapis/python-bigquery-dataframes/commit/867951bcabcff12e2fce88143b45d929d3237088)) -* use direct API for image (#2465) ([8a1a82f7a0fd224f2b075c68ab116d1f580d1d82](https://github.com/googleapis/python-bigquery-dataframes/commit/8a1a82f7a0fd224f2b075c68ab116d1f580d1d82)) -* add bigframes default connection warning (#2471) ([f1bbba23667f01d3b8e7c51b18fe64641a4b135f](https://github.com/googleapis/python-bigquery-dataframes/commit/f1bbba23667f01d3b8e7c51b18fe64641a4b135f)) -* Move readme content to new User Guide section (#2464) ([61a948451baeb1caa323e721ad88b31c7cd0b3cb](https://github.com/googleapis/python-bigquery-dataframes/commit/61a948451baeb1caa323e721ad88b31c7cd0b3cb)) -* Skip inherited methods, use autosummary only for big classes (#2470) ([a9512498ef39b9d5260cad2ca0513c701a6d3592](https://github.com/googleapis/python-bigquery-dataframes/commit/a9512498ef39b9d5260cad2ca0513c701a6d3592)) -* Add code examples to configuration docstrings (#2352) ([3c21993e6fca474c32f3c2371c41ef2be146267e](https://github.com/googleapis/python-bigquery-dataframes/commit/3c21993e6fca474c32f3c2371c41ef2be146267e)) - - -### Features - -* Add cloud_function_cpus option to remote_function (#2475) ([4caf74ccaeb9608d91da864bb80eddf1148a1502](https://github.com/googleapis/python-bigquery-dataframes/commit/4caf74ccaeb9608d91da864bb80eddf1148a1502)) -* Support pd.col simple aggregates (#2480) ([cb00daabce49f067be8e16627166dda00d5d8134](https://github.com/googleapis/python-bigquery-dataframes/commit/cb00daabce49f067be8e16627166dda00d5d8134)) -* add display.render_mode to control DataFrame/Series visualization (#2413) ([7813eaa6fa2ae42943b90583e600c95beaf5d75e](https://github.com/googleapis/python-bigquery-dataframes/commit/7813eaa6fa2ae42943b90583e600c95beaf5d75e)) -* add support for Python 3.14 (#2232) ([c25a6d0151380dde74368a35e13deb7a930b494f](https://github.com/googleapis/python-bigquery-dataframes/commit/c25a6d0151380dde74368a35e13deb7a930b494f)) -* Support pd.col expressions with .loc and getitem (#2473) ([ae5c8b322765aef51eed016bfacaff5a7a917a7b](https://github.com/googleapis/python-bigquery-dataframes/commit/ae5c8b322765aef51eed016bfacaff5a7a917a7b)) -* add dt.tz_localize() (#2469) ([f70f93a1227add1627d522d7e55a37f42fc3549e](https://github.com/googleapis/python-bigquery-dataframes/commit/f70f93a1227add1627d522d7e55a37f42fc3549e)) -* Update bigquery.ai.generate_table output_schema to allow Mapping type (#2463) ([f7fd1895e64a133fe63eddeb90f57a42a35c29b2](https://github.com/googleapis/python-bigquery-dataframes/commit/f7fd1895e64a133fe63eddeb90f57a42a35c29b2)) - - -### Bug Fixes - -* upload local data through write API if nested JSONs detected (#2478) ([01dc5a34e09171351575d5cbdc9f301e505e1567](https://github.com/googleapis/python-bigquery-dataframes/commit/01dc5a34e09171351575d5cbdc9f301e505e1567)) -* allow IsInOp with same dtypes regardless nullable (#2466) ([1d81b414acbc964502ca624eae72cdb8c14e1576](https://github.com/googleapis/python-bigquery-dataframes/commit/1d81b414acbc964502ca624eae72cdb8c14e1576)) - -## [2.36.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.35.0...v2.36.0) (2026-02-17) - - -### Documentation - -* update multimodal dataframe notebook to use public APIs (#2456) ([342fa723c4631d371364a87ae0ddd6fa03360a4b](https://github.com/googleapis/python-bigquery-dataframes/commit/342fa723c4631d371364a87ae0ddd6fa03360a4b)) -* use direct API for pdf chunk and pdf extract (#2452) ([543ce52c18269eab2a89886f226d1478dbabf9ba](https://github.com/googleapis/python-bigquery-dataframes/commit/543ce52c18269eab2a89886f226d1478dbabf9ba)) -* fix generate_text and generate_table input docs (#2455) ([078bd32ebd28af0d2cfba6bb874ba79e904183e2](https://github.com/googleapis/python-bigquery-dataframes/commit/078bd32ebd28af0d2cfba6bb874ba79e904183e2)) -* Update multimodal notebook to use public runtime helpers (#2451) ([e36dd8b492fd7ab433fa4cac732b31774c1e428b](https://github.com/googleapis/python-bigquery-dataframes/commit/e36dd8b492fd7ab433fa4cac732b31774c1e428b)) -* use direct API for audio transcription (#2447) ([59cbc5db66fd178ecce03bf4b8b4a504d7ef3e9f](https://github.com/googleapis/python-bigquery-dataframes/commit/59cbc5db66fd178ecce03bf4b8b4a504d7ef3e9f)) -* Add EXIF metadata extraction example to multimodal notebook (#2429) ([84c6f883aef8048e7013a8b3c03a1bde47e94eea](https://github.com/googleapis/python-bigquery-dataframes/commit/84c6f883aef8048e7013a8b3c03a1bde47e94eea)) - - -### Features - -* Initial support for biglake iceberg tables (#2409) ([ae35a9890a2f9903b12e431488362c091118bbdd](https://github.com/googleapis/python-bigquery-dataframes/commit/ae35a9890a2f9903b12e431488362c091118bbdd)) -* add bigquery.ai.generate_table function (#2453) ([b925aa243dad0e42ad126c9397f42be0aad7152d](https://github.com/googleapis/python-bigquery-dataframes/commit/b925aa243dad0e42ad126c9397f42be0aad7152d)) - -## [2.35.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.34.0...v2.35.0) (2026-02-07) - - -### Documentation - -* fix cast method shown on public docs (#2436) ([ad0f33c65ee01409826c381ae0f70aad65bb6a27](https://github.com/googleapis/python-bigquery-dataframes/commit/ad0f33c65ee01409826c381ae0f70aad65bb6a27)) - - -### Features - -* remove redundant "started." messages from progress output (#2440) ([2017cc2f27f0a432af46f60b3286b231caa4a98b](https://github.com/googleapis/python-bigquery-dataframes/commit/2017cc2f27f0a432af46f60b3286b231caa4a98b)) -* Add bigframes.pandas.col with basic operators (#2405) ([12741677c0391efb5d05281fc756445ccbb1387e](https://github.com/googleapis/python-bigquery-dataframes/commit/12741677c0391efb5d05281fc756445ccbb1387e)) -* Disable progress bars in Anywidget mode (#2444) ([4e2689a1c975c4cabaf36b7d0817dcbedc926853](https://github.com/googleapis/python-bigquery-dataframes/commit/4e2689a1c975c4cabaf36b7d0817dcbedc926853)) -* Disable progress bars in Anywidget mode to reduce notebook clutter (#2437) ([853240daf45301ad534c635c8955cb6ce91d23c2](https://github.com/googleapis/python-bigquery-dataframes/commit/853240daf45301ad534c635c8955cb6ce91d23c2)) -* add bigquery.ai.generate_text function (#2433) ([5bd0029a99e7653843de4ac7d57370c9dffeed4d](https://github.com/googleapis/python-bigquery-dataframes/commit/5bd0029a99e7653843de4ac7d57370c9dffeed4d)) -* Add a bigframes cell magic for ipython (#2395) ([e6de52ded6c5091275a936dec36f01a6cf701233](https://github.com/googleapis/python-bigquery-dataframes/commit/e6de52ded6c5091275a936dec36f01a6cf701233)) -* add `bigframes.bigquery.ai.generate_embedding` (#2343) ([e91536c8a5b2d8d896767510ced80c6fd2a68a97](https://github.com/googleapis/python-bigquery-dataframes/commit/e91536c8a5b2d8d896767510ced80c6fd2a68a97)) -* add bigframe.bigquery.load_data function (#2426) ([4b0f13b2fe10fa5b07d3ca3b7cb1ae1cb95030c7](https://github.com/googleapis/python-bigquery-dataframes/commit/4b0f13b2fe10fa5b07d3ca3b7cb1ae1cb95030c7)) - - -### Bug Fixes - -* suppress JSONDtypeWarning in Anywidget mode and clean up progress output (#2441) ([e0d185ad2c0245b17eac315f71152a46c6da41bb](https://github.com/googleapis/python-bigquery-dataframes/commit/e0d185ad2c0245b17eac315f71152a46c6da41bb)) -* exlcude gcsfs 2026.2.0 (#2445) ([311de31e79227408515f087dafbab7edc54ddf1b](https://github.com/googleapis/python-bigquery-dataframes/commit/311de31e79227408515f087dafbab7edc54ddf1b)) -* always display the results in the `%%bqsql` cell magics output (#2439) ([2d973b54550f30429dbd10894f78db7bb0c57345](https://github.com/googleapis/python-bigquery-dataframes/commit/2d973b54550f30429dbd10894f78db7bb0c57345)) - -## [2.34.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.33.0...v2.34.0) (2026-02-02) - - -### Features - -* add `bigframes.pandas.options.experiments.sql_compiler` for switching the backend compiler (#2417) ([7eba6ee03f07938315d99e2aeaf72368c02074cf](https://github.com/googleapis/python-bigquery-dataframes/commit/7eba6ee03f07938315d99e2aeaf72368c02074cf)) -* add bigquery.ml.generate_embedding function (#2422) ([35f3f5e6f8c64b47e6e7214034f96f047785e647](https://github.com/googleapis/python-bigquery-dataframes/commit/35f3f5e6f8c64b47e6e7214034f96f047785e647)) -* add bigquery.create_external_table method (#2415) ([76db2956e505aec4f1055118ac7ca523facc10ff](https://github.com/googleapis/python-bigquery-dataframes/commit/76db2956e505aec4f1055118ac7ca523facc10ff)) -* add deprecation warnings for .blob accessor and read_gbq_object_table (#2408) ([7261a4ea5cdab6b30f5bc333501648c60e70be59](https://github.com/googleapis/python-bigquery-dataframes/commit/7261a4ea5cdab6b30f5bc333501648c60e70be59)) -* add bigquery.ml.generate_text function (#2403) ([5ac681028624de15e31f0c2ae360b47b2dcf1e8d](https://github.com/googleapis/python-bigquery-dataframes/commit/5ac681028624de15e31f0c2ae360b47b2dcf1e8d)) - - -### Bug Fixes - -* broken job url (#2411) ([fcb5bc1761c656e1aec61dbcf96a36d436833b7a](https://github.com/googleapis/python-bigquery-dataframes/commit/fcb5bc1761c656e1aec61dbcf96a36d436833b7a)) - -## [2.33.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.32.0...v2.33.0) (2026-01-22) - - -### Features - -* add bigquery.ml.transform function (#2394) ([1f9ee373c1f1d0cd08b80169c3063b862ea46465](https://github.com/googleapis/python-bigquery-dataframes/commit/1f9ee373c1f1d0cd08b80169c3063b862ea46465)) -* Add BigQuery ObjectRef functions to `bigframes.bigquery.obj` (#2380) ([9c3bbc36983dffb265454f27b37450df8c5fbc71](https://github.com/googleapis/python-bigquery-dataframes/commit/9c3bbc36983dffb265454f27b37450df8c5fbc71)) -* Stabilize interactive table height to prevent notebook layout shifts (#2378) ([a634e976c0f44087ca2a65f68cf2775ae6f04024](https://github.com/googleapis/python-bigquery-dataframes/commit/a634e976c0f44087ca2a65f68cf2775ae6f04024)) -* Add max_columns control for anywidget mode (#2374) ([34b5975f6911c5aa5ffc64a2fe6967a9f3d86f78](https://github.com/googleapis/python-bigquery-dataframes/commit/34b5975f6911c5aa5ffc64a2fe6967a9f3d86f78)) -* Add dark mode to anywidget mode (#2365) ([2763b41d4b86939e389f76789f5b2acd44f18169](https://github.com/googleapis/python-bigquery-dataframes/commit/2763b41d4b86939e389f76789f5b2acd44f18169)) -* Configure Biome for Consistent Code Style (#2364) ([81e27b3d81da9b1684eae0b7f0b9abfd7badcc4f](https://github.com/googleapis/python-bigquery-dataframes/commit/81e27b3d81da9b1684eae0b7f0b9abfd7badcc4f)) - - -### Bug Fixes - -* Throw if write api commit op has stream_errors (#2385) ([7abfef0598d476ef233364a01f72d73291983c30](https://github.com/googleapis/python-bigquery-dataframes/commit/7abfef0598d476ef233364a01f72d73291983c30)) -* implement retry logic for cloud function endpoint fetching (#2369) ([0f593c27bfee89fe1bdfc880504f9ab0ac28a24e](https://github.com/googleapis/python-bigquery-dataframes/commit/0f593c27bfee89fe1bdfc880504f9ab0ac28a24e)) - ## [2.32.0](https://github.com/googleapis/google-cloud-python/compare/bigframes-v2.31.0...bigframes-v2.32.0) (2026-01-05) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..039f4368120 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,95 @@ + +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of +experience, education, socio-economic status, nationality, personal appearance, +race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, or to ban temporarily or permanently any +contributor for other behaviors that they deem inappropriate, threatening, +offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +This Code of Conduct also applies outside the project spaces when the Project +Steward has a reasonable belief that an individual's behavior may have a +negative impact on the project or its community. + +## Conflict Resolution + +We do not believe that all conflict is bad; healthy debate and disagreement +often yield positive results. However, it is never okay to be disrespectful or +to engage in behavior that violates the project’s code of conduct. + +If you see someone violating the code of conduct, you are encouraged to address +the behavior directly with those involved. Many issues can be resolved quickly +and easily, and this gives people more control over the outcome of their +dispute. If you are unable to resolve the matter for any reason, or if the +behavior is threatening or harassing, report it. We are dedicated to providing +an environment where participants feel welcome and safe. + + +Reports should be directed to *googleapis-stewards@google.com*, the +Project Steward(s) for *Google Cloud Client Libraries*. It is the Project Steward’s duty to +receive and address reported violations of the code of conduct. They will then +work with a committee consisting of representatives from the Open Source +Programs Office and the Google Open Source Strategy team. If for any reason you +are uncomfortable reaching out to the Project Steward, please email +opensource@google.com. + +We will investigate every complaint, but you may not receive a direct response. +We will use our discretion in determining when and how to follow up on reported +incidents, which may range from not taking action to permanent expulsion from +the project and project-sponsored spaces. We will notify the accused of the +report and provide them an opportunity to discuss it before any action is taken. +The identity of the reporter will be omitted from the details of the report +supplied to the accused. In potentially harmful situations, such as ongoing +harassment or threats to anyone's safety, we may take action without notice. + +## Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 1.4, +available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html \ No newline at end of file diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index ba8400eb866..5374e7e3770 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.10, 3.11, 3.12, 3.13 and 3.14 on both UNIX and Windows. + 3.9, 3.10, 3.11, 3.12 and 3.13 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -42,14 +42,14 @@ You'll have to create a development environment using a Git checkout: - Clone your fork of ``python-bigquery-dataframes`` from your GitHub account to your local computer, substituting your account username and specifying the destination - as ``hack-on-google-cloud-python``. E.g.:: + as ``hack-on-python-bigquery-dataframes``. E.g.:: $ cd ${HOME} - $ git clone git@github.com:USERNAME/google-cloud-python.git hack-on-google-cloud-python - $ cd hack-on-google-cloud-python - # Configure remotes such that you can pull changes from the googleapis/google-cloud-python + $ git clone git@github.com:USERNAME/python-bigquery-dataframes.git hack-on-python-bigquery-dataframes + $ cd hack-on-python-bigquery-dataframes + # Configure remotes such that you can pull changes from the googleapis/python-bigquery-dataframes # repository into your local repository. - $ git remote add upstream git@github.com:googleapis/google-cloud-python.git + $ git remote add upstream git@github.com:googleapis/python-bigquery-dataframes.git # fetch and merge changes from upstream into main $ git fetch upstream $ git merge upstream/main @@ -60,7 +60,7 @@ repo, from which you can submit a pull request. To work on the codebase and run the tests, we recommend using ``nox``, but you can also use a ``virtualenv`` of your own creation. -.. _repo: https://github.com/googleapis/google-cloud-python/tree/main/packages/bigframes +.. _repo: https://github.com/googleapis/python-bigquery-dataframes Using ``nox`` ============= @@ -72,7 +72,7 @@ We use `nox `__ to instrument our tests. - To run a single unit test:: - $ nox -s unit-3.14 -- -k + $ nox -s unit-3.13 -- -k .. note:: @@ -143,12 +143,12 @@ Running System Tests $ nox -s system # Run a single system test - $ nox -s system-3.14 -- -k + $ nox -s system-3.13 -- -k .. note:: - System tests are only configured to run under Python 3.10, 3.12 and 3.14. + System tests are only configured to run under Python 3.9, 3.11, 3.12 and 3.13. For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local @@ -232,11 +232,11 @@ configure them just like the System Tests. # Run all tests in a folder $ cd samples/snippets - $ nox -s py-3.10 + $ nox -s py-3.8 # Run a single sample test $ cd samples/snippets - $ nox -s py-3.10 -- -k + $ nox -s py-3.8 -- -k ******************************************** Note About ``README`` as it pertains to PyPI @@ -246,7 +246,7 @@ The `description on PyPI`_ for the project comes directly from the ``README``. Due to the reStructuredText (``rst``) parser used by PyPI, relative links which will work on GitHub (e.g. ``CONTRIBUTING.rst`` instead of -``https://github.com/googleapis/google-cloud-python/blob/main/packages/bigframes/CONTRIBUTING.rst``) +``https://github.com/googleapis/python-bigquery-dataframes/blob/main/CONTRIBUTING.rst``) may cause problems creating links or rendering the description. .. _description on PyPI: https://pypi.org/project/bigframes @@ -258,25 +258,35 @@ Supported Python Versions We support: +- `Python 3.9`_ - `Python 3.10`_ - `Python 3.11`_ - `Python 3.12`_ - `Python 3.13`_ -- `Python 3.14`_ +.. _Python 3.9: https://docs.python.org/3.9/ .. _Python 3.10: https://docs.python.org/3.10/ .. _Python 3.11: https://docs.python.org/3.11/ .. _Python 3.12: https://docs.python.org/3.12/ .. _Python 3.13: https://docs.python.org/3.13/ -.. _Python 3.14: https://docs.python.org/3.14/ Supported versions can be found in our ``noxfile.py`` `config`_. -.. _config: https://github.com/googleapis/google-cloud-python/blob/main/packages/bigframes/noxfile.py +.. _config: https://github.com/googleapis/python-bigquery-dataframes/blob/main/noxfile.py +We also explicitly decided to support Python 3 beginning with version 3.9. +Reasons for this include: +- Encouraging use of newest versions of Python 3 +- Taking the lead of `prominent`_ open-source `projects`_ +- `Unicode literal support`_ which allows for a cleaner codebase that + works in both Python 2 and Python 3 + +.. _prominent: https://docs.djangoproject.com/en/1.9/faq/install/#what-python-version-can-i-use-with-django +.. _projects: http://flask.pocoo.org/docs/0.10/python3/ +.. _Unicode literal support: https://www.python.org/dev/peps/pep-0414/ ********** Versioning diff --git a/.gemini/tasks/scalar_op.md b/GEMINI.md similarity index 63% rename from .gemini/tasks/scalar_op.md rename to GEMINI.md index a9318d54824..0d447f17a48 100644 --- a/.gemini/tasks/scalar_op.md +++ b/GEMINI.md @@ -1,4 +1,75 @@ -## Adding a scalar operator +# Contribution guidelines, tailored for LLM agents + +## Testing + +We use `nox` to instrument our tests. + +- To test your changes, run unit tests with `nox`: + + ```bash + nox -r -s unit + ``` + +- To run a single unit test: + + ```bash + nox -r -s unit-3.13 -- -k + ``` + +- Ignore this step if you lack access to Google Cloud resources. To run system + tests, you can execute:: + + # Run all system tests + $ nox -r -s system + + # Run a single system test + $ nox -r -s system-3.13 -- -k + +- The codebase must have better coverage than it had previously after each + change. You can test coverage via `nox -s unit system cover` (takes a long + time). Omit `system` if you lack access to cloud resources. + +## Code Style + +- We use the automatic code formatter `black`. You can run it using + the nox session `format`. This will eliminate many lint errors. Run via: + + ```bash + nox -r -s format + ``` + +- PEP8 compliance is required, with exceptions defined in the linter configuration. + If you have ``nox`` installed, you can test that you have not introduced + any non-compliant code via: + + ``` + nox -r -s lint + ``` + +- When writing tests, use the idiomatic "pytest" style. + +## Documentation + +If a method or property is implementing the same interface as a third-party +package such as pandas or scikit-learn, place the relevant docstring in the +corresponding `third_party/bigframes_vendored/package_name` directory, not in +the `bigframes` directory. Implementations may be placed in the `bigframes` +directory, though. + +### Testing code samples + +Code samples are very important for accurate documentation. We use the "doctest" +framework to ensure the samples are functioning as expected. After adding a code +sample, please ensure it is correct by running doctest. To run the samples +doctests for just a single method, refer to the following example: + +```bash +pytest --doctest-modules bigframes/pandas/__init__.py::bigframes.pandas.cut +``` + +## Tips for implementing common BigFrames features + +### Adding a scalar operator For an example, see commit [c5b7fdae74a22e581f7705bc0cf5390e928f4425](https://github.com/googleapis/python-bigquery-dataframes/commit/c5b7fdae74a22e581f7705bc0cf5390e928f4425). @@ -65,3 +136,13 @@ To add a new scalar operator, follow these steps: - If you are overriding a pandas or GeoPandas property, add a unit test to ensure the correct behavior (e.g., raising `NotImplementedError` if the functionality is not supported). + + +## Constraints + +- Only add git commits. Do not change git history. +- Follow the spec file for development. + - Check off items in the "Acceptance + criteria" and "Detailed steps" sections with `[x]`. + - Please do this as they are completed. + - Refer back to the spec after each step. diff --git a/README.rst b/README.rst index a3aef5380bb..366062b1d3a 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,8 @@ +:orphan: + BigQuery DataFrames (BigFrames) =============================== - |GA| |pypi| |versions| BigQuery DataFrames (also known as BigFrames) provides a Pythonic DataFrame @@ -16,7 +17,7 @@ for many use cases, including: * `bigframes.bigquery.ai `_ are a collection of powerful AI methods, powered by Gemini. -BigQuery DataFrames is an `open-source package `_. +BigQuery DataFrames is an `open-source package `_. .. |GA| image:: https://img.shields.io/badge/support-GA-gold.svg :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#general-availability @@ -64,15 +65,15 @@ Documentation To learn more about BigQuery DataFrames, visit these pages * `Introduction to BigQuery DataFrames (BigFrames) `_ -* `Sample notebooks `_ +* `Sample notebooks `_ * `API reference `_ -* `Source code (GitHub) `_ +* `Source code (GitHub) `_ License ------- BigQuery DataFrames is distributed with the `Apache-2.0 license -`_. +`_. It also contains code derived from the following third-party packages: @@ -84,7 +85,7 @@ It also contains code derived from the following third-party packages: * `SQLGlot `_ For details, see the `third_party -`_ +`_ directory. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..8b58ae9c01a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,7 @@ +# Security Policy + +To report a security issue, please use [g.co/vulnz](https://g.co/vulnz). + +The Google Security Team will respond within 5 working days of your report on g.co/vulnz. + +We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue. diff --git a/bigframes/__init__.py b/bigframes/__init__.py index 533726343a5..240608ebc2d 100644 --- a/bigframes/__init__.py +++ b/bigframes/__init__.py @@ -14,64 +14,19 @@ """BigQuery DataFrames provides a DataFrame API scaled by the BigQuery engine.""" -import warnings - -# Suppress Python version support warnings from google-cloud libraries. -# These are particularly noisy in Colab which still uses Python 3.10. -warnings.filterwarnings( - "ignore", - category=FutureWarning, - message=".*Google will stop supporting.*Python.*", -) - -# import configuration and types. -# This ensures that when the deeper 'core' modules ask for 'dtypes','options', et. al., -# they are already defined and available. -import bigframes.dtypes # noqa: E402 # isort: skip -import bigframes._config # noqa: E402 # isort: skip -from bigframes._config import option_context, options # noqa: E402 # isort: skip - -import bigframes.enums as enums # noqa: E402 -import bigframes.exceptions as exceptions # noqa: E402 - -# We import operations early to resolve a circular dependency between -# bigframes.core.expression and bigframes.operations. -# This ensures the 'Expression' base class is defined before 'Aggregation' -# subclasses attempt to inherit from it. -import bigframes.operations # noqa: E402 # isort: skip - -# Register pandas extensions -import bigframes.extensions.pandas.dataframe_accessor # noqa: F401, E402 -import bigframes.extensions.pandas.series_accessor # noqa: F401, E402 -from bigframes._config.bigquery_options import BigQueryOptions # noqa: E402 -from bigframes.core.global_session import ( # noqa: E402 - close_session, - execution_history, - get_global_session, -) -from bigframes.session import Session, connect # noqa: E402 -from bigframes.version import __version__ # noqa: E402 - -_MAGIC_NAMES = ["bqsql"] - - -def load_ipython_extension(ipython): - """Called by IPython when this module is loaded as an IPython extension.""" - # Requires IPython to be installed for import to succeed - from bigframes._magics import _cell_magic - - for magic_name in _MAGIC_NAMES: - ipython.register_magic_function( - _cell_magic, magic_kind="cell", magic_name=magic_name - ) - +from bigframes._config import option_context, options +from bigframes._config.bigquery_options import BigQueryOptions +from bigframes.core.global_session import close_session, get_global_session +import bigframes.enums as enums +import bigframes.exceptions as exceptions +from bigframes.session import connect, Session +from bigframes.version import __version__ __all__ = [ "options", "BigQueryOptions", "get_global_session", "close_session", - "execution_history", "enums", "exceptions", "connect", diff --git a/bigframes/_config/__init__.py b/bigframes/_config/__init__.py index cbe369ad58c..1302f6cc039 100644 --- a/bigframes/_config/__init__.py +++ b/bigframes/_config/__init__.py @@ -17,12 +17,12 @@ DataFrames from this package. """ -import bigframes._config.global_options as global_options from bigframes._config.bigquery_options import BigQueryOptions from bigframes._config.compute_options import ComputeOptions from bigframes._config.display_options import DisplayOptions from bigframes._config.experiment_options import ExperimentOptions -from bigframes._config.global_options import Options, option_context +from bigframes._config.global_options import option_context, Options +import bigframes._config.global_options as global_options from bigframes._config.sampling_options import SamplingOptions options = global_options.options diff --git a/bigframes/_config/auth.py b/bigframes/_config/auth.py index f1c069b5310..1574fc48835 100644 --- a/bigframes/_config/auth.py +++ b/bigframes/_config/auth.py @@ -14,7 +14,6 @@ from __future__ import annotations -import os import threading from typing import Optional @@ -22,8 +21,6 @@ import google.auth.transport.requests import pydata_google_auth -import bigframes._config.bigquery_options as bigquery_options - _SCOPES = ["https://www.googleapis.com/auth/cloud-platform"] # Put the lock here rather than in BigQueryOptions so that BigQueryOptions @@ -33,33 +30,7 @@ _cached_project_default: Optional[str] = None -_GOOGLE_CLOUD_PROJECT = "GOOGLE_CLOUD_PROJECT" - - -def resolve_credentials_and_project( - options: bigquery_options.BigQueryOptions, -) -> tuple[google.auth.credentials.Credentials, str]: - project = options.project - credentials = options.credentials - if project is None: - project = os.getenv(_GOOGLE_CLOUD_PROJECT) - - if credentials is None: - credentials, cred_project = _get_default_credentials_with_project() - # This might conflict with explicit project, which will be ignored, credentials project - # only used if nothing else specified - if project is None: - project = cred_project - - if project is None: - raise ValueError( - "Project must be set to initialize BigQuery client. " - "Try setting `bigframes.options.bigquery.project` first." - ) - return credentials, project - - -def _get_default_credentials_with_project() -> tuple[ +def get_default_credentials_with_project() -> tuple[ google.auth.credentials.Credentials, Optional[str] ]: global _AUTH_LOCK, _cached_credentials, _cached_project_default diff --git a/bigframes/_config/bigquery_options.py b/bigframes/_config/bigquery_options.py index 6c5c424240d..648b69dea7f 100644 --- a/bigframes/_config/bigquery_options.py +++ b/bigframes/_config/bigquery_options.py @@ -16,8 +16,8 @@ from __future__ import annotations -import warnings from typing import Literal, Optional, Sequence, Tuple +import warnings import google.auth.credentials import requests.adapters @@ -127,11 +127,6 @@ def application_name(self) -> Optional[str]: The recommended format is ``"application-name/major.minor.patch_version"`` or ``"(gpn:PartnerName;)"`` for official Google partners. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.bigquery.application_name = "my-app/1.0.0" # doctest: +SKIP - Returns: None or str: Application name as a string if exists; otherwise None. @@ -150,13 +145,6 @@ def application_name(self, value: Optional[str]): def credentials(self) -> Optional[google.auth.credentials.Credentials]: """The OAuth2 credentials to use for this client. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import google.auth - >>> credentials, project = google.auth.default() # doctest: +SKIP - >>> bpd.options.bigquery.credentials = credentials # doctest: +SKIP - Returns: None or google.auth.credentials.Credentials: google.auth.credentials.Credentials if exists; otherwise None. @@ -175,11 +163,6 @@ def location(self) -> Optional[str]: For more information, see https://cloud.google.com/bigquery/docs/locations BigQuery locations. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.bigquery.location = "US" # doctest: +SKIP - Returns: None or str: Default location as a string; otherwise None. @@ -196,11 +179,6 @@ def location(self, value: Optional[str]): def project(self) -> Optional[str]: """Google Cloud project ID to use for billing and as the default project. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.bigquery.project = "my-project" # doctest: +SKIP - Returns: None or str: Google Cloud project ID as a string; otherwise None. @@ -228,11 +206,6 @@ def bq_connection(self) -> Optional[str]: If this option isn't provided, or project or location aren't provided, session will use its default project/location/connection_id as default connection. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.bigquery.bq_connection = "my-project.us.my-connection" # doctest: +SKIP - Returns: None or str: Name of the BigQuery connection as a string; otherwise None. @@ -255,11 +228,6 @@ def skip_bq_connection_check(self) -> bool: necessary permissions set up to support BigQuery DataFrames operations, then a runtime error will be reported. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.bigquery.skip_bq_connection_check = True # doctest: +SKIP - Returns: bool: A boolean value, where True indicates a BigQuery connection is @@ -332,12 +300,6 @@ def use_regional_endpoints(self) -> bool: does not promise any guarantee on the request remaining within the location during transit. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.bigquery.location = "europe-west3" # doctest: +SKIP - >>> bpd.options.bigquery.use_regional_endpoints = True # doctest: +SKIP - Returns: bool: A boolean value, where True indicates that regional endpoints @@ -377,11 +339,6 @@ def kms_key_name(self) -> Optional[str]: For more information, see https://cloud.google.com/bigquery/docs/customer-managed-encryption#assign_role Assign the Encrypter/Decrypter. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.bigquery.kms_key_name = "projects/my-project/locations/us/keyRings/my-ring/cryptoKeys/my-key" # doctest: +SKIP - Returns: None or str: Name of the customer managed encryption key as a string; otherwise None. @@ -399,11 +356,6 @@ def kms_key_name(self, value: str): def ordering_mode(self) -> Literal["strict", "partial"]: """Controls whether total row order is always maintained for DataFrame/Series. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.bigquery.ordering_mode = "partial" # doctest: +SKIP - Returns: Literal: A literal string value of either strict or partial ordering mode. @@ -480,14 +432,7 @@ def requests_transport_adapters( @property def enable_polars_execution(self) -> bool: - """If True, will use polars to execute some simple query plans locally. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.bigquery.enable_polars_execution = True # doctest: +SKIP - - """ + """If True, will use polars to execute some simple query plans locally.""" return self._enable_polars_execution @enable_polars_execution.setter diff --git a/bigframes/_config/compute_options.py b/bigframes/_config/compute_options.py index 2ef1e5b7213..7810ee897f5 100644 --- a/bigframes/_config/compute_options.py +++ b/bigframes/_config/compute_options.py @@ -28,30 +28,30 @@ class ComputeOptions: >>> import bigframes.pandas as bpd >>> df = bpd.read_gbq("bigquery-public-data.ml_datasets.penguins") - >>> bpd.options.compute.maximum_bytes_billed = 500 # doctest: +SKIP + >>> bpd.options.compute.maximum_bytes_billed = 500 >>> df.to_pandas() # this should fail # doctest: +SKIP google.api_core.exceptions.InternalServerError: 500 Query exceeded limit for bytes billed: 500. 10485760 or higher required. - >>> bpd.options.compute.maximum_bytes_billed = None # reset option # doctest: +SKIP + >>> bpd.options.compute.maximum_bytes_billed = None # reset option To add multiple extra labels to a query configuration, use the `assign_extra_query_labels` method with keyword arguments: - >>> bpd.options.compute.assign_extra_query_labels(test1=1, test2="abc") # doctest: +SKIP - >>> bpd.options.compute.extra_query_labels # doctest: +SKIP + >>> bpd.options.compute.assign_extra_query_labels(test1=1, test2="abc") + >>> bpd.options.compute.extra_query_labels {'test1': 1, 'test2': 'abc'} Alternatively, you can add labels individually by directly accessing the `extra_query_labels` dictionary: - >>> bpd.options.compute.extra_query_labels["test3"] = False # doctest: +SKIP - >>> bpd.options.compute.extra_query_labels # doctest: +SKIP + >>> bpd.options.compute.extra_query_labels["test3"] = False + >>> bpd.options.compute.extra_query_labels {'test1': 1, 'test2': 'abc', 'test3': False} To remove a label from the configuration, use the `del` keyword on the desired label key: - >>> del bpd.options.compute.extra_query_labels["test1"] # doctest: +SKIP - >>> bpd.options.compute.extra_query_labels # doctest: +SKIP + >>> del bpd.options.compute.extra_query_labels["test1"] + >>> bpd.options.compute.extra_query_labels {'test2': 'abc', 'test3': False} """ @@ -63,11 +63,6 @@ class ComputeOptions: their operations to resume. The default value is 0. Set the value to None to turn off the guard. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.compute.ai_ops_confirmation_threshold = 100 # doctest: +SKIP - Returns: Optional[int]: Number of rows. """ @@ -78,11 +73,6 @@ class ComputeOptions: When set to True, the operation automatically fails without asking for user inputs. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.compute.ai_ops_threshold_autofail = True # doctest: +SKIP - Returns: bool: True if the guard is enabled. """ @@ -95,10 +85,6 @@ class ComputeOptions: 10 GB for potentially faster execution; BigQuery will raise an error if this limit is exceeded. Setting to True removes this result size limit. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.compute.allow_large_results = True # doctest: +SKIP Returns: bool | None: True if results > 10 GB are enabled. @@ -111,10 +97,6 @@ class ComputeOptions: query engine to handle. However this comes at the cost of increase cost and latency. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.compute.enable_multi_query_execution = True # doctest: +SKIP Returns: bool | None: True if enabled. @@ -139,11 +121,6 @@ class ComputeOptions: default. See `maximum_bytes_billed`: https://cloud.google.com/python/docs/reference/bigquery/latest/google.cloud.bigquery.job.QueryJobConfig#google_cloud_bigquery_job_QueryJobConfig_maximum_bytes_billed. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.compute.maximum_bytes_billed = 1000 # doctest: +SKIP - Returns: int | None: Number of bytes, if set. """ @@ -159,15 +136,30 @@ class ComputeOptions: of rows to be downloaded exceeds this limit, a ``bigframes.exceptions.MaximumResultRowsExceeded`` exception is raised. - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.compute.maximum_result_rows = 1000 # doctest: +SKIP - Returns: int | None: Number of rows, if set. """ + semantic_ops_confirmation_threshold: Optional[int] = 0 + """ + Deprecated. + + .. deprecated:: 1.42.0 + Semantic operators are deprecated. Please use the functions in + :mod:`bigframes.bigquery.ai` instead. + + """ + + semantic_ops_threshold_autofail = False + """ + Deprecated. + + .. deprecated:: 1.42.0 + Semantic operators are deprecated. Please use the functions in + :mod:`bigframes.bigquery.ai` instead. + + """ + def assign_extra_query_labels(self, **kwargs: Any) -> None: """ Assigns additional custom labels for query configuration. The method updates the diff --git a/bigframes/_config/experiment_options.py b/bigframes/_config/experiment_options.py index 8f70c8952f6..024de392c06 100644 --- a/bigframes/_config/experiment_options.py +++ b/bigframes/_config/experiment_options.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Optional import warnings -from typing import Literal, Optional import bigframes import bigframes.exceptions as bfe @@ -21,37 +21,39 @@ class ExperimentOptions: """ - Encapsulates the configuration for experiments + Encapsulates the configration for experiments """ def __init__(self): - self._sql_compiler: Literal["legacy", "stable", "experimental"] = "stable" - self._enable_python_transpiler: bool = False + self._semantic_operators: bool = False + self._ai_operators: bool = False @property - def sql_compiler(self) -> Literal["legacy", "stable", "experimental"]: - """Set to 'experimental' to try out the latest in compilation experiments.. + def semantic_operators(self) -> bool: + return self._semantic_operators - **Examples:** + @semantic_operators.setter + def semantic_operators(self, value: bool): + if value is True: + msg = bfe.format_message( + "Semantic operators are deprecated, and will be removed in the future" + ) + warnings.warn(msg, category=FutureWarning) + self._semantic_operators = value - >>> import bigframes.pandas as bpd - >>> bpd.options.experiments.sql_compiler = 'experimental' # doctest: +SKIP - """ - return self._sql_compiler + @property + def ai_operators(self) -> bool: + return self._ai_operators - @sql_compiler.setter - def sql_compiler(self, value: Literal["legacy", "stable", "experimental"]): - if value not in ["legacy", "stable", "experimental"]: - raise ValueError( - "sql_compiler must be one of 'legacy', 'stable', or 'experimental'" - ) - if value == "experimental": + @ai_operators.setter + def ai_operators(self, value: bool): + if value is True: msg = bfe.format_message( - "The experimental SQL compiler is still under experiments, and is subject " + "AI operators are still under experiments, and are subject " "to change in the future." ) - warnings.warn(msg, category=FutureWarning) - self._sql_compiler = value + warnings.warn(msg, category=bfe.PreviewWarning) + self._ai_operators = value @property def blob(self) -> bool: @@ -124,17 +126,3 @@ def blob_display_height(self, value: Optional[int]): warnings.warn(msg, category=bfe.ApiDeprecationWarning) bigframes.options.display.blob_display_height = value - - @property - def enable_python_transpiler(self) -> bool: - return self._enable_python_transpiler - - @enable_python_transpiler.setter - def enable_python_transpiler(self, value: bool): - if value: - msg = bfe.format_message( - "Python transpiler is an unstable, experimental feature, and not yet fully " - "validated, use at your own risk." - ) - warnings.warn(msg, category=bfe.PythonTranspilerPreviewWarning) - self._enable_python_transpiler = value diff --git a/bigframes/_config/global_options.py b/bigframes/_config/global_options.py index 8f742608292..4a3da6d3804 100644 --- a/bigframes/_config/global_options.py +++ b/bigframes/_config/global_options.py @@ -20,8 +20,8 @@ from __future__ import annotations import copy -import threading from dataclasses import dataclass, field +import threading from typing import Optional import bigframes_vendored.pandas._config.config as pandas_config diff --git a/bigframes/_config/sampling_options.py b/bigframes/_config/sampling_options.py index 9746e01f31d..107142c3ba9 100644 --- a/bigframes/_config/sampling_options.py +++ b/bigframes/_config/sampling_options.py @@ -31,11 +31,6 @@ class SamplingOptions: Download size threshold in MB. Default 500. If value set to None, the download size won't be checked. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.sampling.max_download_size = 1000 # doctest: +SKIP """ enable_downsampling: bool = False @@ -45,11 +40,6 @@ class SamplingOptions: If max_download_size is exceeded when downloading data (e.g., to_pandas()), the data will be downsampled if enable_downsampling is True, otherwise, an error will be raised. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.sampling.enable_downsampling = True # doctest: +SKIP """ sampling_method: Literal["head", "uniform"] = "uniform" @@ -60,11 +50,6 @@ class SamplingOptions: the beginning. It is fast and requires minimal computations to perform the downsampling.; "uniform": This algorithm returns uniform random samples of the data. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.sampling.sampling_method = "head" # doctest: +SKIP """ random_state: Optional[int] = None @@ -73,11 +58,6 @@ class SamplingOptions: If provided, the uniform method may take longer to execute and require more computation. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.sampling.random_state = 42 # doctest: +SKIP """ def with_max_download_size(self, max_rows: Optional[int]) -> SamplingOptions: diff --git a/bigframes/_magics.py b/bigframes/_magics.py deleted file mode 100644 index f6b69f35ff5..00000000000 --- a/bigframes/_magics.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from IPython.core import magic_arguments # type: ignore -from IPython.core.getipython import get_ipython -from IPython.display import display - -import bigframes.pandas - - -@magic_arguments.magic_arguments() -@magic_arguments.argument( - "destination_var", - nargs="?", - help=("If provided, save the output to this variable instead of displaying it."), -) -@magic_arguments.argument( - "--dry_run", - action="store_true", - default=False, - help=( - "Sets query to be a dry run to estimate costs. " - "Defaults to executing the query instead of dry run if this argument is not used." - "Does not work with engine 'bigframes'. " - ), -) -def _cell_magic(line, cell): - ipython = get_ipython() - if ipython is None: - raise RuntimeError("BigQuery magic must be run in an IPython environment.") - - args = magic_arguments.parse_argstring(_cell_magic, line) - if not cell: - print("Query is missing.") - return - pyformat_args = ipython.user_ns - dataframe = bigframes.pandas._read_gbq_colab( - cell, pyformat_args=pyformat_args, dry_run=args.dry_run - ) - if args.destination_var: - ipython.push({args.destination_var: dataframe}) - - display(dataframe) diff --git a/bigframes/_tools/docs.py b/bigframes/_tools/docs.py deleted file mode 100644 index 9ecfd61b3c9..00000000000 --- a/bigframes/_tools/docs.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def inherit_docs(source_class): - """ - A class decorator that copies docstrings from source_class to the - decorated class for any methods or attributes that match names. - """ - - def decorator(target_class): - if not target_class.__doc__ and source_class.__doc__: - target_class.__doc__ = source_class.__doc__ - - for name, source_item in vars(source_class).items(): - if name in vars(target_class): - target_item = getattr(target_class, name) - - if hasattr(target_item, "__doc__") and not target_item.__doc__: - if hasattr(source_item, "__doc__") and source_item.__doc__: - try: - target_item.__doc__ = source_item.__doc__ - except AttributeError: - pass - - underlying = None - if isinstance(target_item, property): - underlying = target_item.fget - elif hasattr(target_item, "__func__"): - underlying = target_item.__func__ - elif hasattr(target_item, "func"): - underlying = getattr(target_item, "func", None) - - if underlying is not None: - try: - underlying.__doc__ = source_item.__doc__ - except AttributeError: - pass - - return target_class - - return decorator diff --git a/bigframes/bigquery/__init__.py b/bigframes/bigquery/__init__.py index ade7535c32b..0bbbc418e6d 100644 --- a/bigframes/bigquery/__init__.py +++ b/bigframes/bigquery/__init__.py @@ -12,44 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -Access BigQuery-specific operations and namespaces within BigQuery DataFrames. - -This module provides specialized functions and sub-modules that expose BigQuery's -advanced capabilities to DataFrames and Series. It acts as a bridge between the -pandas-compatible API and the full power of BigQuery SQL. - -Key sub-modules include: - -* :mod:`bigframes.bigquery.ai`: Generative and predictive AI functions (Gemini, BQML). -* :mod:`bigframes.bigquery.ml`: Direct access to BigQuery ML model operations. -* :mod:`bigframes.bigquery.obj`: Support for BigQuery object tables. - -This module also provides direct access to optimized BigQuery functions for: - -* **JSON Processing:** High-performance functions like ``json_extract``, ``json_value``, - and ``parse_json`` for handling semi-structured data. -* **Geospatial Analysis:** Comprehensive geographic functions such as ``st_area``, - ``st_distance``, and ``st_centroid`` (``ST_`` prefixed functions). -* **Array Operations:** Tools for working with BigQuery arrays, including ``array_agg`` - and ``array_length``. -* **Vector Search:** Integration with BigQuery's vector search and indexing - capabilities for high-dimensional data. -* **Custom SQL:** The ``sql_scalar`` function allows embedding raw SQL snippets for - advanced operations not yet directly mapped in the API. - -By using these functions, you can leverage BigQuery's high-performance engine for -domain-specific tasks while maintaining a Python-centric development experience. - -For the full list of BigQuery standard SQL functions, see: -https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-reference -""" +"""This module integrates BigQuery built-in functions for use with DataFrame objects, +such as array functions: +https://cloud.google.com/bigquery/docs/reference/standard-sql/array_functions. """ import sys -from bigframes.bigquery import aead, ai, ml, obj +from bigframes.bigquery import ai, ml, obj from bigframes.bigquery._operations.approx_agg import approx_top_count -from bigframes.bigquery._operations.array import array_agg +from bigframes.bigquery._operations.array import ( + array_agg, + array_length, + array_to_string, +) from bigframes.bigquery._operations.datetime import ( unix_micros, unix_millis, @@ -68,7 +43,6 @@ st_regionstats, st_simplify, ) -from bigframes.bigquery._operations.io import load_data from bigframes.bigquery._operations.json import ( json_extract, json_extract_array, @@ -83,108 +57,18 @@ to_json, to_json_string, ) -from bigframes.bigquery._operations.mathematical import ( - hparam_candidates, - hparam_range, - rand, -) from bigframes.bigquery._operations.search import create_vector_index, vector_search from bigframes.bigquery._operations.sql import sql_scalar from bigframes.bigquery._operations.struct import struct -from bigframes.bigquery._operations.table import create_external_table from bigframes.core.logging import log_adapter -from bigframes.operations.googlesql.global_namespace.aead_encryption import ( - deterministic_decrypt_bytes, - deterministic_decrypt_string, - deterministic_encrypt, -) -from bigframes.operations.googlesql.global_namespace.array import ( - array_concat, - array_first, - array_first_n, - array_includes, - array_includes_all, - array_includes_any, - array_is_distinct, - array_last, - array_length, - array_reverse, - array_slice, - array_to_string, - flatten, - generate_array, -) -from bigframes.operations.googlesql.global_namespace.bit import ( - bit_count, -) -from bigframes.operations.googlesql.global_namespace.conversion import ( - bool_, - double, - float64, - int64, - parse_bignumeric, - parse_numeric, - string, -) -from bigframes.operations.googlesql.global_namespace.date import ( - current_date, - date, - date_add, - date_diff, - date_from_unix_date, - date_sub, - date_trunc, - extract, - format_date, - generate_date_array, - last_day, - parse_date, - unix_date, -) _functions = [ # approximate aggregate ops approx_top_count, # array ops array_agg, - array_concat, - array_first, - array_first_n, - array_includes, - array_includes_all, - array_includes_any, - array_is_distinct, - array_last, array_length, - array_reverse, - array_slice, array_to_string, - flatten, - generate_array, - # bit ops - bit_count, - # conversion ops - bool_, - double, - float64, - int64, - parse_bignumeric, - parse_numeric, - string, - # date ops - current_date, - date, - date_add, - date_diff, - date_from_unix_date, - date_sub, - date_trunc, - extract, - format_date, - generate_date_array, - last_day, - parse_date, - unix_date, # datetime ops unix_micros, unix_millis, @@ -201,10 +85,6 @@ st_length, st_regionstats, st_simplify, - # deterministic encryption ops - deterministic_decrypt_bytes, - deterministic_decrypt_string, - deterministic_encrypt, # json ops json_extract, json_extract_array, @@ -217,10 +97,6 @@ parse_json, to_json, to_json_string, - # mathematical ops - hparam_candidates, - hparam_range, - rand, # search ops create_vector_index, vector_search, @@ -228,10 +104,6 @@ sql_scalar, # struct ops struct, - # table ops - create_external_table, - # io ops - load_data, ] _module = sys.modules[__name__] @@ -245,44 +117,8 @@ "approx_top_count", # array ops "array_agg", - "array_concat", - "array_first", - "array_first_n", - "array_includes", - "array_includes_all", - "array_includes_any", - "array_is_distinct", - "array_last", "array_length", - "array_reverse", - "array_slice", "array_to_string", - "flatten", - "generate_array", - # bit ops - "bit_count", - # conversion ops - "bool_", - "double", - "float64", - "int64", - "parse_bignumeric", - "parse_numeric", - "string", - # date ops - "current_date", - "date", - "date_add", - "date_diff", - "date_from_unix_date", - "date_sub", - "date_trunc", - "extract", - "format_date", - "generate_date_array", - "last_day", - "parse_date", - "unix_date", # datetime ops "unix_micros", "unix_millis", @@ -299,10 +135,6 @@ "st_length", "st_regionstats", "st_simplify", - # deterministic encryption ops - "deterministic_decrypt_bytes", - "deterministic_decrypt_string", - "deterministic_encrypt", # json ops "json_extract", "json_extract_array", @@ -316,10 +148,6 @@ "parse_json", "to_json", "to_json_string", - # mathematical ops - "hparam_candidates", - "hparam_range", - "rand", # search ops "create_vector_index", "vector_search", @@ -327,12 +155,7 @@ "sql_scalar", # struct ops "struct", - # table ops - "create_external_table", - # io ops - "load_data", # Modules / SQL namespaces - "aead", "ai", "ml", "obj", diff --git a/bigframes/bigquery/_operations/ai.py b/bigframes/bigquery/_operations/ai.py index 40d5556de40..fd7dafe95fc 100644 --- a/bigframes/bigquery/_operations/ai.py +++ b/bigframes/bigquery/_operations/ai.py @@ -19,18 +19,15 @@ from __future__ import annotations import json -from typing import Any, Dict, Iterable, List, Literal, Mapping, Optional, Tuple, Union +from typing import Any, Iterable, List, Literal, Mapping, Tuple, Union import pandas as pd -from bigframes import dataframe, dtypes, series, session +from bigframes import clients, dataframe, dtypes from bigframes import pandas as bpd -from bigframes.bigquery._operations import obj as bq_obj -from bigframes.bigquery._operations import utils as bq_utils +from bigframes import series, session from bigframes.core import convert -from bigframes.core.compile.sqlglot import sql as sg_sql from bigframes.core.logging import log_adapter -from bigframes.ml import base as ml_base from bigframes.ml import core as ml_core from bigframes.operations import ai_ops, output_schemas @@ -49,7 +46,7 @@ def generate( *, connection_id: str | None = None, endpoint: str | None = None, - request_type: Literal["dedicated", "shared", "unspecified"] | None = None, + request_type: Literal["dedicated", "shared", "unspecified"] = "unspecified", model_params: Mapping[Any, Any] | None = None, output_schema: Mapping[str, str] | None = None, ) -> series.Series: @@ -61,17 +58,17 @@ def generate( >>> import bigframes.pandas as bpd >>> import bigframes.bigquery as bbq >>> country = bpd.Series(["Japan", "Canada"]) - >>> bbq.ai.generate(("What's the capital city of ", country, " one word only")) # doctest: +ELLIPSIS + >>> bbq.ai.generate(("What's the capital city of ", country, " one word only")) # doctest: +SKIP 0 {'result': 'Tokyo', 'full_response': '{"cand... 1 {'result': 'Ottawa', 'full_response': '{"can... dtype: struct>, status: string>[pyarrow] - >>> bbq.ai.generate(("What's the capital city of ", country, " one word only")).struct.field("result") + >>> bbq.ai.generate(("What's the capital city of ", country, " one word only")).struct.field("result") # doctest: +SKIP 0 Tokyo 1 Ottawa Name: result, dtype: string - You get structured output when the ``output_schema`` parameter is set: + You get structured output when the `output_schema` parameter is set: >>> animals = bpd.Series(["Rabbit", "Spider"]) >>> bbq.ai.generate(animals, output_schema={"number_of_legs": "INT64", "is_herbivore": "BOOL"}) @@ -79,15 +76,22 @@ def generate( 1 {'is_herbivore': False, 'number_of_legs': 8, '... dtype: struct>, status: string>[pyarrow] + .. note:: + + This product or feature is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). + Args: prompt (str | Series | List[str|Series] | Tuple[str|Series, ...]): A mixture of Series and string literals that specifies the prompt to send to the model. The Series can be BigFrames Series or pandas Series. connection_id (str, optional): - Specifies the connection to use to communicate with the model. For example, ``myproject.us.myconnection``. + Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`. If not provided, the query uses your end-user credential. endpoint (str, optional): - Specifies the Vertex AI endpoint to use for the model. For example ``"gemini-2.5-flash"``. You can specify any + Specifies the Vertex AI endpoint to use for the model. For example `"gemini-2.5-flash"`. You can specify any generally available or preview Gemini model. If you specify the model name, BigQuery ML automatically identifies and uses the full endpoint of the model. If you don't specify an ENDPOINT value, BigQuery ML selects a recent stable version of Gemini to use. @@ -103,7 +107,7 @@ def generate( Provides additional parameters to the model. The MODEL_PARAMS value must conform to the generateContent request body format. output_schema (Mapping[str, str]): A mapping value that specifies the schema of the output, in the form {field_name: data_type}. Supported data types include - ``STRING``, ``INT64``, ``FLOAT64``, ``BOOL``, ``ARRAY``, and ``STRUCT``. + `STRING`, `INT64`, `FLOAT64`, `BOOL`, `ARRAY`, and `STRUCT`. Returns: bigframes.series.Series: A new struct Series with the result data. The struct contains these fields: @@ -130,7 +134,7 @@ def generate( prompt_context=tuple(prompt_context), connection_id=connection_id, endpoint=endpoint, - request_type=_upper_optional(request_type), + request_type=request_type, model_params=json.dumps(model_params) if model_params else None, output_schema=output_schema_str, ) @@ -144,7 +148,7 @@ def generate_bool( *, connection_id: str | None = None, endpoint: str | None = None, - request_type: Literal["dedicated", "shared", "unspecified"] | None = None, + request_type: Literal["dedicated", "shared", "unspecified"] = "unspecified", model_params: Mapping[Any, Any] | None = None, ) -> series.Series: """ @@ -170,15 +174,22 @@ def generate_bool( 2 False Name: result, dtype: boolean + .. note:: + + This product or feature is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). + Args: prompt (str | Series | List[str|Series] | Tuple[str|Series, ...]): A mixture of Series and string literals that specifies the prompt to send to the model. The Series can be BigFrames Series or pandas Series. connection_id (str, optional): - Specifies the connection to use to communicate with the model. For example, ``myproject.us.myconnection``. + Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`. If not provided, the query uses your end-user credential. endpoint (str, optional): - Specifies the Vertex AI endpoint to use for the model. For example ``"gemini-2.5-flash"``. You can specify any + Specifies the Vertex AI endpoint to use for the model. For example `"gemini-2.5-flash"`. You can specify any generally available or preview Gemini model. If you specify the model name, BigQuery ML automatically identifies and uses the full endpoint of the model. If you don't specify an ENDPOINT value, BigQuery ML selects a recent stable version of Gemini to use. @@ -208,7 +219,7 @@ def generate_bool( prompt_context=tuple(prompt_context), connection_id=connection_id, endpoint=endpoint, - request_type=_upper_optional(request_type), + request_type=request_type, model_params=json.dumps(model_params) if model_params else None, ) @@ -221,7 +232,7 @@ def generate_int( *, connection_id: str | None = None, endpoint: str | None = None, - request_type: Literal["dedicated", "shared", "unspecified"] | None = None, + request_type: Literal["dedicated", "shared", "unspecified"] = "unspecified", model_params: Mapping[Any, Any] | None = None, ) -> series.Series: """ @@ -231,8 +242,8 @@ def generate_int( >>> import bigframes.pandas as bpd >>> import bigframes.bigquery as bbq - >>> animal = bpd.Series(["Ostrich", "Rabbit", "Spider"]) - >>> bbq.ai.generate_int(("How many legs does a ", animal, " have?")) # doctest: +ELLIPSIS + >>> animal = bpd.Series(["Kangaroo", "Rabbit", "Spider"]) + >>> bbq.ai.generate_int(("How many legs does a ", animal, " have?")) 0 {'result': 2, 'full_response': '{"candidates":... 1 {'result': 4, 'full_response': '{"candidates":... 2 {'result': 8, 'full_response': '{"candidates":... @@ -244,15 +255,22 @@ def generate_int( 2 8 Name: result, dtype: Int64 + .. note:: + + This product or feature is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). + Args: prompt (str | Series | List[str|Series] | Tuple[str|Series, ...]): A mixture of Series and string literals that specifies the prompt to send to the model. The Series can be BigFrames Series or pandas Series. connection_id (str, optional): - Specifies the connection to use to communicate with the model. For example, ``myproject.us.myconnection``. + Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`. If not provided, the query uses your end-user credential. endpoint (str, optional): - Specifies the Vertex AI endpoint to use for the model. For example ``"gemini-2.5-flash"``. You can specify any + Specifies the Vertex AI endpoint to use for the model. For example `"gemini-2.5-flash"`. You can specify any generally available or preview Gemini model. If you specify the model name, BigQuery ML automatically identifies and uses the full endpoint of the model. If you don't specify an ENDPOINT value, BigQuery ML selects a recent stable version of Gemini to use. @@ -282,7 +300,7 @@ def generate_int( prompt_context=tuple(prompt_context), connection_id=connection_id, endpoint=endpoint, - request_type=_upper_optional(request_type), + request_type=request_type, model_params=json.dumps(model_params) if model_params else None, ) @@ -295,7 +313,7 @@ def generate_double( *, connection_id: str | None = None, endpoint: str | None = None, - request_type: Literal["dedicated", "shared", "unspecified"] | None = None, + request_type: Literal["dedicated", "shared", "unspecified"] = "unspecified", model_params: Mapping[Any, Any] | None = None, ) -> series.Series: """ @@ -305,8 +323,8 @@ def generate_double( >>> import bigframes.pandas as bpd >>> import bigframes.bigquery as bbq - >>> animal = bpd.Series(["Ostrich", "Rabbit", "Spider"]) - >>> bbq.ai.generate_double(("How many legs does a ", animal, " have?")) # doctest: +ELLIPSIS + >>> animal = bpd.Series(["Kangaroo", "Rabbit", "Spider"]) + >>> bbq.ai.generate_double(("How many legs does a ", animal, " have?")) 0 {'result': 2.0, 'full_response': '{"candidates... 1 {'result': 4.0, 'full_response': '{"candidates... 2 {'result': 8.0, 'full_response': '{"candidates... @@ -318,15 +336,22 @@ def generate_double( 2 8.0 Name: result, dtype: Float64 + .. note:: + + This product or feature is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). + Args: prompt (str | Series | List[str|Series] | Tuple[str|Series, ...]): A mixture of Series and string literals that specifies the prompt to send to the model. The Series can be BigFrames Series or pandas Series. connection_id (str, optional): - Specifies the connection to use to communicate with the model. For example, ``myproject.us.myconnection``. + Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`. If not provided, the query uses your end-user credential. endpoint (str, optional): - Specifies the Vertex AI endpoint to use for the model. For example ``"gemini-2.5-flash"``. You can specify any + Specifies the Vertex AI endpoint to use for the model. For example `"gemini-2.5-flash"`. You can specify any generally available or preview Gemini model. If you specify the model name, BigQuery ML automatically identifies and uses the full endpoint of the model. If you don't specify an ENDPOINT value, BigQuery ML selects a recent stable version of Gemini to use. @@ -356,433 +381,21 @@ def generate_double( prompt_context=tuple(prompt_context), connection_id=connection_id, endpoint=endpoint, - request_type=_upper_optional(request_type), + request_type=request_type, model_params=json.dumps(model_params) if model_params else None, ) return series_list[0]._apply_nary_op(operator, series_list[1:]) -@log_adapter.method_logger(custom_base_name="bigquery_ai") -def generate_embedding( - model: Union[ml_base.BaseEstimator, str, pd.Series], - data: Union[dataframe.DataFrame, series.Series, pd.DataFrame, pd.Series], - *, - output_dimensionality: Optional[int] = None, - task_type: Optional[str] = None, - start_second: Optional[float] = None, - end_second: Optional[float] = None, - interval_seconds: Optional[float] = None, - trial_id: Optional[int] = None, -) -> dataframe.DataFrame: - """ - Creates embeddings that describe an entity—for example, a piece of text or an image. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - >>> df = bpd.DataFrame({"content": ["apple", "bear", "pear"]}) - >>> bbq.ai.generate_embedding( # doctest: +SKIP - ... "project.dataset.model_name", - ... df - ... ) - - Args: - model (ml_base.BaseEstimator or str): - The model to use for text embedding. - data (bigframes.pandas.DataFrame or bigframes.pandas.Series): - The data to generate embeddings for. If a Series is provided, it is - treated as the 'content' column. If a DataFrame is provided, it - must contain a 'content' column, or you must rename the column you - wish to embed to 'content'. - output_dimensionality (int, optional): - An INT64 value that specifies the number of dimensions to use when - generating embeddings. For example, if you specify 256 AS - output_dimensionality, then the embedding output column contains a - 256-dimensional embedding for each input value. To find the - supported range of output dimensions, read about the available - `Google text embedding models `_. - task_type (str, optional): - A STRING literal that specifies the intended downstream application to - help the model produce better quality embeddings. For a list of - supported task types and how to choose which one to use, see `Choose an - embeddings task type `_. - start_second (float, optional): - The second in the video at which to start the embedding. The default value is 0. - end_second (float, optional): - The second in the video at which to end the embedding. The default value is 120. - interval_seconds (float, optional): - The interval to use when creating embeddings. The default value is 16. - trial_id (int, optional): - An INT64 value that identifies the hyperparameter tuning trial that - you want the function to evaluate. The function uses the optimal - trial by default. Only specify this argument if you ran - hyperparameter tuning when creating the model. - - Returns: - bigframes.pandas.DataFrame: - A new DataFrame with the generated embeddings. See the `SQL - reference for AI.GENERATE_EMBEDDING - `_ - for details. - """ - data = _to_dataframe(data, series_rename="content") - model_name, session = bq_utils.get_model_name_and_session(model, data) - table_sql = bq_utils.to_sql(data) - - struct_fields: Dict[str, Any] = {} - if output_dimensionality is not None: - struct_fields["OUTPUT_DIMENSIONALITY"] = output_dimensionality - if task_type is not None: - struct_fields["TASK_TYPE"] = task_type - if start_second is not None: - struct_fields["START_SECOND"] = start_second - if end_second is not None: - struct_fields["END_SECOND"] = end_second - if interval_seconds is not None: - struct_fields["INTERVAL_SECONDS"] = interval_seconds - if trial_id is not None: - struct_fields["TRIAL_ID"] = trial_id - - # Construct the TVF query - query = f""" - SELECT * - FROM AI.GENERATE_EMBEDDING( - MODEL `{model_name}`, - ({table_sql}), - {sg_sql.to_sql(sg_sql.literal(struct_fields))} - ) - """ - - if session is None: - return bpd.read_gbq_query(query) - else: - return session.read_gbq_query(query) - - -@log_adapter.method_logger(custom_base_name="bigquery_ai") -def generate_text( - model: Union[ml_base.BaseEstimator, str, pd.Series], - data: Union[dataframe.DataFrame, series.Series, pd.DataFrame, pd.Series], - *, - temperature: Optional[float] = None, - max_output_tokens: Optional[int] = None, - top_k: Optional[int] = None, - top_p: Optional[float] = None, - stop_sequences: Optional[List[str]] = None, - ground_with_google_search: Optional[bool] = None, - request_type: Optional[str] = None, -) -> dataframe.DataFrame: - """ - Generates text using a BigQuery ML model. - - See the `BigQuery ML GENERATE_TEXT function syntax - `_ - for additional reference. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - >>> df = bpd.DataFrame({"prompt": ["write a poem about apples"]}) - >>> bbq.ai.generate_text( # doctest: +SKIP - ... "project.dataset.model_name", - ... df - ... ) - - Args: - model (ml_base.BaseEstimator or str): - The model to use for text generation. - data (bigframes.pandas.DataFrame or bigframes.pandas.Series): - The data to generate text for. If a Series is provided, it is - treated as the 'prompt' column. If a DataFrame is provided, it - must contain a 'prompt' column, or you must rename the column you - wish to generate text to 'prompt'. - temperature (float, optional): - A FLOAT64 value that is used for sampling promiscuity. The value - must be in the range ``[0.0, 1.0]``. A lower temperature works well - for prompts that expect a more deterministic and less open-ended - or creative response, while a higher temperature can lead to more - diverse or creative results. A temperature of ``0`` is - deterministic, meaning that the highest probability response is - always selected. - max_output_tokens (int, optional): - An INT64 value that sets the maximum number of tokens in the - generated text. - top_k (int, optional): - An INT64 value that changes how the model selects tokens for - output. A ``top_k`` of ``1`` means the next selected token is the - most probable among all tokens in the model's vocabulary. A - ``top_k`` of ``3`` means that the next token is selected from - among the three most probable tokens by using temperature. The - default value is ``40``. - top_p (float, optional): - A FLOAT64 value that changes how the model selects tokens for - output. Tokens are selected from most probable to least probable - until the sum of their probabilities equals the ``top_p`` value. - For example, if tokens A, B, and C have a probability of 0.3, 0.2, - and 0.1 and the ``top_p`` value is ``0.5``, then the model will - select either A or B as the next token by using temperature. The - default value is ``0.95``. - stop_sequences (List[str], optional): - An ARRAY value that contains the stop sequences for the model. - ground_with_google_search (bool, optional): - A BOOL value that determines whether to ground the model with Google Search. - request_type (str, optional): - A STRING value that contains the request type for the model. - - Returns: - bigframes.pandas.DataFrame: - The generated text. - """ - data = _to_dataframe(data, series_rename="prompt") - model_name, session = bq_utils.get_model_name_and_session(model, data) - table_sql = bq_utils.to_sql(data) - - struct_fields: Dict[ - str, - Union[str, int, float, bool, Mapping[str, str], List[str], Mapping[str, Any]], - ] = {} - if temperature is not None: - struct_fields["TEMPERATURE"] = temperature - if max_output_tokens is not None: - struct_fields["MAX_OUTPUT_TOKENS"] = max_output_tokens - if top_k is not None: - struct_fields["TOP_K"] = top_k - if top_p is not None: - struct_fields["TOP_P"] = top_p - if stop_sequences is not None: - struct_fields["STEP_SEQUENCES"] = stop_sequences - if ground_with_google_search is not None: - struct_fields["GROUND_WITH_GOOGLE_SEARCH"] = ground_with_google_search - if request_type is not None: - struct_fields["REQUEST_TYPE"] = request_type - - query = f""" - SELECT * - FROM AI.GENERATE_TEXT( - MODEL `{model_name}`, - ({table_sql}), - {sg_sql.to_sql(sg_sql.literal(struct_fields))} - ) - """ - - if session is None: - return bpd.read_gbq_query(query) - else: - return session.read_gbq_query(query) - - -@log_adapter.method_logger(custom_base_name="bigquery_ai") -def generate_table( - model: Union[ml_base.BaseEstimator, str, pd.Series], - data: Union[dataframe.DataFrame, series.Series, pd.DataFrame, pd.Series], - *, - output_schema: Union[str, Mapping[str, str]], - temperature: Optional[float] = None, - top_p: Optional[float] = None, - max_output_tokens: Optional[int] = None, - stop_sequences: Optional[List[str]] = None, - request_type: Optional[str] = None, -) -> dataframe.DataFrame: - """ - Generates a table using a BigQuery ML model. - - See the `AI.GENERATE_TABLE function syntax - `_ - for additional reference. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - >>> # The user is responsible for constructing a DataFrame that contains - >>> # the necessary columns for the model's prompt. For example, a - >>> # DataFrame with a 'prompt' column for text classification. - >>> df = bpd.DataFrame({'prompt': ["some text to classify"]}) - >>> result = bbq.ai.generate_table( # doctest: +SKIP - ... "project.dataset.model_name", - ... data=df, - ... output_schema="category STRING" - ... ) - - Args: - model (ml_base.BaseEstimator or str): - The model to use for table generation. - data (bigframes.pandas.DataFrame or bigframes.pandas.Series): - The data to generate table for. If a Series is provided, it is - treated as the 'prompt' column. If a DataFrame is provided, it - must contain a 'prompt' column, or you must rename the column you - wish to generate table to 'prompt'. - output_schema (str | Mapping[str, str]): - A string defining the output schema (e.g., "col1 STRING, col2 INT64"), - or a mapping value that specifies the schema of the output, in the form {field_name: data_type}. - Supported data types include ``STRING``, ``INT64``, ``FLOAT64``, ``BOOL``, ``ARRAY``, and ``STRUCT``. - temperature (float, optional): - A FLOAT64 value that is used for sampling promiscuity. The value - must be in the range ``[0.0, 1.0]``. - top_p (float, optional): - A FLOAT64 value that changes how the model selects tokens for - output. - max_output_tokens (int, optional): - An INT64 value that sets the maximum number of tokens in the - generated table. - stop_sequences (List[str], optional): - An ARRAY value that contains the stop sequences for the model. - request_type (str, optional): - A STRING value that contains the request type for the model. - - Returns: - bigframes.pandas.DataFrame: - The generated table. - """ - data = _to_dataframe(data, series_rename="prompt") - model_name, session = bq_utils.get_model_name_and_session(model, data) - table_sql = bq_utils.to_sql(data) - - if isinstance(output_schema, Mapping): - output_schema_str = ", ".join( - [f"{name} {sql_type}" for name, sql_type in output_schema.items()] - ) - # Validate user input - output_schemas.parse_sql_fields(output_schema_str) - else: - output_schema_str = output_schema - - struct_fields_bq: Dict[str, Any] = {"output_schema": output_schema_str} - if temperature is not None: - struct_fields_bq["temperature"] = temperature - if top_p is not None: - struct_fields_bq["top_p"] = top_p - if max_output_tokens is not None: - struct_fields_bq["max_output_tokens"] = max_output_tokens - if stop_sequences is not None: - struct_fields_bq["stop_sequences"] = stop_sequences - if request_type is not None: - struct_fields_bq["request_type"] = request_type - - struct_sql = sg_sql.to_sql(sg_sql.literal(struct_fields_bq)) - query = f""" - SELECT * - FROM AI.GENERATE_TABLE( - MODEL `{model_name}`, - ({table_sql}), - {struct_sql} - ) - """ - - if session is None: - return bpd.read_gbq_query(query) - else: - return session.read_gbq_query(query) - - -@log_adapter.method_logger(custom_base_name="bigquery_ai") -def embed( - content: str | series.Series | pd.Series, - *, - endpoint: str | None = None, - model: str | None = None, - task_type: ( - Literal[ - "retrieval_query", - "retrieval_document", - "semantic_similarity", - "classification", - "clustering", - "question_answering", - "fact_verification", - "code_retrieval_query", - ] - | None - ) = None, - title: str | None = None, - model_params: Mapping[Any, Any] | None = None, - connection_id: str | None = None, -) -> series.Series: - """ - Creates embeddings from text or image data in BigQuery. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - >>> bbq.ai.embed("dog", endpoint="text-embedding-005") # doctest: +ELLIPSIS - 0 {'result': array([ 1.78243860e-03, -1.10658340... - dtype: struct, status: string>[pyarrow] - - >>> s = bpd.Series(['dog']) - >>> bbq.ai.embed(s, endpoint='text-embedding-005') # doctest: +ELLIPSIS - 0 {'result': array([ 1.78243860e-03, -1.10658340... - dtype: struct, status: string>[pyarrow] - - Args: - content (str | Series): - A string literal or a Series (either BigFrames series or pandas Series) that provides the text or image to embed. - endpoint (str, optional): - A string value that specifies a supported Vertex AI embedding model endpoint to use. - The endpoint value that you specify must include the model version, for example, - ``"text-embedding-005"``. If you specify this parameter, you can't specify the - ``model`` parameter. - model (str, optional): - A string value that specifies a built-in embedding model. The only supported value is - ``"embeddinggemma-300m"``. If you specify this parameter, you can't specify the ``endpoint``, - ``title``, ``model_params``, or ``connection_id`` parameters. - task_type (str, optional): - A string literal that specifies the intended downstream application to help the model - produce better quality embeddings. Accepts ``"retrieval_query"``, ``"retrieval_document"``, - ``"semantic_similarity"``, ``"classification"``, ``"clustering"``, ``"question_answering"``, - ``"fact_verification"``, ``"code_retrieval_query"``. - title (str, optional): - A string value that specifies the document title, which the model uses to improve - embedding quality. You can only use this parameter if you specify ``"retrieval_document"`` - for the ``task_type`` value. - model_params (Mapping[Any, Any], optional): - A JSON literal that provides additional parameters to the model. For example, - ``{"outputDimensionality": 768}`` lets you specify the number of dimensions to use when - generating embeddings. - connection_id (str, optional): - A STRING value specifying the connection to use to communicate with the model, in the - format ``PROJECT_ID.LOCATION.CONNECTION_ID``. For example, ``myproject.us.myconnection``. - If not provided, the query uses your end-user credential. - - Returns: - bigframes.series.Series: A new struct Series with the result data. The struct contains these fields: - * "result": an ARRAY value containing the generated embeddings. - * "status": a STRING value that contains the API response status for the corresponding row. This value is empty if the operation was successful. - """ - - operator = ai_ops.AIEmbed( - endpoint=endpoint, - model=model, - task_type=_upper_optional(task_type), - title=title, - model_params=json.dumps(model_params) if model_params else None, - connection_id=connection_id, - ) - - if isinstance(content, str): - return series.Series([content])._apply_unary_op(operator) - elif isinstance(content, pd.Series): - return series.Series(content)._apply_unary_op(operator) - elif isinstance(content, series.Series): - return content._apply_unary_op(operator) - else: - raise ValueError(f"Unsupported 'content' parameter type: {type(content)}") - - @log_adapter.method_logger(custom_base_name="bigquery_ai") def if_( prompt: PROMPT_TYPE, *, connection_id: str | None = None, - endpoint: str | None = None, - optimization_mode: Literal["minimize_cost", "maximize_quality"] | None = None, - max_error_ratio: float | None = None, ) -> series.Series: """ - Evaluates the prompt to True or False. Compared to ``ai.generate_bool()``, this function + Evaluates the prompt to True or False. Compared to `ai.generate_bool()`, this function provides optimization such that not all rows are evaluated with the LLM. **Examples:** @@ -801,26 +414,20 @@ def if_( 1 Illinois dtype: string + .. note:: + + This product or feature is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). + Args: prompt (str | Series | List[str|Series] | Tuple[str|Series, ...]): A mixture of Series and string literals that specifies the prompt to send to the model. The Series can be BigFrames Series or pandas Series. connection_id (str, optional): - Specifies the connection to use to communicate with the model. For example, ``myproject.us.myconnection``. - If not provided, the query uses your end-user credential. - endpoint (str, optional): - Specifies the Vertex AI endpoint to use for the model. For example ``"gemini-2.5-flash"``. You can specify any - generally available or preview Gemini model. If you specify the model name, BigQuery ML automatically identifies and - uses the full endpoint of the model. If you don't specify an ENDPOINT value, BigQuery ML dynamically chooses a model based on your query to have the - best cost to quality tradeoff for the task. - optimization_mode (Literal["minimize_cost", "maximize_quality"]): - Specifies the optimization strategy to use. Supported values are: - * "minimize_cost" (default): uses a local, distilled model to process the majority of rows, reducing latency and cost. - * "maximize_quality": always uses the remote LLM for inference. - max_error_ratio (float): - A float value between 0.0 and 1.0 that contains the maximum acceptable ratio of row-level inference failures to - rows processed on this function. If this value is exceeded, then the query fails. The default value is 1.0. - This argument isn't supported when ``optimization_mode`` is set to "minimize_cost". + Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`. + If not provided, the connection from the current session will be used. Returns: bigframes.series.Series: A new series of bools. @@ -831,10 +438,7 @@ def if_( operator = ai_ops.AIIf( prompt_context=tuple(prompt_context), - connection_id=connection_id, - endpoint=endpoint, - optimization_mode=_upper_optional(optimization_mode), - max_error_ratio=max_error_ratio, + connection_id=_resolve_connection_id(series_list[0], connection_id), ) return series_list[0]._apply_nary_op(operator, series_list[1:]) @@ -845,14 +449,7 @@ def classify( input: PROMPT_TYPE, categories: tuple[str, ...] | list[str], *, - examples: list[tuple[str, str]] - | list[tuple[str, list[str] | tuple[str, ...]]] - | None = None, connection_id: str | None = None, - endpoint: str | None = None, - output_mode: Literal["single", "multi"] | None = None, - optimization_mode: Literal["minimize_cost", "maximize_quality"] | None = None, - max_error_ratio: float | None = None, ) -> series.Series: """ Classifies a given input into one of the specified categories. It will always return one of the provided categories best fit the prompt input. @@ -870,59 +467,34 @@ def classify( [2 rows x 2 columns] + .. note:: + + This product or feature is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). + Args: input (str | Series | List[str|Series] | Tuple[str|Series, ...]): A mixture of Series and string literals that specifies the input to send to the model. The Series can be BigFrames Series or pandas Series. categories (tuple[str, ...] | list[str]): Categories to classify the input into. - examples (list[tuple[str, str]] | list[tuple[str, list[str] | tuple[str, ...]]], optional): - An array that contains representative examples of input strings and the output category - that you expect. If ``output_mode`` is ``multi``, each example output must be a list or tuple of strings. - You can provide examples to help the model understand your intended threshold for a condition with nuanced - or subjective logic. We recommend providing at most 5 examples. connection_id (str, optional): - Specifies the connection to use to communicate with the model. For example, ``myproject.us.myconnection``. - If not provided, the query uses your end-user credential. - endpoint (str, optional): - A STRING value that specifies the Vertex AI endpoint to use for the model. You can specify any - generally available or preview Gemini model. If you specify the model name, BigQuery ML automatically - identifies and uses the full endpoint of the model. - output_mode (Literal["single", "multi"], optional): - A STRING value that indicates whether a single input can be classified into multiple categories. - Supported values are ``single`` and ``multi``. - optimization_mode (Literal["minimize_cost", "maximize_quality"], optional): - A STRING value that specifies the optimization strategy to use. Supported values are ``minimize_cost`` - and ``maximize_quality``. - max_error_ratio (float, optional): - A value between ``0.0`` and ``1.0`` that contains the maximum acceptable ratio of row-level - inference failures to rows processed on this function. The default value is 1.0. - This argument isn't supported when ``optimization_mode`` is set to ``minimize_cost``. + Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`. + If not provided, the connection from the current session will be used. Returns: - bigframes.series.Series: A new series of strings (or a series of arrays of strings if ``output_mode`` is specified). + bigframes.series.Series: A new series of strings. """ prompt_context, series_list = _separate_context_and_series(input) assert len(series_list) > 0 - if examples is not None: - example_tuples: Any = tuple( - (ex[0], tuple(ex[1]) if isinstance(ex[1], (list, tuple)) else ex[1]) - for ex in examples - ) - else: - example_tuples = None - operator = ai_ops.AIClassify( prompt_context=tuple(prompt_context), categories=tuple(categories), - examples=example_tuples, - connection_id=connection_id, - endpoint=endpoint, - output_mode=output_mode, - optimization_mode=_upper_optional(optimization_mode), - max_error_ratio=max_error_ratio, + connection_id=_resolve_connection_id(series_list[0], connection_id), ) return series_list[0]._apply_nary_op(operator, series_list[1:]) @@ -933,8 +505,6 @@ def score( prompt: PROMPT_TYPE, *, connection_id: str | None = None, - endpoint: str | None = None, - max_error_ratio: float | None = None, ) -> series.Series: """ Computes a score based on rubrics described in natural language. It will return a double value. @@ -946,27 +516,26 @@ def score( >>> import bigframes.pandas as bpd >>> import bigframes.bigquery as bbq >>> animal = bpd.Series(["Tiger", "Rabbit", "Blue Whale"]) - >>> bbq.ai.score(("Rank the relative weights of ", animal, " on the scale from 1 to 3")) + >>> bbq.ai.score(("Rank the relative weights of ", animal, " on the scale from 1 to 3")) # doctest: +SKIP 0 2.0 1 1.0 2 3.0 dtype: Float64 + .. note:: + + This product or feature is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). + Args: prompt (str | Series | List[str|Series] | Tuple[str|Series, ...]): A mixture of Series and string literals that specifies the prompt to send to the model. The Series can be BigFrames Series or pandas Series. connection_id (str, optional): - Specifies the connection to use to communicate with the model. For example, ``myproject.us.myconnection``. - If not provided, the query uses your end-user credential. - endpoint (str, optional): - Specifies the Vertex AI endpoint to use for the model. For example ``"gemini-2.5-flash"``. You can specify any - generally available or preview Gemini model. If you specify the model name, BigQuery ML automatically identifies and - uses the full endpoint of the model. If you don't specify an endpoint value, BigQuery ML dynamically chooses a model - based on your query to have the best cost to quality tradeoff for the task. - max_error_ratio (float, optional): - A value between ``0.0`` and ``1.0`` that contains the maximum acceptable ratio of row-level inference failures to - rows processed on this function. If this value is exceeded, then the query fails. + Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`. + If not provided, the connection from the current session will be used. Returns: bigframes.series.Series: A new series of double (float) values. @@ -977,89 +546,12 @@ def score( operator = ai_ops.AIScore( prompt_context=tuple(prompt_context), - connection_id=connection_id, - endpoint=endpoint, - max_error_ratio=max_error_ratio, + connection_id=_resolve_connection_id(series_list[0], connection_id), ) return series_list[0]._apply_nary_op(operator, series_list[1:]) -@log_adapter.method_logger(custom_base_name="bigquery_ai") -def similarity( - content1: str | series.Series | pd.Series, - content2: str | series.Series | pd.Series, - *, - endpoint: str | None = None, - model: str | None = None, - model_params: Mapping[Any, Any] | None = None, - connection_id: str | None = None, -) -> series.Series: - """ - Returns a FLOAT64 value that represents the cosine similarity between the two inputs. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - >>> df = bpd.DataFrame({'word': ['happy', 'sad']}) - >>> bbq.ai.similarity(df['word'], 'glad', endpoint='text-embedding-005') - 0 0.916601 - 1 0.660579 - Name: word, dtype: Float64 - - Args: - content1 (str | Series): - A string or series that provides the first value to compare. Both a BigFrames Series or a pandas Series are allowed. - content2 (str | Series): - A string or series that provides the second value to compare. Both a BigFrames Series or a pandas Series are allowed. - endpoint (str, optional): - Specifies the Vertex AI endpoint to use for the text embedding model. - If you specify the model name, such as ``'text-embedding-005'``, rather than a URL, then BigQuery ML automatically identifies the model and uses the model's full endpoint. - model (str, optional): - Specifies a built-in text embedding model. The only supported value is the embeddinggemma-300m model. - If you specify this parameter, you can't specify the ``endpoint``, ``model_params``, or ``connection_id`` parameters. - model_params (Mapping[Any, Any], optional): - Provides additional parameters to the model. You can use any of the parameters object fields. - One of these fields, ``outputDimensionality``, lets you specify the number of dimensions to use when generating embeddings. - connection_id (str, optional): - Specifies the connection to use to communicate with the model. For example, ``myproject.us.myconnection``. - - Returns: - bigframes.series.Series: A new series of FLOAT64 values representing the cosine similarity. - """ - - operator = ai_ops.AISimilarity( - endpoint=endpoint, - model=model, - model_params=json.dumps(model_params) if model_params else None, - connection_id=connection_id, - ) - - # Find a unifying session for the subsequent operations. - bf_session = None - if isinstance(content1, series.Series): - bf_session = content1._session - elif isinstance(content2, series.Series): - bf_session = content2._session - - if isinstance(content1, str) and isinstance(content2, str): - content1 = series.Series([content1], session=bf_session) - return content1._apply_binary_op(content2, operator) - elif isinstance(content1, str): - # content2 must be a series - content2 = convert.to_bf_series( - content2, default_index=None, session=bf_session - ) - return content2._apply_binary_op(content1, operator) - else: - # content1 must be a series. - content1 = convert.to_bf_series( - content1, default_index=None, session=bf_session - ) - return content1._apply_binary_op(content2, operator) - - @log_adapter.method_logger(custom_base_name="bigquery_ai") def forecast( df: dataframe.DataFrame | pd.DataFrame, @@ -1070,30 +562,17 @@ def forecast( id_cols: Iterable[str] | None = None, horizon: int = 10, confidence_level: float = 0.95, - output_historical_time_series: bool = False, context_window: int | None = None, ) -> dataframe.DataFrame: """ Forecast time series at future horizon. Using Google Research's open source TimesFM(https://github.com/google-research/timesfm) model. - **Examples:** - - Forecast using a pandas DataFrame: - - >>> import pandas as pd - >>> import bigframes.pandas as bpd - >>> df = pd.DataFrame({"value": [1, 2, 3], "time": pd.to_datetime(["2020-01-01", "2020-01-02", "2020-01-03"])}) - >>> bpd.options.display.progress_bar = None - >>> forecasted_pandas_df = df.bigquery.ai.forecast(data_col="value", timestamp_col="time", horizon=2) - >>> type(forecasted_pandas_df) # doctest: +ELLIPSIS - - - Forecast using a BigFrames DataFrame: + .. note:: - >>> bf_df = bpd.DataFrame({"value": [1, 2, 3], "time": pd.to_datetime(["2020-01-01", "2020-01-02", "2020-01-03"])}) - >>> forecasted_bf_df = bf_df.bigquery.ai.forecast(data_col="value", timestamp_col="time", horizon=2) - >>> type(forecasted_bf_df) - + This product or feature is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). Args: df (DataFrame): @@ -1117,15 +596,6 @@ def forecast( confidence_level (float, default 0.95): A FLOAT64 value that specifies the percentage of the future values that fall in the prediction interval. The default value is 0.95. The valid input range is [0, 1). - output_historical_time_series (bool, default False): - A BOOL value that determines whether the input data is returned - along with the forecasted data. Set this argument to TRUE to return - input data. The default value is FALSE. - - Returning the input data along with the forecasted data lets you - compare the historical value of the data column with the forecasted - value of the data column, or chart the change in the data column - values over time. context_window (int, optional): An int value that specifies the context window length used by BigQuery ML's built-in TimesFM model. The context window length determines how many of the most recent data points from the input time series are use by the model. @@ -1157,7 +627,6 @@ def forecast( "timestamp_col": timestamp_col, "model": model, "horizon": horizon, - "output_historical_time_series": output_historical_time_series, "confidence_level": confidence_level, } if id_cols: @@ -1178,19 +647,16 @@ def _separate_context_and_series( Input: ("str1", series1, "str2", "str3", series2) Output: ["str1", None, "str2", "str3", None], [series1, series2] """ - if not isinstance(prompt, (str, list, tuple, series.Series, pd.Series)): + if not isinstance(prompt, (str, list, tuple, series.Series)): raise ValueError(f"Unsupported prompt type: {type(prompt)}") if isinstance(prompt, str): return [None], [series.Series([prompt])] - if isinstance(prompt, pd.Series): - return [None], [bpd.read_pandas(prompt)] - if isinstance(prompt, series.Series): if prompt.dtype == dtypes.OBJ_REF_DTYPE: # Multi-model support - return [None], [bq_obj.get_access_url(prompt, mode="R")] + return [None], [prompt.blob.read_url()] return [None], [prompt] prompt_context: List[str | None] = [] @@ -1226,29 +692,14 @@ def _convert_series( result = convert.to_bf_series(s, default_index=None, session=session) if result.dtype == dtypes.OBJ_REF_DTYPE: - # Support multimodal - return bq_obj.get_access_url(result, mode="R") + # Support multimodel + return result.blob.read_url() return result -def _to_dataframe( - data: Union[dataframe.DataFrame, series.Series, pd.DataFrame, pd.Series], - series_rename: str, -) -> dataframe.DataFrame: - if isinstance(data, (pd.DataFrame, pd.Series)): - data = bpd.read_pandas(data) - - if isinstance(data, series.Series): - data = data.copy() - data.name = series_rename - return data.to_frame() - elif isinstance(data, dataframe.DataFrame): - return data - - raise ValueError(f"Unsupported data type: {type(data)}") - - -def _upper_optional(value: str | None) -> str | None: - if value is None: - return None - return value.upper() +def _resolve_connection_id(series: series.Series, connection_id: str | None): + return clients.get_canonical_bq_connection_id( + connection_id or series._session._bq_connection, + series._session._project, + series._session._location, + ) diff --git a/bigframes/bigquery/_operations/array.py b/bigframes/bigquery/_operations/array.py index 0a3c5d66217..6f9dd20b544 100644 --- a/bigframes/bigquery/_operations/array.py +++ b/bigframes/bigquery/_operations/array.py @@ -17,6 +17,7 @@ https://cloud.google.com/bigquery/docs/reference/standard-sql/array_functions """ + from __future__ import annotations import typing @@ -24,6 +25,7 @@ import bigframes_vendored.constants as constants import bigframes.core.groupby as groupby +import bigframes.operations as ops import bigframes.operations.aggregations as agg_ops import bigframes.series as series @@ -31,6 +33,40 @@ import bigframes.dataframe as dataframe +def array_length(series: series.Series) -> series.Series: + """Compute the length of each array element in the Series. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> import bigframes.bigquery as bbq + + >>> s = bpd.Series([[1, 2, 8, 3], [], [3, 4]]) + >>> bbq.array_length(s) + 0 4 + 1 0 + 2 2 + dtype: Int64 + + You can also apply this function directly to Series. + + >>> s.apply(bbq.array_length, by_row=False) + 0 4 + 1 0 + 2 2 + dtype: Int64 + + Args: + series (bigframes.series.Series): A Series with array columns. + + Returns: + bigframes.series.Series: A Series of integer values indicating + the length of each element in the Series. + + """ + return series._apply_unary_op(ops.len_op) + + def array_agg( obj: groupby.SeriesGroupBy | groupby.DataFrameGroupBy, ) -> series.Series | dataframe.DataFrame: @@ -80,3 +116,31 @@ def array_agg( raise ValueError( f"Unsupported type {type(obj)} to apply `array_agg` function. {constants.FEEDBACK_LINK}" ) + + +def array_to_string(series: series.Series, delimiter: str) -> series.Series: + """Converts array elements within a Series into delimited strings. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> import bigframes.bigquery as bbq + + >>> s = bpd.Series([["H", "i", "!"], ["Hello", "World"], np.nan, [], ["Hi"]]) + >>> bbq.array_to_string(s, delimiter=", ") + 0 H, i, ! + 1 Hello, World + 2 + 3 + 4 Hi + dtype: string + + Args: + series (bigframes.series.Series): A Series containing arrays. + delimiter (str): The string used to separate array elements. + + Returns: + bigframes.series.Series: A Series containing delimited strings. + + """ + return series._apply_unary_op(ops.ArrayToStringOp(delimiter=delimiter)) diff --git a/bigframes/bigquery/_operations/geo.py b/bigframes/bigquery/_operations/geo.py index e9ea711c969..f0fda99a166 100644 --- a/bigframes/bigquery/_operations/geo.py +++ b/bigframes/bigquery/_operations/geo.py @@ -19,10 +19,10 @@ import shapely # type: ignore +from bigframes import operations as ops import bigframes.dataframe import bigframes.geopandas import bigframes.series -from bigframes import operations as ops """ Search functions defined from @@ -99,7 +99,7 @@ def st_area( bigframes.pandas.Series: Series of float representing the areas. """ - series = series._apply_nary_op(ops.googlesql.ST_AREA, []) + series = series._apply_unary_op(ops.geo_area_op) series.name = None return series @@ -223,7 +223,7 @@ def st_centroid( bigframes.pandas.Series: A series of geography objects representing the centroids. """ - series = series._apply_nary_op(ops.googlesql.ST_CENTROID, []) + series = series._apply_unary_op(ops.geo_st_centroid_op) series.name = None return series @@ -753,4 +753,6 @@ def st_simplify( Returns: a Series containing the simplified GEOGRAPHY data. """ - return geography._apply_nary_op(ops.googlesql.ST_SIMPLIFY, [tolerance_meters]) + return geography._apply_unary_op( + ops.GeoStSimplifyOp(tolerance_meters=tolerance_meters) + ) diff --git a/bigframes/bigquery/_operations/io.py b/bigframes/bigquery/_operations/io.py deleted file mode 100644 index bf9eae95660..00000000000 --- a/bigframes/bigquery/_operations/io.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from typing import Mapping, Optional, Union - -import pandas as pd - -import bigframes.core.compile.sqlglot.sql as sql -import bigframes.core.logging.log_adapter as log_adapter -import bigframes.session -from bigframes.bigquery._operations.table import _get_table_metadata - - -@log_adapter.method_logger(custom_base_name="bigquery_io") -def load_data( - table_name: str, - *, - write_disposition: str = "INTO", - columns: Optional[Mapping[str, str]] = None, - partition_by: Optional[list[str]] = None, - cluster_by: Optional[list[str]] = None, - table_options: Optional[Mapping[str, Union[str, int, float, bool, list]]] = None, - from_files_options: Mapping[str, Union[str, int, float, bool, list]], - with_partition_columns: Optional[Mapping[str, str]] = None, - connection_name: Optional[str] = None, - session: Optional[bigframes.session.Session] = None, -) -> pd.Series: - """ - Loads data into a BigQuery table. - See the `BigQuery LOAD DATA DDL syntax - `_ - for additional reference. - Args: - table_name (str): - The name of the table in BigQuery. - write_disposition (str, default "INTO"): - Whether to replace the table if it already exists ("OVERWRITE") or append to it ("INTO"). - columns (Mapping[str, str], optional): - The table's schema. - partition_by (list[str], optional): - A list of partition expressions to partition the table by. See https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/load-statements#partition_expression. - cluster_by (list[str], optional): - A list of columns to cluster the table by. - table_options (Mapping[str, Union[str, int, float, bool, list]], optional): - The table options. - from_files_options (Mapping[str, Union[str, int, float, bool, list]]): - The options for loading data from files. - with_partition_columns (Mapping[str, str], optional): - The table's partition columns. - connection_name (str, optional): - The connection to use for the table. - session (bigframes.session.Session, optional): - The session to use. If not provided, the default session is used. - Returns: - pandas.Series: - A Series with object dtype containing the table metadata. Reference - the `BigQuery Table REST API reference - `_ - for available fields. - """ - import bigframes.pandas as bpd - - load_data_expr = sql.load_data( - table_name=table_name, - write_disposition=write_disposition, - columns=columns, - partition_by=partition_by, - cluster_by=cluster_by, - table_options=table_options, - from_files_options=from_files_options, - with_partition_columns=with_partition_columns, - connection_name=connection_name, - ) - sql_text = sql.to_sql(load_data_expr) - - if session is None: - bpd.read_gbq_query(sql_text) - session = bpd.get_global_session() - else: - session.read_gbq_query(sql_text) - - return _get_table_metadata(bqclient=session.bqclient, table_name=table_name) diff --git a/bigframes/bigquery/_operations/json.py b/bigframes/bigquery/_operations/json.py index 8afb234b719..0fc184b2fca 100644 --- a/bigframes/bigquery/_operations/json.py +++ b/bigframes/bigquery/_operations/json.py @@ -18,10 +18,11 @@ https://cloud.google.com/bigquery/docs/reference/standard-sql/json_functions """ + from __future__ import annotations +from typing import Any, cast, Optional, Sequence, Tuple, Union import warnings -from typing import Any, Optional, Sequence, Tuple, Union, cast import bigframes.core.utils as utils import bigframes.dtypes diff --git a/bigframes/bigquery/_operations/mathematical.py b/bigframes/bigquery/_operations/mathematical.py deleted file mode 100644 index 5e6a299f83f..00000000000 --- a/bigframes/bigquery/_operations/mathematical.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from typing import Sequence - -import bigframes.core.col -import bigframes.core.expression -from bigframes import dtypes -from bigframes import operations as ops -from bigframes.operations import googlesql - - -def rand() -> bigframes.core.col.Expression: - """ - Generates a pseudo-random value of type FLOAT64 in the range of [0, 1), - inclusive of 0 and exclusive of 1. - - .. warning:: - This method introduces non-determinism to the expression. Reading the - same column twice may result in different results. The value might - change. Do not use this value or any value derived from it as a join - key. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - >>> df = bpd.DataFrame({"a": [1, 2, 3]}) - >>> df['random'] = bbq.rand() - >>> # Resulting column 'random' will contain random floats between 0 and 1. - - Returns: - bigframes.pandas.api.typing.Expression: - An expression that can be used in - :func:`~bigframes.pandas.DataFrame.assign` and other methods. See - :func:`bigframes.pandas.col`. - """ - return bigframes.core.col.Expression( - bigframes.core.expression.OpExpression(googlesql.RAND, ()) - ) - - -def hparam_range(min: float, max: float) -> bigframes.core.col.Expression: - """ - Defines the minimum and maximum bounds of the search space of continuous - values for a hyperparameter. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - >>> # Specify a range of values for a hyperparameter. - >>> learn_rate = bbq.hparam_range(0.0001, 1.0) - - Args: - min (float or int): - The minimum bound of the search space. - max (float or int): - The maximum bound of the search space. - - Returns: - bigframes.pandas.api.typing.Expression: - An expression that can be used in model options. - """ - min_expr = bigframes.core.expression.const(min) - max_expr = bigframes.core.expression.const(max) - - op = ops.SqlScalarOp( - _output_type=dtypes.FLOAT_DTYPE, - sql_template="HPARAM_RANGE({0}, {1})", - is_deterministic=True, - ) - return bigframes.core.col.Expression( - bigframes.core.expression.OpExpression(op, (min_expr, max_expr)) - ) - - -def hparam_candidates( - candidates: Sequence[float | str], -) -> bigframes.core.col.Expression: - """ - Specifies the set of discrete values for the hyperparameter. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - >>> # Specify a set of values for a hyperparameter. - >>> optimizer = bbq.hparam_candidates(['ADAGRAD', 'SGD', 'FTRL']) - - Args: - candidates (Sequence[float | str]): - The set of discrete values for the hyperparameter. - - Returns: - bigframes.pandas.api.typing.Expression: - An expression that can be used in model options. - """ - candidates_expr = bigframes.core.expression.const(tuple(candidates)) - - op = ops.SqlScalarOp( - _output_type=dtypes.STRING_DTYPE, - sql_template="HPARAM_CANDIDATES({0})", - is_deterministic=True, - ) - return bigframes.core.col.Expression( - bigframes.core.expression.OpExpression(op, (candidates_expr,)) - ) diff --git a/bigframes/bigquery/_operations/ml.py b/bigframes/bigquery/_operations/ml.py index c6ef1f8bb7a..c9b48bb5ac7 100644 --- a/bigframes/bigquery/_operations/ml.py +++ b/bigframes/bigquery/_operations/ml.py @@ -14,19 +14,64 @@ from __future__ import annotations -from typing import List, Mapping, Optional, Union +from typing import cast, Mapping, Optional, Union import bigframes_vendored.constants import google.cloud.bigquery import pandas as pd -import bigframes.core.col as col import bigframes.core.logging.log_adapter as log_adapter import bigframes.core.sql.ml import bigframes.dataframe as dataframe import bigframes.ml.base import bigframes.session -from bigframes.bigquery._operations import utils + + +# Helper to convert DataFrame to SQL string +def _to_sql(df_or_sql: Union[pd.DataFrame, dataframe.DataFrame, str]) -> str: + import bigframes.pandas as bpd + + if isinstance(df_or_sql, str): + return df_or_sql + + if isinstance(df_or_sql, pd.DataFrame): + bf_df = bpd.read_pandas(df_or_sql) + else: + bf_df = cast(dataframe.DataFrame, df_or_sql) + + # Cache dataframes to make sure base table is not a snapshot. + # Cached dataframe creates a full copy, never uses snapshot. + # This is a workaround for internal issue b/310266666. + bf_df.cache() + sql, _, _ = bf_df._to_sql_query(include_index=False) + return sql + + +def _get_model_name_and_session( + model: Union[bigframes.ml.base.BaseEstimator, str, pd.Series], + # Other dataframe arguments to extract session from + *dataframes: Optional[Union[pd.DataFrame, dataframe.DataFrame, str]], +) -> tuple[str, Optional[bigframes.session.Session]]: + if isinstance(model, pd.Series): + try: + model_ref = model["modelReference"] + model_name = f"{model_ref['projectId']}.{model_ref['datasetId']}.{model_ref['modelId']}" # type: ignore + except KeyError: + raise ValueError("modelReference must be present in the pandas Series.") + elif isinstance(model, str): + model_name = model + else: + if model._bqml_model is None: + raise ValueError("Model must be fitted to be used in ML operations.") + return model._bqml_model.model_name, model._bqml_model.session + + session = None + for df in dataframes: + if isinstance(df, dataframe.DataFrame): + session = df._session + break + + return model_name, session def _get_model_metadata( @@ -51,9 +96,7 @@ def create_model( input_schema: Optional[Mapping[str, str]] = None, output_schema: Optional[Mapping[str, str]] = None, connection_name: Optional[str] = None, - options: Optional[ - Mapping[str, Union[str, int, float, bool, list, "col.Expression"]] - ] = None, + options: Optional[Mapping[str, Union[str, int, float, bool, list]]] = None, training_data: Optional[Union[pd.DataFrame, dataframe.DataFrame, str]] = None, custom_holiday: Optional[Union[pd.DataFrame, dataframe.DataFrame, str]] = None, session: Optional[bigframes.session.Session] = None, @@ -81,7 +124,7 @@ def create_model( The OUTPUT clause, which specifies the schema of the output data. connection_name (str, optional): The connection to use for the model. - options (Mapping[str, Union[str, int, float, bool, list, bigframes.core.col.Expression]], optional): + options (Mapping[str, Union[str, int, float, bool, list]], optional): The OPTIONS clause, which specifies the model options. training_data (Union[bigframes.pandas.DataFrame, str], optional): The query or DataFrame to use for training the model. @@ -100,12 +143,8 @@ def create_model( """ import bigframes.pandas as bpd - training_data_sql = ( - utils.to_sql(training_data) if training_data is not None else None - ) - custom_holiday_sql = ( - utils.to_sql(custom_holiday) if custom_holiday is not None else None - ) + training_data_sql = _to_sql(training_data) if training_data is not None else None + custom_holiday_sql = _to_sql(custom_holiday) if custom_holiday is not None else None # Determine session from DataFrames if not provided if session is None: @@ -134,9 +173,9 @@ def create_model( if session is None: bpd.read_gbq_query(sql) session = bpd.get_global_session() - assert session is not None, ( - f"Missing connection to BigQuery. Please report how you encountered this error at {bigframes_vendored.constants.FEEDBACK_LINK}." - ) + assert ( + session is not None + ), f"Missing connection to BigQuery. Please report how you encountered this error at {bigframes_vendored.constants.FEEDBACK_LINK}." else: session.read_gbq_query(sql) @@ -188,8 +227,8 @@ def evaluate( """ import bigframes.pandas as bpd - model_name, session = utils.get_model_name_and_session(model, input_) - table_sql = utils.to_sql(input_) if input_ is not None else None + model_name, session = _get_model_name_and_session(model, input_) + table_sql = _to_sql(input_) if input_ is not None else None sql = bigframes.core.sql.ml.evaluate( model_name=model_name, @@ -242,8 +281,8 @@ def predict( """ import bigframes.pandas as bpd - model_name, session = utils.get_model_name_and_session(model, input_) - table_sql = utils.to_sql(input_) + model_name, session = _get_model_name_and_session(model, input_) + table_sql = _to_sql(input_) sql = bigframes.core.sql.ml.predict( model_name=model_name, @@ -301,8 +340,8 @@ def explain_predict( """ import bigframes.pandas as bpd - model_name, session = utils.get_model_name_and_session(model, input_) - table_sql = utils.to_sql(input_) + model_name, session = _get_model_name_and_session(model, input_) + table_sql = _to_sql(input_) sql = bigframes.core.sql.ml.explain_predict( model_name=model_name, @@ -344,7 +383,7 @@ def global_explain( """ import bigframes.pandas as bpd - model_name, session = utils.get_model_name_and_session(model) + model_name, session = _get_model_name_and_session(model) sql = bigframes.core.sql.ml.global_explain( model_name=model_name, class_level_explain=class_level_explain, @@ -354,223 +393,3 @@ def global_explain( return bpd.read_gbq_query(sql) else: return session.read_gbq_query(sql) - - -@log_adapter.method_logger(custom_base_name="bigquery_ml") -def transform( - model: Union[bigframes.ml.base.BaseEstimator, str, pd.Series], - input_: Union[pd.DataFrame, dataframe.DataFrame, str], -) -> dataframe.DataFrame: - """ - Transforms input data using a BigQuery ML model. - - See the `BigQuery ML TRANSFORM function syntax - `_ - for additional reference. - - Args: - model (bigframes.ml.base.BaseEstimator or str): - The model to use for transformation. - input_ (Union[bigframes.pandas.DataFrame, str]): - The DataFrame or query to use for transformation. - - Returns: - bigframes.pandas.DataFrame: - The transformed data. - """ - import bigframes.pandas as bpd - - model_name, session = utils.get_model_name_and_session(model, input_) - table_sql = utils.to_sql(input_) - - sql = bigframes.core.sql.ml.transform( - model_name=model_name, - table=table_sql, - ) - - if session is None: - return bpd.read_gbq_query(sql) - else: - return session.read_gbq_query(sql) - - -@log_adapter.method_logger(custom_base_name="bigquery_ml") -def generate_text( - model: Union[bigframes.ml.base.BaseEstimator, str, pd.Series], - input_: Union[pd.DataFrame, dataframe.DataFrame, str], - *, - temperature: Optional[float] = None, - max_output_tokens: Optional[int] = None, - top_k: Optional[int] = None, - top_p: Optional[float] = None, - flatten_json_output: Optional[bool] = None, - stop_sequences: Optional[List[str]] = None, - ground_with_google_search: Optional[bool] = None, - request_type: Optional[str] = None, -) -> dataframe.DataFrame: - """ - Generates text using a BigQuery ML model. - - See the `BigQuery ML GENERATE_TEXT function syntax - `_ - for additional reference. - - Args: - model (bigframes.ml.base.BaseEstimator or str): - The model to use for text generation. - input_ (Union[bigframes.pandas.DataFrame, str]): - The DataFrame or query to use for text generation. - temperature (float, optional): - A FLOAT64 value that is used for sampling promiscuity. The value - must be in the range ``[0.0, 1.0]``. A lower temperature works well - for prompts that expect a more deterministic and less open-ended - or creative response, while a higher temperature can lead to more - diverse or creative results. A temperature of ``0`` is - deterministic, meaning that the highest probability response is - always selected. - max_output_tokens (int, optional): - An INT64 value that sets the maximum number of tokens in the - generated text. - top_k (int, optional): - An INT64 value that changes how the model selects tokens for - output. A ``top_k`` of ``1`` means the next selected token is the - most probable among all tokens in the model's vocabulary. A - ``top_k`` of ``3`` means that the next token is selected from - among the three most probable tokens by using temperature. The - default value is ``40``. - top_p (float, optional): - A FLOAT64 value that changes how the model selects tokens for - output. Tokens are selected from most probable to least probable - until the sum of their probabilities equals the ``top_p`` value. - For example, if tokens A, B, and C have a probability of 0.3, 0.2, - and 0.1 and the ``top_p`` value is ``0.5``, then the model will - select either A or B as the next token by using temperature. The - default value is ``0.95``. - flatten_json_output (bool, optional): - A BOOL value that determines the content of the generated JSON column. - stop_sequences (List[str], optional): - An ARRAY value that contains the stop sequences for the model. - ground_with_google_search (bool, optional): - A BOOL value that determines whether to ground the model with Google Search. - request_type (str, optional): - A STRING value that contains the request type for the model. - - Returns: - bigframes.pandas.DataFrame: - The generated text. - """ - import bigframes.pandas as bpd - - model_name, session = utils.get_model_name_and_session(model, input_) - table_sql = utils.to_sql(input_) - - sql = bigframes.core.sql.ml.generate_text( - model_name=model_name, - table=table_sql, - temperature=temperature, - max_output_tokens=max_output_tokens, - top_k=top_k, - top_p=top_p, - flatten_json_output=flatten_json_output, - stop_sequences=stop_sequences, - ground_with_google_search=ground_with_google_search, - request_type=request_type, - ) - - if session is None: - return bpd.read_gbq_query(sql) - else: - return session.read_gbq_query(sql) - - -@log_adapter.method_logger(custom_base_name="bigquery_ml") -def get_insights( - model: Union[bigframes.ml.base.BaseEstimator, str, pd.Series], -) -> dataframe.DataFrame: - """ - Gets insights from a BigQuery ML model. - - See the `BigQuery ML GET_INSIGHTS function syntax - `_ - for additional reference. - - Args: - model (bigframes.ml.base.BaseEstimator, str, or pd.Series): - The model to get insights from. - - Returns: - bigframes.pandas.DataFrame: - The insights. - """ - import bigframes.pandas as bpd - - model_name, session = utils.get_model_name_and_session(model) - - sql = bigframes.core.sql.ml.get_insights( - model_name=model_name, - ) - - if session is None: - return bpd.read_gbq_query(sql) - else: - return session.read_gbq_query(sql) - - -@log_adapter.method_logger(custom_base_name="bigquery_ml") -def generate_embedding( - model: Union[bigframes.ml.base.BaseEstimator, str, pd.Series], - input_: Union[pd.DataFrame, dataframe.DataFrame, str], - *, - flatten_json_output: Optional[bool] = None, - task_type: Optional[str] = None, - output_dimensionality: Optional[int] = None, -) -> dataframe.DataFrame: - """ - Generates text embedding using a BigQuery ML model. - - See the `BigQuery ML GENERATE_EMBEDDING function syntax - `_ - for additional reference. - - Args: - model (bigframes.ml.base.BaseEstimator or str): - The model to use for text embedding. - input_ (Union[bigframes.pandas.DataFrame, str]): - The DataFrame or query to use for text embedding. - flatten_json_output (bool, optional): - A BOOL value that determines the content of the generated JSON column. - task_type (str, optional): - A STRING value that specifies the intended downstream application task. - Supported values are: - - `RETRIEVAL_QUERY` - - `RETRIEVAL_DOCUMENT` - - `SEMANTIC_SIMILARITY` - - `CLASSIFICATION` - - `CLUSTERING` - - `QUESTION_ANSWERING` - - `FACT_VERIFICATION` - - `CODE_RETRIEVAL_QUERY` - output_dimensionality (int, optional): - An INT64 value that specifies the size of the output embedding. - - Returns: - bigframes.pandas.DataFrame: - The generated text embedding. - """ - import bigframes.pandas as bpd - - model_name, session = utils.get_model_name_and_session(model, input_) - table_sql = utils.to_sql(input_) - - sql = bigframes.core.sql.ml.generate_embedding( - model_name=model_name, - table=table_sql, - flatten_json_output=flatten_json_output, - task_type=task_type, - output_dimensionality=output_dimensionality, - ) - - if session is None: - return bpd.read_gbq_query(sql) - else: - return session.read_gbq_query(sql) diff --git a/bigframes/bigquery/_operations/obj.py b/bigframes/bigquery/_operations/obj.py index ca09d7ab1ce..5aef00e73bd 100644 --- a/bigframes/bigquery/_operations/obj.py +++ b/bigframes/bigquery/_operations/obj.py @@ -18,6 +18,7 @@ See bigframes.bigquery.obj for public docs. """ + from __future__ import annotations import datetime @@ -26,11 +27,11 @@ import numpy as np import pandas as pd +from bigframes.core import convert +from bigframes.core.logging import log_adapter import bigframes.core.utils as utils import bigframes.operations as ops import bigframes.series as series -from bigframes.core import convert -from bigframes.core.logging import log_adapter @log_adapter.method_logger(custom_base_name="bigquery_obj") diff --git a/bigframes/bigquery/_operations/sql.py b/bigframes/bigquery/_operations/sql.py index 332d558866b..295412fd759 100644 --- a/bigframes/bigquery/_operations/sql.py +++ b/bigframes/bigquery/_operations/sql.py @@ -16,31 +16,19 @@ from __future__ import annotations -from typing import Optional, Sequence, Union, cast +from typing import Sequence import google.cloud.bigquery -import bigframes.dataframe +import bigframes.core.compile.sqlglot.sqlglot_ir as sqlglot_ir import bigframes.dtypes import bigframes.operations import bigframes.series -from bigframes.core.compile.sqlglot import sql - - -def _format_names(sql_template: str, dataframe: bigframes.dataframe.DataFrame): - """Turn sql_template from a template that uses names to one that uses - numbers. - """ - names_to_numbers = {name: f"{{{i}}}" for i, name in enumerate(dataframe.columns)} - numbers = [f"{{{i}}}" for i in range(len(dataframe.columns))] - return sql_template.format(*numbers, **names_to_numbers) def sql_scalar( sql_template: str, - columns: Union[bigframes.dataframe.DataFrame, Sequence[bigframes.series.Series]], - *, - output_dtype: Optional[bigframes.dtypes.Dtype] = None, + columns: Sequence[bigframes.series.Series], ) -> bigframes.series.Series: """Create a Series from a SQL template. @@ -49,9 +37,6 @@ def sql_scalar( >>> import bigframes.pandas as bpd >>> import bigframes.bigquery as bbq - Either pass in a sequence of series, in which case use integers in the - format strings. - >>> s = bpd.Series(["1.5", "2.5", "3.5"]) >>> s = s.astype(pd.ArrowDtype(pa.decimal128(38, 9))) >>> bbq.sql_scalar("ROUND({0}, 0, 'ROUND_HALF_EVEN')", [s]) @@ -60,48 +45,13 @@ def sql_scalar( 2 4.000000000 dtype: decimal128(38, 9)[pyarrow] - Or pass in a DataFrame, in which case use the column names in the format - strings. - - >>> df = bpd.DataFrame({"a": ["1.5", "2.5", "3.5"]}) - >>> df = df.astype({"a": pd.ArrowDtype(pa.decimal128(38, 9))}) - >>> bbq.sql_scalar("ROUND({a}, 0, 'ROUND_HALF_EVEN')", df) - 0 2.000000000 - 1 2.000000000 - 2 4.000000000 - dtype: decimal128(38, 9)[pyarrow] - - You can also use the `.bigquery` DataFrame accessor to apply a SQL scalar function. - - Compute SQL scalar using a pandas DataFrame: - - >>> import pandas as pd - >>> df = pd.DataFrame({"x": [1, 2, 3]}) - >>> bpd.options.display.progress_bar = None # doctest: +SKIP - >>> pandas_s = df.bigquery.sql_scalar("POW({0}, 2)") # doctest: +SKIP - >>> type(pandas_s) # doctest: +SKIP - - - Compute SQL scalar using a BigFrames DataFrame: - - >>> bf_df = bpd.DataFrame({"x": [1, 2, 3]}) - >>> bf_s = bf_df.bigquery.sql_scalar("POW({0}, 2)") # doctest: +SKIP - >>> type(bf_s) # doctest: +SKIP - - - Args: sql_template (str): A SQL format string with Python-style {0} placeholders for each of the Series objects in ``columns``. - columns ( - Sequence[bigframes.pandas.Series] | bigframes.pandas.DataFrame - ): + columns (Sequence[bigframes.pandas.Series]): Series objects representing the column inputs to the ``sql_template``. Must contain at least one Series. - output_dtype (a BigQuery DataFrames compatible dtype, optional): - If provided, BigQuery DataFrames uses this to determine the output - of the returned Series. This avoids a dry run query. Returns: bigframes.pandas.Series: @@ -110,38 +60,31 @@ def sql_scalar( Raises: ValueError: If ``columns`` is empty. """ - if isinstance(columns, bigframes.dataframe.DataFrame): - sql_template = _format_names(sql_template, columns) - columns = [ - cast(bigframes.series.Series, columns[column]) for column in columns.columns - ] - if len(columns) == 0: raise ValueError("Must provide at least one column in columns") - base_series = columns[0] - # To integrate this into our expression trees, we need to get the output # type, so we do some manual compilation and a dry run query to get that. # Another benefit of this is that if there is a syntax error in the SQL # template, then this will fail with an error earlier in the process, # aiding users in debugging. - if output_dtype is None: - literals_sql = [ - sql.to_sql(sql.literal(None, column.dtype)) for column in columns - ] - select_sql = sql_template.format(*literals_sql) - dry_run_sql = f"SELECT {select_sql}" - - # Use the executor directly, because we want the original column IDs, not - # the user-friendly column names that block.to_sql_query() would produce. - bqclient = base_series._session.bqclient - job = bqclient.query( - dry_run_sql, job_config=google.cloud.bigquery.QueryJobConfig(dry_run=True) - ) - _, output_dtype = bigframes.dtypes.convert_schema_field(job.schema[0]) + literals_sql = [ + sqlglot_ir._literal(None, column.dtype).sql(dialect="bigquery") + for column in columns + ] + select_sql = sql_template.format(*literals_sql) + dry_run_sql = f"SELECT {select_sql}" + + # Use the executor directly, because we want the original column IDs, not + # the user-friendly column names that block.to_sql_query() would produce. + base_series = columns[0] + bqclient = base_series._session.bqclient + job = bqclient.query( + dry_run_sql, job_config=google.cloud.bigquery.QueryJobConfig(dry_run=True) + ) + _, output_type = bigframes.dtypes.convert_schema_field(job.schema[0]) op = bigframes.operations.SqlScalarOp( - _output_type=output_dtype, sql_template=sql_template + _output_type=output_type, sql_template=sql_template ) return base_series._apply_nary_op(op, columns[1:]) diff --git a/bigframes/bigquery/_operations/struct.py b/bigframes/bigquery/_operations/struct.py index 2ee760fb8e5..a6304677ef7 100644 --- a/bigframes/bigquery/_operations/struct.py +++ b/bigframes/bigquery/_operations/struct.py @@ -15,7 +15,8 @@ """This module integrates BigQuery built-in functions for use with DataFrame objects, such as array functions: -https://cloud.google.com/bigquery/docs/reference/standard-sql/array_functions.""" +https://cloud.google.com/bigquery/docs/reference/standard-sql/array_functions. """ + from __future__ import annotations @@ -42,9 +43,9 @@ def struct(value: dataframe.DataFrame) -> series.Series: >>> srs = series.Series([{"version": 1, "project": "pandas"}, {"version": 2, "project": "numpy"},]) >>> df = srs.struct.explode() >>> bbq.struct(df) - 0 {'version': 1, 'project': 'pandas'} - 1 {'version': 2, 'project': 'numpy'} - dtype: struct[pyarrow] + 0 {'project': 'pandas', 'version': 1} + 1 {'project': 'numpy', 'version': 2} + dtype: struct[pyarrow] Args: value (bigframes.dataframe.DataFrame): @@ -57,5 +58,5 @@ def struct(value: dataframe.DataFrame) -> series.Series: block, result_id = block.apply_nary_op( block.value_columns, ops.StructOp(column_names=tuple(block.column_labels)) ) - block = block.select_column(result_id).with_column_labels([None]) + block = block.select_column(result_id) return series.Series(block) diff --git a/bigframes/bigquery/_operations/table.py b/bigframes/bigquery/_operations/table.py deleted file mode 100644 index cad025412d5..00000000000 --- a/bigframes/bigquery/_operations/table.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from typing import Mapping, Optional, Union - -import google.cloud.bigquery -import pandas as pd - -import bigframes.core.compile.sqlglot.sql as sg_sql -import bigframes.core.logging.log_adapter as log_adapter -import bigframes.session - - -def _get_table_metadata( - *, - bqclient: google.cloud.bigquery.Client, - table_name: str, -) -> pd.Series: - table_metadata = bqclient.get_table(table_name) - table_dict = table_metadata.to_api_repr() - return pd.Series(table_dict) - - -@log_adapter.method_logger(custom_base_name="bigquery_table") -def create_external_table( - table_name: str, - *, - replace: bool = False, - if_not_exists: bool = False, - columns: Optional[Mapping[str, str]] = None, - partition_columns: Optional[Mapping[str, str]] = None, - connection_name: Optional[str] = None, - options: Mapping[str, Union[str, int, float, bool, list]], - session: Optional[bigframes.session.Session] = None, -) -> pd.Series: - """ - Creates a BigQuery external table. - - See the `BigQuery CREATE EXTERNAL TABLE DDL syntax - `_ - for additional reference. - - Args: - table_name (str): - The name of the table in BigQuery. - replace (bool, default False): - Whether to replace the table if it already exists. - if_not_exists (bool, default False): - Whether to ignore the error if the table already exists. - columns (Mapping[str, str], optional): - The table's schema. - partition_columns (Mapping[str, str], optional): - The table's partition columns. - connection_name (str, optional): - The connection to use for the table. - options (Mapping[str, Union[str, int, float, bool, list]]): - The OPTIONS clause, which specifies the table options. - session (bigframes.session.Session, optional): - The session to use. If not provided, the default session is used. - - Returns: - pandas.Series: - A Series with object dtype containing the table metadata. Reference - the `BigQuery Table REST API reference - `_ - for available fields. - """ - import bigframes.pandas as bpd - - sql = sg_sql.to_sql( - sg_sql.create_external_table( - table_name=table_name, - replace=replace, - if_not_exists=if_not_exists, - columns=columns, - partition_columns=partition_columns, - connection_name=connection_name, - options=options, - ) - ) - - if session is None: - bpd.read_gbq_query(sql) - session = bpd.get_global_session() - else: - session.read_gbq_query(sql) - - return _get_table_metadata(bqclient=session.bqclient, table_name=table_name) diff --git a/bigframes/bigquery/_operations/utils.py b/bigframes/bigquery/_operations/utils.py deleted file mode 100644 index 0bae8f47c7a..00000000000 --- a/bigframes/bigquery/_operations/utils.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Union, cast - -import pandas as pd - -import bigframes -from bigframes import dataframe -from bigframes.ml import base as ml_base - - -def get_model_name_and_session( - model: Union[ml_base.BaseEstimator, str, pd.Series], - # Other dataframe arguments to extract session from - *dataframes: Optional[Union[pd.DataFrame, dataframe.DataFrame, str]], -) -> tuple[str, Optional[bigframes.session.Session]]: - if isinstance(model, pd.Series): - try: - model_ref = model["modelReference"] - model_name = f"{model_ref['projectId']}.{model_ref['datasetId']}.{model_ref['modelId']}" # type: ignore - except KeyError: - raise ValueError("modelReference must be present in the pandas Series.") - elif isinstance(model, str): - model_name = model - else: - if model._bqml_model is None: - raise ValueError("Model must be fitted to be used in ML operations.") - return model._bqml_model.model_name, model._bqml_model.session - - session = None - for df in dataframes: - if isinstance(df, dataframe.DataFrame): - session = df._session - break - - return model_name, session - - -def to_sql(df_or_sql: Union[pd.DataFrame, dataframe.DataFrame, str]) -> str: - """ - Helper to convert DataFrame to SQL string - """ - import bigframes.pandas as bpd - - if isinstance(df_or_sql, str): - return df_or_sql - - if isinstance(df_or_sql, pd.DataFrame): - bf_df = bpd.read_pandas(df_or_sql) - else: - bf_df = cast(dataframe.DataFrame, df_or_sql) - - # Cache dataframes to make sure base table is not a snapshot. - # Cached dataframe creates a full copy, never uses snapshot. - # This is a workaround for internal issue b/310266666. - bf_df.cache() - sql, _, _ = bf_df._to_sql_query(include_index=False) - return sql diff --git a/bigframes/bigquery/ai.py b/bigframes/bigquery/ai.py index 6dd3d116635..3af52205a65 100644 --- a/bigframes/bigquery/ai.py +++ b/bigframes/bigquery/ai.py @@ -12,78 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -Integrate BigQuery built-in AI functions into your BigQuery DataFrames workflow. - -The ``bigframes.bigquery.ai`` module provides a Pythonic interface to leverage BigQuery ML's -generative AI and predictive functions directly on BigQuery DataFrames and Series objects. -These functions enable you to perform advanced AI tasks at scale without moving data -out of BigQuery. - -Key capabilities include: - -* **Generative AI:** Use :func:`bigframes.bigquery.ai.generate` (Gemini) to - perform text analysis, translation, or - content generation. Specialized versions like - :func:`~bigframes.bigquery.ai.generate_bool`, - :func:`~bigframes.bigquery.ai.generate_int`, and - :func:`~bigframes.bigquery.ai.generate_double` are available for structured - outputs. -* **Embeddings:** Generate vector embeddings for text using - :func:`~bigframes.bigquery.ai.generate_embedding`, which are essential for - semantic search and retrieval-augmented generation (RAG) workflows. -* **Classification and Scoring:** Apply machine learning models to your data for - predictive tasks with :func:`~bigframes.bigquery.ai.classify` and - :func:`~bigframes.bigquery.ai.score`. -* **Forecasting:** Predict future values in time-series data using - :func:`~bigframes.bigquery.ai.forecast`. - -**Example usage:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - - >>> df = bpd.DataFrame({ - ... "text_input": [ - ... "Is this a positive review? The food was terrible.", - ... ], - ... }) # doctest: +SKIP - - >>> # Assuming a Gemini model has been created in BigQuery as 'my_gemini_model' - >>> result = bq.ai.generate_text("my_gemini_model", df["text_input"]) # doctest: +SKIP - -For more information on the underlying BigQuery ML syntax, see: -https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-ai-generate-bool -""" +"""This module integrates BigQuery built-in AI functions for use with Series/DataFrame objects, +such as AI.GENERATE_BOOL: +https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-ai-generate-bool""" from bigframes.bigquery._operations.ai import ( classify, - embed, forecast, generate, generate_bool, generate_double, - generate_embedding, generate_int, - generate_table, - generate_text, if_, score, - similarity, ) __all__ = [ "classify", - "embed", "forecast", "generate", "generate_bool", "generate_double", - "generate_embedding", "generate_int", - "generate_table", - "generate_text", "if_", "score", - "similarity", ] diff --git a/bigframes/bigquery/ml.py b/bigframes/bigquery/ml.py index 9b0d77d5b89..93b0670ba5e 100644 --- a/bigframes/bigquery/ml.py +++ b/bigframes/bigquery/ml.py @@ -23,12 +23,8 @@ create_model, evaluate, explain_predict, - generate_embedding, - generate_text, - get_insights, global_explain, predict, - transform, ) __all__ = [ @@ -37,8 +33,4 @@ "predict", "explain_predict", "global_explain", - "transform", - "generate_text", - "generate_embedding", - "get_insights", ] diff --git a/bigframes/blob/_functions.py b/bigframes/blob/_functions.py new file mode 100644 index 00000000000..3dfe38811b1 --- /dev/null +++ b/bigframes/blob/_functions.py @@ -0,0 +1,728 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +import inspect +import typing +from typing import Callable, Iterable, Union + +import google.cloud.bigquery as bigquery + +import bigframes.session +import bigframes.session._io.bigquery as bf_io_bigquery + +_PYTHON_TO_BQ_TYPES = { + int: "INT64", + float: "FLOAT64", + str: "STRING", + bytes: "BYTES", + bool: "BOOL", +} + + +@dataclass(frozen=True) +class FunctionDef: + """Definition of a Python UDF.""" + + func: Callable # function body + requirements: Iterable[str] # required packages + + +# TODO(garrettwu): migrate to bigframes UDF when it is available +class TransformFunction: + """Simple transform function class to deal with Python UDF.""" + + def __init__( + self, + func_def: FunctionDef, + session: bigframes.session.Session, + connection: str, + max_batching_rows: int, + container_cpu: Union[float, int], + container_memory: str, + ): + self._func = func_def.func + self._requirements = func_def.requirements + self._session = session + self._connection = connection + self._max_batching_rows = ( + int(max_batching_rows) if max_batching_rows > 1 else max_batching_rows + ) + self._container_cpu = container_cpu + self._container_memory = container_memory + + def _input_bq_signature(self): + sig = inspect.signature(self._func) + inputs = [] + for k, v in sig.parameters.items(): + inputs.append(f"{k} {_PYTHON_TO_BQ_TYPES[v.annotation]}") + return ", ".join(inputs) + + def _output_bq_type(self): + sig = inspect.signature(self._func) + return_annotation = sig.return_annotation + origin = typing.get_origin(return_annotation) + if origin is Union: + args = typing.get_args(return_annotation) + if len(args) == 2 and args[1] is type(None): + return _PYTHON_TO_BQ_TYPES[args[0]] + return _PYTHON_TO_BQ_TYPES[sig.return_annotation] + + def _create_udf(self): + """Create Python UDF in BQ. Return name of the UDF.""" + udf_name = str( + self._session._anon_dataset_manager.generate_unique_resource_id() + ) + + func_body = "import typing\n" + inspect.getsource(self._func) + func_name = self._func.__name__ + packages = str(list(self._requirements)) + + sql = f""" +CREATE OR REPLACE FUNCTION `{udf_name}`({self._input_bq_signature()}) +RETURNS {self._output_bq_type()} LANGUAGE python +WITH CONNECTION `{self._connection}` +OPTIONS (entry_point='{func_name}', runtime_version='python-3.11', packages={packages}, max_batching_rows={self._max_batching_rows}, container_cpu={self._container_cpu}, container_memory='{self._container_memory}') +AS r\"\"\" + + +{func_body} + + +\"\"\" + """ + + bf_io_bigquery.start_query_with_client( + self._session.bqclient, + sql, + job_config=bigquery.QueryJobConfig(), + metrics=self._session._metrics, + location=None, + project=None, + timeout=None, + query_with_job=True, + publisher=self._session._publisher, + ) + + return udf_name + + def udf(self): + """Create and return the UDF object.""" + udf_name = self._create_udf() + + # TODO(b/404605969): remove cleanups when UDF fixes dataset deletion. + self._session._function_session._update_temp_artifacts(udf_name, "") + return self._session.read_gbq_function(udf_name) + + +def exif_func(src_obj_ref_rt: str, verbose: bool) -> str: + try: + import io + import json + + from PIL import ExifTags, Image + import requests + from requests import adapters + + session = requests.Session() + session.mount("https://", adapters.HTTPAdapter(max_retries=3)) + + src_obj_ref_rt_json = json.loads(src_obj_ref_rt) + src_url = src_obj_ref_rt_json["access_urls"]["read_url"] + + response = session.get(src_url, timeout=30) + response.raise_for_status() + bts = response.content + + image = Image.open(io.BytesIO(bts)) + exif_data = image.getexif() + exif_dict = {} + + if exif_data: + for tag, value in exif_data.items(): + tag_name = ExifTags.TAGS.get(tag, tag) + # Convert non-serializable types to strings + try: + json.dumps(value) + exif_dict[tag_name] = value + except (TypeError, ValueError): + exif_dict[tag_name] = str(value) + + if verbose: + return json.dumps({"status": "", "content": json.dumps(exif_dict)}) + else: + return json.dumps(exif_dict) + + except Exception as e: + # Return error as JSON with error field + error_result = {"status": f"{type(e).__name__}: {str(e)}", "content": "{}"} + if verbose: + return json.dumps(error_result) + else: + return "{}" + + +exif_func_def = FunctionDef(exif_func, ["pillow", "requests"]) + + +# Blur images. Takes ObjectRefRuntime as JSON string. Outputs ObjectRefRuntime JSON string. +def image_blur_func( + src_obj_ref_rt: str, + dst_obj_ref_rt: str, + ksize_x: int, + ksize_y: int, + ext: str, + verbose: bool, +) -> typing.Optional[str]: + try: + import json + + import cv2 as cv # type: ignore + import numpy as np + import requests + from requests import adapters + + session = requests.Session() + session.mount("https://", adapters.HTTPAdapter(max_retries=3)) + + ext = ext or ".jpeg" + + src_obj_ref_rt_json = json.loads(src_obj_ref_rt) + dst_obj_ref_rt_json = json.loads(dst_obj_ref_rt) + + src_url = src_obj_ref_rt_json["access_urls"]["read_url"] + dst_url = dst_obj_ref_rt_json["access_urls"]["write_url"] + + response = session.get(src_url, timeout=30) + response.raise_for_status() # Raise exception for HTTP errors + bts = response.content + + nparr = np.frombuffer(bts, np.uint8) + img = cv.imdecode(nparr, cv.IMREAD_UNCHANGED) + + if img is None: + raise ValueError( + "Failed to decode image - possibly corrupted or unsupported format" + ) + + img_blurred = cv.blur(img, ksize=(ksize_x, ksize_y)) + + success, encoded = cv.imencode(ext, img_blurred) + if not success: + raise ValueError(f"Failed to encode image with extension {ext}") + + bts = encoded.tobytes() + + ext = ext.replace(".", "") + ext_mappings = {"jpg": "jpeg", "tif": "tiff"} + ext = ext_mappings.get(ext, ext) + content_type = "image/" + ext + + put_response = session.put( + url=dst_url, + data=bts, + headers={"Content-Type": content_type}, + timeout=30, + ) + put_response.raise_for_status() + + if verbose: + return json.dumps({"status": "", "content": dst_obj_ref_rt}) + else: + return dst_obj_ref_rt + + except Exception as e: + if verbose: + error_result = { + "status": f"Error: {type(e).__name__}: {str(e)}", + "content": "", + } + return json.dumps(error_result) + else: + return None + + +image_blur_def = FunctionDef(image_blur_func, ["opencv-python", "numpy", "requests"]) + + +def image_blur_to_bytes_func( + src_obj_ref_rt: str, ksize_x: int, ksize_y: int, ext: str, verbose: bool +) -> str: + import base64 + import json + + try: + import cv2 as cv # type: ignore + import numpy as np + import requests + from requests import adapters + + session = requests.Session() + session.mount("https://", adapters.HTTPAdapter(max_retries=3)) + + ext = ext or ".jpeg" + + src_obj_ref_rt_json = json.loads(src_obj_ref_rt) + src_url = src_obj_ref_rt_json["access_urls"]["read_url"] + + response = session.get(src_url, timeout=30) + response.raise_for_status() + bts = response.content + + nparr = np.frombuffer(bts, np.uint8) + img = cv.imdecode(nparr, cv.IMREAD_UNCHANGED) + if img is None: + raise ValueError( + "Failed to decode image - possibly corrupted or unsupported format" + ) + img_blurred = cv.blur(img, ksize=(ksize_x, ksize_y)) + success, encoded = cv.imencode(ext, img_blurred) + if not success: + raise ValueError(f"Failed to encode image with extension {ext}") + content = encoded.tobytes() + + encoded_content = base64.b64encode(content).decode("utf-8") + result_dict = {"status": "", "content": encoded_content} + if verbose: + return json.dumps(result_dict) + else: + return result_dict["content"] + + except Exception as e: + status = f"Error: {type(e).__name__}: {str(e)}" + encoded_content = base64.b64encode(b"").decode("utf-8") + result_dict = {"status": status, "content": encoded_content} + if verbose: + return json.dumps(result_dict) + else: + return result_dict["content"] + + +image_blur_to_bytes_def = FunctionDef( + image_blur_to_bytes_func, ["opencv-python", "numpy", "requests"] +) + + +def image_resize_func( + src_obj_ref_rt: str, + dst_obj_ref_rt: str, + dsize_x: int, + dsize_y: int, + fx: float, + fy: float, + ext: str, + verbose: bool, +) -> typing.Optional[str]: + try: + import json + + import cv2 as cv # type: ignore + import numpy as np + import requests + from requests import adapters + + session = requests.Session() + session.mount("https://", adapters.HTTPAdapter(max_retries=3)) + + ext = ext or ".jpeg" + + src_obj_ref_rt_json = json.loads(src_obj_ref_rt) + dst_obj_ref_rt_json = json.loads(dst_obj_ref_rt) + + src_url = src_obj_ref_rt_json["access_urls"]["read_url"] + dst_url = dst_obj_ref_rt_json["access_urls"]["write_url"] + + response = session.get(src_url, timeout=30) + response.raise_for_status() + bts = response.content + + nparr = np.frombuffer(bts, np.uint8) + img = cv.imdecode(nparr, cv.IMREAD_UNCHANGED) + if img is None: + raise ValueError( + "Failed to decode image - possibly corrupted or unsupported format" + ) + img_resized = cv.resize(img, dsize=(dsize_x, dsize_y), fx=fx, fy=fy) + + success, encoded = cv.imencode(ext, img_resized) + if not success: + raise ValueError(f"Failed to encode image with extension {ext}") + bts = encoded.tobytes() + + ext = ext.replace(".", "") + ext_mappings = {"jpg": "jpeg", "tif": "tiff"} + ext = ext_mappings.get(ext, ext) + content_type = "image/" + ext + + put_response = session.put( + url=dst_url, + data=bts, + headers={ + "Content-Type": content_type, + }, + timeout=30, + ) + put_response.raise_for_status() + + if verbose: + return json.dumps({"status": "", "content": dst_obj_ref_rt}) + else: + return dst_obj_ref_rt + + except Exception as e: + if verbose: + error_result = { + "status": f"Error: {type(e).__name__}: {str(e)}", + "content": "", + } + return json.dumps(error_result) + else: + return None + + +image_resize_def = FunctionDef( + image_resize_func, ["opencv-python", "numpy", "requests"] +) + + +def image_resize_to_bytes_func( + src_obj_ref_rt: str, + dsize_x: int, + dsize_y: int, + fx: float, + fy: float, + ext: str, + verbose: bool, +) -> str: + import base64 + import json + + try: + import cv2 as cv # type: ignore + import numpy as np + import requests + from requests import adapters + + session = requests.Session() + session.mount("https://", adapters.HTTPAdapter(max_retries=3)) + + ext = ext or ".jpeg" + + src_obj_ref_rt_json = json.loads(src_obj_ref_rt) + src_url = src_obj_ref_rt_json["access_urls"]["read_url"] + + response = session.get(src_url, timeout=30) + response.raise_for_status() + bts = response.content + + nparr = np.frombuffer(bts, np.uint8) + img = cv.imdecode(nparr, cv.IMREAD_UNCHANGED) + if img is None: + raise ValueError( + "Failed to decode image - possibly corrupted or unsupported format" + ) + img_resized = cv.resize(img, dsize=(dsize_x, dsize_y), fx=fx, fy=fy) + success, encoded = cv.imencode(ext, img_resized) + if not success: + raise ValueError(f"Failed to encode image with extension {ext}") + content = encoded.tobytes() + + encoded_content = base64.b64encode(content).decode("utf-8") + result_dict = {"status": "", "content": encoded_content} + if verbose: + return json.dumps(result_dict) + else: + return result_dict["content"] + + except Exception as e: + status = f"Error: {type(e).__name__}: {str(e)}" + encoded_content = base64.b64encode(b"").decode("utf-8") + result_dict = {"status": status, "content": encoded_content} + if verbose: + return json.dumps(result_dict) + else: + return result_dict["content"] + + +image_resize_to_bytes_def = FunctionDef( + image_resize_to_bytes_func, ["opencv-python", "numpy", "requests"] +) + + +def image_normalize_func( + src_obj_ref_rt: str, + dst_obj_ref_rt: str, + alpha: float, + beta: float, + norm_type: str, + ext: str, + verbose: bool, +) -> typing.Optional[str]: + try: + import json + + import cv2 as cv # type: ignore + import numpy as np + import requests + from requests import adapters + + session = requests.Session() + session.mount("https://", adapters.HTTPAdapter(max_retries=3)) + + ext = ext or ".jpeg" + + norm_type_mapping = { + "inf": cv.NORM_INF, + "l1": cv.NORM_L1, + "l2": cv.NORM_L2, + "minmax": cv.NORM_MINMAX, + } + + src_obj_ref_rt_json = json.loads(src_obj_ref_rt) + dst_obj_ref_rt_json = json.loads(dst_obj_ref_rt) + + src_url = src_obj_ref_rt_json["access_urls"]["read_url"] + dst_url = dst_obj_ref_rt_json["access_urls"]["write_url"] + + response = session.get(src_url, timeout=30) + response.raise_for_status() + bts = response.content + + nparr = np.frombuffer(bts, np.uint8) + img = cv.imdecode(nparr, cv.IMREAD_UNCHANGED) + if img is None: + raise ValueError( + "Failed to decode image - possibly corrupted or unsupported format" + ) + img_normalized = cv.normalize( + img, None, alpha=alpha, beta=beta, norm_type=norm_type_mapping[norm_type] + ) + + success, encoded = cv.imencode(ext, img_normalized) + if not success: + raise ValueError(f"Failed to encode image with extension {ext}") + bts = encoded.tobytes() + + ext = ext.replace(".", "") + ext_mappings = {"jpg": "jpeg", "tif": "tiff"} + ext = ext_mappings.get(ext, ext) + content_type = "image/" + ext + + put_response = session.put( + url=dst_url, + data=bts, + headers={ + "Content-Type": content_type, + }, + timeout=30, + ) + put_response.raise_for_status() + + if verbose: + return json.dumps({"status": "", "content": dst_obj_ref_rt}) + else: + return dst_obj_ref_rt + + except Exception as e: + if verbose: + error_result = { + "status": f"Error: {type(e).__name__}: {str(e)}", + "content": "", + } + return json.dumps(error_result) + else: + return None + + +image_normalize_def = FunctionDef( + image_normalize_func, ["opencv-python", "numpy", "requests"] +) + + +def image_normalize_to_bytes_func( + src_obj_ref_rt: str, + alpha: float, + beta: float, + norm_type: str, + ext: str, + verbose: bool, +) -> str: + import base64 + import json + + try: + import cv2 as cv # type: ignore + import numpy as np + import requests + from requests import adapters + + session = requests.Session() + session.mount("https://", adapters.HTTPAdapter(max_retries=3)) + + ext = ext or ".jpeg" + + norm_type_mapping = { + "inf": cv.NORM_INF, + "l1": cv.NORM_L1, + "l2": cv.NORM_L2, + "minmax": cv.NORM_MINMAX, + } + + src_obj_ref_rt_json = json.loads(src_obj_ref_rt) + src_url = src_obj_ref_rt_json["access_urls"]["read_url"] + + response = session.get(src_url, timeout=30) + response.raise_for_status() + bts = response.content + + nparr = np.frombuffer(bts, np.uint8) + img = cv.imdecode(nparr, cv.IMREAD_UNCHANGED) + if img is None: + raise ValueError( + "Failed to decode image - possibly corrupted or unsupported format" + ) + img_normalized = cv.normalize( + img, None, alpha=alpha, beta=beta, norm_type=norm_type_mapping[norm_type] + ) + success, encoded = cv.imencode(ext, img_normalized) + if not success: + raise ValueError(f"Failed to encode image with extension {ext}") + content = encoded.tobytes() + + encoded_content = base64.b64encode(content).decode("utf-8") + result_dict = {"status": "", "content": encoded_content} + + if verbose: + return json.dumps(result_dict) + else: + return result_dict["content"] + + except Exception as e: + status = f"Error: {type(e).__name__}: {str(e)}" + encoded_content = base64.b64encode(b"").decode("utf-8") + result_dict = {"status": status, "content": encoded_content} + if verbose: + return json.dumps(result_dict) + else: + return result_dict["content"] + + +image_normalize_to_bytes_def = FunctionDef( + image_normalize_to_bytes_func, ["opencv-python", "numpy", "requests"] +) + + +# Extracts all text from a PDF url +def pdf_extract_func(src_obj_ref_rt: str, verbose: bool) -> str: + try: + import io + import json + + from pypdf import PdfReader # type: ignore + import requests + from requests import adapters + + session = requests.Session() + session.mount("https://", adapters.HTTPAdapter(max_retries=3)) + + src_obj_ref_rt_json = json.loads(src_obj_ref_rt) + src_url = src_obj_ref_rt_json["access_urls"]["read_url"] + + response = session.get(src_url, timeout=30, stream=True) + response.raise_for_status() + pdf_bytes = response.content + + pdf_file = io.BytesIO(pdf_bytes) + reader = PdfReader(pdf_file, strict=False) + + all_text = "" + for page in reader.pages: + page_extract_text = page.extract_text() + if page_extract_text: + all_text += page_extract_text + + result_dict = {"status": "", "content": all_text} + + except Exception as e: + result_dict = {"status": str(e), "content": ""} + + if verbose: + return json.dumps(result_dict) + else: + return result_dict["content"] + + +pdf_extract_def = FunctionDef( + pdf_extract_func, ["pypdf>=5.3.1,<6.0.0", "requests", "cryptography==43.0.3"] +) + + +# Extracts text from a PDF url and chunks it simultaneously +def pdf_chunk_func( + src_obj_ref_rt: str, chunk_size: int, overlap_size: int, verbose: bool +) -> str: + try: + import io + import json + + from pypdf import PdfReader # type: ignore + import requests + from requests import adapters + + session = requests.Session() + session.mount("https://", adapters.HTTPAdapter(max_retries=3)) + + src_obj_ref_rt_json = json.loads(src_obj_ref_rt) + src_url = src_obj_ref_rt_json["access_urls"]["read_url"] + + response = session.get(src_url, timeout=30, stream=True) + response.raise_for_status() + pdf_bytes = response.content + + pdf_file = io.BytesIO(pdf_bytes) + reader = PdfReader(pdf_file, strict=False) + # extract and chunk text simultaneously + all_text_chunks = [] + curr_chunk = "" + for page in reader.pages: + page_text = page.extract_text() + if page_text: + curr_chunk += page_text + # split the accumulated text into chunks of a specific size with overlaop + # this loop implements a sliding window approach to create chunks + while len(curr_chunk) >= chunk_size: + split_idx = curr_chunk.rfind(" ", 0, chunk_size) + if split_idx == -1: + split_idx = chunk_size + actual_chunk = curr_chunk[:split_idx] + all_text_chunks.append(actual_chunk) + overlap = curr_chunk[split_idx + 1 : split_idx + 1 + overlap_size] + curr_chunk = overlap + curr_chunk[split_idx + 1 + overlap_size :] + if curr_chunk: + all_text_chunks.append(curr_chunk) + + result_dict = {"status": "", "content": all_text_chunks} + + except Exception as e: + result_dict = {"status": str(e), "content": []} + + if verbose: + return json.dumps(result_dict) + else: + return json.dumps(result_dict["content"]) + + +pdf_chunk_def = FunctionDef( + pdf_chunk_func, ["pypdf>=5.3.1,<6.0.0", "requests", "cryptography==43.0.3"] +) diff --git a/bigframes/clients.py b/bigframes/clients.py index b724843c133..e6ddd5c6cbe 100644 --- a/bigframes/clients.py +++ b/bigframes/clients.py @@ -19,7 +19,7 @@ import logging import textwrap import time -from typing import Optional, cast +from typing import cast, Optional import google.api_core.exceptions import google.api_core.retry diff --git a/bigframes/core/agg_expressions.py b/bigframes/core/agg_expressions.py index 6d126c92420..125e3fef630 100644 --- a/bigframes/core/agg_expressions.py +++ b/bigframes/core/agg_expressions.py @@ -19,12 +19,12 @@ import functools import itertools import typing -from typing import Callable, Hashable, Mapping, Tuple, TypeVar +from typing import Callable, Mapping, Tuple, TypeVar -import bigframes.core.identifiers as ids -import bigframes.operations.aggregations as agg_ops from bigframes import dtypes from bigframes.core import expression, window_spec +import bigframes.core.identifiers as ids +import bigframes.operations.aggregations as agg_ops TExpression = TypeVar("TExpression", bound="Aggregation") @@ -60,14 +60,15 @@ def output_type(self) -> dtypes.ExpressionType: @abc.abstractmethod def inputs( self, - ) -> typing.Tuple[expression.Expression, ...]: ... + ) -> typing.Tuple[expression.Expression, ...]: + ... @property def children(self) -> Tuple[expression.Expression, ...]: return self.inputs @property - def free_variables(self) -> typing.Tuple[Hashable, ...]: + def free_variables(self) -> typing.Tuple[str, ...]: return tuple( itertools.chain.from_iterable(map(lambda x: x.free_variables, self.inputs)) ) @@ -81,7 +82,8 @@ def is_scalar_expr(self) -> bool: return False @abc.abstractmethod - def replace_args(self: TExpression, *arg) -> TExpression: ... + def replace_args(self: TExpression, *arg) -> TExpression: + ... def transform_children( self: TExpression, t: Callable[[expression.Expression], expression.Expression] @@ -90,7 +92,7 @@ def transform_children( def bind_variables( self: TExpression, - bindings: Mapping[Hashable, expression.Expression], + bindings: Mapping[str, expression.Expression], allow_partial_bindings: bool = False, ) -> TExpression: return self.transform_children( @@ -190,7 +192,7 @@ def children(self) -> Tuple[expression.Expression, ...]: return self.inputs @property - def free_variables(self) -> typing.Tuple[Hashable, ...]: + def free_variables(self) -> typing.Tuple[str, ...]: return tuple( itertools.chain.from_iterable(map(lambda x: x.free_variables, self.inputs)) ) @@ -214,7 +216,7 @@ def transform_children( def bind_variables( self: WindowExpression, - bindings: Mapping[Hashable, expression.Expression], + bindings: Mapping[str, expression.Expression], allow_partial_bindings: bool = False, ) -> WindowExpression: return self.transform_children( diff --git a/bigframes/core/array_value.py b/bigframes/core/array_value.py index d7fb186ae91..7901243e4b0 100644 --- a/bigframes/core/array_value.py +++ b/bigframes/core/array_value.py @@ -13,34 +13,35 @@ # limitations under the License. from __future__ import annotations +from dataclasses import dataclass import datetime import functools import typing -from dataclasses import dataclass -from typing import Iterable, List, Mapping, Optional, Sequence, Tuple, Union +from typing import Iterable, List, Mapping, Optional, Sequence, Tuple +import google.cloud.bigquery import pandas import pyarrow as pa +from bigframes.core import ( + agg_expressions, + bq_data, + expression_factoring, + join_def, + local_data, +) import bigframes.core.expression as ex import bigframes.core.guid import bigframes.core.identifiers as ids import bigframes.core.nodes as nodes +from bigframes.core.ordering import OrderingExpression import bigframes.core.ordering as orderings import bigframes.core.schema as schemata import bigframes.core.tree_properties +from bigframes.core.window_spec import WindowSpec import bigframes.dtypes import bigframes.operations as ops import bigframes.operations.aggregations as agg_ops -from bigframes.core import ( - agg_expressions, - bq_data, - expression_factoring, - join_def, - local_data, -) -from bigframes.core.ordering import OrderingExpression -from bigframes.core.window_spec import WindowSpec if typing.TYPE_CHECKING: from bigframes.session import Session @@ -90,7 +91,7 @@ def from_range(cls, start, end, step): @classmethod def from_table( cls, - table: Union[bq_data.BiglakeIcebergTable, bq_data.GbqNativeTable], + table: google.cloud.bigquery.Table, session: Session, *, columns: Optional[Sequence[str]] = None, @@ -102,6 +103,8 @@ def from_table( ): if offsets_col and primary_key: raise ValueError("must set at most one of 'offests', 'primary_key'") + # define data source only for needed columns, this makes row-hashing cheaper + table_def = bq_data.GbqTable.from_table(table, columns=columns or ()) # create ordering from info ordering = None @@ -112,9 +115,7 @@ def from_table( [ids.ColumnId(key_part) for key_part in primary_key] ) - bf_schema = schemata.ArraySchema.from_bq_schema( - table.physical_schema, columns=columns - ) + bf_schema = schemata.ArraySchema.from_bq_table(table, columns=columns) # Scan all columns by default, we define this list as it can be pruned while preserving source_def scan_list = nodes.ScanList( tuple( @@ -123,7 +124,7 @@ def from_table( ) ) source_def = bq_data.BigqueryDataSource( - table=table, + table=table_def, schema=bf_schema, at_time=at_time, sql_predicate=predicate, @@ -204,25 +205,14 @@ def filter_by_id(self, predicate_id: str, keep_null: bool = False) -> ArrayValue return self.filter(predicate) def filter(self, predicate: ex.Expression): - if predicate.is_scalar_expr: - return ArrayValue(nodes.FilterNode(child=self.node, predicate=predicate)) - else: - arr, filter_ids = self.compute_general_expression([predicate]) - arr = arr.filter_by_id(filter_ids[0]) - return arr.drop_columns(filter_ids) + return ArrayValue(nodes.FilterNode(child=self.node, predicate=predicate)) def order_by( - self, - by: Sequence[OrderingExpression], - is_total_order: bool = False, - stable: bool = True, + self, by: Sequence[OrderingExpression], is_total_order: bool = False ) -> ArrayValue: return ArrayValue( nodes.OrderByNode( - child=self.node, - by=tuple(by), - is_total_order=is_total_order, - stable=stable, + child=self.node, by=tuple(by), is_total_order=is_total_order ) ) @@ -547,7 +537,6 @@ def relational_join( for l_col, r_col in conditions ), type=type, - nulls_equal=True, # pandas semantics propogate_order=propogate_order or self.session._strictly_ordered, ) return ArrayValue(join_node), (l_mapping, r_mapping) diff --git a/bigframes/core/bigframe_node.py b/bigframes/core/bigframe_node.py index c48605dc248..7e40248a009 100644 --- a/bigframes/core/bigframe_node.py +++ b/bigframes/core/bigframe_node.py @@ -22,9 +22,9 @@ import typing from typing import Callable, Dict, Generator, Iterable, Mapping, Sequence, Tuple +from bigframes.core import expression, field, identifiers import bigframes.core.schema as schemata import bigframes.dtypes -from bigframes.core import expression, field, identifiers COLUMN_SET = frozenset[identifiers.ColumnId] @@ -142,7 +142,8 @@ def roots(self) -> typing.Set[BigFrameNode]: # TODO: Store some local data lazily for select, aggregate nodes. @property @abc.abstractmethod - def fields(self) -> Sequence[field.Field]: ... + def fields(self) -> Sequence[field.Field]: + ... @property def ids(self) -> Iterable[identifiers.ColumnId]: @@ -299,9 +300,9 @@ def iter_nodes_topo( self: BigFrameNode, ) -> Generator[BigFrameNode, None, None]: """Returns nodes in reverse topological order, using Kahn's algorithm.""" - child_to_parents: Dict[BigFrameNode, list[BigFrameNode]] = ( - collections.defaultdict(list) - ) + child_to_parents: Dict[ + BigFrameNode, list[BigFrameNode] + ] = collections.defaultdict(list) out_degree: Dict[BigFrameNode, int] = collections.defaultdict(int) queue: collections.deque["BigFrameNode"] = collections.deque() @@ -329,30 +330,20 @@ def top_down( """ Perform a top-down transformation of the BigFrameNode tree. """ + to_process = [self] results: Dict[BigFrameNode, BigFrameNode] = {} - # Each stack entry is (node, t_node). t_node is None until transform(node) is called. - stack: list[tuple[BigFrameNode, typing.Optional[BigFrameNode]]] = [(self, None)] - while stack: - node, t_node = stack[-1] - - if t_node is None: - if node in results: - stack.pop() - continue - t_node = transform(node) - stack[-1] = (node, t_node) - - all_done = True - for child in reversed(t_node.child_nodes): - if child not in results: - stack.append((child, None)) - all_done = False - break - - if all_done: - results[node] = t_node.transform_children(lambda x: results[x]) - stack.pop() + while to_process: + item = to_process.pop() + if item not in results.keys(): + item_result = transform(item) + results[item] = item_result + to_process.extend(item_result.child_nodes) + + to_process = [self] + # for each processed item, replace its children + for item in reversed(list(results.keys())): + results[item] = results[item].transform_children(lambda x: results[x]) return results[self] diff --git a/bigframes/core/block_transforms.py b/bigframes/core/block_transforms.py index c919b88614d..5c6395d1714 100644 --- a/bigframes/core/block_transforms.py +++ b/bigframes/core/block_transforms.py @@ -14,122 +14,22 @@ from __future__ import annotations import functools -import inspect import typing -from typing import Callable, Hashable, Optional, Sequence +from typing import Optional, Sequence import bigframes_vendored.constants as constants import pandas as pd import bigframes.constants +from bigframes.core import agg_expressions import bigframes.core as core import bigframes.core.blocks as blocks -import bigframes.core.bytecode as bytecode import bigframes.core.expression as ex import bigframes.core.ordering as ordering -import bigframes.core.window_spec as window_specs +import bigframes.core.window_spec as windows import bigframes.dtypes as dtypes -import bigframes.functions import bigframes.operations as ops import bigframes.operations.aggregations as agg_ops -from bigframes._config import options -from bigframes.core import agg_expressions, py_expressions - - -def compile_udf( - block: blocks.Block, - func: Callable, - args: tuple = (), - kwargs: dict | None = None, - col_series_args: typing.Mapping[str, str] | None = None, - window_spec: Optional[window_specs.WindowSpec] = None, -) -> ex.Expression: - """Compile a python function to a BigFrames expression in the context of a block.""" - if kwargs is None: - kwargs = {} - expr = bytecode._compile_bytecode_to_py_expr(func) - sig = inspect.signature(func) - - bindings: dict[Hashable, ex.Expression] = {} - - bound_args = sig.bind(*(None, *args), **kwargs) - bound_args.apply_defaults() - bound_params = bound_args.arguments - for name, value in bound_params.items(): - bindings[name] = ex.const(value) - - series_arg = next(iter(sig.parameters.keys())) - - if col_series_args is not None: - expr = py_expressions.resolve_py_exprs( - expr, - series_arg=series_arg, - col_series_args=col_series_args, - window_spec=window_spec, - ) - else: - series_attrs: dict = {} - for i, (col_id, label) in enumerate( - zip(block.value_columns, block.column_labels) - ): - series_attrs[i] = col_id - if label is not None: - series_attrs[label] = col_id - - expr = py_expressions.resolve_py_exprs( - expr, - series_arg=series_arg, - series_attrs=series_attrs, - window_spec=window_spec, - ) - - expr = expr.bind_variables(bindings) - return expr - - -def is_transpiler_eligible(func: typing.Any) -> bool: - """Return True if func is eligible for Python transpilation.""" - return ( - options.experiments.enable_python_transpiler - and callable(func) - and not isinstance(func, bigframes.functions.Udf) - ) - - -def compile_column_udf( - block: blocks.Block, - func: Callable, - column_id: str, - args: tuple = (), - kwargs: dict | None = None, - window_spec: Optional[window_specs.WindowSpec] = None, -) -> tuple[ex.Expression, str]: - """Compile a column-wise python UDF in block context and return (expr, name).""" - sig = inspect.signature(func) - series_arg = next(iter(sig.parameters.keys())) - expr = compile_udf( - block, - func, - args=args, - kwargs=kwargs, - col_series_args={series_arg: column_id}, - window_spec=window_spec, - ) - name = getattr(func, "__name__", "") - return expr, name - - -def apply_to_block_rows( - func: Callable, block: blocks.Block, *args, **kwargs -) -> blocks.Block: - """ - Apply the given function to each row of the block. - - The function is applied to each row of the block, and the result is returned - as a new block with the same index. - """ - expr = compile_udf(block, func, args, kwargs) - return block.project_exprs([expr], labels=[None], drop=True) def equals(block1: blocks.Block, block2: blocks.Block) -> bool: @@ -171,13 +71,13 @@ def indicate_duplicates( agg_expressions.NullaryAggregation( agg_ops.RowNumberOp(), ), - window=window_specs.unbound(grouping_keys=tuple(columns)), + window=windows.unbound(grouping_keys=tuple(columns)), ) count = agg_expressions.WindowExpression( agg_expressions.NullaryAggregation( agg_ops.SizeOp(), ), - window=window_specs.unbound(grouping_keys=tuple(columns)), + window=windows.unbound(grouping_keys=tuple(columns)), ) if keep == "first": @@ -211,7 +111,7 @@ def quantile( dropna: bool = False, ) -> blocks.Block: # TODO: handle windowing and more interpolation methods - window = window_specs.unbound( + window = windows.unbound( grouping_keys=tuple(grouping_column_ids), ) quantile_cols = [] @@ -312,8 +212,8 @@ def _interpolate_column( if interpolate_method not in ["linear", "nearest", "ffill"]: raise ValueError("interpolate method not supported") window_ordering = (ordering.OrderingExpression(ex.deref(x_values)),) - backwards_window = window_specs.rows(end=0, ordering=window_ordering) - forwards_window = window_specs.rows(start=0, ordering=window_ordering) + backwards_window = windows.rows(end=0, ordering=window_ordering) + forwards_window = windows.rows(start=0, ordering=window_ordering) # Note, this method may block, notnull = block.apply_unary_op(column, ops.notnull_op) @@ -465,7 +365,7 @@ def value_counts( ) count_id = block.value_columns[0] if normalize: - unbound_window = window_specs.unbound(grouping_keys=tuple(grouping_keys)) + unbound_window = windows.unbound(grouping_keys=tuple(grouping_keys)) block, total_count_id = block.apply_window_op( count_id, agg_ops.sum_op, unbound_window ) @@ -493,7 +393,7 @@ def pct_change(block: blocks.Block, periods: int = 1) -> blocks.Block: column_labels = block.column_labels # Window framing clause is not allowed for analytic function lag. - window_spec = window_specs.unbound() + window_spec = windows.unbound() original_columns = block.value_columns exprs = [] @@ -548,9 +448,9 @@ def rank( ) window_op = agg_ops.dense_rank_op if method == "dense" else agg_ops.count_op window_spec = ( - window_specs.unbound(grouping_keys=grouping_cols, ordering=window_ordering) + windows.unbound(grouping_keys=grouping_cols, ordering=window_ordering) if method == "dense" - else window_specs.rows( + else windows.rows( end=0, ordering=window_ordering, grouping_keys=grouping_cols ) ) @@ -562,7 +462,7 @@ def rank( result_expr, agg_expressions.WindowExpression( agg_expressions.UnaryAggregation(agg_ops.max_op, result_expr), - window_specs.unbound(grouping_keys=grouping_cols), + windows.unbound(grouping_keys=grouping_cols), ), ) # Step 2: Apply aggregate to groups of like input values. @@ -575,7 +475,7 @@ def rank( }[method] result_expr = agg_expressions.WindowExpression( agg_expressions.UnaryAggregation(agg_op, result_expr), - window_specs.unbound(grouping_keys=(col, *grouping_cols)), + windows.unbound(grouping_keys=(col, *grouping_cols)), ) # Pandas masks all values where any grouping column is null # Note: we use pd.NA instead of float('nan') @@ -676,7 +576,7 @@ def nsmallest( block, counter = block.apply_window_op( column_ids[0], agg_ops.rank_op, - window_spec=window_specs.unbound(ordering=tuple(order_refs)), + window_spec=windows.unbound(ordering=tuple(order_refs)), ) block, condition = block.project_expr(ops.le_op.as_expr(counter, ex.const(n))) block = block.filter_by_id(condition) @@ -706,7 +606,7 @@ def nlargest( block, counter = block.apply_window_op( column_ids[0], agg_ops.rank_op, - window_spec=window_specs.unbound(ordering=tuple(order_refs)), + window_spec=windows.unbound(ordering=tuple(order_refs)), ) block, condition = block.project_expr(ops.le_op.as_expr(counter, ex.const(n))) block = block.filter_by_id(condition) @@ -718,6 +618,7 @@ def skew( skew_column_ids: typing.Sequence[str], grouping_column_ids: typing.Sequence[str] = (), ) -> blocks.Block: + original_columns = skew_column_ids column_labels = block.select_columns(original_columns).column_labels @@ -977,7 +878,7 @@ def _idx_extrema( for idx_col in original_block.index_columns ], ] - window_spec = window_specs.unbound(ordering=tuple(order_refs)) + window_spec = windows.unbound(ordering=tuple(order_refs)) idx_col = original_block.index_columns[0] block, result_col = block.apply_window_op( idx_col, agg_ops.first_op, window_spec diff --git a/bigframes/core/blocks.py b/bigframes/core/blocks.py index 8522a4d97be..5bac1a06f1e 100644 --- a/bigframes/core/blocks.py +++ b/bigframes/core/blocks.py @@ -28,7 +28,6 @@ import itertools import random import typing -import warnings from typing import ( Iterable, Iterator, @@ -40,6 +39,7 @@ Tuple, Union, ) +import warnings import bigframes_vendored.constants as constants import google.cloud.bigquery as bigquery @@ -47,7 +47,10 @@ import pandas as pd import pyarrow as pa +from bigframes import session +from bigframes._config import sampling_options import bigframes.constants +from bigframes.core import agg_expressions, local_data import bigframes.core as core import bigframes.core.agg_expressions as ex_types import bigframes.core.expression as ex @@ -63,9 +66,6 @@ import bigframes.exceptions as bfe import bigframes.operations as ops import bigframes.operations.aggregations as agg_ops -from bigframes import session -from bigframes._config import sampling_options -from bigframes.core import agg_expressions, local_data from bigframes.session import dry_runs, execution_spec from bigframes.session import executor as executors from bigframes.session._io import pandas as io_pandas @@ -140,7 +140,6 @@ def __init__( column_labels: typing.Union[pd.Index, typing.Iterable[Label]], index_labels: typing.Union[pd.Index, typing.Iterable[Label], None] = None, *, - value_columns: Optional[Iterable[str]] = None, transpose_cache: Optional[Block] = None, ): """Construct a block object, will create default index if no index columns specified.""" @@ -159,13 +158,7 @@ def __init__( if index_labels else tuple([None for _ in index_columns]) ) - if value_columns is None: - value_columns = [ - col_id for col_id in expr.column_ids if col_id not in index_columns - ] - self._expr = self._normalize_expression( - expr, self._index_columns, value_columns - ) + self._expr = self._normalize_expression(expr, self._index_columns) # Use pandas index to more easily replicate column indexing, especially for hierarchical column index self._column_labels = ( column_labels.copy() @@ -395,10 +388,9 @@ def cols_matching_label(self, partial_label: Label) -> typing.Sequence[str]: def order_by( self, by: typing.Sequence[ordering.OrderingExpression], - stable: bool = True, ) -> Block: return Block( - self._expr.order_by(by, stable=stable), + self._expr.order_by(by), index_columns=self.index_columns, column_labels=self.column_labels, index_labels=self.index.names, @@ -658,6 +650,7 @@ def _get_sampling_option( sampling_method: Optional[str] = None, random_state: Optional[int] = None, ) -> sampling_options.SamplingOptions: + if (sampling_method is not None) and (sampling_method not in _SAMPLING_METHODS): raise NotImplementedError( f"The downsampling method {sampling_method} is not implemented, " @@ -696,13 +689,11 @@ def to_pandas_batches( page_size: Optional[int] = None, max_results: Optional[int] = None, allow_large_results: Optional[bool] = None, - cell_execution_count: Optional[int] = None, ) -> PandasBatches: """Download results one message at a time. page_size and max_results determine the size and number of batches, - see https://cloud.google.com/python/docs/reference/bigquery/latest/google.cloud.bigquery.job.QueryJob#google_cloud_bigquery_job_QueryJob_result - """ + see https://cloud.google.com/python/docs/reference/bigquery/latest/google.cloud.bigquery.job.QueryJob#google_cloud_bigquery_job_QueryJob_result""" under_10gb = ( (not allow_large_results) @@ -714,7 +705,6 @@ def to_pandas_batches( execution_spec.ExecutionSpec( promise_under_10gb=under_10gb, ordered=True, - cell_execution_count=cell_execution_count, ), ) result_batches = execution_result.batches() @@ -850,7 +840,7 @@ def _materialize_local( else: raw_df = result_batches.to_pandas() df = self._copy_index_to_pandas(raw_df) - df.columns = self.column_labels + df.set_axis(self.column_labels, axis=1, copy=False) return df, execute_result.query_job def split( @@ -1093,14 +1083,14 @@ def multi_apply_window_op( def multi_apply_unary_op( self, - op: Union[ops.UnaryOp, ops.NaryOp, ex.Expression], + op: Union[ops.UnaryOp, ex.Expression], ) -> Block: - if isinstance(op, (ops.UnaryOp, ops.NaryOp)): + if isinstance(op, ops.UnaryOp): input_varname = guid.generate_guid() expr = op.as_expr(ex.free_var(input_varname)) else: input_varnames = op.free_variables - assert len(set(input_varnames)) == 1 + assert len(input_varnames) == 1 expr = op input_varname = input_varnames[0] @@ -1124,15 +1114,13 @@ def project_exprs( labels: Union[Sequence[Label], pd.Index], drop=False, ) -> Block: - new_array, new_cols = self.expr.compute_values(exprs) + new_array, _ = self.expr.compute_values(exprs) if drop: new_array = new_array.drop_columns(self.value_columns) - new_val_cols = new_cols if drop else (*self.value_columns, *new_cols) return Block( new_array, index_columns=self.index_columns, - value_columns=new_val_cols, column_labels=labels if drop else self.column_labels.append(pd.Index(labels)), @@ -1554,13 +1542,17 @@ def _get_labels_for_columns(self, column_ids: typing.Sequence[str]) -> pd.Index: def _normalize_expression( self, expr: core.ArrayValue, - index_columns: Iterable[str], - value_columns: Iterable[str], + index_columns: typing.Sequence[str], + assert_value_size: typing.Optional[int] = None, ): """Normalizes expression by moving index columns to left.""" - normalized_ids = (*index_columns, *value_columns) - if tuple(expr.column_ids) == normalized_ids: - return expr + value_columns = [ + col_id for col_id in expr.column_ids if col_id not in index_columns + ] + if (assert_value_size is not None) and ( + len(value_columns) != assert_value_size + ): + raise ValueError("Unexpected number of value columns.") return expr.select_columns([*index_columns, *value_columns]) def grouped_head( @@ -1825,9 +1817,9 @@ def melt( Arguments correspond to pandas.melt arguments. """ # TODO: Implement col_level and ignore_index - value_labels: pd.Index = self.column_labels[ - [self.value_columns.index(col_id) for col_id in value_vars] - ] + value_labels: pd.Index = pd.Index( + [self.col_id_to_label[col_id] for col_id in value_vars] + ) id_labels = [self.col_id_to_label[col_id] for col_id in id_vars] unpivot_expr, (var_col_ids, unpivot_out, passthrough_cols) = unpivot( @@ -1991,10 +1983,6 @@ def _generate_resample_label( ) level = level or 0 col_id = self.index.resolve_level(level)[0] - if isinstance(level, int): - resample_label = self.index.names[level] - else: - resample_label = level # Reset index to make the resampling level a column, then drop all other index columns. # This simplifies processing by focusing solely on the column required for resampling. block = self.reset_index(drop=False) @@ -2013,7 +2001,6 @@ def _generate_resample_label( raise KeyError(f"The grouper name {on} is not found") col_id = matches[0] - resample_label = on block = self if level is None: dtype = self._column_type(col_id) @@ -2106,7 +2093,6 @@ def _generate_resample_label( block.value_columns[0], block.value_columns[1], op=ops.IntegerLabelToDatetimeOp(freq=freq, label=label, origin=origin), - result_label=resample_label, ) # After multiple merges, the columns: @@ -2421,13 +2407,13 @@ def _align_both_axes( rcol_indexer if (rcol_indexer is not None) else range(len(columns)) ) - left_input_lookup = lambda index: ( - ex.deref(get_column_left[self.value_columns[index]]) + left_input_lookup = ( + lambda index: ex.deref(get_column_left[self.value_columns[index]]) if index != -1 else ex.const(None) ) - righ_input_lookup = lambda index: ( - ex.deref(get_column_right[other.value_columns[index]]) + righ_input_lookup = ( + lambda index: ex.deref(get_column_right[other.value_columns[index]]) if index != -1 else ex.const(None) ) @@ -2480,13 +2466,15 @@ def _align_series_block_axis_1( rcol_indexer if (rcol_indexer is not None) else range(len(columns)) ) - left_input_lookup = lambda index: ( - ex.deref(get_column_left[self.value_columns[index]]) + left_input_lookup = ( + lambda index: ex.deref(get_column_left[self.value_columns[index]]) if index != -1 else ex.const(None) ) - righ_input_lookup = lambda index: ( - ex.deref(get_column_right[other.transpose().value_columns[index]]) + righ_input_lookup = ( + lambda index: ex.deref( + get_column_right[other.transpose().value_columns[index]] + ) if index != -1 else ex.const(None) ) @@ -2513,11 +2501,13 @@ def _align_pd_series_axis_1( rcol_indexer if (rcol_indexer is not None) else range(len(columns)) ) - left_input_lookup = lambda index: ( - ex.deref(self.value_columns[index]) if index != -1 else ex.const(None) + left_input_lookup = ( + lambda index: ex.deref(self.value_columns[index]) + if index != -1 + else ex.const(None) ) - righ_input_lookup = lambda index: ( - ex.const(other.iloc[index]) if index != -1 else ex.const(None) + righ_input_lookup = ( + lambda index: ex.const(other.iloc[index]) if index != -1 else ex.const(None) ) left_inputs = [left_input_lookup(i) for i in lcol_indexer] @@ -2550,10 +2540,7 @@ def join( sort: bool = False, block_identity_join: bool = False, always_order: bool = False, - ) -> Tuple[ - Block, - Tuple[Mapping[str, str], Mapping[str, str]], - ]: + ) -> Tuple[Block, Tuple[Mapping[str, str], Mapping[str, str]],]: """ Join two blocks objects together, and provide mappings between source columns and output columns. @@ -2801,11 +2788,8 @@ def _is_monotonic( ) block = block.drop_columns([equal_monotonic_id, strict_monotonic_id]) - assert last_result_id is not None block, monotonic_result_id = block.apply_binary_op( - last_result_id, - last_notna_id, - ops.and_op, # type: ignore + last_result_id, last_notna_id, ops.and_op # type: ignore ) if last_result_id is not None: block = block.drop_columns([last_result_id, last_notna_id]) @@ -2974,12 +2958,7 @@ def is_uniquely_named(self: BlockIndexProperties): def try_new_row_join( left: Block, right: Block -) -> Optional[ - Tuple[ - Block, - Tuple[Mapping[str, str], Mapping[str, str]], - ] -]: +) -> Optional[Tuple[Block, Tuple[Mapping[str, str], Mapping[str, str]],]]: join_keys = tuple( (left_id, right_id) for left_id, right_id in zip(left.index_columns, right.index_columns) @@ -3010,12 +2989,7 @@ def try_legacy_row_join( right: Block, *, how="left", -) -> Optional[ - Tuple[ - Block, - Tuple[Mapping[str, str], Mapping[str, str]], - ] -]: +) -> Optional[Tuple[Block, Tuple[Mapping[str, str], Mapping[str, str]],]]: """Joins two blocks that have a common root expression by merging the projections.""" left_expr = left.expr right_expr = right.expr @@ -3069,10 +3043,7 @@ def try_legacy_row_join( def join_with_single_row( left: Block, single_row_block: Block, -) -> Tuple[ - Block, - Tuple[Mapping[str, str], Mapping[str, str]], -]: +) -> Tuple[Block, Tuple[Mapping[str, str], Mapping[str, str]],]: """ Special join case where other is a single row block. This property is not validated, caller responsible for not passing multi-row block. @@ -3107,10 +3078,7 @@ def join_mono_indexed( how="left", sort: bool = False, propogate_order: bool = False, -) -> Tuple[ - Block, - Tuple[Mapping[str, str], Mapping[str, str]], -]: +) -> Tuple[Block, Tuple[Mapping[str, str], Mapping[str, str]],]: left_expr = left.expr right_expr = right.expr @@ -3157,10 +3125,7 @@ def join_multi_indexed( how="left", sort: bool = False, propogate_order: bool = False, -) -> Tuple[ - Block, - Tuple[Mapping[str, str], Mapping[str, str]], -]: +) -> Tuple[Block, Tuple[Mapping[str, str], Mapping[str, str]],]: if not (left.index.is_uniquely_named() and right.index.is_uniquely_named()): raise ValueError("Joins not supported on indices with non-unique level names") @@ -3447,7 +3412,6 @@ def unpivot( joined_array, (labels_mapping, column_mapping) = labels_array.relational_join( array_value, type="cross" ) - new_passthrough_cols = [column_mapping[col] for col in passthrough_columns] # Last column is offsets index_col_ids = [labels_mapping[col] for col in labels_array.column_ids[:-1]] @@ -3457,24 +3421,20 @@ def unpivot( unpivot_exprs: List[ex.Expression] = [] # Supports producing multiple stacked ouput columns for stacking only part of hierarchical index for input_ids in unpivot_columns: - col_expr: ex.Expression - if not input_ids: - col_expr = ex.const(None, dtype=bigframes.dtypes.INT_DTYPE) - else: - # row explode offset used to choose the input column - # we use offset instead of label as labels are not necessarily unique - cases = itertools.chain( - *( - ( - ops.eq_op.as_expr(explode_offsets_id, ex.const(i)), - ex.deref(column_mapping[id_or_null]) - if (id_or_null is not None) - else ex.const(None), - ) - for i, id_or_null in enumerate(input_ids) + # row explode offset used to choose the input column + # we use offset instead of label as labels are not necessarily unique + cases = itertools.chain( + *( + ( + ops.eq_op.as_expr(explode_offsets_id, ex.const(i)), + ex.deref(column_mapping[id_or_null]) + if (id_or_null is not None) + else ex.const(None), ) + for i, id_or_null in enumerate(input_ids) ) - col_expr = ops.case_when_op.as_expr(*cases) + ) + col_expr = ops.case_when_op.as_expr(*cases) unpivot_exprs.append(col_expr) joined_array, unpivot_col_ids = joined_array.compute_values(unpivot_exprs) @@ -3492,43 +3452,19 @@ def _pd_index_to_array_value( Create an ArrayValue from a list of label tuples. The last column will be row offsets. """ - id_gen = bigframes.core.identifiers.standard_id_strings() - col_ids = [next(id_gen) for _ in range(index.nlevels)] - offset_id = next(id_gen) - rows = [] labels_as_tuples = utils.index_as_tuples(index) for row_offset in range(len(index)): + id_gen = bigframes.core.identifiers.standard_id_strings() row_label = labels_as_tuples[row_offset] row_label = (row_label,) if not isinstance(row_label, tuple) else row_label row = {} - for label_part, col_id in zip(row_label, col_ids): - row[col_id] = label_part if pd.notnull(label_part) else None - row[offset_id] = row_offset + for label_part, id in zip(row_label, id_gen): + row[id] = label_part if pd.notnull(label_part) else None + row[next(id_gen)] = row_offset rows.append(row) - if not rows: - dtypes_list = getattr(index, "dtypes", None) - if dtypes_list is None: - dtypes_list = ( - [index.dtype] if hasattr(index, "dtype") else [pd.Float64Dtype()] - ) - - fields = [] - for col_id, dtype in zip(col_ids, dtypes_list): - try: - pa_type = bigframes.dtypes.bigframes_dtype_to_arrow_dtype(dtype) - except Exception: - pa_type = pa.string() - fields.append(pa.field(col_id, pa_type)) - fields.append(pa.field(offset_id, pa.int64())) - schema = pa.schema(fields) - pt = pa.Table.from_pylist([], schema=schema) - else: - pt = pa.Table.from_pylist(rows) - pt = pt.rename_columns([*col_ids, offset_id]) - - return core.ArrayValue.from_pyarrow(pt, session=session) + return core.ArrayValue.from_pyarrow(pa.Table.from_pylist(rows), session=session) def _resolve_index_col( diff --git a/bigframes/core/bq_data.py b/bigframes/core/bq_data.py index 55ac1270b6c..3b42ff7c031 100644 --- a/bigframes/core/bq_data.py +++ b/bigframes/core/bq_data.py @@ -22,214 +22,74 @@ import queue import threading import typing -from typing import Any, Iterator, List, Literal, Optional, Sequence, Tuple, Union +from typing import Any, Iterator, Optional, Sequence, Tuple +from google.cloud import bigquery_storage_v1 import google.cloud.bigquery as bq import google.cloud.bigquery_storage_v1.types as bq_storage_types -import pyarrow as pa -from google.cloud import bigquery_storage_v1 from google.protobuf import timestamp_pb2 +import pyarrow as pa -import bigframes.constants -import bigframes.core.schema from bigframes.core import pyarrow_utils +import bigframes.core.schema if typing.TYPE_CHECKING: import bigframes.core.ordering as orderings -def _resolve_standard_gcp_region(bq_region: str): - """ - Resolve bq regions to standardized - """ - if bq_region.casefold() == "US": - return "us-central1" - elif bq_region.casefold() == "EU": - return "europe-west4" - return bq_region - - -def is_irc_table(table_id: str): - """ - Determines if a table id should be resolved through the iceberg rest catalog. - """ - return len(table_id.split(".")) == 4 - - -def is_compatible( - data_region: Union[GcsRegion, BigQueryRegion], session_location: str -) -> bool: - # based on https://docs.cloud.google.com/bigquery/docs/locations#storage-location-considerations - if isinstance(data_region, BigQueryRegion): - return data_region.name == session_location - else: - assert isinstance(data_region, GcsRegion) - # TODO(b/463675088): Multi-regions don't yet support rest catalog tables - if session_location in bigframes.constants.BIGQUERY_MULTIREGIONS: - return False - return _resolve_standard_gcp_region(session_location) in data_region.included - - -def get_default_bq_region(data_region: Union[GcsRegion, BigQueryRegion]) -> str: - if isinstance(data_region, BigQueryRegion): - return data_region.name - elif isinstance(data_region, GcsRegion): - # should maybe try to track and prefer primary replica? - return data_region.included[0] - - -@dataclasses.dataclass(frozen=True) -class BigQueryRegion: - name: str - - -@dataclasses.dataclass(frozen=True) -class GcsRegion: - # this is the name of gcs regions, which may be names for multi-regions, so shouldn't be compared with non-gcs locations - storage_regions: tuple[str, ...] - # this tracks all the included standard, specific regions (eg us-east1), and should be comparable to bq regions (except non-standard US, EU, omni regions) - included: tuple[str, ...] - - -# what is the line between metadata and core fields? Mostly metadata fields are optional or unreliable, but its fuzzy @dataclasses.dataclass(frozen=True) -class TableMetadata: - # this size metadata might be stale, don't use where strict correctness is needed - location: Union[BigQueryRegion, GcsRegion] - type: Literal["TABLE", "EXTERNAL", "VIEW", "MATERIALIZE_VIEW", "SNAPSHOT"] - numBytes: Optional[int] = None - numRows: Optional[int] = None - created_time: Optional[datetime.datetime] = None - modified_time: Optional[datetime.datetime] = None - - -@dataclasses.dataclass(frozen=True) -class GbqNativeTable: +class GbqTable: project_id: str = dataclasses.field() dataset_id: str = dataclasses.field() table_id: str = dataclasses.field() physical_schema: Tuple[bq.SchemaField, ...] = dataclasses.field() - metadata: TableMetadata = dataclasses.field() - partition_col: Optional[str] = None - cluster_cols: typing.Optional[Tuple[str, ...]] = None - primary_key: Optional[Tuple[str, ...]] = None + is_physically_stored: bool = dataclasses.field() + cluster_cols: typing.Optional[Tuple[str, ...]] @staticmethod - def from_table(table: bq.Table, columns: Sequence[str] = ()) -> GbqNativeTable: + def from_table(table: bq.Table, columns: Sequence[str] = ()) -> GbqTable: # Subsetting fields with columns can reduce cost of row-hash default ordering if columns: schema = tuple(item for item in table.schema if item.name in columns) else: schema = tuple(table.schema) - - metadata = TableMetadata( - numBytes=table.num_bytes, - numRows=table.num_rows, - location=BigQueryRegion(table.location), # type: ignore - type=table.table_type or "TABLE", # type: ignore - created_time=table.created, - modified_time=table.modified, - ) - partition_col = None - if table.range_partitioning: - partition_col = table.range_partitioning.field - elif table.time_partitioning: - partition_col = table.time_partitioning.field - - return GbqNativeTable( + return GbqTable( project_id=table.project, dataset_id=table.dataset_id, table_id=table.table_id, physical_schema=schema, - partition_col=partition_col, + is_physically_stored=(table.table_type in ["TABLE", "MATERIALIZED_VIEW"]), cluster_cols=None - if (table.clustering_fields is None) + if table.clustering_fields is None else tuple(table.clustering_fields), - primary_key=tuple(_get_primary_keys(table)), - metadata=metadata, ) @staticmethod def from_ref_and_schema( table_ref: bq.TableReference, schema: Sequence[bq.SchemaField], - location: str, - table_type: Literal["TABLE"] = "TABLE", cluster_cols: Optional[Sequence[str]] = None, - ) -> GbqNativeTable: - return GbqNativeTable( + ) -> GbqTable: + return GbqTable( project_id=table_ref.project, dataset_id=table_ref.dataset_id, table_id=table_ref.table_id, - metadata=TableMetadata(location=BigQueryRegion(location), type=table_type), physical_schema=tuple(schema), + is_physically_stored=True, cluster_cols=tuple(cluster_cols) if cluster_cols else None, ) - @property - def is_physically_stored(self) -> bool: - return self.metadata.type in ["TABLE", "MATERIALIZED_VIEW"] - def get_table_ref(self) -> bq.TableReference: return bq.TableReference( bq.DatasetReference(self.project_id, self.dataset_id), self.table_id ) - def get_full_id(self, quoted: bool = False) -> str: - if quoted: - return f"`{self.project_id}`.`{self.dataset_id}`.`{self.table_id}`" - return f"{self.project_id}.{self.dataset_id}.{self.table_id}" - @property @functools.cache def schema_by_id(self): return {col.name: col for col in self.physical_schema} -@dataclasses.dataclass(frozen=True) -class BiglakeIcebergTable: - project_id: str = dataclasses.field() - catalog_id: str = dataclasses.field() - namespace_id: str = dataclasses.field() - table_id: str = dataclasses.field() - physical_schema: Tuple[bq.SchemaField, ...] = dataclasses.field() - cluster_cols: typing.Optional[Tuple[str, ...]] - metadata: TableMetadata - - def get_full_id(self, quoted: bool = False) -> str: - if quoted: - return f"`{self.project_id}`.`{self.catalog_id}`.`{self.namespace_id}`.`{self.table_id}`" - return ( - f"{self.project_id}.{self.catalog_id}.{self.namespace_id}.{self.table_id}" - ) - - @property - @functools.cache - def schema_by_id(self): - return {col.name: col for col in self.physical_schema} - - @property - def partition_col(self) -> Optional[str]: - # TODO: Use iceberg partition metadata - return None - - @property - def dataset_id(self) -> str: - """ - Not a true dataset, but serves as the dataset component of the identifer in sql queries - """ - return f"{self.catalog_id}.{self.namespace_id}" - - @property - def primary_key(self) -> Optional[Tuple[str, ...]]: - return None - - def get_table_ref(self) -> bq.TableReference: - return bq.TableReference( - bq.DatasetReference(self.project_id, self.dataset_id), self.table_id - ) - - @dataclasses.dataclass(frozen=True) class BigqueryDataSource: """ @@ -244,18 +104,15 @@ def __post_init__(self): self.schema.names ) - table: Union[GbqNativeTable, BiglakeIcebergTable] + table: GbqTable schema: bigframes.core.schema.ArraySchema at_time: typing.Optional[datetime.datetime] = None # Added for backwards compatibility, not validated sql_predicate: typing.Optional[str] = None ordering: typing.Optional[orderings.RowOrdering] = None - # Optimization field, must be correct if set, don't put maybe-stale number here + # Optimization field n_rows: Optional[int] = None - def with_ordering(self, ordering: orderings.RowOrdering) -> BigqueryDataSource: - return dataclasses.replace(self, ordering=ordering) - _WORKER_TIME_INCREMENT = 0.05 @@ -331,8 +188,6 @@ def get_arrow_batches( project_id: str, sample_rate: Optional[float] = None, ) -> ReadResult: - assert isinstance(data.table, GbqNativeTable) - table_mod_options = {} read_options_dict: dict[str, Any] = {"selected_fields": list(columns)} @@ -390,21 +245,3 @@ def process_batch(pa_batch): return ReadResult( batches, session.estimated_row_count, session.estimated_total_bytes_scanned ) - - -def _get_primary_keys( - table: bq.Table, -) -> List[str]: - """Get primary keys from table if they are set.""" - - primary_keys: List[str] = [] - if ( - (table_constraints := getattr(table, "table_constraints", None)) is not None - and (primary_key := table_constraints.primary_key) is not None - # This will be False for either None or empty list. - # We want primary_keys = None if no primary keys are set. - and (columns := primary_key.columns) - ): - primary_keys = columns if columns is not None else [] - - return primary_keys diff --git a/bigframes/core/bytecode.py b/bigframes/core/bytecode.py deleted file mode 100644 index f657ce707ea..00000000000 --- a/bigframes/core/bytecode.py +++ /dev/null @@ -1,913 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import dataclasses -import dis -import operator -import sys -from types import ModuleType -from typing import Callable - -import bigframes.core.py_expressions as py_exprs -from bigframes.core import expression -from bigframes.operations import generic_ops - -_BINARY_OP_MAP = { - "+": operator.add, - "-": operator.sub, - "*": operator.mul, - "/": operator.truediv, - "//": operator.floordiv, - "%": operator.mod, - "**": operator.pow, - "[]": operator.getitem, -} - -_COMPARE_OP_MAP = { - "==": operator.eq, - "!=": operator.ne, - "<": operator.lt, - "<=": operator.le, - ">": operator.gt, - ">=": operator.ge, -} - -_OLD_BINARY_OP_MAP = { - "BINARY_ADD": operator.add, - "INPLACE_ADD": operator.add, - "BINARY_SUBTRACT": operator.sub, - "INPLACE_SUBTRACT": operator.sub, - "BINARY_MULTIPLY": operator.mul, - "INPLACE_MULTIPLY": operator.mul, - "BINARY_TRUE_DIVIDE": operator.truediv, - "INPLACE_TRUE_DIVIDE": operator.truediv, - "BINARY_FLOOR_DIVIDE": operator.floordiv, - "INPLACE_FLOOR_DIVIDE": operator.floordiv, - "BINARY_MODULO": operator.mod, - "INPLACE_MODULO": operator.mod, - "BINARY_POWER": operator.pow, - "INPLACE_POWER": operator.pow, -} - - -_NULL = py_exprs.PyObject(None) - - -_RETURN_OPNAMES = {"RETURN_VALUE", "RETURN_CONST"} - -_UNCONDITIONAL_JUMP_OPNAMES = { - "JUMP_FORWARD", - "JUMP_ABSOLUTE", - "JUMP_BACKWARD", - "JUMP_BACKWARD_NO_INTERRUPT", - "JUMP", - "JUMP_NO_INTERRUPT", -} - -_JUMP_IF_FALSE_OPNAMES = { - "POP_JUMP_IF_FALSE", - "POP_JUMP_FORWARD_IF_FALSE", - "POP_JUMP_BACKWARD_IF_FALSE", -} - -_JUMP_IF_TRUE_OPNAMES = { - "POP_JUMP_IF_TRUE", - "POP_JUMP_FORWARD_IF_TRUE", - "POP_JUMP_BACKWARD_IF_TRUE", -} - -_JUMP_IF_NONE_OPNAMES = { - "POP_JUMP_IF_NONE", - "POP_JUMP_FORWARD_IF_NONE", - "POP_JUMP_BACKWARD_IF_NONE", -} - -_JUMP_IF_NOT_NONE_OPNAMES = { - "POP_JUMP_IF_NOT_NONE", - "POP_JUMP_FORWARD_IF_NOT_NONE", - "POP_JUMP_BACKWARD_IF_NOT_NONE", -} - -_CONDITIONAL_JUMP_OPNAMES = ( - _JUMP_IF_FALSE_OPNAMES - | _JUMP_IF_TRUE_OPNAMES - | _JUMP_IF_NONE_OPNAMES - | _JUMP_IF_NOT_NONE_OPNAMES - | { - "JUMP_IF_FALSE_OR_POP", - "JUMP_IF_TRUE_OR_POP", - } -) - -_ALL_JUMP_OPNAMES = _UNCONDITIONAL_JUMP_OPNAMES | _CONDITIONAL_JUMP_OPNAMES - - -@dataclasses.dataclass -class BasicBlock: - start_offset: int - instructions: list[dis.Instruction] - successors: list[int] = dataclasses.field(default_factory=list) - predecessors: list[int] = dataclasses.field(default_factory=list) - - -def get_block_starts(instructions: list[dis.Instruction]) -> set[int]: - starts = {0} - for i, inst in enumerate(instructions): - opname = inst.opname - if opname in _ALL_JUMP_OPNAMES: - if isinstance(inst.argval, int): - starts.add(inst.argval) - if i + 1 < len(instructions): - starts.add(instructions[i + 1].offset) - elif opname in _RETURN_OPNAMES: - if i + 1 < len(instructions): - starts.add(instructions[i + 1].offset) - return starts - - -def get_block_successors(block: BasicBlock, next_offsets: dict[int, int]) -> list[int]: - if not block.instructions: - return [] - last_inst = block.instructions[-1] - opname = last_inst.opname - offset = last_inst.offset - - next_offset = next_offsets.get(offset) - - if opname in _RETURN_OPNAMES: - return [] - - if opname in _UNCONDITIONAL_JUMP_OPNAMES: - return [last_inst.argval] - - if opname in _CONDITIONAL_JUMP_OPNAMES: - successors = [last_inst.argval] - if next_offset is not None: - successors.append(next_offset) - return successors - - if next_offset is not None: - return [next_offset] - return [] - - -def build_cfg( - instructions: list[dis.Instruction], next_offsets: dict[int, int] -) -> dict[int, BasicBlock]: - starts = sorted(list(get_block_starts(instructions))) - - blocks: dict[int, BasicBlock] = {} - for i, start in enumerate(starts): - end = starts[i + 1] if i + 1 < len(starts) else None - block_insts = [ - inst - for inst in instructions - if start <= inst.offset and (end is None or inst.offset < end) - ] - blocks[start] = BasicBlock(start_offset=start, instructions=block_insts) - - for block in blocks.values(): - successors = get_block_successors(block, next_offsets) - block.successors = successors - for succ in successors: - blocks[succ].predecessors.append(block.start_offset) - - return blocks - - -def topological_sort(blocks: dict[int, BasicBlock]) -> list[int]: - in_degree = {offset: len(block.predecessors) for offset, block in blocks.items()} - queue = [offset for offset, deg in in_degree.items() if deg == 0] - order = [] - - while queue: - queue.sort() - curr = queue.pop(0) - order.append(curr) - for succ in blocks[curr].successors: - in_degree[succ] -= 1 - if in_degree[succ] == 0: - queue.append(succ) - - # TODO(b/521549179): Support limited loop analysis (eg unroll loops over a constant range). - if len(order) != len(blocks): - raise ValueError( - "Loops are not supported in the Python function for transpilation." - ) - - return order - - -def merge_values( - pairs: list[tuple[expression.Expression, expression.Expression]], -) -> expression.Expression: - if not pairs: - raise ValueError("Cannot merge empty list of values") - if len(pairs) == 1: - return pairs[0][0] - - val = pairs[-1][0] - for next_val, next_cond in reversed(pairs[:-1]): - val = py_exprs.Call( - py_exprs.PyObject(generic_ops.where_op), (next_val, next_cond, val) - ) - return val - - -def _compile_bytecode_to_py_expr(func: Callable) -> expression.Expression: - instructions = list(dis.get_instructions(func)) - next_offsets = { - inst.offset: next_inst.offset - for inst, next_inst in zip(instructions, instructions[1:]) - } - - blocks = build_cfg(instructions, next_offsets) - order = topological_sort(blocks) - - stack: list[expression.Expression] - local_vars: dict[str, expression.Expression] - - globals_dict = func.__globals__ - import builtins - - builtins_dict = builtins.__dict__ - closure_dict = {} - if func.__closure__: - free_vars = func.__code__.co_freevars - for var, cell in zip(free_vars, func.__closure__): - try: - closure_dict[var] = cell.cell_contents - except ValueError: - pass - - block_outputs: dict[ - int, tuple[list[expression.Expression], dict[str, expression.Expression]] - ] = {} - block_reach_conditions: dict[int, expression.Expression] = { - 0: py_exprs.PyObject(True) - } - edge_conditions: dict[tuple[int, int], expression.Expression] = {} - edge_stacks: dict[tuple[int, int], list[expression.Expression]] = {} - returns: list[tuple[expression.Expression, expression.Expression]] = [] - - co = func.__code__ - param_names = list(co.co_varnames[: co.co_argcount]) - kwonly_argcount = co.co_kwonlyargcount - param_names.extend( - co.co_varnames[co.co_argcount : co.co_argcount + kwonly_argcount] - ) - - initial_local_vars: dict[str, expression.Expression] = { - name: expression.UnboundVariableExpression(name) for name in param_names - } - - for offset in order: - block = blocks[offset] - - reach_cond: expression.Expression - if offset == 0: - reach_cond = py_exprs.PyObject(True) - else: - incoming = [ - edge_conditions[(pred, offset)] - for pred in block.predecessors - if (pred, offset) in edge_conditions - ] - if not incoming: - continue - - reach_cond = incoming[0] - for cond in incoming[1:]: - reach_cond = py_exprs.Call( - py_exprs.PyObject(operator.or_), (reach_cond, cond) - ) - - block_reach_conditions[offset] = reach_cond - - if offset == 0: - stack = [] - local_vars = initial_local_vars.copy() - else: - reachable_preds = [ - pred for pred in block.predecessors if (pred, offset) in edge_stacks - ] - if not reachable_preds: - continue - - h = len(edge_stacks[(reachable_preds[0], offset)]) - stack = [] - for i in range(h): - pairs = [ - (edge_stacks[(p, offset)][i], edge_conditions[(p, offset)]) - for p in reachable_preds - ] - stack.append(merge_values(pairs)) - - all_vars: set[str] = set() - for p in reachable_preds: - all_vars.update(block_outputs[p][1].keys()) - - local_vars = {} - for var in all_vars: - pairs = [ - ( - block_outputs[p][1].get( - var, expression.UnboundVariableExpression(var) - ), - edge_conditions[(p, offset)], - ) - for p in reachable_preds - ] - local_vars[var] = merge_values(pairs) - - jumped = False - for inst in block.instructions: - opname = inst.opname - - match opname: - case "RESUME" | "PRECALL" | "COPY_FREE_VARS" | "NOT_TAKEN" | "NOP": - continue - - case "LOAD_FAST_LOAD_FAST" | "LOAD_FAST_BORROW_LOAD_FAST_BORROW": - var1, var2 = inst.argval - stack.append( - local_vars.get(var1, expression.UnboundVariableExpression(var1)) - ) - stack.append( - local_vars.get(var2, expression.UnboundVariableExpression(var2)) - ) - - case ( - "LOAD_FAST" - | "LOAD_FAST_CHECK" - | "LOAD_FAST_AND_CLEAR" - | "LOAD_FAST_BORROW" - ): - stack.append( - local_vars.get( - inst.argval, - expression.UnboundVariableExpression(inst.argval), - ) - ) - - case "STORE_FAST": - if not stack: - raise ValueError("Stack is empty") - local_vars[inst.argval] = stack.pop() - - case "LOAD_CONST" | "LOAD_SMALL_INT": - stack.append(py_exprs.PyObject(inst.argval)) - - case "LOAD_DEREF" | "LOAD_FROM_DICT_OR_DEREF": - name = inst.argval - found = False - val = None - if name in closure_dict: - val = closure_dict[name] - found = True - elif name in globals_dict: - val = globals_dict[name] - found = True - elif name in builtins_dict: - val = builtins_dict[name] - found = True - - if found: - if isinstance(val, ModuleType): - stack.append(py_exprs.Module(val)) - else: - stack.append(py_exprs.PyObject(val)) - else: - stack.append(expression.UnboundVariableExpression(name)) - - case "LOAD_GLOBAL": - if ( - sys.version_info >= (3, 11) - and inst.arg is not None - and (inst.arg & 1) - ): - stack.append(_NULL) - name = inst.argval - found = False - val = None - if name in closure_dict: - val = closure_dict[name] - found = True - elif name in globals_dict: - val = globals_dict[name] - found = True - elif name in builtins_dict: - val = builtins_dict[name] - found = True - - if found: - if isinstance(val, ModuleType): - stack.append(py_exprs.Module(val)) - else: - stack.append(py_exprs.PyObject(val)) - else: - stack.append(expression.UnboundVariableExpression(name)) - - case "LOAD_ATTR" | "LOAD_METHOD": - if not stack: - raise ValueError("Stack is empty") - target = stack.pop() - stack.append(py_exprs.GetAttr(target, inst.argval)) - - is_method_lookup = (opname == "LOAD_METHOD") or ( - opname == "LOAD_ATTR" - and sys.version_info >= (3, 12) - and inst.arg is not None - and (inst.arg & 1) - ) - if is_method_lookup: - if isinstance(target, py_exprs.Module) or ( - isinstance(target, py_exprs.PyObject) - and isinstance(target.value, type) - ): - stack.append(_NULL) - else: - stack.append(target) - - case "PUSH_NULL": - stack.append(_NULL) - - case "TO_BOOL": - if not stack: - raise ValueError("Stack is empty") - val = stack.pop() - stack.append( - py_exprs.Call( - py_exprs.PyObject(generic_ops.coerce_to_bool_op), - (val,), - ) - ) - - case "FORMAT_SIMPLE": - if not stack: - raise ValueError("Stack is empty") - value = stack.pop() - stack.append(py_exprs.Call(py_exprs.PyObject(str), (value,))) - - case "CONVERT_VALUE": - flags = inst.arg - assert flags is not None - value = stack.pop() - if flags == 1: - stack.append(py_exprs.Call(py_exprs.PyObject(str), (value,))) - else: - raise NotImplementedError( - "repr() and ascii() conversions are not supported" - ) - - case "FORMAT_VALUE": - flags = inst.arg - assert flags is not None - if (flags & 0x04) == 0x04: - stack.pop() - raise NotImplementedError( - "Formatting with specifier is not supported" - ) - - value = stack.pop() - conversion = flags & 0x03 - if conversion == 0 or conversion == 1: - stack.append(py_exprs.Call(py_exprs.PyObject(str), (value,))) - else: - raise NotImplementedError( - "repr() and ascii() conversions are not supported" - ) - - case "FORMAT_WITH_SPEC": - raise NotImplementedError( - "Formatting with specifier is not supported" - ) - - case "BUILD_STRING": - count = inst.arg - assert count is not None - if len(stack) < count: - raise ValueError( - "Stack has fewer elements than BUILD_STRING count" - ) - - if count == 0: - stack.append(py_exprs.PyObject("")) - else: - strings = [stack.pop() for _ in range(count)][::-1] - result = strings[0] - for s in strings[1:]: - result = py_exprs.Call( - py_exprs.PyObject(operator.add), - (result, s), - ) - stack.append(result) - - case "COPY": - idx = inst.arg - if idx is None or idx < 1 or len(stack) < idx: - raise ValueError( - f"Invalid COPY index or stack too small: {idx}" - ) - stack.append(stack[-idx]) - - case "UNARY_NOT": - if not stack: - raise ValueError("Stack is empty") - val = stack.pop() - val_bool = py_exprs.Call( - py_exprs.PyObject(generic_ops.coerce_to_bool_op), - (val,), - ) - stack.append( - py_exprs.Call( - py_exprs.PyObject(operator.not_), - (val_bool,), - ) - ) - - case "SWAP": - idx = inst.arg - if idx is None or idx < 1 or len(stack) < idx: - raise ValueError( - f"Invalid SWAP index or stack too small: {idx}" - ) - stack[-1], stack[-idx] = stack[-idx], stack[-1] - - case "ROT_TWO": - if len(stack) < 2: - raise ValueError("Stack has < 2 elements") - stack[-1], stack[-2] = stack[-2], stack[-1] - - case "ROT_THREE": - if len(stack) < 3: - raise ValueError("Stack has < 3 elements") - stack[-1], stack[-2], stack[-3] = stack[-2], stack[-3], stack[-1] - - case "DUP_TOP": - if not stack: - raise ValueError("Stack is empty") - stack.append(stack[-1]) - - case "BINARY_OP": - if len(stack) < 2: - raise ValueError("Stack is empty") - right = stack.pop() - left = stack.pop() - op_symbol = inst.argrepr - if not op_symbol and isinstance(inst.argval, str): - op_symbol = inst.argval - if op_symbol and op_symbol.endswith("="): - op_symbol = op_symbol[:-1] - - if op_symbol not in _BINARY_OP_MAP: - raise ValueError(f"Unsupported binary operator: {op_symbol}") - stack.append( - py_exprs.Call( - py_exprs.PyObject(_BINARY_OP_MAP[op_symbol]), - (left, right), - ) - ) - - case "BINARY_SUBSCR": - if len(stack) < 2: - raise ValueError("Stack has < 2 elements") - key = stack.pop() - container = stack.pop() - stack.append( - py_exprs.Call( - py_exprs.PyObject(operator.getitem), - (container, key), - ) - ) - - case name if name in _OLD_BINARY_OP_MAP: - if len(stack) < 2: - raise ValueError("Stack has < 2 elements") - right = stack.pop() - left = stack.pop() - stack.append( - py_exprs.Call( - py_exprs.PyObject(_OLD_BINARY_OP_MAP[opname]), - (left, right), - ) - ) - - case "IS_OP": - if len(stack) < 2: - raise ValueError("Stack has < 2 elements") - right = stack.pop() - left = stack.pop() - invert = inst.arg - - def is_none_const(expr) -> bool: - if isinstance(expr, py_exprs.PyObject) and expr.value is None: - return True - if ( - isinstance(expr, expression.ScalarConstantExpression) - and expr.value is None - ): - return True - return False - - if is_none_const(right): - op = ( - generic_ops.isnull_op - if not invert - else generic_ops.notnull_op - ) - stack.append(py_exprs.Call(py_exprs.PyObject(op), (left,))) - elif is_none_const(left): - op = ( - generic_ops.isnull_op - if not invert - else generic_ops.notnull_op - ) - stack.append(py_exprs.Call(py_exprs.PyObject(op), (right,))) - else: - raise NotImplementedError( - "Identity comparison (is/is not) is only supported for None" - ) - - case "COMPARE_OP": - if len(stack) < 2: - raise ValueError("Stack has < 2 elements") - right = stack.pop() - left = stack.pop() - op_symbol = inst.argval - if op_symbol not in _COMPARE_OP_MAP: - raise ValueError(f"Unsupported compare operator: {op_symbol}") - stack.append( - py_exprs.Call( - py_exprs.PyObject(_COMPARE_OP_MAP[op_symbol]), - (left, right), - ) - ) - - case "UNARY_NEGATIVE" | "UNARY_INVERT": - if not stack: - raise ValueError("Stack is empty") - target = stack.pop() - stack.append( - py_exprs.Call( - py_exprs.PyObject( - operator.neg - if opname == "UNARY_NEGATIVE" - else operator.invert - ), - (target,), - ) - ) - - case "UNARY_POSITIVE": - if not stack: - raise ValueError("Stack is empty") - target = stack.pop() - stack.append( - py_exprs.Call(py_exprs.PyObject(operator.pos), (target,)) - ) - - case "CALL_INTRINSIC_1": - if inst.argrepr == "INTRINSIC_UNARY_POSITIVE": - if not stack: - raise ValueError("Stack is empty") - target = stack.pop() - stack.append( - py_exprs.Call(py_exprs.PyObject(operator.pos), (target,)) - ) - else: - raise ValueError(f"Unsupported intrinsic: {inst.argrepr}") - - case "CALL" | "CALL_FUNCTION" | "CALL_METHOD": - num_args = inst.arg - assert num_args is not None - if len(stack) < num_args: - raise ValueError(f"Stack has fewer than {num_args} elements") - args = [stack.pop() for _ in range(num_args)][::-1] - - is_method_call = False - if opname == "CALL" or opname == "CALL_METHOD": - if len(stack) >= 2 and stack[-2] == _NULL: - stack[-1], stack[-2] = stack[-2], stack[-1] - if stack and stack[-1] == _NULL: - stack.pop() - is_method_call = False - else: - is_method_call = True - elif opname == "CALL_FUNCTION": - is_method_call = False - - if is_method_call: - if ( - stack - and stack[-1] != _NULL - and isinstance(stack[-1], expression.Expression) - ): - self_arg = stack.pop() - args = [self_arg] + args - - if not stack: - raise ValueError("Stack is empty") - callable_expr = stack.pop() - stack.append(py_exprs.Call(callable_expr, tuple(args))) - - case "RETURN_VALUE": - if not stack: - raise ValueError("Stack is empty") - returns.append((stack[-1], reach_cond)) - jumped = True - break - - case "RETURN_CONST": - returns.append((py_exprs.PyObject(inst.argval), reach_cond)) - jumped = True - break - - case "POP_TOP": - if stack: - stack.pop() - - case name if name in _UNCONDITIONAL_JUMP_OPNAMES: - dest = inst.argval - edge_conditions[(offset, dest)] = reach_cond - edge_stacks[(offset, dest)] = stack.copy() - jumped = True - break - - case "JUMP_IF_FALSE_OR_POP" | "JUMP_IF_TRUE_OR_POP": - if not stack: - raise ValueError("Stack is empty") - cond_expr = stack[-1] - cond_bool = py_exprs.Call( - py_exprs.PyObject(generic_ops.coerce_to_bool_op), - (cond_expr,), - ) - dest = inst.argval - next_offset = next_offsets.get(inst.offset) - if opname == "JUMP_IF_FALSE_OR_POP": - not_cond_bool = py_exprs.Call( - py_exprs.PyObject(operator.not_), (cond_bool,) - ) - edge_conditions[(offset, dest)] = py_exprs.Call( - py_exprs.PyObject(operator.and_), - (reach_cond, not_cond_bool), - ) - edge_stacks[(offset, dest)] = stack.copy() - if next_offset is not None: - edge_conditions[(offset, next_offset)] = py_exprs.Call( - py_exprs.PyObject(operator.and_), - (reach_cond, cond_bool), - ) - edge_stacks[(offset, next_offset)] = stack[:-1] - else: # JUMP_IF_TRUE_OR_POP - edge_conditions[(offset, dest)] = py_exprs.Call( - py_exprs.PyObject(operator.and_), - (reach_cond, cond_bool), - ) - edge_stacks[(offset, dest)] = stack.copy() - if next_offset is not None: - not_cond_bool = py_exprs.Call( - py_exprs.PyObject(operator.not_), (cond_bool,) - ) - edge_conditions[(offset, next_offset)] = py_exprs.Call( - py_exprs.PyObject(operator.and_), - (reach_cond, not_cond_bool), - ) - edge_stacks[(offset, next_offset)] = stack[:-1] - jumped = True - break - - case name if ( - name in _JUMP_IF_FALSE_OPNAMES or name in _JUMP_IF_TRUE_OPNAMES - ): - if not stack: - raise ValueError("Stack is empty") - cond_expr = stack.pop() - cond_expr = py_exprs.Call( - py_exprs.PyObject(generic_ops.coerce_to_bool_op), - (cond_expr,), - ) - - dest = inst.argval - next_offset = next_offsets.get(inst.offset) - - if opname in _JUMP_IF_FALSE_OPNAMES: - not_cond_expr = py_exprs.Call( - py_exprs.PyObject(operator.not_), (cond_expr,) - ) - edge_conditions[(offset, dest)] = py_exprs.Call( - py_exprs.PyObject(operator.and_), - (reach_cond, not_cond_expr), - ) - edge_stacks[(offset, dest)] = stack.copy() - if next_offset is not None: - edge_conditions[(offset, next_offset)] = py_exprs.Call( - py_exprs.PyObject(operator.and_), - (reach_cond, cond_expr), - ) - edge_stacks[(offset, next_offset)] = stack.copy() - else: # opname in _JUMP_IF_TRUE_OPNAMES - not_cond_expr = py_exprs.Call( - py_exprs.PyObject(operator.not_), (cond_expr,) - ) - edge_conditions[(offset, dest)] = py_exprs.Call( - py_exprs.PyObject(operator.and_), - (reach_cond, cond_expr), - ) - edge_stacks[(offset, dest)] = stack.copy() - if next_offset is not None: - edge_conditions[(offset, next_offset)] = py_exprs.Call( - py_exprs.PyObject(operator.and_), - (reach_cond, not_cond_expr), - ) - edge_stacks[(offset, next_offset)] = stack.copy() - jumped = True - break - - case name if ( - name in _JUMP_IF_NONE_OPNAMES or name in _JUMP_IF_NOT_NONE_OPNAMES - ): - if not stack: - raise ValueError("Stack is empty") - cond_expr = stack.pop() - cond_bool = py_exprs.Call( - py_exprs.PyObject(generic_ops.isnull_op), - (cond_expr,), - ) - - dest = inst.argval - next_offset = next_offsets.get(inst.offset) - - if opname in _JUMP_IF_NONE_OPNAMES: - not_cond_bool = py_exprs.Call( - py_exprs.PyObject(operator.not_), (cond_bool,) - ) - edge_conditions[(offset, dest)] = py_exprs.Call( - py_exprs.PyObject(operator.and_), - (reach_cond, cond_bool), - ) - edge_stacks[(offset, dest)] = stack.copy() - if next_offset is not None: - edge_conditions[(offset, next_offset)] = py_exprs.Call( - py_exprs.PyObject(operator.and_), - (reach_cond, not_cond_bool), - ) - edge_stacks[(offset, next_offset)] = stack.copy() - else: # opname in _JUMP_IF_NOT_NONE_OPNAMES - not_cond_bool = py_exprs.Call( - py_exprs.PyObject(operator.not_), (cond_bool,) - ) - edge_conditions[(offset, dest)] = py_exprs.Call( - py_exprs.PyObject(operator.and_), - (reach_cond, not_cond_bool), - ) - edge_stacks[(offset, dest)] = stack.copy() - if next_offset is not None: - edge_conditions[(offset, next_offset)] = py_exprs.Call( - py_exprs.PyObject(operator.and_), - (reach_cond, cond_bool), - ) - edge_stacks[(offset, next_offset)] = stack.copy() - jumped = True - break - - case name if name in _ALL_JUMP_OPNAMES: - raise ValueError(f"Unsupported jump opcode: {opname}") - - case _: - raise ValueError(f"Unsupported opcode: {opname}") - - if not jumped: - next_offset = next_offsets.get(block.instructions[-1].offset) - if next_offset is not None: - edge_conditions[(offset, next_offset)] = reach_cond - edge_stacks[(offset, next_offset)] = stack.copy() - - block_outputs[offset] = (stack, local_vars) - - if not returns: - raise ValueError("No return value found") - - return merge_values(returns) - - -def py_to_expression(func: Callable) -> expression.Expression: - """ - Try to convert a python function to a BigQuery expression. - - This is "best effort" - if the function contains operations that cannot - be converted to BigQuery expressions, it will raise an Exception. - """ - py_expr = _compile_bytecode_to_py_expr(func) - return py_exprs.resolve_py_exprs(py_expr) diff --git a/bigframes/core/col.py b/bigframes/core/col.py deleted file mode 100644 index 50968dfbf94..00000000000 --- a/bigframes/core/col.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import annotations - -import dataclasses -from typing import TYPE_CHECKING, Any, Hashable, Literal - -import bigframes_vendored.pandas.core.col as pd_col -import numpy - -import bigframes.core.expression as bf_expression -import bigframes.operations as bf_ops -import bigframes.operations.aggregations as agg_ops -from bigframes.core import agg_expressions, window_spec - -if TYPE_CHECKING: - import bigframes.operations.datetimes as datetimes - import bigframes.operations.strings as strings - - -# Not to be confused with the Expression class in `bigframes.core.expressions` -# Name collision unintended -@dataclasses.dataclass(frozen=True) -class Expression: - __doc__ = pd_col.Expression.__doc__ - - _value: bf_expression.Expression - - def _apply_unary_op(self, op: bf_ops.UnaryOp) -> Expression: - return Expression(op.as_expr(self._value)) - - def _apply_unary_agg(self, op: agg_ops.UnaryAggregateOp) -> Expression: - # We probably shouldn't need to windowize here, but block apis expect pre-windowized expressions - # Later on, we will probably have col expressions in windowed context, so will need to defer windowization - # instead of automatically applying the default unbound window - agg_expr = op.as_expr(self._value) - return Expression( - agg_expressions.WindowExpression(agg_expr, window_spec.unbound()) - ) - - # alignment is purely for series compatibility, and is ignored here - def _apply_binary_op( - self, - other: Any, - op: bf_ops.BinaryOp, - alignment: Literal["outer", "left"] = "outer", - reverse: bool = False, - ): - if reverse: - return Expression(op.as_expr(_as_bf_expr(other), self._value)) - else: - return Expression(op.as_expr(self._value, _as_bf_expr(other))) - - def __add__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.add_op) - - def __radd__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.add_op, reverse=True) - - def __sub__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.sub_op) - - def __rsub__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.sub_op, reverse=True) - - def __mul__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.mul_op) - - def __rmul__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.mul_op, reverse=True) - - def __truediv__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.div_op) - - def __rtruediv__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.div_op, reverse=True) - - def __floordiv__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.floordiv_op) - - def __rfloordiv__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.floordiv_op, reverse=True) - - def __ge__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.ge_op) - - def __gt__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.gt_op) - - def __le__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.le_op) - - def __lt__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.lt_op) - - def __eq__(self, other: object) -> Expression: # type: ignore - return self._apply_binary_op(other, bf_ops.eq_op) - - def __ne__(self, other: object) -> Expression: # type: ignore - return self._apply_binary_op(other, bf_ops.ne_op) - - def __mod__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.mod_op) - - def __rmod__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.mod_op, reverse=True) - - def __and__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.and_op) - - def __rand__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.and_op, reverse=True) - - def __or__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.or_op) - - def __ror__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.or_op, reverse=True) - - def __xor__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.xor_op) - - def __rxor__(self, other: Any) -> Expression: - return self._apply_binary_op(other, bf_ops.xor_op, reverse=True) - - def __invert__(self) -> Expression: - return self._apply_unary_op(bf_ops.invert_op) - - def sum(self) -> Expression: - return self._apply_unary_agg(agg_ops.sum_op) - - def mean(self) -> Expression: - return self._apply_unary_agg(agg_ops.mean_op) - - def var(self) -> Expression: - return self._apply_unary_agg(agg_ops.var_op) - - def std(self) -> Expression: - return self._apply_unary_agg(agg_ops.std_op) - - def min(self) -> Expression: - return self._apply_unary_agg(agg_ops.min_op) - - def max(self) -> Expression: - return self._apply_unary_agg(agg_ops.max_op) - - @property - def dt(self) -> datetimes.DatetimeSimpleMethods: - import bigframes.operations.datetimes as datetimes - - return datetimes.DatetimeSimpleMethods(self) - - def __array_ufunc__( - self, ufunc: numpy.ufunc, method: str, *inputs, **kwargs - ) -> Expression: - """Used to support numpy ufuncs. - See: https://numpy.org/doc/stable/reference/ufuncs.html - """ - # Only __call__ supported with zero arguments - if method != "__call__" or len(inputs) > 2 or len(kwargs) > 0: - return NotImplemented - - if len(inputs) == 1 and ufunc in bf_ops.NUMPY_TO_OP: - op = bf_ops.NUMPY_TO_OP[ufunc] - return Expression(op.as_expr(self._value)) - if len(inputs) == 2 and ufunc in bf_ops.NUMPY_TO_BINOP: - binop = bf_ops.NUMPY_TO_BINOP[ufunc] - if inputs[0] is self: - return Expression(binop.as_expr(self._value, _as_bf_expr(inputs[1]))) - else: - return Expression(binop.as_expr(_as_bf_expr(inputs[0]), self._value)) - - return NotImplemented - - # keep this last as str declaration can shadow builtins.str - @property - def str(self) -> strings.StringMethods: - import bigframes.operations.strings as strings - - return strings.StringMethods(self) - - -def _as_bf_expr(arg: Any) -> bf_expression.Expression: - if isinstance(arg, Expression): - return arg._value - return bf_expression.const(arg) - - -def col(col_name: Hashable) -> Expression: - return Expression(bf_expression.free_var(col_name)) - - -col.__doc__ = pd_col.col.__doc__ diff --git a/bigframes/core/compile/__init__.py b/bigframes/core/compile/__init__.py index c1b9c5d9022..68c36df2889 100644 --- a/bigframes/core/compile/__init__.py +++ b/bigframes/core/compile/__init__.py @@ -13,30 +13,13 @@ # limitations under the License. from __future__ import annotations -from typing import Literal - from bigframes.core.compile.api import test_only_ibis_inferred_schema from bigframes.core.compile.configs import CompileRequest, CompileResult - - -def compile_sql( - request: CompileRequest, - compiler_name: Literal["sqlglot", "ibis"] = "sqlglot", -) -> CompileResult: - """Compiles a BigFrameNode according to the request into SQL.""" - if compiler_name == "sqlglot": - import bigframes.core.compile.sqlglot.compiler as sqlglot_compiler - - return sqlglot_compiler.compile_sql(request) - else: - import bigframes.core.compile.ibis_compiler.ibis_compiler as ibis_compiler - - return ibis_compiler.compile_sql(request) - +from bigframes.core.compile.ibis_compiler.ibis_compiler import compile_sql __all__ = [ "test_only_ibis_inferred_schema", + "compile_sql", "CompileRequest", "CompileResult", - "compile_sql", ] diff --git a/bigframes/core/compile/api.py b/bigframes/core/compile/api.py index 82672fc95b5..dde6f3a3258 100644 --- a/bigframes/core/compile/api.py +++ b/bigframes/core/compile/api.py @@ -21,9 +21,9 @@ def test_only_ibis_inferred_schema(node: bigframes.core.nodes.BigFrameNode): """Use only for testing paths to ensure ibis inferred schema does not diverge from bigframes inferred schema.""" + from bigframes.core.compile.ibis_compiler import ibis_compiler import bigframes.core.rewrite import bigframes.core.schema - from bigframes.core.compile.ibis_compiler import ibis_compiler node = ibis_compiler._replace_unsupported_ops(node) node = bigframes.core.rewrite.bake_order(node) diff --git a/bigframes/core/compile/compiled.py b/bigframes/core/compile/compiled.py index fea94f6e6ed..f8be331d59b 100644 --- a/bigframes/core/compile/compiled.py +++ b/bigframes/core/compile/compiled.py @@ -13,6 +13,7 @@ # limitations under the License. from __future__ import annotations +import functools import itertools import typing from typing import Literal, Optional, Sequence @@ -23,20 +24,22 @@ import bigframes_vendored.ibis.expr.datatypes as ibis_dtypes import bigframes_vendored.ibis.expr.operations as ibis_ops import bigframes_vendored.ibis.expr.types as ibis_types -import bigframes_vendored.sqlglot.expressions as sge -import pyarrow as pa from google.cloud import bigquery +import pyarrow as pa +from bigframes.core import agg_expressions import bigframes.core.agg_expressions as ex_types +import bigframes.core.compile.googlesql import bigframes.core.compile.ibis_compiler.aggregate_compiler as agg_compiler import bigframes.core.compile.ibis_compiler.scalar_op_compiler as op_compilers import bigframes.core.compile.ibis_types import bigframes.core.expression as ex -import bigframes.core.sql -import bigframes.dtypes -from bigframes.core import agg_expressions, rewrite from bigframes.core.ordering import OrderingExpression +import bigframes.core.sql from bigframes.core.window_spec import WindowSpec +import bigframes.dtypes +import bigframes.operations as ops +import bigframes.operations.aggregations as agg_ops op_compiler = op_compilers.scalar_op_compiler @@ -56,8 +59,7 @@ def __init__( column.resolve(table) # type:ignore # TODO(https://github.com/ibis-project/ibis/issues/7613): use # public API to refer to Deferred type. - if isinstance(column, ibis_deferred.Deferred) - else column + if isinstance(column, ibis_deferred.Deferred) else column for column in columns ) # To allow for more efficient lookup by column name, create a @@ -83,21 +85,13 @@ def to_sql( ) if order_by or limit or not is_noop_selection: - # selections are (ref.id.sql, name) where ref.id.sql is escaped identifier - to_select = [ - sge.Alias( - this=sge.to_identifier(src, quoted=True), - alias=sge.to_identifier(alias, quoted=True), - ) - if src != alias - else sge.to_identifier(src, quoted=True) - for src, alias in selection_strings - ] - # Use string formatting for FROM clause to avoid re-parsing potentially complex SQL (like ARRAY>) - # that sqlglot might not handle perfectly when parsing BigQuery dialect strings. - select_sql = sge.Select().select(*to_select).sql(dialect="bigquery") - ibis_sql = ibis_bigquery.Backend().compile(ibis_table) - sql = f"{select_sql} FROM ({ibis_sql}) AS `t`" + sql = ibis_bigquery.Backend().compile(ibis_table) + sql = ( + bigframes.core.compile.googlesql.Select() + .from_(sql) + .select(selection_strings) + .sql() + ) # Single row frames may not have any ordering columns if len(order_by) > 0: @@ -108,7 +102,7 @@ def to_sql( raise TypeError(f"Limit param: {limit} must be an int.") sql += f"\nLIMIT {limit}" else: - sql = ibis_bigquery.Backend().compile(ibis_table) + sql = ibis_bigquery.Backend().compile(self._to_ibis_expr()) return typing.cast(str, sql) @property @@ -381,7 +375,6 @@ def isin_join( new_column = ( (left_table[conditions[0]]) .isin((right_table[conditions[1]])) - .fillna(False) .name(indicator_col) ) @@ -431,11 +424,59 @@ def project_window_op( output_name, ) - rewritten_expr = rewrite.simplify_complex_windows( - agg_expressions.WindowExpression(expression, window_spec) + if expression.op.order_independent and window_spec.is_unbounded: + # notably percentile_cont does not support ordering clause + window_spec = window_spec.without_order() + + # TODO: Turn this logic into a true rewriter + result_expr: ex.Expression = agg_expressions.WindowExpression( + expression, window_spec ) + clauses: list[tuple[ex.Expression, ex.Expression]] = [] + if window_spec.min_periods and len(expression.inputs) > 0: + if not expression.op.nulls_count_for_min_values: + is_observation = ops.notnull_op.as_expr() + + # Most operations do not count NULL values towards min_periods + per_col_does_count = ( + ops.notnull_op.as_expr(input) for input in expression.inputs + ) + # All inputs must be non-null for observation to count + is_observation = functools.reduce( + lambda x, y: ops.and_op.as_expr(x, y), per_col_does_count + ) + observation_sentinel = ops.AsTypeOp(bigframes.dtypes.INT_DTYPE).as_expr( + is_observation + ) + observation_count_expr = agg_expressions.WindowExpression( + ex_types.UnaryAggregation(agg_ops.sum_op, observation_sentinel), + window_spec, + ) + else: + # Operations like count treat even NULLs as valid observations for the sake of min_periods + # notnull is just used to convert null values to non-null (FALSE) values to be counted + is_observation = ops.notnull_op.as_expr(expression.inputs[0]) + observation_count_expr = agg_expressions.WindowExpression( + agg_ops.count_op.as_expr(is_observation), + window_spec, + ) + clauses.append( + ( + ops.lt_op.as_expr( + observation_count_expr, ex.const(window_spec.min_periods) + ), + ex.const(None), + ) + ) + if clauses: + case_inputs = [ + *itertools.chain.from_iterable(clauses), + ex.const(True), + result_expr, + ] + result_expr = ops.CaseWhenOp().as_expr(*case_inputs) - ibis_expr = op_compiler.compile_expression(rewritten_expr, self._ibis_bindings) + ibis_expr = op_compiler.compile_expression(result_expr, self._ibis_bindings) return UnorderedIR(self._table, (*self.columns, ibis_expr.name(output_name))) diff --git a/bigframes/core/compile/configs.py b/bigframes/core/compile/configs.py index 62c28f87cae..5ffca0cf43b 100644 --- a/bigframes/core/compile/configs.py +++ b/bigframes/core/compile/configs.py @@ -34,4 +34,3 @@ class CompileResult: sql: str sql_schema: typing.Sequence[google.cloud.bigquery.SchemaField] row_order: typing.Optional[ordering.RowOrdering] - encoded_type_refs: str diff --git a/bigframes/core/compile/googlesql/__init__.py b/bigframes/core/compile/googlesql/__init__.py new file mode 100644 index 00000000000..add0c5ec445 --- /dev/null +++ b/bigframes/core/compile/googlesql/__init__.py @@ -0,0 +1,61 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Python classes representing GoogleSQL syntax nodes, adhering to the official syntax: +https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax""" + +from __future__ import annotations + +from bigframes.core.compile.googlesql.datatype import DataType +from bigframes.core.compile.googlesql.expression import ( + _escape_chars, + AliasExpression, + ColumnExpression, + CTEExpression, + identifier, + StarExpression, + TableExpression, +) +from bigframes.core.compile.googlesql.function import Cast +from bigframes.core.compile.googlesql.query import ( + AsAlias, + FromClause, + FromItem, + NonRecursiveCTE, + QueryExpr, + Select, + SelectAll, + SelectExpression, +) + +__all__ = [ + "_escape_chars", + "identifier", + "AliasExpression", + "AsAlias", + "Cast", + "ColumnExpression", + "CTEExpression", + "DataType", + "FromClause", + "FromItem", + "NonRecursiveCTE", + "QueryExpr", + "Select", + "SelectAll", + "SelectExpression", + "StarExpression", + "StringType", + "TableExpression", +] diff --git a/bigframes/bigquery/aead.py b/bigframes/core/compile/googlesql/abc.py similarity index 72% rename from bigframes/bigquery/aead.py rename to bigframes/core/compile/googlesql/abc.py index c4243a5c010..081836467c2 100644 --- a/bigframes/bigquery/aead.py +++ b/bigframes/core/compile/googlesql/abc.py @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""AEAD encryption functions""" - from __future__ import annotations -from bigframes.operations.googlesql.aead import decrypt_bytes, decrypt_string, encrypt +import abc + + +class SQLSyntax(abc.ABC): + """Abstract base class provides GoogleSQL syntax.""" -__all__ = [ - "decrypt_bytes", - "decrypt_string", - "encrypt", -] + @abc.abstractmethod + def sql(self): + ... diff --git a/bigframes/extensions/core/__init__.py b/bigframes/core/compile/googlesql/datatype.py similarity index 69% rename from bigframes/extensions/core/__init__.py rename to bigframes/core/compile/googlesql/datatype.py index 41b554c99ef..ccf3ff4d41b 100644 --- a/bigframes/extensions/core/__init__.py +++ b/bigframes/core/compile/googlesql/datatype.py @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from bigframes.extensions.core.dataframe_accessor import ( - AIAccessor, - BigQueryDataFrameAccessor, -) +import enum -__all__ = ["AIAccessor", "BigQueryDataFrameAccessor"] +"""This module represents all GoogleSQL for BigQuery data types: +https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types""" + + +class DataType(enum.Enum): + STRING = 1 + FLOAT64 = 2 diff --git a/bigframes/core/compile/googlesql/expression.py b/bigframes/core/compile/googlesql/expression.py new file mode 100644 index 00000000000..581ab67718a --- /dev/null +++ b/bigframes/core/compile/googlesql/expression.py @@ -0,0 +1,124 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import dataclasses +import typing + +import bigframes.core.compile.googlesql.abc as abc + +"""This module represents GoogleSQL `expression` and its extensions. +Core class: + +* `expression`: Models basic SQL expressions. + +Extended classes (not part of standard GoogleSQL syntax, but added for convenience): + +* `ColumnExpression`: Represents column references. +* `TableExpression`: Represents table references. +* `AliasExpression`: Represents aliased expressions. +* ... +""" + + +@dataclasses.dataclass +class Expression(abc.SQLSyntax): + pass + + +@dataclasses.dataclass +class ColumnExpression(Expression): + name: str + parent: typing.Optional[TableExpression | AliasExpression | CTEExpression] = None + + def sql(self) -> str: + if self.parent is not None: + return f"{self.parent.sql()}.{identifier(self.name)}" + return identifier(self.name) + + +@dataclasses.dataclass +class StarExpression(Expression): + parent: typing.Optional[TableExpression | AliasExpression | CTEExpression] = None + + def sql(self) -> str: + if self.parent is not None: + return f"{self.parent.sql()}.*" + return "*" + + +@dataclasses.dataclass +class TableExpression(Expression): + table_id: str + dataset_id: typing.Optional[str] = None + project_id: typing.Optional[str] = None + + def __post_init__(self): + if self.project_id is not None and self.dataset_id is None: + raise ValueError("The `dataset_id` is missing.") + + def sql(self) -> str: + text = [] + if self.project_id is not None: + text.append(identifier(self.project_id)) + if self.dataset_id is not None: + text.append(identifier(self.dataset_id)) + text.append(identifier(self.table_id)) + return ".".join(text) + + +@dataclasses.dataclass +class AliasExpression(Expression): + alias: str + + def sql(self) -> str: + return identifier(self.alias) + + +@dataclasses.dataclass +class CTEExpression(Expression): + name: str + + def sql(self) -> str: + return identifier(self.name) + + +def identifier(id: str) -> str: + """Return a string representing column reference in a SQL.""" + # https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#identifiers + # Just always escape, otherwise need to check against every reserved sql keyword + return f"`{_escape_chars(id)}`" + + +def _escape_chars(value: str): + """Escapes all special charactesrs""" + # https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#string_and_bytes_literals + trans_table = str.maketrans( + { + "\a": r"\a", + "\b": r"\b", + "\f": r"\f", + "\n": r"\n", + "\r": r"\r", + "\t": r"\t", + "\v": r"\v", + "\\": r"\\", + "?": r"\?", + '"': r"\"", + "'": r"\'", + "`": r"\`", + } + ) + return value.translate(trans_table) diff --git a/bigframes/core/sentinels.py b/bigframes/core/compile/googlesql/function.py similarity index 50% rename from bigframes/core/sentinels.py rename to bigframes/core/compile/googlesql/function.py index ff9913f7c6f..19b61f2fc99 100644 --- a/bigframes/core/sentinels.py +++ b/bigframes/core/compile/googlesql/function.py @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,20 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Sentinel values used throughout BigFrames.""" +import dataclasses -from __future__ import annotations +import bigframes.core.compile.googlesql.datatype as datatype +import bigframes.core.compile.googlesql.expression as expr -import enum +# Conversion functions: +# https://cloud.google.com/bigquery/docs/reference/standard-sql/conversion_functions -class Sentinel(enum.Enum): - """Default values used throughout BigFrames.""" +@dataclasses.dataclass +class Cast(expr.Expression): + """This class represents the `cast` function.""" - """Default value for an optional argument. + expression: expr.ColumnExpression + type: datatype.DataType - When a parameter is set to this, that parameter is explicitly omitted - from the SQL text. This allows for NULL (None in Python) to be explicitly - passed in to optional parameters. - """ - ARGUMENT_DEFAULT = enum.auto() + def sql(self) -> str: + return f"CAST ({self.expression.sql()} AS {self.type.name})" diff --git a/bigframes/core/compile/googlesql/query.py b/bigframes/core/compile/googlesql/query.py new file mode 100644 index 00000000000..f591216b3a8 --- /dev/null +++ b/bigframes/core/compile/googlesql/query.py @@ -0,0 +1,231 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import dataclasses +import typing + +import google.cloud.bigquery as bigquery + +import bigframes.core.compile.googlesql.abc as abc +import bigframes.core.compile.googlesql.expression as expr + +"""This module provides a structured representation of GoogleSQL syntax using nodes. +Each node's name and child nodes are designed to strictly follow the official GoogleSQL +syntax rules outlined in the documentation: +https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax""" + +TABLE_SOURCE_TYPE = typing.Union[str, bigquery.TableReference] + + +@dataclasses.dataclass +class QueryExpr(abc.SQLSyntax): + """This class represents GoogleSQL `query_expr` syntax.""" + + select: Select + with_cte_list: typing.Sequence[NonRecursiveCTE] = () + + def sql(self) -> str: + text = [] + if len(self.with_cte_list) > 0: + with_cte_text = ",\n".join( + [with_cte.sql() for with_cte in self.with_cte_list] + ) + text.append(f"WITH {with_cte_text}") + + text.append(self.select.sql()) + return "\n".join(text) + + +@dataclasses.dataclass +class Select(abc.SQLSyntax): + """This class represents GoogleSQL `select` syntax.""" + + select_list: typing.Sequence[ + typing.Union[SelectExpression, SelectAll] + ] = dataclasses.field(default_factory=list) + from_clause_list: typing.Sequence[FromClause] = dataclasses.field( + default_factory=list + ) + distinct: bool = False + + def select( + self, + columns: typing.Union[ + typing.Iterable[str], typing.Iterable[tuple[str, str]], str, None + ] = None, + distinct: bool = False, + ) -> Select: + if isinstance(columns, str): + columns = [columns] + self.select_list: typing.List[typing.Union[SelectExpression, SelectAll]] = ( + [self._select_field(column) for column in columns] + if columns + else [SelectAll(expression=expr.StarExpression())] + ) + self.distinct = distinct + return self + + def _select_field(self, field) -> SelectExpression: + if isinstance(field, str): + return SelectExpression(expression=expr.ColumnExpression(name=field)) + + else: + alias = ( + expr.AliasExpression(field[1]) + if isinstance(field[1], str) + else field[1] + if (field[0] != field[1]) + else None + ) + return SelectExpression( + expression=expr.ColumnExpression(name=field[0]), alias=alias + ) + + def from_( + self, + sources: typing.Union[TABLE_SOURCE_TYPE, typing.Iterable[TABLE_SOURCE_TYPE]], + ) -> Select: + if (not isinstance(sources, typing.Iterable)) or isinstance(sources, str): + sources = [sources] + self.from_clause_list = [ + FromClause(FromItem.from_source(source)) for source in sources + ] + return self + + def sql(self) -> str: + if (self.select_list is not None) and (not self.select_list): + raise ValueError("Select clause has not been properly initialized.") + + text = ["SELECT"] + + if self.distinct: + text.append("DISTINCT") + + select_list_sql = ",\n".join([select.sql() for select in self.select_list]) + text.append(select_list_sql) + + if self.from_clause_list: + from_clauses_sql = ",\n".join( + [clause.sql() for clause in self.from_clause_list] + ) + text.append(f"FROM\n{from_clauses_sql}") + return "\n".join(text) + + +@dataclasses.dataclass(frozen=True) +class SelectExpression(abc.SQLSyntax): + """This class represents `select_expression`.""" + + expression: expr.ColumnExpression + alias: typing.Optional[expr.AliasExpression] = None + + def sql(self) -> str: + if self.alias is None: + return self.expression.sql() + else: + return f"{self.expression.sql()} AS {self.alias.sql()}" + + +@dataclasses.dataclass +class SelectAll(abc.SQLSyntax): + """This class represents `select_all` (aka. `SELECT *`).""" + + expression: expr.StarExpression + + def sql(self) -> str: + return self.expression.sql() + + +@dataclasses.dataclass +class FromClause(abc.SQLSyntax): + """This class represents GoogleSQL `from_clause` syntax.""" + + from_item: FromItem + + def sql(self) -> str: + return self.from_item.sql() + + +@dataclasses.dataclass +class FromItem(abc.SQLSyntax): + """This class represents GoogleSQL `from_item` syntax.""" + + # Note: Temporarily introduces the `str` type to interact with pre-existing, + # compiled SQL strings. + expression: typing.Union[expr.TableExpression, QueryExpr, str, expr.CTEExpression] + as_alias: typing.Optional[AsAlias] = None + + @classmethod + def from_source( + cls, + subquery_or_tableref: typing.Union[bigquery.TableReference, str], + as_alias: typing.Optional[AsAlias] = None, + ): + if isinstance(subquery_or_tableref, bigquery.TableReference): + return cls( + expression=expr.TableExpression( + table_id=subquery_or_tableref.table_id, + dataset_id=subquery_or_tableref.dataset_id, + project_id=subquery_or_tableref.project, + ), + as_alias=as_alias, + ) + elif isinstance(subquery_or_tableref, str): + return cls( + expression=subquery_or_tableref, + as_alias=as_alias, + ) + else: + raise ValueError("The source must be bigquery.TableReference or str.") + + def sql(self) -> str: + if isinstance(self.expression, (expr.TableExpression, expr.CTEExpression)): + text = self.expression.sql() + elif isinstance(self.expression, str): + text = f"({self.expression})" + elif isinstance(self.expression, QueryExpr): + text = f"({self.expression.sql()})" + else: + raise ValueError( + f"Unsupported expression type {type(self.expression).__name__};" + "expected one of TableExpression, QueryExpr, str, or CTEExpression." + ) + + if self.as_alias is None: + return text + else: + return f"{text} {self.as_alias.sql()}" + + +@dataclasses.dataclass +class NonRecursiveCTE(abc.SQLSyntax): + """This class represents GoogleSQL `non_recursive_cte` syntax.""" + + cte_name: expr.CTEExpression + query_expr: QueryExpr + + def sql(self) -> str: + return f"{self.cte_name.sql()} AS (\n{self.query_expr.sql()}\n)" + + +@dataclasses.dataclass +class AsAlias(abc.SQLSyntax): + """This class represents GoogleSQL `as_alias` syntax.""" + + alias: expr.AliasExpression + + def sql(self) -> str: + return f"AS {self.alias.sql()}" diff --git a/bigframes/core/compile/ibis_compiler/aggregate_compiler.py b/bigframes/core/compile/ibis_compiler/aggregate_compiler.py index 94607bf04bc..0106b150e22 100644 --- a/bigframes/core/compile/ibis_compiler/aggregate_compiler.py +++ b/bigframes/core/compile/ibis_compiler/aggregate_compiler.py @@ -16,27 +16,27 @@ import functools import typing -from typing import List, Optional, cast +from typing import cast, List, Optional import bigframes_vendored.constants as constants import bigframes_vendored.ibis +from bigframes_vendored.ibis.expr import builders as ibis_expr_builders import bigframes_vendored.ibis.expr.api as ibis_api import bigframes_vendored.ibis.expr.datatypes as ibis_dtypes +from bigframes_vendored.ibis.expr.operations import window as ibis_expr_window import bigframes_vendored.ibis.expr.operations as ibis_ops import bigframes_vendored.ibis.expr.operations.udf as ibis_udf import bigframes_vendored.ibis.expr.types as ibis_types import pandas as pd -from bigframes_vendored.ibis.expr import builders as ibis_expr_builders -from bigframes_vendored.ibis.expr.operations import window as ibis_expr_window +from bigframes.core import agg_expressions +from bigframes.core.compile import constants as compiler_constants import bigframes.core.compile.ibis_compiler.scalar_op_compiler as scalar_compilers import bigframes.core.compile.ibis_types as compile_ibis_types import bigframes.core.utils +from bigframes.core.window_spec import RangeWindowBounds, RowsWindowBounds, WindowSpec import bigframes.core.window_spec as window_spec import bigframes.operations.aggregations as agg_ops -from bigframes.core import agg_expressions -from bigframes.core.compile import constants as compiler_constants -from bigframes.core.window_spec import RangeWindowBounds, RowsWindowBounds, WindowSpec scalar_compiler = scalar_compilers.scalar_op_compiler @@ -528,10 +528,8 @@ def _( column: ibis_types.Column, window=None, ) -> ibis_types.Value: - # Ibis FirstNonNullValue expects Value[Any, Columnar], Mypy struggles to see Column as compatible. return _apply_window_if_present( - ibis_ops.FirstNonNullValue(column).to_expr(), # type: ignore[arg-type] - window, # type: ignore + ibis_ops.FirstNonNullValue(column).to_expr(), window # type: ignore ) @@ -550,10 +548,8 @@ def _( column: ibis_types.Column, window=None, ) -> ibis_types.Value: - # Ibis LastNonNullValue expects Value[Any, Columnar], Mypy struggles to see Column as compatible. return _apply_window_if_present( - ibis_ops.LastNonNullValue(column).to_expr(), # type: ignore[arg-type] - window, # type: ignore + ibis_ops.LastNonNullValue(column).to_expr(), window # type: ignore ) @@ -805,10 +801,8 @@ def _to_ibis_boundary( ) -> Optional[ibis_expr_window.WindowBoundary]: if boundary is None: return None - # WindowBoundary expects Value[Any, Any], ibis_types.literal returns Scalar which Mypy doesn't see as compatible. return ibis_expr_window.WindowBoundary( - ibis_types.literal(boundary if boundary >= 0 else -boundary), # type: ignore[arg-type] - preceding=boundary <= 0, # type:ignore + abs(boundary), preceding=boundary <= 0 # type:ignore ) diff --git a/bigframes/core/compile/ibis_compiler/default_ordering.py b/bigframes/core/compile/ibis_compiler/default_ordering.py index 84ce52851c4..3f2628d10cd 100644 --- a/bigframes/core/compile/ibis_compiler/default_ordering.py +++ b/bigframes/core/compile/ibis_compiler/default_ordering.py @@ -18,7 +18,7 @@ from __future__ import annotations -from typing import Sequence, cast +from typing import cast, Sequence import bigframes_vendored.ibis import bigframes_vendored.ibis.expr.datatypes as ibis_dtypes diff --git a/bigframes/core/compile/ibis_compiler/ibis_compiler.py b/bigframes/core/compile/ibis_compiler/ibis_compiler.py index 938759ae181..31cd9a0456b 100644 --- a/bigframes/core/compile/ibis_compiler/ibis_compiler.py +++ b/bigframes/core/compile/ibis_compiler/ibis_compiler.py @@ -16,13 +16,15 @@ import dataclasses import functools import typing -from typing import Optional, cast +from typing import cast, Optional import bigframes_vendored.ibis.backends.bigquery as ibis_bigquery import bigframes_vendored.ibis.expr.api as ibis_api import bigframes_vendored.ibis.expr.datatypes as ibis_dtypes import bigframes_vendored.ibis.expr.types as ibis_types +from bigframes import dtypes, operations +from bigframes.core import bq_data, expression, pyarrow_utils import bigframes.core.compile.compiled as compiled import bigframes.core.compile.concat as concat_impl import bigframes.core.compile.configs as configs @@ -30,10 +32,6 @@ import bigframes.core.nodes as nodes import bigframes.core.ordering as bf_ordering import bigframes.core.rewrite as rewrites -import bigframes.core.rewrite.schema_binding as schema_binding -from bigframes import dtypes, operations -from bigframes.core import bq_data, expression, pyarrow_utils -from bigframes.core.logging import data_types as data_type_logger if typing.TYPE_CHECKING: import bigframes.core @@ -50,8 +48,7 @@ def compile_sql(request: configs.CompileRequest) -> configs.CompileResult: # Can only pullup slice if we are doing ORDER BY in outermost SELECT # Need to do this before replacing unsupported ops, as that will rewrite slice ops result_node = rewrites.pull_up_limits(result_node) - result_node = cast(nodes.ResultNode, _replace_unsupported_ops(result_node)) - result_node = cast(nodes.ResultNode, result_node.bottom_up(rewrites.simplify_join)) + result_node = _replace_unsupported_ops(result_node) # prune before pulling up order to avoid unnnecessary row_number() ops result_node = cast(nodes.ResultNode, rewrites.column_pruning(result_node)) result_node = rewrites.defer_order( @@ -59,30 +56,15 @@ def compile_sql(request: configs.CompileRequest) -> configs.CompileResult: ) if request.sort_rows: result_node = cast(nodes.ResultNode, rewrites.column_pruning(result_node)) - encoded_type_refs = data_type_logger.encode_type_refs(result_node) - # Have to bind schema as the final step before compilation. - # Probably, should defer even further - result_node = typing.cast( - nodes.ResultNode, schema_binding.bind_schema_to_tree(result_node) - ) sql = compile_result_node(result_node) return configs.CompileResult( - sql, - result_node.schema.to_bigquery(), - result_node.order_by, - encoded_type_refs, + sql, result_node.schema.to_bigquery(), result_node.order_by ) ordering: Optional[bf_ordering.RowOrdering] = result_node.order_by result_node = dataclasses.replace(result_node, order_by=None) result_node = cast(nodes.ResultNode, rewrites.column_pruning(result_node)) result_node = cast(nodes.ResultNode, rewrites.defer_selection(result_node)) - encoded_type_refs = data_type_logger.encode_type_refs(result_node) - # Have to bind schema as the final step before compilation. - # Probably, should defer even further - result_node = typing.cast( - nodes.ResultNode, schema_binding.bind_schema_to_tree(result_node) - ) sql = compile_result_node(result_node) # Return the ordering iff no extra columns are needed to define the row order if ordering is not None: @@ -90,9 +72,7 @@ def compile_sql(request: configs.CompileRequest) -> configs.CompileResult: ordering if ordering.referenced_columns.issubset(result_node.ids) else None ) assert (not request.materialize_all_order_keys) or (output_order is not None) - return configs.CompileResult( - sql, result_node.schema.to_bigquery(), output_order, encoded_type_refs - ) + return configs.CompileResult(sql, result_node.schema.to_bigquery(), output_order) def _replace_unsupported_ops(node: nodes.BigFrameNode): @@ -100,7 +80,6 @@ def _replace_unsupported_ops(node: nodes.BigFrameNode): node = nodes.bottom_up(node, rewrites.rewrite_slice) node = nodes.bottom_up(node, rewrites.rewrite_timedelta_expressions) node = nodes.bottom_up(node, rewrites.rewrite_range_rolling) - node = nodes.bottom_up(node, rewrites.lower_udfs) return node @@ -228,7 +207,9 @@ def _table_to_ibis( source: bq_data.BigqueryDataSource, scan_cols: typing.Sequence[str], ) -> ibis_types.Table: - full_table_name = source.table.get_full_id(quoted=False) + full_table_name = ( + f"{source.table.project_id}.{source.table.dataset_id}.{source.table.table_id}" + ) # Physical schema might include unused columns, unsupported datatypes like JSON physical_schema = ibis_bigquery.BigQuerySchema.to_ibis( list(source.table.physical_schema) diff --git a/bigframes/core/compile/ibis_compiler/operations/geo_ops.py b/bigframes/core/compile/ibis_compiler/operations/geo_ops.py index 772752112a4..0ca69726ffe 100644 --- a/bigframes/core/compile/ibis_compiler/operations/geo_ops.py +++ b/bigframes/core/compile/ibis_compiler/operations/geo_ops.py @@ -16,11 +16,11 @@ from typing import cast +from bigframes_vendored import ibis +from bigframes_vendored.ibis.expr import types as ibis_types import bigframes_vendored.ibis.expr.datatypes as ibis_dtypes import bigframes_vendored.ibis.expr.operations.geospatial as ibis_geo import bigframes_vendored.ibis.expr.operations.udf as ibis_udf -from bigframes_vendored import ibis -from bigframes_vendored.ibis.expr import types as ibis_types from bigframes.core.compile.ibis_compiler import scalar_op_compiler from bigframes.operations import geo_ops as ops @@ -30,6 +30,11 @@ # Geo Ops +@register_unary_op(ops.geo_area_op) +def geo_area_op_impl(x: ibis_types.Value): + return cast(ibis_types.GeoSpatialValue, x).area() + + @register_unary_op(ops.geo_st_astext_op) def geo_st_astext_op_impl(x: ibis_types.Value): return cast(ibis_types.GeoSpatialValue, x).as_text() @@ -50,6 +55,11 @@ def geo_st_buffer_op_impl(x: ibis_types.Value, op: ops.GeoStBufferOp): ) +@register_unary_op(ops.geo_st_centroid_op, pass_op=False) +def geo_st_centroid_op_impl(x: ibis_types.Value): + return cast(ibis_types.GeoSpatialValue, x).centroid() + + @register_unary_op(ops.geo_st_convexhull_op, pass_op=False) def geo_st_convexhull_op_impl(x: ibis_types.Value): return st_convexhull(x) @@ -122,6 +132,12 @@ def geo_st_regionstats_op_impl( ).to_expr() +@register_unary_op(ops.GeoStSimplifyOp, pass_op=True) +def st_simplify_op_impl(x: ibis_types.Value, op: ops.GeoStSimplifyOp): + x = cast(ibis_types.GeoSpatialValue, x) + return st_simplify(x, op.tolerance_meters) + + @register_unary_op(ops.geo_x_op) def geo_x_op_impl(x: ibis_types.Value): return cast(ibis_types.GeoSpatialValue, x).x() @@ -165,11 +181,7 @@ def st_buffer( @ibis_udf.scalar.builtin -def st_distance( - a: ibis_dtypes.geography, # type: ignore - b: ibis_dtypes.geography, # type: ignore - use_spheroid: bool, # type: ignore -) -> ibis_dtypes.float: # type: ignore +def st_distance(a: ibis_dtypes.geography, b: ibis_dtypes.geography, use_spheroid: bool) -> ibis_dtypes.float: # type: ignore """Convert string to geography.""" diff --git a/bigframes/core/compile/ibis_compiler/scalar_op_compiler.py b/bigframes/core/compile/ibis_compiler/scalar_op_compiler.py index 31a8459923c..8a027ca2965 100644 --- a/bigframes/core/compile/ibis_compiler/scalar_op_compiler.py +++ b/bigframes/core/compile/ibis_compiler/scalar_op_compiler.py @@ -21,13 +21,11 @@ from typing import TYPE_CHECKING import bigframes_vendored.ibis -import bigframes_vendored.ibis.expr.operations.generic as ibis_generic import bigframes_vendored.ibis.expr.types as ibis_types +from bigframes.core import agg_expressions, ordering import bigframes.core.compile.ibis_types import bigframes.core.expression as ex -from bigframes.core import agg_expressions, ordering -from bigframes.operations import googlesql as gsql_ops from bigframes.operations import numeric_ops if TYPE_CHECKING: @@ -94,20 +92,8 @@ def _( self.compile_expression(sub_expr, bindings) for sub_expr in expression.inputs ] - if isinstance(expression.op, gsql_ops.GoogleSqlScalarOp): - return googlesql_scalar_op_impl( - *inputs, op=expression.op, output_type=expression.output_type - ) return self.compile_row_op(expression.op, inputs) - @compile_expression.register - def _( - self, - expression: ex.OmittedArg, - bindings: typing.Dict[str, ibis_types.Value], - ) -> ibis_types.Value: - return bigframes_vendored.ibis.omitted() - def compile_row_op( self, op: ops.RowOp, inputs: typing.Sequence[ibis_types.Value] ) -> ibis_types.Value: @@ -292,39 +278,3 @@ def isnanornull(arg): @scalar_op_compiler.register_unary_op(numeric_ops.isfinite_op) def isfinite(arg): return arg.isinf().negate() & arg.isnan().negate() - - -def googlesql_scalar_op_impl( - *operands: ibis_types.Value, op: ops.GoogleSqlScalarOp, output_type -): - final_operands: list[ibis_types.Value] = [] - arg_templates = [] - for i, operand in enumerate(operands): - if i < len(op.args): - arg_spec = op.args[i] - else: - assert op.args[-1].is_vararg, ( - f"Too many arguments, for {op.sql_name}, expected {len(op.args)}" - ) - arg_spec = op.args[-1] - if isinstance(operand.op(), ibis_generic.OmittedArg): - assert arg_spec.optional, "Argument omitted, but not optional" - continue - - target_idx = len(final_operands) - final_operands.append(operand) - if arg_spec.arg_name: - arg_templates.append(f"{arg_spec.arg_name} => {{{target_idx}}}") - else: - arg_templates.append(f"{{{target_idx}}}") - args_template = ", ".join(arg_templates) - sql_template = f"{op.sql_name}({args_template})" - return ibis_generic.SqlScalar( - sql_template, - values=tuple( - typing.cast(ibis_generic.Value, expr.op()) for expr in final_operands - ), - output_type=bigframes.core.compile.ibis_types.bigframes_dtype_to_ibis_dtype( - output_type - ), - ).to_expr() diff --git a/bigframes/core/compile/ibis_compiler/scalar_op_registry.py b/bigframes/core/compile/ibis_compiler/scalar_op_registry.py index 530d23a8b06..519b2c94426 100644 --- a/bigframes/core/compile/ibis_compiler/scalar_op_registry.py +++ b/bigframes/core/compile/ibis_compiler/scalar_op_registry.py @@ -16,8 +16,9 @@ import functools import typing -from typing import Any, cast +from typing import cast +from bigframes_vendored import ibis import bigframes_vendored.ibis.expr.api as ibis_api import bigframes_vendored.ibis.expr.datatypes as ibis_dtypes import bigframes_vendored.ibis.expr.operations.ai_ops as ai_ops @@ -26,15 +27,14 @@ import bigframes_vendored.ibis.expr.types as ibis_types import numpy as np import pandas as pd -from bigframes_vendored import ibis -import bigframes.core.compile.ibis_compiler.default_ordering -import bigframes.core.compile.ibis_types -import bigframes.operations as ops from bigframes.core.compile.constants import UNIT_TO_US_CONVERSION_FACTORS +import bigframes.core.compile.ibis_compiler.default_ordering from bigframes.core.compile.ibis_compiler.scalar_op_compiler import ( scalar_op_compiler, # TODO(tswast): avoid import of variables ) +import bigframes.core.compile.ibis_types +import bigframes.operations as ops _ZERO = typing.cast(ibis_types.NumericValue, ibis_types.literal(0)) _NAN = typing.cast(ibis_types.NumericValue, ibis_types.literal(np.nan)) @@ -169,8 +169,6 @@ def arctanh_op_impl(x: ibis_types.Value): @scalar_op_compiler.register_unary_op(ops.floor_op) def floor_op_impl(x: ibis_types.Value): x_numeric = typing.cast(ibis_types.NumericValue, x) - if x_numeric.type().is_boolean(): - return x_numeric.cast(ibis_dtypes.Int64()).cast(ibis_dtypes.Float64()) if x_numeric.type().is_integer(): return x_numeric.cast(ibis_dtypes.Float64()) if x_numeric.type().is_floating(): @@ -183,8 +181,6 @@ def floor_op_impl(x: ibis_types.Value): @scalar_op_compiler.register_unary_op(ops.ceil_op) def ceil_op_impl(x: ibis_types.Value): x_numeric = typing.cast(ibis_types.NumericValue, x) - if x_numeric.type().is_boolean(): - return x_numeric.cast(ibis_dtypes.Int64()).cast(ibis_dtypes.Float64()) if x_numeric.type().is_integer(): return x_numeric.cast(ibis_dtypes.Float64()) if x_numeric.type().is_floating(): @@ -363,6 +359,14 @@ def contains_regex_op_impl(x: ibis_types.Value, op: ops.StrContainsRegexOp): return typing.cast(ibis_types.StringValue, x).re_search(op.pat) +@scalar_op_compiler.register_unary_op(ops.StrGetOp, pass_op=True) +def strget_op_impl(x: ibis_types.Value, op: ops.StrGetOp): + substr = typing.cast( + ibis_types.StringValue, typing.cast(ibis_types.StringValue, x)[op.i] + ) + return substr.nullif(ibis_types.literal("")) + + @scalar_op_compiler.register_unary_op(ops.StrPadOp, pass_op=True) def strpad_op_impl(x: ibis_types.Value, op: ops.StrPadOp): str_val = typing.cast(ibis_types.StringValue, x) @@ -655,7 +659,7 @@ def datetime_to_integer_label_non_fixed_frequency( .else_((x_int - first - 1) // us + 1) # type: ignore .end() ) - elif rule_code in ("M", "ME"): # Monthly + elif rule_code == "ME": # Monthly x_int = x.year() * 12 + x.month() - 1 # type: ignore first = y.year() * 12 + y.month() - 1 # type: ignore x_int_label = ( @@ -664,7 +668,7 @@ def datetime_to_integer_label_non_fixed_frequency( .else_((x_int - first - 1) // n + 1) # type: ignore .end() ) - elif rule_code in ("Q-DEC", "QE-DEC"): # Quarterly + elif rule_code == "QE-DEC": # Quarterly x_int = x.year() * 4 + x.quarter() - 1 # type: ignore first = y.year() * 4 + y.quarter() - 1 # type: ignore x_int_label = ( @@ -673,7 +677,7 @@ def datetime_to_integer_label_non_fixed_frequency( .else_((x_int - first - 1) // n + 1) # type: ignore .end() ) - elif rule_code in ("A-DEC", "Y-DEC", "YE-DEC"): # Yearly + elif rule_code == "YE-DEC": # Yearly x_int = x.year() # type: ignore first = y.year() # type: ignore x_int_label = ( @@ -741,7 +745,7 @@ def integer_label_to_datetime_op_non_fixed_frequency( .cast(ibis_dtypes.Timestamp(timezone="UTC")) .cast(y.type()) ) - elif rule_code in ("M", "ME"): # Monthly + elif rule_code == "ME": # Monthly one = ibis_types.literal(1) twelve = ibis_types.literal(12) first = y.year() * twelve + y.month() - one # type: ignore @@ -761,7 +765,7 @@ def integer_label_to_datetime_op_non_fixed_frequency( 0, ) x_label = next_month_date - ibis_api.interval(days=1) - elif rule_code in ("Q-DEC", "QE-DEC"): # Quarterly + elif rule_code == "QE-DEC": # Quarterly one = ibis_types.literal(1) three = ibis_types.literal(3) four = ibis_types.literal(4) @@ -784,7 +788,7 @@ def integer_label_to_datetime_op_non_fixed_frequency( ) x_label = next_month_date - ibis_api.interval(days=1) - elif rule_code in ("A-DEC", "Y-DEC", "YE-DEC"): # Yearly + elif rule_code == "YE-DEC": # Yearly one = ibis_types.literal(1) first = y.year() # type: ignore x = x * n + first # type: ignore @@ -876,25 +880,6 @@ def numeric_to_datetime( ) -@scalar_op_compiler.register_unary_op(ops.coerce_to_bool_op) -def coerce_to_bool_op_impl(x: ibis_types.Value): - x_type = x.type() - if x_type.is_boolean(): - res = x - elif x_type.is_numeric(): - res = x != 0 # type: ignore - elif x_type.is_string(): - res = x.length() > 0 # type: ignore - elif x_type.is_binary(): - res = x.length() > 0 # type: ignore - elif isinstance(x_type, ibis_dtypes.Array): - res = x.length() > 0 # type: ignore - else: - res = x.notnull() - - return res.fill_null(False) # type: ignore - - @scalar_op_compiler.register_unary_op(ops.AsTypeOp, pass_op=True) def astype_op_impl(x: ibis_types.Value, op: ops.AsTypeOp): to_type = bigframes.core.compile.ibis_types.bigframes_dtype_to_ibis_dtype( @@ -933,6 +918,35 @@ def astype_op_impl(x: ibis_types.Value, op: ops.AsTypeOp): elif to_type == ibis_dtypes.time: return x_converted.time() + if to_type == ibis_dtypes.json: + if x.type() == ibis_dtypes.string: + return parse_json_in_safe(x) if op.safe else parse_json(x) + if x.type() == ibis_dtypes.bool: + x_bool = typing.cast( + ibis_types.StringValue, + bigframes.core.compile.ibis_types.cast_ibis_value( + x, ibis_dtypes.string, safe=op.safe + ), + ).lower() + return parse_json_in_safe(x_bool) if op.safe else parse_json(x_bool) + if x.type() in (ibis_dtypes.int64, ibis_dtypes.float64): + x_str = bigframes.core.compile.ibis_types.cast_ibis_value( + x, ibis_dtypes.string, safe=op.safe + ) + return parse_json_in_safe(x_str) if op.safe else parse_json(x_str) + + if x.type() == ibis_dtypes.json: + if to_type == ibis_dtypes.int64: + return cast_json_to_int64_in_safe(x) if op.safe else cast_json_to_int64(x) + if to_type == ibis_dtypes.float64: + return ( + cast_json_to_float64_in_safe(x) if op.safe else cast_json_to_float64(x) + ) + if to_type == ibis_dtypes.bool: + return cast_json_to_bool_in_safe(x) if op.safe else cast_json_to_bool(x) + if to_type == ibis_dtypes.string: + return cast_json_to_string_in_safe(x) if op.safe else cast_json_to_string(x) + # TODO: either inline this function, or push rest of this op into the function return bigframes.core.compile.ibis_types.cast_ibis_value(x, to_type, safe=op.safe) @@ -948,7 +962,7 @@ def isin_op_impl(x: ibis_types.Value, op: ops.IsInOp): # to actually cast it, as that could be lossy (eg float -> int) item_inferred_type = ibis_types.literal(item).type() if ( - x.type().name == item_inferred_type.name + x.type() == item_inferred_type or x.type().is_numeric() and item_inferred_type.is_numeric() ): @@ -964,9 +978,7 @@ def isin_op_impl(x: ibis_types.Value, op: ops.IsInOp): @scalar_op_compiler.register_unary_op(ops.ToDatetimeOp, pass_op=True) def to_datetime_op_impl(x: ibis_types.Value, op: ops.ToDatetimeOp): - if x.type() == ibis_dtypes.Timestamp(None): # type: ignore - return x # already a timestamp, no-op - elif x.type() in (ibis_dtypes.str, ibis_dtypes.Timestamp("UTC")): # type: ignore + if x.type() == ibis_dtypes.str: return x.try_cast(ibis_dtypes.Timestamp(None)) # type: ignore else: # Numerical inputs. @@ -989,8 +1001,6 @@ def to_timestamp_op_impl(x: ibis_types.Value, op: ops.ToTimestampOp): if op.format else timestamp(x) ) - elif x.type() == ibis_dtypes.Timestamp(None): # type: ignore - return timestamp(x) else: # Numerical inputs. if op.format: @@ -1013,24 +1023,61 @@ def to_timedelta_op_impl(x: ibis_types.Value, op: ops.ToTimedeltaOp): @scalar_op_compiler.register_unary_op(ops.timedelta_floor_op) def timedelta_floor_op_impl(x: ibis_types.NumericValue): - return ibis_api.case().when(x > ibis.literal(0), x.floor()).else_(x.ceil()).end() + return x.floor() -@scalar_op_compiler.register_nary_op(ops.RemoteFunctionOp, pass_op=True) -def remote_function_op_impl(*values: ibis_types.Value, op: ops.RemoteFunctionOp): +@scalar_op_compiler.register_unary_op(ops.RemoteFunctionOp, pass_op=True) +def remote_function_op_impl(x: ibis_types.Value, op: ops.RemoteFunctionOp): udf_sig = op.function_def.signature - assert not udf_sig.is_virtual # should have been devirtualized in lowering pass - ibis_py_sig = (tuple(arg.py_type for arg in udf_sig.inputs), udf_sig.output.py_type) - arg_names = tuple(arg.name for arg in udf_sig.inputs) + ibis_py_sig = (udf_sig.py_input_types, udf_sig.py_output_type) + + @ibis_udf.scalar.builtin( + name=str(op.function_def.routine_ref), signature=ibis_py_sig + ) + def udf(input): + ... + + x_transformed = udf(x) + if not op.apply_on_null: + return ibis_api.case().when(x.isnull(), x).else_(x_transformed).end() + return x_transformed + + +@scalar_op_compiler.register_binary_op(ops.BinaryRemoteFunctionOp, pass_op=True) +def binary_remote_function_op_impl( + x: ibis_types.Value, y: ibis_types.Value, op: ops.BinaryRemoteFunctionOp +): + udf_sig = op.function_def.signature + ibis_py_sig = (udf_sig.py_input_types, udf_sig.py_output_type) + + @ibis_udf.scalar.builtin( + name=str(op.function_def.routine_ref), signature=ibis_py_sig + ) + def udf(input1, input2): + ... + + x_transformed = udf(x, y) + return x_transformed + + +@scalar_op_compiler.register_nary_op(ops.NaryRemoteFunctionOp, pass_op=True) +def nary_remote_function_op_impl( + *operands: ibis_types.Value, op: ops.NaryRemoteFunctionOp +): + udf_sig = op.function_def.signature + ibis_py_sig = (udf_sig.py_input_types, udf_sig.py_output_type) + arg_names = tuple(arg.name for arg in udf_sig.input_types) @ibis_udf.scalar.builtin( name=str(op.function_def.routine_ref), signature=ibis_py_sig, param_name_overrides=arg_names, ) - def udf(*inputs): ... + def udf(*inputs): + ... - return udf(*values) + result = udf(*operands) + return result @scalar_op_compiler.register_unary_op(ops.MapOp, pass_op=True) @@ -1051,39 +1098,13 @@ def array_to_string_op_impl(x: ibis_types.Value, op: ops.ArrayToStringOp): return typing.cast(ibis_types.ArrayValue, x).join(op.delimiter) -@scalar_op_compiler.register_unary_op(ops.GetItemOp, pass_op=True) -def getitem_op_impl(x: ibis_types.Value, op: ops.GetItemOp): - if x.type().is_struct(): - struct_value = typing.cast(ibis_types.StructValue, x) - if isinstance(op.key, str): - name = op.key - else: - name = struct_value.names[op.key] - result = struct_value[name] - return result.cast(result.type()(nullable=True)).name(name) - elif x.type().is_array(): - key = typing.cast(int, op.key) - res = typing.cast(ibis_types.ArrayValue, x)[key] - return res - elif x.type().is_string(): - key = typing.cast(int, op.key) - res = typing.cast(ibis_types.StringValue, x)[key] - return _null_or_value(res, res != ibis_types.literal("")) - else: - raise TypeError(f"Cannot subscript input of type {x.type()}") - - -@scalar_op_compiler.register_binary_op(ops.DynamicGetItemOp) -def dynamic_getitem_op_impl(left: ibis_types.Value, right: ibis_types.Value): - if left.type().is_array(): - int_right = typing.cast(ibis_types.IntegerValue, right) - return typing.cast(ibis_types.ArrayValue, left)[int_right] - elif left.type().is_string(): - scalar_right = typing.cast(ibis_types.IntegerScalar, right) - res = typing.cast(ibis_types.StringValue, left)[scalar_right] +@scalar_op_compiler.register_unary_op(ops.ArrayIndexOp, pass_op=True) +def array_index_op_impl(x: ibis_types.Value, op: ops.ArrayIndexOp): + res = typing.cast(ibis_types.ArrayValue, x)[op.index] + if x.type().is_string(): return _null_or_value(res, res != ibis_types.literal("")) else: - raise TypeError(f"Cannot dynamically subscript input of type {left.type()}") + return res @scalar_op_compiler.register_unary_op(ops.ArraySliceOp, pass_op=True) @@ -1124,13 +1145,6 @@ def array_reduce_op_impl(x: ibis_types.Value, op: ops.ArrayReduceOp): ) -@scalar_op_compiler.register_unary_op(ops.ArrayMapOp, pass_op=True) -def array_map_op_impl(x: ibis_types.Value, op: ops.ArrayMapOp): - return typing.cast(ibis_types.ArrayValue, x).map( - lambda arr_vals: scalar_op_compiler.compile_row_op(op.map_op, (arr_vals,)) - ) - - # JSON Ops @scalar_op_compiler.register_binary_op(ops.JSONSet, pass_op=True) def json_set_op_impl(x: ibis_types.Value, y: ibis_types.Value, op: ops.JSONSet): @@ -1201,27 +1215,9 @@ def parse_json_op_impl(x: ibis_types.Value, op: ops.ParseJSON): return parse_json(json_str=x) -@scalar_op_compiler.register_unary_op(ops.ToJSON, pass_op=True) -def to_json_op_impl(x: ibis_types.Value, op: ops.ToJSON): - if x.type() == ibis_dtypes.string: - return parse_json_in_safe(x) if op.safe else parse_json(x) - return x.isnull().ifelse(ibis.null().cast(ibis_dtypes.json), to_json(x)) - - -@scalar_op_compiler.register_unary_op(ops.JSONDecode, pass_op=True) -def json_decode_op_impl(x: ibis_types.Value, op: ops.JSONDecode): - to_type = bigframes.core.compile.ibis_types.bigframes_dtype_to_ibis_dtype( - op.to_type - ) - if to_type == ibis_dtypes.int64: - return cast_json_to_int64_in_safe(x) if op.safe else cast_json_to_int64(x) - if to_type == ibis_dtypes.float64: - return cast_json_to_float64_in_safe(x) if op.safe else cast_json_to_float64(x) - if to_type == ibis_dtypes.bool: - return cast_json_to_bool_in_safe(x) if op.safe else cast_json_to_bool(x) - if to_type == ibis_dtypes.string: - return cast_json_to_string_in_safe(x) if op.safe else cast_json_to_string(x) - raise TypeError(f"Cannot cast from JSON to type {to_type}") +@scalar_op_compiler.register_unary_op(ops.ToJSON) +def to_json_op_impl(json_obj: ibis_types.Value): + return to_json(json_obj=json_obj) @scalar_op_compiler.register_unary_op(ops.ToJSONString) @@ -1905,11 +1901,12 @@ def struct_op_impl( def ai_generate( *values: ibis_types.Value, op: ops.AIGenerate ) -> ibis_types.StructValue: + return ai_ops.AIGenerate( _construct_prompt(values, op.prompt_context), # type: ignore op.connection_id, # type: ignore op.endpoint, # type: ignore - op.request_type, # type: ignore + op.request_type.upper(), # type: ignore op.model_params, # type: ignore op.output_schema, # type: ignore ).to_expr() @@ -1919,11 +1916,12 @@ def ai_generate( def ai_generate_bool( *values: ibis_types.Value, op: ops.AIGenerateBool ) -> ibis_types.StructValue: + return ai_ops.AIGenerateBool( _construct_prompt(values, op.prompt_context), # type: ignore op.connection_id, # type: ignore op.endpoint, # type: ignore - op.request_type, # type: ignore + op.request_type.upper(), # type: ignore op.model_params, # type: ignore ).to_expr() @@ -1932,11 +1930,12 @@ def ai_generate_bool( def ai_generate_int( *values: ibis_types.Value, op: ops.AIGenerateInt ) -> ibis_types.StructValue: + return ai_ops.AIGenerateInt( _construct_prompt(values, op.prompt_context), # type: ignore op.connection_id, # type: ignore op.endpoint, # type: ignore - op.request_type, # type: ignore + op.request_type.upper(), # type: ignore op.model_params, # type: ignore ).to_expr() @@ -1945,36 +1944,22 @@ def ai_generate_int( def ai_generate_double( *values: ibis_types.Value, op: ops.AIGenerateDouble ) -> ibis_types.StructValue: + return ai_ops.AIGenerateDouble( _construct_prompt(values, op.prompt_context), # type: ignore op.connection_id, # type: ignore op.endpoint, # type: ignore - op.request_type, # type: ignore + op.request_type.upper(), # type: ignore op.model_params, # type: ignore ).to_expr() -@scalar_op_compiler.register_unary_op(ops.AIEmbed, pass_op=True) -def ai_embed(value: ibis_types.Value, op: ops.AIEmbed) -> ibis_types.StructValue: - return ai_ops.AIEmbed( - value, # type: ignore - connection_id=op.connection_id, # type: ignore - endpoint=op.endpoint, # type: ignore - model=op.model, # type: ignore - task_type=op.task_type, # type: ignore - title=op.title, # type: ignore - model_params=op.model_params, # type: ignore - ).to_expr() - - @scalar_op_compiler.register_nary_op(ops.AIIf, pass_op=True) def ai_if(*values: ibis_types.Value, op: ops.AIIf) -> ibis_types.StructValue: + return ai_ops.AIIf( _construct_prompt(values, op.prompt_context), # type: ignore op.connection_id, # type: ignore - op.endpoint, # type: ignore - op.optimization_mode, # type: ignore - op.max_error_ratio, # type: ignore ).to_expr() @@ -1982,39 +1967,20 @@ def ai_if(*values: ibis_types.Value, op: ops.AIIf) -> ibis_types.StructValue: def ai_classify( *values: ibis_types.Value, op: ops.AIClassify ) -> ibis_types.StructValue: + return ai_ops.AIClassify( _construct_prompt(values, op.prompt_context), # type: ignore op.categories, # type: ignore - _construct_examples(op.examples), # type: ignore op.connection_id, # type: ignore - op.endpoint, # type: ignore - op.output_mode, # type: ignore - op.optimization_mode, # type: ignore - op.max_error_ratio, # type: ignore ).to_expr() @scalar_op_compiler.register_nary_op(ops.AIScore, pass_op=True) def ai_score(*values: ibis_types.Value, op: ops.AIScore) -> ibis_types.StructValue: + return ai_ops.AIScore( _construct_prompt(values, op.prompt_context), # type: ignore op.connection_id, # type: ignore - op.endpoint, # type: ignore - op.max_error_ratio, # type: ignore - ).to_expr() - - -@scalar_op_compiler.register_binary_op(ops.AISimilarity, pass_op=True) -def ai_similarity( - content1: ibis_types.Value, content2: ibis_types.Value, op: ops.AISimilarity -) -> ibis_types.Value: - return ai_ops.AISimilarity( - content1, # type: ignore - content2, # type: ignore - op.endpoint, # type: ignore - op.model, # type: ignore - op.model_params, # type: ignore - op.connection_id, # type: ignore ).to_expr() @@ -2034,25 +2000,6 @@ def _construct_prompt( return ibis.struct(prompt) -def _construct_examples( - examples: tuple[tuple[str, str | tuple[str, ...]], ...] | None, -) -> ibis_types.ArrayValue | None: - if examples is None: - return None - - results: list[ibis_types.StructValue] = [] - - for example in examples: - value: Any = example[1] - if isinstance(example[1], (list, tuple)): - value = list(example[1]) - - ibis_example = ibis.struct({"_field_1": example[0], "_field_2": value}) - results.append(ibis_example) - - return ibis.array(results) - - @scalar_op_compiler.register_nary_op(ops.RowKey, pass_op=True) def rowkey_op_impl(*values: ibis_types.Value, op: ops.RowKey) -> ibis_types.Value: return bigframes.core.compile.ibis_compiler.default_ordering.gen_row_key(values) @@ -2069,8 +2016,8 @@ def _ibis_num(number: float): @ibis_udf.scalar.builtin -def timestamp(a) -> ibis_dtypes.timestamp: # type: ignore - """Convert string or a datetime to timestamp.""" +def timestamp(a: str) -> ibis_dtypes.timestamp: # type: ignore + """Convert string to timestamp.""" @ibis_udf.scalar.builtin @@ -2213,12 +2160,7 @@ def obj_make_ref_json(objectref_json: ibis_dtypes.JSON) -> _OBJ_REF_IBIS_DTYPE: @ibis_udf.scalar.builtin(name="OBJ.GET_ACCESS_URL") -# Stub for BigQuery UDF, empty body is intentional. -# _OBJ_REF_IBIS_DTYPE is a variable holding a type, Mypy complains about it being used as type hint. -def obj_get_access_url( # type: ignore[empty-body] - obj_ref: _OBJ_REF_IBIS_DTYPE, # type: ignore[valid-type] - mode: ibis_dtypes.String, -) -> ibis_dtypes.JSON: +def obj_get_access_url(obj_ref: _OBJ_REF_IBIS_DTYPE, mode: ibis_dtypes.String) -> ibis_dtypes.JSON: # type: ignore """Get access url (as ObjectRefRumtime JSON) from ObjectRef.""" diff --git a/bigframes/core/compile/ibis_types.py b/bigframes/core/compile/ibis_types.py index 788f6db4435..25b59d4582c 100644 --- a/bigframes/core/compile/ibis_types.py +++ b/bigframes/core/compile/ibis_types.py @@ -13,7 +13,7 @@ # limitations under the License. from __future__ import annotations -from typing import Dict, Iterable, Optional, Tuple, Union, cast +from typing import cast, Dict, Iterable, Optional, Tuple, Union import bigframes_vendored.constants as constants import bigframes_vendored.ibis diff --git a/bigframes/core/compile/polars/__init__.py b/bigframes/core/compile/polars/__init__.py index 027582c7ded..215d6b088e6 100644 --- a/bigframes/core/compile/polars/__init__.py +++ b/bigframes/core/compile/polars/__init__.py @@ -16,7 +16,6 @@ Make sure to import all polars implementations here so that they get registered. """ - from __future__ import annotations import warnings @@ -24,7 +23,6 @@ # The ops imports appear first so that the implementations can be registered. # polars shouldn't be needed at import time, as register is a no-op if polars # isn't installed. -import bigframes.core.compile.polars.operations.array_ops # noqa: F401 import bigframes.core.compile.polars.operations.generic_ops # noqa: F401 import bigframes.core.compile.polars.operations.numeric_ops # noqa: F401 import bigframes.core.compile.polars.operations.struct_ops # noqa: F401 diff --git a/bigframes/core/compile/polars/compiler.py b/bigframes/core/compile/polars/compiler.py index 65b217602b9..1f0ca592e53 100644 --- a/bigframes/core/compile/polars/compiler.py +++ b/bigframes/core/compile/polars/compiler.py @@ -16,11 +16,13 @@ import dataclasses import functools import itertools -from typing import TYPE_CHECKING, Literal, Optional, Sequence, Tuple, Type, cast +from typing import cast, Literal, Optional, Sequence, Tuple, Type, TYPE_CHECKING import pandas as pd import bigframes.core +from bigframes.core import agg_expressions, identifiers, nodes, ordering, window_spec +from bigframes.core.compile.polars import lowering import bigframes.core.expression as ex import bigframes.core.guid as guid import bigframes.core.rewrite @@ -37,11 +39,7 @@ import bigframes.operations.generic_ops as gen_ops import bigframes.operations.json_ops as json_ops import bigframes.operations.numeric_ops as num_ops -import bigframes.operations.remote_function_ops as remote_function_ops import bigframes.operations.string_ops as string_ops -import bigframes.operations.struct_ops as struct_ops -from bigframes.core import agg_expressions, identifiers, nodes, ordering, window_spec -from bigframes.core.compile.polars import lowering polars_installed = True if TYPE_CHECKING: @@ -124,7 +122,7 @@ def _bigframes_dtype_to_polars_dtype( ] ) if bigframes.dtypes.is_array_like(dtype): - return pl.List( + return pl.Array( inner=_bigframes_dtype_to_polars_dtype( bigframes.dtypes.get_array_inner_type(dtype) ) @@ -140,20 +138,11 @@ class PolarsExpressionCompiler: Should be extended to dispatch based on bigframes schema types. """ - _expr_types: dict[int, bigframes.dtypes.ExpressionType] = dataclasses.field( - default_factory=dict, init=False, compare=False - ) - - def compile_expression(self, expression: ex.Expression) -> pl.Expr: - res = self._compile_expression(expression) - self._expr_types[id(res)] = expression.output_type - return res - @functools.singledispatchmethod - def _compile_expression(self, expression: ex.Expression) -> pl.Expr: + def compile_expression(self, expression: ex.Expression) -> pl.Expr: raise NotImplementedError(f"Cannot compile expression: {expression}") - @_compile_expression.register + @compile_expression.register def _( self, expression: ex.ScalarConstantExpression, @@ -170,78 +159,27 @@ def _( return pl.lit(value, _bigframes_dtype_to_polars_dtype(expression.dtype)) - @_compile_expression.register + @compile_expression.register def _( self, expression: ex.DerefOp, ) -> pl.Expr: return pl.col(expression.id.sql) - @_compile_expression.register + @compile_expression.register def _( self, expression: ex.ResolvedDerefOp, ) -> pl.Expr: return pl.col(expression.id.sql) - @_compile_expression.register + @compile_expression.register def _( self, expression: ex.OpExpression, ) -> pl.Expr: - import datetime - - import pyarrow as pa - - op = expression.op - - # Polars panics on nulls from pandas objects in timezone-aware - # datetimes for certain ops. Convert to timezone-naive temporarily - # to avoid this issue. - # TODO(tswast): Remove workaround when - # https://github.com/pola-rs/polars/issues/27862 has been fixed. - is_problematic_op = type(op) in ( - date_ops.YearOp, - date_ops.QuarterOp, - date_ops.MonthOp, - date_ops.DayOp, - date_ops.IsoWeekOp, - ) - - if is_problematic_op and len(expression.inputs) == 1: - input_expr = expression.inputs[0] - if ( - input_expr.is_resolved - and isinstance(input_expr.output_type, pd.ArrowDtype) - and isinstance( - input_expr.output_type.pyarrow_dtype, pa.TimestampType - ) - and input_expr.output_type.pyarrow_dtype.tz is not None - ): - tz_str = input_expr.output_type.pyarrow_dtype.tz - if tz_str == "UTC": - dummy_tz = datetime.timezone.utc - else: - try: - from zoneinfo import ZoneInfo - - dummy_tz = ZoneInfo(tz_str) # type: ignore - except Exception: - dummy_tz = datetime.timezone.utc - - dummy_val = datetime.datetime(1970, 1, 1, tzinfo=dummy_tz) - - compiled_input = self.compile_expression(input_expr) - filled_input = compiled_input.fill_null(dummy_val) - compiled_op_with_fill = self.compile_op(op, filled_input) - - return ( - pl.when(compiled_input.is_null()) - .then(None) - .otherwise(compiled_op_with_fill) - ) - # TODO: Complete the implementation + op = expression.op args = tuple(map(self.compile_expression, expression.inputs)) return self.compile_op(op, *args) @@ -372,28 +310,6 @@ def _( ) -> pl.Expr: return pl.when(condition).then(original).otherwise(otherwise) - @compile_op.register(gen_ops.CoerceToBoolOp) - def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: - assert isinstance(op, gen_ops.CoerceToBoolOp) - from_type = self._expr_types.get(id(input)) - if from_type is None: - return input.cast(pl.Boolean).fill_null(False) - - if from_type == bigframes.dtypes.BOOL_DTYPE: - res = input - elif bigframes.dtypes.is_numeric(from_type): - res = input != 0 - elif from_type == bigframes.dtypes.BYTES_DTYPE: - res = input.bin.size() > 0 - elif bigframes.dtypes.is_string_like(from_type): - res = input.str.len_chars() > 0 - elif bigframes.dtypes.is_array_like(from_type): - res = input.list.len() > 0 - else: - res = input.is_not_null() - - return res.fill_null(False) - @compile_op.register(gen_ops.AsTypeOp) def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: assert isinstance(op, gen_ops.AsTypeOp) @@ -454,54 +370,6 @@ def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: else: return pl.any_horizontal(*(input.str.ends_with(pat) for pat in op.pat)) - @compile_op.register(string_ops.CapitalizeOp) - def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: - assert isinstance(op, string_ops.CapitalizeOp) - return ( - input.str.slice(0, 1).str.to_uppercase() - + input.str.slice(1).str.to_lowercase() - ) - - @compile_op.register(string_ops.IsAlnumOp) - def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: - assert isinstance(op, string_ops.IsAlnumOp) - return input.str.contains(r"^[a-zA-Z0-9]+$") - - @compile_op.register(string_ops.IsAlphaOp) - def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: - assert isinstance(op, string_ops.IsAlphaOp) - return input.str.contains(r"^[a-zA-Z]+$") - - @compile_op.register(string_ops.IsDigitOp) - def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: - assert isinstance(op, string_ops.IsDigitOp) - return input.str.contains(r"^[0-9]+$") - - @compile_op.register(string_ops.IsSpaceOp) - def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: - assert isinstance(op, string_ops.IsSpaceOp) - return input.str.contains(r"^\s+$") - - @compile_op.register(string_ops.IsDecimalOp) - def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: - assert isinstance(op, string_ops.IsDecimalOp) - return input.str.contains(r"^[0-9]+$") - - @compile_op.register(string_ops.IsNumericOp) - def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: - assert isinstance(op, string_ops.IsNumericOp) - return input.str.contains(r"^[0-9]+$") - - @compile_op.register(string_ops.IsLowerOp) - def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: - assert isinstance(op, string_ops.IsLowerOp) - return input.str.contains(r"[a-z]") & ~input.str.contains(r"[A-Z]") - - @compile_op.register(string_ops.IsUpperOp) - def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: - assert isinstance(op, string_ops.IsUpperOp) - return input.str.contains(r"[A-Z]") & ~input.str.contains(r"[a-z]") - @compile_op.register(freq_ops.FloorDtOp) def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: assert isinstance(op, freq_ops.FloorDtOp) @@ -559,65 +427,10 @@ def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: ) @compile_op.register(json_ops.JSONDecode) - def _(self, op: json_ops.JSONDecode, input: pl.Expr) -> pl.Expr: + def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr: assert isinstance(op, json_ops.JSONDecode) return input.str.json_decode(_DTYPE_MAPPING[op.to_type]) - @compile_op.register(json_ops.ToJSON) - def _(self, op: json_ops.ToJSON, input: pl.Expr) -> pl.Expr: - from_type = self._expr_types.get(id(input)) - if from_type in ( - bigframes.dtypes.STRING_DTYPE, - bigframes.dtypes.JSON_DTYPE, - ): - return input - else: - return input.cast(pl.String()) - - @compile_op.register(json_ops.ToJSONString) - def _(self, op: json_ops.ToJSONString, input: pl.Expr) -> pl.Expr: - from_type = self._expr_types.get(id(input)) - - def preprocess_binary( - expr: pl.Expr, dtype: bigframes.dtypes.ExpressionType - ) -> pl.Expr: - if dtype == bigframes.dtypes.BYTES_DTYPE: - return expr.bin.encode("base64") - if bigframes.dtypes.is_struct_like(dtype): - fields = bigframes.dtypes.get_struct_fields(dtype) - return pl.struct( - *[ - preprocess_binary( - expr.struct.field(name), field_type - ).alias(name) - for name, field_type in fields.items() - ] - ) - if bigframes.dtypes.is_array_like(dtype): - inner_type = bigframes.dtypes.get_array_inner_type(dtype) - return expr.list.eval(preprocess_binary(pl.element(), inner_type)) - return expr - - preprocessed = preprocess_binary(input, from_type) - - if bigframes.dtypes.is_struct_like(from_type): - result = preprocessed.struct.json_encode() - elif from_type == bigframes.dtypes.INT_DTYPE: - result = preprocessed.cast(pl.String) - elif from_type == bigframes.dtypes.BOOL_DTYPE: - result = ( - pl.when(preprocessed) - .then(pl.lit("true")) - .otherwise(pl.lit("false")) - ) - elif from_type == bigframes.dtypes.BYTES_DTYPE: - result = pl.lit('"') + preprocessed + pl.lit('"') - else: - wrapped = pl.struct(value=preprocessed).struct.json_encode() - result = wrapped.str.slice(9, wrapped.str.len_chars() - 10) - - return pl.when(input.is_null()).then(pl.lit("null")).otherwise(result) - @compile_op.register(arr_ops.ToArrayOp) def _(self, op: ops.ToArrayOp, *inputs: pl.Expr) -> pl.Expr: return pl.concat_list(*inputs) @@ -648,36 +461,6 @@ def _(self, op: ops.ArrayReduceOp, input: pl.Expr) -> pl.Expr: f"Haven't implemented array aggregation: {op.aggregation}" ) - @compile_op.register(struct_ops.StructOp) - def _(self, op: struct_ops.StructOp, *inputs: pl.Expr) -> pl.Expr: - return pl.struct(**{col: inp for col, inp in zip(op.column_names, inputs)}) # type: ignore - - @compile_op.register(struct_ops.StructFieldOp) - def _(self, op: struct_ops.StructFieldOp, *inputs: pl.Expr) -> pl.Expr: - return inputs[0].struct[op.name_or_index] - - @compile_op.register(remote_function_ops.PythonUdfOp) - def _(self, op: ops.PythonUdfOp, *inputs: pl.Expr) -> pl.Expr: - from bigframes.functions import function_template - - code = op.function_def.code.to_callable() - if op.function_def.signature.is_row_processor: - - def handler(py_struct): - args = list(py_struct.values()) - series_arg = function_template.get_pd_series(args[0]) - return code(series_arg, *args[1:]) - else: - - def handler(py_struct): - return code(*(field for field in py_struct.values())) - - return pl.struct(*inputs).map_elements( - handler, - return_dtype=_bigframes_dtype_to_polars_dtype(op.output_type()), - skip_nulls=False, - ) - @dataclasses.dataclass(frozen=True) class PolarsAggregateCompiler: scalar_compiler = PolarsExpressionCompiler() diff --git a/bigframes/core/compile/polars/lowering.py b/bigframes/core/compile/polars/lowering.py index 5b3d9154b73..bf617d6879f 100644 --- a/bigframes/core/compile/polars/lowering.py +++ b/bigframes/core/compile/polars/lowering.py @@ -18,7 +18,6 @@ import numpy as np import pandas as pd -import bigframes.operations as ops from bigframes import dtypes from bigframes.core import bigframe_node, expression from bigframes.core.rewrite import op_lowering @@ -26,9 +25,11 @@ comparison_ops, datetime_ops, generic_ops, + json_ops, numeric_ops, string_ops, ) +import bigframes.operations as ops # TODO: Would be more precise to actually have separate op set for polars ops (where they diverge from the original ops) @@ -173,10 +174,12 @@ def lower(self, expr: expression.OpExpression) -> expression.Expression: divisor.output_type ): # exact same as floordiv impl for timedelta - numeric_result = ops.div_op.as_expr( + numeric_result = ops.floordiv_op.as_expr( ops.AsTypeOp(to_type=dtypes.INT_DTYPE).as_expr(dividend), divisor ) - return _numeric_to_timedelta(numeric_result) + int_result = ops.AsTypeOp(to_type=dtypes.INT_DTYPE).as_expr(numeric_result) + return ops.AsTypeOp(to_type=dtypes.TIMEDELTA_DTYPE).as_expr(int_result) + if ( dividend.output_type == dtypes.BOOL_DTYPE and divisor.output_type == dtypes.BOOL_DTYPE @@ -223,10 +226,11 @@ def lower(self, expr: expression.OpExpression) -> expression.Expression: divisor.output_type ): # this is pretty fragile as zero will break it, and must fit back into int - numeric_result = ops.div_op.as_expr( + numeric_result = expr.op.as_expr( ops.AsTypeOp(to_type=dtypes.INT_DTYPE).as_expr(dividend), divisor ) - return _numeric_to_timedelta(numeric_result) + int_result = ops.AsTypeOp(to_type=dtypes.INT_DTYPE).as_expr(numeric_result) + return ops.AsTypeOp(to_type=dtypes.TIMEDELTA_DTYPE).as_expr(int_result) if dividend.output_type == dtypes.BOOL_DTYPE: dividend = ops.AsTypeOp(to_type=dtypes.INT_DTYPE).as_expr(dividend) @@ -315,32 +319,6 @@ def lower(self, expr: expression.OpExpression) -> expression.Expression: return expr -class LowerCeilOp(op_lowering.OpLoweringRule): - @property - def op(self) -> type[ops.ScalarOp]: - return numeric_ops.CeilOp - - def lower(self, expr: expression.OpExpression) -> expression.Expression: - assert isinstance(expr.op, numeric_ops.CeilOp) - arg = expr.children[0] - if arg.output_type in (dtypes.INT_DTYPE, dtypes.BOOL_DTYPE): - return expr.op.as_expr(ops.AsTypeOp(dtypes.FLOAT_DTYPE).as_expr(arg)) - return expr - - -class LowerFloorOp(op_lowering.OpLoweringRule): - @property - def op(self) -> type[ops.ScalarOp]: - return numeric_ops.FloorOp - - def lower(self, expr: expression.OpExpression) -> expression.Expression: - assert isinstance(expr.op, numeric_ops.FloorOp) - arg = expr.children[0] - if arg.output_type in (dtypes.INT_DTYPE, dtypes.BOOL_DTYPE): - return expr.op.as_expr(ops.AsTypeOp(dtypes.FLOAT_DTYPE).as_expr(arg)) - return expr - - class LowerIsinOp(op_lowering.OpLoweringRule): @property def op(self) -> type[ops.ScalarOp]: @@ -411,6 +389,9 @@ def _coerce_comparables( def _lower_cast(cast_op: ops.AsTypeOp, arg: expression.Expression): if arg.output_type == cast_op.to_type: return arg + + if arg.output_type == dtypes.JSON_DTYPE: + return json_ops.JSONDecode(cast_op.to_type).as_expr(arg) if ( arg.output_type == dtypes.STRING_DTYPE and cast_op.to_type == dtypes.DATETIME_DTYPE @@ -484,21 +465,8 @@ def _lower_cast(cast_op: ops.AsTypeOp, arg: expression.Expression): LowerInvertOp(), LowerIsinOp(), LowerLenOp(), - LowerCeilOp(), - LowerFloorOp(), ) def lower_ops_to_polars(root: bigframe_node.BigFrameNode) -> bigframe_node.BigFrameNode: return op_lowering.lower_ops(root, rules=POLARS_LOWERING_RULES) - - -def _numeric_to_timedelta(expr: expression.Expression) -> expression.Expression: - """rounding logic used for emulating timedelta ops""" - rounded_value = ops.where_op.as_expr( - ops.floor_op.as_expr(expr), - ops.gt_op.as_expr(expr, expression.const(0)), - ops.ceil_op.as_expr(expr), - ) - int_value = ops.AsTypeOp(to_type=dtypes.INT_DTYPE).as_expr(rounded_value) - return ops.AsTypeOp(to_type=dtypes.TIMEDELTA_DTYPE).as_expr(int_value) diff --git a/bigframes/core/compile/polars/operations/array_ops.py b/bigframes/core/compile/polars/operations/array_ops.py deleted file mode 100644 index 1f2960471db..00000000000 --- a/bigframes/core/compile/polars/operations/array_ops.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -BigFrames -> Polars compilation for the operations in bigframes.operations.array_ops. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -import bigframes.core.compile.polars.compiler as polars_compiler -import bigframes.dtypes as dtypes -from bigframes.operations import generic_ops - -if TYPE_CHECKING: - import polars as pl - - -@polars_compiler.register_op(generic_ops.GetItemOp) -def getitem_op_impl( - compiler: polars_compiler.PolarsExpressionCompiler, - op: generic_ops.GetItemOp, # type: ignore - input: pl.Expr, -) -> pl.Expr: - input_type = compiler._expr_types.get(id(input)) - if input_type is not None and dtypes.is_struct_like(input_type): - if isinstance(op.key, str): - return input.struct.field(op.key) - else: - raise NotImplementedError( - "Referencing a struct field by number not implemented in polars compiler." - ) - elif input_type is not None and dtypes.is_string_like(input_type): - return input.str.slice(op.key, 1) - else: - return input.list.get(op.key) - - -@polars_compiler.register_op(generic_ops.DynamicGetItemOp) -def dynamic_getitem_op_impl( - compiler: polars_compiler.PolarsExpressionCompiler, - op: generic_ops.DynamicGetItemOp, # type: ignore - left: pl.Expr, - right: pl.Expr, -) -> pl.Expr: - left_type = compiler._expr_types.get(id(left)) - if left_type is not None and dtypes.is_string_like(left_type): - return left.str.slice(right, 1) - else: - return left.list.get(right) diff --git a/bigframes/core/compile/sqlglot/__init__.py b/bigframes/core/compile/sqlglot/__init__.py index fa515e4f15a..9e3f123807f 100644 --- a/bigframes/core/compile/sqlglot/__init__.py +++ b/bigframes/core/compile/sqlglot/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. from __future__ import annotations +from bigframes.core.compile.sqlglot.compiler import compile_sql import bigframes.core.compile.sqlglot.expressions.ai_ops # noqa: F401 import bigframes.core.compile.sqlglot.expressions.array_ops # noqa: F401 import bigframes.core.compile.sqlglot.expressions.blob_ops # noqa: F401 @@ -27,6 +28,5 @@ import bigframes.core.compile.sqlglot.expressions.string_ops # noqa: F401 import bigframes.core.compile.sqlglot.expressions.struct_ops # noqa: F401 import bigframes.core.compile.sqlglot.expressions.timedelta_ops # noqa: F401 -from bigframes.core.compile.sqlglot.compiler import compile_sql __all__ = ["compile_sql"] diff --git a/bigframes/core/compile/sqlglot/aggregate_compiler.py b/bigframes/core/compile/sqlglot/aggregate_compiler.py index c0781e260c6..d2ecfaade01 100644 --- a/bigframes/core/compile/sqlglot/aggregate_compiler.py +++ b/bigframes/core/compile/sqlglot/aggregate_compiler.py @@ -15,7 +15,6 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler from bigframes.core import agg_expressions, window_spec from bigframes.core.compile.sqlglot.aggregations import ( binary_compiler, @@ -24,6 +23,7 @@ unary_compiler, ) from bigframes.core.compile.sqlglot.expressions import typed_expr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler def compile_aggregate( @@ -35,7 +35,7 @@ def compile_aggregate( return nullary_compiler.compile(aggregate.op) if isinstance(aggregate, agg_expressions.UnaryAggregation): column = typed_expr.TypedExpr( - expression_compiler.expression_compiler.compile_expression(aggregate.arg), + scalar_compiler.scalar_op_compiler.compile_expression(aggregate.arg), aggregate.arg.output_type, ) if not aggregate.op.order_independent: @@ -46,11 +46,11 @@ def compile_aggregate( return unary_compiler.compile(aggregate.op, column) elif isinstance(aggregate, agg_expressions.BinaryAggregation): left = typed_expr.TypedExpr( - expression_compiler.expression_compiler.compile_expression(aggregate.left), + scalar_compiler.scalar_op_compiler.compile_expression(aggregate.left), aggregate.left.output_type, ) right = typed_expr.TypedExpr( - expression_compiler.expression_compiler.compile_expression(aggregate.right), + scalar_compiler.scalar_op_compiler.compile_expression(aggregate.right), aggregate.right.output_type, ) return binary_compiler.compile(aggregate.op, left, right) @@ -66,9 +66,11 @@ def compile_analytic( return nullary_compiler.compile(aggregate.op, window) if isinstance(aggregate, agg_expressions.UnaryAggregation): column = typed_expr.TypedExpr( - expression_compiler.expression_compiler.compile_expression(aggregate.arg), + scalar_compiler.scalar_op_compiler.compile_expression(aggregate.arg), aggregate.arg.output_type, ) return unary_compiler.compile(aggregate.op, column, window) + elif isinstance(aggregate, agg_expressions.BinaryAggregation): + raise NotImplementedError("binary analytic operations not yet supported") else: raise ValueError(f"Unexpected analytic operation: {aggregate}") diff --git a/bigframes/core/compile/sqlglot/aggregations/binary_compiler.py b/bigframes/core/compile/sqlglot/aggregations/binary_compiler.py index df8437fe76f..51ff1ceeccb 100644 --- a/bigframes/core/compile/sqlglot/aggregations/binary_compiler.py +++ b/bigframes/core/compile/sqlglot/aggregations/binary_compiler.py @@ -18,10 +18,10 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.aggregations.op_registration as reg -import bigframes.core.compile.sqlglot.expressions.typed_expr as typed_expr from bigframes.core import window_spec +import bigframes.core.compile.sqlglot.aggregations.op_registration as reg from bigframes.core.compile.sqlglot.aggregations.windows import apply_window_if_present +import bigframes.core.compile.sqlglot.expressions.typed_expr as typed_expr from bigframes.operations import aggregations as agg_ops BINARY_OP_REGISTRATION = reg.OpRegistration() @@ -33,8 +33,6 @@ def compile( right: typed_expr.TypedExpr, window: typing.Optional[window_spec.WindowSpec] = None, ) -> sge.Expression: - if op.order_independent and (window is not None) and window.is_unbounded: - window = window.without_order() return BINARY_OP_REGISTRATION[op](op, left, right, window=window) diff --git a/bigframes/core/compile/sqlglot/aggregations/nullary_compiler.py b/bigframes/core/compile/sqlglot/aggregations/nullary_compiler.py index f2f1978908f..58ab7ec5135 100644 --- a/bigframes/core/compile/sqlglot/aggregations/nullary_compiler.py +++ b/bigframes/core/compile/sqlglot/aggregations/nullary_compiler.py @@ -18,8 +18,8 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.aggregations.op_registration as reg from bigframes.core import window_spec +import bigframes.core.compile.sqlglot.aggregations.op_registration as reg from bigframes.core.compile.sqlglot.aggregations.windows import apply_window_if_present from bigframes.operations import aggregations as agg_ops @@ -30,8 +30,6 @@ def compile( op: agg_ops.WindowOp, window: typing.Optional[window_spec.WindowSpec] = None, ) -> sge.Expression: - if op.order_independent and (window is not None) and window.is_unbounded: - window = window.without_order() return NULLARY_OP_REGISTRATION[op](op, window=window) diff --git a/bigframes/core/compile/sqlglot/aggregations/unary_compiler.py b/bigframes/core/compile/sqlglot/aggregations/unary_compiler.py index 417faef34aa..647e86d28ac 100644 --- a/bigframes/core/compile/sqlglot/aggregations/unary_compiler.py +++ b/bigframes/core/compile/sqlglot/aggregations/unary_compiler.py @@ -16,17 +16,16 @@ import typing -import bigframes_vendored.sqlglot as sg import bigframes_vendored.sqlglot.expressions as sge import pandas as pd -import bigframes.core.compile.sqlglot.aggregations.op_registration as reg -import bigframes.core.compile.sqlglot.expressions.typed_expr as typed_expr from bigframes import dtypes from bigframes.core import window_spec -from bigframes.core.compile.sqlglot import sql +import bigframes.core.compile.sqlglot.aggregations.op_registration as reg from bigframes.core.compile.sqlglot.aggregations.windows import apply_window_if_present from bigframes.core.compile.sqlglot.expressions import constants +import bigframes.core.compile.sqlglot.expressions.typed_expr as typed_expr +import bigframes.core.compile.sqlglot.sqlglot_ir as ir from bigframes.operations import aggregations as agg_ops UNARY_OP_REGISTRATION = reg.OpRegistration() @@ -37,8 +36,6 @@ def compile( column: typed_expr.TypedExpr, window: typing.Optional[window_spec.WindowSpec] = None, ) -> sge.Expression: - if op.order_independent and (window is not None) and window.is_unbounded: - window = window.without_order() return UNARY_OP_REGISTRATION[op](op, column, window=window) @@ -157,9 +154,9 @@ def _cut_ops_w_int_bins( for this_bin in range(bins): value: sge.Expression if op.labels is False: - value = sql.literal(this_bin, dtypes.INT_DTYPE) + value = ir._literal(this_bin, dtypes.INT_DTYPE) elif isinstance(op.labels, typing.Iterable): - value = sql.literal(list(op.labels)[this_bin], dtypes.STRING_DTYPE) + value = ir._literal(list(op.labels)[this_bin], dtypes.STRING_DTYPE) else: left_adj: sge.Expression = ( adj if this_bin == 0 and op.right else sge.convert(0) @@ -190,10 +187,7 @@ def _cut_ops_w_int_bins( condition: sge.Expression if this_bin == bins - 1: - condition = sge.Is( - this=sge.paren(column.expr, copy=False), - expression=sg.not_(sge.Null(), copy=False), - ) + condition = sge.Is(this=column.expr, expression=sge.Not(this=sge.Null())) else: if op.right: condition = sge.LTE( @@ -217,10 +211,10 @@ def _cut_ops_w_intervals( ) -> sge.Case: case_expr = sge.Case() for this_bin, interval in enumerate(bins): - left: sge.Expression = sql.literal( + left: sge.Expression = ir._literal( interval[0], dtypes.infer_literal_type(interval[0]) ) - right: sge.Expression = sql.literal( + right: sge.Expression = ir._literal( interval[1], dtypes.infer_literal_type(interval[1]) ) condition: sge.Expression @@ -237,9 +231,9 @@ def _cut_ops_w_intervals( value: sge.Expression if op.labels is False: - value = sql.literal(this_bin, dtypes.INT_DTYPE) + value = ir._literal(this_bin, dtypes.INT_DTYPE) elif isinstance(op.labels, typing.Iterable): - value = sql.literal(list(op.labels)[this_bin], dtypes.STRING_DTYPE) + value = ir._literal(list(op.labels)[this_bin], dtypes.STRING_DTYPE) else: if op.right: left_identifier = sge.Identifier(this="left_exclusive", quoted=True) @@ -533,7 +527,7 @@ def _( else: result = apply_window_if_present(result, window) - if op.should_floor_result or column.dtype == dtypes.TIMEDELTA_DTYPE: + if op.should_floor_result: result = sge.Cast(this=sge.func("FLOOR", result), to="INT64") return result @@ -609,7 +603,7 @@ def _( # Will be null if all inputs are null. Pandas defaults to zero sum though. zero = pd.to_timedelta(0) if column.dtype == dtypes.TIMEDELTA_DTYPE else 0 - return sge.func("IFNULL", expr, sql.literal(zero, column.dtype)) + return sge.func("IFNULL", expr, ir._literal(zero, column.dtype)) @UNARY_OP_REGISTRATION.register(agg_ops.VarOp) diff --git a/bigframes/core/compile/sqlglot/aggregations/windows.py b/bigframes/core/compile/sqlglot/aggregations/windows.py index cb4a2e70edd..9c327885850 100644 --- a/bigframes/core/compile/sqlglot/aggregations/windows.py +++ b/bigframes/core/compile/sqlglot/aggregations/windows.py @@ -17,11 +17,11 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler +from bigframes.core import utils, window_spec +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler import bigframes.core.expression as ex import bigframes.core.ordering as ordering_spec import bigframes.dtypes as dtypes -from bigframes.core import utils, window_spec def apply_window_if_present( @@ -116,7 +116,7 @@ def get_window_order_by( order_by = [] for ordering_spec_item in ordering: - expr = expression_compiler.expression_compiler.compile_expression( + expr = scalar_compiler.scalar_op_compiler.compile_expression( ordering_spec_item.scalar_expression ) desc = not ordering_spec_item.direction.is_ascending @@ -191,15 +191,15 @@ def _get_window_bounds( def _compile_group_by_key(key: ex.Expression) -> sge.Expression: - expr = expression_compiler.expression_compiler.compile_expression(key) + expr = scalar_compiler.scalar_op_compiler.compile_expression(key) # The group_by keys has been rewritten by bind_schema_to_node - assert key.is_scalar_expr and key.is_resolved + assert isinstance(key, ex.ResolvedDerefOp) # Some types need to be converted to another type to enable groupby - if key.output_type == dtypes.FLOAT_DTYPE: + if key.dtype == dtypes.FLOAT_DTYPE: expr = sge.Cast(this=expr, to="STRING") - elif key.output_type == dtypes.GEO_DTYPE: + elif key.dtype == dtypes.GEO_DTYPE: expr = sge.func("ST_ASBINARY", expr) - elif key.output_type == dtypes.JSON_DTYPE: + elif key.dtype == dtypes.JSON_DTYPE: expr = sge.func("TO_JSON_STRING", expr) return expr diff --git a/bigframes/core/compile/sqlglot/compiler.py b/bigframes/core/compile/sqlglot/compiler.py index 393d10ec825..e77370892c0 100644 --- a/bigframes/core/compile/sqlglot/compiler.py +++ b/bigframes/core/compile/sqlglot/compiler.py @@ -19,30 +19,30 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.aggregate_compiler as aggregate_compiler -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler -import bigframes.core.ordering as bf_ordering -from bigframes import dtypes from bigframes.core import ( + agg_expressions, expression, guid, identifiers, nodes, pyarrow_utils, rewrite, - sql_nodes, ) from bigframes.core.compile import configs -from bigframes.core.compile.sqlglot import sql, sqlglot_ir +import bigframes.core.compile.sqlglot.aggregate_compiler as aggregate_compiler from bigframes.core.compile.sqlglot.aggregations import windows from bigframes.core.compile.sqlglot.expressions import typed_expr -from bigframes.core.logging import data_types as data_type_logger +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler +import bigframes.core.compile.sqlglot.sqlglot_ir as ir +import bigframes.core.ordering as bf_ordering from bigframes.core.rewrite import schema_binding def compile_sql(request: configs.CompileRequest) -> configs.CompileResult: """Compiles a BigFrameNode according to the request into SQL using SQLGlot.""" + # Generator for unique identifiers. + uid_gen = guid.SequentialUIDGenerator() output_names = tuple((expression.DerefOp(id), id.sql) for id in request.node.ids) result_node = nodes.ResultNode( request.node, @@ -53,10 +53,7 @@ def compile_sql(request: configs.CompileRequest) -> configs.CompileResult: # Can only pullup slice if we are doing ORDER BY in outermost SELECT # Need to do this before replacing unsupported ops, as that will rewrite slice ops result_node = rewrite.pull_up_limits(result_node) - result_node = typing.cast(nodes.ResultNode, _replace_unsupported_ops(result_node)) - result_node = typing.cast( - nodes.ResultNode, result_node.bottom_up(rewrite.simplify_join) - ) + result_node = _replace_unsupported_ops(result_node) # prune before pulling up order to avoid unnnecessary row_number() ops result_node = typing.cast(nodes.ResultNode, rewrite.column_pruning(result_node)) result_node = rewrite.defer_order( @@ -64,33 +61,29 @@ def compile_sql(request: configs.CompileRequest) -> configs.CompileResult: ) if request.sort_rows: result_node = typing.cast(nodes.ResultNode, rewrite.column_pruning(result_node)) - encoded_type_refs = data_type_logger.encode_type_refs(result_node) - # TODO: Extract CTEs earlier - result_node = typing.cast(nodes.ResultNode, rewrite.extract_ctes(result_node)) - sql = _compile_result_node(result_node) + result_node = _remap_variables(result_node, uid_gen) + result_node = typing.cast( + nodes.ResultNode, rewrite.defer_selection(result_node) + ) + sql = _compile_result_node(result_node, uid_gen) return configs.CompileResult( - sql, - result_node.schema.to_bigquery(), - result_node.order_by, - encoded_type_refs, + sql, result_node.schema.to_bigquery(), result_node.order_by ) ordering: typing.Optional[bf_ordering.RowOrdering] = result_node.order_by result_node = dataclasses.replace(result_node, order_by=None) result_node = typing.cast(nodes.ResultNode, rewrite.column_pruning(result_node)) - encoded_type_refs = data_type_logger.encode_type_refs(result_node) - # TODO: Extract CTEs earlier - result_node = typing.cast(nodes.ResultNode, rewrite.extract_ctes(result_node)) - sql = _compile_result_node(result_node) + + result_node = _remap_variables(result_node, uid_gen) + result_node = typing.cast(nodes.ResultNode, rewrite.defer_selection(result_node)) + sql = _compile_result_node(result_node, uid_gen) # Return the ordering iff no extra columns are needed to define the row order if ordering is not None: output_order = ( ordering if ordering.referenced_columns.issubset(result_node.ids) else None ) assert (not request.materialize_all_order_keys) or (output_order is not None) - return configs.CompileResult( - sql, result_node.schema.to_bigquery(), output_order, encoded_type_refs - ) + return configs.CompileResult(sql, result_node.schema.to_bigquery(), output_order) def _remap_variables( @@ -101,38 +94,50 @@ def _remap_variables( result_node, _ = rewrite.remap_variables( node, map(identifiers.ColumnId, uid_gen.get_uid_stream("bfcol_")) ) - result_node.validate_tree() return typing.cast(nodes.ResultNode, result_node) -def _compile_result_node(root: nodes.ResultNode) -> str: - # Create UIDs to standardize variable names and ensure consistent compilation - # of nodes using the same generator. - uid_gen = guid.SequentialUIDGenerator() - root = _remap_variables(root, uid_gen) - # Remap variables creates too mayn new - # root = rewrite.select_pullup(root, prefer_source_names=False) - root = typing.cast(nodes.ResultNode, rewrite.defer_selection(root)) - +def _compile_result_node( + root: nodes.ResultNode, uid_gen: guid.SequentialUIDGenerator +) -> str: # Have to bind schema as the final step before compilation. - # Probably, should defer even further root = typing.cast(nodes.ResultNode, schema_binding.bind_schema_to_tree(root)) + selected_cols: tuple[tuple[str, sge.Expression], ...] = tuple( + (name, scalar_compiler.scalar_op_compiler.compile_expression(ref)) + for ref, name in root.output_cols + ) + sqlglot_ir = compile_node(root.child, uid_gen).select(selected_cols) + + if root.order_by is not None: + ordering_cols = tuple( + sge.Ordered( + this=scalar_compiler.scalar_op_compiler.compile_expression( + ordering.scalar_expression + ), + desc=ordering.direction.is_ascending is False, + nulls_first=ordering.na_last is False, + ) + for ordering in root.order_by.all_ordering_columns + ) + sqlglot_ir = sqlglot_ir.order_by(ordering_cols) + + if root.limit is not None: + sqlglot_ir = sqlglot_ir.limit(root.limit) - # TODO: Bake all IDs in tree, stop passing uid_gen to emitters - sqlglot_ir_obj = compile_node(rewrite.as_sql_nodes(root, uid_gen), uid_gen) - return sqlglot_ir_obj.sql + return sqlglot_ir.sql +@functools.lru_cache(maxsize=5000) def compile_node( node: nodes.BigFrameNode, uid_gen: guid.SequentialUIDGenerator -) -> sqlglot_ir.SQLGlotIR: +) -> ir.SQLGlotIR: """Compiles the given BigFrameNode from bottem-up into SQLGlotIR.""" - bf_to_sqlglot: dict[nodes.BigFrameNode, sqlglot_ir.SQLGlotIR] = {} - child_results: tuple[sqlglot_ir.SQLGlotIR, ...] = () + bf_to_sqlglot: dict[nodes.BigFrameNode, ir.SQLGlotIR] = {} + child_results: tuple[ir.SQLGlotIR, ...] = () for current_node in list(node.iter_nodes_topo()): if current_node.child_nodes == (): # For leaf node, generates a dumpy child to pass the UID generator. - child_results = tuple([sqlglot_ir.SQLGlotIR.empty(uid_gen=uid_gen)]) + child_results = tuple([ir.SQLGlotIR(uid_gen=uid_gen)]) else: # Child nodes should have been compiled in the reverse topological order. child_results = tuple( @@ -146,49 +151,14 @@ def compile_node( @functools.singledispatch def _compile_node( - node: nodes.BigFrameNode, *compiled_children: sqlglot_ir.SQLGlotIR -) -> sqlglot_ir.SQLGlotIR: + node: nodes.BigFrameNode, *compiled_children: ir.SQLGlotIR +) -> ir.SQLGlotIR: """Defines transformation but isn't cached, always use compile_node instead""" raise ValueError(f"Can't compile unrecognized node: {node}") @_compile_node.register -def compile_sql_select(node: sql_nodes.SqlSelectNode, child: sqlglot_ir.SQLGlotIR): - ordering_cols = tuple( - sge.Ordered( - this=expression_compiler.expression_compiler.compile_expression( - ordering.scalar_expression - ), - desc=ordering.direction.is_ascending is False, - nulls_first=ordering.na_last is False, - ) - for ordering in node.sorting - ) - - projected_cols: tuple[tuple[str, sge.Expression], ...] = tuple() - if not node.is_star_selection: - projected_cols = tuple( - ( - cdef.id.sql, - expression_compiler.expression_compiler.compile_expression( - cdef.expression - ), - ) - for cdef in node.selections - ) - - sge_predicates = tuple( - expression_compiler.expression_compiler.compile_expression(expression) - for expression in node.predicates - ) - - return child.select(projected_cols, sge_predicates, ordering_cols, node.limit) - - -@_compile_node.register -def compile_readlocal( - node: nodes.ReadLocalNode, child: sqlglot_ir.SQLGlotIR -) -> sqlglot_ir.SQLGlotIR: +def compile_readlocal(node: nodes.ReadLocalNode, child: ir.SQLGlotIR) -> ir.SQLGlotIR: pa_table = node.local_data_source.data pa_table = pa_table.select([item.source_id for item in node.scan_list.items]) pa_table = pa_table.rename_columns([item.id.sql for item in node.scan_list.items]) @@ -197,42 +167,64 @@ def compile_readlocal( if offsets: pa_table = pyarrow_utils.append_offsets(pa_table, offsets) - return sqlglot_ir.SQLGlotIR.from_pyarrow( - pa_table, node.schema, uid_gen=child.uid_gen - ) + return ir.SQLGlotIR.from_pyarrow(pa_table, node.schema, uid_gen=child.uid_gen) @_compile_node.register -def compile_readtable(node: sql_nodes.SqlDataSource, child: sqlglot_ir.SQLGlotIR): - table_obj = node.source.table - columns = () if node.is_star_selection else node.source.schema.names - return sqlglot_ir.SQLGlotIR.from_table( - table_obj.project_id, - table_obj.dataset_id, - table_obj.table_id, +def compile_readtable(node: nodes.ReadTableNode, child: ir.SQLGlotIR): + table = node.source.table + return ir.SQLGlotIR.from_table( + table.project_id, + table.dataset_id, + table.table_id, + col_names=[col.source_id for col in node.scan_list.items], + alias_names=[col.id.sql for col in node.scan_list.items], uid_gen=child.uid_gen, - columns=columns, sql_predicate=node.source.sql_predicate, system_time=node.source.at_time, ) +@_compile_node.register +def compile_selection(node: nodes.SelectionNode, child: ir.SQLGlotIR) -> ir.SQLGlotIR: + selected_cols: tuple[tuple[str, sge.Expression], ...] = tuple( + (id.sql, scalar_compiler.scalar_op_compiler.compile_expression(expr)) + for expr, id in node.input_output_pairs + ) + return child.select(selected_cols) + + +@_compile_node.register +def compile_projection(node: nodes.ProjectionNode, child: ir.SQLGlotIR) -> ir.SQLGlotIR: + projected_cols: tuple[tuple[str, sge.Expression], ...] = tuple( + (id.sql, scalar_compiler.scalar_op_compiler.compile_expression(expr)) + for expr, id in node.assignments + ) + return child.project(projected_cols) + + +@_compile_node.register +def compile_filter(node: nodes.FilterNode, child: ir.SQLGlotIR) -> ir.SQLGlotIR: + condition = scalar_compiler.scalar_op_compiler.compile_expression(node.predicate) + return child.filter(tuple([condition])) + + @_compile_node.register def compile_join( - node: nodes.JoinNode, left: sqlglot_ir.SQLGlotIR, right: sqlglot_ir.SQLGlotIR -) -> sqlglot_ir.SQLGlotIR: + node: nodes.JoinNode, left: ir.SQLGlotIR, right: ir.SQLGlotIR +) -> ir.SQLGlotIR: conditions = tuple( ( typed_expr.TypedExpr( - expression_compiler.expression_compiler.compile_expression(left_expr), - left_expr.output_type, + scalar_compiler.scalar_op_compiler.compile_expression(left), + left.output_type, ), typed_expr.TypedExpr( - expression_compiler.expression_compiler.compile_expression(right_expr), - right_expr.output_type, + scalar_compiler.scalar_op_compiler.compile_expression(right), + right.output_type, ), ) - for left_expr, right_expr in node.conditions + for left, right in node.conditions ) return left.join( @@ -245,16 +237,16 @@ def compile_join( @_compile_node.register def compile_isin_join( - node: nodes.InNode, left: sqlglot_ir.SQLGlotIR, right: sqlglot_ir.SQLGlotIR -) -> sqlglot_ir.SQLGlotIR: + node: nodes.InNode, left: ir.SQLGlotIR, right: ir.SQLGlotIR +) -> ir.SQLGlotIR: right_field = node.right_child.fields[0] conditions = ( typed_expr.TypedExpr( - expression_compiler.expression_compiler.compile_expression(node.left_col), + scalar_compiler.scalar_op_compiler.compile_expression(node.left_col), node.left_col.output_type, ), typed_expr.TypedExpr( - expression_compiler.expression_compiler.compile_expression( + scalar_compiler.scalar_op_compiler.compile_expression( expression.DerefOp(right_field.id) ), right_field.dtype, @@ -270,81 +262,34 @@ def compile_isin_join( @_compile_node.register -def compile_cte_ref_node(node: sql_nodes.SqlCteRefNode, child: sqlglot_ir.SQLGlotIR): - return sqlglot_ir.SQLGlotIR.from_cte_ref( - node.cte_name, - uid_gen=child.uid_gen, - ) - - -@_compile_node.register -def compile_with_ctes_node( - node: sql_nodes.SqlWithCtesNode, - child: sqlglot_ir.SQLGlotIR, - *ctes: sqlglot_ir.SQLGlotIR, -): - return child.with_ctes(tuple(zip(node.cte_names, ctes))) - - -@_compile_node.register -def compile_concat( - node: nodes.ConcatNode, *children: sqlglot_ir.SQLGlotIR -) -> sqlglot_ir.SQLGlotIR: +def compile_concat(node: nodes.ConcatNode, *children: ir.SQLGlotIR) -> ir.SQLGlotIR: assert len(children) >= 1 uid_gen = children[0].uid_gen - # BigQuery `UNION` query takes the column names from the first `SELECT` clause. - default_output_ids = [field.id.sql for field in node.child_nodes[0].fields] - output_aliases = [ - (default_output_id, output_id.sql) - for default_output_id, output_id in zip(default_output_ids, node.output_ids) - ] - - return sqlglot_ir.SQLGlotIR.from_union( - [child.expr.as_select_all() for child in children], - output_aliases=output_aliases, + output_ids = [id.sql for id in node.output_ids] + return ir.SQLGlotIR.from_union( + [child.expr for child in children], + output_ids=output_ids, uid_gen=uid_gen, ) @_compile_node.register -def compile_explode( - node: nodes.ExplodeNode, child: sqlglot_ir.SQLGlotIR -) -> sqlglot_ir.SQLGlotIR: +def compile_explode(node: nodes.ExplodeNode, child: ir.SQLGlotIR) -> ir.SQLGlotIR: offsets_col = node.offsets_col.sql if (node.offsets_col is not None) else None columns = tuple(ref.id.sql for ref in node.column_ids) return child.explode(columns, offsets_col) -@_compile_node.register -def compile_fromrange( - node: nodes.FromRangeNode, start: sqlglot_ir.SQLGlotIR, end: sqlglot_ir.SQLGlotIR -) -> sqlglot_ir.SQLGlotIR: - start_col_id = node.start.fields[0].id - end_col_id = node.end.fields[0].id - - start_expr = expression_compiler.expression_compiler.compile_expression( - expression.DerefOp(start_col_id) - ) - end_expr = expression_compiler.expression_compiler.compile_expression( - expression.DerefOp(end_col_id) - ) - step_expr = sql.literal(node.step, dtypes.INT_DTYPE) - - return start.resample(end, node.output_id.sql, start_expr, end_expr, step_expr) - - @_compile_node.register def compile_random_sample( - node: nodes.RandomSampleNode, child: sqlglot_ir.SQLGlotIR -) -> sqlglot_ir.SQLGlotIR: + node: nodes.RandomSampleNode, child: ir.SQLGlotIR +) -> ir.SQLGlotIR: return child.sample(node.fraction) @_compile_node.register -def compile_aggregate( - node: nodes.AggregateNode, child: sqlglot_ir.SQLGlotIR -) -> sqlglot_ir.SQLGlotIR: +def compile_aggregate(node: nodes.AggregateNode, child: ir.SQLGlotIR) -> ir.SQLGlotIR: # The BigQuery ordered aggregation cannot support for NULL FIRST/LAST, # so we need to add extra expressions to enforce the null ordering. ordering_cols = windows.get_window_order_by(node.order_by, override_null_order=True) @@ -358,7 +303,7 @@ def compile_aggregate( for agg, id in node.aggregations ) by_cols: tuple[sge.Expression, ...] = tuple( - expression_compiler.expression_compiler.compile_expression(by_col) + scalar_compiler.scalar_op_compiler.compile_expression(by_col) for by_col in node.by_column_ids ) @@ -371,8 +316,80 @@ def compile_aggregate( return child.aggregate(aggregations, by_cols, tuple(dropna_cols)) +@_compile_node.register +def compile_window(node: nodes.WindowOpNode, child: ir.SQLGlotIR) -> ir.SQLGlotIR: + window_spec = node.window_spec + result = child + for cdef in node.agg_exprs: + assert isinstance(cdef.expression, agg_expressions.Aggregation) + if cdef.expression.op.order_independent and window_spec.is_unbounded: + # notably percentile_cont does not support ordering clause + window_spec = window_spec.without_order() + + window_op = aggregate_compiler.compile_analytic(cdef.expression, window_spec) + + inputs: tuple[sge.Expression, ...] = tuple( + scalar_compiler.scalar_op_compiler.compile_expression( + expression.DerefOp(column) + ) + for column in cdef.expression.column_references + ) + + clauses: list[tuple[sge.Expression, sge.Expression]] = [] + if window_spec.min_periods and len(inputs) > 0: + if not cdef.expression.op.nulls_count_for_min_values: + # Most operations do not count NULL values towards min_periods + not_null_columns = [ + sge.Not(this=sge.Is(this=column, expression=sge.Null())) + for column in inputs + ] + # All inputs must be non-null for observation to count + if not not_null_columns: + is_observation_expr: sge.Expression = sge.convert(True) + else: + is_observation_expr = not_null_columns[0] + for expr in not_null_columns[1:]: + is_observation_expr = sge.And( + this=is_observation_expr, expression=expr + ) + is_observation = ir._cast(is_observation_expr, "INT64") + observation_count = windows.apply_window_if_present( + sge.func("SUM", is_observation), window_spec + ) + observation_count = sge.func( + "COALESCE", observation_count, sge.convert(0) + ) + else: + # Operations like count treat even NULLs as valid observations + # for the sake of min_periods notnull is just used to convert + # null values to non-null (FALSE) values to be counted. + is_observation = ir._cast( + sge.Not(this=sge.Is(this=inputs[0], expression=sge.Null())), + "INT64", + ) + observation_count = windows.apply_window_if_present( + sge.func("COUNT", is_observation), window_spec + ) + + clauses.append( + ( + observation_count < sge.convert(window_spec.min_periods), + sge.Null(), + ) + ) + if clauses: + when_expressions = [sge.When(this=cond, true=res) for cond, res in clauses] + window_op = sge.Case(ifs=when_expressions, default=window_op) + + # TODO: check if we can directly window the expression. + result = result.window( + window_op=window_op, + output_column_id=cdef.id.sql, + ) + return result + + def _replace_unsupported_ops(node: nodes.BigFrameNode): node = nodes.bottom_up(node, rewrite.rewrite_slice) node = nodes.bottom_up(node, rewrite.rewrite_range_rolling) - node = nodes.bottom_up(node, rewrite.lower_udfs) return node diff --git a/bigframes/core/compile/sqlglot/expressions/ai_ops.py b/bigframes/core/compile/sqlglot/expressions/ai_ops.py index d092f662f0f..748f15b8677 100644 --- a/bigframes/core/compile/sqlglot/expressions/ai_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/ai_ops.py @@ -15,17 +15,14 @@ from __future__ import annotations from dataclasses import asdict -from typing import Any import bigframes_vendored.sqlglot.expressions as sge from bigframes import operations as ops -from bigframes.core.compile.sqlglot import expression_compiler +from bigframes.core.compile.sqlglot import scalar_compiler from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr -register_nary_op = expression_compiler.expression_compiler.register_nary_op -register_binary_op = expression_compiler.expression_compiler.register_binary_op -register_unary_op = expression_compiler.expression_compiler.register_unary_op +register_nary_op = scalar_compiler.scalar_op_compiler.register_nary_op @register_nary_op(ops.AIGenerate, pass_op=True) @@ -56,13 +53,6 @@ def _(*exprs: TypedExpr, op: ops.AIGenerateDouble) -> sge.Expression: return sge.func("AI.GENERATE_DOUBLE", *args) -@register_unary_op(ops.AIEmbed, pass_op=True) -def _(expr: TypedExpr, op: ops.AIEmbed) -> sge.Expression: - args: list[Any] = [expr.expr] + _construct_named_args(op) - - return sge.func("AI.EMBED", *args) - - @register_nary_op(ops.AIIf, pass_op=True) def _(*exprs: TypedExpr, op: ops.AIIf) -> sge.Expression: args = [_construct_prompt(exprs, op.prompt_context)] + _construct_named_args(op) @@ -72,8 +62,14 @@ def _(*exprs: TypedExpr, op: ops.AIIf) -> sge.Expression: @register_nary_op(ops.AIClassify, pass_op=True) def _(*exprs: TypedExpr, op: ops.AIClassify) -> sge.Expression: + category_literals = [sge.Literal.string(cat) for cat in op.categories] + categories_arg = sge.Kwarg( + this="categories", expression=sge.array(*category_literals) + ) + args = [ _construct_prompt(exprs, op.prompt_context, param_name="input"), + categories_arg, ] + _construct_named_args(op) return sge.func("AI.CLASSIFY", *args) @@ -86,16 +82,6 @@ def _(*exprs: TypedExpr, op: ops.AIScore) -> sge.Expression: return sge.func("AI.SCORE", *args) -@register_binary_op(ops.AISimilarity, pass_op=True) -def _(content1: TypedExpr, content2: TypedExpr, op: ops.AISimilarity) -> sge.Expression: - args = [ - sge.Kwarg(this="content1", expression=content1.expr), - sge.Kwarg(this="content2", expression=content2.expr), - ] + _construct_named_args(op) - - return sge.func("AI.SIMILARITY", *args) - - def _construct_prompt( exprs: tuple[TypedExpr, ...], prompt_context: tuple[str | None, ...], @@ -111,51 +97,52 @@ def _construct_prompt( else: prompt.append(sge.Literal.string(elem)) - # Need Struct rather than tuple syntax, as tuple syntax is ambiguous for single arg - return sge.Kwarg(this=param_name, expression=sge.Struct(expressions=prompt)) + return sge.Kwarg(this=param_name, expression=sge.Tuple(expressions=prompt)) -def _construct_named_args(op: ops.ScalarOp) -> list[sge.Kwarg]: +def _construct_named_args(op: ops.NaryOp) -> list[sge.Kwarg]: args = [] op_args = asdict(op) - for field, value in op_args.items(): - if value is None or field == "prompt_context": - continue + connection_id = op_args.get("connection_id", None) + if connection_id is not None: + args.append( + sge.Kwarg( + this="connection_id", expression=sge.Literal.string(connection_id) + ) + ) + + endpoit = op_args.get("endpoint", None) + if endpoit is not None: + args.append(sge.Kwarg(this="endpoint", expression=sge.Literal.string(endpoit))) - if field == "categories": - category_literals = [sge.Literal.string(cat) for cat in value] - categories_arg = sge.Kwarg( - this="categories", expression=sge.array(*category_literals) + request_type = op_args.get("request_type", None) + if request_type is not None: + args.append( + sge.Kwarg( + this="request_type", expression=sge.Literal.string(request_type.upper()) ) - args.append(categories_arg) - elif field == "model_params": - # model_params is a JSON string, so we need to use the JSON function to pass it as a named argument. - args.append( - sge.Kwarg( - this="model_params", - # sge.JSON requires the SQLGlot version to be at least 25.18.0 - # PARSE_JSON won't work as the function requires a JSON literal. - expression=sge.JSON(this=sge.Literal.string(value)), - ) + ) + + model_params = op_args.get("model_params", None) + if model_params is not None: + args.append( + sge.Kwarg( + this="model_params", + # sge.JSON requires the SQLGlot version to be at least 25.18.0 + # PARSE_JSON won't work as the function requires a JSON literal. + expression=sge.JSON(this=sge.Literal.string(model_params)), ) - elif field == "examples": - example_expressions = [] - for key, val in value: - if isinstance(val, (list, tuple)): - val_expr: sge.Array | sge.Literal = sge.array( - *[sge.Literal.string(v) for v in val] - ) - else: - val_expr = sge.Literal.string(val) - example_expressions.append( - sge.Tuple(expressions=[sge.Literal.string(key), val_expr]) - ) - args.append( - sge.Kwarg(this=field, expression=sge.array(*example_expressions)) + ) + + output_schema = op_args.get("output_schema", None) + if output_schema is not None: + args.append( + sge.Kwarg( + this="output_schema", + expression=sge.Literal.string(output_schema), ) - else: - args.append(sge.Kwarg(this=field, expression=sge.convert(value))) + ) return args diff --git a/bigframes/core/compile/sqlglot/expressions/array_ops.py b/bigframes/core/compile/sqlglot/expressions/array_ops.py index 56ffbf24cb3..e83a6ea99a9 100644 --- a/bigframes/core/compile/sqlglot/expressions/array_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/array_ops.py @@ -18,72 +18,31 @@ import bigframes_vendored.sqlglot as sg import bigframes_vendored.sqlglot.expressions as sge -import pandas as pd -import pyarrow as pa -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler -import bigframes.dtypes as dtypes from bigframes import operations as ops from bigframes.core.compile.sqlglot.expressions.string_ops import ( string_index, string_slice, ) from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler +import bigframes.dtypes as dtypes -register_unary_op = expression_compiler.expression_compiler.register_unary_op -register_nary_op = expression_compiler.expression_compiler.register_nary_op - - -@register_unary_op(ops.GetItemOp, pass_op=True) -def _(expr: TypedExpr, op: ops.GetItemOp) -> sge.Expression: - if dtypes.is_struct_like(expr.dtype): - if isinstance(op.key, str): - name = op.key - else: - pa_type = typing.cast(pd.ArrowDtype, expr.dtype) - pa_struct_type = typing.cast(pa.StructType, pa_type.pyarrow_dtype) - name = pa_struct_type.field(op.key).name +register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op +register_nary_op = scalar_compiler.scalar_op_compiler.register_nary_op - return sge.Column( - this=sge.to_identifier(name, quoted=True), - catalog=expr.expr, - ) - elif dtypes.is_array_like(expr.dtype): - return sge.Bracket( - this=expr.expr, - expressions=[sge.convert(op.key)], - safe=True, - offset=False, - ) - elif expr.dtype == dtypes.STRING_DTYPE: - return string_index(expr, typing.cast(int, op.key)) - else: - raise TypeError(f"Cannot subscript input of type {expr.dtype}") +@register_unary_op(ops.ArrayIndexOp, pass_op=True) +def _(expr: TypedExpr, op: ops.ArrayIndexOp) -> sge.Expression: + if expr.dtype == dtypes.STRING_DTYPE: + return string_index(expr, op.index) -@register_nary_op(ops.DynamicGetItemOp) # type: ignore[arg-type] -def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if dtypes.is_array_like(left.dtype): - return sge.Bracket( - this=left.expr, - expressions=[right.expr], - safe=True, - offset=False, - ) - elif left.dtype == dtypes.STRING_DTYPE: - start_expr = sge.Add(this=right.expr, expression=sge.convert(1)) - sub_str = sge.Substring( - this=left.expr, - start=start_expr, - length=sge.convert(1), - ) - return sge.If( - this=sge.NEQ(this=sub_str, expression=sge.convert("")), - true=sub_str, - false=sge.Null(), - ) - else: - raise TypeError(f"Cannot dynamically subscript input of type {left.dtype}") + return sge.Bracket( + this=expr.expr, + expressions=[sge.convert(op.index)], + safe=True, + offset=False, + ) @register_unary_op(ops.ArrayReduceOp, pass_op=True) @@ -114,28 +73,6 @@ def _(expr: TypedExpr, op: ops.ArrayReduceOp) -> sge.Expression: ) -@register_unary_op(ops.ArrayMapOp, pass_op=True) -def _(expr: TypedExpr, op: ops.ArrayMapOp) -> sge.Expression: - sub_expr = sg.to_identifier("bf_arr_map_uid") - sub_type = dtypes.get_array_inner_type(expr.dtype) - - # TODO: Expression should be provided instead of invoking compiler manually - map_expr = expression_compiler.expression_compiler.compile_row_op( - op.map_op, (TypedExpr(sub_expr, sub_type),) - ) - - return sge.array( - sge.select(map_expr) - .from_( - sge.Unnest( - expressions=[expr.expr], - alias=sge.TableAlias(columns=[sub_expr]), - ) - ) - .subquery() - ) - - @register_unary_op(ops.ArraySliceOp, pass_op=True) def _(expr: TypedExpr, op: ops.ArraySliceOp) -> sge.Expression: if expr.dtype == dtypes.STRING_DTYPE: @@ -146,7 +83,7 @@ def _(expr: TypedExpr, op: ops.ArraySliceOp) -> sge.Expression: @register_unary_op(ops.ArrayToStringOp, pass_op=True) def _(expr: TypedExpr, op: ops.ArrayToStringOp) -> sge.Expression: - return sge.ArrayToString(this=expr.expr, expression=sge.convert(op.delimiter)) + return sge.ArrayToString(this=expr.expr, expression=f"'{op.delimiter}'") @register_nary_op(ops.ToArrayOp) @@ -168,6 +105,31 @@ def _coerce_bool_to_int(typed_expr: TypedExpr) -> sge.Expression: return typed_expr.expr +def _string_slice(expr: TypedExpr, op: ops.ArraySliceOp) -> sge.Expression: + # local name for each element in the array + el = sg.to_identifier("el") + # local name for the index in the array + slice_idx = sg.to_identifier("slice_idx") + + conditions: typing.List[sge.Predicate] = [slice_idx >= op.start] + if op.stop is not None: + conditions.append(slice_idx < op.stop) + + selected_elements = ( + sge.select(el) + .from_( + sge.Unnest( + expressions=[expr.expr], + alias=sge.TableAlias(columns=[el]), + offset=slice_idx, + ) + ) + .where(*conditions) + ) + + return sge.array(selected_elements) + + def _array_slice(expr: TypedExpr, op: ops.ArraySliceOp) -> sge.Expression: # local name for each element in the array el = sg.to_identifier("el") diff --git a/bigframes/core/compile/sqlglot/expressions/blob_ops.py b/bigframes/core/compile/sqlglot/expressions/blob_ops.py index 01b4f7a1617..3105cd8e303 100644 --- a/bigframes/core/compile/sqlglot/expressions/blob_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/blob_ops.py @@ -16,12 +16,12 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler from bigframes import operations as ops from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler -register_unary_op = expression_compiler.expression_compiler.register_unary_op -register_binary_op = expression_compiler.expression_compiler.register_binary_op +register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op +register_binary_op = scalar_compiler.scalar_op_compiler.register_binary_op @register_unary_op(ops.obj_fetch_metadata_op) diff --git a/bigframes/core/compile/sqlglot/expressions/bool_ops.py b/bigframes/core/compile/sqlglot/expressions/bool_ops.py index 7e31646b295..6fee3f4278e 100644 --- a/bigframes/core/compile/sqlglot/expressions/bool_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/bool_ops.py @@ -16,13 +16,12 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler from bigframes import dtypes from bigframes import operations as ops -from bigframes.core.compile.sqlglot import sql from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler -register_binary_op = expression_compiler.expression_compiler.register_binary_op +register_binary_op = scalar_compiler.scalar_op_compiler.register_binary_op @register_binary_op(ops.and_op) @@ -30,10 +29,10 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: # For AND, when we encounter a NULL value, we only know when the result is FALSE, # otherwise the result is unknown (NULL). See: truth table at # https://en.wikibooks.org/wiki/Structured_Query_Language/NULLs_and_the_Three_Valued_Logic#AND,_OR - if sql.is_null_literal(left.expr): + if left.expr == sge.null(): condition = sge.EQ(this=right.expr, expression=sge.convert(False)) return sge.If(this=condition, true=right.expr, false=sge.null()) - if sql.is_null_literal(right.expr): + if right.expr == sge.null(): condition = sge.EQ(this=left.expr, expression=sge.convert(False)) return sge.If(this=condition, true=left.expr, false=sge.null()) @@ -47,10 +46,10 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: # For OR, when we encounter a NULL value, we only know when the result is TRUE, # otherwise the result is unknown (NULL). See: truth table at # https://en.wikibooks.org/wiki/Structured_Query_Language/NULLs_and_the_Three_Valued_Logic#AND,_OR - if sql.is_null_literal(left.expr): + if left.expr == sge.null(): condition = sge.EQ(this=right.expr, expression=sge.convert(True)) return sge.If(this=condition, true=right.expr, false=sge.null()) - if sql.is_null_literal(right.expr): + if right.expr == sge.null(): condition = sge.EQ(this=left.expr, expression=sge.convert(True)) return sge.If(this=condition, true=left.expr, false=sge.null()) @@ -65,12 +64,12 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: # maintains the boolean data type. left_expr = left.expr left_dtype = left.dtype - if sql.is_null_literal(left_expr): + if left_expr == sge.null(): left_expr = sge.Cast(this=sge.convert(None), to="BOOLEAN") left_dtype = dtypes.BOOL_DTYPE right_expr = right.expr right_dtype = right.dtype - if sql.is_null_literal(right_expr): + if right_expr == sge.null(): right_expr = sge.Cast(this=sge.convert(None), to="BOOLEAN") right_dtype = dtypes.BOOL_DTYPE diff --git a/bigframes/core/compile/sqlglot/expressions/common.py b/bigframes/core/compile/sqlglot/expressions/common.py deleted file mode 100644 index 067ca070edf..00000000000 --- a/bigframes/core/compile/sqlglot/expressions/common.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import bigframes_vendored.sqlglot.expressions as sge - - -def round_towards_zero(expr: sge.Expression): - """ - Round a float value to to an integer, always rounding towards zero. - - This is used to handle duration/timedelta emulation mostly. - """ - return sge.Cast( - this=sge.If( - this=sge.GT(this=expr, expression=sge.convert(0)), - true=sge.Floor(this=expr), - false=sge.Ceil(this=expr), - ), - to="INT64", - ) diff --git a/bigframes/core/compile/sqlglot/expressions/comparison_ops.py b/bigframes/core/compile/sqlglot/expressions/comparison_ops.py index a3331ce6fb5..8c201f6a068 100644 --- a/bigframes/core/compile/sqlglot/expressions/comparison_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/comparison_ops.py @@ -20,60 +20,53 @@ import bigframes_vendored.sqlglot.expressions as sge import pandas as pd -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler from bigframes import dtypes from bigframes import operations as ops -from bigframes.core.compile.sqlglot import sql +from bigframes.core.compile.sqlglot import sqlglot_ir from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler -register_unary_op = expression_compiler.expression_compiler.register_unary_op -register_binary_op = expression_compiler.expression_compiler.register_binary_op +register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op +register_binary_op = scalar_compiler.scalar_op_compiler.register_binary_op @register_unary_op(ops.IsInOp, pass_op=True) def _(expr: TypedExpr, op: ops.IsInOp) -> sge.Expression: values = [] - # bools are not comparable to non-bools in SQL, so we need to cast the expression to INT64 if the values contain non-bools. - must_upcast_bools = dtypes.is_numeric(expr.dtype, include_bool=False) or any( - dtypes.is_numeric(dtypes.bigframes_type(type(value)), include_bool=False) - for value in op.values - if not _is_null(value) - ) + is_numeric_expr = dtypes.is_numeric(expr.dtype, include_bool=False) for value in op.values: if _is_null(value): continue dtype = dtypes.bigframes_type(type(value)) - if dtypes.can_compare(expr.dtype, dtype): - if must_upcast_bools and dtype == dtypes.BOOL_DTYPE: - value = int(value) - values.append(sql.literal(value)) - - sg_lexpr: sge.Expression = expr.expr - if expr.dtype == dtypes.BOOL_DTYPE and must_upcast_bools: - sg_lexpr = sge.cast(expr.expr, "INT64") + if ( + expr.dtype == dtype + or is_numeric_expr + and dtypes.is_numeric(dtype, include_bool=False) + ): + values.append(sge.convert(value)) if op.match_nulls: contains_nulls = any(_is_null(value) for value in op.values) if contains_nulls: if len(values) == 0: - return sge.Is(this=sg_lexpr, expression=sge.Null()) - return sge.Is(this=sg_lexpr, expression=sge.Null()) | sge.In( - this=sg_lexpr, expressions=values + return sge.Is(this=expr.expr, expression=sge.Null()) + return sge.Is(this=expr.expr, expression=sge.Null()) | sge.In( + this=expr.expr, expressions=values ) if len(values) == 0: return sge.convert(False) return sge.func( - "COALESCE", sge.In(this=sg_lexpr, expressions=values), sge.convert(False) + "COALESCE", sge.In(this=expr.expr, expressions=values), sge.convert(False) ) @register_binary_op(ops.eq_op) def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if sql.is_null_literal(left.expr): + if sqlglot_ir._is_null_literal(left.expr): return sge.Is(this=right.expr, expression=sge.Null()) - if sql.is_null_literal(right.expr): + if sqlglot_ir._is_null_literal(right.expr): return sge.Is(this=left.expr, expression=sge.Null()) left_expr = _coerce_bool_to_int(left) right_expr = _coerce_bool_to_int(right) @@ -102,7 +95,7 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: @register_binary_op(ops.ge_op) def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if sql.is_null_literal(left.expr) or sql.is_null_literal(right.expr): + if left.expr == sge.null() or right.expr == sge.null(): return sge.null() left_expr = _coerce_bool_to_int(left) @@ -112,7 +105,7 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: @register_binary_op(ops.gt_op) def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if sql.is_null_literal(left.expr) or sql.is_null_literal(right.expr): + if left.expr == sge.null() or right.expr == sge.null(): return sge.null() left_expr = _coerce_bool_to_int(left) @@ -122,7 +115,7 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: @register_binary_op(ops.lt_op) def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if sql.is_null_literal(left.expr) or sql.is_null_literal(right.expr): + if left.expr == sge.null() or right.expr == sge.null(): return sge.null() left_expr = _coerce_bool_to_int(left) @@ -132,7 +125,7 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: @register_binary_op(ops.le_op) def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if sql.is_null_literal(left.expr) or sql.is_null_literal(right.expr): + if left.expr == sge.null() or right.expr == sge.null(): return sge.null() left_expr = _coerce_bool_to_int(left) @@ -152,12 +145,12 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: @register_binary_op(ops.ne_op) def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if sql.is_null_literal(left.expr): + if sqlglot_ir._is_null_literal(left.expr): return sge.Is( this=sge.paren(right.expr, copy=False), expression=sg.not_(sge.Null(), copy=False), ) - if sql.is_null_literal(right.expr): + if sqlglot_ir._is_null_literal(right.expr): return sge.Is( this=sge.paren(left.expr, copy=False), expression=sg.not_(sge.Null(), copy=False), diff --git a/bigframes/core/compile/sqlglot/expressions/date_ops.py b/bigframes/core/compile/sqlglot/expressions/date_ops.py index 2410926887b..3de7c4b23b1 100644 --- a/bigframes/core/compile/sqlglot/expressions/date_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/date_ops.py @@ -16,11 +16,11 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler from bigframes import operations as ops from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler -register_unary_op = expression_compiler.expression_compiler.register_unary_op +register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op @register_unary_op(ops.date_op) diff --git a/bigframes/core/compile/sqlglot/expressions/datetime_ops.py b/bigframes/core/compile/sqlglot/expressions/datetime_ops.py index 399b3062273..7f3e8135af1 100644 --- a/bigframes/core/compile/sqlglot/expressions/datetime_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/datetime_ops.py @@ -16,15 +16,15 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler from bigframes import dtypes from bigframes import operations as ops from bigframes.core.compile.constants import UNIT_TO_US_CONVERSION_FACTORS from bigframes.core.compile.sqlglot import sqlglot_types from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler -register_unary_op = expression_compiler.expression_compiler.register_unary_op -register_binary_op = expression_compiler.expression_compiler.register_binary_op +register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op +register_binary_op = scalar_compiler.scalar_op_compiler.register_binary_op @register_binary_op(ops.DatetimeToIntegerLabelOp, pass_op=True) @@ -125,7 +125,7 @@ def _datetime_to_integer_label_non_fixed_frequency( expression=sge.convert(1), ), ) - elif rule_code in ("M", "ME"): # Monthly + elif rule_code == "ME": # Monthly x_int = sge.Paren( # type: ignore this=sge.Add( this=sge.Mul( @@ -182,7 +182,7 @@ def _datetime_to_integer_label_non_fixed_frequency( expression=sge.convert(1), ), ) - elif rule_code in ("Q-DEC", "QE-DEC"): # Quarterly + elif rule_code == "QE-DEC": # Quarterly x_int = sge.Paren( # type: ignore this=sge.Add( this=sge.Mul( @@ -239,7 +239,7 @@ def _datetime_to_integer_label_non_fixed_frequency( expression=sge.convert(1), ), ) - elif rule_code in ("A-DEC", "Y-DEC", "YE-DEC"): # Yearly + elif rule_code == "YE-DEC": # Yearly x_int = sge.Extract(this=sge.Identifier(this="YEAR"), expression=x.expr) first = sge.Extract(this=sge.Identifier(this="YEAR"), expression=y.expr) return sge.Case( @@ -364,23 +364,14 @@ def _(expr: TypedExpr) -> sge.Expression: def _(expr: TypedExpr, op: ops.ToDatetimeOp) -> sge.Expression: if op.format: result = expr.expr - if expr.dtype == dtypes.STRING_DTYPE: - return sge.TryCast(this=result, to="DATETIME") - else: + if expr.dtype != dtypes.STRING_DTYPE: result = sge.Cast(this=result, to="STRING") - result = sge.func( - "PARSE_TIMESTAMP", sge.convert(op.format), result, sge.convert("UTC") - ) - return sge.Cast(this=result, to="DATETIME") - - if expr.dtype == dtypes.TIMESTAMP_DTYPE: - return sge.func("DATETIME", expr.expr, sge.convert("UTC")) + result = sge.func( + "PARSE_TIMESTAMP", sge.convert(op.format), result, sge.convert("UTC") + ) + return sge.Cast(this=result, to="DATETIME") - if expr.dtype in ( - dtypes.STRING_DTYPE, - dtypes.DATETIME_DTYPE, - dtypes.DATE_DTYPE, - ): + if expr.dtype == dtypes.STRING_DTYPE: return sge.TryCast(this=expr.expr, to="DATETIME") value = expr.expr @@ -389,10 +380,9 @@ def _(expr: TypedExpr, op: ops.ToDatetimeOp) -> sge.Expression: if factor != 1: value = sge.Mul(this=value, expression=sge.convert(factor)) value = sge.func("TRUNC", value) - return sge.func( - "DATETIME", - sge.func("TIMESTAMP_MICROS", sge.Cast(this=value, to="INT64")), - sge.convert("UTC"), + return sge.Cast( + this=sge.func("TIMESTAMP_MICROS", sge.Cast(this=value, to="INT64")), + to="DATETIME", ) @@ -403,15 +393,10 @@ def _(expr: TypedExpr, op: ops.ToTimestampOp) -> sge.Expression: if expr.dtype != dtypes.STRING_DTYPE: result = sge.Cast(this=result, to="STRING") return sge.func( - "PARSE_TIMESTAMP", sge.convert(op.format), result, sge.convert("UTC") + "PARSE_TIMESTAMP", sge.convert(op.format), expr.expr, sge.convert("UTC") ) - if expr.dtype in ( - dtypes.STRING_DTYPE, - dtypes.DATETIME_DTYPE, - dtypes.TIMESTAMP_DTYPE, - dtypes.DATE_DTYPE, - ): + if expr.dtype == dtypes.STRING_DTYPE: return sge.func("TIMESTAMP", expr.expr) value = expr.expr @@ -686,49 +671,3 @@ def _integer_label_to_datetime_op_yearly_freq( this=next_month_date, expression=sge.Interval(this=one, unit="DAY") ) return sge.Cast(this=x_label, to=sqlglot_types.from_bigframes_dtype(y.dtype)) - - -@register_binary_op(ops.timestamp_add_op) -def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - return sge.TimestampAdd( - this=left.expr, expression=right.expr, unit=sge.Var(this="MICROSECOND") - ) - - -@register_binary_op(ops.timestamp_sub_op) -def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - return sge.TimestampSub( - this=left.expr, expression=right.expr, unit=sge.Var(this="MICROSECOND") - ) - - -@register_binary_op(ops.timestamp_diff_op) -def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - return sge.TimestampDiff( - this=left.expr, expression=right.expr, unit=sge.Var(this="MICROSECOND") - ) - - -@register_binary_op(ops.date_add_op) -def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - left_expr = sge.Cast(this=left.expr, to="TIMESTAMP") - return sge.TimestampAdd( - this=left_expr, expression=right.expr, unit=sge.Var(this="MICROSECOND") - ) - - -@register_binary_op(ops.date_sub_op) -def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - left_expr = sge.Cast(this=left.expr, to="TIMESTAMP") - return sge.TimestampSub( - this=left_expr, expression=right.expr, unit=sge.Var(this="MICROSECOND") - ) - - -@register_binary_op(ops.date_diff_op) -def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - diff = sge.DateDiff(this=left.expr, expression=right.expr, unit=sge.Var(this="DAY")) - return sge.Mul( - this=diff, - expression=sge.convert(int(UNIT_TO_US_CONVERSION_FACTORS["d"])), - ) diff --git a/bigframes/core/compile/sqlglot/expressions/generic_ops.py b/bigframes/core/compile/sqlglot/expressions/generic_ops.py index 90c8270ae1d..2f486fc9d51 100644 --- a/bigframes/core/compile/sqlglot/expressions/generic_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/generic_ops.py @@ -17,16 +17,16 @@ import bigframes_vendored.sqlglot as sg import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler from bigframes import dtypes from bigframes import operations as ops -from bigframes.core.compile.sqlglot import sql, sqlglot_types +from bigframes.core.compile.sqlglot import sqlglot_ir, sqlglot_types from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler -register_unary_op = expression_compiler.expression_compiler.register_unary_op -register_binary_op = expression_compiler.expression_compiler.register_binary_op -register_nary_op = expression_compiler.expression_compiler.register_nary_op -register_ternary_op = expression_compiler.expression_compiler.register_ternary_op +register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op +register_binary_op = scalar_compiler.scalar_op_compiler.register_binary_op +register_nary_op = scalar_compiler.scalar_op_compiler.register_nary_op +register_ternary_op = scalar_compiler.scalar_op_compiler.register_ternary_op @register_unary_op(ops.AsTypeOp, pass_op=True) @@ -36,14 +36,20 @@ def _(expr: TypedExpr, op: ops.AsTypeOp) -> sge.Expression: sg_to_type = sqlglot_types.from_bigframes_dtype(to_type) sg_expr = expr.expr + if to_type == dtypes.JSON_DTYPE: + return _cast_to_json(expr, op) + + if from_type == dtypes.JSON_DTYPE: + return _cast_from_json(expr, op) + if to_type == dtypes.INT_DTYPE: result = _cast_to_int(expr, op) if result is not None: return result if to_type == dtypes.FLOAT_DTYPE and from_type == dtypes.BOOL_DTYPE: - sg_expr = sql.cast(sg_expr, "INT64", op.safe) - return sql.cast(sg_expr, sg_to_type, op.safe) + sg_expr = _cast(sg_expr, "INT64", op.safe) + return _cast(sg_expr, sg_to_type, op.safe) if to_type == dtypes.BOOL_DTYPE: if from_type == dtypes.BOOL_DTYPE: @@ -52,16 +58,16 @@ def _(expr: TypedExpr, op: ops.AsTypeOp) -> sge.Expression: return sge.NEQ(this=sg_expr, expression=sge.convert(0)) if to_type == dtypes.STRING_DTYPE: - sg_expr = sql.cast(sg_expr, sg_to_type, op.safe) + sg_expr = _cast(sg_expr, sg_to_type, op.safe) if from_type == dtypes.BOOL_DTYPE: sg_expr = sge.func("INITCAP", sg_expr) return sg_expr if dtypes.is_time_like(to_type) and from_type == dtypes.INT_DTYPE: sg_expr = sge.func("TIMESTAMP_MICROS", sg_expr) - return sql.cast(sg_expr, sg_to_type, op.safe) + return _cast(sg_expr, sg_to_type, op.safe) - return sql.cast(sg_expr, sg_to_type, op.safe) + return _cast(sg_expr, sg_to_type, op.safe) @register_unary_op(ops.hash_op) @@ -76,27 +82,6 @@ def _(expr: TypedExpr) -> sge.Expression: return sge.BitwiseNot(this=sge.paren(expr.expr)) -@register_nary_op(ops.GoogleSqlScalarOp, pass_op=True) -def _(*operands: TypedExpr, op: ops.GoogleSqlScalarOp) -> sge.Expression: - args: list[sge.Expression] = [] - for i, operand in enumerate(operands): - if i < len(op.args): - arg_spec = op.args[i] - else: - assert op.args[-1].is_vararg, ( - f"Too many arguments, for {op.sql_name}, expected {len(op.args)}" - ) - arg_spec = op.args[-1] - if operand.is_omitted: - assert arg_spec.optional, "Argument omitted, but not optional" - continue - elif arg_spec.arg_name: - args.append(sge.Kwarg(this=arg_spec.arg_name, expression=operand.expr)) - else: - args.append(operand.expr) - return sg.func(op.sql_name, *args) - - @register_nary_op(ops.SqlScalarOp, pass_op=True) def _(*operands: TypedExpr, op: ops.SqlScalarOp) -> sge.Expression: return sg.parse_one( @@ -109,7 +94,7 @@ def _(*operands: TypedExpr, op: ops.SqlScalarOp) -> sge.Expression: @register_unary_op(ops.isnull_op) def _(expr: TypedExpr) -> sge.Expression: - return sge.Is(this=sge.paren(expr.expr), expression=sge.Null()) + return sge.Is(this=expr.expr, expression=sge.Null()) @register_unary_op(ops.MapOp, pass_op=True) @@ -119,19 +104,17 @@ def _(expr: TypedExpr, op: ops.MapOp) -> sge.Expression: mappings = [ ( - sql.literal(key, dtypes.is_compatible(key, expr.dtype)), - sql.literal(value, dtypes.is_compatible(value, expr.dtype)), + sqlglot_ir._literal(key, dtypes.is_compatible(key, expr.dtype)), + sqlglot_ir._literal(value, dtypes.is_compatible(value, expr.dtype)), ) for key, value in op.mappings ] return sge.Case( ifs=[ sge.If( - this=( - sge.EQ(this=expr.expr, expression=key) - if not sql.is_null_literal(key) - else sge.Is(this=expr.expr, expression=sge.Null()) - ), + this=sge.EQ(this=expr.expr, expression=key) + if not sqlglot_ir._is_null_literal(key) + else sge.Is(this=expr.expr, expression=sge.Null()), true=value, ) for key, value in mappings @@ -142,32 +125,7 @@ def _(expr: TypedExpr, op: ops.MapOp) -> sge.Expression: @register_unary_op(ops.notnull_op) def _(expr: TypedExpr) -> sge.Expression: - return sge.Is( - this=sge.paren(expr.expr, copy=False), - expression=sg.not_(sge.Null(), copy=False), - ) - - -@register_unary_op(ops.coerce_to_bool_op) -def _(expr: TypedExpr) -> sge.Expression: - from_type = expr.dtype - sg_expr = expr.expr - - if from_type == dtypes.BOOL_DTYPE: - res = sg_expr - elif dtypes.is_numeric(from_type): - res = sge.NEQ(this=sg_expr, expression=sge.convert(0)) - elif dtypes.is_string_like(from_type): - res = sge.GT(this=sge.func("LENGTH", sg_expr), expression=sge.convert(0)) - elif dtypes.is_array_like(from_type): - res = sge.GT(this=sge.func("ARRAY_LENGTH", sg_expr), expression=sge.convert(0)) - else: - res = sge.Is( - this=sge.paren(sg_expr, copy=False), - expression=sg.not_(sge.Null(), copy=False), - ) - - return sge.Coalesce(this=res, expressions=[sge.convert(False)]) + return sge.Not(this=sge.Is(this=expr.expr, expression=sge.Null())) @register_ternary_op(ops.where_op) @@ -202,9 +160,33 @@ def _get_remote_function_name(op): ) -@register_nary_op(ops.RemoteFunctionOp, pass_op=True) -def _(*values: TypedExpr, op: ops.RemoteFunctionOp) -> sge.Expression: - return sge.func(_get_remote_function_name(op), *(value.expr for value in values)) +@register_unary_op(ops.RemoteFunctionOp, pass_op=True) +def _(expr: TypedExpr, op: ops.RemoteFunctionOp) -> sge.Expression: + func_name = _get_remote_function_name(op) + func = sge.func(func_name, expr.expr) + + if not op.apply_on_null: + return sge.If( + this=sge.Is(this=expr.expr, expression=sge.Null()), + true=expr.expr, + false=func, + ) + + return func + + +@register_binary_op(ops.BinaryRemoteFunctionOp, pass_op=True) +def _( + left: TypedExpr, right: TypedExpr, op: ops.BinaryRemoteFunctionOp +) -> sge.Expression: + func_name = _get_remote_function_name(op) + return sge.func(func_name, left.expr, right.expr) + + +@register_nary_op(ops.NaryRemoteFunctionOp, pass_op=True) +def _(*operands: TypedExpr, op: ops.NaryRemoteFunctionOp) -> sge.Expression: + func_name = _get_remote_function_name(op) + return sge.func(func_name, *(operand.expr for operand in operands)) @register_nary_op(ops.case_when_op) @@ -216,14 +198,12 @@ def _(*cases_and_outputs: TypedExpr) -> sge.Expression: ) if do_upcast_bool: result_values = tuple( - ( - TypedExpr( - sge.Cast(this=val.expr, to="INT64"), - dtypes.INT_DTYPE, - ) - if val.dtype == dtypes.BOOL_DTYPE - else val + TypedExpr( + sge.Cast(this=val.expr, to="INT64"), + dtypes.INT_DTYPE, ) + if val.dtype == dtypes.BOOL_DTYPE + else val for val in result_values ) @@ -267,6 +247,35 @@ def _(*values: TypedExpr) -> sge.Expression: # Helper functions +def _cast_to_json(expr: TypedExpr, op: ops.AsTypeOp) -> sge.Expression: + from_type = expr.dtype + sg_expr = expr.expr + + if from_type == dtypes.STRING_DTYPE: + func_name = "PARSE_JSON_IN_SAFE" if op.safe else "PARSE_JSON" + return sge.func(func_name, sg_expr) + if from_type in (dtypes.INT_DTYPE, dtypes.BOOL_DTYPE, dtypes.FLOAT_DTYPE): + sg_expr = sge.Cast(this=sg_expr, to="STRING") + return sge.func("PARSE_JSON", sg_expr) + raise TypeError(f"Cannot cast from {from_type} to {dtypes.JSON_DTYPE}") + + +def _cast_from_json(expr: TypedExpr, op: ops.AsTypeOp) -> sge.Expression: + to_type = op.to_type + sg_expr = expr.expr + func_name = "" + if to_type == dtypes.INT_DTYPE: + func_name = "INT64" + elif to_type == dtypes.FLOAT_DTYPE: + func_name = "FLOAT64" + elif to_type == dtypes.BOOL_DTYPE: + func_name = "BOOL" + elif to_type == dtypes.STRING_DTYPE: + func_name = "STRING" + if func_name: + func_name = "SAFE." + func_name if op.safe else func_name + return sge.func(func_name, sg_expr) + raise TypeError(f"Cannot cast from {dtypes.JSON_DTYPE} to {to_type}") def _cast_to_int(expr: TypedExpr, op: ops.AsTypeOp) -> sge.Expression | None: @@ -274,23 +283,30 @@ def _cast_to_int(expr: TypedExpr, op: ops.AsTypeOp) -> sge.Expression | None: sg_expr = expr.expr # Cannot cast DATETIME to INT directly so need to convert to TIMESTAMP first. if from_type == dtypes.DATETIME_DTYPE: - sg_expr = sql.cast(sg_expr, "TIMESTAMP", op.safe) + sg_expr = _cast(sg_expr, "TIMESTAMP", op.safe) return sge.func("UNIX_MICROS", sg_expr) if from_type == dtypes.TIMESTAMP_DTYPE: return sge.func("UNIX_MICROS", sg_expr) if from_type == dtypes.TIME_DTYPE: return sge.func( "TIME_DIFF", - sql.cast(sg_expr, "TIME", op.safe), + _cast(sg_expr, "TIME", op.safe), sge.convert("00:00:00"), "MICROSECOND", ) if from_type == dtypes.NUMERIC_DTYPE or from_type == dtypes.FLOAT_DTYPE: sg_expr = sge.func("TRUNC", sg_expr) - return sql.cast(sg_expr, "INT64", op.safe) + return _cast(sg_expr, "INT64", op.safe) return None +def _cast(expr: sge.Expression, to: str, safe: bool): + if safe: + return sge.TryCast(this=expr, to=to) + else: + return sge.Cast(this=expr, to=to) + + def _convert_to_nonnull_string_sqlglot(expr: TypedExpr) -> sge.Expression: col_type = expr.dtype sg_expr = expr.expr diff --git a/bigframes/core/compile/sqlglot/expressions/geo_ops.py b/bigframes/core/compile/sqlglot/expressions/geo_ops.py index 8c353988ae3..9c6ba33ea54 100644 --- a/bigframes/core/compile/sqlglot/expressions/geo_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/geo_ops.py @@ -16,12 +16,17 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler from bigframes import operations as ops from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler -register_unary_op = expression_compiler.expression_compiler.register_unary_op -register_binary_op = expression_compiler.expression_compiler.register_binary_op +register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op +register_binary_op = scalar_compiler.scalar_op_compiler.register_binary_op + + +@register_unary_op(ops.geo_area_op) +def _(expr: TypedExpr) -> sge.Expression: + return sge.func("ST_AREA", expr.expr) @register_unary_op(ops.geo_st_astext_op) @@ -45,6 +50,11 @@ def _(expr: TypedExpr, op: ops.GeoStBufferOp) -> sge.Expression: ) +@register_unary_op(ops.geo_st_centroid_op) +def _(expr: TypedExpr) -> sge.Expression: + return sge.func("ST_CENTROID", expr.expr) + + @register_unary_op(ops.geo_st_convexhull_op) def _(expr: TypedExpr) -> sge.Expression: return sge.func("ST_CONVEXHULL", expr.expr) @@ -87,6 +97,15 @@ def _( return sge.func("ST_REGIONSTATS", *args) +@register_unary_op(ops.GeoStSimplifyOp, pass_op=True) +def _(expr: TypedExpr, op: ops.GeoStSimplifyOp) -> sge.Expression: + return sge.func( + "ST_SIMPLIFY", + expr.expr, + sge.convert(op.tolerance_meters), + ) + + @register_unary_op(ops.geo_x_op) def _(expr: TypedExpr) -> sge.Expression: return sge.func("ST_X", expr.expr) diff --git a/bigframes/core/compile/sqlglot/expressions/json_ops.py b/bigframes/core/compile/sqlglot/expressions/json_ops.py index f9a92d3d7a6..d2008b45bf9 100644 --- a/bigframes/core/compile/sqlglot/expressions/json_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/json_ops.py @@ -16,13 +16,12 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler -from bigframes import dtypes from bigframes import operations as ops from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler -register_unary_op = expression_compiler.expression_compiler.register_unary_op -register_binary_op = expression_compiler.expression_compiler.register_binary_op +register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op +register_binary_op = scalar_compiler.scalar_op_compiler.register_binary_op @register_unary_op(ops.JSONExtract, pass_op=True) @@ -70,39 +69,9 @@ def _(expr: TypedExpr) -> sge.Expression: return sge.func("PARSE_JSON", expr.expr) -@register_unary_op(ops.ToJSON, pass_op=True) -def _(expr: TypedExpr, op: ops.ToJSON) -> sge.Expression: - from_type = expr.dtype - sg_expr = expr.expr - - # Parsing really should be a distinct operation from serialization, but - # this was the way things were intially launched. - if from_type == dtypes.STRING_DTYPE: - func_name = "SAFE.PARSE_JSON" if op.safe else "PARSE_JSON" - return sge.func(func_name, sg_expr) - else: - return sge.func( - "IF", sg_expr.is_(sge.Null()), sge.Null(), sge.func("TO_JSON", sg_expr) - ) - - -@register_unary_op(ops.JSONDecode, pass_op=True) -def _(expr: TypedExpr, op: ops.JSONDecode) -> sge.Expression: - to_type = op.to_type - sg_expr = expr.expr - func_name = "" - if to_type == dtypes.INT_DTYPE: - func_name = "INT64" - elif to_type == dtypes.FLOAT_DTYPE: - func_name = "FLOAT64" - elif to_type == dtypes.BOOL_DTYPE: - func_name = "BOOL" - elif to_type == dtypes.STRING_DTYPE: - func_name = "STRING" - if func_name: - func_name = "SAFE." + func_name if op.safe else func_name - return sge.func(func_name, sg_expr) - raise TypeError(f"Cannot cast from {dtypes.JSON_DTYPE} to {to_type}") +@register_unary_op(ops.ToJSON) +def _(expr: TypedExpr) -> sge.Expression: + return sge.func("TO_JSON", expr.expr) @register_unary_op(ops.ToJSONString) diff --git a/bigframes/core/compile/sqlglot/expressions/numeric_ops.py b/bigframes/core/compile/sqlglot/expressions/numeric_ops.py index d62a93111be..28d3532b8b8 100644 --- a/bigframes/core/compile/sqlglot/expressions/numeric_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/numeric_ops.py @@ -17,17 +17,15 @@ import bigframes_vendored.constants as bf_constants import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler -import bigframes.core.compile.sqlglot.expressions.constants as constants from bigframes import dtypes from bigframes import operations as ops -from bigframes.core.compile.sqlglot import sql -from bigframes.core.compile.sqlglot.expressions.common import round_towards_zero +import bigframes.core.compile.sqlglot.expressions.constants as constants from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler from bigframes.operations import numeric_ops -register_unary_op = expression_compiler.expression_compiler.register_unary_op -register_binary_op = expression_compiler.expression_compiler.register_binary_op +register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op +register_binary_op = scalar_compiler.scalar_op_compiler.register_binary_op @register_unary_op(ops.abs_op) @@ -261,9 +259,6 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: def _int_pow_op( left_expr: sge.Expression, right_expr: sge.Expression ) -> sge.Expression: - if sql.is_null_literal(left_expr) or sql.is_null_literal(right_expr): - return sge.null() - overflow_cond = sge.and_( sge.NEQ(this=left_expr, expression=sge.convert(0)), sge.GT( @@ -296,9 +291,6 @@ def _int_pow_op( def _float_pow_op( left_expr: sge.Expression, right_expr: sge.Expression ) -> sge.Expression: - if sql.is_null_literal(left_expr) or sql.is_null_literal(right_expr): - return sge.null() - # Most conditions here seek to prevent calling BQ POW with inputs that would generate errors. # See: https://cloud.google.com/bigquery/docs/reference/standard-sql/mathematical_functions#pow overflow_cond = sge.and_( @@ -370,7 +362,7 @@ def _float_pow_op( sge.If( this=sge.and_( sge.LT(this=left_expr, expression=constants._ZERO), - sge.Not(this=sge.paren(exponent_is_whole)), + sge.Not(this=exponent_is_whole), ), true=constants._NAN, ), @@ -432,7 +424,7 @@ def _(expr: TypedExpr) -> sge.Expression: @register_binary_op(ops.add_op) def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if sql.is_null_literal(left.expr) or sql.is_null_literal(right.expr): + if left.expr == sge.null() or right.expr == sge.null(): return sge.null() if left.dtype == dtypes.STRING_DTYPE and right.dtype == dtypes.STRING_DTYPE: @@ -470,15 +462,12 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: @register_binary_op(ops.div_op) def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if sql.is_null_literal(left.expr) or sql.is_null_literal(right.expr): - return sge.null() - left_expr = _coerce_bool_to_int(left) right_expr = _coerce_bool_to_int(right) result = sge.func("IEEE_DIVIDE", left_expr, right_expr) if left.dtype == dtypes.TIMEDELTA_DTYPE and dtypes.is_numeric(right.dtype): - return round_towards_zero(result) + return sge.Cast(this=sge.Floor(this=result), to="INT64") else: return result @@ -492,7 +481,7 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: @register_binary_op(ops.floordiv_op) def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if sql.is_null_literal(left.expr) or sql.is_null_literal(right.expr): + if left.expr == sge.null() or right.expr == sge.null(): return sge.null() left_expr = _coerce_bool_to_int(left) @@ -521,7 +510,7 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: ) if dtypes.is_numeric(right.dtype) and left.dtype == dtypes.TIMEDELTA_DTYPE: - result = round_towards_zero(sge.func("IEEE_DIVIDE", left_expr, right_expr)) + result = sge.Cast(this=sge.Floor(this=result), to="INT64") return result @@ -535,9 +524,6 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: @register_binary_op(ops.mod_op) def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if sql.is_null_literal(left.expr) or sql.is_null_literal(right.expr): - return sge.null() - # In BigQuery returned value has the same sign as X. In pandas, the sign of y is used, so we need to flip the result if sign(x) != sign(y) left_expr = _coerce_bool_to_int(left) right_expr = _coerce_bool_to_int(right) @@ -581,7 +567,7 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: @register_binary_op(ops.mul_op) def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if sql.is_null_literal(left.expr) or sql.is_null_literal(right.expr): + if left.expr == sge.null() or right.expr == sge.null(): return sge.null() left_expr = _coerce_bool_to_int(left) @@ -592,7 +578,7 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: if (dtypes.is_numeric(left.dtype) and right.dtype == dtypes.TIMEDELTA_DTYPE) or ( left.dtype == dtypes.TIMEDELTA_DTYPE and dtypes.is_numeric(right.dtype) ): - return round_towards_zero(result) + return sge.Cast(this=sge.Floor(this=result), to="INT64") else: return result @@ -607,7 +593,7 @@ def _(expr: TypedExpr, n_digits: TypedExpr) -> sge.Expression: @register_binary_op(ops.sub_op) def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: - if sql.is_null_literal(left.expr) or sql.is_null_literal(right.expr): + if left.expr == sge.null() or right.expr == sge.null(): return sge.null() if dtypes.is_numeric(left.dtype) and dtypes.is_numeric(right.dtype): diff --git a/bigframes/core/compile/sqlglot/expressions/string_ops.py b/bigframes/core/compile/sqlglot/expressions/string_ops.py index 65a13a45f8b..242dd7ae20d 100644 --- a/bigframes/core/compile/sqlglot/expressions/string_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/string_ops.py @@ -19,13 +19,13 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler from bigframes import dtypes from bigframes import operations as ops from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler -register_unary_op = expression_compiler.expression_compiler.register_unary_op -register_binary_op = expression_compiler.expression_compiler.register_binary_op +register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op +register_binary_op = scalar_compiler.scalar_op_compiler.register_binary_op @register_unary_op(ops.capitalize_op) @@ -256,6 +256,11 @@ def _(expr: TypedExpr, op: ops.StringSplitOp) -> sge.Expression: return sge.Split(this=expr.expr, expression=sge.convert(op.pat)) +@register_unary_op(ops.StrGetOp, pass_op=True) +def _(expr: TypedExpr, op: ops.StrGetOp) -> sge.Expression: + return string_index(expr, op.i) + + @register_unary_op(ops.StrSliceOp, pass_op=True) def _(expr: TypedExpr, op: ops.StrSliceOp) -> sge.Expression: return string_slice(expr, op.start, op.end) @@ -361,16 +366,10 @@ def string_slice( column_length + sge.convert(start + 1), ] ) - length_expr = sge.Greatest( + length_expr = sge.convert(op_end) - sge.Greatest( expressions=[ sge.convert(0), - sge.convert(op_end) - - sge.Greatest( - expressions=[ - sge.convert(0), - column_length + sge.convert(start), - ] - ), + column_length + sge.convert(start), ] ) else: diff --git a/bigframes/core/compile/sqlglot/expressions/struct_ops.py b/bigframes/core/compile/sqlglot/expressions/struct_ops.py index 01022210182..5048941f149 100644 --- a/bigframes/core/compile/sqlglot/expressions/struct_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/struct_ops.py @@ -20,12 +20,12 @@ import pandas as pd import pyarrow as pa -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler from bigframes import operations as ops from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler -register_nary_op = expression_compiler.expression_compiler.register_nary_op -register_unary_op = expression_compiler.expression_compiler.register_unary_op +register_nary_op = scalar_compiler.scalar_op_compiler.register_nary_op +register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op @register_unary_op(ops.StructFieldOp, pass_op=True) diff --git a/bigframes/core/compile/sqlglot/expressions/timedelta_ops.py b/bigframes/core/compile/sqlglot/expressions/timedelta_ops.py index fbc982829ca..b442fa8175d 100644 --- a/bigframes/core/compile/sqlglot/expressions/timedelta_ops.py +++ b/bigframes/core/compile/sqlglot/expressions/timedelta_ops.py @@ -16,13 +16,13 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler from bigframes import dtypes from bigframes import operations as ops from bigframes.core.compile.constants import UNIT_TO_US_CONVERSION_FACTORS from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler -register_unary_op = expression_compiler.expression_compiler.register_unary_op +register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op @register_unary_op(ops.timedelta_floor_op) diff --git a/bigframes/core/compile/sqlglot/expressions/typed_expr.py b/bigframes/core/compile/sqlglot/expressions/typed_expr.py index d8c38c2e718..4623b8c9b43 100644 --- a/bigframes/core/compile/sqlglot/expressions/typed_expr.py +++ b/bigframes/core/compile/sqlglot/expressions/typed_expr.py @@ -25,6 +25,3 @@ class TypedExpr: expr: sge.Expression dtype: dtypes.ExpressionType - - # kludge to support optional args in argument lists - is_omitted: bool = False diff --git a/bigframes/core/compile/sqlglot/expression_compiler.py b/bigframes/core/compile/sqlglot/scalar_compiler.py similarity index 90% rename from bigframes/core/compile/sqlglot/expression_compiler.py rename to bigframes/core/compile/sqlglot/scalar_compiler.py index b412249a39f..317141b6cc4 100644 --- a/bigframes/core/compile/sqlglot/expression_compiler.py +++ b/bigframes/core/compile/sqlglot/scalar_compiler.py @@ -18,14 +18,13 @@ import bigframes_vendored.sqlglot.expressions as sge -import bigframes.core.agg_expressions as agg_exprs +from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.sqlglot_ir as ir import bigframes.core.expression as ex import bigframes.operations as ops -from bigframes.core.compile.sqlglot import sql -from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr -class ExpressionCompiler: +class ScalarOpCompiler: # Mapping of operation name to implemenations _registry: dict[ str, @@ -48,10 +47,6 @@ class ExpressionCompiler: sge.LT, sge.EQ, sge.NEQ, - sge.Like, - sge.RegexpLike, - sge.In, - sge.Between, # Logical operations sge.And, sge.Or, @@ -81,23 +76,13 @@ def _(self, expr: ex.DerefOp) -> sge.Expression: @compile_expression.register def _(self, expr: ex.ScalarConstantExpression) -> sge.Expression: - return sql.literal(expr.value, expr.dtype) - - @compile_expression.register - def _(self, expr: agg_exprs.WindowExpression) -> sge.Expression: - import bigframes.core.compile.sqlglot.aggregate_compiler as agg_compile - - return agg_compile.compile_analytic( - expr.analytic_expr, - expr.window, - ) + return ir._literal(expr.value, expr.dtype) @compile_expression.register def _(self, expr: ex.OpExpression) -> sge.Expression: + # Non-recursively compiles the children scalar expressions. inputs = tuple( TypedExpr(self.compile_expression(sub_expr), sub_expr.output_type) - if not isinstance(sub_expr, ex.OmittedArg) - else TypedExpr(sge.Null(), None, is_omitted=True) for sub_expr in expr.inputs ) return self.compile_row_op(expr.op, inputs) @@ -233,4 +218,4 @@ def _add_parentheses(cls, expr: TypedExpr) -> TypedExpr: # Singleton compiler -expression_compiler = ExpressionCompiler() +scalar_op_compiler = ScalarOpCompiler() diff --git a/bigframes/core/compile/sqlglot/sql/__init__.py b/bigframes/core/compile/sqlglot/sql/__init__.py deleted file mode 100644 index 751c3cfc3a5..00000000000 --- a/bigframes/core/compile/sqlglot/sql/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import annotations - -from bigframes.core.compile.sqlglot.sql.base import ( - cast, - identifier, - is_null_literal, - literal, - table, - to_sql, -) -from bigframes.core.compile.sqlglot.sql.ddl import create_external_table, load_data -from bigframes.core.compile.sqlglot.sql.dml import insert, replace - -__all__ = [ - # From base.py - "cast", - "identifier", - "is_null_literal", - "literal", - "table", - "to_sql", - # From ddl.py - "create_external_table", - "load_data", - # From dml.py - "insert", - "replace", -] diff --git a/bigframes/core/compile/sqlglot/sql/base.py b/bigframes/core/compile/sqlglot/sql/base.py deleted file mode 100644 index f77dcbee4d9..00000000000 --- a/bigframes/core/compile/sqlglot/sql/base.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import typing - -import bigframes_vendored.sqlglot as sg -import bigframes_vendored.sqlglot.expressions as sge -import numpy as np -import pandas as pd -import pyarrow as pa -from google.cloud import bigquery - -import bigframes.core.compile.sqlglot.sqlglot_types as sgt -from bigframes import dtypes -from bigframes.core import utils -from bigframes.core.compile.sqlglot.expressions import constants - -# shapely.wkt.dumps was moved to shapely.io.to_wkt in 2.0. -try: - from shapely.io import to_wkt # type: ignore -except ImportError: - from shapely.wkt import dumps # type: ignore - - to_wkt = dumps - - -QUOTED: bool = True -"""Whether to quote identifiers in the generated SQL.""" - -PRETTY: bool = True -"""Whether to pretty-print the generated SQL.""" - -DIALECT = sg.dialects.bigquery.BigQuery -"""The SQL dialect used for generation.""" - - -def to_sql(expr: sge.Expression) -> str: - """Generate SQL string from the given expression.""" - return expr.sql(dialect=DIALECT, pretty=PRETTY) - - -def identifier(id: str) -> sge.Identifier: - """Return a string representing column reference in a SQL.""" - return sge.to_identifier(id, quoted=QUOTED) - - -def literal(value: typing.Any, dtype: dtypes.Dtype | None = None) -> sge.Expression: - """Return a string representing column reference in a SQL.""" - if dtype is None: - dtype = dtypes.infer_literal_type(value) - - sqlglot_type = sgt.from_bigframes_dtype(dtype) if dtype else None - if sqlglot_type is None: - if not pd.isna(value): - raise ValueError(f"Cannot infer SQLGlot type from None dtype: {value}") - return sge.Null() - - if value is None: - if str(sqlglot_type).upper() == "NULL": - return sge.Null() - return cast(sge.Null(), sqlglot_type) - if dtypes.is_struct_like(dtype): - items = [ - literal(value=value[field_name], dtype=field_dtype).as_( - field_name, quoted=True - ) - for field_name, field_dtype in dtypes.get_struct_fields(dtype).items() - ] - return sge.Struct.from_arg_list(items) - elif dtypes.is_array_like(dtype): - value_type = dtypes.get_array_inner_type(dtype) - values = sge.Array( - expressions=[literal(value=v, dtype=value_type) for v in value] - ) - return values if len(value) > 0 else cast(values, sqlglot_type) - elif dtype == dtypes.FLOAT_DTYPE: - if pd.isna(value): - if isinstance(value, (float, np.floating)) and np.isnan(value): - return constants._NAN - return cast(sge.Null(), sqlglot_type) - if np.isinf(value): - return constants._INF if value > 0 else constants._NEG_INF - return sge.convert(value) - elif pd.isna(value) or (isinstance(value, pa.Scalar) and not value.is_valid): - return cast(sge.Null(), sqlglot_type) - elif dtype == dtypes.JSON_DTYPE: - return sge.ParseJSON(this=sge.convert(str(value))) - elif dtype == dtypes.BYTES_DTYPE: - return cast(str(value), sqlglot_type) - elif dtypes.is_time_like(dtype): - if isinstance(value, str): - return cast(sge.convert(value), sqlglot_type) - if isinstance(value, np.generic): - value = value.item() - return cast(sge.convert(value.isoformat()), sqlglot_type) - elif dtype in (dtypes.NUMERIC_DTYPE, dtypes.BIGNUMERIC_DTYPE): - return cast(sge.convert(value), sqlglot_type) - elif dtypes.is_geo_like(dtype): - wkt = value if isinstance(value, str) else to_wkt(value) - return sge.func("ST_GEOGFROMTEXT", sge.convert(wkt)) - elif dtype == dtypes.TIMEDELTA_DTYPE: - return sge.convert(utils.timedelta_to_micros(value)) - else: - if isinstance(value, np.generic): - value = value.item() - if isinstance(value, pa.Scalar): - value = value.as_py() - return sge.convert(value) - - -def cast(arg: typing.Any, to: str, safe: bool = False) -> sge.Cast | sge.TryCast: - """Return a SQL expression that casts the given argument to the specified type.""" - if safe: - return sge.TryCast(this=arg, to=to) - else: - return sge.Cast(this=arg, to=to) - - -def table(table: bigquery.TableReference) -> sge.Table: - """Return a SQLGlot Table expression representing the given BigQuery table reference.""" - return sge.Table( - this=sge.to_identifier(table.table_id, quoted=True), - db=sge.to_identifier(table.dataset_id, quoted=True), - catalog=sge.to_identifier(table.project, quoted=True), - ) - - -def is_null_literal(expr: sge.Expression) -> bool: - """Checks if the given expression is a NULL literal.""" - if isinstance(expr, sge.Null): - return True - if isinstance(expr, sge.Cast) and isinstance(expr.this, sge.Null): - return True - return False diff --git a/bigframes/core/compile/sqlglot/sql/ddl.py b/bigframes/core/compile/sqlglot/sql/ddl.py deleted file mode 100644 index 1a63d016d5e..00000000000 --- a/bigframes/core/compile/sqlglot/sql/ddl.py +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from typing import Mapping, Optional, Union - -import bigframes_vendored.sqlglot as sg -import bigframes_vendored.sqlglot.expressions as sge - -from bigframes.core.compile.sqlglot.sql import base - - -def load_data( - table_name: str, - *, - write_disposition: str = "INTO", - columns: Optional[Mapping[str, str]] = None, - partition_by: Optional[list[str]] = None, - cluster_by: Optional[list[str]] = None, - table_options: Optional[Mapping[str, Union[str, int, float, bool, list]]] = None, - from_files_options: Mapping[str, Union[str, int, float, bool, list]], - with_partition_columns: Optional[Mapping[str, str]] = None, - connection_name: Optional[str] = None, -) -> sge.LoadData: - """Generates the LOAD DATA DDL statement.""" - # We use a Table with a simple identifier for the table name. - # Quoting is handled by the dialect. - table_expr = sge.Table(this=base.identifier(table_name)) - - sge_partition_by = ( - sge.PartitionedByProperty( - this=base.identifier(partition_by[0]) - if len(partition_by) == 1 - else sge.Tuple(expressions=[base.identifier(col) for col in partition_by]) - ) - if partition_by - else None - ) - - sge_cluster_by = ( - sge.Cluster(expressions=[base.identifier(col) for col in cluster_by]) - if cluster_by - else None - ) - - sge_from_files = sge.Tuple( - expressions=[ - sge.Property(this=base.identifier(k), value=base.literal(v)) - for k, v in from_files_options.items() - ] - ) - - sge_connection = base.identifier(connection_name) if connection_name else None - - return sge.LoadData( - this=table_expr, - overwrite=(write_disposition == "OVERWRITE"), - inpath=sge.convert("fake"), # satisfy sqlglot's required inpath arg - columns=_get_sge_schema(columns), - partition_by=sge_partition_by, - cluster_by=sge_cluster_by, - options=_get_sge_properties(table_options), - from_files=sge_from_files, - with_partition_columns=_get_sge_schema(with_partition_columns), - connection=sge_connection, - ) - - -def create_external_table( - table_name: str, - *, - replace: bool = False, - if_not_exists: bool = False, - columns: Optional[Mapping[str, str]] = None, - partition_columns: Optional[Mapping[str, str]] = None, - connection_name: Optional[str] = None, - options: Optional[Mapping[str, Union[str, int, float, bool, list]]] = None, -) -> sge.Create: - """Generates the CREATE EXTERNAL TABLE DDL statement.""" - sge_connection = base.identifier(connection_name) if connection_name else None - - table_expr = sge.Table(this=base.identifier(table_name)) - - # sqlglot.expressions.Create usually takes 'this' (Table or Schema) - sge_schema = _get_sge_schema(columns) - this: sge.Table | sge.Schema - if sge_schema: - sge_schema.set("this", table_expr) - this = sge_schema - else: - this = table_expr - - return sge.Create( - this=this, - kind="EXTERNAL TABLE", - replace=replace, - exists_ok=if_not_exists, - properties=_get_sge_properties(options), - connection=sge_connection, - partition_columns=_get_sge_schema(partition_columns), - ) - - -def _get_sge_schema( - columns: Optional[Mapping[str, str]] = None, -) -> Optional[sge.Schema]: - if not columns: - return None - - return sge.Schema( - this=None, - expressions=[ - sge.ColumnDef( - this=base.identifier(name), - kind=sge.DataType.build(typ, dialect=base.DIALECT), - ) - for name, typ in columns.items() - ], - ) - - -def _get_sge_properties( - options: Optional[Mapping[str, Union[str, int, float, bool, list]]] = None, -) -> Optional[sge.Properties]: - if not options: - return None - - return sge.Properties( - expressions=[ - sge.Property(this=base.identifier(k), value=base.literal(v)) - for k, v in options.items() - ] - ) - - -def _loaddata_sql(self: sg.Generator, expression: sge.LoadData) -> str: - out = ["LOAD DATA"] - if expression.args.get("overwrite"): - out.append("OVERWRITE") - - out.append(f"INTO {self.sql(expression, 'this').strip()}") - - # We ignore inpath as it's just a dummy to satisfy sqlglot requirements - # but BigQuery uses FROM FILES instead. - - columns = self.sql(expression, "columns").strip() - if columns: - out.append(columns) - - partition_by = self.sql(expression, "partition_by").strip() - if partition_by: - out.append(partition_by) - - cluster_by = self.sql(expression, "cluster_by").strip() - if cluster_by: - out.append(cluster_by) - - options = self.sql(expression, "options").strip() - if options: - out.append(options) - - from_files = self.sql(expression, "from_files").strip() - if from_files: - out.append(f"FROM FILES {from_files}") - - with_partition_columns = self.sql(expression, "with_partition_columns").strip() - if with_partition_columns: - out.append(f"WITH PARTITION COLUMNS {with_partition_columns}") - - connection = self.sql(expression, "connection").strip() - if connection: - out.append(f"WITH CONNECTION {connection}") - - return " ".join(out) - - -def _create_sql(self: sg.Generator, expression: sge.Create) -> str: - kind = expression.args.get("kind") - if kind != "EXTERNAL TABLE": - return self.create_sql(expression) - - out = ["CREATE"] - if expression.args.get("replace"): - out.append("OR REPLACE") - out.append("EXTERNAL TABLE") - if expression.args.get("exists_ok"): - out.append("IF NOT EXISTS") - - out.append(self.sql(expression, "this")) - - connection = self.sql(expression, "connection").strip() - if connection: - out.append(f"WITH CONNECTION {connection}") - - partition_columns = self.sql(expression, "partition_columns").strip() - if partition_columns: - out.append(f"WITH PARTITION COLUMNS {partition_columns}") - - properties = self.sql(expression, "properties").strip() - if properties: - out.append(properties) - - return " ".join(out) - - -# Register the transform for BigQuery generator -base.DIALECT.Generator.TRANSFORMS[sge.LoadData] = _loaddata_sql -base.DIALECT.Generator.TRANSFORMS[sge.Create] = _create_sql diff --git a/bigframes/core/compile/sqlglot/sql/dml.py b/bigframes/core/compile/sqlglot/sql/dml.py deleted file mode 100644 index 0f0ae9dff2b..00000000000 --- a/bigframes/core/compile/sqlglot/sql/dml.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import typing - -import bigframes_vendored.sqlglot.expressions as sge -from google.cloud import bigquery - -from bigframes import dtypes -from bigframes.core.compile.sqlglot.sql import base - - -def insert( - query_or_table: typing.Union[sge.Select, sge.Table], - destination: bigquery.TableReference, -) -> sge.Insert: - """Generates an INSERT INTO SQL statement from the given SELECT statement or - table reference.""" - return sge.insert(_as_from_item(query_or_table), base.table(destination)) - - -def replace( - query_or_table: typing.Union[sge.Select, sge.Table], - destination: bigquery.TableReference, -) -> sge.Merge: - """Generates a MERGE statement to replace the contents of the destination table.""" - return sge.Merge( - this=base.table(destination), - using=_as_from_item(query_or_table), - on=base.literal(False, dtypes.BOOL_DTYPE), - whens=sge.Whens( - expressions=[ - sge.When(matched=False, source=True, then=sge.Delete()), - sge.When(matched=False, then=sge.Insert(this=sge.Var(this="ROW"))), - ] - ), - ) - - -def _as_from_item( - query_or_table: typing.Union[sge.Select, sge.Table], -) -> typing.Union[sge.Subquery, sge.Table]: - if isinstance(query_or_table, sge.Select): - return query_or_table.subquery() - else: # table - return query_or_table diff --git a/bigframes/core/compile/sqlglot/sqlglot_ir.py b/bigframes/core/compile/sqlglot/sqlglot_ir.py index b29a23cd84b..d4dc4ecc064 100644 --- a/bigframes/core/compile/sqlglot/sqlglot_ir.py +++ b/bigframes/core/compile/sqlglot/sqlglot_ir.py @@ -21,13 +21,15 @@ import bigframes_vendored.sqlglot as sg import bigframes_vendored.sqlglot.expressions as sge +from google.cloud import bigquery +import numpy as np +import pandas as pd import pyarrow as pa -import bigframes.core.compile.sqlglot.sqlglot_types as sgt from bigframes import dtypes -from bigframes.core import guid, local_data, schema -from bigframes.core.compile.sqlglot import sql -from bigframes.core.compile.sqlglot.expressions import typed_expr +from bigframes.core import guid, local_data, schema, utils +from bigframes.core.compile.sqlglot.expressions import constants, typed_expr +import bigframes.core.compile.sqlglot.sqlglot_types as sgt # shapely.wkt.dumps was moved to shapely.io.to_wkt in 2.0. try: @@ -38,92 +40,29 @@ to_wkt = dumps -class SelectFragment: - def __init__(self, select_expr: sge.Select): - self.select_expr = select_expr - - def as_select_all(self) -> sge.Select: - return self.select_expr - - def select(self, *items: sge.Expression) -> sge.Select: - return sge.Select().select(*items).from_(self.select_expr.subquery()) - - def as_from_item(self) -> sge.Expression: - return self.select_expr.subquery() - - -class TableFragment: - def __init__(self, table: sge.Table | sge.Unnest): - self.table = table - - def as_select_all(self) -> sge.Select: - return sge.Select().select(sge.Star()).from_(self.table) - - def select(self, *items: sge.Expression) -> sge.Select: - return sge.Select().select(*items).from_(self.table) - - def as_from_item(self) -> sge.Expression: - return self.table - - -class DeferredSelectFragment: - def __init__(self, select_supplier: typing.Callable[[sge.Select], sge.Select]): - self.select_supplier = select_supplier - - def as_select_all(self) -> sge.Select: - return self.select_supplier(sge.Select().select(sge.Star())) - - def select(self, *items: sge.Expression) -> sge.Select: - return self.select_supplier(sge.Select().select(*items)) - - def as_from_item(self) -> sge.Expression: - return self.select_supplier(sge.Select().select(sge.Star())).subquery() - - -ExprT = SelectFragment | TableFragment | DeferredSelectFragment - - @dataclasses.dataclass(frozen=True) class SQLGlotIR: """Helper class to build SQLGlot Query and generate SQL string.""" - expr: ExprT + expr: sge.Select = sg.select() """The SQLGlot expression representing the query.""" + dialect = sg.dialects.bigquery.BigQuery + """The SQL dialect used for generation.""" + + quoted: bool = True + """Whether to quote identifiers in the generated SQL.""" + + pretty: bool = True + """Whether to pretty-print the generated SQL.""" + uid_gen: guid.SequentialUIDGenerator = guid.SequentialUIDGenerator() """Generator for unique identifiers.""" @property def sql(self) -> str: """Generate SQL string from the given expression.""" - return sql.to_sql(self.expr.as_select_all()) - - @classmethod - def empty( - cls, uid_gen: guid.SequentialUIDGenerator = guid.SequentialUIDGenerator() - ) -> SQLGlotIR: - return cls(expr=SelectFragment(sge.select()), uid_gen=uid_gen) - - @classmethod - def from_expr( - cls, - expr: sge.Expression, - uid_gen: guid.SequentialUIDGenerator = guid.SequentialUIDGenerator(), - ) -> SQLGlotIR: - if isinstance(expr, sge.Select): - return cls(expr=SelectFragment(expr), uid_gen=uid_gen) - elif isinstance(expr, (sge.Table, sge.Unnest)): - return cls(expr=TableFragment(expr), uid_gen=uid_gen) - else: - raise ValueError(f"Unsupported expression type: {type(expr)}") - - @classmethod - def from_func( - cls, - select_handler: typing.Callable[[sge.Select], sge.Select], - uid_gen: guid.SequentialUIDGenerator = guid.SequentialUIDGenerator(), - ): - return cls(expr=DeferredSelectFragment(select_handler), uid_gen=uid_gen) + return self.expr.sql(dialect=self.dialect, pretty=self.pretty) @classmethod def from_pyarrow( @@ -150,7 +89,7 @@ def from_pyarrow( data_expr = [ sge.Struct( expressions=tuple( - sql.literal( + _literal( value=value, dtype=field.dtype, ) @@ -169,7 +108,7 @@ def from_pyarrow( ), ], ) - return cls.from_expr(expr=expr, uid_gen=uid_gen) + return cls(expr=sg.select(sge.Star()).from_(expr), uid_gen=uid_gen) @classmethod def from_table( @@ -177,8 +116,9 @@ def from_table( project_id: str, dataset_id: str, table_id: str, - uid_gen: guid.SequentialUIDGenerator | None = None, - columns: typing.Sequence[str] = (), + col_names: typing.Sequence[str], + alias_names: typing.Sequence[str], + uid_gen: guid.SequentialUIDGenerator, sql_predicate: typing.Optional[str] = None, system_time: typing.Optional[datetime.datetime] = None, ) -> SQLGlotIR: @@ -188,140 +128,201 @@ def from_table( project_id (str): The project ID of the BigQuery table. dataset_id (str): The dataset ID of the BigQuery table. table_id (str): The table ID of the BigQuery table. + col_names (typing.Sequence[str]): The names of the columns to select. + alias_names (typing.Sequence[str]): The aliases for the selected columns. uid_gen (guid.SequentialUIDGenerator): A generator for unique identifiers. - columns (typing.Sequence[str]): The names of the columns to select. sql_predicate (typing.Optional[str]): An optional SQL predicate for filtering. system_time (typing.Optional[str]): An optional system time for time-travel queries. """ + selections = [ + sge.Alias( + this=sge.to_identifier(col_name, quoted=cls.quoted), + alias=sge.to_identifier(alias_name, quoted=cls.quoted), + ) + if col_name != alias_name + else sge.to_identifier(col_name, quoted=cls.quoted) + for col_name, alias_name in zip(col_names, alias_names) + ] version = ( sge.Version( - this=sge.Identifier(this="SYSTEM_TIME", quoted=False), - expression=sge.Literal.string(system_time.isoformat()), + this="TIMESTAMP", + expression=sge.Literal(this=system_time.isoformat(), is_string=True), kind="AS OF", ) if system_time else None ) - if uid_gen is None: - uid_gen = guid.SequentialUIDGenerator() - table_alias = next(uid_gen.get_uid_stream("bft_")) table_expr = sge.Table( - this=sql.identifier(table_id), - db=sql.identifier(dataset_id), - catalog=sql.identifier(project_id), + this=sg.to_identifier(table_id, quoted=cls.quoted), + db=sg.to_identifier(dataset_id, quoted=cls.quoted), + catalog=sg.to_identifier(project_id, quoted=cls.quoted), version=version, - alias=sql.identifier(table_alias), - ) - - if not columns and not sql_predicate: - return cls.from_expr(expr=table_expr, uid_gen=uid_gen) - - select_items: list[sge.Identifier | sge.Star] = ( - [sql.identifier(col) for col in columns] if columns else [sge.Star()] ) - select_expr = sge.Select().select(*select_items).from_(table_expr) - + select_expr = sge.Select().select(*selections).from_(table_expr) if sql_predicate: select_expr = select_expr.where( - sg.parse_one(sql_predicate, dialect=sql.base.DIALECT), append=False + sg.parse_one(sql_predicate, dialect="bigquery"), append=False ) - - return cls.from_expr(expr=select_expr, uid_gen=uid_gen) + return cls(expr=select_expr, uid_gen=uid_gen) @classmethod - def from_cte_ref( - cls, - cte_ref: str, - uid_gen: guid.SequentialUIDGenerator, - ) -> SQLGlotIR: - table_expr = sge.Table( - this=sql.identifier(cte_ref), - ) - return cls.from_expr(expr=table_expr, uid_gen=uid_gen) - - def select( - self, - selections: tuple[tuple[str, sge.Expression], ...] = (), - predicates: tuple[sge.Expression, ...] = (), - sorting: tuple[sge.Ordered, ...] = (), - limit: typing.Optional[int] = None, - ) -> SQLGlotIR: - # TODO: Explicitly insert CTEs into plan - if len(selections) > 0: - to_select = [ - expr - if (isinstance(expr, sge.Alias) and expr.alias == id) - or (isinstance(expr, sge.Column) and expr.name == id) - else sge.Alias( - this=expr.this if isinstance(expr, sge.Alias) else expr, - alias=sql.identifier(id), - ) - for id, expr in selections - ] - new_expr = self.expr.select(*to_select) - else: - new_expr = self.expr.as_select_all() - - if len(sorting) > 0: - new_expr = new_expr.order_by(*sorting) - - if len(predicates) > 0: - condition = _and(predicates) - new_expr = new_expr.where(condition, append=False) - if limit is not None: - new_expr = new_expr.limit(limit) - - return SQLGlotIR.from_expr(expr=new_expr, uid_gen=self.uid_gen) - - @classmethod - def from_unparsed_query( + def from_query_string( cls, query_string: str, ) -> SQLGlotIR: - """Builds a SQLGlot expression from a query string. Wrapping the query - in a CTE can avoid the query parsing issue for unsupported syntax in - SQLGlot.""" + """Builds a SQLGlot expression from a query string""" uid_gen: guid.SequentialUIDGenerator = guid.SequentialUIDGenerator() - cte_name = sql.identifier(next(uid_gen.get_uid_stream("bfcte_"))) + cte_name = sge.to_identifier( + next(uid_gen.get_uid_stream("bfcte_")), quoted=cls.quoted + ) cte = sge.CTE( this=query_string, alias=cte_name, ) select_expr = sge.Select().select(sge.Star()).from_(sge.Table(this=cte_name)) select_expr = _set_query_ctes(select_expr, [cte]) - return cls.from_expr(expr=select_expr, uid_gen=uid_gen) + return cls(expr=select_expr, uid_gen=uid_gen) @classmethod def from_union( cls, selects: typing.Sequence[sge.Select], - output_aliases: typing.Sequence[typing.Tuple[str, str]], + output_ids: typing.Sequence[str], uid_gen: guid.SequentialUIDGenerator, ) -> SQLGlotIR: """Builds a SQLGlot expression by unioning of multiple select expressions.""" - assert len(list(selects)) >= 2, ( - f"At least two select expressions must be provided, but got {selects}." - ) - union_expr: sge.Query = selects[0].subquery() - for select in selects[1:]: - union_expr = sge.Union( - this=union_expr, - expression=select.subquery(), - distinct=False, - copy=False, + assert ( + len(list(selects)) >= 2 + ), f"At least two select expressions must be provided, but got {selects}." + + existing_ctes: list[sge.CTE] = [] + union_selects: list[sge.Expression] = [] + for select in selects: + assert isinstance( + select, sge.Select + ), f"All provided expressions must be of type sge.Select, but got {type(select)}" + + select_expr = select.copy() + select_expr, select_ctes = _pop_query_ctes(select_expr) + existing_ctes = [*existing_ctes, *select_ctes] + + new_cte_name = sge.to_identifier( + next(uid_gen.get_uid_stream("bfcte_")), quoted=cls.quoted + ) + new_cte = sge.CTE( + this=select_expr, + alias=new_cte_name, + ) + existing_ctes = [*existing_ctes, new_cte] + + selections = [ + sge.Alias( + this=sge.to_identifier(expr.alias_or_name, quoted=cls.quoted), + alias=sge.to_identifier(output_id, quoted=cls.quoted), + ) + for expr, output_id in zip(select_expr.expressions, output_ids) + ] + union_selects.append( + sge.Select().select(*selections).from_(sge.Table(this=new_cte_name)) ) + union_expr = typing.cast( + sge.Select, + functools.reduce( + lambda x, y: sge.Union( + this=x, expression=y, distinct=False, copy=False + ), + union_selects, + ), + ) + final_select_expr = sge.Select().select(sge.Star()).from_(union_expr.subquery()) + final_select_expr = _set_query_ctes(final_select_expr, existing_ctes) + return cls(expr=final_select_expr, uid_gen=uid_gen) + + def select( + self, + selected_cols: tuple[tuple[str, sge.Expression], ...], + ) -> SQLGlotIR: + """Replaces new selected columns of the current SELECT clause.""" selections = [ sge.Alias( - this=sql.identifier(old_name), - alias=sql.identifier(new_name), + this=expr, + alias=sge.to_identifier(id, quoted=self.quoted), ) - for old_name, new_name in output_aliases + if expr.alias_or_name != id + else expr + for id, expr in selected_cols ] - final_select_expr = ( - sge.Select().select(*selections).from_(union_expr.subquery()) + + new_expr = _select_to_cte( + self.expr, + sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcte_")), quoted=self.quoted + ), + ) + new_expr = new_expr.select(*selections, append=False) + return SQLGlotIR(expr=new_expr, uid_gen=self.uid_gen) + + def project( + self, + projected_cols: tuple[tuple[str, sge.Expression], ...], + ) -> SQLGlotIR: + """Adds new columns to the SELECT clause.""" + projected_cols_expr = [ + sge.Alias( + this=expr, + alias=sge.to_identifier(id, quoted=self.quoted), + ) + for id, expr in projected_cols + ] + new_expr = _select_to_cte( + self.expr, + sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcte_")), quoted=self.quoted + ), + ) + new_expr = new_expr.select(*projected_cols_expr, append=True) + return SQLGlotIR(expr=new_expr, uid_gen=self.uid_gen) + + def order_by( + self, + ordering: tuple[sge.Ordered, ...], + ) -> SQLGlotIR: + """Adds an ORDER BY clause to the query.""" + if len(ordering) == 0: + return SQLGlotIR(expr=self.expr.copy(), uid_gen=self.uid_gen) + new_expr = self.expr.order_by(*ordering) + return SQLGlotIR(expr=new_expr, uid_gen=self.uid_gen) + + def limit( + self, + limit: int | None, + ) -> SQLGlotIR: + """Adds a LIMIT clause to the query.""" + if limit is not None: + new_expr = self.expr.limit(limit) + else: + new_expr = self.expr.copy() + return SQLGlotIR(expr=new_expr, uid_gen=self.uid_gen) + + def filter( + self, + conditions: tuple[sge.Expression, ...], + ) -> SQLGlotIR: + """Filters the query by adding a WHERE clause.""" + condition = _and(conditions) + if condition is None: + return SQLGlotIR(expr=self.expr.copy(), uid_gen=self.uid_gen) + + new_expr = _select_to_cte( + self.expr, + sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcte_")), quoted=self.quoted + ), + ) + return SQLGlotIR( + expr=new_expr.where(condition, append=False), uid_gen=self.uid_gen ) - return cls.from_expr(expr=final_select_expr, uid_gen=uid_gen) def join( self, @@ -332,8 +333,19 @@ def join( joins_nulls: bool = True, ) -> SQLGlotIR: """Joins the current query with another SQLGlotIR instance.""" - left_from = self.expr.as_from_item() - right_from = right.expr.as_from_item() + left_cte_name = sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcte_")), quoted=self.quoted + ) + right_cte_name = sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcte_")), quoted=self.quoted + ) + + left_select = _select_to_cte(self.expr, left_cte_name) + right_select = _select_to_cte(right.expr, right_cte_name) + + left_select, left_ctes = _pop_query_ctes(left_select) + right_select, right_ctes = _pop_query_ctes(right_select) + merged_ctes = [*left_ctes, *right_ctes] join_on = _and( tuple( @@ -342,12 +354,15 @@ def join( ) join_type_str = join_type if join_type != "outer" else "full outer" - return SQLGlotIR.from_func( - lambda select: select.from_(left_from).join( - right_from, on=join_on, join_type=join_type_str - ), - uid_gen=self.uid_gen, + new_expr = ( + sge.Select() + .select(sge.Star()) + .from_(sge.Table(this=left_cte_name)) + .join(sge.Table(this=right_cte_name), on=join_on, join_type=join_type_str) ) + new_expr = _set_query_ctes(new_expr, merged_ctes) + + return SQLGlotIR(expr=new_expr, uid_gen=self.uid_gen) def isin_join( self, @@ -357,53 +372,59 @@ def isin_join( joins_nulls: bool = True, ) -> SQLGlotIR: """Joins the current query with another SQLGlotIR instance.""" - left_from = self.expr.as_from_item() + left_cte_name = sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcte_")), quoted=self.quoted + ) + + left_select = _select_to_cte(self.expr, left_cte_name) + # Prefer subquery over CTE for the IN clause's right side to improve SQL readability. + right_select = right.expr + + left_select, left_ctes = _pop_query_ctes(left_select) + right_select, right_ctes = _pop_query_ctes(right_select) + merged_ctes = [*left_ctes, *right_ctes] + + left_condition = typed_expr.TypedExpr( + sge.Column(this=conditions[0].expr, table=left_cte_name), + conditions[0].dtype, + ) new_column: sge.Expression if joins_nulls: - force_float_domain = False - if ( - conditions[0].dtype == dtypes.FLOAT_DTYPE - or conditions[1].dtype == dtypes.FLOAT_DTYPE - ): - force_float_domain = True - left_expr1, left_expr2 = _value_to_non_null_identity( - conditions[0], force_float_domain + right_table_name = sge.to_identifier( + next(self.uid_gen.get_uid_stream("bft_")), quoted=self.quoted ) - right_expr1, right_expr2 = _value_to_non_null_identity( - conditions[1], force_float_domain + right_condition = typed_expr.TypedExpr( + sge.Column(this=conditions[1].expr, table=right_table_name), + conditions[1].dtype, ) - - # Use EXISTS for better performance. - # We use COALESCE on both sides in the WHERE clause as requested. new_column = sge.Exists( this=sge.Select() .select(sge.convert(1)) - .from_(right.expr.as_from_item()) + .from_(sge.Alias(this=right_select.subquery(), alias=right_table_name)) .where( - sge.and_( - sge.EQ(this=left_expr1, expression=right_expr1), - sge.EQ(this=left_expr2, expression=right_expr2), - ) + _join_condition(left_condition, right_condition, joins_nulls=True) ) ) else: - new_column = sge.func( - "COALESCE", - sge.In( - this=conditions[0].expr, - expressions=[right._as_subquery()], - ), - sql.literal(False, dtypes.BOOL_DTYPE), + new_column = sge.In( + this=left_condition.expr, + expressions=[right_select.subquery()], ) new_column = sge.Alias( this=new_column, - alias=sql.identifier(indicator_col), + alias=sge.to_identifier(indicator_col, quoted=self.quoted), ) - new_expr = sge.Select().select(sge.Star(), new_column).from_(left_from) - return SQLGlotIR.from_expr(expr=new_expr, uid_gen=self.uid_gen) + new_expr = ( + sge.Select() + .select(sge.Column(this=sge.Star(), table=left_cte_name), new_column) + .from_(sge.Table(this=left_cte_name)) + ) + new_expr = _set_query_ctes(new_expr, merged_ctes) + + return SQLGlotIR(expr=new_expr, uid_gen=self.uid_gen) def explode( self, @@ -420,13 +441,22 @@ def explode( def sample(self, fraction: float) -> SQLGlotIR: """Uniform samples a fraction of the rows.""" + uuid_col = sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcol_")), quoted=self.quoted + ) + uuid_expr = sge.Alias(this=sge.func("RAND"), alias=uuid_col) condition = sge.LT( - this=sge.func("RAND"), - expression=sql.literal(fraction, dtypes.FLOAT_DTYPE), + this=uuid_col, + expression=_literal(fraction, dtypes.FLOAT_DTYPE), ) - new_expr = self.expr.as_select_all().where(condition, append=False) - return SQLGlotIR.from_expr(expr=new_expr, uid_gen=self.uid_gen) + new_cte_name = sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcte_")), quoted=self.quoted + ) + new_expr = _select_to_cte( + self.expr.select(uuid_expr, append=True), new_cte_name + ).where(condition, append=False) + return SQLGlotIR(expr=new_expr, uid_gen=self.uid_gen) def aggregate( self, @@ -444,12 +474,20 @@ def aggregate( aggregations_expr = [ sge.Alias( this=expr, - alias=sql.identifier(id), + alias=sge.to_identifier(id, quoted=self.quoted), ) for id, expr in aggregations ] - new_expr = self.expr.select(*[*by_cols, *aggregations_expr]).group_by(*by_cols) + new_expr = _select_to_cte( + self.expr, + sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcte_")), quoted=self.quoted + ), + ) + new_expr = new_expr.group_by(*by_cols).select( + *[*by_cols, *aggregations_expr], append=False + ) condition = _and( tuple( @@ -459,66 +497,59 @@ def aggregate( ) if condition is not None: new_expr = new_expr.where(condition, append=False) - return SQLGlotIR.from_expr(expr=new_expr, uid_gen=self.uid_gen) + return SQLGlotIR(expr=new_expr, uid_gen=self.uid_gen) - def with_ctes( + def window( self, - ctes: tuple[tuple[str, SQLGlotIR], ...], + window_op: sge.Expression, + output_column_id: str, ) -> SQLGlotIR: - sge_ctes = [ - sge.CTE( - this=cte.expr.as_select_all(), - alias=sql.identifier(cte_name), - ) - for cte_name, cte in ctes - ] - select_expr = _set_query_ctes(self.expr.as_select_all(), sge_ctes) - return SQLGlotIR.from_expr(expr=select_expr, uid_gen=self.uid_gen) + return self.project(((output_column_id, window_op),)) - def resample( + def insert( self, - right: SQLGlotIR, - array_col_name: str, - start_expr: sge.Expression, - stop_expr: sge.Expression, - step_expr: sge.Expression, - ) -> SQLGlotIR: - generate_array = sge.func( - "GENERATE_ARRAY", - start_expr, - stop_expr, - step_expr, + destination: bigquery.TableReference, + ) -> str: + """Generates an INSERT INTO SQL statement from the current SELECT clause.""" + return sge.insert(self.expr.subquery(), _table(destination)).sql( + dialect=self.dialect, pretty=self.pretty ) - unnested_column_alias = sql.identifier( - next(self.uid_gen.get_uid_stream("bfcol_")) - ) - unnest_expr = sge.Unnest( - expressions=[generate_array], - alias=sge.TableAlias(columns=[unnested_column_alias]), - ) - - final_col_id = sql.identifier(array_col_name) - - # Build final expression by joining everything directly in a single SELECT - new_expr = ( - sge.Select() - .select(unnested_column_alias.as_(final_col_id)) - .from_(self.expr.as_from_item()) - .join(right.expr.as_from_item(), join_type="cross") - .join(unnest_expr, join_type="cross") + def replace( + self, + destination: bigquery.TableReference, + ) -> str: + """Generates a MERGE statement to replace the destination table's contents. + by the current SELECT clause. + """ + # Workaround for SQLGlot breaking change: + # https://github.com/tobymao/sqlglot/pull/4495 + whens_expr = [ + sge.When(matched=False, source=True, then=sge.Delete()), + sge.When(matched=False, then=sge.Insert(this=sge.Var(this="ROW"))), + ] + whens_str = "\n".join( + when_expr.sql(dialect=self.dialect, pretty=self.pretty) + for when_expr in whens_expr ) - return SQLGlotIR.from_expr(expr=new_expr, uid_gen=self.uid_gen) + merge_str = sge.Merge( + this=_table(destination), + using=self.expr.subquery(), + on=_literal(False, dtypes.BOOL_DTYPE), + ).sql(dialect=self.dialect, pretty=self.pretty) + return f"{merge_str}\n{whens_str}" def _explode_single_column( self, column_name: str, offsets_col: typing.Optional[str] ) -> SQLGlotIR: """Helper method to handle the case of exploding a single column.""" - offset = sql.identifier(offsets_col) if offsets_col else None - column = sql.identifier(column_name) - unnested_column_alias = sql.identifier( - next(self.uid_gen.get_uid_stream("bfcol_")) + offset = ( + sge.to_identifier(offsets_col, quoted=self.quoted) if offsets_col else None + ) + column = sge.to_identifier(column_name, quoted=self.quoted) + unnested_column_alias = sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcol_")), quoted=self.quoted ) unnest_expr = sge.Unnest( expressions=[column], @@ -527,9 +558,17 @@ def _explode_single_column( ) selection = sge.Star(replace=[unnested_column_alias.as_(column)]) + new_expr = _select_to_cte( + self.expr, + sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcte_")), quoted=self.quoted + ), + ) # Use LEFT JOIN to preserve rows when unnesting empty arrays. - new_expr = self.expr.select(selection).join(unnest_expr, join_type="LEFT") - return SQLGlotIR.from_expr(expr=new_expr, uid_gen=self.uid_gen) + new_expr = new_expr.select(selection, append=False).join( + unnest_expr, join_type="LEFT" + ) + return SQLGlotIR(expr=new_expr, uid_gen=self.uid_gen) def _explode_multiple_columns( self, @@ -537,19 +576,27 @@ def _explode_multiple_columns( offsets_col: typing.Optional[str], ) -> SQLGlotIR: """Helper method to handle the case of exploding multiple columns.""" - offset = sql.identifier(offsets_col) if offsets_col else None - columns = [sql.identifier(column_name) for column_name in column_names] + offset = ( + sge.to_identifier(offsets_col, quoted=self.quoted) if offsets_col else None + ) + columns = [ + sge.to_identifier(column_name, quoted=self.quoted) + for column_name in column_names + ] # If there are multiple columns, we need to unnest by zipping the arrays: # https://cloud.google.com/bigquery/docs/arrays#zipping_arrays - column_lengths = [sge.func("ARRAY_LENGTH", column) - 1 for column in columns] + column_lengths = [ + sge.func("ARRAY_LENGTH", sge.to_identifier(column, quoted=self.quoted)) - 1 + for column in columns + ] generate_array = sge.func( "GENERATE_ARRAY", sge.convert(0), sge.func("LEAST", *column_lengths), ) - unnested_offset_alias = sql.identifier( - next(self.uid_gen.get_uid_stream("bfcol_")) + unnested_offset_alias = sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcol_")), quoted=self.quoted ) unnest_expr = sge.Unnest( expressions=[generate_array], @@ -567,13 +614,105 @@ def _explode_multiple_columns( for column in columns ] ) + new_expr = _select_to_cte( + self.expr, + sge.to_identifier( + next(self.uid_gen.get_uid_stream("bfcte_")), quoted=self.quoted + ), + ) # Use LEFT JOIN to preserve rows when unnesting empty arrays. - new_expr = self.expr.select(selection).join(unnest_expr, join_type="LEFT") - return SQLGlotIR.from_expr(expr=new_expr, uid_gen=self.uid_gen) + new_expr = new_expr.select(selection, append=False).join( + unnest_expr, join_type="LEFT" + ) + return SQLGlotIR(expr=new_expr, uid_gen=self.uid_gen) + + +def _select_to_cte(expr: sge.Select, cte_name: sge.Identifier) -> sge.Select: + """Transforms a given sge.Select query by pushing its main SELECT statement + into a new CTE and then generates a 'SELECT * FROM new_cte_name' + for the new query.""" + select_expr = expr.copy() + select_expr, existing_ctes = _pop_query_ctes(select_expr) + new_cte = sge.CTE( + this=select_expr, + alias=cte_name, + ) + new_select_expr = sge.Select().select(sge.Star()).from_(sge.Table(this=cte_name)) + new_select_expr = _set_query_ctes(new_select_expr, [*existing_ctes, new_cte]) + return new_select_expr + + +def _is_null_literal(expr: sge.Expression) -> bool: + """Checks if the given expression is a NULL literal.""" + if isinstance(expr, sge.Null): + return True + if isinstance(expr, sge.Cast) and isinstance(expr.this, sge.Null): + return True + return False + + +def _literal(value: typing.Any, dtype: dtypes.Dtype) -> sge.Expression: + sqlglot_type = sgt.from_bigframes_dtype(dtype) if dtype else None + if sqlglot_type is None: + if not pd.isna(value): + raise ValueError(f"Cannot infer SQLGlot type from None dtype: {value}") + return sge.Null() + + if value is None: + return _cast(sge.Null(), sqlglot_type) + if dtypes.is_struct_like(dtype): + items = [ + _literal(value=value[field_name], dtype=field_dtype).as_( + field_name, quoted=True + ) + for field_name, field_dtype in dtypes.get_struct_fields(dtype).items() + ] + return sge.Struct.from_arg_list(items) + elif dtypes.is_array_like(dtype): + value_type = dtypes.get_array_inner_type(dtype) + values = sge.Array( + expressions=[_literal(value=v, dtype=value_type) for v in value] + ) + return values if len(value) > 0 else _cast(values, sqlglot_type) + elif pd.isna(value) or (isinstance(value, pa.Scalar) and not value.is_valid): + return _cast(sge.Null(), sqlglot_type) + elif dtype == dtypes.JSON_DTYPE: + return sge.ParseJSON(this=sge.convert(str(value))) + elif dtype == dtypes.BYTES_DTYPE: + return _cast(str(value), sqlglot_type) + elif dtypes.is_time_like(dtype): + if isinstance(value, str): + return _cast(sge.convert(value), sqlglot_type) + if isinstance(value, np.generic): + value = value.item() + return _cast(sge.convert(value.isoformat()), sqlglot_type) + elif dtype in (dtypes.NUMERIC_DTYPE, dtypes.BIGNUMERIC_DTYPE): + return _cast(sge.convert(value), sqlglot_type) + elif dtypes.is_geo_like(dtype): + wkt = value if isinstance(value, str) else to_wkt(value) + return sge.func("ST_GEOGFROMTEXT", sge.convert(wkt)) + elif dtype == dtypes.TIMEDELTA_DTYPE: + return sge.convert(utils.timedelta_to_micros(value)) + elif dtype == dtypes.FLOAT_DTYPE: + if np.isinf(value): + return constants._INF if value > 0 else constants._NEG_INF + return sge.convert(value) + else: + if isinstance(value, np.generic): + value = value.item() + return sge.convert(value) + - def _as_subquery(self) -> sge.Subquery: - # Sometimes explicitly need a subquery, e.g. for IN expressions. - return self.expr.as_select_all().subquery() +def _cast(arg: typing.Any, to: str) -> sge.Cast: + return sge.Cast(this=arg, to=to) + + +def _table(table: bigquery.TableReference) -> sge.Table: + return sge.Table( + this=sg.to_identifier(table.table_id, quoted=True), + db=sg.to_identifier(table.dataset_id, quoted=True), + catalog=sg.to_identifier(table.project, quoted=True), + ) def _and(conditions: tuple[sge.Expression, ...]) -> typing.Optional[sge.Expression]: @@ -606,57 +745,77 @@ def _join_condition( joins_nulls: If True, generates complex logic to handle nulls/NaNs. Otherwise, uses a simple equality check where appropriate. """ - if not joins_nulls: + is_floating_types = ( + left.dtype == dtypes.FLOAT_DTYPE and right.dtype == dtypes.FLOAT_DTYPE + ) + if not is_floating_types and not joins_nulls: return sge.EQ(this=left.expr, expression=right.expr) - force_float_domain = False - if left.dtype == dtypes.FLOAT_DTYPE or right.dtype == dtypes.FLOAT_DTYPE: - force_float_domain = True - left_expr1, left_expr2 = _value_to_non_null_identity(left, force_float_domain) - right_expr1, right_expr2 = _value_to_non_null_identity(right, force_float_domain) + is_numeric_types = dtypes.is_numeric( + left.dtype, include_bool=False + ) and dtypes.is_numeric(right.dtype, include_bool=False) + if is_numeric_types: + return _join_condition_for_numeric(left, right) + else: + return _join_condition_for_others(left, right) + + +def _join_condition_for_others( + left: typed_expr.TypedExpr, + right: typed_expr.TypedExpr, +) -> sge.And: + """Generates a join condition for non-numeric types to match pandas's + null-handling logic. + """ + left_str = _cast(left.expr, "STRING") + right_str = _cast(right.expr, "STRING") + left_0 = sge.func("COALESCE", left_str, _literal("0", dtypes.STRING_DTYPE)) + left_1 = sge.func("COALESCE", left_str, _literal("1", dtypes.STRING_DTYPE)) + right_0 = sge.func("COALESCE", right_str, _literal("0", dtypes.STRING_DTYPE)) + right_1 = sge.func("COALESCE", right_str, _literal("1", dtypes.STRING_DTYPE)) return sge.And( - this=sge.EQ(this=left_expr1, expression=right_expr1), - expression=sge.EQ(this=left_expr2, expression=right_expr2), + this=sge.EQ(this=left_0, expression=right_0), + expression=sge.EQ(this=left_1, expression=right_1), ) -def _value_to_non_null_identity( - value: typed_expr.TypedExpr, force_float_domain: bool = False -) -> tuple[sge.Expression, sge.Expression]: - # normal_value -> (normal_value, normal_value) - # null_value -> (0, 1) - # nan_value -> (2, 3) - if dtypes.is_numeric(value.dtype, include_bool=False): - dtype = dtypes.FLOAT_DTYPE if force_float_domain else value.dtype - expr1 = sge.func( - "COALESCE", value.expr, sql.literal(0.0 if force_float_domain else 0, dtype) - ) - expr2 = sge.func( - "COALESCE", value.expr, sql.literal(1.0 if force_float_domain else 1, dtype) - ) - if value.dtype == dtypes.FLOAT_DTYPE: - expr1 = sge.If( - this=sge.IsNan(this=value.expr), - true=sql.literal(2.0, value.dtype), - false=expr1, - ) - expr2 = sge.If( - this=sge.IsNan(this=value.expr), - true=sql.literal(3, value.dtype), - false=expr2, - ) - else: # general case, convert to string and coalesce - expr1 = sge.func( - "COALESCE", - sql.cast(value.expr, "STRING"), - sql.literal("0", dtypes.STRING_DTYPE), - ) - expr2 = sge.func( - "COALESCE", - sql.cast(value.expr, "STRING"), - sql.literal("1", dtypes.STRING_DTYPE), +def _join_condition_for_numeric( + left: typed_expr.TypedExpr, + right: typed_expr.TypedExpr, +) -> sge.And: + """Generates a join condition for non-numeric types to match pandas's + null-handling logic. Specifically for FLOAT types, Pandas treats NaN aren't + equal so need to coalesce as well with different constants. + """ + is_floating_types = ( + left.dtype == dtypes.FLOAT_DTYPE and right.dtype == dtypes.FLOAT_DTYPE + ) + left_0 = sge.func("COALESCE", left.expr, _literal(0, left.dtype)) + left_1 = sge.func("COALESCE", left.expr, _literal(1, left.dtype)) + right_0 = sge.func("COALESCE", right.expr, _literal(0, right.dtype)) + right_1 = sge.func("COALESCE", right.expr, _literal(1, right.dtype)) + if not is_floating_types: + return sge.And( + this=sge.EQ(this=left_0, expression=right_0), + expression=sge.EQ(this=left_1, expression=right_1), ) - return expr1, expr2 + + left_2 = sge.If( + this=sge.IsNan(this=left.expr), true=_literal(2, left.dtype), false=left_0 + ) + left_3 = sge.If( + this=sge.IsNan(this=left.expr), true=_literal(3, left.dtype), false=left_1 + ) + right_2 = sge.If( + this=sge.IsNan(this=right.expr), true=_literal(2, right.dtype), false=right_0 + ) + right_3 = sge.If( + this=sge.IsNan(this=right.expr), true=_literal(3, right.dtype), false=right_1 + ) + return sge.And( + this=sge.EQ(this=left_2, expression=right_2), + expression=sge.EQ(this=left_3, expression=right_3), + ) def _set_query_ctes( @@ -674,3 +833,17 @@ def _set_query_ctes( else: raise ValueError("The expression does not support CTEs.") return new_expr + + +def _pop_query_ctes( + expr: sge.Select, +) -> tuple[sge.Select, list[sge.CTE]]: + """Pops the CTEs of a given sge.Select expression.""" + if "with" in expr.arg_types.keys(): + expr_ctes = expr.args.pop("with", []) + return expr, expr_ctes + elif "with_" in expr.arg_types.keys(): + expr_ctes = expr.args.pop("with_", []) + return expr, expr_ctes + else: + raise ValueError("The expression does not support CTEs.") diff --git a/bigframes/core/eval.py b/bigframes/core/eval.py index aba0f836b7a..82add992589 100644 --- a/bigframes/core/eval.py +++ b/bigframes/core/eval.py @@ -53,10 +53,7 @@ def eval(df: dataframe.DataFrame, expr: str, target: Optional[dataframe.DataFram } # 3 Levels: user -> logging wrapper -> dataframe -> eval helper (this) return vendored_pandas_eval.eval( - expr=expr, - level=3, - target=target, - resolvers=(index_resolver, column_resolver), # type: ignore + expr=expr, level=3, target=target, resolvers=(index_resolver, column_resolver) # type: ignore ) diff --git a/bigframes/core/events.py b/bigframes/core/events.py index d6cef860f6d..d0e5f7ad69b 100644 --- a/bigframes/core/events.py +++ b/bigframes/core/events.py @@ -14,13 +14,11 @@ from __future__ import annotations -import asyncio -import concurrent.futures import dataclasses import datetime import threading +from typing import Any, Callable, Optional, Set import uuid -from typing import Any, Callable, Literal, Optional, Set import google.cloud.bigquery._job_helpers import google.cloud.bigquery.job.query @@ -28,19 +26,9 @@ import bigframes.session.executor -_DEFAULT: Literal["default"] = "default" - -ProgressBarType = Literal["default", "auto", "notebook", "terminal"] | None -QueryPlanType = list[google.cloud.bigquery.job.query.QueryPlanEntry] | None - class Subscriber: - def __init__( - self, - callback: Callable[[EventEnvelope], None], - *, - publisher: Publisher, - ): + def __init__(self, callback: Callable[[Event], None], *, publisher: Publisher): self._publisher = publisher self._callback = callback self._subscriber_id = uuid.uuid4() @@ -67,12 +55,10 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): if exc_value is not None: self( - EventEnvelope( - UnknownErrorEvent( - exc_type=exc_type, - exc_value=exc_value, - traceback=traceback, - ) + UnknownErrorEvent( + exc_type=exc_type, + exc_value=exc_value, + traceback=traceback, ) ) self.close() @@ -82,14 +68,8 @@ class Publisher: def __init__(self): self._subscribers_lock = threading.Lock() self._subscribers: Set[Subscriber] = set() - self._executor: concurrent.futures.Executor = ( - concurrent.futures.ThreadPoolExecutor() - ) - def subscribe( - self, - callback: Callable[[EventEnvelope], None], - ) -> Subscriber: + def subscribe(self, callback: Callable[[Event], None]) -> Subscriber: # TODO(b/448176657): figure out how to handle subscribers/publishers in # a background thread. Maybe subscribers should be thread-local? subscriber = Subscriber(callback, publisher=self) @@ -101,50 +81,16 @@ def unsubscribe(self, subscriber: Subscriber): with self._subscribers_lock: self._subscribers.remove(subscriber) - def publish(self, envelope: EventEnvelope | Event): - if not isinstance(envelope, EventEnvelope): - envelope = EventEnvelope(event=envelope) + def publish(self, event: Event): with self._subscribers_lock: for subscriber in self._subscribers: - subscriber(envelope) - - async def publish_async(self, envelope: EventEnvelope | Event): - if not isinstance(envelope, EventEnvelope): - envelope = EventEnvelope(event=envelope) - with self._subscribers_lock: - subscribers_snapshot = list(self._subscribers) - loop = asyncio.get_running_loop() - tasks = [ - loop.run_in_executor(self._executor, subscriber, envelope) - for subscriber in subscribers_snapshot - ] - return await asyncio.gather(*tasks, return_exceptions=True) + subscriber(event) class Event: pass -@dataclasses.dataclass(frozen=True) -class EventEnvelope: - """An envelope that wraps an execution event with metadata and display options. - - Attributes: - event: - The actual execution event details (e.g., ExecutionStarted, BigQuerySentEvent). - progress_bar: - Specifies the style of progress bar to display during execution. - cell_execution_count: - The 1-indexed IPython/Jupyter notebook cell execution number (e.g. the 'x' in 'In [x]'). - This is NOT a job count, but rather the sequential number of the cell execution in the - current notebook session, used to group and filter execution history on a per-cell basis. - """ - - event: Event - progress_bar: ProgressBarType = _DEFAULT - cell_execution_count: Optional[int] = None - - @dataclasses.dataclass(frozen=True) class SessionClosed(Event): session_id: str @@ -160,7 +106,7 @@ class ExecutionRunning(Event): @dataclasses.dataclass(frozen=True) class ExecutionFinished(Event): - result: bigframes.session.executor.ExecuteResult | None = None + result: Optional[bigframes.session.executor.ExecuteResult] = None @dataclasses.dataclass(frozen=True) @@ -175,16 +121,13 @@ class BigQuerySentEvent(ExecutionRunning): """Query sent to BigQuery.""" query: str - billing_project: str | None = None - location: str | None = None - job_id: str | None = None - request_id: str | None = None + billing_project: Optional[str] = None + location: Optional[str] = None + job_id: Optional[str] = None + request_id: Optional[str] = None @classmethod - def from_bqclient( - cls, - event: google.cloud.bigquery._job_helpers.QuerySentEvent, - ): + def from_bqclient(cls, event: google.cloud.bigquery._job_helpers.QuerySentEvent): return cls( query=event.query, billing_project=event.billing_project, @@ -199,16 +142,13 @@ class BigQueryRetryEvent(ExecutionRunning): """Query sent another time because the previous attempt failed.""" query: str - billing_project: str | None = None - location: str | None = None - job_id: str | None = None - request_id: str | None = None + billing_project: Optional[str] = None + location: Optional[str] = None + job_id: Optional[str] = None + request_id: Optional[str] = None @classmethod - def from_bqclient( - cls, - event: google.cloud.bigquery._job_helpers.QueryRetryEvent, - ): + def from_bqclient(cls, event: google.cloud.bigquery._job_helpers.QueryRetryEvent): return cls( query=event.query, billing_project=event.billing_project, @@ -222,20 +162,19 @@ def from_bqclient( class BigQueryReceivedEvent(ExecutionRunning): """Query received and acknowledged by the BigQuery API.""" - billing_project: str | None = None - location: str | None = None - job_id: str | None = None - statement_type: str | None = None - state: str | None = None - query_plan: QueryPlanType = None - created: datetime.datetime | None = None - started: datetime.datetime | None = None - ended: datetime.datetime | None = None + billing_project: Optional[str] = None + location: Optional[str] = None + job_id: Optional[str] = None + statement_type: Optional[str] = None + state: Optional[str] = None + query_plan: Optional[list[google.cloud.bigquery.job.query.QueryPlanEntry]] = None + created: Optional[datetime.datetime] = None + started: Optional[datetime.datetime] = None + ended: Optional[datetime.datetime] = None @classmethod def from_bqclient( - cls, - event: google.cloud.bigquery._job_helpers.QueryReceivedEvent, + cls, event: google.cloud.bigquery._job_helpers.QueryReceivedEvent ): return cls( billing_project=event.billing_project, @@ -254,22 +193,21 @@ def from_bqclient( class BigQueryFinishedEvent(ExecutionRunning): """Query finished successfully.""" - billing_project: str | None = None - location: str | None = None - query_id: str | None = None - job_id: str | None = None - destination: google.cloud.bigquery.table.TableReference | None = None - total_rows: int | None = None - total_bytes_processed: int | None = None - slot_millis: int | None = None - created: datetime.datetime | None = None - started: datetime.datetime | None = None - ended: datetime.datetime | None = None + billing_project: Optional[str] = None + location: Optional[str] = None + query_id: Optional[str] = None + job_id: Optional[str] = None + destination: Optional[google.cloud.bigquery.table.TableReference] = None + total_rows: Optional[int] = None + total_bytes_processed: Optional[int] = None + slot_millis: Optional[int] = None + created: Optional[datetime.datetime] = None + started: Optional[datetime.datetime] = None + ended: Optional[datetime.datetime] = None @classmethod def from_bqclient( - cls, - event: google.cloud.bigquery._job_helpers.QueryFinishedEvent, + cls, event: google.cloud.bigquery._job_helpers.QueryFinishedEvent ): return cls( billing_project=event.billing_project, diff --git a/bigframes/core/explode.py b/bigframes/core/explode.py index ddd290b0f84..142536a931c 100644 --- a/bigframes/core/explode.py +++ b/bigframes/core/explode.py @@ -14,7 +14,7 @@ """Utility functions for implementing 'explode' functions.""" -from typing import Sequence, Union, cast +from typing import cast, Sequence, Union import bigframes.core.blocks as blocks import bigframes.core.utils as utils diff --git a/bigframes/core/expression.py b/bigframes/core/expression.py index 6c27dfc120b..89bcb9b9207 100644 --- a/bigframes/core/expression.py +++ b/bigframes/core/expression.py @@ -19,17 +19,14 @@ import functools import itertools import typing -from typing import Callable, Generator, Hashable, Mapping, TypeVar, Union +from typing import Callable, Generator, Mapping, TypeVar, Union import pandas as pd -import bigframes.core.identifiers as ids -import bigframes.operations from bigframes import dtypes from bigframes.core import field - -if typing.TYPE_CHECKING: - import bigframes.operations +import bigframes.core.identifiers as ids +import bigframes.operations def const( @@ -42,7 +39,7 @@ def deref(name: str) -> DerefOp: return DerefOp(ids.ColumnId(name)) -def free_var(id: Hashable) -> UnboundVariableExpression: +def free_var(id: str) -> UnboundVariableExpression: return UnboundVariableExpression(id) @@ -55,7 +52,7 @@ class Expression(abc.ABC): """An expression represents a computation taking N scalar inputs and producing a single output scalar.""" @property - def free_variables(self) -> typing.Tuple[Hashable, ...]: + def free_variables(self) -> typing.Tuple[str, ...]: return () @property @@ -74,7 +71,8 @@ def nullable(self) -> bool: @property @abc.abstractmethod - def column_references(self) -> typing.Tuple[ids.ColumnId, ...]: ... + def column_references(self) -> typing.Tuple[ids.ColumnId, ...]: + ... def remap_column_refs( self: TExpression, @@ -88,7 +86,8 @@ def remap_column_refs( @property @abc.abstractmethod - def is_const(self) -> bool: ... + def is_const(self) -> bool: + ... @property @abc.abstractmethod @@ -100,7 +99,8 @@ def is_resolved(self) -> bool: @property @abc.abstractmethod - def output_type(self) -> dtypes.ExpressionType: ... + def output_type(self) -> dtypes.ExpressionType: + ... @abc.abstractmethod def bind_refs( @@ -116,9 +116,7 @@ def bind_refs( @abc.abstractmethod def bind_variables( - self, - bindings: Mapping[Hashable, Expression], - allow_partial_bindings: bool = False, + self, bindings: Mapping[str, Expression], allow_partial_bindings: bool = False ) -> Expression: """Replace variables with expression given in `bindings`. @@ -145,9 +143,8 @@ def is_scalar_expr(self) -> bool: return all(expr.is_scalar_expr for expr in self.children) @abc.abstractmethod - def transform_children( - self, t: Callable[[Expression], Expression] - ) -> Expression: ... + def transform_children(self, t: Callable[[Expression], Expression]) -> Expression: + ... def bottom_up(self, t: Callable[[Expression], Expression]) -> Expression: expr = self.transform_children(lambda child: child.bottom_up(t)) @@ -194,9 +191,7 @@ def output_type(self) -> dtypes.ExpressionType: return self.dtype def bind_variables( - self, - bindings: Mapping[Hashable, Expression], - allow_partial_bindings: bool = False, + self, bindings: Mapping[str, Expression], allow_partial_bindings: bool = False ) -> Expression: return self @@ -231,10 +226,10 @@ def transform_children(self, t: Callable[[Expression], Expression]) -> Expressio class UnboundVariableExpression(Expression): """A variable expression representing an unbound variable.""" - id: Hashable + id: str @property - def free_variables(self) -> typing.Tuple[Hashable, ...]: + def free_variables(self) -> typing.Tuple[str, ...]: return (self.id,) @property @@ -261,9 +256,7 @@ def bind_refs( return self def bind_variables( - self, - bindings: Mapping[Hashable, Expression], - allow_partial_bindings: bool = False, + self, bindings: Mapping[str, Expression], allow_partial_bindings: bool = False ) -> Expression: if self.id in bindings.keys(): return bindings[self.id] @@ -311,9 +304,7 @@ def output_type(self) -> dtypes.ExpressionType: raise ValueError(f"Type of variable {self.id} has not been fixed.") def bind_variables( - self, - bindings: Mapping[Hashable, Expression], - allow_partial_bindings: bool = False, + self, bindings: Mapping[str, Expression], allow_partial_bindings: bool = False ) -> Expression: return self @@ -364,56 +355,6 @@ def output_type(self) -> dtypes.ExpressionType: return self.dtype -@dataclasses.dataclass(frozen=True) -class OmittedArg(Expression): - """Represents an omitted optional arg used calling a function.""" - - @property - def free_variables(self) -> typing.Tuple[Hashable, ...]: - return () - - @property - def is_const(self) -> bool: - return True - - @property - def column_references(self) -> typing.Tuple[ids.ColumnId, ...]: - return () - - @property - def is_resolved(self): - return True # vacuously - - @property - def output_type(self) -> dtypes.ExpressionType: - return None - - def bind_refs( - self, - bindings: Mapping[ids.ColumnId, Expression], - allow_partial_bindings: bool = False, - ) -> OmittedArg: - return self - - def bind_variables( - self, - bindings: Mapping[Hashable, Expression], - allow_partial_bindings: bool = False, - ) -> Expression: - return self - - @property - def is_bijective(self) -> bool: - return True - - @property - def is_identity(self) -> bool: - return True - - def transform_children(self, t: Callable[[Expression], Expression]) -> Expression: - return self - - @dataclasses.dataclass(frozen=True) class OpExpression(Expression): """An expression representing a scalar operation applied to 1 or more argument sub-expressions.""" @@ -432,7 +373,7 @@ def column_references( ) @property - def free_variables(self) -> typing.Tuple[Hashable, ...]: + def free_variables(self) -> typing.Tuple[str, ...]: return tuple( itertools.chain.from_iterable(map(lambda x: x.free_variables, self.inputs)) ) @@ -467,9 +408,7 @@ def output_type(self) -> dtypes.ExpressionType: return self.op.output_type(*input_types) def bind_variables( - self, - bindings: Mapping[Hashable, Expression], - allow_partial_bindings: bool = False, + self, bindings: Mapping[str, Expression], allow_partial_bindings: bool = False ) -> OpExpression: return OpExpression( self.op, diff --git a/bigframes/core/expression_factoring.py b/bigframes/core/expression_factoring.py index 22f1433c8a4..b58330f5a45 100644 --- a/bigframes/core/expression_factoring.py +++ b/bigframes/core/expression_factoring.py @@ -19,6 +19,7 @@ import itertools from typing import ( Callable, + cast, Dict, Generator, Hashable, @@ -29,7 +30,6 @@ Sequence, Tuple, TypeVar, - cast, ) from bigframes.core import ( @@ -64,9 +64,9 @@ def iter_nodes_topo( roots: Sequence[expression.Expression], ) -> Generator[expression.Expression, None, None]: """Returns nodes in reverse topological order, using Kahn's algorithm.""" - child_to_parents: Dict[expression.Expression, list[expression.Expression]] = ( - collections.defaultdict(list) - ) + child_to_parents: Dict[ + expression.Expression, list[expression.Expression] + ] = collections.defaultdict(list) out_degree: Dict[expression.Expression, int] = collections.defaultdict(int) queue: collections.deque[expression.Expression] = collections.deque() @@ -243,13 +243,7 @@ def factor_aggregation(root: nodes.ColumnDef) -> FactoredAggregation: } root_scalar_expr = nodes.ColumnDef( - sub_expressions( - root.expression, - cast( - Mapping[expression.Expression, expression.Expression], agg_outputs_dict - ), - ), - root.id, # type: ignore + sub_expressions(root.expression, agg_outputs_dict), root.id # type: ignore ) return FactoredAggregation( @@ -358,7 +352,9 @@ def push_into_tree( def graph_extract_scalar_exprs() -> Sequence[nodes.ColumnDef]: results: dict[identifiers.ColumnId, expression.Expression] = dict() - while True: # Will converge as each loop either reduces graph size, or fails to find any candidate and breaks + while ( + True + ): # Will converge as each loop either reduces graph size, or fails to find any candidate and breaks candidate_ids = list( id for id in graph.sinks diff --git a/bigframes/core/global_session.py b/bigframes/core/global_session.py index a38280e6447..b055bdb854e 100644 --- a/bigframes/core/global_session.py +++ b/bigframes/core/global_session.py @@ -18,8 +18,8 @@ import threading import traceback +from typing import Callable, Optional, TYPE_CHECKING, TypeVar import warnings -from typing import TYPE_CHECKING, Callable, Iterable, Optional, TypeVar import google.auth.exceptions @@ -124,22 +124,6 @@ def with_default_session(func_: Callable[..., _T], *args, **kwargs) -> _T: return func_(get_global_session(), *args, **kwargs) -def execution_history( - *, - events: Optional[Iterable[bigframes.core.events.Event]] = None, - job_ids: Optional[Iterable[str]] = None, - all_cells: bool = True, -) -> "bigframes.session._ExecutionHistory": - import bigframes.session - - return with_default_session( - bigframes.session.Session.execution_history, - events=events, - job_ids=job_ids, - all_cells=all_cells, - ) - - class _GlobalSessionContext: """ Context manager for testing that sets global session. diff --git a/bigframes/core/googlesql.py b/bigframes/core/googlesql.py deleted file mode 100644 index 8869fbff3ef..00000000000 --- a/bigframes/core/googlesql.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utilities for working with GoogleSqlScalarOps.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Optional, Union - -import pandas as pd - -import bigframes.core.col -import bigframes.core.expression as ex -import bigframes.core.global_session as global_session -import bigframes.core.sentinels as sentinels -import bigframes.series as series -from bigframes.operations import googlesql - -if TYPE_CHECKING: - import bigframes.session - - -def _is_pandas_series(arg: Any) -> bool: - return isinstance(arg, pd.Series) - - -def _find_session(*args: Any) -> Optional[bigframes.session.Session]: - import bigframes.core.indexes as indexes - import bigframes.dataframe as dataframe - - for arg in args: - if isinstance(arg, (series.Series, dataframe.DataFrame, indexes.Index)): - return arg._session - return None - - -def _get_session(*args: Any) -> bigframes.session.Session: - session = _find_session(*args) - if session is not None: - return session - return global_session.get_global_session() - - -def apply_googlesql_scalar_op( - op: googlesql.GoogleSqlScalarOp, - *args: Any, -) -> Union[series.Series, bigframes.core.col.Expression]: - """Applies a GoogleSQL scalar operator to the given arguments. - - Handles a mix of Series, Expression, and literal inputs. - - Args: - op (googlesql.GoogleSqlScalarOp): - The operator to apply. - *args (Any): - The arguments to apply the operator to. - - Returns: - bigframes.pandas.Series | bigframes.core.col.Expression: - The result of the operation. If any of ``args`` is a Series, returns - a Series. Otherwise, returns an Expression. - """ - has_pandas_series = any(_is_pandas_series(arg) for arg in args) - - if has_pandas_series: - session = _get_session(*args) - args = tuple( - session.read_pandas(arg) if _is_pandas_series(arg) else arg for arg in args - ) - - # Find the first Series to use for alignment - first_series = None - for arg in args: - if isinstance(arg, series.Series): - first_series = arg - break - - if first_series is not None: - processed_args: list[Union[bigframes.core.col.Expression, series.Series]] = [] - block = first_series._block - for arg in args: - if isinstance(arg, bigframes.core.col.Expression): - block, col_id = block.project_expr(bigframes.core.col._as_bf_expr(arg)) - processed_args.append(series.Series(block.select_column(col_id))) - elif arg is sentinels.Sentinel.ARGUMENT_DEFAULT: - processed_args.append(bigframes.core.col.Expression(ex.OmittedArg())) - else: - processed_args.append(arg) - - # Apply the n-ary op. _apply_nary_op handles alignment of Series and literals. - result = first_series._apply_nary_op(op, processed_args, ignore_self=True) - result.name = None - return result - - # No Series, return an Expression - expr_args = [] - for arg in args: - if isinstance(arg, bigframes.core.col.Expression): - expr_args.append(bigframes.core.col._as_bf_expr(arg)) - elif arg is sentinels.Sentinel.ARGUMENT_DEFAULT: - expr_args.append(ex.OmittedArg()) - else: - expr_args.append(ex.const(arg)) - - return bigframes.core.col.Expression(ex.OpExpression(op, tuple(expr_args))) diff --git a/bigframes/core/groupby/dataframe_group_by.py b/bigframes/core/groupby/dataframe_group_by.py index 7cc61d43a02..7f9e5d627ab 100644 --- a/bigframes/core/groupby/dataframe_group_by.py +++ b/bigframes/core/groupby/dataframe_group_by.py @@ -23,12 +23,17 @@ import numpy import pandas as pd +from bigframes import session +from bigframes.core import agg_expressions +from bigframes.core import expression as ex import bigframes.core.block_transforms as block_ops -import bigframes.core.block_transforms as block_transforms import bigframes.core.blocks as blocks +from bigframes.core.groupby import aggs, group_by, series_group_by +from bigframes.core.logging import log_adapter import bigframes.core.ordering as order import bigframes.core.utils as utils import bigframes.core.validations as validations +from bigframes.core.window import rolling import bigframes.core.window as windows import bigframes.core.window_spec as window_specs import bigframes.dataframe as df @@ -36,18 +41,12 @@ import bigframes.operations import bigframes.operations.aggregations as agg_ops import bigframes.series as series -from bigframes import session -from bigframes._tools import docs -from bigframes.core import agg_expressions -from bigframes.core import expression as ex -from bigframes.core.groupby import aggs, group_by, series_group_by -from bigframes.core.logging import log_adapter -from bigframes.core.window import rolling @log_adapter.class_logger -@docs.inherit_docs(vendored_pandas_groupby.DataFrameGroupBy) -class DataFrameGroupBy: +class DataFrameGroupBy(vendored_pandas_groupby.DataFrameGroupBy): + __doc__ = vendored_pandas_groupby.GroupBy.__doc__ + def __init__( self, block: blocks.Block, @@ -581,43 +580,11 @@ def agg(self, func=None, **kwargs) -> typing.Union[df.DataFrame, series.Series]: else: return self._agg_named(**kwargs) - def transform(self, func, *args, **kwargs) -> df.DataFrame: - if block_transforms.is_transpiler_eligible(func): - window_spec = window_specs.unbound(grouping_keys=tuple(self._by_col_ids)) - target_cols, labels = self._aggregated_columns() - exprs = [] - for col_id in target_cols: - expr, _ = block_transforms.compile_column_udf( - self._block, - func, - col_id, - args=args, - kwargs=kwargs, - window_spec=window_spec, - ) - exprs.append(expr) - - block = self._block.project_block_exprs( - exprs, - labels=labels, - drop=True, - ) - return df.DataFrame(block) - - raise NotImplementedError( - "DataFrameGroupBy.transform is only supported when experiments.enable_python_transpiler is True and a transpiler-compatible python function is provided." - ) - def _agg_func(self, func) -> df.DataFrame: ids, labels = self._aggregated_columns() - aggregations = [] - for col_id in ids: - if block_transforms.is_transpiler_eligible(func): - expr, _ = block_transforms.compile_column_udf(self._block, func, col_id) - aggregations.append(expr) - else: - aggregations.append(aggs.agg(col_id, agg_ops.lookup_agg_func(func)[0])) - + aggregations = [ + aggs.agg(col_id, agg_ops.lookup_agg_func(func)[0]) for col_id in ids + ] agg_block = self._block.aggregate( by_column_ids=self._by_col_ids, aggregations=aggregations, @@ -628,7 +595,7 @@ def _agg_func(self, func) -> df.DataFrame: return dataframe if self._as_index else self._convert_index(dataframe) def _agg_dict(self, func: typing.Mapping) -> df.DataFrame: - aggregations: typing.List[ex.Expression] = [] + aggregations: typing.List[agg_expressions.Aggregation] = [] column_labels = [] function_labels = [] @@ -640,18 +607,10 @@ def _agg_dict(self, func: typing.Mapping) -> df.DataFrame: funcs_for_id if utils.is_list_like(funcs_for_id) else [funcs_for_id] ) for f in func_list: - if block_transforms.is_transpiler_eligible(f): - expr, name = block_transforms.compile_column_udf( - self._block, f, col_id - ) - aggregations.append(expr) - column_labels.append(label) - function_labels.append(name) - else: - f_op, f_label = agg_ops.lookup_agg_func(f) - aggregations.append(aggs.agg(col_id, f_op)) - column_labels.append(label) - function_labels.append(f_label) + f_op, f_label = agg_ops.lookup_agg_func(f) + aggregations.append(aggs.agg(col_id, f_op)) + column_labels.append(label) + function_labels.append(f_label) agg_block = self._block.aggregate( by_column_ids=self._by_col_ids, aggregations=aggregations, @@ -671,34 +630,23 @@ def _agg_dict(self, func: typing.Mapping) -> df.DataFrame: def _agg_list(self, func: typing.Sequence) -> df.DataFrame: ids, labels = self._aggregated_columns() - aggregations = [] - fn_labels = [] - - for f in func: - if block_transforms.is_transpiler_eligible(f): - fn_labels.append(getattr(f, "__name__", "")) - else: - fn_labels.append(agg_ops.lookup_agg_func(f)[1]) - - for col_id in ids: - for f in func: - if block_transforms.is_transpiler_eligible(f): - expr, _ = block_transforms.compile_column_udf( - self._block, f, col_id - ) - aggregations.append(expr) - else: - aggregations.append(aggs.agg(col_id, agg_ops.lookup_agg_func(f)[0])) + aggregations = [ + aggs.agg(col_id, agg_ops.lookup_agg_func(f)[0]) + for col_id in ids + for f in func + ] if self._block.column_labels.nlevels > 1: + # Restructure MultiIndex for proper format: (idx1, idx2, func) + # rather than ((idx1, idx2), func). column_labels = [ - tuple(label) + (fn_lbl,) + tuple(label) + (agg_ops.lookup_agg_func(f)[1],) for label in labels.to_frame(index=False).to_numpy() - for fn_lbl in fn_labels + for f in func ] else: # Single-level index column_labels = [ - (label, fn_lbl) for label in labels for fn_lbl in fn_labels + (label, agg_ops.lookup_agg_func(f)[1]) for label in labels for f in func ] agg_block = self._block.aggregate( diff --git a/bigframes/core/groupby/group_by.py b/bigframes/core/groupby/group_by.py index 34786e4fd88..1d24e615458 100644 --- a/bigframes/core/groupby/group_by.py +++ b/bigframes/core/groupby/group_by.py @@ -19,10 +19,10 @@ import pandas as pd -import bigframes.enums -import bigframes.operations as ops from bigframes.core import blocks from bigframes.core import expression as ex +import bigframes.enums +import bigframes.operations as ops def block_groupby_iter( diff --git a/bigframes/core/groupby/series_group_by.py b/bigframes/core/groupby/series_group_by.py index fb7845f36d2..a8900cf5455 100644 --- a/bigframes/core/groupby/series_group_by.py +++ b/bigframes/core/groupby/series_group_by.py @@ -23,12 +23,16 @@ import numpy import pandas +from bigframes import session +from bigframes.core import expression as ex import bigframes.core.block_transforms as block_ops -import bigframes.core.block_transforms as block_transforms import bigframes.core.blocks as blocks +from bigframes.core.groupby import aggs, group_by +from bigframes.core.logging import log_adapter import bigframes.core.ordering as order import bigframes.core.utils as utils import bigframes.core.validations as validations +from bigframes.core.window import rolling import bigframes.core.window as windows import bigframes.core.window_spec as window_specs import bigframes.dataframe as df @@ -36,17 +40,12 @@ import bigframes.operations import bigframes.operations.aggregations as agg_ops import bigframes.series as series -from bigframes import session -from bigframes._tools import docs -from bigframes.core import expression as ex -from bigframes.core.groupby import aggs, group_by -from bigframes.core.logging import log_adapter -from bigframes.core.window import rolling @log_adapter.class_logger -@docs.inherit_docs(vendored_pandas_groupby.SeriesGroupBy) class SeriesGroupBy(vendored_pandas_groupby.SeriesGroupBy): + __doc__ = vendored_pandas_groupby.GroupBy.__doc__ + def __init__( self, block: blocks.Block, @@ -259,27 +258,18 @@ def prod(self, *args) -> series.Series: return self._aggregate(agg_ops.product_op) def agg(self, func=None) -> typing.Union[df.DataFrame, series.Series]: + column_names: list[str] = [] if utils.is_dict_like(func): raise NotImplementedError( f"Aggregate with {func} not supported. {constants.FEEDBACK_LINK}" ) - is_single_func = not utils.is_list_like(func) - if is_single_func: + if not utils.is_list_like(func): func = [func] - aggregations = [] - column_labels = [] - for f in func: - if block_transforms.is_transpiler_eligible(f): - expr, name = block_transforms.compile_column_udf( - self._block, f, self._value_column - ) - aggregations.append(expr) - column_labels.append(self._value_name if is_single_func else name) - else: - agg_op, label = agg_ops.lookup_agg_func(f) - aggregations.append(aggs.agg(self._value_column, agg_op)) - column_labels.append(label if not is_single_func else self._value_name) + aggregations = [ + aggs.agg(self._value_column, agg_ops.lookup_agg_func(f)[0]) for f in func + ] + column_names = [agg_ops.lookup_agg_func(f)[1] for f in func] agg_block = self._block.aggregate( by_column_ids=self._by_col_ids, @@ -287,8 +277,8 @@ def agg(self, func=None) -> typing.Union[df.DataFrame, series.Series]: dropna=self._dropna, ) - if column_labels: - agg_block = agg_block.with_column_labels(column_labels) + if column_names: + agg_block = agg_block.with_column_labels(column_names) if len(aggregations) == 1: return series.Series(agg_block) @@ -296,29 +286,6 @@ def agg(self, func=None) -> typing.Union[df.DataFrame, series.Series]: aggregate = agg - def transform(self, func, *args, **kwargs) -> series.Series: - if block_transforms.is_transpiler_eligible(func): - window_spec = window_specs.unbound(grouping_keys=tuple(self._by_col_ids)) - expr, _ = block_transforms.compile_column_udf( - self._block, - func, - self._value_column, - args=args, - kwargs=kwargs, - window_spec=window_spec, - ) - - block = self._block.project_block_exprs( - [expr], - labels=[self._value_name], - drop=True, - ) - return series.Series(block) - - raise NotImplementedError( - "SeriesGroupBy.transform is only supported when experiments.enable_python_transpiler is True and a transpiler-compatible python function is provided." - ) - def value_counts( self, normalize: bool = False, diff --git a/bigframes/core/indexers.py b/bigframes/core/indexers.py index faf76f7d4f0..c60e40880b7 100644 --- a/bigframes/core/indexers.py +++ b/bigframes/core/indexers.py @@ -14,25 +14,19 @@ from __future__ import annotations -import numbers import typing +from typing import Tuple, Union import warnings -from typing import Any, Sequence, Tuple, Union, cast import bigframes_vendored.constants as constants import bigframes_vendored.ibis.common.exceptions as ibis_exceptions -import numpy as np import pandas as pd -import pyarrow as pa -import pyarrow.types # type: ignore import bigframes.core.blocks -import bigframes.core.col import bigframes.core.expression as ex import bigframes.core.guid as guid import bigframes.core.indexes as indexes import bigframes.core.scalar -import bigframes.core.validations as validations import bigframes.core.window_spec as windows import bigframes.dataframe import bigframes.dtypes @@ -42,17 +36,10 @@ if typing.TYPE_CHECKING: LocSingleKey = Union[ - bigframes.series.Series, - indexes.Index, - slice, - bigframes.core.scalar.Scalar, - bigframes.core.col.Expression, + bigframes.series.Series, indexes.Index, slice, bigframes.core.scalar.Scalar ] -_DATAFRAME_ILOC_ERROR = "Only DataFrame.iloc[:, col_indexer] = value is supported." - - class LocSeriesIndexer: def __init__(self, series: bigframes.series.Series): self._series = series @@ -110,9 +97,6 @@ def __getitem__( Other key types are not yet supported. """ - if not _is_noop_slice(key): - validations.enforce_ordered(self._series, "iloc") - return _iloc_getitem_series_or_dataframe(self._series, key) @@ -121,9 +105,9 @@ def __init__(self, series: bigframes.series.Series): self._series = series def __getitem__(self, key: int) -> bigframes.core.scalar.Scalar: - if not _is_integer_scalar(key): + if not isinstance(key, int): raise ValueError("Series iAt based indexing can only have integer indexers") - return self._series.iloc[_to_python_int(key)] + return self._series.iloc[key] class AtSeriesIndexer: @@ -155,14 +139,16 @@ def __init__(self, dataframe: bigframes.dataframe.DataFrame): @typing.overload def __getitem__( self, key: LocSingleKey - ) -> Union[bigframes.dataframe.DataFrame, pd.Series]: ... + ) -> Union[bigframes.dataframe.DataFrame, pd.Series]: + ... # Technically this is wrong since we can have duplicate column labels, but # this is expected to be rare. @typing.overload def __getitem__( self, key: Tuple[LocSingleKey, str] - ) -> Union[bigframes.series.Series, bigframes.core.scalar.Scalar]: ... + ) -> Union[bigframes.series.Series, bigframes.core.scalar.Scalar]: + ... def __getitem__(self, key): # TODO(tbergeron): Pandas will try both splitting 2-tuple into row, index or as 2-part @@ -197,7 +183,14 @@ def __setitem__( key: Tuple[slice, str], value: bigframes.dataframe.SingleItemValue, ): - if isinstance(key, tuple) and len(key) == 2 and _is_noop_slice(key[0]): + if ( + isinstance(key, tuple) + and len(key) == 2 + and isinstance(key[0], slice) + and (key[0].start is None or key[0].start == 0) + and (key[0].step is None or key[0].step == 1) + and key[0].stop is None + ): # TODO(swast): Support setting multiple columns with key[1] as a list # of labels and value as a DataFrame. df = self._dataframe.assign(**{key[1]: value}) @@ -248,41 +241,8 @@ def __getitem__(self, key) -> Union[bigframes.dataframe.DataFrame, pd.Series]: Other key types are not yet supported. """ - requires_ordering = True - if isinstance(key, tuple): - if len(key) > 0: - row_indexer = key[0] - if _is_noop_slice(row_indexer): - requires_ordering = False - elif _is_noop_slice(key): - requires_ordering = False - - if requires_ordering: - validations.enforce_ordered(self._dataframe, "iloc") - return _iloc_getitem_series_or_dataframe(self._dataframe, key) - def __setitem__( - self, - key: Tuple[ - slice, Union[int, typing.Sequence[int], slice, typing.Sequence[bool]] - ], - value: Union[ - bigframes.dataframe.SingleItemValue, bigframes.dataframe.DataFrame - ], - ): - if not (isinstance(key, tuple) and len(key) == 2): - raise NotImplementedError(_DATAFRAME_ILOC_ERROR) - - row_indexer, col_indexer = key - - if not _is_noop_slice(row_indexer): - raise NotImplementedError(_DATAFRAME_ILOC_ERROR) - - col_offsets = _iloc_col_indexer_to_offsets(self._dataframe, col_indexer) - df = self._dataframe._assign_multi_items_by_offsets(col_offsets, value) - self._dataframe._set_block(df._get_block()) - class IatDataFrameIndexer: def __init__(self, dataframe: bigframes.dataframe.DataFrame): @@ -291,21 +251,19 @@ def __init__(self, dataframe: bigframes.dataframe.DataFrame): def __getitem__(self, key: tuple) -> bigframes.core.scalar.Scalar: error_message = "DataFrame.iat should be indexed by a tuple of exactly 2 ints" # we raise TypeError or ValueError under the same conditions that pandas does - if _is_integer_scalar(key): + if isinstance(key, int): raise TypeError(error_message) if not isinstance(key, tuple): raise ValueError(error_message) - key_values_are_ints = [_is_integer_scalar(key_value) for key_value in key] + key_values_are_ints = [isinstance(key_value, int) for key_value in key] if not all(key_values_are_ints): raise ValueError(error_message) if len(key) != 2: raise TypeError(error_message) - row_idx = _to_python_int(key[0]) - col_idx = _to_python_int(key[1]) block: bigframes.core.blocks.Block = self._dataframe._block - column_block = block.select_columns([block.value_columns[col_idx]]) + column_block = block.select_columns([block.value_columns[key[1]]]) column = bigframes.series.Series(column_block) - return column.iloc[row_idx] + return column.iloc[key[0]] class AtDataFrameIndexer: @@ -322,26 +280,18 @@ def __getitem__( return self._dataframe.loc[key] -def _is_noop_slice(key: Any) -> bool: - """Return True if key is a slice selecting all elements in the original order.""" - return ( - isinstance(key, slice) - and (key.start is None or key.start == 0) - and (key.step is None or key.step == 1) - and key.stop is None - ) - - @typing.overload def _loc_getitem_series_or_dataframe( series_or_dataframe: bigframes.series.Series, key -) -> Union[bigframes.core.scalar.Scalar, bigframes.series.Series]: ... +) -> Union[bigframes.core.scalar.Scalar, bigframes.series.Series]: + ... @typing.overload def _loc_getitem_series_or_dataframe( series_or_dataframe: bigframes.dataframe.DataFrame, key -) -> Union[bigframes.dataframe.DataFrame, pd.Series]: ... +) -> Union[bigframes.dataframe.DataFrame, pd.Series]: + ... def _loc_getitem_series_or_dataframe( @@ -353,23 +303,12 @@ def _loc_getitem_series_or_dataframe( pd.Series, bigframes.core.scalar.Scalar, ]: - if _is_noop_slice(key): - return series_or_dataframe.copy() - if isinstance(key, slice): + if (key.start is None) and (key.stop is None) and (key.step is None): + return series_or_dataframe.copy() raise NotImplementedError( f"loc does not yet support indexing with a slice. {constants.FEEDBACK_LINK}" ) - - if isinstance(key, bigframes.core.col.Expression): - label_to_col_ref = { - label: ex.deref(id) - for id, label in series_or_dataframe._block.col_id_to_label.items() - } - resolved_expr = key._value.bind_variables(label_to_col_ref) - result = series_or_dataframe.copy() - result._set_block(series_or_dataframe._block.filter(resolved_expr)) - return result if callable(key): raise NotImplementedError( f"loc does not yet support indexing with a callable. {constants.FEEDBACK_LINK}" @@ -414,7 +353,8 @@ def _perform_loc_list_join( series_or_dataframe: bigframes.series.Series, keys_index: indexes.Index, drop_levels: bool = False, -) -> bigframes.series.Series: ... +) -> bigframes.series.Series: + ... @typing.overload @@ -422,7 +362,8 @@ def _perform_loc_list_join( series_or_dataframe: bigframes.dataframe.DataFrame, keys_index: indexes.Index, drop_levels: bool = False, -) -> bigframes.dataframe.DataFrame: ... +) -> bigframes.dataframe.DataFrame: + ... def _perform_loc_list_join( @@ -477,129 +418,18 @@ def _struct_accessor_check_and_warn( warnings.warn(msg, stacklevel=7, category=bfe.BadIndexerKeyWarning) -def _to_python_int(value: Any) -> int: - if isinstance(value, pa.Scalar): - return int(value.as_py()) - return int(value) - - -def _iloc_clip_to_offset(index: Any, length: int, name: str) -> int: - """Support negative values for offsets.""" - if not _is_integer_scalar(index): - raise TypeError(f"got unexpected {type(index)} for {name}") - offset = _to_python_int(index) - if offset < 0: - offset += length - - if offset < 0 or offset >= length: - raise IndexError(f"{name} {index} is out-of-bounds") - - return offset - - -def _is_integer_scalar(value: Any) -> bool: - return not ( - isinstance(value, bool) - or isinstance(value, np.bool_) - or (isinstance(value, pa.Scalar) and pyarrow.types.is_boolean(value.type)) - ) and ( - isinstance(value, numbers.Integral) - or (isinstance(value, pa.Scalar) and pyarrow.types.is_integer(value.type)) - ) - - -def _is_boolean_scalar(value: Any) -> bool: - return ( - isinstance(value, bool) - or isinstance(value, np.bool_) - or (isinstance(value, pa.Scalar) and pyarrow.types.is_boolean(value.type)) - ) - - -def _truth_val(value: Any) -> bool: - if value is None or value is pd.NA or pd.isna(value): - return False - if isinstance(value, pa.Scalar): - return bool(value.as_py()) if value.is_valid else False - return bool(value) - - -def _is_boolean_indexer(indexer: Any) -> bool: - if hasattr(indexer, "dtype") and pd.api.types.is_bool_dtype(indexer.dtype): - return True - if ( - hasattr(indexer, "type") - and isinstance(indexer.type, pa.DataType) - and pyarrow.types.is_boolean(indexer.type) - ): - return True - if pd.api.types.is_list_like(indexer): - lst = ( - list(indexer) - if not isinstance(indexer, (bigframes.series.Series, indexes.Index)) - else list(indexer.to_pandas()) - ) - if len(lst) > 0 and all( - _is_boolean_scalar(x) or (x is None) or (x is pd.NA) or pd.isna(x) - for x in lst - ): - return any(_is_boolean_scalar(x) for x in lst) - return False - - -def _iloc_col_indexer_to_offsets( - df: bigframes.dataframe.DataFrame, col_indexer: Any -) -> Sequence[int]: - """Convert col_indexer from one of the many pandas-compatible formats to a list of offsets.""" - n_cols = len(df.columns) - - if _is_integer_scalar(col_indexer): - col_offset = _to_python_int(col_indexer) - return [ - _iloc_clip_to_offset( - col_offset, n_cols, "single positional iloc column indexer" - ) - ] - - elif isinstance(col_indexer, slice): - return list(range(*col_indexer.indices(n_cols))) - - elif _is_boolean_indexer(col_indexer): - col_indexer_list = list(col_indexer) - if len(col_indexer_list) != n_cols: - raise ValueError( - f"Boolean iloc column indexer has wrong length: {len(col_indexer_list)} instead of {n_cols}" - ) - return [i for i, val in enumerate(col_indexer_list) if _truth_val(val)] - - elif pd.api.types.is_list_like(col_indexer): - col_indexer_list = list(col_indexer) - return [ - _iloc_clip_to_offset(idx, n_cols, "iloc column indexer") - for idx in col_indexer_list - ] - - raise TypeError(f"got unexpected {type(col_indexer)} for iloc column indexer") - - -def _iloc_df_from_column_offsets( - df: bigframes.dataframe.DataFrame, key: Sequence[int] -) -> bigframes.dataframe.DataFrame: - block = df._block - selected_ids = tuple(block.value_columns[offset] for offset in key) - return bigframes.dataframe.DataFrame(block.select_columns(selected_ids)) - - @typing.overload def _iloc_getitem_series_or_dataframe( series_or_dataframe: bigframes.series.Series, key -) -> Union[bigframes.series.Series, bigframes.core.scalar.Scalar]: ... +) -> Union[bigframes.series.Series, bigframes.core.scalar.Scalar]: + ... @typing.overload def _iloc_getitem_series_or_dataframe( series_or_dataframe: bigframes.dataframe.DataFrame, key -) -> Union[bigframes.dataframe.DataFrame, pd.Series, bigframes.core.scalar.Scalar]: ... +) -> Union[bigframes.dataframe.DataFrame, pd.Series, bigframes.core.scalar.Scalar]: + ... def _iloc_getitem_series_or_dataframe( @@ -611,10 +441,9 @@ def _iloc_getitem_series_or_dataframe( bigframes.core.scalar.Scalar, pd.Series, ]: - if _is_integer_scalar(key): - key_int = _to_python_int(key) - stop_key = key_int + 1 if key_int != -1 else None - internal_slice_result = series_or_dataframe._slice(key_int, stop_key, 1) + if isinstance(key, int): + stop_key = key + 1 if key != -1 else None + internal_slice_result = series_or_dataframe._slice(key, stop_key, 1) result_pd_df = internal_slice_result.to_pandas() if result_pd_df.empty: raise IndexError("single positional indexer is out-of-bounds") @@ -635,22 +464,14 @@ def _iloc_getitem_series_or_dataframe( # len(key) == 2 df = typing.cast(bigframes.dataframe.DataFrame, series_or_dataframe) - if _is_integer_scalar(key[0]) and _is_integer_scalar(key[1]): + if isinstance(key[1], int): return df.iat[key] - - row_indexer, column_indexer = key - column_offsets = _iloc_col_indexer_to_offsets(df, column_indexer) - df_subset = _iloc_df_from_column_offsets(df, column_offsets) - - if _is_integer_scalar(column_indexer): - selected_columns = cast( - Union[bigframes.dataframe.DataFrame, bigframes.series.Series], - df_subset[df_subset.columns[0]], - ) - else: - selected_columns = df_subset - - return _iloc_getitem_series_or_dataframe(selected_columns, row_indexer) + elif isinstance(key[1], list): + columns = df.columns[key[1]] + return _iloc_getitem_series_or_dataframe(df[columns], key[0]) + raise NotImplementedError( + f"iloc does not yet support indexing with {key}. {constants.FEEDBACK_LINK}" + ) elif pd.api.types.is_list_like(key): if len(key) == 0: return typing.cast( @@ -658,26 +479,6 @@ def _iloc_getitem_series_or_dataframe( series_or_dataframe.iloc[0:0], ) - if _is_boolean_indexer(key): - key_list = ( - list(key) - if not isinstance(key, (bigframes.series.Series, indexes.Index)) - else list(key.to_pandas()) - ) - n_rows = len(series_or_dataframe) - if len(key_list) != n_rows: - raise IndexError( - f"Boolean index has wrong length: {len(key_list)} instead of {n_rows}" - ) - key = [i for i, val in enumerate(key_list) if _truth_val(val)] - if len(key) == 0: - return typing.cast( - Union[bigframes.dataframe.DataFrame, bigframes.series.Series], - series_or_dataframe.iloc[0:0], - ) - else: - key = [_to_python_int(k) for k in list(key)] - # Check if both positive index and negative index are necessary if isinstance(key, (bigframes.series.Series, indexes.Index)): # Avoid data download diff --git a/bigframes/core/indexes/base.py b/bigframes/core/indexes/base.py index 32279d36c9a..383534fa4df 100644 --- a/bigframes/core/indexes/base.py +++ b/bigframes/core/indexes/base.py @@ -18,7 +18,7 @@ import functools import typing -from typing import Hashable, Literal, Optional, Sequence, Union, cast, overload +from typing import cast, Hashable, Literal, Optional, overload, Sequence, Union import bigframes_vendored.constants as constants import bigframes_vendored.pandas.core.indexes.base as vendored_pandas_index @@ -26,6 +26,7 @@ import numpy as np import pandas +from bigframes import dtypes import bigframes.core.agg_expressions as ex_types import bigframes.core.block_transforms as block_ops import bigframes.core.blocks as blocks @@ -39,8 +40,6 @@ import bigframes.operations.aggregations as agg_ops import bigframes.series import bigframes.session.execution_spec as ex_spec -from bigframes import dtypes -from bigframes._tools import docs if typing.TYPE_CHECKING: import bigframes.dataframe @@ -48,8 +47,8 @@ import bigframes.series -@docs.inherit_docs(vendored_pandas_index.Index) -class Index: +class Index(vendored_pandas_index.Index): + __doc__ = vendored_pandas_index.Index.__doc__ _query_job = None _block: blocks.Block _linked_frame: Union[ @@ -211,6 +210,7 @@ def is_monotonic_increasing(self) -> bool: @property @validations.requires_ordering() def is_monotonic_decreasing(self) -> bool: + return typing.cast( bool, self._block.is_monotonic_decreasing(self._block.index_columns), @@ -255,6 +255,12 @@ def query_job(self) -> bigquery.QueryJob: self._query_job = query_job return self._query_job + @property + def str(self) -> bigframes.operations.strings.StringMethods: + import bigframes.operations.strings + + return bigframes.operations.strings.StringMethods(self) + def get_loc(self, key) -> typing.Union[int, slice, "bigframes.series.Series"]: """Get integer location, slice or boolean mask for requested label. @@ -293,8 +299,7 @@ def get_loc(self, key) -> typing.Union[int, slice, "bigframes.series.Series"]: count_scalar = ( self._block.session._executor.execute( - count_result, - ex_spec.ExecutionSpec(promise_under_10gb=True), + count_result, ex_spec.ExecutionSpec(promise_under_10gb=True) ) .batches() .to_py_scalar() @@ -309,8 +314,7 @@ def get_loc(self, key) -> typing.Union[int, slice, "bigframes.series.Series"]: position_result = filtered_block._expr.aggregate([(min_agg, "position")]) position_scalar = ( self._block.session._executor.execute( - position_result, - ex_spec.ExecutionSpec(promise_under_10gb=True), + position_result, ex_spec.ExecutionSpec(promise_under_10gb=True) ) .batches() .to_py_scalar() @@ -325,7 +329,6 @@ def get_loc(self, key) -> typing.Union[int, slice, "bigframes.series.Series"]: # Return boolean mask for non-monotonic duplicates mask_block = block_with_offsets.select_columns([match_col_id]) mask_block = mask_block.reset_index(drop=True) - mask_block = mask_block.with_column_labels([None]) result_series = bigframes.series.Series(mask_block) return result_series.astype("boolean") @@ -433,8 +436,7 @@ def sort_values( *, inplace: bool = False, ascending: bool = True, - kind: str | None = None, - na_position: str = "last", + na_position: __builtins__.str = "last", ) -> Index: if na_position not in ["first", "last"]: raise ValueError("Param na_position must be one of 'first' or 'last'") @@ -446,8 +448,7 @@ def sort_values( else order.descending_over(column, na_last) for column in index_columns ] - is_stable = (kind or constants.DEFAULT_SORT_KIND) in constants.STABLE_SORT_KINDS - return Index(self._block.order_by(ordering, stable=is_stable)) + return Index(self._block.order_by(ordering)) def astype( self, @@ -541,7 +542,8 @@ def fillna(self, value=None) -> Index: def rename( self, name: Union[blocks.Label, Sequence[blocks.Label]], - ) -> Index: ... + ) -> Index: + ... @overload def rename( @@ -549,7 +551,8 @@ def rename( name: Union[blocks.Label, Sequence[blocks.Label]], *, inplace: Literal[False], - ) -> Index: ... + ) -> Index: + ... @overload def rename( @@ -557,7 +560,8 @@ def rename( name: Union[blocks.Label, Sequence[blocks.Label]], *, inplace: Literal[True], - ) -> None: ... + ) -> None: + ... def rename( self, @@ -716,12 +720,14 @@ def to_pandas( # type: ignore[overload-overlap] *, allow_large_results: Optional[bool] = ..., dry_run: Literal[False] = ..., - ) -> pandas.Index: ... + ) -> pandas.Index: + ... @overload def to_pandas( self, *, allow_large_results: Optional[bool] = ..., dry_run: Literal[True] = ... - ) -> pandas.Series: ... + ) -> pandas.Series: + ... def to_pandas( self, @@ -771,11 +777,6 @@ def to_list(self, *, allow_large_results: Optional[bool] = None) -> list: def __len__(self): return self.shape[0] - def __bool__(self): - raise ValueError( - "Cannot convert Index into bool. Consider using .empty(), .item(), .any(), or .all() methods." - ) - def item(self): # Docstring is in third_party/bigframes_vendored/pandas/core/indexes/base.py return self.to_series().peek(2).item() @@ -839,13 +840,6 @@ def _apply_binary_op( else: return NotImplemented - # last so as to not shadow __builtins__.str - @property - def str(self) -> bigframes.operations.strings.StringMethods: - import bigframes.operations.strings - - return bigframes.operations.strings.StringMethods(self) - def _should_create_datetime_index(block: blocks.Block) -> bool: if len(block.index.dtypes) != 1: diff --git a/bigframes/core/indexes/datetimes.py b/bigframes/core/indexes/datetimes.py index 763e44be095..23ad8b03b4d 100644 --- a/bigframes/core/indexes/datetimes.py +++ b/bigframes/core/indexes/datetimes.py @@ -20,14 +20,14 @@ datetimes as vendored_pandas_datetime_index, ) -from bigframes._tools import docs from bigframes.core import expression as ex from bigframes.core.indexes.base import Index from bigframes.operations import date_ops -@docs.inherit_docs(vendored_pandas_datetime_index.DatetimeIndex) -class DatetimeIndex(Index): +class DatetimeIndex(Index, vendored_pandas_datetime_index.DatetimeIndex): + __doc__ = vendored_pandas_datetime_index.DatetimeIndex.__doc__ + # Must be above 5000 for pandas to delegate to bigframes for binops __pandas_priority__ = 12000 diff --git a/bigframes/core/indexes/multi.py b/bigframes/core/indexes/multi.py index 0b9681b55f6..cfabd9e70d1 100644 --- a/bigframes/core/indexes/multi.py +++ b/bigframes/core/indexes/multi.py @@ -14,12 +14,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Hashable, Iterable, Optional, Sequence, cast +from typing import cast, Hashable, Iterable, Optional, Sequence, TYPE_CHECKING import bigframes_vendored.pandas.core.indexes.multi as vendored_pandas_multindex import pandas -from bigframes._tools import docs from bigframes.core import blocks from bigframes.core import expression as ex from bigframes.core.indexes.base import Index @@ -28,8 +27,9 @@ import bigframes.session -@docs.inherit_docs(vendored_pandas_multindex.MultiIndex) -class MultiIndex(Index): +class MultiIndex(Index, vendored_pandas_multindex.MultiIndex): + __doc__ = vendored_pandas_multindex.MultiIndex.__doc__ + @classmethod def from_tuples( cls, diff --git a/bigframes/core/interchange.py b/bigframes/core/interchange.py index 4dacedd0142..f6f0bdd1038 100644 --- a/bigframes/core/interchange.py +++ b/bigframes/core/interchange.py @@ -15,10 +15,10 @@ import dataclasses import functools -from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Sequence +from typing import Any, Dict, Iterable, Optional, Sequence, TYPE_CHECKING -import bigframes.enums from bigframes.core import blocks +import bigframes.enums if TYPE_CHECKING: import bigframes.dataframe diff --git a/bigframes/core/local_data.py b/bigframes/core/local_data.py index c05bda7a7fb..0ef24089b2b 100644 --- a/bigframes/core/local_data.py +++ b/bigframes/core/local_data.py @@ -21,8 +21,8 @@ import io import itertools import json +from typing import Any, Callable, cast, Generator, Iterable, Literal, Optional, Union import uuid -from typing import Any, Callable, Generator, Iterable, Literal, Optional, Union, cast import geopandas # type: ignore import numpy @@ -31,9 +31,9 @@ import pyarrow as pa import pyarrow.parquet # type: ignore +from bigframes.core import pyarrow_utils import bigframes.core.schema as schemata import bigframes.dtypes -from bigframes.core import identifiers, pyarrow_utils @dataclasses.dataclass(frozen=True) @@ -154,9 +154,6 @@ def to_arrow( else: return schema, batches - def is_nullable(self, column_id: identifiers.ColumnId) -> bool: - return self.data.column(column_id.name).null_count > 0 - def to_pyarrow_table( self, *, @@ -248,11 +245,10 @@ def iter_array( elif dtype == bigframes.dtypes.TIMEDELTA_DTYPE: if duration_type == "int": yield from map( - lambda x: ( - ((x.days * 3600 * 24) + x.seconds) * 1_000_000 + x.microseconds - if x is not None - else x - ), + lambda x: ((x.days * 3600 * 24) + x.seconds) * 1_000_000 + + x.microseconds + if x is not None + else x, values, ) else: @@ -424,7 +420,7 @@ def _get_managed_storage_type(dtype: bigframes.dtypes.Dtype) -> pa.DataType: def _recursive_map_types( - f: Callable[[pa.DataType], pa.DataType], + f: Callable[[pa.DataType], pa.DataType] ) -> Callable[[pa.DataType], pa.DataType]: @functools.wraps(f) def recursive_f(type: pa.DataType) -> pa.DataType: diff --git a/bigframes/core/logging/log_adapter.py b/bigframes/core/logging/log_adapter.py index 83d300a708b..77c09437c0e 100644 --- a/bigframes/core/logging/log_adapter.py +++ b/bigframes/core/logging/log_adapter.py @@ -17,8 +17,8 @@ import threading from typing import List, Optional -import pandas from google.cloud import bigquery +import pandas _lock = threading.Lock() @@ -172,14 +172,14 @@ def wrapper(*args, **kwargs): base_name = custom_base_name full_method_name = f"{base_name.lower()}-{api_method_name}" + # Track directly called methods + if len(_call_stack) == 0: + session = _find_session(*args, **kwargs) + add_api_method(full_method_name, session=session) + _call_stack.append(full_method_name) try: - # Track directly called methods - if len(_call_stack) == 1: - session = _find_session(*args, **kwargs) - add_api_method(full_method_name, session=session) - return method(*args, **kwargs) except (NotImplementedError, TypeError) as e: # Log method parameters that are implemented in pandas but either missing (TypeError) @@ -220,12 +220,12 @@ def wrapped(*args, **kwargs): property_name = prop.__name__ full_property_name = f"{class_name.lower()}-{property_name.lower()}" + if len(_call_stack) == 0: + session = _find_session(*args, **kwargs) + add_api_method(full_property_name, session=session) + _call_stack.append(full_property_name) try: - if len(_call_stack) == 1: - session = _find_session(*args, **kwargs) - add_api_method(full_property_name, session=session) - return prop(*args, **kwargs) finally: _call_stack.pop() @@ -309,9 +309,7 @@ def _is_session_initialized(session): Because the method logger could get called before Session.__init__ has a chance to run, we use the globals in that case. """ - return hasattr(session, "_api_methods_lock") and isinstance( - getattr(session, "_api_methods", None), list - ) + return hasattr(session, "_api_methods_lock") and hasattr(session, "_api_methods") def _find_session(*args, **kwargs): @@ -319,13 +317,13 @@ def _find_session(*args, **kwargs): # imports log_adapter. from bigframes.session import Session - for arg in args: - if isinstance(arg, Session) and _is_session_initialized(arg): - return arg - if hasattr(arg, "__dict__") and "_block" in arg.__dict__: - session = getattr(arg, "_session", None) - if isinstance(session, Session) and _is_session_initialized(session): - return session + session = args[0] if args else None + if ( + session is not None + and isinstance(session, Session) + and _is_session_initialized(session) + ): + return session session = kwargs.get("session") if ( diff --git a/bigframes/core/nodes.py b/bigframes/core/nodes.py index e88a78fae5c..ddccb39ef98 100644 --- a/bigframes/core/nodes.py +++ b/bigframes/core/nodes.py @@ -22,24 +22,25 @@ from typing import ( AbstractSet, Callable, + cast, Iterable, Mapping, Optional, Sequence, Tuple, - cast, ) +from bigframes.core import agg_expressions, bq_data, identifiers, local_data, sequences +from bigframes.core.bigframe_node import BigFrameNode, COLUMN_SET import bigframes.core.expression as ex +from bigframes.core.field import Field +from bigframes.core.ordering import OrderingExpression, RowOrdering import bigframes.core.slices as slices import bigframes.core.window_spec as window import bigframes.dtypes -from bigframes.core import agg_expressions, bq_data, identifiers, local_data, sequences -from bigframes.core.bigframe_node import COLUMN_SET, BigFrameNode -from bigframes.core.field import Field -from bigframes.core.ordering import OrderingExpression, RowOrdering if typing.TYPE_CHECKING: + import bigframes.core.ordering as orderings import bigframes.session @@ -68,14 +69,17 @@ class AdditiveNode: @property @abc.abstractmethod - def added_fields(self) -> Tuple[Field, ...]: ... + def added_fields(self) -> Tuple[Field, ...]: + ... @property @abc.abstractmethod - def additive_base(self) -> BigFrameNode: ... + def additive_base(self) -> BigFrameNode: + ... @abc.abstractmethod - def replace_additive_base(self, BigFrameNode) -> BigFrameNode: ... + def replace_additive_base(self, BigFrameNode) -> BigFrameNode: + ... @dataclasses.dataclass(frozen=True, eq=False) @@ -203,8 +207,6 @@ class InNode(BigFrameNode, AdditiveNode): right_child: BigFrameNode left_col: ex.DerefOp indicator_col: identifiers.ColumnId - # For matching left_col to right_child[0], if true, nulls match nulls, if false, nulls don't match nulls - nulls_equal: bool = True def _validate(self): assert len(self.right_child.fields) == 1 @@ -272,7 +274,10 @@ def additive_base(self) -> BigFrameNode: @property def joins_nulls(self) -> bool: - return self.nulls_equal + left_nullable = self.left_child.field_by_id[self.left_col.id].nullable + # assumption: right side has one column + right_nullable = self.right_child.fields[0].nullable + return left_nullable or right_nullable @property def _node_expressions(self): @@ -314,15 +319,12 @@ class JoinNode(BigFrameNode): right_child: BigFrameNode conditions: typing.Tuple[typing.Tuple[ex.DerefOp, ex.DerefOp], ...] type: typing.Literal["inner", "outer", "left", "right", "cross"] - # choose to treat nulls as equal or not for purposes of the join - # pandas treats nulls as equal, sql does not - nulls_equal: bool propogate_order: bool def _validate(self): - assert not (set(self.left_child.ids) & set(self.right_child.ids)), ( - "Join ids collide" - ) + assert not ( + set(self.left_child.ids) & set(self.right_child.ids) + ), "Join ids collide" @property def row_preserving(self) -> bool: @@ -356,7 +358,13 @@ def fields(self) -> Sequence[Field]: @property def joins_nulls(self) -> bool: - return self.nulls_equal + for left_ref, right_ref in self.conditions: + if ( + self.left_child.field_by_id[left_ref.id].nullable + and self.right_child.field_by_id[right_ref.id].nullable + ): + return True + return False @functools.cached_property def variables_introduced(self) -> int: @@ -670,13 +678,7 @@ class ReadLocalNode(LeafNode): @property def fields(self) -> Sequence[Field]: fields = tuple( - Field( - col_id, - self.local_data_source.schema.get_type(source_id), - nullable=self.local_data_source.is_nullable( - identifiers.ColumnId(source_id) - ), - ) + Field(col_id, self.local_data_source.schema.get_type(source_id)) for col_id, source_id in self.scan_list.items ) @@ -823,7 +825,9 @@ def variables_introduced(self) -> int: @property def row_count(self) -> typing.Optional[int]: - return self.source.n_rows + if self.source.sql_predicate is None and self.source.table.is_physically_stored: + return self.source.n_rows + return None @property def node_defined_ids(self) -> Tuple[identifiers.ColumnId, ...]: @@ -845,10 +849,10 @@ def remap_refs( ) -> ReadTableNode: return self - def pull_out_order(self): + def with_order_cols(self): # Maybe the ordering should be required to always be in the scan list, and then we won't need this? if self.source.ordering is None: - return self, RowOrdering() + return self, orderings.RowOrdering() order_cols = {col.sql for col in self.source.ordering.referenced_columns} scan_cols = {col.source_id for col in self.scan_list.items} @@ -862,18 +866,10 @@ def pull_out_order(self): ] new_scan_list = ScanList(items=(*self.scan_list.items, *new_scan_cols)) new_order = self.source.ordering.remap_column_refs( - { - identifiers.ColumnId(item.source_id): item.id - for item in new_scan_list.items - }, + {identifiers.ColumnId(item.source_id): item.id for item in new_scan_cols}, allow_partial_bindings=True, ) - new_node = dataclasses.replace( - self, - scan_list=new_scan_list, - source=self.source.with_ordering(RowOrdering()), - ) - return new_node, new_order + return dataclasses.replace(self, scan_list=new_scan_list), new_order @dataclasses.dataclass(frozen=True, eq=False) @@ -998,8 +994,7 @@ def remap_refs( @dataclasses.dataclass(frozen=True, eq=False) class OrderByNode(UnaryNode): by: Tuple[OrderingExpression, ...] - stable: bool = True - # This is an optimization, if true, can discard previous orderings, even if doing a stable sort + # This is an optimization, if true, can discard previous orderings. # might be a total ordering even if false is_total_order: bool = False @@ -1720,39 +1715,6 @@ def _node_expressions(self): return tuple(ref for ref, _ in self.output_cols) -@dataclasses.dataclass(frozen=True, eq=False) -class CteNode(UnaryNode): - """ - Semantically a no-op, used to indicate shared subtrees and act as optimization boundary. - """ - - @property - def fields(self) -> Sequence[Field]: - return self.child.fields - - @property - def variables_introduced(self) -> int: - return 0 - - @property - def row_count(self) -> Optional[int]: - return self.child.row_count - - @property - def node_defined_ids(self) -> Tuple[identifiers.ColumnId, ...]: - return () - - def remap_vars( - self, mappings: Mapping[identifiers.ColumnId, identifiers.ColumnId] - ) -> CteNode: - return self - - def remap_refs( - self, mappings: Mapping[identifiers.ColumnId, identifiers.ColumnId] - ) -> CteNode: - return self - - # Tree operators def top_down( root: BigFrameNode, diff --git a/bigframes/core/ordering.py b/bigframes/core/ordering.py index 7ad8b6f567a..50b3cee8aa6 100644 --- a/bigframes/core/ordering.py +++ b/bigframes/core/ordering.py @@ -14,9 +14,9 @@ from __future__ import annotations -import typing from dataclasses import dataclass, field from enum import Enum +import typing from typing import Callable, Mapping, Optional, Sequence, Set, Union import bigframes.core.expression as expression @@ -341,13 +341,15 @@ def remap_column_refs( def join( self, other: TotalOrdering, - ) -> TotalOrdering: ... + ) -> TotalOrdering: + ... @typing.overload def join( self, other: RowOrdering, - ) -> RowOrdering: ... + ) -> RowOrdering: + ... def join( self, diff --git a/bigframes/core/pruning.py b/bigframes/core/pruning.py index f98b8eb5d58..2542c8b6f00 100644 --- a/bigframes/core/pruning.py +++ b/bigframes/core/pruning.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import TYPE_CHECKING, Set +from typing import Set, TYPE_CHECKING import bigframes.core.expression as ex import bigframes.core.identifiers as ids diff --git a/bigframes/core/py_expressions.py b/bigframes/core/py_expressions.py deleted file mode 100644 index da937677050..00000000000 --- a/bigframes/core/py_expressions.py +++ /dev/null @@ -1,595 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import dataclasses -import itertools -import operator -from types import ModuleType -from typing import Callable, Hashable, Mapping, Optional, Tuple - -import bigframes.core.agg_expressions as agg_exprs -import bigframes.operations.aggregations as agg_ops -import bigframes.operations.python_op_maps as python_op_maps -from bigframes import dtypes -from bigframes.core import identifiers -from bigframes.core import window_spec as window_specs -from bigframes.core.expression import ( - Expression, - OpExpression, - ScalarConstantExpression, - UnboundVariableExpression, - const, - deref, -) -from bigframes.operations import ( - NUMPY_TO_BINOP, - NUMPY_TO_OP, - ScalarOp, - generic_ops, - numeric_ops, -) - -_CALLABLE_TO_OP = { - **NUMPY_TO_OP, - **NUMPY_TO_BINOP, -} - -_BUILTIN_CALLABLES = { - str: generic_ops.AsTypeOp(dtypes.STRING_DTYPE), - abs: numeric_ops.abs_op, -} - - -@dataclasses.dataclass(frozen=True) -class GetAttr(Expression): - input: Expression - attr: str - - @property - def column_references( - self, - ) -> Tuple[identifiers.ColumnId, ...]: - return self.input.column_references - - @property - def free_variables(self) -> tuple[Hashable, ...]: - return self.input.free_variables - - @property - def is_const(self) -> bool: - return False - - @property - def children(self): - return (self.input,) - - @property - def nullable(self) -> bool: - return True - - @property - def is_resolved(self) -> bool: - return False - - @property - def output_type(self) -> dtypes.ExpressionType: - raise ValueError(f"Type of expression {self} has not been fixed.") - - @property - def is_bijective(self) -> bool: - # TODO: Mark individual functions as bijective? - return False - - @property - def deterministic(self) -> bool: - return True - - def transform_children(self, t: Callable[[Expression], Expression]) -> Expression: - new_input = t(self.input) - if new_input != self.input: - return dataclasses.replace(self, input=new_input) - return self - - def bind_variables( - self, - bindings: Mapping[Hashable, Expression], - allow_partial_bindings: bool = False, - ) -> GetAttr: - return GetAttr( - self.input.bind_variables( - bindings, allow_partial_bindings=allow_partial_bindings - ), - self.attr, - ) - - def bind_refs( - self, - bindings: Mapping[identifiers.ColumnId, Expression], - allow_partial_bindings: bool = False, - ) -> GetAttr: - return GetAttr( - self.input.bind_refs( - bindings, allow_partial_bindings=allow_partial_bindings - ), - self.attr, - ) - - -@dataclasses.dataclass(frozen=True) -class GetItem(Expression): - input: Expression - key: Expression - - @property - def column_references(self) -> Tuple[identifiers.ColumnId, ...]: - return self.input.column_references + self.key.column_references - - @property - def free_variables(self) -> tuple[Hashable, ...]: - return self.input.free_variables + self.key.free_variables - - @property - def is_const(self) -> bool: - return False - - @property - def children(self): - return (self.input, self.key) - - @property - def nullable(self) -> bool: - return True - - @property - def is_resolved(self) -> bool: - return False - - @property - def output_type(self) -> dtypes.ExpressionType: - raise ValueError(f"Type of expression {self} has not been fixed.") - - @property - def is_bijective(self) -> bool: - return False - - @property - def deterministic(self) -> bool: - return True - - def transform_children(self, t: Callable[[Expression], Expression]) -> Expression: - new_input = t(self.input) - new_key = t(self.key) - if new_input != self.input or new_key != self.key: - return dataclasses.replace(self, input=new_input, key=new_key) - return self - - def bind_variables( - self, - bindings: Mapping[Hashable, Expression], - allow_partial_bindings: bool = False, - ) -> GetItem: - return GetItem( - self.input.bind_variables( - bindings, allow_partial_bindings=allow_partial_bindings - ), - self.key.bind_variables( - bindings, allow_partial_bindings=allow_partial_bindings - ), - ) - - def bind_refs( - self, - bindings: Mapping[identifiers.ColumnId, Expression], - allow_partial_bindings: bool = False, - ) -> GetItem: - return GetItem( - self.input.bind_refs( - bindings, allow_partial_bindings=allow_partial_bindings - ), - self.key.bind_refs(bindings, allow_partial_bindings=allow_partial_bindings), - ) - - -@dataclasses.dataclass(frozen=True) -class Module(Expression): - """An expression representing a module reference.""" - - module: ModuleType - - @property - def is_const(self) -> bool: - return True - - @property - def column_references(self) -> Tuple[identifiers.ColumnId, ...]: - return () - - @property - def nullable(self) -> bool: - return True # type: ignore - - @property - def is_resolved(self) -> bool: - return False - - @property - def output_type(self) -> dtypes.ExpressionType: - raise ValueError("Module expression does not have a type.") - - def bind_variables( - self, - bindings: Mapping[Hashable, Expression], - allow_partial_bindings: bool = False, - ) -> Expression: - return self - - def bind_refs( - self, - bindings: Mapping[identifiers.ColumnId, Expression], - allow_partial_bindings: bool = False, - ) -> Module: - return self - - @property - def is_bijective(self) -> bool: - # () <-> value - return True - - def transform_children(self, t: Callable[[Expression], Expression]) -> Expression: - return self - - -@dataclasses.dataclass(frozen=True) -class PyObject(Expression): - """An expression representing a module reference.""" - - value: Hashable - - @property - def is_const(self) -> bool: - return True - - @property - def column_references(self) -> Tuple[identifiers.ColumnId, ...]: - return () - - @property - def nullable(self) -> bool: - return True # type: ignore - - @property - def is_resolved(self) -> bool: - return False - - @property - def output_type(self) -> dtypes.ExpressionType: - raise ValueError("PyObject expression does not have a type.") - - def bind_variables( - self, - bindings: Mapping[Hashable, Expression], - allow_partial_bindings: bool = False, - ) -> Expression: - return self - - def bind_refs( - self, - bindings: Mapping[identifiers.ColumnId, Expression], - allow_partial_bindings: bool = False, - ) -> PyObject: - return self - - @property - def is_bijective(self) -> bool: - # () <-> value - return True - - def transform_children(self, t: Callable[[Expression], Expression]) -> Expression: - return self - - -@dataclasses.dataclass(frozen=True) -class Call(Expression): - """An expression representing a scalar constant.""" - - # TODO: Further constrain? - callable: Expression - inputs: Tuple[Expression, ...] - - @property - def column_references( - self, - ) -> Tuple[identifiers.ColumnId, ...]: - return tuple( - itertools.chain.from_iterable( - map(lambda x: x.column_references, self.children) - ) - ) - - @property - def free_variables(self) -> tuple[Hashable, ...]: - return tuple( - itertools.chain.from_iterable( - map(lambda x: x.free_variables, self.children) - ) - ) - - @property - def is_const(self) -> bool: - return False - - @property - def children(self): - return (self.callable, *self.inputs) - - @property - def nullable(self) -> bool: - return True - - @property - def is_resolved(self) -> bool: - return False - - @property - def output_type(self) -> dtypes.ExpressionType: - raise ValueError(f"Type of expression {self} has not been fixed.") - - @property - def is_bijective(self) -> bool: - # TODO: Mark individual functions as bijective? - return False - - @property - def deterministic(self) -> bool: - return True - - def transform_children(self, t: Callable[[Expression], Expression]) -> Expression: - return dataclasses.replace( - self, - callable=t(self.callable), - inputs=tuple(t(input) for input in self.inputs), - ) - - def bind_variables( - self, - bindings: Mapping[Hashable, Expression], - allow_partial_bindings: bool = False, - ) -> Call: - return Call( - callable=self.callable.bind_variables( - bindings, allow_partial_bindings=allow_partial_bindings - ), - inputs=tuple( - input.bind_variables( - bindings, allow_partial_bindings=allow_partial_bindings - ) - for input in self.inputs - ), - ) - - def bind_refs( - self, - bindings: Mapping[identifiers.ColumnId, Expression], - allow_partial_bindings: bool = False, - ) -> Call: - return Call( - callable=self.callable.bind_refs( - bindings, allow_partial_bindings=allow_partial_bindings - ), - inputs=tuple( - input.bind_refs(bindings, allow_partial_bindings=allow_partial_bindings) - for input in self.inputs - ), - ) - - -# TODO: Mode that resolves free variable attrs as columns -def resolve_py_exprs( - expression: Expression, - series_arg: Optional[str] = None, - series_attrs: Mapping[Hashable, str] | None = None, - col_series_args: Mapping[str, str] | None = None, - window_spec: window_specs.WindowSpec | None = None, -) -> Expression: - """ - Replace all PyObject, attribute, item, and call expressions bottom-up. - - This function translates unresolved python expressions (like GetAttr, GetItem, - Call, PyObject) into resolved BigQuery expressions (like OpExpression, DerefOp, - Aggregation, ScalarConstantExpression) by binding them to the specified context. - - Args: - expression: The unresolved python expression to translate. - series_arg: The name of the parameter representing the row (for row-wise UDFs like - apply axis=1) or the DataFrame group (for DataFrameGroupBy.apply). - series_attrs: A mapping of attribute/item names to column IDs for the series_arg. - When GetAttr(series_arg, attr) or GetItem(series_arg, key) is encountered, - it is resolved to deref(column_id). - col_series_args: A mapping of parameter names to column IDs for parameters that - represent a single Series/column directly (for SeriesGroupBy.apply). When - UnboundVariableExpression(arg_name) is encountered and arg_name is in - col_series_args, it is resolved directly to deref(column_id). - window_spec: Optional window spec. When provided, aggregations inside calls will - be converted to WindowExpression using this spec. - - Returns: - The resolved BigQuery Expression. - """ - - def resolve_expr_if_call(expr: Expression) -> Expression: - if isinstance(expr, Call): - return resolve_call(expr, window_spec=window_spec) - return expr - - def resolve_attrs(expr: Expression) -> Expression: - if isinstance(expr, GetAttr): - return _resolve_getattr(expr, series_arg, series_attrs) - if isinstance(expr, GetItem): - return _resolve_getitem(expr, series_arg, series_attrs, col_series_args) - return expr - - def resolve_series_var(expr: Expression) -> Expression: - if ( - col_series_args is not None - and isinstance(expr, UnboundVariableExpression) - and isinstance(expr.id, str) - and expr.id in col_series_args - ): - return deref(col_series_args[expr.id]) - return expr - - def resolve_pyobjs(expr: Expression) -> Expression: - if isinstance(expr, PyObject): - return const(expr.value) - return expr - - wo_calls = expression.bottom_up(resolve_expr_if_call) - wo_attrs = wo_calls.bottom_up(resolve_attrs) - wo_vars = wo_attrs.bottom_up(resolve_series_var) - return wo_vars.bottom_up(resolve_pyobjs) - - -def _resolve_getattr( - expression: GetAttr, - series_arg: Optional[str], - series_attrs: Mapping[Hashable, str] | None, -) -> Expression: - if isinstance(expression.input, Module): - # resolves things like Math.pi - return PyObject(getattr(expression.input.module, expression.attr)) - # Resolve attribute access on the series/row argument - if ( - series_arg is not None - and series_attrs is not None - and isinstance(expression.input, UnboundVariableExpression) - and expression.input.id == series_arg - and expression.attr in series_attrs - ): - return deref(series_attrs[expression.attr]) - return expression - - -def _resolve_getitem( - expression: GetItem, - series_arg: Optional[str], - series_attrs: Mapping[Hashable, str] | None, - col_series_args: Mapping[str, str] | None, -) -> Expression: - # Resolve subscript/item access on the series/row argument - key_val = None - if isinstance(expression.key, PyObject): - key_val = expression.key.value - elif isinstance(expression.key, ScalarConstantExpression): - key_val = expression.key.value - - is_series_var = ( - series_arg is not None - and isinstance(expression.input, UnboundVariableExpression) - and expression.input.id == series_arg - ) - - if is_series_var and series_attrs is not None: - if key_val is None: - raise NotImplementedError("Dynamic column lookup is not supported.") - if key_val in series_attrs: - return deref(series_attrs[key_val]) - else: - raise KeyError(f"Column '{key_val}' not found.") - - is_columnar_var = ( - col_series_args is not None - and isinstance(expression.input, UnboundVariableExpression) - and expression.input.id in col_series_args - ) - - if is_columnar_var: - raise NotImplementedError( - "Subscripting a Series/column is not supported in this UDF context." - ) - - if key_val is not None: - if isinstance(key_val, (str, int)): - return OpExpression(generic_ops.GetItemOp(key_val), (expression.input,)) - else: - raise NotImplementedError( - f"Subscript key of type '{type(key_val).__name__}' is not supported." - ) - else: - return OpExpression( - generic_ops.DynamicGetItemOp(), (expression.input, expression.key) - ) - - -def resolve_call( - call: Call, window_spec: window_specs.WindowSpec | None = None -) -> Expression: - callable = call.callable - if isinstance(callable, GetAttr): - attr = callable.attr - if isinstance(callable.input, Module): - fn = getattr(callable.input.module, attr) - if fn in python_op_maps.PYTHON_TO_BIGFRAMES: - op = python_op_maps.PYTHON_TO_BIGFRAMES[fn] - return OpExpression(op, call.inputs) - if fn in _CALLABLE_TO_OP: - op = _CALLABLE_TO_OP[fn] - return OpExpression(op, call.inputs) - elif isinstance(callable.input, PyObject) and isinstance( - callable.input.value, type - ): - fn = getattr(callable.input.value, attr, None) - if fn in python_op_maps.PYTHON_TO_BIGFRAMES: - op = python_op_maps.PYTHON_TO_BIGFRAMES[fn] - return OpExpression(op, call.inputs) - else: - # Method call on an expression (e.g. df.col.sum() or s.mean()) - try: - agg_op, _ = agg_ops.lookup_agg_func(attr) - - if isinstance(agg_op, agg_ops.UnaryAggregateOp): - agg_expr: agg_exprs.Aggregation = agg_exprs.UnaryAggregation( - agg_op, callable.input - ) - if window_spec is not None: - return agg_exprs.WindowExpression(agg_expr, window_spec) - return agg_expr - elif isinstance(agg_op, agg_ops.NullaryAggregateOp): - agg_expr = agg_exprs.NullaryAggregation(agg_op) - if window_spec is not None: - return agg_exprs.WindowExpression(agg_expr, window_spec) - return agg_expr - except ValueError: - pass - - # Support common scalar method calls on Series/expressions - if (method_op := python_op_maps.SERIES_METHOD_TO_OP.get(attr)) is not None: - if isinstance(method_op, ScalarOp): - return OpExpression(method_op, (callable.input,)) - - elif isinstance(callable, PyObject): - if callable.value == operator.getitem: - return GetItem(call.inputs[0], call.inputs[1]) - if isinstance(callable.value, ScalarOp): - return OpExpression(callable.value, call.inputs) - if callable.value in python_op_maps.PYTHON_TO_BIGFRAMES: - op = python_op_maps.PYTHON_TO_BIGFRAMES[callable.value] # type: ignore - return OpExpression(op, call.inputs) - if callable.value in _BUILTIN_CALLABLES: - return OpExpression(_BUILTIN_CALLABLES[callable.value], call.inputs) - - raise NotImplementedError( - f"No implementation available for call expression: {call}" - ) diff --git a/bigframes/core/pyformat.py b/bigframes/core/pyformat.py index dfd91ba1ad0..8f49556ff4c 100644 --- a/bigframes/core/pyformat.py +++ b/bigframes/core/pyformat.py @@ -21,15 +21,15 @@ import string import typing -from typing import Any, Optional, Tuple, Union +from typing import Any, Optional, Union import google.cloud.bigquery import pandas -import bigframes.core.local_data -import bigframes.session from bigframes.core import utils +import bigframes.core.local_data from bigframes.core.tools import bigquery_schema +import bigframes.session _BQ_TABLE_TYPES = Union[ google.cloud.bigquery.Table, @@ -39,11 +39,7 @@ def _table_to_sql(table: _BQ_TABLE_TYPES) -> str: - # BiglakeIcebergTable IDs have 4 parts. BigFrames packs catalog.namespace - # into the dataset_id. - dataset_parts = table.dataset_id.split(".") - dataset_sql = ".".join(f"`{part}`" for part in dataset_parts) - return f"`{table.project}`.{dataset_sql}.`{table.table_id}`" + return f"`{table.project}`.`{table.dataset_id}`.`{table.table_id}`" def _pandas_df_to_sql_dry_run(pd_df: pandas.DataFrame) -> str: @@ -93,7 +89,7 @@ def _field_to_template_value( dry_run: bool = False, ) -> str: """Convert value to something embeddable in a SQL string.""" - import bigframes.core.compile.sqlglot.sql as sql # Avoid circular imports + import bigframes.core.sql # Avoid circular imports import bigframes.dataframe # Avoid circular imports _validate_type(name, value) @@ -106,43 +102,25 @@ def _field_to_template_value( return _pandas_df_to_sql(value, session=session, dry_run=dry_run, name=name) if isinstance(value, bigframes.dataframe.DataFrame): - import bigframes.core.bq_data as bq_data - import bigframes.core.nodes as nodes - - # TODO(b/493608478): Remove this workaround for BigLake/Iceberg tables, - # which cannot currently be used in views, once a fix rolls out. - def is_biglake( - node: nodes.BigFrameNode, child_results: Tuple[bool, ...] - ) -> bool: - if isinstance(node, nodes.ReadTableNode): - return isinstance(node.source.table, bq_data.BiglakeIcebergTable) - return any(child_results) - - contains_biglake = value._block.expr.node.reduce_up(is_biglake) - - if contains_biglake: - sql_query, _, _ = value._to_sql_query(include_index=True) - return f"({sql_query})" - return _table_to_sql(value._to_placeholder_table(dry_run=dry_run)) if isinstance(value, str): return value - return sql.to_sql(sql.literal(value)) + return bigframes.core.sql.simple_literal(value) def _validate_type(name: str, value: Any): """Raises TypeError if value is unsupported.""" + import bigframes.core.sql # Avoid circular imports import bigframes.dataframe # Avoid circular imports - import bigframes.dtypes # Avoid circular imports if value is None: return # None can't be used in isinstance, but is a valid literal. supported_types = ( typing.get_args(_BQ_TABLE_TYPES) - + bigframes.dtypes.SUPPORTED_LITERAL_TYPES + + typing.get_args(bigframes.core.sql.SIMPLE_LITERAL_TYPES) + (bigframes.dataframe.DataFrame,) + (pandas.DataFrame,) ) @@ -162,160 +140,6 @@ def _parse_fields(sql_template: str) -> list[str]: ] -def _is_escaped_open_brace(sql_template: str, idx: int, literal_char: str) -> bool: - """Checks if the character at idx in sql_template is an escaped open brace '{{'.""" - return sql_template[idx : idx + 2] == "{{" and literal_char == "{" - - -def _is_escaped_close_brace(sql_template: str, idx: int, literal_char: str) -> bool: - """Checks if the character at idx in sql_template is an escaped close brace '}}'.""" - return sql_template[idx : idx + 2] == "}}" and literal_char == "}" - - -def _consume_literal(sql_template: str, current_idx: int, literal_text: str) -> int: - """Advances current_idx past literal_text in sql_template, accounting for escaped braces. - - A **literal** (or literal text) is the static part of the template string that - does not contain formatting placeholders. The string.Formatter parser resolves - escaped braces ('{{' and '}}') into single braces ('{' and '}') in its output - literal_text. - - This function aligns the resolved literal_text back to the original - sql_template by consuming 2 characters from sql_template ('{{' or '}}') for - every single escaped brace character in literal_text, and 1 character for - everything else. - - Returns: - int: the advanced current_idx in sql_template. - """ - lit_idx = 0 - while lit_idx < len(literal_text): - if _is_escaped_open_brace(sql_template, current_idx, literal_text[lit_idx]): - current_idx += 2 - lit_idx += 1 - elif _is_escaped_close_brace(sql_template, current_idx, literal_text[lit_idx]): - current_idx += 2 - lit_idx += 1 - elif ( - current_idx < len(sql_template) - and sql_template[current_idx] == literal_text[lit_idx] - ): - current_idx += 1 - lit_idx += 1 - else: - raise RuntimeError( - "Internal error: failed to align parsed SQL template with original query. " - f"Expected {literal_text[lit_idx]!r} at position {current_idx} in template, " - f"but found {sql_template[current_idx : current_idx + 2]!r}." - ) - return current_idx - - -def _is_escaped_brace(sql_template: str, idx: int) -> bool: - """Checks if the template has an escaped brace ('{{' or '}}') at the given index.""" - return sql_template[idx : idx + 2] in ("{{", "}}") - - -def _advance_past_field(sql_template: str, current_idx: int) -> int: - """Advances current_idx past the format field starting at current_idx. - - A **field** (or replacement field) is a placeholder in the template enclosed - in braces (e.g., "{my_var}" or "{json_col: { "val": 1 } }"). - - This function assumes current_idx points to the opening '{' of a field. - It parses forward, tracking nested braces to find the matching closing '}' - that terminates the field, while ignoring escaped braces ('{{' and '}}') - which do not affect the nesting level. - - Returns: - int: the index immediately after the closing '}' of the field. - """ - assert sql_template[current_idx] == "{" - brace_count = 1 - current_idx += 1 # past '{' - - while brace_count > 0 and current_idx < len(sql_template): - if _is_escaped_brace(sql_template, current_idx): - current_idx += 2 - elif sql_template[current_idx] == "{": - brace_count += 1 - current_idx += 1 - elif sql_template[current_idx] == "}": - brace_count -= 1 - current_idx += 1 - else: - current_idx += 1 - - return current_idx - - -def _find_all_field_positions(sql_template: str) -> dict[tuple[str, int], int]: - """Finds the character positions of all fields in the sql_template. - - Returns: - dict: a dict mapping (field_name, occurrence_idx) to character index. - """ - formatter = string.Formatter() - current_idx = 0 - seen_counts: dict[str, int] = {} - positions: dict[tuple[str, int], int] = {} - - for literal_text, field_name, _, _ in formatter.parse(sql_template): - current_idx = _consume_literal(sql_template, current_idx, literal_text) - - if field_name is not None: - occurrence_idx = seen_counts.get(field_name, 0) - seen_counts[field_name] = occurrence_idx + 1 - - positions[(field_name, occurrence_idx)] = current_idx - - current_idx = _advance_past_field(sql_template, current_idx) - - return positions - - -def get_error_context_at_pos(sql_template: str, pos: int) -> str: - """Create a helpful 'pointer' to where the problematic position is - in the original SQL. - - This should make the error message a lot friendlier, by providing more - context towards the problematic syntax. - """ - if pos == -1: - return "" - - lines = sql_template.splitlines(keepends=True) - - char_count = 0 - target_line_idx = -1 - for i, line in enumerate(lines): - if char_count <= pos < char_count + len(line): - target_line_idx = i - break - char_count += len(line) - - if target_line_idx == -1: - return "" - - col_offset = pos - char_count - - context_lines = [] - start_line = max(0, target_line_idx - 2) - end_line = min(len(lines), target_line_idx + 3) - - for i in range(start_line, end_line): - line_num = i + 1 - line_content = lines[i].rstrip("\r\n") - if i == target_line_idx: - context_lines.append(f"{line_num:4d}: {line_content}") - indent = 6 + col_offset - context_lines.append(" " * indent + "^") - else: - context_lines.append(f"{line_num:4d}: {line_content}") - - return "\n".join(context_lines) - - def pyformat( sql_template: str, *, @@ -339,36 +163,13 @@ def pyformat( Raises: TypeError: if a referenced variable is not of a supported type. - ValueError: - if a referenced variable is not found (KeyError is caught and raised - as ValueError with context). + KeyError: if a referenced variable is not found. """ - try: - fields = _parse_fields(sql_template) - except ValueError as e: - raise ValueError( - "Failed to parse SQL template. " - "Did you mean to escape '{' and '}' by doubling them?\n" - f"Error details: {e}" - ) from e - - format_kwargs: dict[str, str] = {} - seen_counts: dict[str, int] = {} - for name in fields: - seen_counts[name] = seen_counts.get(name, 0) + 1 - try: - value = pyformat_args[name] - except KeyError as e: - positions = _find_all_field_positions(sql_template) - occurrence_idx = seen_counts[name] - 1 - pos = positions.get((name, occurrence_idx), -1) - context = get_error_context_at_pos(sql_template, pos) - raise ValueError( - f"Undetected variable {name!r} in SQL template. " - "Did you mean to escape '{' and '}' by doubling them?\n" - f"{context}" - ) from e + fields = _parse_fields(sql_template) + format_kwargs = {} + for name in fields: + value = pyformat_args[name] format_kwargs[name] = _field_to_template_value( name, value, session=session, dry_run=dry_run ) diff --git a/bigframes/core/reshape/concat.py b/bigframes/core/reshape/concat.py index cc81319ae68..a42488cbe80 100644 --- a/bigframes/core/reshape/concat.py +++ b/bigframes/core/reshape/concat.py @@ -31,7 +31,8 @@ def concat( axis: typing.Literal["index", 0] = ..., join=..., ignore_index=..., -) -> bigframes.series.Series: ... +) -> bigframes.series.Series: + ... @typing.overload @@ -41,7 +42,8 @@ def concat( axis: typing.Literal["index", 0] = ..., join=..., ignore_index=..., -) -> bigframes.dataframe.DataFrame: ... +) -> bigframes.dataframe.DataFrame: + ... @typing.overload @@ -51,7 +53,8 @@ def concat( axis: typing.Literal["columns", 1], join=..., ignore_index=..., -) -> bigframes.dataframe.DataFrame: ... +) -> bigframes.dataframe.DataFrame: + ... @typing.overload @@ -61,7 +64,8 @@ def concat( axis=..., join=..., ignore_index=..., -) -> Union[bigframes.dataframe.DataFrame, bigframes.series.Series]: ... +) -> Union[bigframes.dataframe.DataFrame, bigframes.series.Series]: + ... def concat( diff --git a/bigframes/core/reshape/merge.py b/bigframes/core/reshape/merge.py index 55e3abe0c6e..2afeb2a1064 100644 --- a/bigframes/core/reshape/merge.py +++ b/bigframes/core/reshape/merge.py @@ -20,8 +20,8 @@ from typing import Literal, Sequence -import bigframes_vendored.pandas.core.reshape.merge as vendored_pandas_merge from bigframes_vendored import constants +import bigframes_vendored.pandas.core.reshape.merge as vendored_pandas_merge from bigframes import dataframe, series from bigframes.core import blocks, utils diff --git a/bigframes/core/reshape/pivot.py b/bigframes/core/reshape/pivot.py index 082948728f6..c69c7f11aba 100644 --- a/bigframes/core/reshape/pivot.py +++ b/bigframes/core/reshape/pivot.py @@ -13,7 +13,7 @@ # limitations under the License. from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import Optional, TYPE_CHECKING import bigframes_vendored.pandas.core.reshape.pivot as vendored_pandas_pivot import pandas as pd diff --git a/bigframes/core/reshape/tile.py b/bigframes/core/reshape/tile.py index 61f869f2797..a2efa8f9274 100644 --- a/bigframes/core/reshape/tile.py +++ b/bigframes/core/reshape/tile.py @@ -15,7 +15,7 @@ from __future__ import annotations import typing -from typing import TYPE_CHECKING, Optional +from typing import Optional, TYPE_CHECKING import bigframes_vendored.constants as constants import bigframes_vendored.pandas.core.reshape.tile as vendored_pandas_tile diff --git a/bigframes/core/rewrite/__init__.py b/bigframes/core/rewrite/__init__.py index ae4b142b1a4..4e5295ae9d3 100644 --- a/bigframes/core/rewrite/__init__.py +++ b/bigframes/core/rewrite/__init__.py @@ -12,14 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from bigframes.core.rewrite.as_sql import as_sql_nodes -from bigframes.core.rewrite.ctes import extract_ctes from bigframes.core.rewrite.fold_row_count import fold_row_counts from bigframes.core.rewrite.identifiers import remap_variables from bigframes.core.rewrite.implicit_align import try_row_join from bigframes.core.rewrite.legacy_align import legacy_join_as_projection -from bigframes.core.rewrite.nullity import simplify_join -from bigframes.core.rewrite.order import bake_order, defer_order, pull_out_order +from bigframes.core.rewrite.order import bake_order, defer_order from bigframes.core.rewrite.pruning import column_pruning from bigframes.core.rewrite.scan_reduction import ( try_reduce_to_local_scan, @@ -28,16 +25,9 @@ from bigframes.core.rewrite.select_pullup import defer_selection from bigframes.core.rewrite.slices import pull_out_limit, pull_up_limits, rewrite_slice from bigframes.core.rewrite.timedeltas import rewrite_timedelta_expressions -from bigframes.core.rewrite.udfs import lower_udfs -from bigframes.core.rewrite.windows import ( - pull_out_window_order, - rewrite_range_rolling, - simplify_complex_windows, -) +from bigframes.core.rewrite.windows import pull_out_window_order, rewrite_range_rolling __all__ = [ - "as_sql_nodes", - "extract_ctes", "legacy_join_as_projection", "try_row_join", "rewrite_slice", @@ -50,12 +40,8 @@ "rewrite_range_rolling", "try_reduce_to_table_scan", "bake_order", - "pull_out_order", "try_reduce_to_local_scan", "fold_row_counts", "pull_out_window_order", "defer_selection", - "simplify_complex_windows", - "lower_udfs", - "simplify_join", ] diff --git a/bigframes/core/rewrite/as_sql.py b/bigframes/core/rewrite/as_sql.py deleted file mode 100644 index eb823d1fed1..00000000000 --- a/bigframes/core/rewrite/as_sql.py +++ /dev/null @@ -1,308 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import annotations - -import dataclasses -import itertools -from typing import Optional, Sequence, Union - -import bigframes.core.rewrite -from bigframes.core import ( - agg_expressions, - expression, - guid, - identifiers, - nodes, - ordering, - sql_nodes, -) - - -def _limit(select: sql_nodes.SqlSelectNode, limit: int) -> sql_nodes.SqlSelectNode: - new_limit = limit if select.limit is None else min([select.limit, limit]) - return dataclasses.replace(select, limit=new_limit) - - -def _try_sort( - select: sql_nodes.SqlSelectNode, sort_by: Sequence[ordering.OrderingExpression] -) -> Optional[sql_nodes.SqlSelectNode]: - new_order_exprs = [] - for sort_expr in sort_by: - new_expr = _try_bind( - sort_expr.scalar_expression, select.get_id_mapping(), analytic_allowed=False - ) - if new_expr is None: - return None - new_order_exprs.append( - dataclasses.replace(sort_expr, scalar_expression=new_expr) - ) - return dataclasses.replace(select, sorting=tuple(new_order_exprs)) - - -def _sort( - node: nodes.BigFrameNode, sort_by: Sequence[ordering.OrderingExpression] -) -> sql_nodes.SqlSelectNode: - if isinstance(node, sql_nodes.SqlSelectNode): - merged = _try_sort(node, sort_by) - if merged: - return merged - result = _try_sort(_create_noop_select(node), sort_by) - assert result is not None - return result - - -def _try_bind( - expr: expression.Expression, - bindings: dict[identifiers.ColumnId, expression.Expression], - analytic_allowed: bool = False, # means block binding to an analytic even if original is scalar -) -> Optional[expression.Expression]: - if not expr.is_scalar_expr or not analytic_allowed: - for ref in expr.column_references: - if ref in bindings and not bindings[ref].is_scalar_expr: - return None - return expr.bind_refs(bindings) - - -def _try_add_cdefs( - select: sql_nodes.SqlSelectNode, cdefs: Sequence[nodes.ColumnDef] -) -> Optional[sql_nodes.SqlSelectNode]: - # TODO: add up complexity measure while inlining refs - new_defs = [] - for cdef in cdefs: - cdef_expr = cdef.expression - merged_expr = _try_bind( - cdef_expr, select.get_id_mapping(), analytic_allowed=True - ) - if merged_expr is None: - return None - new_defs.append(nodes.ColumnDef(merged_expr, cdef.id)) - - return dataclasses.replace(select, selections=(*select.selections, *new_defs)) - - -def _add_cdefs( - node: nodes.BigFrameNode, cdefs: Sequence[nodes.ColumnDef] -) -> sql_nodes.SqlSelectNode: - if isinstance(node, sql_nodes.SqlSelectNode): - merged = _try_add_cdefs(node, cdefs) - if merged: - return merged - # Otherwise, wrap the child in a SELECT and add the columns - result = _try_add_cdefs(_create_noop_select(node), cdefs) - assert result is not None - return result - - -def _try_add_filter( - select: sql_nodes.SqlSelectNode, predicates: Sequence[expression.Expression] -) -> Optional[sql_nodes.SqlSelectNode]: - # Filter implicitly happens first, so merging it into ths select will modify non-scalar col expressions - if not all(cdef.expression.is_scalar_expr for cdef in select.selections): - return None - if not all( - sort_expr.scalar_expression.is_scalar_expr for sort_expr in select.sorting - ): - return None - # Constraint: filters can only be merged if they are scalar expression after binding - new_predicates = [] - # bind variables, merge predicates - for predicate in predicates: - merged_pred = _try_bind(predicate, select.get_id_mapping()) - if not merged_pred: - return None - new_predicates.append(merged_pred) - return dataclasses.replace(select, predicates=(*select.predicates, *new_predicates)) - - -def _add_filter( - node: nodes.BigFrameNode, predicates: Sequence[expression.Expression] -) -> sql_nodes.SqlSelectNode: - if isinstance(node, sql_nodes.SqlSelectNode): - result = _try_add_filter(node, predicates) - if result: - return result - new_node = _try_add_filter(_create_noop_select(node), predicates) - assert new_node is not None - return new_node - - -def _create_noop_select(node: nodes.BigFrameNode) -> sql_nodes.SqlSelectNode: - return sql_nodes.SqlSelectNode( - node, - selections=tuple( - nodes.ColumnDef(expression.ResolvedDerefOp.from_field(field), field.id) - for field in node.fields - ), - ) - - -def _try_remap_select_cols( - select: sql_nodes.SqlSelectNode, cols: Sequence[nodes.AliasedRef] -): - new_defs = [] - for aliased_ref in cols: - new_defs.append( - nodes.ColumnDef(select.get_id_mapping()[aliased_ref.ref.id], aliased_ref.id) - ) - - return dataclasses.replace(select, selections=tuple(new_defs)) - - -def _remap_select_cols(node: nodes.BigFrameNode, cols: Sequence[nodes.AliasedRef]): - if isinstance(node, sql_nodes.SqlSelectNode): - result = _try_remap_select_cols(node, cols) - if result: - return result - new_node = _try_remap_select_cols(_create_noop_select(node), cols) - assert new_node is not None - return new_node - - -def _get_added_cdefs(node: Union[nodes.ProjectionNode, nodes.WindowOpNode]): - # TODO: InNode - if isinstance(node, nodes.ProjectionNode): - return tuple(nodes.ColumnDef(expr, id) for expr, id in node.assignments) - if isinstance(node, nodes.WindowOpNode): - new_cdefs = [] - for cdef in node.agg_exprs: - assert isinstance(cdef.expression, agg_expressions.Aggregation) - window_expr = agg_expressions.WindowExpression( - cdef.expression, node.window_spec - ) - # TODO: we probably should do this as another step - rewritten_window_expr = bigframes.core.rewrite.simplify_complex_windows( - window_expr - ) - new_cdefs.append(nodes.ColumnDef(rewritten_window_expr, cdef.id)) - return tuple(new_cdefs) - else: - raise ValueError(f"Unexpected node type: {type(node)}") - - -def _as_sql_node(node: nodes.BigFrameNode) -> nodes.BigFrameNode: - # case one, can be converted to select - if isinstance(node, nodes.ReadTableNode): - leaf = sql_nodes.SqlDataSource(source=node.source) - mappings = [ - nodes.AliasedRef(expression.deref(scan_item.source_id), scan_item.id) - for scan_item in node.scan_list.items - ] - return _remap_select_cols(leaf, mappings) - elif isinstance(node, (nodes.ProjectionNode, nodes.WindowOpNode)): - cdefs = _get_added_cdefs(node) - return _add_cdefs(node.child, cdefs) - elif isinstance(node, (nodes.SelectionNode)): - return _remap_select_cols(node.child, node.input_output_pairs) - elif isinstance(node, nodes.FilterNode): - return _add_filter(node.child, [node.predicate]) - elif isinstance(node, nodes.ResultNode): - result = node.child - if node.order_by is not None: - result = _sort(result, node.order_by.all_ordering_columns) - result = _remap_select_cols( - result, - [ - nodes.AliasedRef(ref, identifiers.ColumnId(name)) - for ref, name in node.output_cols - ], - ) - if node.limit is not None: - result = _limit(result, node.limit) # type: ignore - return result - else: - return node - - -# In the future, we will have sql nodes for each of these node types. -_LOGICAL_NODE_TYPES_TO_WRAP = ( - nodes.ReadLocalNode, - nodes.ExplodeNode, - nodes.InNode, - nodes.AggregateNode, - nodes.FromRangeNode, - nodes.ConcatNode, - sql_nodes.SqlSelectNode, -) - - -def _insert_cte_markers(root: nodes.BigFrameNode) -> nodes.BigFrameNode: - # important not to wrap nodes that are already wrapped - wrapped_nodes = set( - node.child for node in root.unique_nodes() if isinstance(node, nodes.CteNode) - ) - # don't wrap child nodes of ConcatNode - union_child_nodes = set( - itertools.chain.from_iterable( - node.child_nodes - for node in root.unique_nodes() - if isinstance(node, nodes.ConcatNode) - ) - ) - - def maybe_insert_cte_marker(node: nodes.BigFrameNode) -> nodes.BigFrameNode: - if node == root: - return node - if ( - isinstance(node, _LOGICAL_NODE_TYPES_TO_WRAP) - and node not in wrapped_nodes - and node not in union_child_nodes - ): - wrapped_nodes.add(node) - return nodes.CteNode(node) - return node - - return root.top_down(maybe_insert_cte_marker) - - -def _extract_ctes_to_with_expr( - root: nodes.BigFrameNode, uid_gen: guid.SequentialUIDGenerator -) -> nodes.BigFrameNode: - topological_ctes = list( - filter( - lambda n: isinstance(n, nodes.CteNode), - root.iter_nodes_topo(), - ) - ) - cte_names = tuple( - next(uid_gen.get_uid_stream("bfcte_")) for _ in range(len(topological_ctes)) - ) - - if len(topological_ctes) == 0: - return root - - mapping = { - cte_node: sql_nodes.SqlCteRefNode(cte_name, tuple(cte_node.fields)) - for cte_node, cte_name in zip(topological_ctes, cte_names) - } - - # Replace all CTEs with CTE references and wrap the new root in a WITH clause - return sql_nodes.SqlWithCtesNode( - root.top_down(lambda x: mapping.get(x, x)), - cte_names, - tuple( - # Mypy loses context that cte_node is a CteNode with a child attribute, despite the isinstance filter above. - cte_node.child.top_down(lambda x: mapping.get(x, x)) # type: ignore[attr-defined] - for cte_node in topological_ctes - ), - ) - - -def as_sql_nodes( - root: nodes.BigFrameNode, uid_gen: guid.SequentialUIDGenerator -) -> nodes.BigFrameNode: - root = nodes.bottom_up(root, _as_sql_node) - # Insert CTE markers to indicate where we want to split the query. - root = _insert_cte_markers(root) - root = _extract_ctes_to_with_expr(root, uid_gen) - return root diff --git a/bigframes/core/rewrite/ctes.py b/bigframes/core/rewrite/ctes.py deleted file mode 100644 index a5afd19bb35..00000000000 --- a/bigframes/core/rewrite/ctes.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import annotations - -from collections import defaultdict - -from bigframes.core import nodes - - -def extract_ctes(root: nodes.BigFrameNode) -> nodes.BigFrameNode: - # identify candidates - node_parents: dict[nodes.BigFrameNode, int] = defaultdict(int) - for parent in root.unique_nodes(): - for child in parent.child_nodes: - node_parents[child] += 1 - - # everywhere a multi-parent node is referenced, wrap it in a CTE node - def insert_cte_markers(node: nodes.BigFrameNode) -> nodes.BigFrameNode: - def _add_cte_if_needed(child: nodes.BigFrameNode) -> nodes.BigFrameNode: - if node_parents[child] > 1: - return nodes.CteNode(child) - return child - - if isinstance(node, nodes.CteNode): - # don't re-wrap CTE nodes - return node - - return node.transform_children(_add_cte_if_needed) - - return root.top_down(insert_cte_markers) diff --git a/bigframes/core/rewrite/identifiers.py b/bigframes/core/rewrite/identifiers.py index 7b1d1d9a512..da43fdf8b93 100644 --- a/bigframes/core/rewrite/identifiers.py +++ b/bigframes/core/rewrite/identifiers.py @@ -13,42 +13,13 @@ # limitations under the License. from __future__ import annotations +import dataclasses import typing from bigframes.core import identifiers, nodes -def _create_mapping_operator( - id_def_remapping_by_node: dict[ - nodes.BigFrameNode, dict[identifiers.ColumnId, identifiers.ColumnId] - ], - id_ref_remapping_by_node: dict[ - nodes.BigFrameNode, dict[identifiers.ColumnId, identifiers.ColumnId] - ], -): - """ - Builds a remapping operator that uses predefined local remappings for ids. - - Args: - id_remapping_by_node: A mapping from nodes to their local remappings. - - Returns: - A remapping operator. - """ - - def _mapping_operator(node: nodes.BigFrameNode) -> nodes.BigFrameNode: - # Step 1: Get the local remapping for the current node. - local_def_remaps = id_def_remapping_by_node[node] - local_ref_remaps = id_ref_remapping_by_node[node] - - result = node.remap_vars(local_def_remaps) - result = result.remap_refs(local_ref_remaps) - - return result - - return _mapping_operator - - +# TODO: May as well just outright remove selection nodes in this process. def remap_variables( root: nodes.BigFrameNode, id_generator: typing.Iterator[identifiers.ColumnId], @@ -71,47 +42,52 @@ def remap_variables( A tuple of the new root node and a mapping from old to new column IDs visible to the parent node. """ - # step 1: defined remappings for each individual unique node - # step 2: top down traversal to apply remappings (mappings are value-based, so bottom-up doesn't work) - - id_def_remaps: dict[ - nodes.BigFrameNode, dict[identifiers.ColumnId, identifiers.ColumnId] - ] = {} - id_ref_remaps: dict[ - nodes.BigFrameNode, dict[identifiers.ColumnId, identifiers.ColumnId] - ] = {} - for node in root.iter_nodes_topo(): # bottom up - local_def_remaps = { - col_id: next(id_generator) for col_id in node.node_defined_ids - } - id_def_remaps[node] = local_def_remaps - - local_ref_remaps = {} - - # InNode is special case as ID scope inherited purely from left side - inheriting_nodes = ( - [node.child_nodes[0]] - if isinstance(node, nodes.InNode) - else node.child_nodes + # Step 1: Recursively remap children to get their new nodes and ID mappings. + new_child_nodes: list[nodes.BigFrameNode] = [] + new_child_mappings: list[dict[identifiers.ColumnId, identifiers.ColumnId]] = [] + for child in root.child_nodes: + new_child, child_mappings = remap_variables(child, id_generator=id_generator) + new_child_nodes.append(new_child) + new_child_mappings.append(child_mappings) + + # Step 2: Transform children to use their new nodes. + remapped_children: dict[nodes.BigFrameNode, nodes.BigFrameNode] = { + child: new_child for child, new_child in zip(root.child_nodes, new_child_nodes) + } + new_root = root.transform_children(lambda node: remapped_children[node]) + + # Step 3: Transform the current node using the mappings from its children. + # "reversed" is required for InNode so that in case of a duplicate column ID, + # the left child's mapping is the one that's kept. + downstream_mappings: dict[identifiers.ColumnId, identifiers.ColumnId] = { + k: v for mapping in reversed(new_child_mappings) for k, v in mapping.items() + } + if isinstance(new_root, nodes.InNode): + new_root = typing.cast(nodes.InNode, new_root) + new_root = dataclasses.replace( + new_root, + left_col=new_root.left_col.remap_column_refs( + new_child_mappings[0], allow_partial_bindings=True + ), ) - for child in inheriting_nodes: # inherit ref and def mappings from children - if not child.defines_namespace: # these nodes represent new id spaces - local_ref_remaps.update( - { - old_id: new_id - for old_id, new_id in id_ref_remaps[child].items() - if old_id in child.ids - } - ) - local_ref_remaps.update(id_def_remaps[child]) - id_ref_remaps[node] = local_ref_remaps - - # have to do top down to preserve node identities - return ( - root.top_down(_create_mapping_operator(id_def_remaps, id_ref_remaps)), - # Only used by unit tests - { - old_id: (id_def_remaps[root] | id_ref_remaps[root])[old_id] - for old_id in root.ids - }, - ) + else: + new_root = new_root.remap_refs(downstream_mappings) + + # Step 4: Create new IDs for columns defined by the current node. + node_defined_mappings = { + old_id: next(id_generator) for old_id in root.node_defined_ids + } + new_root = new_root.remap_vars(node_defined_mappings) + + new_root._validate() + + # Step 5: Determine which mappings to propagate up to the parent. + if root.defines_namespace: + # If a node defines a new namespace (e.g., a join), mappings from its + # children are not visible to its parents. + mappings_for_parent = node_defined_mappings + else: + # Otherwise, pass up the combined mappings from children and the current node. + mappings_for_parent = downstream_mappings | node_defined_mappings + + return new_root, mappings_for_parent diff --git a/bigframes/core/rewrite/legacy_align.py b/bigframes/core/rewrite/legacy_align.py index 26ee71d2ec5..573a7026e45 100644 --- a/bigframes/core/rewrite/legacy_align.py +++ b/bigframes/core/rewrite/legacy_align.py @@ -252,9 +252,9 @@ def legacy_join_as_projection( # Most likely because join keys didn't match return None merged = left_side.merge(right_side, how, join_keys, mappings) - assert merged is not None, ( - "Couldn't merge nodes. This shouldn't happen. Please share full stacktrace with the BigQuery DataFrames team at bigframes-feedback@google.com." - ) + assert ( + merged is not None + ), "Couldn't merge nodes. This shouldn't happen. Please share full stacktrace with the BigQuery DataFrames team at bigframes-feedback@google.com." return merged.expand() else: return None diff --git a/bigframes/core/rewrite/nullity.py b/bigframes/core/rewrite/nullity.py deleted file mode 100644 index 6307b12ec27..00000000000 --- a/bigframes/core/rewrite/nullity.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import dataclasses - -from bigframes.core import nodes - - -def simplify_join(node: nodes.BigFrameNode) -> nodes.BigFrameNode: - """Simplify a join node by removing nullity checks.""" - # if join conditions are provably non-null, we can set nulls_equal=False - if isinstance(node, nodes.JoinNode): - # even better, we can always make nulls_equal false, but wrap the join keys in coalesce - # to handle nulls correctly, this is more granular than the current implementation - for left_ref, right_ref in node.conditions: - if ( - node.left_child.field_by_id[left_ref.id].nullable - and node.right_child.field_by_id[right_ref.id].nullable - ): - return node - return dataclasses.replace(node, nulls_equal=False) - elif isinstance(node, nodes.InNode): - if ( - node.left_child.field_by_id[node.left_col.id].nullable - and node.right_child.fields[0].nullable - ): - return node - return dataclasses.replace(node, nulls_equal=False) - else: - return node diff --git a/bigframes/core/rewrite/op_lowering.py b/bigframes/core/rewrite/op_lowering.py index 013fc48c06a..6473c3bf8a0 100644 --- a/bigframes/core/rewrite/op_lowering.py +++ b/bigframes/core/rewrite/op_lowering.py @@ -16,17 +16,19 @@ import abc from typing import Sequence -import bigframes.operations as ops from bigframes.core import bigframe_node, expression, nodes +import bigframes.operations as ops class OpLoweringRule(abc.ABC): @property @abc.abstractmethod - def op(self) -> type[ops.ScalarOp]: ... + def op(self) -> type[ops.ScalarOp]: + ... @abc.abstractmethod - def lower(self, expr: expression.OpExpression) -> expression.Expression: ... + def lower(self, expr: expression.OpExpression) -> expression.Expression: + ... def lower_ops( diff --git a/bigframes/core/rewrite/order.py b/bigframes/core/rewrite/order.py index b61fca82182..6741dfddadd 100644 --- a/bigframes/core/rewrite/order.py +++ b/bigframes/core/rewrite/order.py @@ -15,10 +15,10 @@ import functools from typing import Mapping, Tuple +from bigframes.core import agg_expressions, expression, identifiers import bigframes.core.nodes import bigframes.core.ordering import bigframes.core.window_spec -from bigframes.core import agg_expressions, expression, identifiers from bigframes.operations import aggregations as agg_ops @@ -47,15 +47,6 @@ def bake_order( return node -def pull_out_order( - node: bigframes.core.nodes.BigFrameNode, -) -> Tuple[bigframes.core.nodes.BigFrameNode, bigframes.core.ordering.RowOrdering]: - import bigframes.core.rewrite.slices - - node = node.bottom_up(bigframes.core.rewrite.slices.rewrite_slice) - return _pull_up_order(node, order_root=True) - - # Makes ordering explicit in window definitions def _pull_up_order( root: bigframes.core.nodes.BigFrameNode, @@ -80,8 +71,7 @@ def pull_up_order_inner( child_result, child_order = pull_up_order_inner(node.child) return child_result, child_order.with_reverse() elif isinstance(node, bigframes.core.nodes.OrderByNode): - # unstable sorts don't care about previous order, total orders override previous order - if (not node.stable) or node.is_total_order: + if node.is_total_order: new_node = remove_order(node.child) else: new_node, child_order = pull_up_order_inner(node.child) @@ -116,10 +106,6 @@ def pull_up_order_inner( ), ) ) - elif not node.stable: - new_order = bigframes.core.ordering.RowOrdering( - ordering_value_columns=tuple(new_by), - ) else: assert child_order new_order = child_order.with_ordering_columns(new_by) @@ -162,7 +148,7 @@ def pull_up_order_inner( ) elif isinstance(node, bigframes.core.nodes.ReadTableNode): if node.source.ordering is not None: - return node.pull_out_order() + return node.with_order_cols() else: # No defined ordering return node, bigframes.core.ordering.RowOrdering() @@ -281,7 +267,7 @@ def pull_up_order_inner( offsets_id ) return new_explode, child_order.join(inner_order) - raise ValueError(f"Unexpected node type {type(node).__name__}") + raise ValueError(f"Unexpected node: {node}") def pull_order_concat( node: bigframes.core.nodes.ConcatNode, diff --git a/bigframes/core/rewrite/pruning.py b/bigframes/core/rewrite/pruning.py index 29744d66cd6..7695ace3b33 100644 --- a/bigframes/core/rewrite/pruning.py +++ b/bigframes/core/rewrite/pruning.py @@ -67,7 +67,7 @@ def prune_selection_child( # Important to check this first if list(selection.ids) == list(child.ids): - if all(ref.ref.id == ref.id for ref in selection.input_output_pairs): + if (ref.ref.id == ref.id for ref in selection.input_output_pairs): # selection is no-op so just remove it entirely return child @@ -75,7 +75,6 @@ def prune_selection_child( return selection.remap_refs( {id: ref.id for ref, id in child.input_output_pairs} ).replace_child(child.child) - elif isinstance(child, nodes.AdditiveNode): if not set(field.id for field in child.added_fields) & selection.consumed_ids: return selection.replace_child(child.additive_base) @@ -163,7 +162,8 @@ def prune_aggregate( def prune_leaf( node: nodes.BigFrameNode, used_cols: typing.AbstractSet[identifiers.ColumnId], -): ... +): + ... @prune_leaf.register diff --git a/bigframes/core/rewrite/scan_reduction.py b/bigframes/core/rewrite/scan_reduction.py index da609c1ea1f..b0729337e7b 100644 --- a/bigframes/core/rewrite/scan_reduction.py +++ b/bigframes/core/rewrite/scan_reduction.py @@ -15,8 +15,8 @@ import functools from typing import Optional -import bigframes.core.rewrite.slices from bigframes.core import nodes +import bigframes.core.rewrite.slices def try_reduce_to_table_scan(root: nodes.BigFrameNode) -> Optional[nodes.ReadTableNode]: diff --git a/bigframes/core/rewrite/schema_binding.py b/bigframes/core/rewrite/schema_binding.py index 14755d34a41..d874c7c5989 100644 --- a/bigframes/core/rewrite/schema_binding.py +++ b/bigframes/core/rewrite/schema_binding.py @@ -15,8 +15,9 @@ import dataclasses import typing -from bigframes.core import agg_expressions, bigframe_node, nodes, ordering +from bigframes.core import agg_expressions, bigframe_node from bigframes.core import expression as ex +from bigframes.core import nodes, ordering def bind_schema_to_tree( @@ -123,9 +124,9 @@ def _bind_schema_to_aggregation_expr( aggregation: agg_expressions.Aggregation, child: bigframe_node.BigFrameNode, ) -> agg_expressions.Aggregation: - assert isinstance(aggregation, agg_expressions.Aggregation), ( - f"Expected Aggregation, got {type(aggregation)}" - ) + assert isinstance( + aggregation, agg_expressions.Aggregation + ), f"Expected Aggregation, got {type(aggregation)}" if isinstance(aggregation, agg_expressions.UnaryAggregation): return typing.cast( diff --git a/bigframes/core/rewrite/select_pullup.py b/bigframes/core/rewrite/select_pullup.py index a15aba7663f..415182f8840 100644 --- a/bigframes/core/rewrite/select_pullup.py +++ b/bigframes/core/rewrite/select_pullup.py @@ -54,12 +54,13 @@ def pull_up_source_ids(node: nodes.ReadTableNode) -> nodes.BigFrameNode: if all(id.sql == source_id for id, source_id in node.scan_list.items): return node else: + source_ids = sorted( + set(scan_item.source_id for scan_item in node.scan_list.items) + ) new_scan_list = nodes.ScanList.from_items( [ - nodes.ScanItem( - identifiers.ColumnId(scan_item.source_id), scan_item.source_id - ) - for scan_item in node.scan_list.items + nodes.ScanItem(identifiers.ColumnId(source_id), source_id) + for source_id in source_ids ] ) new_source = dataclasses.replace(node, scan_list=new_scan_list) diff --git a/bigframes/core/rewrite/timedeltas.py b/bigframes/core/rewrite/timedeltas.py index 7544963732e..7190810f714 100644 --- a/bigframes/core/rewrite/timedeltas.py +++ b/bigframes/core/rewrite/timedeltas.py @@ -206,12 +206,12 @@ def _rewrite_div_op(left: _TypedExpr, right: _TypedExpr) -> _TypedExpr: def _rewrite_floordiv_op(left: _TypedExpr, right: _TypedExpr) -> _TypedExpr: + result = _TypedExpr.create_op_expr(ops.floordiv_op, left, right) + if left.dtype == dtypes.TIMEDELTA_DTYPE and dtypes.is_numeric(right.dtype): - return _TypedExpr.create_op_expr( - ops.timedelta_floor_op, _TypedExpr.create_op_expr(ops.div_op, left, right) - ) + return _TypedExpr.create_op_expr(ops.timedelta_floor_op, result) - return _TypedExpr.create_op_expr(ops.floordiv_op, left, right) + return result def _rewrite_to_timedelta_op(op: ops.ToTimedeltaOp, arg: _TypedExpr): diff --git a/bigframes/core/rewrite/udfs.py b/bigframes/core/rewrite/udfs.py deleted file mode 100644 index 286a9d9d940..00000000000 --- a/bigframes/core/rewrite/udfs.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import annotations - -import dataclasses - -import bigframes.functions.udf_def as udf_def -import bigframes.operations as ops -from bigframes.core import bigframe_node, expression -from bigframes.core.rewrite import op_lowering - - -@dataclasses.dataclass -class LowerRemoteFunctionRule(op_lowering.OpLoweringRule): - @property - def op(self) -> type[ops.ScalarOp]: - return ops.RemoteFunctionOp - - def lower(self, expr: expression.OpExpression) -> expression.Expression: - assert isinstance(expr.op, ops.RemoteFunctionOp) - func_def = expr.op.function_def - devirtualized_expr = ops.RemoteFunctionOp( - func_def.with_devirtualize(), - ).as_expr(*expr.children) - if isinstance(func_def.signature.output, udf_def.VirtualListTypeV1): - return func_def.signature.output.out_expr(devirtualized_expr) - else: - return devirtualized_expr - - -UDF_LOWERING_RULES = (LowerRemoteFunctionRule(),) - - -def lower_udfs(root: bigframe_node.BigFrameNode) -> bigframe_node.BigFrameNode: - return op_lowering.lower_ops(root, rules=UDF_LOWERING_RULES) diff --git a/bigframes/core/rewrite/windows.py b/bigframes/core/rewrite/windows.py index 4d271a072d4..6e9ba0dd3d0 100644 --- a/bigframes/core/rewrite/windows.py +++ b/bigframes/core/rewrite/windows.py @@ -15,72 +15,9 @@ from __future__ import annotations import dataclasses -import functools -import itertools -import bigframes.dtypes from bigframes import operations as ops -from bigframes.core import ( - agg_expressions, - expression, - guid, - identifiers, - nodes, - ordering, -) -from bigframes.operations import aggregations as agg_ops - - -def simplify_complex_windows( - window_expr: agg_expressions.WindowExpression, -) -> expression.Expression: - result_expr: expression.Expression = window_expr - agg_expr = window_expr.analytic_expr - window_spec = window_expr.window - clauses: list[tuple[expression.Expression, expression.Expression]] = [] - if window_spec.min_periods and len(agg_expr.inputs) > 0: - if not agg_expr.op.nulls_count_for_min_values: - is_observation = ops.notnull_op.as_expr() - - # Most operations do not count NULL values towards min_periods - per_col_does_count = ( - ops.notnull_op.as_expr(input) for input in agg_expr.inputs - ) - # All inputs must be non-null for observation to count - is_observation = functools.reduce( - lambda x, y: ops.and_op.as_expr(x, y), per_col_does_count - ) - observation_sentinel = ops.AsTypeOp(bigframes.dtypes.INT_DTYPE).as_expr( - is_observation - ) - observation_count_expr = agg_expressions.WindowExpression( - agg_expressions.UnaryAggregation(agg_ops.sum_op, observation_sentinel), - window_spec, - ) - else: - # Operations like count treat even NULLs as valid observations for the sake of min_periods - # notnull is just used to convert null values to non-null (FALSE) values to be counted - is_observation = ops.notnull_op.as_expr(agg_expr.inputs[0]) - observation_count_expr = agg_expressions.WindowExpression( - agg_ops.count_op.as_expr(is_observation), - window_spec, - ) - clauses.append( - ( - ops.lt_op.as_expr( - observation_count_expr, expression.const(window_spec.min_periods) - ), - expression.const(None), - ) - ) - if clauses: - case_inputs = [ - *itertools.chain.from_iterable(clauses), - expression.const(True), - result_expr, - ] - result_expr = ops.CaseWhenOp().as_expr(*case_inputs) - return result_expr +from bigframes.core import guid, identifiers, nodes, ordering def rewrite_range_rolling(node: nodes.BigFrameNode) -> nodes.BigFrameNode: diff --git a/bigframes/core/schema.py b/bigframes/core/schema.py index ab30b9bff14..395ad55f492 100644 --- a/bigframes/core/schema.py +++ b/bigframes/core/schema.py @@ -14,10 +14,10 @@ from __future__ import annotations +from dataclasses import dataclass import functools import typing -from dataclasses import dataclass -from typing import Dict, Optional, Sequence +from typing import Dict, List, Optional, Sequence import google.cloud.bigquery import pyarrow @@ -40,16 +40,31 @@ class ArraySchema: def __iter__(self): yield from self.items + @classmethod + def from_bq_table( + cls, + table: google.cloud.bigquery.Table, + column_type_overrides: Optional[ + typing.Dict[str, bigframes.dtypes.Dtype] + ] = None, + columns: Optional[Sequence[str]] = None, + ): + if not columns: + fields = table.schema + else: + lookup = {field.name: field for field in table.schema} + fields = [lookup[col] for col in columns] + + return ArraySchema.from_bq_schema( + fields, column_type_overrides=column_type_overrides + ) + @classmethod def from_bq_schema( cls, - schema: Sequence[google.cloud.bigquery.SchemaField], + schema: List[google.cloud.bigquery.SchemaField], column_type_overrides: Optional[Dict[str, bigframes.dtypes.Dtype]] = None, - columns: Optional[Sequence[str]] = None, ): - if columns: - lookup = {field.name: field for field in schema} - schema = [lookup[col] for col in columns] if column_type_overrides is None: column_type_overrides = {} items = tuple( diff --git a/bigframes/core/sql/__init__.py b/bigframes/core/sql/__init__.py index b28d5921695..ccd2a16ddcd 100644 --- a/bigframes/core/sql/__init__.py +++ b/bigframes/core/sql/__init__.py @@ -11,28 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations """ Utility functions for SQL construction. """ -from __future__ import annotations - +import datetime +import decimal import json -from typing import ( - TYPE_CHECKING, - Any, - Collection, - Iterable, - Mapping, - Optional, - Union, - cast, -) +import math +from typing import cast, Collection, Iterable, Mapping, Optional, TYPE_CHECKING, Union -import bigframes_vendored.sqlglot.expressions as sge +import shapely.geometry.base # type: ignore -from bigframes.core.compile.sqlglot import sql +import bigframes.core.compile.googlesql as googlesql if TYPE_CHECKING: import google.cloud.bigquery as bigquery @@ -49,52 +42,83 @@ to_wkt = dumps -def identifier(name: str) -> str: - if len(name) > 256: - raise ValueError("Identifier must be less than 256 characters") - return f"`{escape_chars(name)}`" - - -def escape_chars(value: str): - """Escapes all special characters""" - # TODO: Reuse literal's escaping logic instead of re-implementing it here. - # https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#string_and_bytes_literals - trans_table = str.maketrans( - { - "\a": r"\a", - "\b": r"\b", - "\f": r"\f", - "\n": r"\n", - "\r": r"\r", - "\t": r"\t", - "\v": r"\v", - "\\": r"\\", - "?": r"\?", - '"': r"\"", - "'": r"\'", - "`": r"\`", - } - ) - return value.translate(trans_table) +SIMPLE_LITERAL_TYPES = Union[ + bytes, + str, + int, + bool, + float, + datetime.datetime, + datetime.date, + datetime.time, + decimal.Decimal, + list, +] + + +### Writing SQL Values (literals, column references, table references, etc.) +def simple_literal(value: Union[SIMPLE_LITERAL_TYPES, None]) -> str: + """Return quoted input string.""" + + # https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#literals + if value is None: + return "NULL" + elif isinstance(value, str): + # Single quoting seems to work nicer with ibis than double quoting + return f"'{googlesql._escape_chars(value)}'" + elif isinstance(value, bytes): + return repr(value) + elif isinstance(value, (bool, int)): + return str(value) + elif isinstance(value, float): + # https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#floating_point_literals + if math.isnan(value): + return 'CAST("nan" as FLOAT)' + if value == math.inf: + return 'CAST("+inf" as FLOAT)' + if value == -math.inf: + return 'CAST("-inf" as FLOAT)' + return str(value) + # Check datetime first as it is a subclass of date + elif isinstance(value, datetime.datetime): + if value.tzinfo is None: + return f"DATETIME('{value.isoformat()}')" + else: + return f"TIMESTAMP('{value.isoformat()}')" + elif isinstance(value, datetime.date): + return f"DATE('{value.isoformat()}')" + elif isinstance(value, datetime.time): + return f"TIME(DATETIME('1970-01-01 {value.isoformat()}'))" + elif isinstance(value, shapely.geometry.base.BaseGeometry): + return f"ST_GEOGFROMTEXT({simple_literal(to_wkt(value))})" + elif isinstance(value, decimal.Decimal): + # TODO: disambiguate BIGNUMERIC based on scale and/or precision + return f"CAST('{str(value)}' AS NUMERIC)" + elif isinstance(value, list): + simple_literals = [simple_literal(i) for i in value] + return f"[{', '.join(simple_literals)}]" + + else: + raise ValueError(f"Cannot produce literal for {value}") -def multi_literal(*values: Any): - literal_strings = [sql.to_sql(sql.literal(i)) for i in values] +def multi_literal(*values: str): + literal_strings = [simple_literal(i) for i in values] return "(" + ", ".join(literal_strings) + ")" def cast_as_string(column_name: str) -> str: """Return a string representing string casting of a column.""" - return sge.Cast(this=sge.to_identifier(column_name, quoted=True), to="STRING").sql( - dialect="bigquery" - ) + return googlesql.Cast( + googlesql.ColumnExpression(column_name), googlesql.DataType.STRING + ).sql() def to_json_string(column_name: str) -> str: """Return a string representing JSON version of a column.""" - return f"TO_JSON_STRING({sql.to_sql(sql.identifier(column_name))})" + return f"TO_JSON_STRING({googlesql.identifier(column_name)})" def csv(values: Iterable[str]) -> str: @@ -108,29 +132,11 @@ def infix_op(opname: str, left_arg: str, right_arg: str): def is_distinct_sql(columns: Iterable[str], table_ref: bigquery.TableReference) -> str: - table_expr = sge.Table( - this=sge.Identifier(this=table_ref.table_id, quoted=True), - db=sge.Identifier(this=table_ref.dataset_id, quoted=True), - catalog=sge.Identifier(this=table_ref.project, quoted=True), - ) - to_select = [sge.to_identifier(col, quoted=True) for col in columns] - - full_table_sql = ( - sge.Select().select(*to_select).from_(table_expr).sql(dialect="bigquery") - ) - distinct_table_sql = ( - sge.Select() - .select(*to_select) - .distinct() - .from_(table_expr) - .sql(dialect="bigquery") - ) - is_unique_sql = f"""WITH full_table AS ( - {full_table_sql} + {googlesql.Select().from_(table_ref).select(columns).sql()} ), distinct_table AS ( - {distinct_table_sql} + {googlesql.Select().from_(table_ref).select(columns, distinct=True).sql()} ) SELECT (SELECT COUNT(*) FROM full_table) AS `total_count`, @@ -177,7 +183,7 @@ def create_vector_index_ddl( if len(stored_column_names) > 0: escaped_stored = [ - f"{sql.to_sql(sql.identifier(name))}" for name in stored_column_names + f"{googlesql.identifier(name)}" for name in stored_column_names ] storing = f"STORING({', '.join(escaped_stored)}) " else: @@ -185,14 +191,14 @@ def create_vector_index_ddl( rendered_options = ", ".join( [ - f"{option_name} = {sql.to_sql(sql.literal(option_value))}" + f"{option_name} = {simple_literal(option_value)}" for option_name, option_value in options.items() ] ) return f""" - {create} {sql.to_sql(sql.identifier(index_name))} - ON {sql.to_sql(sql.identifier(table_name))}({sql.to_sql(sql.identifier(column_name))}) + {create} {googlesql.identifier(index_name)} + ON {googlesql.identifier(table_name)}({googlesql.identifier(column_name)}) {storing} OPTIONS({rendered_options}); """ @@ -211,27 +217,25 @@ def create_vector_search_sql( """Encode the VECTOR SEARCH statement for BigQuery Vector Search.""" vector_search_args = [ - f"TABLE {sql.to_sql(sql.identifier(cast(str, base_table)))}", - f"{sql.to_sql(sql.literal(column_to_search))}", + f"TABLE {googlesql.identifier(cast(str, base_table))}", + f"{simple_literal(column_to_search)}", f"({sql_string})", ] if query_column_to_search is not None: vector_search_args.append( - f"query_column_to_search => {sql.to_sql(sql.literal(query_column_to_search))}" + f"query_column_to_search => {simple_literal(query_column_to_search)}" ) if top_k is not None: - vector_search_args.append(f"top_k=> {sql.to_sql(sql.literal(top_k))}") + vector_search_args.append(f"top_k=> {simple_literal(top_k)}") if distance_type is not None: - vector_search_args.append( - f"distance_type => {sql.to_sql(sql.literal(distance_type))}" - ) + vector_search_args.append(f"distance_type => {simple_literal(distance_type)}") if options is not None: vector_search_args.append( - f"options => {sql.to_sql(sql.literal(json.dumps(options, indent=None)))}" + f"options => {simple_literal(json.dumps(options, indent=None))}" ) args_str = ",\n".join(vector_search_args) diff --git a/bigframes/core/sql/ml.py b/bigframes/core/sql/ml.py index 8d971e6c3e8..ec55fe04269 100644 --- a/bigframes/core/sql/ml.py +++ b/bigframes/core/sql/ml.py @@ -14,11 +14,10 @@ from __future__ import annotations -from typing import Any, Dict, List, Mapping, Optional, Union +from typing import Dict, Mapping, Optional, Union -import bigframes.core.col as col -from bigframes.core.compile.sqlglot import sql as sg_sql -from bigframes.core.compile.sqlglot.expression_compiler import expression_compiler +import bigframes.core.compile.googlesql as googlesql +import bigframes.core.sql def create_model_ddl( @@ -30,9 +29,7 @@ def create_model_ddl( input_schema: Optional[Mapping[str, str]] = None, output_schema: Optional[Mapping[str, str]] = None, connection_name: Optional[str] = None, - options: Optional[ - Mapping[str, Union[str, int, float, bool, list, "col.Expression"]] - ] = None, + options: Optional[Mapping[str, Union[str, int, float, bool, list]]] = None, training_data: Optional[str] = None, custom_holiday: Optional[str] = None, ) -> str: @@ -48,7 +45,7 @@ def create_model_ddl( else: create = "CREATE MODEL " - ddl = f"{create}{sg_sql.to_sql(sg_sql.identifier(model_name))}\n" + ddl = f"{create}{googlesql.identifier(model_name)}\n" # [TRANSFORM (select_list)] if transform: @@ -68,22 +65,19 @@ def create_model_ddl( if connection_name.upper() == "DEFAULT": ddl += "REMOTE WITH CONNECTION DEFAULT\n" else: - ddl += f"REMOTE WITH CONNECTION {sg_sql.to_sql(sg_sql.identifier(connection_name))}\n" + ddl += f"REMOTE WITH CONNECTION {googlesql.identifier(connection_name)}\n" # [OPTIONS(model_option_list)] if options: rendered_options = [] for option_name, option_value in options.items(): - if isinstance(option_value, col.Expression): - sg_expr = expression_compiler.compile_expression(option_value._value) - rendered_val = sg_sql.to_sql(sg_expr) - elif isinstance(option_value, (list, tuple)): + if isinstance(option_value, (list, tuple)): # Handle list options like model_registry="vertex_ai" # wait, usually options are key=value. # if value is list, it is [val1, val2] - rendered_val = sg_sql.to_sql(sg_sql.literal(list(option_value))) + rendered_val = bigframes.core.sql.simple_literal(list(option_value)) else: - rendered_val = sg_sql.to_sql(sg_sql.literal(option_value)) + rendered_val = bigframes.core.sql.simple_literal(option_value) rendered_options.append(f"{option_name} = {rendered_val}") @@ -106,14 +100,16 @@ def create_model_ddl( def _build_struct_sql( - struct_options: Mapping[ - str, - Union[str, int, float, bool, Mapping[str, str], List[str], Mapping[str, Any]], - ], + struct_options: Mapping[str, Union[str, int, float, bool]] ) -> str: if not struct_options: return "" - return f", {sg_sql.to_sql(sg_sql.literal(struct_options))}" + + rendered_options = [] + for option_name, option_value in struct_options.items(): + rendered_val = bigframes.core.sql.simple_literal(option_value) + rendered_options.append(f"{rendered_val} AS {option_name}") + return f", STRUCT({', '.join(rendered_options)})" def evaluate( @@ -135,7 +131,7 @@ def evaluate( if confidence_level is not None: struct_options["confidence_level"] = confidence_level - sql = f"SELECT * FROM ML.EVALUATE(MODEL {sg_sql.to_sql(sg_sql.identifier(model_name))}" + sql = f"SELECT * FROM ML.EVALUATE(MODEL {googlesql.identifier(model_name)}" if table: sql += f", ({table})" @@ -155,7 +151,7 @@ def predict( """Encode the ML.PREDICT statement. See https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-predict for reference. """ - struct_options: Dict[str, Union[str, int, float, bool]] = {} + struct_options = {} if threshold is not None: struct_options["threshold"] = threshold if keep_original_columns is not None: @@ -163,7 +159,9 @@ def predict( if trial_id is not None: struct_options["trial_id"] = trial_id - sql = f"SELECT * FROM ML.PREDICT(MODEL {sg_sql.to_sql(sg_sql.identifier(model_name))}, ({table})" + sql = ( + f"SELECT * FROM ML.PREDICT(MODEL {googlesql.identifier(model_name)}, ({table})" + ) sql += _build_struct_sql(struct_options) sql += ")\n" return sql @@ -187,13 +185,13 @@ def explain_predict( if threshold is not None: struct_options["threshold"] = threshold if integrated_gradients_num_steps is not None: - struct_options["integrated_gradients_num_steps"] = ( - integrated_gradients_num_steps - ) + struct_options[ + "integrated_gradients_num_steps" + ] = integrated_gradients_num_steps if approx_feature_contrib is not None: struct_options["approx_feature_contrib"] = approx_feature_contrib - sql = f"SELECT * FROM ML.EXPLAIN_PREDICT(MODEL {sg_sql.to_sql(sg_sql.identifier(model_name))}, ({table})" + sql = f"SELECT * FROM ML.EXPLAIN_PREDICT(MODEL {googlesql.identifier(model_name)}, ({table})" sql += _build_struct_sql(struct_options) sql += ")\n" return sql @@ -207,103 +205,11 @@ def global_explain( """Encode the ML.GLOBAL_EXPLAIN statement. See https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-global-explain for reference. """ - struct_options: Dict[str, Union[str, int, float, bool]] = {} + struct_options = {} if class_level_explain is not None: struct_options["class_level_explain"] = class_level_explain - sql = f"SELECT * FROM ML.GLOBAL_EXPLAIN(MODEL {sg_sql.to_sql(sg_sql.identifier(model_name))}" - sql += _build_struct_sql(struct_options) - sql += ")\n" - return sql - - -def transform( - model_name: str, - table: str, -) -> str: - """Encode the ML.TRANSFORM statement. - See https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-transform for reference. - """ - sql = f"SELECT * FROM ML.TRANSFORM(MODEL {sg_sql.to_sql(sg_sql.identifier(model_name))}, ({table}))\n" - return sql - - -def generate_text( - model_name: str, - table: str, - *, - temperature: Optional[float] = None, - max_output_tokens: Optional[int] = None, - top_k: Optional[int] = None, - top_p: Optional[float] = None, - flatten_json_output: Optional[bool] = None, - stop_sequences: Optional[List[str]] = None, - ground_with_google_search: Optional[bool] = None, - request_type: Optional[str] = None, -) -> str: - """Encode the ML.GENERATE_TEXT statement. - See https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-generate-text for reference. - """ - struct_options: Dict[ - str, - Union[str, int, float, bool, Mapping[str, str], List[str], Mapping[str, Any]], - ] = {} - if temperature is not None: - struct_options["temperature"] = temperature - if max_output_tokens is not None: - struct_options["max_output_tokens"] = max_output_tokens - if top_k is not None: - struct_options["top_k"] = top_k - if top_p is not None: - struct_options["top_p"] = top_p - if flatten_json_output is not None: - struct_options["flatten_json_output"] = flatten_json_output - if stop_sequences is not None: - struct_options["stop_sequences"] = stop_sequences - if ground_with_google_search is not None: - struct_options["ground_with_google_search"] = ground_with_google_search - if request_type is not None: - struct_options["request_type"] = request_type - - sql = f"SELECT * FROM ML.GENERATE_TEXT(MODEL {sg_sql.to_sql(sg_sql.identifier(model_name))}, ({table})" - sql += _build_struct_sql(struct_options) - sql += ")\n" - return sql - - -def get_insights( - model_name: str, -) -> str: - """Encode the ML.GET_INSIGHTS statement. - See https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-get-insights for reference. - """ - sql = f"SELECT * FROM ML.GET_INSIGHTS(MODEL {sg_sql.to_sql(sg_sql.identifier(model_name))})\n" - return sql - - -def generate_embedding( - model_name: str, - table: str, - *, - flatten_json_output: Optional[bool] = None, - task_type: Optional[str] = None, - output_dimensionality: Optional[int] = None, -) -> str: - """Encode the ML.GENERATE_EMBEDDING statement. - See https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-generate-embedding for reference. - """ - struct_options: Dict[ - str, - Union[str, int, float, bool, Mapping[str, str], List[str], Mapping[str, Any]], - ] = {} - if flatten_json_output is not None: - struct_options["flatten_json_output"] = flatten_json_output - if task_type is not None: - struct_options["task_type"] = task_type - if output_dimensionality is not None: - struct_options["output_dimensionality"] = output_dimensionality - - sql = f"SELECT * FROM ML.GENERATE_EMBEDDING(MODEL {sg_sql.to_sql(sg_sql.identifier(model_name))}, ({table})" + sql = f"SELECT * FROM ML.GLOBAL_EXPLAIN(MODEL {googlesql.identifier(model_name)}" sql += _build_struct_sql(struct_options) sql += ")\n" return sql diff --git a/bigframes/core/sql_nodes.py b/bigframes/core/sql_nodes.py deleted file mode 100644 index c7a05a082f2..00000000000 --- a/bigframes/core/sql_nodes.py +++ /dev/null @@ -1,300 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import dataclasses -import functools -from typing import Callable, Mapping, Optional, Sequence, Tuple - -import bigframes.core.expression as ex -import bigframes.dtypes -from bigframes.core import bq_data, identifiers, nodes -from bigframes.core.ordering import OrderingExpression - -# SQL Nodes are generally terminal, so don't support rich transformation methods -# like remap_vars, remap_refs, etc. -# Still, fields should be defined on them, as typing info is still used for -# dispatching some operators in the emitter, and for validation. - - -# TODO: Join node, union node -@dataclasses.dataclass(frozen=True) -class SqlDataSource(nodes.LeafNode): - source: bq_data.BigqueryDataSource - - @functools.cached_property - def fields(self) -> Sequence[nodes.Field]: - return tuple( - nodes.Field( - identifiers.ColumnId(source_id), - self.source.schema.get_type(source_id), - self.source.table.schema_by_id[source_id].is_nullable, - ) - for source_id in self.source.schema.names - ) - - @property - def is_star_selection(self) -> bool: - return tuple(self.source.schema.names) == tuple( - field.name for field in self.source.table.physical_schema - ) - - @property - def variables_introduced(self) -> int: - # This operation only renames variables, doesn't actually create new ones - return 0 - - @property - def defines_namespace(self) -> bool: - return True - - @property - def explicitly_ordered(self) -> bool: - return False - - @property - def order_ambiguous(self) -> bool: - return True - - @property - def row_count(self) -> Optional[int]: - return self.source.n_rows - - @property - def node_defined_ids(self) -> Tuple[identifiers.ColumnId, ...]: - return tuple(self.ids) - - @property - def consumed_ids(self): - return () - - @property - def _node_expressions(self): - return () - - def remap_vars( - self, mappings: Mapping[identifiers.ColumnId, identifiers.ColumnId] - ) -> SqlSelectNode: - raise NotImplementedError() - - def remap_refs( - self, mappings: Mapping[identifiers.ColumnId, identifiers.ColumnId] - ) -> SqlSelectNode: - raise NotImplementedError() # type: ignore - - -@dataclasses.dataclass(frozen=True) -class SqlWithCtesNode(nodes.BigFrameNode): - # def, name pairs - child: nodes.BigFrameNode - cte_names: tuple[str, ...] - cte_defs: tuple[nodes.BigFrameNode, ...] - - @property - def child_nodes(self) -> Sequence[nodes.BigFrameNode]: - return (self.child, *self.cte_defs) - - @property - def fields(self) -> Sequence[nodes.Field]: - return self.child.fields - - @property - def variables_introduced(self) -> int: - # This operation only renames variables, doesn't actually create new ones - return 0 - - @property - def defines_namespace(self) -> bool: - return True - - @property - def explicitly_ordered(self) -> bool: - return False - - @property - def order_ambiguous(self) -> bool: - return True - - @property - def row_count(self) -> Optional[int]: - return self.child.row_count - - @property - def node_defined_ids(self) -> Tuple[identifiers.ColumnId, ...]: - return tuple(self.ids) - - @property - def consumed_ids(self): - return () - - @property - def _node_expressions(self): - return () - - def remap_vars( - self, mappings: Mapping[identifiers.ColumnId, identifiers.ColumnId] - ) -> SqlWithCtesNode: - raise NotImplementedError() - - def remap_refs( - self, mappings: Mapping[identifiers.ColumnId, identifiers.ColumnId] - ) -> SqlWithCtesNode: - raise NotImplementedError() # type: ignore - - def transform_children( - self, transform: Callable[[nodes.BigFrameNode], nodes.BigFrameNode] - ) -> SqlWithCtesNode: - return SqlWithCtesNode( - transform(self.child), - self.cte_names, - tuple(transform(cte) for cte in self.cte_defs), - ) - - -@dataclasses.dataclass(frozen=True) -class SqlCteRefNode(nodes.LeafNode): - cte_name: str - cte_schema: tuple[nodes.Field, ...] - - @property - def fields(self) -> Sequence[nodes.Field]: - return self.cte_schema - - @property - def variables_introduced(self) -> int: - # This operation only renames variables, doesn't actually create new ones - return 0 - - @property - def defines_namespace(self) -> bool: - return True - - @property - def explicitly_ordered(self) -> bool: - return False - - @property - def order_ambiguous(self) -> bool: - return True - - @property - def row_count(self) -> Optional[int]: - raise NotImplementedError() - - @property - def node_defined_ids(self) -> Tuple[identifiers.ColumnId, ...]: - return tuple(self.ids) - - @property - def consumed_ids(self): - return () - - @property - def _node_expressions(self): - return () - - def remap_vars( - self, mappings: Mapping[identifiers.ColumnId, identifiers.ColumnId] - ) -> SqlCteRefNode: - raise NotImplementedError() - - def remap_refs( - self, mappings: Mapping[identifiers.ColumnId, identifiers.ColumnId] - ) -> SqlCteRefNode: - raise NotImplementedError() # type: ignore - - -@dataclasses.dataclass(frozen=True) -class SqlSelectNode(nodes.UnaryNode): - selections: tuple[nodes.ColumnDef, ...] = () - predicates: tuple[ex.Expression, ...] = () - sorting: tuple[OrderingExpression, ...] = () - limit: Optional[int] = None - - @functools.cached_property - def fields(self) -> Sequence[nodes.Field]: - fields = [] - for cdef in self.selections: - bound_expr = ex.bind_schema_fields(cdef.expression, self.child.field_by_id) - field = nodes.Field( - cdef.id, - bigframes.dtypes.dtype_for_etype(bound_expr.output_type), - nullable=bound_expr.nullable, - ) - - # Special case until we get better nullability inference in expression objects themselves - if bound_expr.is_identity and not any( - self.child.field_by_id[id].nullable - for id in cdef.expression.column_references - ): - field = field.with_nonnull() - fields.append(field) - - return tuple(fields) - - @property - def variables_introduced(self) -> int: - # This operation only renames variables, doesn't actually create new ones - return 0 - - @property - def defines_namespace(self) -> bool: - return True - - @property - def row_count(self) -> Optional[int]: - if self.child.row_count is not None: - if self.limit is not None: - return min([self.limit, self.child.row_count]) - return self.child.row_count - - return None - - @property - def node_defined_ids(self) -> Tuple[identifiers.ColumnId, ...]: - return tuple(cdef.id for cdef in self.selections) - - @property - def consumed_ids(self): - raise NotImplementedError() - - @property - def _node_expressions(self): - raise NotImplementedError() - - @property - def is_star_selection(self) -> bool: - if tuple(self.ids) != tuple(self.child.ids): - return False - for cdef in self.selections: - if not isinstance(cdef.expression, ex.DerefOp): - return False - if cdef.expression.id != cdef.id: - return False - return True - - @functools.cache - def get_id_mapping(self) -> dict[identifiers.ColumnId, ex.Expression]: - return {cdef.id: cdef.expression for cdef in self.selections} - - def remap_vars( - self, mappings: Mapping[identifiers.ColumnId, identifiers.ColumnId] - ) -> SqlSelectNode: - raise NotImplementedError() - - def remap_refs( - self, mappings: Mapping[identifiers.ColumnId, identifiers.ColumnId] - ) -> SqlSelectNode: - raise NotImplementedError() # type: ignore diff --git a/bigframes/core/tools/datetimes.py b/bigframes/core/tools/datetimes.py index 0cdda67693d..0e5594d498b 100644 --- a/bigframes/core/tools/datetimes.py +++ b/bigframes/core/tools/datetimes.py @@ -16,7 +16,7 @@ from collections.abc import Mapping from datetime import date, datetime -from typing import TYPE_CHECKING, Optional, Union +from typing import Optional, TYPE_CHECKING, Union import bigframes_vendored.constants as constants import bigframes_vendored.pandas.core.tools.datetimes as vendored_pandas_datetimes @@ -60,17 +60,10 @@ def to_datetime( arg = bigframes.series.Series(arg, session=session) - if ( - format - and unit - and arg.dtype in (bigframes.dtypes.INT_DTYPE, bigframes.dtypes.FLOAT_DTYPE) - ): # type: ignore + if format and unit and arg.dtype in (bigframes.dtypes.INT_DTYPE, bigframes.dtypes.FLOAT_DTYPE): # type: ignore raise ValueError("cannot specify both format and unit") - if unit and arg.dtype not in ( - bigframes.dtypes.INT_DTYPE, - bigframes.dtypes.FLOAT_DTYPE, - ): # type: ignore + if unit and arg.dtype not in (bigframes.dtypes.INT_DTYPE, bigframes.dtypes.FLOAT_DTYPE): # type: ignore raise NotImplementedError( f"Unit parameter is not supported for non-numerical input types. {constants.FEEDBACK_LINK}" ) diff --git a/bigframes/core/tree_properties.py b/bigframes/core/tree_properties.py index 225cfc2f437..baf4b12566b 100644 --- a/bigframes/core/tree_properties.py +++ b/bigframes/core/tree_properties.py @@ -15,13 +15,10 @@ import functools import itertools -from typing import TYPE_CHECKING, Callable, Dict, Optional, Sequence +from typing import Callable, Dict, Optional, Sequence import bigframes.core.nodes as nodes -if TYPE_CHECKING: - import bigframes.session.execution_cache as execution_cache - def is_trivially_executable(node: nodes.BigFrameNode) -> bool: if local_only(node): @@ -68,7 +65,7 @@ def select_cache_target( root: nodes.BigFrameNode, min_complexity: float, max_complexity: float, - cache: execution_cache.ExecutionCache, + cache: dict[nodes.BigFrameNode, nodes.BigFrameNode], heuristic: Callable[[int, int], float], ) -> Optional[nodes.BigFrameNode]: """Take tree, and return candidate nodes with (# of occurences, post-caching planning complexity). @@ -78,7 +75,7 @@ def select_cache_target( @functools.cache def _with_caching(subtree: nodes.BigFrameNode) -> nodes.BigFrameNode: - return cache.subsitute_cached_subplans(subtree) + return nodes.top_down(subtree, lambda x: cache.get(x, x)) def _combine_counts( left: Dict[nodes.BigFrameNode, int], right: Dict[nodes.BigFrameNode, int] @@ -109,7 +106,6 @@ def _node_counts_inner( if len(node_counts) == 0: raise ValueError("node counts should be non-zero") - # for each considered node, calculate heuristic value, and return node with max value return max( node_counts.keys(), key=lambda node: heuristic( diff --git a/bigframes/core/utils.py b/bigframes/core/utils.py index 641fbcc9ac4..dd37a352a7c 100644 --- a/bigframes/core/utils.py +++ b/bigframes/core/utils.py @@ -15,8 +15,8 @@ import functools import re import typing -import warnings from typing import Hashable, Iterable, List +import warnings import bigframes_vendored.pandas.io.common as vendored_pandas_io_common import numpy as np @@ -113,13 +113,13 @@ def get_standardized_ids( """ col_ids = [ UNNAMED_COLUMN_ID - if pd.isna(col_label) # type: ignore + if col_label is None else label_to_identifier(col_label, strict=strict) for col_label in col_labels ] idx_ids = [ UNNAMED_INDEX_ID - if pd.isna(idx_label) # type: ignore + if idx_label is None else label_to_identifier(idx_label, strict=strict) for idx_label in idx_labels ] @@ -234,7 +234,7 @@ def wrapper(*args, **kwargs): def timedelta_to_micros( - timedelta: typing.Union[pd.Timedelta, datetime.timedelta, np.timedelta64], + timedelta: typing.Union[pd.Timedelta, datetime.timedelta, np.timedelta64] ) -> int: if isinstance(timedelta, pd.Timedelta): # pd.Timedelta.value returns total nanoseconds. @@ -249,16 +249,3 @@ def timedelta_to_micros( ) * 1_000_000 + timedelta.microseconds raise TypeError(f"Unrecognized input type: {type(timedelta)}") - - -def get_ipython_execution_count() -> typing.Optional[int]: - """Returns the current IPython cell execution count if running in a notebook, else None.""" - try: - from IPython.core.interactiveshell import InteractiveShell - - if InteractiveShell.initialized(): - ipy = InteractiveShell.instance() - return getattr(ipy, "execution_count", None) - except (ImportError, NameError): - pass - return None diff --git a/bigframes/core/validations.py b/bigframes/core/validations.py index 84f802cd740..e6fdcb7bd5d 100644 --- a/bigframes/core/validations.py +++ b/bigframes/core/validations.py @@ -17,7 +17,7 @@ from __future__ import annotations import functools -from typing import TYPE_CHECKING, Optional, Protocol, Union +from typing import Optional, Protocol, TYPE_CHECKING, Union import bigframes_vendored.constants as constants @@ -32,10 +32,12 @@ class HasSession(Protocol): @property - def _session(self) -> Session: ... + def _session(self) -> Session: + ... @property - def _block(self) -> Block: ... + def _block(self) -> Block: + ... def requires_index(meth): diff --git a/bigframes/core/window/rolling.py b/bigframes/core/window/rolling.py index a3660954dfb..b7bb62372cc 100644 --- a/bigframes/core/window/rolling.py +++ b/bigframes/core/window/rolling.py @@ -15,20 +15,20 @@ from __future__ import annotations import datetime -from typing import TYPE_CHECKING, Literal, Mapping, Sequence, Union +from typing import Literal, Mapping, Sequence, TYPE_CHECKING, Union import bigframes_vendored.pandas.core.window.rolling as vendored_pandas_rolling import numpy import pandas -import bigframes.core.blocks as blocks -import bigframes.operations.aggregations as agg_ops from bigframes import dtypes -from bigframes._tools import docs -from bigframes.core import agg_expressions, ordering, utils, window_spec +from bigframes.core import agg_expressions from bigframes.core import expression as ex +from bigframes.core import ordering, utils, window_spec +import bigframes.core.blocks as blocks from bigframes.core.logging import log_adapter from bigframes.core.window import ordering as window_ordering +import bigframes.operations.aggregations as agg_ops if TYPE_CHECKING: import bigframes.dataframe as df @@ -36,8 +36,9 @@ @log_adapter.class_logger -@docs.inherit_docs(vendored_pandas_rolling.Window) -class Window: +class Window(vendored_pandas_rolling.Window): + __doc__ = vendored_pandas_rolling.Window.__doc__ + def __init__( self, block: blocks.Block, @@ -217,6 +218,7 @@ def create_range_window( grouping_keys: Sequence[str] = tuple(), drop_null_groups: bool = True, ) -> Window: + if on is None: # Rolling on index index_dtypes = block.index.dtypes diff --git a/bigframes/core/window_spec.py b/bigframes/core/window_spec.py index 509dd954b9f..9e4ee171031 100644 --- a/bigframes/core/window_spec.py +++ b/bigframes/core/window_spec.py @@ -13,9 +13,9 @@ # limitations under the License. from __future__ import annotations +from dataclasses import dataclass, replace import datetime import itertools -from dataclasses import dataclass, replace from typing import Callable, Literal, Mapping, Optional, Sequence, Set, Tuple, Union import numpy as np diff --git a/bigframes/dataframe.py b/bigframes/dataframe.py index 1ddf858509f..e1ad4f3e75d 100644 --- a/bigframes/dataframe.py +++ b/bigframes/dataframe.py @@ -23,11 +23,10 @@ import sys import textwrap import typing -import warnings from typing import ( - TYPE_CHECKING, Any, Callable, + cast, Dict, Hashable, Iterable, @@ -35,32 +34,30 @@ Literal, Mapping, Optional, + overload, Sequence, Tuple, - TypeVar, Union, - cast, - overload, ) +import warnings import bigframes_vendored.constants as constants import bigframes_vendored.pandas.core.frame as vendored_pandas_frame import bigframes_vendored.pandas.pandas._typing as vendored_pandas_typing import google.api_core.exceptions import google.cloud.bigquery as bigquery -import google.cloud.bigquery.job -import google.cloud.bigquery.table import numpy import pandas +from pandas.api import extensions as pd_ext +import pandas.io.formats.format import pyarrow import tabulate -from pandas.api import extensions as pd_ext import bigframes.constants import bigframes.core +from bigframes.core import agg_expressions import bigframes.core.block_transforms as block_ops import bigframes.core.blocks as blocks -import bigframes.core.col import bigframes.core.convert import bigframes.core.explode import bigframes.core.expression as ex @@ -69,48 +66,40 @@ import bigframes.core.indexers as indexers import bigframes.core.indexes as indexes import bigframes.core.interchange +from bigframes.core.logging import log_adapter import bigframes.core.ordering as order import bigframes.core.utils as utils import bigframes.core.validations as validations import bigframes.core.window +from bigframes.core.window import rolling import bigframes.core.window_spec as windows import bigframes.dtypes import bigframes.exceptions as bfe import bigframes.formatting_helpers as formatter import bigframes.functions +from bigframes.functions import function_typing import bigframes.operations as ops import bigframes.operations.aggregations as agg_ops +import bigframes.operations.ai import bigframes.operations.plotting as plotting +import bigframes.operations.semantics import bigframes.operations.structs import bigframes.series import bigframes.session._io.bigquery import bigframes.session.execution_spec as ex_spec -from bigframes._tools import docs -from bigframes.core import agg_expressions -from bigframes.core.logging import log_adapter -from bigframes.core.window import rolling -from bigframes.functions import function_typing -if TYPE_CHECKING: +if typing.TYPE_CHECKING: from _typeshed import SupportsRichComparison - import bigframes.extensions.bigframes.dataframe_accessor as bigquery_accessor import bigframes.session SingleItemValue = Union[ - bigframes.series.Series, - int, - float, - str, - pandas.Timedelta, - Callable, - bigframes.core.col.Expression, + bigframes.series.Series, int, float, str, pandas.Timedelta, Callable ] MultiItemValue = Union[ "DataFrame", Sequence[int | float | str | pandas.Timedelta | Callable] ] -U = TypeVar("U") LevelType = typing.Hashable LevelsType = typing.Union[LevelType, typing.Sequence[LevelType]] @@ -124,8 +113,8 @@ # Inherits from pandas DataFrame so that we can use the same docstrings. @log_adapter.class_logger -@docs.inherit_docs(vendored_pandas_frame.DataFrame) -class DataFrame: +class DataFrame(vendored_pandas_frame.DataFrame): + __doc__ = vendored_pandas_frame.DataFrame.__doc__ # internal flag to disable cache at all _disable_cache_override: bool = False # Must be above 5000 for pandas to delegate to bigframes for binops @@ -145,7 +134,7 @@ def __init__( ): global bigframes - self._query_job: Optional[google.cloud.bigquery.job.QueryJob] = None + self._query_job: Optional[bigquery.QueryJob] = None if copy is not None and not copy: raise ValueError( @@ -314,6 +303,7 @@ def loc(self) -> indexers.LocDataFrameIndexer: return indexers.LocDataFrameIndexer(self) @property + @validations.requires_ordering() def iloc(self) -> indexers.ILocDataFrameIndexer: return indexers.ILocDataFrameIndexer(self) @@ -335,7 +325,7 @@ def dtypes(self) -> pandas.Series: @property def columns(self) -> pandas.Index: - return self._block.column_labels + return self.dtypes.index @columns.setter def columns(self, labels: pandas.Index): @@ -346,10 +336,6 @@ def columns(self, labels: pandas.Index): def shape(self) -> Tuple[int, int]: return self._block.shape - @property - def axes(self) -> list: - return [self.index, self.columns] - @property def size(self) -> int: rows, cols = self.shape @@ -376,25 +362,6 @@ def bqclient(self) -> bigframes.Session: def _session(self) -> bigframes.Session: return self._get_block().expr.session - @property - def bigquery( - self, - ) -> bigquery_accessor.BigframesBigQueryDataFrameAccessor: - """ - Accessor for BigQuery functionality. - - Returns: - bigframes.extensions.core.dataframe_accessor.BigQueryDataFrameAccessor: - Accessor that exposes BigQuery functionality on a DataFrame, - with method names closer to SQL. - """ - # Import the accessor here to avoid circular imports. - import bigframes.extensions.bigframes.dataframe_accessor - - return bigframes.extensions.bigframes.dataframe_accessor.BigframesBigQueryDataFrameAccessor( - self - ) - @property def _has_index(self) -> bool: return len(self._block.index_columns) > 0 @@ -413,10 +380,7 @@ def __len__(self): rows, _ = self.shape return rows - def __bool__(self): - raise ValueError( - "Cannot convert dataframe into bool. Consider using .empty(), .any(), or .all() methods." - ) + __len__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__len__) def __iter__(self): return iter(self.columns) @@ -438,41 +402,17 @@ def astype( if errors not in ["raise", "null"]: raise ValueError("Arg 'error' must be one of 'raise' or 'null'") - if isinstance(dtype, dict): - for col in dtype: - if col not in self.columns: - raise KeyError( - f"Only Column Names are allowed in dtypes dict. '{col}' is not in the columns." - ) - safe_cast = errors == "null" - exprs: list[ex.Expression] = [] - for col_id, col_label in zip( - self._block.value_columns, self._block.column_labels - ): - from_type = self._block._column_type(col_id) - - if isinstance(dtype, dict): - if col_label not in dtype: - exprs.append(ex.deref(col_id)) - continue - to_type = bigframes.dtypes.bigframes_type(dtype[col_label]) - else: - to_type = bigframes.dtypes.bigframes_type(dtype) - - op: ops.UnaryOp - if to_type == bigframes.dtypes.JSON_DTYPE: - op = ops.ToJSON(safe=safe_cast) - elif from_type == bigframes.dtypes.JSON_DTYPE: - op = ops.JSONDecode(to_type=to_type, safe=safe_cast) - else: - op = ops.AsTypeOp(to_type=to_type, safe=safe_cast) + if isinstance(dtype, dict): + result = self.copy() + for col, to_type in dtype.items(): + result[col] = result[col].astype(to_type) + return result - exprs.append(op.as_expr(ex.deref(col_id))) + dtype = bigframes.dtypes.bigframes_type(dtype) - block = self._block.project_exprs(exprs, labels=self.columns, drop=True) - return DataFrame(block) + return self._apply_unary_op(ops.AsTypeOp(dtype, safe_cast)) def _should_sql_have_index(self) -> bool: """Should the SQL we pass to BQML and other I/O include the index?""" @@ -481,9 +421,7 @@ def _should_sql_have_index(self) -> bool: self.index.name is not None or len(self.index.names) > 1 ) - def _to_placeholder_table( - self, dry_run: bool = False - ) -> google.cloud.bigquery.table.TableReference: + def _to_placeholder_table(self, dry_run: bool = False) -> bigquery.TableReference: """Compiles this DataFrame's expression tree to SQL and saves it to a (temporary) view or table (in the case of a dry run). """ @@ -533,11 +471,11 @@ def sql(self) -> str: ) from e @property - def query_job(self) -> Optional[google.cloud.bigquery.job.QueryJob]: + def query_job(self) -> Optional[bigquery.QueryJob]: """BigQuery job metadata for the most recent query. Returns: - None or google.cloud.bigquery.job.QueryJob: + None or google.cloud.bigquery.QueryJob: The most recent `QueryJob `_. """ @@ -651,11 +589,48 @@ def select_dtypes(self, include=None, exclude=None) -> DataFrame: ) return DataFrame(self._block.select_columns(selected_columns)) - def _set_internal_query_job( - self, query_job: Optional[google.cloud.bigquery.job.QueryJob] - ): + def _set_internal_query_job(self, query_job: Optional[bigquery.QueryJob]): self._query_job = query_job + @overload + def __getitem__( + self, + key: bigframes.series.Series, + ) -> DataFrame: + ... + + @overload + def __getitem__( + self, + key: slice, + ) -> DataFrame: + ... + + @overload + def __getitem__( + self, + key: List[str], + ) -> DataFrame: + ... + + @overload + def __getitem__( + self, + key: List[blocks.Label], + ) -> DataFrame: + ... + + @overload + def __getitem__(self, key: pandas.Index) -> DataFrame: + ... + + @overload + def __getitem__( + self, + key: blocks.Label, + ) -> bigframes.series.Series: + ... + def __getitem__( self, key: Union[ @@ -670,18 +645,13 @@ def __getitem__( ): # No return type annotations (like pandas) as type cannot always be determined statically # NOTE: This implements the operations described in # https://pandas.pydata.org/docs/getting_started/intro_tutorials/03_subset_data.html - import bigframes.core.col - import bigframes.pandas - if isinstance(key, bigframes.pandas.Series): + if isinstance(key, bigframes.series.Series): return self._getitem_bool_series(key) if isinstance(key, slice): return self.iloc[key] - if isinstance(key, bigframes.core.col.Expression): - return self.loc[key] - # TODO(tswast): Fix this pylance warning: Class overlaps "Hashable" # unsafely and could produce a match at runtime if isinstance(key, blocks.Label): @@ -693,6 +663,8 @@ def __getitem__( # TODO(tswast): What case is this supposed to be handling? return self._getitem_columns([cast(Hashable, key)]) + __getitem__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__getitem__) + def _getitem_columns(self, key: Sequence[blocks.Label]) -> DataFrame: selected_ids: Tuple[str, ...] = () for label in key: @@ -730,12 +702,9 @@ def _getitem_bool_series(self, key: bigframes.series.Series) -> DataFrame: f"Only boolean series currently supported for indexing. {constants.FEEDBACK_LINK}" ) # TODO: enforce stricter alignment - ( - combined_index, - ( - get_column_left, - get_column_right, - ), + combined_index, ( + get_column_left, + get_column_right, ) = self._block.join(key._block, how="left") block = combined_index filter_col_id = get_column_right[key._value_column] @@ -753,7 +722,7 @@ def __getattr__(self, key: str): # https://github.com/googleapis/python-bigquery-dataframes/issues/728 # and # https://nedbatchelder.com/blog/201010/surprising_getattr_recursion.html - if "_block" not in self.__dict__ or key == "_block": + if key == "_block": raise AttributeError(key) if key in self._block.column_labels: @@ -839,27 +808,22 @@ def __repr__(self) -> str: column_count=len(self.columns), ) - def _prepare_display_df(self) -> DataFrame: - """Process ObjectRef and JSON/nested JSON columns for display.""" - import bigframes.bigquery as bbq - + def _get_display_df_and_blob_cols(self) -> tuple[DataFrame, list[str]]: + """Process blob columns for display.""" df = self - # Arrow/Pandas to_pandas_batches does not support raw JSON/nested JSON - # columns. Pre-serialize them to string format to bypass this limit. - # Using TO_JSON_STRING via SqlScalarOp handles complex nested STRUCT - # types correctly. Use the offset so that we can handle duplicate and - # non-string column names. - json_col_indexes = [ - col_index - for col_index, col in enumerate(df.columns) - if bigframes.dtypes.contains_db_dtypes_json_dtype(df[col].dtype) - ] - if json_col_indexes: - df.iloc[:, json_col_indexes] = cast( - DataFrame, - df.iloc[:, json_col_indexes].apply(bbq.to_json_string), # type: ignore - ) - return df + blob_cols = [] + if bigframes.options.display.blob_display: + blob_cols = [ + series_name + for series_name, series in self.items() + if series.dtype == bigframes.dtypes.OBJ_REF_DTYPE + ] + if blob_cols: + df = self.copy() + for col in blob_cols: + # TODO(garrettwu): Not necessary to get access urls for all the rows. Update when having a to get URLs from local data. + df[col] = df[col].blob._get_runtime(mode="R", with_metadata=True) + return df, blob_cols def _repr_mimebundle_(self, include=None, exclude=None): """ @@ -887,6 +851,8 @@ def __setitem__( df = self._assign_single_item(key, value) self._set_block(df._get_block()) + __setitem__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__setitem__) + def _apply_binop( self, other: float | int | bigframes.series.Series | DataFrame, @@ -997,39 +963,53 @@ def eq(self, other: typing.Any, axis: str | int = "columns") -> DataFrame: def __eq__(self, other) -> DataFrame: # type: ignore return self.eq(other) + __eq__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__eq__) + def ne(self, other: typing.Any, axis: str | int = "columns") -> DataFrame: return self._apply_binop(other, ops.ne_op, axis=axis) def __ne__(self, other) -> DataFrame: # type: ignore return self.ne(other) + __ne__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__ne__) + def __invert__(self) -> DataFrame: return self._apply_unary_op(ops.invert_op) + __invert__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__invert__) + def le(self, other: typing.Any, axis: str | int = "columns") -> DataFrame: return self._apply_binop(other, ops.le_op, axis=axis) def __le__(self, other) -> DataFrame: return self.le(other) + __le__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__le__) + def lt(self, other: typing.Any, axis: str | int = "columns") -> DataFrame: return self._apply_binop(other, ops.lt_op, axis=axis) def __lt__(self, other) -> DataFrame: return self.lt(other) + __lt__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__lt__) + def ge(self, other: typing.Any, axis: str | int = "columns") -> DataFrame: return self._apply_binop(other, ops.ge_op, axis=axis) def __ge__(self, other) -> DataFrame: return self.ge(other) + __ge__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__ge__) + def gt(self, other: typing.Any, axis: str | int = "columns") -> DataFrame: return self._apply_binop(other, ops.gt_op, axis=axis) def __gt__(self, other) -> DataFrame: return self.gt(other) + __gt__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__gt__) + def add( self, other: float | int | bigframes.series.Series | DataFrame, @@ -1051,9 +1031,13 @@ def radd( def __add__(self, other) -> DataFrame: return self.add(other) + __add__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__add__) + def __radd__(self, other) -> DataFrame: return self.radd(other) + __radd__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__radd__) + def sub( self, other: float | int | bigframes.series.Series | DataFrame, @@ -1062,10 +1046,13 @@ def sub( return self._apply_binop(other, ops.sub_op, axis=axis) subtract = sub + subtract.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.sub) def __sub__(self, other): return self.sub(other) + __sub__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__sub__) + def rsub( self, other: float | int | bigframes.series.Series | DataFrame, @@ -1076,6 +1063,8 @@ def rsub( def __rsub__(self, other): return self.rsub(other) + __rsub__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__rsub__) + def mul( self, other: float | int | bigframes.series.Series | DataFrame, @@ -1084,10 +1073,13 @@ def mul( return self._apply_binop(other, ops.mul_op, axis=axis) multiply = mul + multiply.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.mul) def __mul__(self, other): return self.mul(other) + __mul__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__mul__) + def rmul( self, other: float | int | bigframes.series.Series | DataFrame, @@ -1098,6 +1090,8 @@ def rmul( def __rmul__(self, other): return self.rmul(other) + __rmul__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__rmul__) + def truediv( self, other: float | int | bigframes.series.Series | DataFrame, @@ -1105,11 +1099,14 @@ def truediv( ) -> DataFrame: return self._apply_binop(other, ops.div_op, axis=axis) + truediv.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.truediv) div = divide = truediv def __truediv__(self, other): return self.truediv(other) + __truediv__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__truediv__) + def rtruediv( self, other: float | int | bigframes.series.Series | DataFrame, @@ -1118,10 +1115,13 @@ def rtruediv( return self._apply_binop(other, ops.div_op, axis=axis, reverse=True) rdiv = rtruediv + rdiv.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.rtruediv) def __rtruediv__(self, other): return self.rtruediv(other) + __rtruediv__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__rtruediv__) + def floordiv( self, other: float | int | bigframes.series.Series | DataFrame, @@ -1132,6 +1132,8 @@ def floordiv( def __floordiv__(self, other): return self.floordiv(other) + __floordiv__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__floordiv__) + def rfloordiv( self, other: float | int | bigframes.series.Series | DataFrame, @@ -1142,26 +1144,26 @@ def rfloordiv( def __rfloordiv__(self, other): return self.rfloordiv(other) - def mod( - self, - other: int | bigframes.series.Series | DataFrame, - axis: str | int = "columns", - ) -> DataFrame: # type: ignore + __rfloordiv__.__doc__ = inspect.getdoc( + vendored_pandas_frame.DataFrame.__rfloordiv__ + ) + + def mod(self, other: int | bigframes.series.Series | DataFrame, axis: str | int = "columns") -> DataFrame: # type: ignore return self._apply_binop(other, ops.mod_op, axis=axis) def __mod__(self, other): return self.mod(other) - def rmod( - self, - other: int | bigframes.series.Series | DataFrame, - axis: str | int = "columns", - ) -> DataFrame: # type: ignore + __mod__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__mod__) + + def rmod(self, other: int | bigframes.series.Series | DataFrame, axis: str | int = "columns") -> DataFrame: # type: ignore return self._apply_binop(other, ops.mod_op, axis=axis, reverse=True) def __rmod__(self, other): return self.rmod(other) + __rmod__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__rmod__) + def pow( self, other: int | bigframes.series.Series, axis: str | int = "columns" ) -> DataFrame: @@ -1170,6 +1172,8 @@ def pow( def __pow__(self, other): return self.pow(other) + __pow__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__pow__) + def rpow( self, other: int | bigframes.series.Series, axis: str | int = "columns" ) -> DataFrame: @@ -1178,19 +1182,27 @@ def rpow( def __rpow__(self, other): return self.rpow(other) + __rpow__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__rpow__) + def __and__(self, other: bool | int | bigframes.series.Series) -> DataFrame: return self._apply_binop(other, ops.and_op) + __and__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__and__) + __rand__ = __and__ def __or__(self, other: bool | int | bigframes.series.Series) -> DataFrame: return self._apply_binop(other, ops.or_op) + __or__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__or__) + __ror__ = __or__ def __xor__(self, other: bool | int | bigframes.series.Series) -> DataFrame: return self._apply_binop(other, ops.xor_op) + __xor__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__xor__) + __rxor__ = __xor__ def __pos__(self) -> DataFrame: @@ -1202,6 +1214,8 @@ def __neg__(self) -> DataFrame: def __abs__(self) -> DataFrame: return self._apply_unary_op(ops.abs_op) + __abs__.__doc__ = abs.__doc__ + def align( self, other: typing.Union[DataFrame, bigframes.series.Series], @@ -1636,7 +1650,8 @@ def to_pandas( # type: ignore[overload-overlap] ordered: bool = ..., dry_run: Literal[False] = ..., allow_large_results: Optional[bool] = ..., - ) -> pandas.DataFrame: ... + ) -> pandas.DataFrame: + ... @overload def to_pandas( @@ -1648,7 +1663,8 @@ def to_pandas( ordered: bool = ..., dry_run: Literal[True] = ..., allow_large_results: Optional[bool] = ..., - ) -> pandas.Series: ... + ) -> pandas.Series: + ... def to_pandas( self, @@ -1768,8 +1784,7 @@ def to_pandas( ) if query_job: self._set_internal_query_job(query_job) - df.columns = self._block.column_labels - return df + return df.set_axis(self._block.column_labels, axis=1, copy=False) def to_pandas_batches( self, @@ -1777,7 +1792,6 @@ def to_pandas_batches( max_results: Optional[int] = None, *, allow_large_results: Optional[bool] = None, - cell_execution_count: Optional[int] = None, ) -> blocks.PandasBatches: """Stream DataFrame results to an iterable of pandas DataFrame. @@ -1830,7 +1844,6 @@ def to_pandas_batches( page_size=page_size, max_results=max_results, allow_large_results=allow_large_results, - cell_execution_count=cell_execution_count, ) def _to_pandas_batches( @@ -1839,16 +1852,14 @@ def _to_pandas_batches( max_results: Optional[int] = None, *, allow_large_results: Optional[bool] = None, - cell_execution_count: Optional[int] = None, ) -> blocks.PandasBatches: return self._block.to_pandas_batches( page_size=page_size, max_results=max_results, allow_large_results=allow_large_results, - cell_execution_count=cell_execution_count, ) - def _compute_dry_run(self) -> google.cloud.bigquery.job.QueryJob: + def _compute_dry_run(self) -> bigquery.QueryJob: _, query_job = self._block._compute_dry_run() return query_job @@ -1899,8 +1910,7 @@ def peek( raise ValueError( "Cannot peek efficiently when data has aggregates, joins or window functions applied. Use force=True to fully compute dataframe." ) - maybe_result.columns = self._block.column_labels - return maybe_result + return maybe_result.set_axis(self._block.column_labels, axis=1, copy=False) def nlargest( self, @@ -1966,7 +1976,8 @@ def drop( columns: Union[blocks.Label, Sequence[blocks.Label]] = None, level: typing.Optional[LevelType] = None, inplace: Literal[False] = False, - ) -> DataFrame: ... + ) -> DataFrame: + ... @overload def drop( @@ -1978,7 +1989,8 @@ def drop( columns: Union[blocks.Label, Sequence[blocks.Label]] = None, level: typing.Optional[LevelType] = None, inplace: Literal[True], - ) -> None: ... + ) -> None: + ... def drop( self, @@ -2122,17 +2134,20 @@ def _resolve_levels(self, level: LevelsType) -> typing.Sequence[str]: return self._block.index.resolve_level(level) @overload - def rename(self, *, columns: Mapping[blocks.Label, blocks.Label]) -> DataFrame: ... + def rename(self, *, columns: Mapping[blocks.Label, blocks.Label]) -> DataFrame: + ... @overload def rename( self, *, columns: Mapping[blocks.Label, blocks.Label], inplace: Literal[False] - ) -> DataFrame: ... + ) -> DataFrame: + ... @overload def rename( self, *, columns: Mapping[blocks.Label, blocks.Label], inplace: Literal[True] - ) -> None: ... + ) -> None: + ... def rename( self, *, columns: Mapping[blocks.Label, blocks.Label], inplace: bool = False @@ -2149,7 +2164,8 @@ def rename( def rename_axis( self, mapper: typing.Union[blocks.Label, typing.Sequence[blocks.Label]], - ) -> DataFrame: ... + ) -> DataFrame: + ... @overload def rename_axis( @@ -2158,7 +2174,8 @@ def rename_axis( *, inplace: Literal[False], **kwargs, - ) -> DataFrame: ... + ) -> DataFrame: + ... @overload def rename_axis( @@ -2167,7 +2184,8 @@ def rename_axis( *, inplace: Literal[True], **kwargs, - ) -> None: ... + ) -> None: + ... def rename_axis( self, @@ -2218,13 +2236,6 @@ def _assign_single_item( ) -> DataFrame: if isinstance(v, bigframes.series.Series): return self._assign_series_join_on_index(k, v) - elif isinstance(v, bigframes.core.col.Expression): - label_to_col_ref = { - label: ex.deref(id) for id, label in self._block.col_id_to_label.items() - } - resolved_expr = v._value.bind_variables(label_to_col_ref) - block = self._block.project_block_exprs([resolved_expr], labels=[k]) - return DataFrame(block) elif isinstance(v, bigframes.dataframe.DataFrame): v_df_col_count = len(v._block.value_columns) if v_df_col_count != 1: @@ -2241,47 +2252,10 @@ def _assign_single_item( else: return self._assign_scalar(k, v) # type: ignore - def _assign_single_item_by_offset( - self, - offset: int, - value: SingleItemValue | MultiItemValue, - ) -> DataFrame: - if isinstance(value, bigframes.series.Series): - return self._assign_series_join_on_index_by_offset(offset, value) - elif isinstance(value, bigframes.core.col.Expression): - label_to_col_ref = { - label: ex.deref(id) for id, label in self._block.col_id_to_label.items() - } - resolved_expr = value._value.bind_variables(label_to_col_ref) - block, new_col_id = self._block.project_expr(resolved_expr) - target_col_id = self._block.value_columns[offset] - block = block.copy_values(new_col_id, target_col_id).drop_columns( - [new_col_id] - ) - return DataFrame(block) - elif isinstance(value, DataFrame): - v_df_col_count = len(value._block.value_columns) - if v_df_col_count != 1: - raise ValueError( - f"Cannot set a DataFrame with {v_df_col_count} columns to the single column at offset {offset}" - ) - return self._assign_series_join_on_index_by_offset( - offset, cast(bigframes.series.Series, value[value.columns[0]]) - ) - elif callable(value): - raise NotImplementedError( - "Callable assignment is not supported by column offset." - ) - elif utils.is_list_like(value): - return self._assign_single_item_listlike_by_offset(offset, value) - else: - return self._assign_scalar_by_offset(offset, value) # type: ignore - - def _assign_multi_items_helper( + def _assign_multi_items( self, - k: Sequence[Any] | pandas.Index, + k: list[str] | pandas.Index, v: SingleItemValue | MultiItemValue, - assign_single_fn: Callable[[DataFrame, Any, Any], DataFrame], ) -> DataFrame: value_sources: Sequence[Any] = [] if isinstance(v, DataFrame): @@ -2299,35 +2273,13 @@ def _assign_multi_items_helper( raise ValueError("Columns must be same length as key") # Repeatedly assign columns in order. - result = assign_single_fn(self, k[0], value_sources[0]) + result = self._assign_single_item(k[0], value_sources[0]) for target, source in zip(k[1:], value_sources[1:]): - result = assign_single_fn(result, target, source) + result = result._assign_single_item(target, source) return result - def _assign_multi_items( - self, - k: list[str] | pandas.Index, - v: SingleItemValue | MultiItemValue, - ) -> DataFrame: - return self._assign_multi_items_helper(k, v, DataFrame._assign_single_item) - - def _assign_multi_items_by_offsets( - self, - k: Sequence[int], - v: SingleItemValue | MultiItemValue, - ) -> DataFrame: - return self._assign_multi_items_helper( - k, v, DataFrame._assign_single_item_by_offset - ) - - _assign_multi_items_by_offset = _assign_multi_items_by_offsets - _assign_multi_items_by_label = _assign_multi_items - _assign_multi_items_by_labels = _assign_multi_items - - def _assign_single_item_listlike_to_col_ids( - self, col_ids: Sequence[str], label: Optional[str], value: Sequence - ) -> DataFrame: - given_rows = len(value) + def _assign_single_item_listlike(self, k: str, v: Sequence) -> DataFrame: + given_rows = len(v) actual_rows = len(self) assigning_to_empty_df = len(self.columns) == 0 and actual_rows == 0 if not assigning_to_empty_df and given_rows != actual_rows: @@ -2335,14 +2287,7 @@ def _assign_single_item_listlike_to_col_ids( f"Length of values ({given_rows}) does not match length of index ({actual_rows})" ) - temp_col_name = ( - label - if label is not None - else bigframes.core.guid.generate_guid("listlike_col_") - ) - local_df = DataFrame( - {temp_col_name: value}, session=self._get_block().expr.session - ) + local_df = DataFrame({k: v}, session=self._get_block().expr.session) # local_df is likely (but not guaranteed) to be cached locally # since the original list came from memory and so is probably < MAX_INLINE_DF_SIZE @@ -2350,24 +2295,17 @@ def _assign_single_item_listlike_to_col_ids( original_index_column_ids = self._block.index_columns self_block = self._block.reset_index(drop=False) if assigning_to_empty_df: - if label is None: - raise ValueError( - "Label required when assigning listlike to empty DataFrame." - ) if len(self._block.index_columns) > 1: # match error raised by pandas here raise ValueError( "Assigning listlike to a first column under multiindex is not supported." ) result_block = new_column_block.with_index_labels(self._block.index.names) - result_block = result_block.with_column_labels([label]) + result_block = result_block.with_column_labels([k]) else: - ( - result_block, - ( - get_column_left, - get_column_right, - ), + result_block, ( + get_column_left, + get_column_right, ) = self_block.join(new_column_block, how="left", block_identity_join=True) result_block = result_block.set_index( [get_column_left[col_id] for col_id in original_index_column_ids], @@ -2375,6 +2313,7 @@ def _assign_single_item_listlike_to_col_ids( ) src_col = get_column_right[new_column_block.value_columns[0]] # Check to see if key exists, and modify in place + col_ids = self._block.cols_matching_label(k) for col_id in col_ids: result_block = result_block.copy_values( src_col, get_column_left[col_id] @@ -2383,22 +2322,9 @@ def _assign_single_item_listlike_to_col_ids( result_block = result_block.drop_columns([src_col]) return DataFrame(result_block) - def _assign_single_item_listlike(self, k: str, v: Sequence) -> DataFrame: - col_ids = self._block.cols_matching_label(k) - return self._assign_single_item_listlike_to_col_ids(col_ids, k, v) - - def _assign_single_item_listlike_by_offset( - self, offset: int, value: Sequence - ) -> DataFrame: - col_ids = [self._block.value_columns[offset]] - return self._assign_single_item_listlike_to_col_ids(col_ids, None, value) + def _assign_scalar(self, label: str, value: Union[int, float, str]) -> DataFrame: + col_ids = self._block.cols_matching_label(label) - def _assign_scalar_to_col_ids( - self, - col_ids: Sequence[str], - label: Optional[str], - value: Union[int, float, str], - ) -> DataFrame: block, constant_col_id = self._block.create_constant(value, label) for col_id in col_ids: block = block.copy_values(constant_col_id, col_id) @@ -2408,40 +2334,25 @@ def _assign_scalar_to_col_ids( return DataFrame(block) - def _assign_scalar(self, label: str, value: Union[int, float, str]) -> DataFrame: - col_ids = self._block.cols_matching_label(label) - return self._assign_scalar_to_col_ids(col_ids, label, value) - - def _assign_scalar_by_offset( - self, offset: int, value: Union[int, float, str] - ) -> DataFrame: - col_ids = [self._block.value_columns[offset]] - return self._assign_scalar_to_col_ids(col_ids, None, value) - - def _assign_series_join_on_index_to_col_ids( - self, - column_ids: Sequence[str], - label: Optional[str], - series: bigframes.series.Series, + def _assign_series_join_on_index( + self, label: str, series: bigframes.series.Series ) -> DataFrame: block, (get_column_left, get_column_right) = self._block.join( series._block, how="left" ) - mapped_column_ids = [get_column_left[col_id] for col_id in column_ids] + column_ids = [ + get_column_left[col_id] for col_id in self._block.cols_matching_label(label) + ] source_column = get_column_right[series._value_column] - # Replace each column matching the ids - for column_id in mapped_column_ids: - block = block.copy_values(source_column, column_id) - if label is not None: - block = block.assign_label(column_id, label) + # Replace each column matching the label + for column_id in column_ids: + block = block.copy_values(source_column, column_id).assign_label( + column_id, label + ) - if not mapped_column_ids: - if label is None: - raise ValueError( - "Label required when appending a new column from Series." - ) + if not column_ids: # Append case, so new column needs appropriate label block = block.assign_label(source_column, label) else: @@ -2450,18 +2361,6 @@ def _assign_series_join_on_index_to_col_ids( return DataFrame(block.with_index_labels(self._block.index.names)) - def _assign_series_join_on_index( - self, label: str, series: bigframes.series.Series - ) -> DataFrame: - column_ids = self._block.cols_matching_label(label) - return self._assign_series_join_on_index_to_col_ids(column_ids, label, series) - - def _assign_series_join_on_index_by_offset( - self, offset: int, series: bigframes.series.Series - ) -> DataFrame: - column_ids = [self._block.value_columns[offset]] - return self._assign_series_join_on_index_to_col_ids(column_ids, None, series) - @overload # type: ignore[override] def reset_index( self, @@ -2472,7 +2371,8 @@ def reset_index( col_fill: Hashable = ..., allow_duplicates: Optional[bool] = ..., names: Union[None, Hashable, Sequence[Hashable]] = ..., - ) -> DataFrame: ... + ) -> DataFrame: + ... @overload def reset_index( @@ -2484,7 +2384,8 @@ def reset_index( col_fill: Hashable = ..., allow_duplicates: Optional[bool] = ..., names: Union[None, Hashable, Sequence[Hashable]] = ..., - ) -> None: ... + ) -> None: + ... @overload def reset_index( @@ -2496,7 +2397,8 @@ def reset_index( col_fill: Hashable = ..., allow_duplicates: Optional[bool] = ..., names: Union[None, Hashable, Sequence[Hashable]] = ..., - ) -> Optional[DataFrame]: ... + ) -> Optional[DataFrame]: + ... def reset_index( self, @@ -2558,9 +2460,9 @@ def sort_index( *, ascending: bool = ..., inplace: Literal[False] = ..., - kind: str | None = ..., na_position: Literal["first", "last"] = ..., - ) -> DataFrame: ... + ) -> DataFrame: + ... @overload def sort_index( @@ -2568,9 +2470,9 @@ def sort_index( *, ascending: bool = ..., inplace: Literal[True] = ..., - kind: str | None = ..., na_position: Literal["first", "last"] = ..., - ) -> None: ... + ) -> None: + ... def sort_index( self, @@ -2578,7 +2480,6 @@ def sort_index( axis: Union[int, str] = 0, ascending: bool = True, inplace: bool = False, - kind: str | None = None, na_position: Literal["first", "last"] = "last", ) -> Optional[DataFrame]: if utils.get_axis_number(axis) == 0: @@ -2592,15 +2493,10 @@ def sort_index( else order.descending_over(column, na_last) for column in index_columns ] - is_stable = ( - kind or constants.DEFAULT_SORT_KIND - ) in constants.STABLE_SORT_KINDS - block = self._block.order_by(ordering, stable=is_stable) + block = self._block.order_by(ordering) else: # axis=1 _, indexer = self.columns.sort_values( - return_indexer=True, - ascending=ascending, - na_position=na_position, # type: ignore + return_indexer=True, ascending=ascending, na_position=na_position # type: ignore ) block = self._block.select_columns( [self._block.value_columns[i] for i in indexer] @@ -2618,9 +2514,10 @@ def sort_values( *, inplace: Literal[False] = ..., ascending: bool | typing.Sequence[bool] = ..., - kind: str | None = ..., + kind: str = ..., na_position: typing.Literal["first", "last"] = ..., - ) -> DataFrame: ... + ) -> DataFrame: + ... @overload def sort_values( @@ -2629,9 +2526,10 @@ def sort_values( *, inplace: Literal[True] = ..., ascending: bool | typing.Sequence[bool] = ..., - kind: str | None = ..., + kind: str = ..., na_position: typing.Literal["first", "last"] = ..., - ) -> None: ... + ) -> None: + ... def sort_values( self, @@ -2639,7 +2537,7 @@ def sort_values( *, inplace: bool = False, ascending: bool | typing.Sequence[bool] = True, - kind: str | None = None, + kind: str = "quicksort", na_position: typing.Literal["first", "last"] = "last", ) -> Optional[DataFrame]: if isinstance(by, (bigframes.series.Series, indexes.Index, DataFrame)): @@ -2671,8 +2569,7 @@ def sort_values( if is_ascending else order.descending_over(column_id, na_last) ) - is_stable = (kind or constants.DEFAULT_SORT_KIND) in constants.STABLE_SORT_KINDS - block = self._block.order_by(ordering, stable=is_stable) + block = self._block.order_by(ordering) if inplace: self._set_block(block) return None @@ -2724,9 +2621,9 @@ def take( if not utils.is_list_like(indices): raise ValueError("indices should be a list-like object.") if axis == 0 or axis == "index": - return typing.cast(DataFrame, self.iloc[indices]) + return self.iloc[indices] elif axis == 1 or axis == "columns": - return typing.cast(DataFrame, self.iloc[:, indices]) + return self.iloc[:, indices] else: raise ValueError(f"No axis named {axis} for object type DataFrame") @@ -2915,11 +2812,11 @@ def replace( ): if utils.is_dict_like(value): return self.apply( - lambda x: ( - x.replace(to_replace=to_replace, value=value[x.name], regex=regex) - if (x.name in value) - else x + lambda x: x.replace( + to_replace=to_replace, value=value[x.name], regex=regex ) + if (x.name in value) + else x ) return self.apply( lambda x: x.replace(to_replace=to_replace, value=value, regex=regex) @@ -2989,7 +2886,7 @@ def _apply_callable(self, condition): """Executes the possible callable condition as needed.""" if callable(condition): # When it's a bigframes function. - if isinstance(condition, bigframes.functions.Udf): + if hasattr(condition, "bigframes_bigquery_function"): return self.apply(condition, axis=1) # When it's a plain Python function. @@ -3211,9 +3108,7 @@ def quantile( frame = self._drop_non_numeric() multi_q = utils.is_list_like(q) result = block_ops.quantile( - frame._block, - frame._block.value_columns, - qs=tuple(q) if multi_q else (q,), # type: ignore + frame._block, frame._block.value_columns, qs=tuple(q) if multi_q else (q,) # type: ignore ) if multi_q: return DataFrame(result.stack()).droplevel(0) @@ -3277,6 +3172,7 @@ def prod( return bigframes.series.Series(block) product = prod + product.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.prod) def count(self, *, numeric_only: bool = False) -> bigframes.series.Series: if not numeric_only: @@ -3355,6 +3251,7 @@ def agg(self, func) -> DataFrame | bigframes.series.Series: ) aggregate = agg + aggregate.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.agg) @validations.requires_index @validations.requires_ordering() @@ -3427,6 +3324,7 @@ def kurt(self, *, numeric_only: bool = False): return bigframes.series.Series(result_block) kurtosis = kurt + kurtosis.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.kurt) def _pivot( self, @@ -3946,22 +3844,6 @@ def expanding(self, min_periods: int = 1) -> bigframes.core.window.Window: self._block, window, self._block.value_columns ) - def pipe( - self, - func: Union[Callable[..., U], tuple[Callable[..., U], str]], - *args, - **kwargs, - ) -> U: - import bigframes_vendored.pandas.core.common as common - - return common.pipe(self, func, *args, **kwargs) - - def get(self, key, default=None): - try: - return self[key] - except (KeyError, ValueError, IndexError): - return default - def groupby( self, by: typing.Union[ @@ -4027,12 +3909,9 @@ def _groupby_series( col_ids: typing.Sequence[str] = [] for key in by: if isinstance(key, bigframes.series.Series): - ( - block, - ( - get_column_left, - get_column_right, - ), + block, ( + get_column_left, + get_column_right, ) = block.join(key._block, how="inner" if dropna else "left") col_ids = [ *[get_column_left[value] for value in col_ids], @@ -4073,14 +3952,12 @@ def round(self, decimals: Union[int, dict[Hashable, int]] = 0) -> DataFrame: bigframes.dtypes.BOOL_DTYPE }: if is_mapping: - decimals_dict = typing.cast(dict[typing.Hashable, int], decimals) - if label in decimals_dict: + if label in decimals: # type: ignore exprs.append( ops.round_op.as_expr( col_id, ex.const( - decimals_dict[label], - dtype=bigframes.dtypes.INT_DTYPE, # type: ignore + decimals[label], dtype=bigframes.dtypes.INT_DTYPE # type: ignore ), ) ) @@ -4107,11 +3984,13 @@ def isna(self) -> DataFrame: return self._apply_unary_op(ops.isnull_op) isnull = isna + isnull.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.isna) def notna(self) -> DataFrame: return self._apply_unary_op(ops.notnull_op) notnull = notna + notnull.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.notna) @validations.requires_ordering() def cumsum(self): @@ -4514,6 +4393,8 @@ def __array__(self, dtype=None, copy: Optional[bool] = None) -> numpy.ndarray: raise ValueError("Cannot convert to array without copy.") return self.to_numpy(dtype=dtype) + __array__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__array__) + def to_parquet( self, path=None, @@ -4567,9 +4448,7 @@ def to_dict( allow_large_results: Optional[bool] = None, **kwargs, ) -> dict | list[dict]: - return self.to_pandas(allow_large_results=allow_large_results).to_dict( - orient=orient, into=into, **kwargs - ) # type: ignore + return self.to_pandas(allow_large_results=allow_large_results).to_dict(orient=orient, into=into, **kwargs) # type: ignore def to_excel( self, @@ -4580,7 +4459,7 @@ def to_excel( **kwargs, ) -> None: return self.to_pandas(allow_large_results=allow_large_results).to_excel( - excel_writer, sheet_name=sheet_name, **kwargs + excel_writer, sheet_name, **kwargs ) def to_latex( @@ -4594,11 +4473,7 @@ def to_latex( **kwargs, ) -> str | None: return self.to_pandas(allow_large_results=allow_large_results).to_latex( - buf, - columns=typing.cast(typing.Optional[list[str]], columns), - header=typing.cast(typing.Union[bool, list[str]], header), - index=index, - **kwargs, # type: ignore + buf, columns=columns, header=header, index=index, **kwargs # type: ignore ) def to_records( @@ -4722,9 +4597,7 @@ def to_markdown( allow_large_results: Optional[bool] = None, **kwargs, ) -> str | None: - return self.to_pandas(allow_large_results=allow_large_results).to_markdown( - buf, mode=mode, index=index, **kwargs - ) # type: ignore + return self.to_pandas(allow_large_results=allow_large_results).to_markdown(buf, mode=mode, index=index, **kwargs) # type: ignore def to_pickle(self, path, *, allow_large_results=None, **kwargs) -> None: return self.to_pandas(allow_large_results=allow_large_results).to_pickle( @@ -4823,24 +4696,18 @@ def _prepare_export( return array_value, id_overrides def map(self, func, na_action: Optional[str] = None) -> DataFrame: - from bigframes._config import options - - if not isinstance(func, bigframes.functions.Udf) and not ( - options.experiments.enable_python_transpiler and callable(func) - ): + if not isinstance(func, bigframes.functions.BigqueryCallableRoutine): raise TypeError("the first argument must be callable") if na_action not in {None, "ignore"}: raise ValueError(f"na_action={na_action} not supported") - expr = ops.func_to_expr(func).apply(ex.free_var("input")) - if na_action == "ignore": - # True case, predicate, False case - expr = ops.where_op.as_expr( - expr, ops.notnull_op.as_expr(ex.free_var("input")), ex.const(None) + # TODO(shobs): Support **kwargs + return self._apply_unary_op( + ops.RemoteFunctionOp( + function_def=func.udf_def, apply_on_null=(na_action is None) ) - - return DataFrame(self._block.multi_apply_unary_op(expr)) + ) def apply(self, func, *, axis=0, args: typing.Tuple = (), **kwargs): # In Bigframes BigQuery function, DataFrame '.apply' method is specifically @@ -4853,26 +4720,18 @@ def apply(self, func, *, axis=0, args: typing.Tuple = (), **kwargs): ) warnings.warn(msg, category=bfe.FunctionAxisOnePreviewWarning) - from bigframes._config import options - - if not isinstance(func, bigframes.functions.Udf) and not ( - options.experiments.enable_python_transpiler and callable(func) + if not isinstance( + func, + ( + bigframes.functions.BigqueryCallableRoutine, + bigframes.functions.BigqueryCallableRowRoutine, + ), ): raise ValueError( "For axis=1 a BigFrames BigQuery function must be used." ) - if ( - not isinstance(func, bigframes.functions.Udf) - and options.experiments.enable_python_transpiler - and callable(func) - ): - result_block = block_ops.apply_to_block_rows( - func, self._block, *args, **kwargs - ) - return bigframes.series.Series(result_block) - - if func.udf_def.signature.is_row_processor: + if func.is_row_processor: # Early check whether the dataframe dtypes are currently supported # in the bigquery function # NOTE: Keep in sync with the value converters used in the gcf code @@ -4925,17 +4784,17 @@ def apply(self, func, *, axis=0, args: typing.Tuple = (), **kwargs): ) # Apply the function - expr = ops.func_to_expr(func).expr - if not ( - isinstance(expr, ex.OpExpression) - and isinstance(expr.op, ops.NaryOp) - ): - raise TypeError(f"Expected OpExpression with NaryOp, got {expr}") - result_series = rows_as_json_series._apply_nary_op( - expr.op, - list(args), - ) - + if args: + result_series = rows_as_json_series._apply_nary_op( + ops.NaryRemoteFunctionOp(function_def=func.udf_def), + list(args), + ) + else: + result_series = rows_as_json_series._apply_unary_op( + ops.RemoteFunctionOp( + function_def=func.udf_def, apply_on_null=True + ) + ) else: # This is a special case where we are providing not-pandas-like # extension. If the bigquery function can take one or more @@ -4950,9 +4809,7 @@ def apply(self, func, *, axis=0, args: typing.Tuple = (), **kwargs): # compatible with the data types of the input params. # 3. The order of the columns in the dataframe must correspond # to the order of the input params in the function. - udf_input_dtypes = tuple( - arg.bf_type for arg in func.udf_def.signature.inputs - ) + udf_input_dtypes = func.udf_def.signature.bf_input_types if not args and len(udf_input_dtypes) != len(self.columns): raise ValueError( f"Parameter count mismatch: BigFrames BigQuery function" @@ -4992,16 +4849,17 @@ def apply(self, func, *, axis=0, args: typing.Tuple = (), **kwargs): series_list = [self[col] for col in self.columns] op_list = series_list[1:] + list(args) - result_series = series_list[0]._apply_callable_expr( - ops.func_to_expr(func), op_list + result_series = series_list[0]._apply_nary_op( + ops.NaryRemoteFunctionOp(function_def=func.udf_def), op_list ) result_series.name = None + result_series = func._post_process_series(result_series) return result_series # At this point column-wise or element-wise bigquery function operation will # be performed (not supported). - if isinstance(func, bigframes.functions.Udf): + if hasattr(func, "bigframes_bigquery_function"): raise formatter.create_exception_with_feedback_link( NotImplementedError, "BigFrames DataFrame '.apply()' does not support BigFrames " @@ -5074,6 +4932,7 @@ def first_valid_index(self): return applymap = map + applymap.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.map) def _slice( self, @@ -5275,6 +5134,8 @@ def scatter( def __matmul__(self, other) -> DataFrame: return self.dot(other) + __matmul__.__doc__ = inspect.getdoc(vendored_pandas_frame.DataFrame.__matmul__) + @property def struct(self): return bigframes.operations.structs.StructFrameAccessor(self) @@ -5284,3 +5145,20 @@ def _throw_if_null_index(self, opname: str): raise bigframes.exceptions.NullIndexError( f"DataFrame cannot perform {opname} as it has no index. Set an index using set_index." ) + + @property + def semantics(self): + msg = bfe.format_message( + "The 'semantics' property will be removed. Please use 'bigframes.bigquery.ai' instead." + ) + warnings.warn(msg, category=FutureWarning) + return bigframes.operations.semantics.Semantics(self) + + @property + def ai(self): + """Returns the accessor for AI operators.""" + msg = bfe.format_message( + "The 'ai' property will be removed. Please use 'bigframes.bigquery.ai' instead." + ) + warnings.warn(msg, category=FutureWarning) + return bigframes.operations.ai.AIAccessor(self) diff --git a/bigframes/display/anywidget.py b/bigframes/display/anywidget.py index 01135d6670b..be0d2b45d09 100644 --- a/bigframes/display/anywidget.py +++ b/bigframes/display/anywidget.py @@ -17,24 +17,20 @@ from __future__ import annotations import dataclasses +from importlib import resources import functools -import logging - -logger = logging.getLogger(__name__) import math import threading -import uuid -import warnings -from importlib import resources from typing import Any, Iterator, Optional +import uuid import pandas as pd import bigframes +from bigframes.core import blocks import bigframes.dataframe import bigframes.display.html import bigframes.dtypes as dtypes -from bigframes.core import blocks # anywidget and traitlets are optional dependencies. We don't want the import of # this module to fail if they aren't installed, though. Instead, we try to @@ -61,19 +57,6 @@ class _SortState: ascending: tuple[bool, ...] -@dataclasses.dataclass -class _ExecutionResult: - df_to_set: Optional[bigframes.dataframe.DataFrame] = None - orderable_cols: Optional[list[str]] = None - batches: Optional[blocks.PandasBatches] = None - batch_iter: Optional[Iterator[pd.DataFrame]] = None - cached_batches: Optional[list[pd.DataFrame]] = None - all_data_loaded: bool = False - total_rows: Optional[int] = None - initial_html: Optional[str] = None - error_message: Optional[str] = None - - class TableWidget(_WIDGET_BASE): """An interactive, paginated table widget for BigFrames DataFrames. @@ -93,19 +76,8 @@ class TableWidget(_WIDGET_BASE): _error_message = traitlets.Unicode(allow_none=True, default_value=None).tag( sync=True ) - start_execution = traitlets.Bool(False).tag(sync=True) - is_deferred_mode = traitlets.Bool(False).tag(sync=True) - dry_run_info = traitlets.Unicode("").tag(sync=True) - ping = traitlets.Int(0).tag(sync=True) - - def __init__( - self, - dataframe: ( - bigframes.dataframe.DataFrame - | bigframes.session.deferred.DeferredBigQueryDataFrame - ), - dry_run_info: Optional[str] = None, - ): + + def __init__(self, dataframe: bigframes.dataframe.DataFrame): """Initialize the TableWidget. Args: @@ -117,52 +89,16 @@ def __init__( "`pip install 'bigframes[anywidget]'` to use TableWidget." ) - # Enable third-party widgets manager in Google Colab environment. - try: - import sys - - if "google.colab" in sys.modules: - from google.colab import output - - output.enable_custom_widget_manager() - except Exception: - pass - - from bigframes.session import deferred - - is_deferred = False - deferred_df = None - df = None - - if isinstance(dataframe, deferred.DeferredBigQueryDataFrame): - is_deferred = True - deferred_df = dataframe - elif bigframes.options.display.repr_mode == "deferred": - is_deferred = True - df = dataframe - else: - df = dataframe - - from bigframes.core.utils import get_ipython_execution_count - - self._cell_execution_count = get_ipython_execution_count() + self._dataframe = dataframe super().__init__() - self.is_deferred_mode = is_deferred - self._deferred_dataframe = deferred_df - self._dataframe = df - - if dry_run_info: - self.dry_run_info = dry_run_info - # Initialize attributes that might be needed by observers first self._table_id = str(uuid.uuid4()) self._all_data_loaded = False self._batch_iter: Optional[Iterator[pd.DataFrame]] = None self._cached_batches: list[pd.DataFrame] = [] self._last_sort_state: Optional[_SortState] = None - self._execution_result: Optional[_ExecutionResult] = None # Lock to ensure only one thread at a time is updating the table HTML. self._setting_html_lock = threading.Lock() @@ -170,209 +106,52 @@ def __init__( initial_page_size = bigframes.options.display.max_rows initial_max_columns = bigframes.options.display.max_columns + # set traitlets properties that trigger observers + # TODO(b/462525985): Investigate and improve TableWidget UX for DataFrames with a large number of columns. self.page_size = initial_page_size self.max_columns = initial_max_columns - if not self.is_deferred_mode: - self._initialize_from_dataframe() - - # Signals to the frontend that the initial data load is complete. - # Also used as a guard to prevent observers from firing during initialization. - self._initial_load_complete = True - - @traitlets.observe("start_execution") - def _on_start_execution(self, change: dict[str, Any]): - if change["new"]: - import asyncio - - try: - loop = asyncio.get_running_loop() - except RuntimeError: - try: - import tornado.ioloop # type: ignore[import-not-found] - - loop = tornado.ioloop.IOLoop.current().asyncio_loop # type: ignore[attr-defined] - except Exception: - loop = None - - def run_execution(): - try: - self._error_message = None - df = None - if self.is_deferred_mode: - if self._deferred_dataframe is not None: - result = self._deferred_dataframe.execute() - if isinstance(result, bigframes.series.Series): - df = result.to_frame() - elif isinstance(result, bigframes.dataframe.DataFrame): - df = result - else: - raise TypeError( - f"Unexpected result type: {type(result)}" - ) - elif self._dataframe is not None: - df = self._dataframe - else: - df = self._dataframe - - if df is None: - raise ValueError("No DataFrame to execute.") - - df_to_set = df._prepare_display_df() - orderable_cols = self._get_orderable_columns(df_to_set) - - with bigframes.option_context("display.progress_bar", None): - batches = df_to_set.to_pandas_batches( - page_size=self.page_size, - cell_execution_count=self._cell_execution_count, - ) - - total_rows = getattr(batches, "total_rows", None) - - # Fetch the first batch - batch_iter = iter(batches) - try: - initial_batch = next(batch_iter) - cached_batches = [initial_batch] - all_data_loaded = False - except StopIteration: - initial_batch = pd.DataFrame(columns=df_to_set.columns) - cached_batches = [] - all_data_loaded = True - - # Render the HTML - page_data = initial_batch.copy() - start = 0 - if df_to_set._block.has_index: - is_unnamed_single_index = ( - page_data.index.name is None - and not isinstance(page_data.index, pd.MultiIndex) - ) - page_data = page_data.reset_index() - if is_unnamed_single_index and "index" in page_data.columns: - page_data.rename(columns={"index": ""}, inplace=True) - else: - page_data.insert( - 0, "Row", range(start + 1, start + len(page_data) + 1) - ) - - initial_html = bigframes.display.html.render_html( - dataframe=page_data, - table_id=f"table-{self._table_id}", - orderable_columns=orderable_cols, - max_columns=self.max_columns, - ) - - self._execution_result = _ExecutionResult( - df_to_set=df_to_set, - orderable_cols=orderable_cols, - batches=batches, - batch_iter=batch_iter, - cached_batches=cached_batches, - all_data_loaded=all_data_loaded, - total_rows=total_rows, - initial_html=initial_html, - ) - except Exception as e: - logger.warning(f"Error in background execution: {e}") - self._execution_result = _ExecutionResult(error_message=str(e)) - - import sys - - is_colab = "google.colab" in sys.modules - - if loop is not None and loop.is_running() and not is_colab: - loop.call_soon_threadsafe(self._apply_execution_result) - elif is_colab: - # In Google Colab, background thread updates to traitlets are not automatically - # synchronized to the frontend. We rely on the frontend's active pinging - # (which triggers `_on_ping` on the main kernel thread) to apply the result. - pass - else: - self._apply_execution_result() - - self._execution_thread = threading.Thread(target=run_execution, daemon=True) - self._execution_thread.start() - - def _apply_execution_result(self) -> None: - if self._execution_result is None: - return - - result = self._execution_result - self._execution_result = None - - with self.hold_sync(): - if result.error_message is not None: - self._error_message = result.error_message - self.start_execution = False - else: - self._dataframe = result.df_to_set - self.orderable_columns = result.orderable_cols or [] - self._batches = result.batches - self._batch_iter = result.batch_iter - self._cached_batches = result.cached_batches or [] - self._all_data_loaded = result.all_data_loaded - self._last_sort_state = _SortState((), ()) - self.row_count = result.total_rows - self.table_html = result.initial_html or "" - self.is_deferred_mode = False - self.start_execution = False - - @traitlets.observe("ping") - def _on_ping(self, _change: dict[str, Any]): - self._apply_execution_result() - - def _initialize_from_dataframe(self): - if self._dataframe is None: - return - - self.orderable_columns = self._get_orderable_columns(self._dataframe) - - self._initial_load() - - def _get_orderable_columns( - self, dataframe: bigframes.dataframe.DataFrame - ) -> list[str]: - """Determine which columns can be used for client-side sorting.""" # TODO(b/469861913): Nested columns from structs (e.g., 'struct_col.name') are not currently sortable. # TODO(b/463754889): Support non-string column labels for sorting. - if not all(isinstance(col, str) for col in dataframe.columns): - return [] - - with warnings.catch_warnings(): - warnings.simplefilter("ignore", bigframes.exceptions.JSONDtypeWarning) - warnings.simplefilter("ignore", category=FutureWarning) - return [ + if all(isinstance(col, str) for col in dataframe.columns): + self.orderable_columns = [ str(col_name) for col_name, dtype in dataframe.dtypes.items() if dtypes.is_orderable(dtype) ] + else: + self.orderable_columns = [] + + self._initial_load() + + # Signals to the frontend that the initial data load is complete. + # Also used as a guard to prevent observers from firing during initialization. + self._initial_load_complete = True def _initial_load(self) -> None: """Get initial data and row count.""" # obtain the row counts # TODO(b/428238610): Start iterating over the result of `to_pandas_batches()` # before we get here so that the count might already be cached. - with bigframes.option_context("display.progress_bar", None): - self._reset_batches_for_new_page_size() + self._reset_batches_for_new_page_size() - if self._batches is None: - self._error_message = ( - "Could not retrieve data batches. Data might be unavailable or " - "an error occurred." - ) - self.row_count = None - elif self._batches.total_rows is None: - # Total rows is unknown, this is an expected state. - # TODO(b/461536343): Cheaply discover if we have exactly 1 page. - # There are cases where total rows is not set, but there are no additional - # pages. We could disable the "next" button in these cases. - self.row_count = None - else: - self.row_count = self._batches.total_rows + if self._batches is None: + self._error_message = ( + "Could not retrieve data batches. Data might be unavailable or " + "an error occurred." + ) + self.row_count = None + elif self._batches.total_rows is None: + # Total rows is unknown, this is an expected state. + # TODO(b/461536343): Cheaply discover if we have exactly 1 page. + # There are cases where total rows is not set, but there are no additional + # pages. We could disable the "next" button in these cases. + self.row_count = None + else: + self.row_count = self._batches.total_rows - # get the initial page - self._set_table_html() + # get the initial page + self._set_table_html() @traitlets.observe("_initial_load_complete") def _on_initial_load_complete(self, change: dict[str, Any]): @@ -381,8 +160,8 @@ def _on_initial_load_complete(self, change: dict[str, Any]): @functools.cached_property def _esm(self): - """Load JavaScript code from the compiled Angular hybrid bundle.""" - return resources.read_text(bigframes.display, "table_widget_angular.js") + """Load JavaScript code from external file.""" + return resources.read_text(bigframes.display, "table_widget.js") @functools.cached_property def _css(self): @@ -484,9 +263,7 @@ def _batch_iterator(self) -> Iterator[pd.DataFrame]: def _cached_data(self) -> pd.DataFrame: """Combine all cached batches into a single DataFrame.""" if not self._cached_batches: - if self._dataframe is not None: - return pd.DataFrame(columns=self._dataframe.columns) - return pd.DataFrame() + return pd.DataFrame(columns=self._dataframe.columns) return pd.concat(self._cached_batches) def _reset_batch_cache(self) -> None: @@ -497,36 +274,21 @@ def _reset_batch_cache(self) -> None: def _reset_batches_for_new_page_size(self) -> None: """Reset the batch iterator when page size changes.""" - if self._dataframe is None: - return - with bigframes.option_context("display.progress_bar", None): - self._batches = self._dataframe.to_pandas_batches( - page_size=self.page_size, - cell_execution_count=self._cell_execution_count, - ) + self._batches = self._dataframe.to_pandas_batches(page_size=self.page_size) self._reset_batch_cache() def _set_table_html(self) -> None: """Sets the current html data based on the current page and page size.""" - if self.is_deferred_mode: - return - new_page = None - with ( - self._setting_html_lock, - bigframes.option_context("display.progress_bar", None), - ): + with self._setting_html_lock: if self._error_message: self.table_html = ( - f"
{self._error_message}
" + f"
" + f"{self._error_message}
" ) return - if self._dataframe is None: - self.table_html = "
Internal Error: DataFrame is missing.
" - return - # Apply sorting if a column is selected df_to_display = self._dataframe sort_columns = [item["column"] for item in self.sort_context] @@ -542,8 +304,7 @@ def _set_table_html(self) -> None: current_sort_state = _SortState(tuple(sort_columns), tuple(sort_ascending)) if self._last_sort_state != current_sort_state: self._batches = df_to_display.to_pandas_batches( - page_size=self.page_size, - cell_execution_count=self._cell_execution_count, + page_size=self.page_size ) self._reset_batch_cache() self._last_sort_state = current_sort_state diff --git a/bigframes/display/html.py b/bigframes/display/html.py index 603d53e6866..6102d1512c6 100644 --- a/bigframes/display/html.py +++ b/bigframes/display/html.py @@ -20,16 +20,16 @@ import json import traceback import typing -import warnings from typing import Any, Union +import warnings import pandas as pd import pandas.api.types import bigframes -import bigframes.formatting_helpers as formatter from bigframes._config import display_options, options from bigframes.display import plaintext +import bigframes.formatting_helpers as formatter if typing.TYPE_CHECKING: import bigframes.dataframe @@ -181,7 +181,7 @@ def _obj_ref_rt_to_html(obj_ref_rt: str) -> str: url = obj_ref_rt_json["access_urls"]["read_url"] return f'' - return f"uri: {obj_ref_rt_json['objectref']['uri']}, authorizer: {obj_ref_rt_json['objectref']['authorizer']}" + return f'uri: {obj_ref_rt_json["objectref"]["uri"]}, authorizer: {obj_ref_rt_json["objectref"]["authorizer"]}' def create_html_representation( @@ -189,15 +189,20 @@ def create_html_representation( pandas_df: pd.DataFrame, total_rows: int, total_columns: int, + blob_cols: list[str], ) -> str: """Create an HTML representation of the DataFrame or Series.""" - import bigframes.series + from bigframes.series import Series opts = options.display with display_options.pandas_repr(opts): - if isinstance(obj, bigframes.series.Series): + if isinstance(obj, Series): + # Some pandas objects may not have a _repr_html_ method, or it might + # fail in certain environments. We fall back to a pre-formatted + # string representation to ensure something is always displayed. pd_series = pandas_df.iloc[:, 0] try: + # TODO(b/464053870): Support rich display for blob Series. html_string = pd_series._repr_html_() except AttributeError: html_string = f"
{pd_series.to_string()}
" @@ -207,8 +212,26 @@ def create_html_representation( html_string += f"

[{total_rows} rows]

" return html_string else: - # _repr_html_ stub is missing so mypy thinks it's a Series. Ignore mypy. - html_string = pandas_df._repr_html_() # type:ignore + # It's a DataFrame + # TODO(shuowei, b/464053870): Escaping HTML would be useful, but + # `escape=False` is needed to show images. We may need to implement + # a full-fledged repr module to better support types not in pandas. + if options.display.blob_display and blob_cols: + formatters = {blob_col: _obj_ref_rt_to_html for blob_col in blob_cols} + + # set max_colwidth so not to truncate the image url + with pandas.option_context("display.max_colwidth", None): + html_string = pandas_df.to_html( + escape=False, + notebook=True, + max_rows=pandas.get_option("display.max_rows"), + max_cols=pandas.get_option("display.max_columns"), + show_dimensions=pandas.get_option("display.show_dimensions"), + formatters=formatters, # type: ignore + ) + else: + # _repr_html_ stub is missing so mypy thinks it's a Series. Ignore mypy. + html_string = pandas_df._repr_html_() # type:ignore html_string += f"[{total_rows} rows x {total_columns} columns in total]" return html_string @@ -217,9 +240,9 @@ def create_html_representation( def _get_obj_metadata( obj: Union[bigframes.dataframe.DataFrame, bigframes.series.Series], ) -> tuple[bool, bool]: - import bigframes.series + from bigframes.series import Series - is_series = isinstance(obj, bigframes.series.Series) + is_series = isinstance(obj, Series) if is_series: has_index = len(obj._block.index_columns) > 0 else: @@ -231,31 +254,20 @@ def get_anywidget_bundle( obj: Union[bigframes.dataframe.DataFrame, bigframes.series.Series], include=None, exclude=None, - dry_run_info: str | None = None, ) -> tuple[dict[str, Any], dict[str, Any]]: """ Helper method to create and return the anywidget mimebundle. This function encapsulates the logic for anywidget display. """ - import bigframes.series from bigframes import display + from bigframes.series import Series - if isinstance(obj, bigframes.series.Series): + if isinstance(obj, Series): df = obj.to_frame() else: - df = obj - - from bigframes.session import deferred - - if ( - not isinstance(df, deferred.DeferredBigQueryDataFrame) - and bigframes.options.display.repr_mode != "deferred" - ): - display_df = df._prepare_display_df() - else: - display_df = df + df, blob_cols = obj._get_display_df_and_blob_cols() - widget = display.TableWidget(display_df, dry_run_info=dry_run_info) + widget = display.TableWidget(df) widget_repr_result = widget._repr_mimebundle_(include=include, exclude=exclude) if isinstance(widget_repr_result, tuple): @@ -271,23 +283,21 @@ def get_anywidget_bundle( total_rows = widget.row_count total_columns = len(df.columns) - if dry_run_info: - widget_repr["text/plain"] = dry_run_info - else: - widget_repr["text/html"] = create_html_representation( - obj, - cached_pd, - total_rows, - total_columns, - ) - is_series, has_index = _get_obj_metadata(obj) - widget_repr["text/plain"] = plaintext.create_text_representation( - cached_pd, - total_rows, - is_series=is_series, - has_index=has_index, - column_count=len(df.columns) if not is_series else 0, - ) + widget_repr["text/html"] = create_html_representation( + obj, + cached_pd, + total_rows, + total_columns, + blob_cols if "blob_cols" in locals() else [], + ) + is_series, has_index = _get_obj_metadata(obj) + widget_repr["text/plain"] = plaintext.create_text_representation( + cached_pd, + total_rows, + is_series=is_series, + has_index=has_index, + column_count=len(df.columns) if not is_series else 0, + ) return widget_repr, widget_metadata @@ -304,23 +314,27 @@ def repr_mimebundle_deferred( def repr_mimebundle_head( obj: Union[bigframes.dataframe.DataFrame, bigframes.series.Series], ) -> dict[str, str]: - import bigframes.series + from bigframes.series import Series opts = options.display - if isinstance(obj, bigframes.series.Series): - df = obj.to_frame() + blob_cols: list[str] + if isinstance(obj, Series): + pandas_df, row_count, query_job = obj._block.retrieve_repr_request_results( + opts.max_rows + ) + blob_cols = [] else: - df = obj - - df = df._prepare_display_df() - pandas_df, row_count, query_job = df._block.retrieve_repr_request_results( - opts.max_rows - ) + df, blob_cols = obj._get_display_df_and_blob_cols() + pandas_df, row_count, query_job = df._block.retrieve_repr_request_results( + opts.max_rows + ) obj._set_internal_query_job(query_job) column_count = len(pandas_df.columns) - html_string = create_html_representation(obj, pandas_df, row_count, column_count) + html_string = create_html_representation( + obj, pandas_df, row_count, column_count, blob_cols + ) is_series, has_index = _get_obj_metadata(obj) text_representation = plaintext.create_text_representation( @@ -344,43 +358,20 @@ def repr_mimebundle( # BQ Studio, but there is a known compatibility issue with Marimo that needs to be addressed. opts = options.display - if ( - opts.render_mode == "anywidget" - or opts.repr_mode == "anywidget" - or opts.repr_mode == "deferred" - ): + if opts.repr_mode == "deferred": + return repr_mimebundle_deferred(obj) + + if opts.repr_mode == "anywidget": try: - with bigframes.option_context("display.progress_bar", None): - with warnings.catch_warnings(): - warnings.simplefilter( - "ignore", category=bigframes.exceptions.JSONDtypeWarning - ) - warnings.simplefilter("ignore", category=FutureWarning) - dry_run_info = None - if opts.repr_mode == "deferred": - dry_run_job = obj._compute_dry_run() - dry_run_info = formatter.repr_query_job(dry_run_job) - return get_anywidget_bundle( - obj, - include=include, - exclude=exclude, - dry_run_info=dry_run_info, - ) - except Exception: + return get_anywidget_bundle(obj, include=include, exclude=exclude) + except ImportError: # Anywidget is an optional dependency, so warn rather than fail. # TODO(shuowei): When Anywidget becomes the default for all repr modes, # remove this warning. warnings.warn( - "Anywidget mode is not available or failed to load. " - "Please `pip install anywidget traitlets` or " - "`pip install 'bigframes[anywidget]'` to use interactive tables. " + "Anywidget mode is not available. " + "Please `pip install anywidget traitlets` or `pip install 'bigframes[anywidget]'` to use interactive tables. " f"Falling back to static HTML. Error: {traceback.format_exc()}" ) - if opts.repr_mode == "deferred": - return repr_mimebundle_deferred(obj) - - bundle = repr_mimebundle_head(obj) - if opts.render_mode == "plaintext": - bundle.pop("text/html", None) - return bundle + return repr_mimebundle_head(obj) diff --git a/bigframes/display/table_widget_angular.js b/bigframes/display/table_widget_angular.js deleted file mode 100644 index ad1697def54..00000000000 --- a/bigframes/display/table_widget_angular.js +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var Ba=Object.defineProperty,qa=Object.defineProperties,Ua=Object.getOwnPropertyDescriptors,Hi=Object.getOwnPropertySymbols,Za=Object.prototype.hasOwnProperty,$a=Object.prototype.propertyIsEnumerable,zi=(e,t,n)=>t in e?Ba(e,t,{enumerable:!0,configurable:!0,writable:!0,value:n}):e[t]=n,$=(e,t)=>{for(var n in t||={})Za.call(t,n)&&zi(e,n,t[n]);if(Hi)for(var n of Hi(t))$a.call(t,n)&&zi(e,n,t[n]);return e},Q=(e,t)=>qa(e,Ua(t)),R=null,Ft=!1,Wr=1,Qa=null,ne=Symbol("SIGNAL");function m(e){let t=R;return R=e,t}function Wa(){return R}var St={version:0,lastCleanEpoch:0,dirty:!1,producers:void 0,producersTail:void 0,consumers:void 0,consumersTail:void 0,recomputing:!1,consumerAllowSignalWrites:!1,consumerIsAlwaysLive:!1,kind:"unknown",producerMustRecompute:()=>!1,producerRecomputeValue:()=>{},consumerMarkedDirty:()=>{},consumerOnSignalRead:()=>{}};function Wo(e){if(Ft)throw new Error("");if(R===null)return;R.consumerOnSignalRead(e);let t=R.producersTail;if(t!==void 0&&t.producer===e)return;let n,r=R.recomputing;if(r&&(n=t!==void 0?t.nextProducer:R.producers,n!==void 0&&n.producer===e)){R.producersTail=n,n.lastReadVersion=e.version;return}let i=e.consumersTail;if(i!==void 0&&i.consumer===R&&(!r||Ja(i,R)))return;let o=tt(R),s={producer:e,consumer:R,nextProducer:n,prevConsumer:i,lastReadVersion:e.version,nextConsumer:void 0};R.producersTail=s,t!==void 0?t.nextProducer=s:R.producers=s,o&&Xo(e,s)}function Ga(){Wr++}function Go(e){if(!(tt(e)&&!e.dirty)&&!(!e.dirty&&e.lastCleanEpoch===Wr)){if(!e.producerMustRecompute(e)&&!Yr(e)){Bi(e);return}e.producerRecomputeValue(e),Bi(e)}}function Yo(e){if(e.consumers===void 0)return;let t=Ft;Ft=!0;try{for(let n=e.consumers;n!==void 0;n=n.nextConsumer){let r=n.consumer;r.dirty||Ya(r)}}finally{Ft=t}}function Ko(){return R?.consumerAllowSignalWrites!==!1}function Ya(e){e.dirty=!0,Yo(e),e.consumerMarkedDirty?.(e)}function Bi(e){e.dirty=!1,e.lastCleanEpoch=Wr}function Gt(e){return e&&Ka(e),m(e)}function Ka(e){e.producersTail=void 0,e.recomputing=!0}function Gr(e,t){m(t),e&&Xa(e)}function Xa(e){e.recomputing=!1;let t=e.producersTail,n=t!==void 0?t.nextProducer:e.producers;if(n!==void 0){if(tt(e))do n=Kr(n);while(n!==void 0);t!==void 0?t.nextProducer=void 0:e.producers=void 0}}function Yr(e){for(let t=e.producers;t!==void 0;t=t.nextProducer){let n=t.producer,r=t.lastReadVersion;if(r!==n.version||(Go(n),r!==n.version))return!0}return!1}function yn(e){if(tt(e)){let t=e.producers;for(;t!==void 0;)t=Kr(t)}e.producers=void 0,e.producersTail=void 0,e.consumers=void 0,e.consumersTail=void 0}function Xo(e,t){let n=e.consumersTail,r=tt(e);if(n!==void 0?(t.nextConsumer=n.nextConsumer,n.nextConsumer=t):(t.nextConsumer=void 0,e.consumers=t),t.prevConsumer=n,e.consumersTail=t,!r)for(let i=e.producers;i!==void 0;i=i.nextProducer)Xo(i.producer,i)}function Kr(e){let t=e.producer,n=e.nextProducer,r=e.nextConsumer,i=e.prevConsumer;if(e.nextConsumer=void 0,e.prevConsumer=void 0,r!==void 0?r.prevConsumer=i:t.consumersTail=i,i!==void 0)i.nextConsumer=r;else if(t.consumers=r,!tt(t)){let o=t.producers;for(;o!==void 0;)o=Kr(o)}return n}function tt(e){return e.consumerIsAlwaysLive||e.consumers!==void 0}function Jo(e){Qa?.(e)}function Ja(e,t){let n=t.producersTail;if(n!==void 0){let r=t.producers;do{if(r===e)return!0;if(r===n)break;r=r.nextProducer}while(r!==void 0)}return!1}function es(e,t){return Object.is(e,t)}function eu(e,t){let n=Object.create(tu);n.computation=e,t!==void 0&&(n.equal=t);let r=()=>{if(Go(n),Wo(n),n.value===Ht)throw n.error;return n.value};return r[ne]=n,Jo(n),r}var Ln=Symbol("UNSET"),jn=Symbol("COMPUTING"),Ht=Symbol("ERRORED"),tu=Q($({},St),{value:Ln,dirty:!0,error:null,equal:es,kind:"computed",producerMustRecompute(e){return e.value===Ln||e.value===jn},producerRecomputeValue(e){if(e.value===jn)throw new Error("");let t=e.value;e.value=jn;let n=Gt(e),r,i=!1;try{r=e.computation(),m(null),i=t!==Ln&&t!==Ht&&r!==Ht&&e.equal(t,r)}catch(o){r=Ht,e.error=o}finally{Gr(e,n)}if(i){e.value=t;return}e.value=r,e.version++}});function nu(){throw new Error}var ts=nu;function ns(e){ts(e)}function ru(e){ts=e}var iu=null;function ou(e,t){let n=Object.create(au);n.value=e,t!==void 0&&(n.equal=t);let r=()=>su(n);return r[ne]=n,Jo(n),[r,i=>rs(n,i),i=>lu(n,i)]}function su(e){return Wo(e),e.value}function rs(e,t){Ko()||ns(e),e.equal(e.value,t)||(e.value=t,uu(e))}function lu(e,t){Ko()||ns(e),rs(e,t(e.value))}var au=Q($({},St),{equal:es,value:void 0,kind:"signal"});function uu(e){e.version++,Ga(),Yo(e),iu?.(e)}var cu=Q($({},St),{consumerIsAlwaysLive:!0,consumerAllowSignalWrites:!0,dirty:!0,kind:"effect"});function du(e){if(e.dirty=!1,e.version>0&&!Yr(e))return;e.version++;let t=Gt(e);try{e.cleanup(),e.fn()}finally{Gr(e,t)}}function Y(e){return typeof e=="function"}function is(e){let t=e(n=>{Error.call(n),n.stack=new Error().stack});return t.prototype=Object.create(Error.prototype),t.prototype.constructor=t,t}var Fn=is(e=>function(t){e(this),this.message=t?`${t.length} errors occurred during unsubscription: -${t.map((n,r)=>`${r+1}) ${n.toString()}`).join(` - `)}`:"",this.name="UnsubscriptionError",this.errors=t});function or(e,t){if(e){let n=e.indexOf(t);0<=n&&e.splice(n,1)}}var me=class sr{constructor(t){this.initialTeardown=t,this.closed=!1,this._parentage=null,this._finalizers=null}unsubscribe(){let t;if(!this.closed){this.closed=!0;let{_parentage:n}=this;if(n)if(this._parentage=null,Array.isArray(n))for(let o of n)o.remove(this);else n.remove(this);let{initialTeardown:r}=this;if(Y(r))try{r()}catch(o){t=o instanceof Fn?o.errors:[o]}let{_finalizers:i}=this;if(i){this._finalizers=null;for(let o of i)try{qi(o)}catch(s){t=t??[],s instanceof Fn?t=[...t,...s.errors]:t.push(s)}}if(t)throw new Fn(t)}}add(t){var n;if(t&&t!==this)if(this.closed)qi(t);else{if(t instanceof sr){if(t.closed||t._hasParent(this))return;t._addParent(this)}(this._finalizers=(n=this._finalizers)!==null&&n!==void 0?n:[]).push(t)}}_hasParent(t){let{_parentage:n}=this;return n===t||Array.isArray(n)&&n.includes(t)}_addParent(t){let{_parentage:n}=this;this._parentage=Array.isArray(n)?(n.push(t),n):n?[n,t]:t}_removeParent(t){let{_parentage:n}=this;n===t?this._parentage=null:Array.isArray(n)&&or(n,t)}remove(t){let{_finalizers:n}=this;n&&or(n,t),t instanceof sr&&t._removeParent(this)}};me.EMPTY=(()=>{let e=new me;return e.closed=!0,e})();var os=me.EMPTY;function ss(e){return e instanceof me||e&&"closed"in e&&Y(e.remove)&&Y(e.add)&&Y(e.unsubscribe)}function qi(e){Y(e)?e():e.unsubscribe()}var je={onUnhandledError:null,onStoppedNotification:null,Promise:void 0,useDeprecatedSynchronousErrorHandling:!1,useDeprecatedNextContext:!1},Yt={setTimeout(e,t,...n){let{delegate:r}=Yt;return r?.setTimeout?r.setTimeout(e,t,...n):setTimeout(e,t,...n)},clearTimeout(e){let{delegate:t}=Yt;return(t?.clearTimeout||clearTimeout)(e)},delegate:void 0};function fu(e){Yt.setTimeout(()=>{let{onUnhandledError:t}=je;if(t)t(e);else throw e})}function Ui(){}var hu=Xr("C",void 0,void 0);function pu(e){return Xr("E",void 0,e)}function gu(e){return Xr("N",e,void 0)}function Xr(e,t,n){return{kind:e,value:t,error:n}}var ke=null;function zt(e){if(je.useDeprecatedSynchronousErrorHandling){let t=!ke;if(t&&(ke={errorThrown:!1,error:null}),e(),t){let{errorThrown:n,error:r}=ke;if(ke=null,n)throw r}}else e()}function mu(e){je.useDeprecatedSynchronousErrorHandling&&ke&&(ke.errorThrown=!0,ke.error=e)}var Jr=class extends me{constructor(e){super(),this.isStopped=!1,e?(this.destination=e,ss(e)&&e.add(this)):this.destination=wu}static create(e,t,n){return new lr(e,t,n)}next(e){this.isStopped?zn(gu(e),this):this._next(e)}error(e){this.isStopped?zn(pu(e),this):(this.isStopped=!0,this._error(e))}complete(){this.isStopped?zn(hu,this):(this.isStopped=!0,this._complete())}unsubscribe(){this.closed||(this.isStopped=!0,super.unsubscribe(),this.destination=null)}_next(e){this.destination.next(e)}_error(e){try{this.destination.error(e)}finally{this.unsubscribe()}}_complete(){try{this.destination.complete()}finally{this.unsubscribe()}}},vu=Function.prototype.bind;function Hn(e,t){return vu.call(e,t)}var yu=class{constructor(e){this.partialObserver=e}next(e){let{partialObserver:t}=this;if(t.next)try{t.next(e)}catch(n){At(n)}}error(e){let{partialObserver:t}=this;if(t.error)try{t.error(e)}catch(n){At(n)}else At(e)}complete(){let{partialObserver:e}=this;if(e.complete)try{e.complete()}catch(t){At(t)}}},lr=class extends Jr{constructor(e,t,n){super();let r;if(Y(e)||!e)r={next:e??void 0,error:t??void 0,complete:n??void 0};else{let i;this&&je.useDeprecatedNextContext?(i=Object.create(e),i.unsubscribe=()=>this.unsubscribe(),r={next:e.next&&Hn(e.next,i),error:e.error&&Hn(e.error,i),complete:e.complete&&Hn(e.complete,i)}):r=e}this.destination=new yu(r)}};function At(e){je.useDeprecatedSynchronousErrorHandling?mu(e):fu(e)}function bu(e){throw e}function zn(e,t){let{onStoppedNotification:n}=je;n&&Yt.setTimeout(()=>n(e,t))}var wu={closed:!0,next:Ui,error:bu,complete:Ui},_u=typeof Symbol=="function"&&Symbol.observable||"@@observable";function Cu(e){return e}function xu(e){return e.length===0?Cu:e.length===1?e[0]:function(t){return e.reduce((n,r)=>r(n),t)}}var ar=(()=>{class e{constructor(n){n&&(this._subscribe=n)}lift(n){let r=new e;return r.source=this,r.operator=n,r}subscribe(n,r,i){let o=Su(n)?n:new lr(n,r,i);return zt(()=>{let{operator:s,source:l}=this;o.add(s?s.call(o,l):l?this._subscribe(o):this._trySubscribe(o))}),o}_trySubscribe(n){try{return this._subscribe(n)}catch(r){n.error(r)}}forEach(n,r){return r=Zi(r),new r((i,o)=>{let s=new lr({next:l=>{try{n(l)}catch(a){o(a),s.unsubscribe()}},error:o,complete:i});this.subscribe(s)})}_subscribe(n){var r;return(r=this.source)===null||r===void 0?void 0:r.subscribe(n)}[_u](){return this}pipe(...n){return xu(n)(this)}toPromise(n){return n=Zi(n),new n((r,i)=>{let o;this.subscribe(s=>o=s,s=>i(s),()=>r(o))})}}return e.create=t=>new e(t),e})();function Zi(e){var t;return(t=e??je.Promise)!==null&&t!==void 0?t:Promise}function ku(e){return e&&Y(e.next)&&Y(e.error)&&Y(e.complete)}function Su(e){return e&&e instanceof Jr||ku(e)&&ss(e)}function Eu(e){return Y(e?.lift)}function Iu(e){return t=>{if(Eu(t))return t.lift(function(n){try{return e(n,this)}catch(r){this.error(r)}});throw new TypeError("Unable to lift unknown Observable type")}}function Tu(e,t,n,r,i){return new Ou(e,t,n,r,i)}var Ou=class extends Jr{constructor(e,t,n,r,i,o){super(e),this.onFinalize=i,this.shouldUnsubscribe=o,this._next=t?function(s){try{t(s)}catch(l){e.error(l)}}:super._next,this._error=r?function(s){try{r(s)}catch(l){e.error(l)}finally{this.unsubscribe()}}:super._error,this._complete=n?function(){try{n()}catch(s){e.error(s)}finally{this.unsubscribe()}}:super._complete}unsubscribe(){var e;if(!this.shouldUnsubscribe||this.shouldUnsubscribe()){let{closed:t}=this;super.unsubscribe(),!t&&((e=this.onFinalize)===null||e===void 0||e.call(this))}}},Du=is(e=>function(){e(this),this.name="ObjectUnsubscribedError",this.message="object unsubscribed"}),Et=(()=>{class e extends ar{constructor(){super(),this.closed=!1,this.currentObservers=null,this.observers=[],this.isStopped=!1,this.hasError=!1,this.thrownError=null}lift(n){let r=new $i(this,this);return r.operator=n,r}_throwIfClosed(){if(this.closed)throw new Du}next(n){zt(()=>{if(this._throwIfClosed(),!this.isStopped){this.currentObservers||(this.currentObservers=Array.from(this.observers));for(let r of this.currentObservers)r.next(n)}})}error(n){zt(()=>{if(this._throwIfClosed(),!this.isStopped){this.hasError=this.isStopped=!0,this.thrownError=n;let{observers:r}=this;for(;r.length;)r.shift().error(n)}})}complete(){zt(()=>{if(this._throwIfClosed(),!this.isStopped){this.isStopped=!0;let{observers:n}=this;for(;n.length;)n.shift().complete()}})}unsubscribe(){this.isStopped=this.closed=!0,this.observers=this.currentObservers=null}get observed(){var n;return((n=this.observers)===null||n===void 0?void 0:n.length)>0}_trySubscribe(n){return this._throwIfClosed(),super._trySubscribe(n)}_subscribe(n){return this._throwIfClosed(),this._checkFinalizedStatuses(n),this._innerSubscribe(n)}_innerSubscribe(n){let{hasError:r,isStopped:i,observers:o}=this;return r||i?os:(this.currentObservers=null,o.push(n),new me(()=>{this.currentObservers=null,or(o,n)}))}_checkFinalizedStatuses(n){let{hasError:r,thrownError:i,isStopped:o}=this;r?n.error(i):o&&n.complete()}asObservable(){let n=new ar;return n.source=this,n}}return e.create=(t,n)=>new $i(t,n),e})(),$i=class extends Et{constructor(e,t){super(),this.destination=e,this.source=t}next(e){var t,n;(n=(t=this.destination)===null||t===void 0?void 0:t.next)===null||n===void 0||n.call(t,e)}error(e){var t,n;(n=(t=this.destination)===null||t===void 0?void 0:t.error)===null||n===void 0||n.call(t,e)}complete(){var e,t;(t=(e=this.destination)===null||e===void 0?void 0:e.complete)===null||t===void 0||t.call(e)}_subscribe(e){var t,n;return(n=(t=this.source)===null||t===void 0?void 0:t.subscribe(e))!==null&&n!==void 0?n:os}},Mu=class extends Et{constructor(e){super(),this._value=e}get value(){return this.getValue()}_subscribe(e){let t=super._subscribe(e);return!t.closed&&e.next(this._value),t}getValue(){let{hasError:e,thrownError:t,_value:n}=this;if(e)throw t;return this._throwIfClosed(),n}next(e){super.next(this._value=e)}};function Pu(e,t){return Iu((n,r)=>{let i=0;n.subscribe(Tu(r,o=>{r.next(e.call(t,o,i++))}))})}var ur;function ls(){return ur}function fe(e){let t=ur;return ur=e,t}var Nu=Symbol("NotFound");function ei(e){return e===Nu||e?.name==="\u0275NotFound"}var as="https://angular.dev/best-practices/security#preventing-cross-site-scripting-xss",w=class extends Error{code;constructor(e,t){super(Vu(e,t)),this.code=e}};function Au(e){return`NG0${Math.abs(e)}`}function Vu(e,t){return`${Au(e)}${t?": "+t:""}`}var Kt=globalThis;function k(e){for(let t in e)if(e[t]===k)return t;throw Error("")}function us(e){if(typeof e=="string")return e;if(Array.isArray(e))return`[${e.map(us).join(", ")}]`;if(e==null)return""+e;let t=e.overriddenName||e.name;if(t)return`${t}`;let n=e.toString();if(n==null)return""+n;let r=n.indexOf(` -`);return r>=0?n.slice(0,r):n}function Qi(e,t){return e?t?`${e} ${t}`:e:t||""}var Ru=k({__forward_ref__:k});function cs(e){return e.__forward_ref__=cs,e}function F(e){return Lu(e)?e():e}function Lu(e){return typeof e=="function"&&e.hasOwnProperty(Ru)&&e.__forward_ref__===cs}function D(e){return{token:e.token,providedIn:e.providedIn||null,factory:e.factory,value:void 0}}function ti(e){return ju(e,ds)}function ju(e,t){return e.hasOwnProperty(t)&&e[t]||null}function Fu(e){return(e?.[ds]??null)||null}function Wi(e){return e&&e.hasOwnProperty(Gi)?e[Gi]:null}var ds=k({\u0275prov:k}),Gi=k({\u0275inj:k}),E=class{_desc;ngMetadataName="InjectionToken";\u0275prov;constructor(e,t){this._desc=e,this.\u0275prov=void 0,typeof t=="number"?this.__NG_ELEMENT_ID__=t:t!==void 0&&(this.\u0275prov=D({token:this,providedIn:t.providedIn||"root",factory:t.factory}))}get multi(){return this}toString(){return`InjectionToken ${this._desc}`}};function fs(e){return e&&!!e.\u0275providers}var Hu=k({\u0275cmp:k}),zu=k({\u0275dir:k}),Bu=k({\u0275pipe:k}),Yi=k({\u0275fac:k}),ht=k({__NG_ELEMENT_ID__:k}),Ki=k({__NG_ENV_ID__:k});function gt(e){return ni(e,"@Component"),e[Hu]||null}function hs(e){return ni(e,"@Directive"),e[zu]||null}function qu(e){return ni(e,"@Pipe"),e[Bu]||null}function ni(e,t){if(e==null)throw new w(-919,!1)}function ps(e){return typeof e=="string"?e:e==null?"":String(e)}var gs=k({ngErrorCode:k}),Uu=k({ngErrorMessage:k}),Zu=k({ngTokenPath:k});function ms(e,t){return vs("",-200,t)}function ri(e,t){throw new w(-201,!1)}function vs(e,t,n){let r=new w(t,e);return r[gs]=t,r[Uu]=e,n&&(r[Zu]=n),r}function $u(e){return e[gs]}var cr;function ys(){return cr}function z(e){let t=cr;return cr=e,t}function bs(e,t,n){let r=ti(e);if(r&&r.providedIn=="root")return r.value===void 0?r.value=r.factory():r.value;if(n&8)return null;if(t!==void 0)return t;ri(e,"")}var Qu={},Ee=Qu,Wu="__NG_DI_FLAG__",Gu=class{injector;constructor(e){this.injector=e}retrieve(e,t){let n=mt(t)||0;try{return this.injector.get(e,n&8?null:Ee,n)}catch(r){if(ei(r))return r;throw r}}};function Yu(e,t=0){let n=ls();if(n===void 0)throw new w(-203,!1);if(n===null)return bs(e,void 0,t);{let r=Ku(t),i=n.retrieve(e,r);if(ei(i)){if(r.optional)return null;throw i}return i}}function C(e,t=0){return(ys()||Yu)(F(e),t)}function b(e,t){return C(e,mt(t))}function mt(e){return typeof e>"u"||typeof e=="number"?e:0|(e.optional&&8)|(e.host&&1)|(e.self&&2)|(e.skipSelf&&4)}function Ku(e){return{optional:!!(e&8),host:!!(e&1),self:!!(e&2),skipSelf:!!(e&4)}}function dr(e){let t=[];for(let n=0;nArray.isArray(n)?ii(n,t):t(n))}function ws(e,t,n){t>=e.length?e.push(n):e.splice(t,0,n)}function Xt(e,t){return t>=e.length-1?e.pop():e.splice(t,1)[0]}function tc(e,t,n,r){let i=e.length;if(i==t)e.push(n,r);else if(i===1)e.push(r,e[0]),e[0]=n;else{for(i--,e.push(e[i-1],e[i]);i>t;){let o=i-2;e[i]=e[o],i--}e[t]=n,e[t+1]=r}}function nc(e,t,n){let r=It(e,t);return r>=0?e[r|1]=n:(r=~r,tc(e,r,t,n)),r}function Bn(e,t){let n=It(e,t);if(n>=0)return e[n|1]}function It(e,t){return rc(e,t,1)}function rc(e,t,n){let r=0,i=e.length>>n;for(;i!==r;){let o=r+(i-r>>1),s=e[o<t?i=o:r=o+1}return~(i<{n.push(s)};return ii(t,s=>{let l=s;fr(l,o,[],r)&&(i||=[],i.push(l))}),i!==void 0&&Ss(i,o),n}function Ss(e,t){for(let n=0;n{t(o,r)})}}function fr(e,t,n,r){if(e=F(e),!e)return!1;let i=null,o=Wi(e),s=!o&>(e);if(!o&&!s){let a=e.ngModule;if(o=Wi(a),o)i=a;else return!1}else{if(s&&!s.standalone)return!1;i=e}let l=r.has(i);if(s){if(l)return!1;if(r.add(i),s.dependencies){let a=typeof s.dependencies=="function"?s.dependencies():s.dependencies;for(let u of a)fr(u,t,n,r)}}else if(o){if(o.imports!=null&&!l){r.add(i);let u;ii(o.imports,c=>{fr(c,t,n,r)&&(u||=[],u.push(c))}),u!==void 0&&Ss(u,t)}if(!l){let u=vt(i)||(()=>new i);t({provide:i,useFactory:u,deps:Te},i),t({provide:Cs,useValue:i,multi:!0},i),t({provide:bn,useValue:()=>C(i),multi:!0},i)}let a=o.providers;if(a!=null&&!l){let u=e;si(a,c=>{t(c,u)})}}else return!1;return i!==e&&e.providers!==void 0}function si(e,t){for(let n of e)fs(n)&&(n=n.\u0275providers),Array.isArray(n)?si(n,t):t(n)}var sc=k({provide:String,useValue:k});function Es(e){return e!==null&&typeof e=="object"&&sc in e}function lc(e){return!!(e&&e.useExisting)}function ac(e){return!!(e&&e.useFactory)}function Ke(e){return typeof e=="function"}function uc(e){return!!e.useClass}var li=new E(""),Bt={},Xi={},qn;function ai(){return qn===void 0&&(qn=new xs),qn}var ve=class{},ui=class extends ve{parent;source;scopes;records=new Map;_ngOnDestroyHooks=new Set;_onDestroyHooks=[];get destroyed(){return this._destroyed}_destroyed=!1;injectorDefTypes;constructor(e,t,n,r){super(),this.parent=t,this.source=n,this.scopes=r,pr(e,o=>this.processProvider(o)),this.records.set(_s,$e(void 0,this)),r.has("environment")&&this.records.set(ve,$e(void 0,this));let i=this.records.get(li);i!=null&&typeof i.value=="string"&&this.scopes.add(i.value),this.injectorDefTypes=new Set(this.get(Cs,Te,{self:!0}))}retrieve(e,t){let n=mt(t)||0;try{return this.get(e,Ee,n)}catch(r){if(ei(r))return r;throw r}}destroy(){ut(this),this._destroyed=!0;let e=m(null);try{for(let n of this._ngOnDestroyHooks)n.ngOnDestroy();let t=this._onDestroyHooks;this._onDestroyHooks=[];for(let n of t)n()}finally{this.records.clear(),this._ngOnDestroyHooks.clear(),this.injectorDefTypes.clear(),m(e)}}onDestroy(e){return ut(this),this._onDestroyHooks.push(e),()=>this.removeOnDestroy(e)}runInContext(e){ut(this);let t=fe(this),n=z(void 0),r;try{return e()}finally{fe(t),z(n)}}get(e,t=Ee,n){if(ut(this),e.hasOwnProperty(Ki))return e[Ki](this);let r=mt(n),i,o=fe(this),s=z(void 0);try{if(!(r&4)){let a=this.records.get(e);if(a===void 0){let u=pc(e)&&ti(e);u&&this.injectableDefInScope(u)?a=$e(hr(e),Bt):a=null,this.records.set(e,a)}if(a!=null)return this.hydrate(e,a,r)}let l=r&2?ai():this.parent;return t=r&8&&t===Ee?null:t,l.get(e,t)}catch(l){let a=$u(l);throw a===-200||a===-201?new w(a,null):l}finally{z(s),fe(o)}}resolveInjectorInitializers(){let e=m(null),t=fe(this),n=z(void 0),r;try{let i=this.get(bn,Te,{self:!0});for(let o of i)o()}finally{fe(t),z(n),m(e)}}toString(){return"R3Injector[...]"}processProvider(e){e=F(e);let t=Ke(e)?e:F(e&&e.provide),n=dc(e);if(!Ke(e)&&e.multi===!0){let r=this.records.get(t);r||(r=$e(void 0,Bt,!0),r.factory=()=>dr(r.multi),this.records.set(t,r)),t=e,r.multi.push(e)}this.records.set(t,n)}hydrate(e,t,n){let r=m(null);try{if(t.value===Xi)throw ms("");return t.value===Bt&&(t.value=Xi,t.value=t.factory(void 0,n)),typeof t.value=="object"&&t.value&&hc(t.value)&&this._ngOnDestroyHooks.add(t.value),t.value}finally{m(r)}}injectableDefInScope(e){if(!e.providedIn)return!1;let t=F(e.providedIn);return typeof t=="string"?t==="any"||this.scopes.has(t):this.injectorDefTypes.has(t)}removeOnDestroy(e){let t=this._onDestroyHooks.indexOf(e);t!==-1&&this._onDestroyHooks.splice(t,1)}};function hr(e){let t=ti(e),n=t!==null?t.factory:vt(e);if(n!==null)return n;if(e instanceof E)throw new w(-204,!1);if(e instanceof Function)return cc(e);throw new w(-204,!1)}function cc(e){if(e.length>0)throw new w(-204,!1);let t=Fu(e);return t!==null?()=>t.factory(e):()=>new e}function dc(e){if(Es(e))return $e(void 0,e.useValue);{let t=Is(e);return $e(t,Bt)}}function Is(e,t,n){let r;if(Ke(e)){let i=F(e);return vt(i)||hr(i)}else if(Es(e))r=()=>F(e.useValue);else if(ac(e))r=()=>e.useFactory(...dr(e.deps||[]));else if(lc(e))r=(i,o)=>C(F(e.useExisting),o!==void 0&&o&8?8:void 0);else{let i=F(e&&(e.useClass||e.provide));if(fc(e))r=()=>new i(...dr(e.deps));else return vt(i)||hr(i)}return r}function ut(e){if(e.destroyed)throw new w(-205,!1)}function $e(e,t,n=!1){return{factory:e,value:t,multi:n?[]:void 0}}function fc(e){return!!e.deps}function hc(e){return e!==null&&typeof e=="object"&&typeof e.ngOnDestroy=="function"}function pc(e){return typeof e=="function"||typeof e=="object"&&e.ngMetadataName==="InjectionToken"}function pr(e,t){for(let n of e)Array.isArray(n)?pr(n,t):n&&fs(n)?pr(n.\u0275providers,t):t(n)}function Ts(e,t){let n;e instanceof ui?(ut(e),n=e):n=new Gu(e);let r,i=fe(n),o=z(void 0);try{return t()}finally{fe(i),z(o)}}function gc(){return ys()!==void 0||ls()!=null}var le=0,g=1,v=2,A=3,Z=4,W=5,yt=6,Jt=7,O=8,ye=9,ie=10,V=11,bt=12,Ji=13,nt=14,K=15,Oe=16,Qe=17,oe=18,be=19,Os=20,ge=21,Un=22,De=23,q=24,Zn=25,Me=26,H=27,Ds=1,eo=6,Pe=7,en=8,Xe=9,T=10;function Ie(e){return Array.isArray(e)&&typeof e[Ds]=="object"}function ae(e){return Array.isArray(e)&&e[Ds]===!0}function Ms(e){return(e.flags&4)!==0}function wn(e){return e.componentOffset>-1}function Ps(e){return(e.flags&1)===1}function rt(e){return!!e.template}function tn(e){return(e[v]&512)!==0}function it(e){return(e[v]&256)===256}var mc="svg",vc="math";function X(e){for(;Array.isArray(e);)e=e[le];return e}function Ns(e,t){return X(t[e])}function ue(e,t){return X(t[e.index])}function ci(e,t){return e.data[t]}function Ne(e,t){let n=t[e];return Ie(n)?n:n[le]}function yc(e){return(e[v]&4)===4}function di(e){return(e[v]&128)===128}function bc(e){return ae(e[A])}function se(e,t){return t==null?null:e[t]}function As(e){e[Qe]=0}function Vs(e){e[v]&1024||(e[v]|=1024,di(e)&&Tt(e))}function wc(e,t){for(;e>0;)t=t[nt],e--;return t}function nn(e){return!!(e[v]&9216||e[q]?.dirty)}function gr(e){e[ie].changeDetectionScheduler?.notify(8),e[v]&64&&(e[v]|=1024),nn(e)&&Tt(e)}function Tt(e){e[ie].changeDetectionScheduler?.notify(0);let t=Ae(e);for(;t!==null&&!(t[v]&8192||(t[v]|=8192,!di(t)));)t=Ae(t)}function Rs(e,t){if(it(e))throw new w(911,!1);e[ge]===null&&(e[ge]=[]),e[ge].push(t)}function _c(e,t){if(e[ge]===null)return;let n=e[ge].indexOf(t);n!==-1&&e[ge].splice(n,1)}function Ae(e){let t=e[A];return ae(t)?t[A]:t}function Ls(e){return e[Jt]??=[]}function js(e){return e.cleanup??=[]}function Cc(e,t,n,r){let i=Ls(t);i.push(n),e.firstCreatePass&&js(e).push(r,i.length-1)}var y={lFrame:Zs(null),bindingsEnabled:!0,skipHydrationRootTNode:null},mr=!1;function xc(){return y.lFrame.elementDepthCount}function kc(){y.lFrame.elementDepthCount++}function Sc(){y.lFrame.elementDepthCount--}function Ec(){return y.skipHydrationRootTNode!==null}function Ic(e){return y.skipHydrationRootTNode===e}function Tc(){y.skipHydrationRootTNode=null}function S(){return y.lFrame.lView}function U(){return y.lFrame.tView}function qe(e){return y.lFrame.contextLView=e,e[O]}function Ue(e){return y.lFrame.contextLView=null,e}function ce(){let e=Fs();for(;e!==null&&e.type===64;)e=e.parent;return e}function Fs(){return y.lFrame.currentTNode}function Oc(){let e=y.lFrame,t=e.currentTNode;return e.isParent?t:t.parent}function Ot(e,t){let n=y.lFrame;n.currentTNode=e,n.isParent=t}function Hs(){return y.lFrame.isParent}function Dc(){y.lFrame.isParent=!1}function zs(){return mr}function rn(e){let t=mr;return mr=e,t}function Mc(e){return y.lFrame.bindingIndex=e}function _n(){return y.lFrame.bindingIndex++}function Pc(e){let t=y.lFrame,n=t.bindingIndex;return t.bindingIndex=t.bindingIndex+e,n}function Nc(){return y.lFrame.inI18n}function Ac(e,t){let n=y.lFrame;n.bindingIndex=n.bindingRootIndex=e,vr(t)}function Vc(){return y.lFrame.currentDirectiveIndex}function vr(e){y.lFrame.currentDirectiveIndex=e}function Rc(e){let t=y.lFrame.currentDirectiveIndex;return t===-1?null:e[t]}function Bs(){return y.lFrame.currentQueryIndex}function fi(e){y.lFrame.currentQueryIndex=e}function Lc(e){let t=e[g];return t.type===2?t.declTNode:t.type===1?e[W]:null}function qs(e,t,n){if(n&4){let i=t,o=e;for(;(i=i.parent,i===null&&!(n&1))&&(i=Lc(o),!(i===null||(o=o[nt],i.type&10))););if(i===null)return!1;t=i,e=o}let r=y.lFrame=Us();return r.currentTNode=t,r.lView=e,!0}function hi(e){let t=Us(),n=e[g];y.lFrame=t,t.currentTNode=n.firstChild,t.lView=e,t.tView=n,t.contextLView=e,t.bindingIndex=n.bindingStartIndex,t.inI18n=!1}function Us(){let e=y.lFrame,t=e===null?null:e.child;return t===null?Zs(e):t}function Zs(e){let t={currentTNode:null,isParent:!0,lView:null,tView:null,selectedIndex:-1,contextLView:null,elementDepthCount:0,currentNamespace:null,currentDirectiveIndex:-1,bindingRootIndex:-1,bindingIndex:-1,currentQueryIndex:0,parent:e,child:null,inI18n:!1};return e!==null&&(e.child=t),t}function $s(){let e=y.lFrame;return y.lFrame=e.parent,e.currentTNode=null,e.lView=null,e}var Qs=$s;function pi(){let e=$s();e.isParent=!0,e.tView=null,e.selectedIndex=-1,e.contextLView=null,e.elementDepthCount=0,e.currentDirectiveIndex=-1,e.currentNamespace=null,e.bindingRootIndex=-1,e.bindingIndex=-1,e.currentQueryIndex=0}function jc(e){return(y.lFrame.contextLView=wc(e,y.lFrame.contextLView))[O]}function Fe(){return y.lFrame.selectedIndex}function Ve(e){y.lFrame.selectedIndex=e}function Fc(){let e=y.lFrame;return ci(e.tView,e.selectedIndex)}function Hc(){return y.lFrame.currentNamespace}var Ws=!0;function gi(){return Ws}function mi(e){Ws=e}function to(e,t=null,n=null,r){let i=zc(e,t,n,r);return i.resolveInjectorInitializers(),i}function zc(e,t=null,n=null,r,i=new Set){let o=[n||Te,oc(e)],s;return new ui(o,t||ai(),s||null,i)}var Cn=class Gs{static THROW_IF_NOT_FOUND=Ee;static NULL=new xs;static create(t,n){if(Array.isArray(t))return to({name:""},n,t,"");{let r=t.name??"";return to({name:r},t.parent,t.providers,r)}}static \u0275prov=D({token:Gs,providedIn:"any",factory:()=>C(_s)});static __NG_ELEMENT_ID__=-1},we=new E(""),xn=(()=>{class e{static __NG_ELEMENT_ID__=Bc;static __NG_ENV_ID__=n=>n}return e})(),Ys=class extends xn{_lView;constructor(e){super(),this._lView=e}get destroyed(){return it(this._lView)}onDestroy(e){let t=this._lView;return Rs(t,e),()=>_c(t,e)}};function Bc(){return new Ys(S())}var qc=!1,Uc=new E(""),kn=(()=>{class e{taskId=0;pendingTasks=new Set;destroyed=!1;pendingTask=new Mu(!1);debugTaskTracker=b(Uc,{optional:!0});get hasPendingTasks(){return this.destroyed?!1:this.pendingTask.value}get hasPendingTasksObservable(){return this.destroyed?new ar(n=>{n.next(!1),n.complete()}):this.pendingTask}add(){!this.hasPendingTasks&&!this.destroyed&&this.pendingTask.next(!0);let n=this.taskId++;return this.pendingTasks.add(n),this.debugTaskTracker?.add(n),n}has(n){return this.pendingTasks.has(n)}remove(n){this.pendingTasks.delete(n),this.debugTaskTracker?.remove(n),this.pendingTasks.size===0&&this.hasPendingTasks&&this.pendingTask.next(!1)}ngOnDestroy(){this.pendingTasks.clear(),this.hasPendingTasks&&this.pendingTask.next(!1),this.destroyed=!0,this.pendingTask.unsubscribe()}static \u0275prov=D({token:e,providedIn:"root",factory:()=>new e})}return e})(),Zc=class extends Et{__isAsync;destroyRef=void 0;pendingTasks=void 0;constructor(e=!1){super(),this.__isAsync=e,gc()&&(this.destroyRef=b(xn,{optional:!0})??void 0,this.pendingTasks=b(kn,{optional:!0})??void 0)}emit(e){let t=m(null);try{super.next(e)}finally{m(t)}}subscribe(e,t,n){let r=e,i=t||(()=>null),o=n;if(e&&typeof e=="object"){let l=e;r=l.next?.bind(l),i=l.error?.bind(l),o=l.complete?.bind(l)}this.__isAsync&&(i=this.wrapInTimeout(i),r&&(r=this.wrapInTimeout(r)),o&&(o=this.wrapInTimeout(o)));let s=super.subscribe({next:r,error:i,complete:o});return e instanceof me&&e.add(s),s}wrapInTimeout(e){return t=>{let n=this.pendingTasks?.add();setTimeout(()=>{try{e(t)}finally{n!==void 0&&this.pendingTasks?.remove(n)}})}}},pe=Zc;function on(...e){}function Ks(e){let t,n;function r(){e=on;try{n!==void 0&&typeof cancelAnimationFrame=="function"&&cancelAnimationFrame(n),t!==void 0&&clearTimeout(t)}catch{}}return t=setTimeout(()=>{e(),r()}),typeof requestAnimationFrame=="function"&&(n=requestAnimationFrame(()=>{e(),r()})),()=>r()}function $c(e){return queueMicrotask(()=>e()),()=>{e=on}}var vi="isAngularZone",sn=vi+"_ID",Qc=0,He=class yr{hasPendingMacrotasks=!1;hasPendingMicrotasks=!1;isStable=!0;onUnstable=new pe(!1);onMicrotaskEmpty=new pe(!1);onStable=new pe(!1);onError=new pe(!1);constructor(t){let{enableLongStackTrace:n=!1,shouldCoalesceEventChangeDetection:r=!1,shouldCoalesceRunChangeDetection:i=!1,scheduleInRootZone:o=qc}=t;if(typeof Zone>"u")throw new w(908,!1);Zone.assertZonePatched();let s=this;s._nesting=0,s._outer=s._inner=Zone.current,Zone.TaskTrackingZoneSpec&&(s._inner=s._inner.fork(new Zone.TaskTrackingZoneSpec)),n&&Zone.longStackTraceZoneSpec&&(s._inner=s._inner.fork(Zone.longStackTraceZoneSpec)),s.shouldCoalesceEventChangeDetection=!i&&r,s.shouldCoalesceRunChangeDetection=i,s.callbackScheduled=!1,s.scheduleInRootZone=o,Yc(s)}static isInAngularZone(){return typeof Zone<"u"&&Zone.current.get(vi)===!0}static assertInAngularZone(){if(!yr.isInAngularZone())throw new w(909,!1)}static assertNotInAngularZone(){if(yr.isInAngularZone())throw new w(909,!1)}run(t,n,r){return this._inner.run(t,n,r)}runTask(t,n,r,i){let o=this._inner,s=o.scheduleEventTask("NgZoneEvent: "+i,t,Wc,on,on);try{return o.runTask(s,n,r)}finally{o.cancelTask(s)}}runGuarded(t,n,r){return this._inner.runGuarded(t,n,r)}runOutsideAngular(t){return this._outer.run(t)}},Wc={};function yi(e){if(e._nesting==0&&!e.hasPendingMicrotasks&&!e.isStable)try{e._nesting++,e.onMicrotaskEmpty.emit(null)}finally{if(e._nesting--,!e.hasPendingMicrotasks)try{e.runOutsideAngular(()=>e.onStable.emit(null))}finally{e.isStable=!0}}}function Gc(e){if(e.isCheckStableRunning||e.callbackScheduled)return;e.callbackScheduled=!0;function t(){Ks(()=>{e.callbackScheduled=!1,br(e),e.isCheckStableRunning=!0,yi(e),e.isCheckStableRunning=!1})}e.scheduleInRootZone?Zone.root.run(()=>{t()}):e._outer.run(()=>{t()}),br(e)}function Yc(e){let t=()=>{Gc(e)},n=Qc++;e._inner=e._inner.fork({name:"angular",properties:{[vi]:!0,[sn]:n,[sn+n]:!0},onInvokeTask:(r,i,o,s,l,a)=>{if(Xc(a))return r.invokeTask(o,s,l,a);try{return no(e),r.invokeTask(o,s,l,a)}finally{(e.shouldCoalesceEventChangeDetection&&s.type==="eventTask"||e.shouldCoalesceRunChangeDetection)&&t(),ro(e)}},onInvoke:(r,i,o,s,l,a,u)=>{try{return no(e),r.invoke(o,s,l,a,u)}finally{e.shouldCoalesceRunChangeDetection&&!e.callbackScheduled&&!Jc(a)&&t(),ro(e)}},onHasTask:(r,i,o,s)=>{r.hasTask(o,s),i===o&&(s.change=="microTask"?(e._hasPendingMicrotasks=s.microTask,br(e),yi(e)):s.change=="macroTask"&&(e.hasPendingMacrotasks=s.macroTask))},onHandleError:(r,i,o,s)=>(r.handleError(o,s),e.runOutsideAngular(()=>e.onError.emit(s)),!1)})}function br(e){e._hasPendingMicrotasks||(e.shouldCoalesceEventChangeDetection||e.shouldCoalesceRunChangeDetection)&&e.callbackScheduled===!0?e.hasPendingMicrotasks=!0:e.hasPendingMicrotasks=!1}function no(e){e._nesting++,e.isStable&&(e.isStable=!1,e.onUnstable.emit(null))}function ro(e){e._nesting--,yi(e)}var Kc=class{hasPendingMicrotasks=!1;hasPendingMacrotasks=!1;isStable=!0;onUnstable=new pe;onMicrotaskEmpty=new pe;onStable=new pe;onError=new pe;run(e,t,n){return e.apply(t,n)}runGuarded(e,t,n){return e.apply(t,n)}runOutsideAngular(e){return e()}runTask(e,t,n,r){return e.apply(t,n)}};function Xc(e){return Xs(e,"__ignore_ng_zone__")}function Jc(e){return Xs(e,"__scheduler_tick__")}function Xs(e,t){return!Array.isArray(e)||e.length!==1?!1:e[0]?.data?.[t]===!0}var Sn=class{_console=console;handleError(e){this._console.error("ERROR",e)}},Dt=new E("",{factory:()=>{let e=b(He),t=b(ve),n;return r=>{e.runOutsideAngular(()=>{t.destroyed&&!n?setTimeout(()=>{throw r}):(n??=t.get(Sn),n.handleError(r))})}}}),ed={provide:bn,useValue:()=>{let e=b(Sn,{optional:!0})},multi:!0},td=new E("",{factory:()=>{let e=b(we).defaultView;if(!e)return;let t=b(Dt),n=o=>{t(o.reason),o.preventDefault()},r=o=>{o.error?t(o.error):t(new Error(o.message,{cause:o})),o.preventDefault()},i=()=>{e.addEventListener("unhandledrejection",n),e.addEventListener("error",r)};typeof Zone<"u"?Zone.root.run(i):i(),b(xn).onDestroy(()=>{e.removeEventListener("error",r),e.removeEventListener("unhandledrejection",n)})}});function nd(){return oi([ic(()=>{b(td)})])}function j(e,t){let[n,r,i]=ou(e,t?.equal),o=n,s=o[ne];return o.set=r,o.update=i,o.asReadonly=rd.bind(o),o}function rd(){let e=this[ne];if(e.readonlyFn===void 0){let t=()=>this();t[ne]=e,e.readonlyFn=t}return e.readonlyFn}var Js=(()=>{class e{view;node;constructor(n,r){this.view=n,this.node=r}static __NG_ELEMENT_ID__=id}return e})();function id(){return new Js(S(),ce())}var bi=class{},wi=new E("",{factory:()=>!0}),od=new E(""),el=(()=>{class e{static \u0275prov=D({token:e,providedIn:"root",factory:()=>new sd})}return e})(),sd=class{dirtyEffectCount=0;queues=new Map;add(e){this.enqueue(e),this.schedule(e)}schedule(e){e.dirty&&this.dirtyEffectCount++}remove(e){let t=e.zone,n=this.queues.get(t);n.has(e)&&(n.delete(e),e.dirty&&this.dirtyEffectCount--)}enqueue(e){let t=e.zone;this.queues.has(t)||this.queues.set(t,new Set);let n=this.queues.get(t);n.has(e)||n.add(e)}flush(){for(;this.dirtyEffectCount>0;){let e=!1;for(let[t,n]of this.queues)t===null?e||=this.flushQueue(n):e||=t.run(()=>this.flushQueue(n));e||(this.dirtyEffectCount=0)}}flushQueue(e){let t=!1;for(let n of e)n.dirty&&(this.dirtyEffectCount--,t=!0,n.run());return t}},ld=class{[ne];constructor(e){this[ne]=e}destroy(){this[ne].destroy()}};function $n(e,t){let n=t?.injector??b(Cn),r=t?.manualCleanup!==!0?n.get(xn):null,i,o=n.get(Js,null,{optional:!0}),s=n.get(bi);return o!==null?(i=cd(o.view,s,e),r instanceof Ys&&r._lView===o.view&&(r=null)):i=dd(e,n.get(el),s),i.injector=n,r!==null&&(i.onDestroyFns=[r.onDestroy(()=>i.destroy())]),new ld(i)}var tl=Q($({},cu),{cleanupFns:void 0,zone:null,onDestroyFns:null,run(){let e=rn(!1);try{du(this)}finally{rn(e)}},cleanup(){if(!this.cleanupFns?.length)return;let e=m(null);try{for(;this.cleanupFns.length;)this.cleanupFns.pop()()}finally{this.cleanupFns=[],m(e)}}}),ad=Q($({},tl),{consumerMarkedDirty(){this.scheduler.schedule(this),this.notifier.notify(12)},destroy(){if(yn(this),this.onDestroyFns!==null)for(let e of this.onDestroyFns)e();this.cleanup(),this.scheduler.remove(this)}}),ud=Q($({},tl),{consumerMarkedDirty(){this.view[v]|=8192,Tt(this.view),this.notifier.notify(13)},destroy(){if(yn(this),this.onDestroyFns!==null)for(let e of this.onDestroyFns)e();this.cleanup(),this.view[De]?.delete(this)}});function cd(e,t,n){let r=Object.create(ud);return r.view=e,r.zone=typeof Zone<"u"?Zone.current:null,r.notifier=t,r.fn=nl(r,n),e[De]??=new Set,e[De].add(r),r.consumerMarkedDirty(r),r}function dd(e,t,n){let r=Object.create(ad);return r.fn=nl(r,e),r.scheduler=t,r.notifier=n,r.zone=typeof Zone<"u"?Zone.current:null,r.scheduler.add(r),r.notifier.notify(12),r}function nl(e,t){return()=>{t(n=>(e.cleanupFns??=[]).push(n))}}function fd(e){return{toString:e}.toString()}function hd(e){return typeof e=="function"}function rl(e,t,n,r){t!==null?t.applyValueToInputSignal(t,r):e[n]=r}var pd=class{previousValue;currentValue;firstChange;constructor(e,t,n){this.previousValue=e,this.currentValue=t,this.firstChange=n}isFirstChange(){return this.firstChange}};function gd(e){return e.type.prototype.ngOnChanges&&(e.setInput=vd),md}function md(){let e=ol(this),t=e?.current;if(t){let n=e.previous;if(n===Ye)e.previous=t;else for(let r in t)n[r]=t[r];e.current=null,this.ngOnChanges(t)}}function vd(e,t,n,r,i){let o=this.declaredInputs[r],s=ol(e)||yd(e,{previous:Ye,current:null}),l=s.current||(s.current={}),a=s.previous,u=a[o];l[o]=new pd(u&&u.currentValue,n,a===Ye),rl(e,t,i,n)}var il="__ngSimpleChanges__";function ol(e){return e[il]||null}function yd(e,t){return e[il]=t}var io=[],x=function(e,t=null,n){for(let r=0;r=r)break}else t[a]<0&&(e[Qe]+=65536),(l>14>16&&(e[v]&3)===t&&(e[v]+=16384,oo(l,o)):oo(l,o)}var Ge=-1,Mt=class{factory;name;injectImpl;resolving=!1;canSeeViewProviders;multi;componentProviders;index;providerFactory;constructor(e,t,n,r){this.factory=e,this.name=r,this.canSeeViewProviders=t,this.injectImpl=n}};function Cd(e,t,n){let r=0;for(;rt){s=o-1;break}}}for(;o>16}function an(e,t){let n=kd(e),r=t;for(;n>0;)r=r[nt],n--;return r}var wr=!0;function lo(e){let t=wr;return wr=e,t}var Sd=256,al=Sd-1,ul=5,Ed=0,G={};function Id(e,t,n){let r;typeof n=="string"?r=n.charCodeAt(0)||0:n.hasOwnProperty(ht)&&(r=n[ht]),r==null&&(r=n[ht]=Ed++);let i=r&al,o=1<>ul)]|=o}function un(e,t){let n=cl(e,t);if(n!==-1)return n;let r=t[g];r.firstCreatePass&&(e.injectorIndex=t.length,Wn(r.data,e),Wn(t,null),Wn(r.blueprint,null));let i=_i(e,t),o=e.injectorIndex;if(ll(i)){let s=ln(i),l=an(i,t),a=l[g].data;for(let u=0;u<8;u++)t[o+u]=l[s+u]|a[s+u]}return t[o+8]=i,o}function Wn(e,t){e.push(0,0,0,0,0,0,0,0,t)}function cl(e,t){return e.injectorIndex===-1||e.parent&&e.parent.injectorIndex===e.injectorIndex||t[e.injectorIndex+8]===null?-1:e.injectorIndex}function _i(e,t){if(e.parent&&e.parent.injectorIndex!==-1)return e.parent.injectorIndex;let n=0,r=null,i=t;for(;i!==null;){if(r=gl(i),r===null)return Ge;if(n++,i=i[nt],r.injectorIndex!==-1)return r.injectorIndex|n<<16}return Ge}function _r(e,t,n){Id(e,t,n)}function dl(e,t,n){if(n&8||e!==void 0)return e;ri(t,"NodeInjector")}function fl(e,t,n,r){if(n&8&&r===void 0&&(r=null),(n&3)===0){let i=e[ye],o=z(void 0);try{return i?i.get(t,r,n&8):bs(t,r,n&8)}finally{z(o)}}return dl(r,t,n)}function hl(e,t,n,r=0,i){if(e!==null){if(t[v]&2048&&!(r&2)){let s=Md(e,t,n,r,G);if(s!==G)return s}let o=pl(e,t,n,r,G);if(o!==G)return o}return fl(t,n,r,i)}function pl(e,t,n,r,i){let o=Od(n);if(typeof o=="function"){if(!qs(t,e,r))return r&1?dl(i,n,r):fl(t,n,r,i);try{let s;if(s=o(r),s==null&&!(r&8))ri(n);else return s}finally{Qs()}}else if(typeof o=="number"){let s=null,l=cl(e,t),a=Ge,u=r&1?t[K][W]:null;for((l===-1||r&4)&&(a=l===-1?_i(e,t):t[l+8],a===Ge||!uo(r,!1)?l=-1:(s=t[g],l=ln(a),t=an(a,t)));l!==-1;){let c=t[g];if(ao(o,l,c.data)){let d=Td(l,t,n,s,r,u);if(d!==G)return d}a=t[l+8],a!==Ge&&uo(r,t[g].data[l+8]===u)&&ao(o,l,t)?(s=c,l=ln(a),t=an(a,t)):l=-1}}return i}function Td(e,t,n,r,i,o){let s=t[g],l=s.data[e+8],a=r==null?wn(l)&&wr:r!=s&&(l.type&3)!==0,u=i&1&&o===l,c=Zt(l,s,n,a,u);return c!==null?wt(t,s,c,l,i):G}function Zt(e,t,n,r,i){let o=e.providerIndexes,s=t.data,l=o&1048575,a=e.directiveStart,u=e.directiveEnd,c=o>>20,d=r?l:l+c,h=i?l+c:u;for(let f=d;f=a&&p.type===n)return f}if(i){let f=s[a];if(f&&rt(f)&&f.type===n)return a}return null}function wt(e,t,n,r,i){let o=e[n],s=t.data;if(o instanceof Mt){let l=o;if(l.resolving)throw ms("");let a=lo(l.canSeeViewProviders);l.resolving=!0;let u=s[n].type||s[n],c,d=l.injectImpl?z(l.injectImpl):null,h=qs(e,r,0);try{o=e[n]=l.factory(void 0,i,s,e,r),t.firstCreatePass&&n>=r.directiveStart&&bd(n,s[n],t)}finally{d!==null&&z(d),lo(a),l.resolving=!1,Qs()}}return o}function Od(e){if(typeof e=="string")return e.charCodeAt(0)||0;let t=e.hasOwnProperty(ht)?e[ht]:void 0;return typeof t=="number"?t>=0?t&al:Dd:t}function ao(e,t,n){let r=1<>ul)]&r)}function uo(e,t){return!(e&2)&&!(e&1&&t)}var pt=class{_tNode;_lView;constructor(e,t){this._tNode=e,this._lView=t}get(e,t,n){return hl(this._tNode,this._lView,e,mt(n),t)}};function Dd(){return new pt(ce(),S())}function Md(e,t,n,r,i){let o=e,s=t;for(;o!==null&&s!==null&&s[v]&2048&&!tn(s);){let l=pl(o,s,n,r|2,G);if(l!==G)return l;let a=o.parent;if(!a){let u=s[Os];if(u){let c=u.get(n,G,r&-5);if(c!==G)return c}a=gl(s),s=s[nt]}o=a}return i}function gl(e){let t=e[g],n=t.type;return n===2?t.declTNode:n===1?e[W]:null}function Pd(){return ot(ce(),S())}function ot(e,t){return new In(ue(e,t))}var In=(()=>{class e{nativeElement;constructor(n){this.nativeElement=n}static __NG_ELEMENT_ID__=Pd}return e})();function Nd(e){return e instanceof In?e.nativeElement:e}function Ad(){return this._results[Symbol.iterator]()}var Vd=class{_emitDistinctChangesOnly;dirty=!0;_onDirty=void 0;_results=[];_changesDetected=!1;_changes=void 0;length=0;first=void 0;last=void 0;get changes(){return this._changes??=new Et}constructor(e=!1){this._emitDistinctChangesOnly=e}get(e){return this._results[e]}map(e){return this._results.map(e)}filter(e){return this._results.filter(e)}find(e){return this._results.find(e)}reduce(e,t){return this._results.reduce(e,t)}forEach(e){this._results.forEach(e)}some(e){return this._results.some(e)}toArray(){return this._results.slice()}toString(){return this._results.toString()}reset(e,t){this.dirty=!1;let n=ec(e);(this._changesDetected=!Ju(this._results,n,t))&&(this._results=n,this.length=n.length,this.last=n[this.length-1],this.first=n[0])}notifyOnChanges(){this._changes!==void 0&&(this._changesDetected||!this._emitDistinctChangesOnly)&&this._changes.next(this)}onDirty(e){this._onDirty=e}setDirty(){this.dirty=!0,this._onDirty?.()}destroy(){this._changes!==void 0&&(this._changes.complete(),this._changes.unsubscribe())}[Symbol.iterator]=Ad};function ml(e){return(e.flags&128)===128}var vl=function(e){return e[e.OnPush=0]="OnPush",e[e.Eager=1]="Eager",e[e.Default=1]="Default",e}(vl||{}),yl=new Map,Rd=0;function Ld(){return Rd++}function jd(e){yl.set(e[be],e)}function Cr(e){yl.delete(e[be])}var co="__ngContext__";function Je(e,t){Ie(t)?(e[co]=t[be],jd(t)):e[co]=t}function bl(e){return _l(e[bt])}function wl(e){return _l(e[Z])}function _l(e){for(;e!==null&&!ae(e);)e=e[Z];return e}var xr;function Fd(e){xr=e}function Hd(){if(xr!==void 0)return xr;if(typeof document<"u")return document;throw new w(210,!1)}var Cl=new E("",{factory:()=>zd}),zd="ng",xl=new E(""),kl=new E("",{providedIn:"platform",factory:()=>"unknown"}),Sl=new E("",{factory:()=>b(we).body?.querySelector("[ngCspNonce]")?.getAttribute("ngCspNonce")||null}),Bd="r",qd="di",El=!1,Ud=new E("",{factory:()=>El}),fo=new WeakMap;function Zd(e,t){if(e==null||typeof e!="object")return;let n=fo.get(e);n||(n=new WeakSet,fo.set(e,n)),n.add(t)}var $d=(e,t,n,r)=>{};function Qd(e,t,n,r){$d(e,t,n,r)}function Il(e){return(e.flags&32)===32}var Wd=()=>null;function Tl(e,t,n=!1){return Wd(e,t,n)}function Ol(e,t){let n=e.contentQueries;if(n!==null){let r=m(null);try{for(let i=0;ie,createScript:e=>e,createScriptURL:e=>e})}catch{}return Vt}function Tn(e){return Yd()?.createHTML(e)||e}var Rt;function Kd(){if(Rt===void 0&&(Rt=null,Kt.trustedTypes))try{Rt=Kt.trustedTypes.createPolicy("angular#unsafe-bypass",{createHTML:e=>e,createScript:e=>e,createScriptURL:e=>e})}catch{}return Rt}function ho(e){return Kd()?.createHTML(e)||e}var ze=class{changingThisBreaksApplicationSecurity;constructor(e){this.changingThisBreaksApplicationSecurity=e}toString(){return`SafeValue must use [property]=binding: ${this.changingThisBreaksApplicationSecurity} (see ${as})`}},Xd=class extends ze{getTypeName(){return"HTML"}},Jd=class extends ze{getTypeName(){return"Style"}},ef=class extends ze{getTypeName(){return"Script"}},tf=class extends ze{getTypeName(){return"URL"}},nf=class extends ze{getTypeName(){return"ResourceURL"}};function Ce(e){return e instanceof ze?e.changingThisBreaksApplicationSecurity:e}function Ze(e,t){let n=rf(e);if(n!=null&&n!==t){if(n==="ResourceURL"&&t==="URL")return!0;throw new Error(`Required a safe ${t}, got a ${n} (see ${as})`)}return n===t}function rf(e){return e instanceof ze&&e.getTypeName()||null}function of(e){return new Xd(e)}function sf(e){return new Jd(e)}function lf(e){return new ef(e)}function af(e){return new tf(e)}function uf(e){return new nf(e)}function cf(e){let t=new ff(e);return hf()?new df(t):t}var df=class{inertDocumentHelper;constructor(e){this.inertDocumentHelper=e}getInertBodyElement(e){e=""+e;try{let t=new window.DOMParser().parseFromString(Tn(e),"text/html").body;return t===null?this.inertDocumentHelper.getInertBodyElement(e):(t.firstChild?.remove(),t)}catch{return null}}},ff=class{defaultDoc;inertDocument;constructor(e){this.defaultDoc=e,this.inertDocument=this.defaultDoc.implementation.createHTMLDocument("sanitization-inert")}getInertBodyElement(e){let t=this.inertDocument.createElement("template");return t.innerHTML=Tn(e),t}};function hf(){try{return!!new window.DOMParser().parseFromString(Tn(""),"text/html")}catch{return!1}}var pf=/^(?!javascript:)(?:[a-z0-9+.-]+:|[^&:\/?#]*(?:[\/?#]|$))/i;function Dl(e){return e=String(e),e.match(pf)?e:"unsafe:"+e}function de(e){let t={};for(let n of e.split(","))t[n]=!0;return t}function Pt(...e){let t={};for(let n of e)for(let r in n)n.hasOwnProperty(r)&&(t[r]=!0);return t}var Ml=de("area,br,col,hr,img,wbr"),Pl=de("colgroup,dd,dt,li,p,tbody,td,tfoot,th,thead,tr"),Nl=de("rp,rt"),gf=Pt(Nl,Pl),mf=Pt(Pl,de("address,article,aside,blockquote,caption,center,del,details,dialog,dir,div,dl,figure,figcaption,footer,h1,h2,h3,h4,h5,h6,header,hgroup,hr,ins,main,map,menu,nav,ol,pre,section,summary,table,ul")),vf=Pt(Nl,de("a,abbr,acronym,audio,b,bdi,bdo,big,br,cite,code,del,dfn,em,font,i,img,ins,kbd,label,map,mark,picture,q,ruby,rp,rt,s,samp,small,source,span,strike,strong,sub,sup,time,track,tt,u,var,video")),po=Pt(Ml,mf,vf,gf),Al=de("background,cite,href,itemtype,longdesc,poster,src,xlink:href"),yf=de("abbr,accesskey,align,alt,autoplay,axis,bgcolor,border,cellpadding,cellspacing,class,clear,color,cols,colspan,compact,controls,coords,datetime,default,dir,download,face,headers,height,hidden,hreflang,hspace,ismap,itemscope,itemprop,kind,label,lang,language,loop,media,muted,nohref,nowrap,open,preload,rel,rev,role,rows,rowspan,rules,scope,scrolling,shape,size,sizes,span,srclang,srcset,start,summary,tabindex,target,title,translate,type,usemap,valign,value,vspace,width"),bf=de("aria-activedescendant,aria-atomic,aria-autocomplete,aria-busy,aria-checked,aria-colcount,aria-colindex,aria-colspan,aria-controls,aria-current,aria-describedby,aria-details,aria-disabled,aria-dropeffect,aria-errormessage,aria-expanded,aria-flowto,aria-grabbed,aria-haspopup,aria-hidden,aria-invalid,aria-keyshortcuts,aria-label,aria-labelledby,aria-level,aria-live,aria-modal,aria-multiline,aria-multiselectable,aria-orientation,aria-owns,aria-placeholder,aria-posinset,aria-pressed,aria-readonly,aria-relevant,aria-required,aria-roledescription,aria-rowcount,aria-rowindex,aria-rowspan,aria-selected,aria-setsize,aria-sort,aria-valuemax,aria-valuemin,aria-valuenow,aria-valuetext"),wf=Pt(Al,yf,bf),_f=de("script,style,template"),Cf=class{sanitizedSomething=!1;buf=[];sanitizeChildren(e){let t=e.firstChild,n=!0,r=[];for(;t;){if(t.nodeType===Node.ELEMENT_NODE?n=this.startElement(t):t.nodeType===Node.TEXT_NODE?this.chars(t.nodeValue):this.sanitizedSomething=!0,n&&t.firstChild){r.push(t),t=Sf(t);continue}for(;t;){t.nodeType===Node.ELEMENT_NODE&&this.endElement(t);let i=kf(t);if(i){t=i;break}t=r.pop()}}return this.buf.join("")}startElement(e){let t=go(e).toLowerCase();if(!po.hasOwnProperty(t))return this.sanitizedSomething=!0,!_f.hasOwnProperty(t);this.buf.push("<"),this.buf.push(t);let n=e.attributes;for(let r=0;r"),!0}endElement(e){let t=go(e).toLowerCase();po.hasOwnProperty(t)&&!Ml.hasOwnProperty(t)&&(this.buf.push(""))}chars(e){this.buf.push(mo(e))}};function xf(e,t){return(e.compareDocumentPosition(t)&Node.DOCUMENT_POSITION_CONTAINED_BY)!==Node.DOCUMENT_POSITION_CONTAINED_BY}function kf(e){let t=e.nextSibling;if(t&&e!==t.previousSibling)throw Vl(t);return t}function Sf(e){let t=e.firstChild;if(t&&xf(e,t))throw Vl(t);return t}function go(e){let t=e.nodeName;return typeof t=="string"?t:"FORM"}function Vl(e){return new Error(`Failed to sanitize html because the element is clobbered: ${e.outerHTML}`)}var Ef=/[\uD800-\uDBFF][\uDC00-\uDFFF]/g,If=/([^\#-~ |!])/g;function mo(e){return e.replace(/&/g,"&").replace(Ef,function(t){let n=t.charCodeAt(0),r=t.charCodeAt(1);return"&#"+((n-55296)*1024+(r-56320)+65536)+";"}).replace(If,function(t){return"&#"+t.charCodeAt(0)+";"}).replace(//g,">")}var Lt;function Rl(e,t){let n=null;try{Lt=Lt||cf(e);let r=t?String(t):"";n=Lt.getInertBodyElement(r);let i=5,o=r;do{if(i===0)throw new Error("Failed to sanitize html because the input is unstable");i--,r=o,o=n.innerHTML,n=Lt.getInertBodyElement(r)}while(r!==o);let s=new Cf().sanitizeChildren(vo(n)||n);return Tn(s)}finally{if(n){let r=vo(n)||n;for(;r.firstChild;)r.firstChild.remove()}}}function vo(e){return"content"in e&&Tf(e)?e.content:null}function Tf(e){return e.nodeType===Node.ELEMENT_NODE&&e.nodeName==="TEMPLATE"}function Of(e,t){return e.createText(t)}function Df(e,t,n){e.setValue(t,n)}function Ll(e,t,n){return e.createElement(t,n)}function cn(e,t,n,r,i){e.insertBefore(t,n,r,i)}function jl(e,t,n){e.appendChild(t,n)}function yo(e,t,n,r,i){r!==null?cn(e,t,n,r,i):jl(e,t,n)}function Fl(e,t,n,r){e.removeChild(null,t,n,r)}function Mf(e,t,n){e.setAttribute(t,"style",n)}function Pf(e,t,n){n===""?e.removeAttribute(t,"class"):e.setAttribute(t,"class",n)}function Hl(e,t,n){let{mergedAttrs:r,classes:i,styles:o}=n;r!==null&&Cd(e,t,r),i!==null&&Pf(e,t,i),o!==null&&Mf(e,t,o)}var he=function(e){return e[e.NONE=0]="NONE",e[e.HTML=1]="HTML",e[e.STYLE=2]="STYLE",e[e.SCRIPT=3]="SCRIPT",e[e.URL=4]="URL",e[e.RESOURCE_URL=5]="RESOURCE_URL",e}(he||{});function Nf(e){let t=Af();return t?ho(t.sanitize(he.HTML,e)||""):Ze(e,"HTML")?ho(Ce(e)):Rl(Hd(),ps(e))}function Af(){let e=S();return e&&e[ie].sanitizer}var Vf="ng-template";function Rf(e){return e.type===4&&e.value!==Vf}function Sr(e){return(e&1)===0}function bo(e,t){return e?":not("+t.trim()+")":t}function Lf(e){let t=e[0],n=1,r=2,i="",o=!1;for(;n0?'="'+l+'"':"")+"]"}else r&8?i+="."+s:r&4&&(i+=" "+s);else i!==""&&!Sr(s)&&(t+=bo(o,i),i=""),r=s,o=o||!Sr(r);n++}return i!==""&&(t+=bo(o,i)),t}function jf(e){return e.map(Lf).join(",")}function Ff(e){let t=[],n=[],r=1,i=2;for(;r=0;o--){let s=n[o],l=s.parentNode;s===t?(n.splice(o,1),ct.add(s),s.dispatchEvent(new CustomEvent("animationend",{detail:{cancel:!0}}))):(i&&s===i||l&&r&&l!==r)&&(n.splice(o,1),s.dispatchEvent(new CustomEvent("animationend",{detail:{cancel:!0}})),s.parentNode?.removeChild(s))}}function Zf(e,t){let n=Ir.get(e);n?n.includes(t)||n.push(t):Ir.set(e,[t])}var _t=new Set,Ul=function(e){return e[e.CHANGE_DETECTION=0]="CHANGE_DETECTION",e[e.AFTER_NEXT_RENDER=1]="AFTER_NEXT_RENDER",e}(Ul||{}),Dn=new E(""),wo=new Set;function st(e){wo.has(e)||(wo.add(e),performance?.mark?.("mark_feature_usage",{detail:{feature:e}}))}var $f=(()=>{class e{impl=null;execute(){this.impl?.execute()}static \u0275prov=D({token:e,providedIn:"root",factory:()=>new e})}return e})(),Zl=new E("",{factory:()=>({queue:new Set,isScheduled:!1,scheduler:null,injector:b(ve)})});function $l(e,t,n){let r=e.get(Zl);if(Array.isArray(t))for(let i of t)r.queue.add(i),n?.detachedLeaveAnimationFns?.push(i);else r.queue.add(t),n?.detachedLeaveAnimationFns?.push(t);r.scheduler&&r.scheduler(e)}function Qf(e,t){let n=e.get(Zl);if(t.detachedLeaveAnimationFns){for(let r of t.detachedLeaveAnimationFns)n.queue.delete(r);t.detachedLeaveAnimationFns=void 0}}function Wf(e,t){for(let[n,r]of t)$l(e,r.animateFns)}function _o(e,t,n,r){let i=e?.[Me]?.enter;t!==null&&i&&i.has(n.index)&&Wf(r,i)}function We(e,t,n,r,i,o,s,l){if(i!=null){let a,u=!1;ae(i)?a=i:Ie(i)&&(u=!0,i=i[le]);let c=X(i);e===0&&r!==null?(_o(l,r,o,n),s==null?jl(t,r,c):cn(t,r,c,s||null,!0)):e===1&&r!==null?(_o(l,r,o,n),cn(t,r,c,s||null,!0),Uf(o,c)):e===2?(l?.[Me]?.leave?.has(o.index)&&Zf(o,c),ct.delete(c),Co(l,o,n,d=>{if(ct.has(c)){ct.delete(c);return}Fl(t,c,u,d)})):e===3&&(ct.delete(c),Co(l,o,n,()=>{t.destroyNode(c)})),a!=null&&lh(t,e,n,a,o,r,s)}}function Gf(e,t){Ql(e,t),t[le]=null,t[W]=null}function Yf(e,t,n,r,i,o){r[le]=i,r[W]=t,Pn(e,r,n,1,i,o)}function Ql(e,t){t[ie].changeDetectionScheduler?.notify(9),Pn(e,t,t[V],2,null,null)}function Kf(e){let t=e[bt];if(!t)return Gn(e[g],e);for(;t;){let n=null;if(Ie(t))n=t[bt];else{let r=t[T];r&&(n=r)}if(!n){for(;t&&!t[Z]&&t!==e;)Ie(t)&&Gn(t[g],t),t=t[A];t===null&&(t=e),Ie(t)&&Gn(t[g],t),n=t&&t[Z]}t=n}}function Ei(e,t){let n=e[Xe],r=n.indexOf(t);n.splice(r,1)}function Mn(e,t){if(it(t))return;let n=t[V];n.destroyNode&&Pn(e,t,n,3,null,null),Kf(t)}function Gn(e,t){if(it(t))return;let n=m(null);try{t[v]&=-129,t[v]|=256,t[q]&&yn(t[q]),eh(e,t),Jf(e,t),t[g].type===1&&t[V].destroy();let r=t[Oe];if(r!==null&&ae(t[A])){r!==t[A]&&Ei(r,t);let i=t[oe];i!==null&&i.detachView(e)}Cr(t)}finally{m(n)}}function Co(e,t,n,r){let i=e?.[Me];if(i==null||i.leave==null||!i.leave.has(t.index))return r(!1);e&&_t.add(e[be]),$l(n,()=>{if(i.leave&&i.leave.has(t.index)){let o=i.leave.get(t.index),s=[];if(o){for(let l=0;l{e[Me].running=void 0,_t.delete(e[be]),t(!0)});return}t(!1)}function Jf(e,t){let n=e.cleanup,r=t[Jt];if(n!==null)for(let s=0;s=0?r[l]():r[-l].unsubscribe(),s+=2}else{let l=r[n[s+1]];n[s].call(l)}r!==null&&(t[Jt]=null);let i=t[ge];if(i!==null){t[ge]=null;for(let s=0;sH&&ql(e,t,H,!1);let l=s?_.TemplateUpdateStart:_.TemplateCreateStart;x(l,i,n),n(r,i)}finally{Ve(o);let l=s?_.TemplateUpdateEnd:_.TemplateCreateEnd;x(l,i,n)}}function uh(e,t,n){ph(e,t,n),(n.flags&64)===64&&gh(e,t,n)}function Yl(e,t,n=ue){let r=t.localNames;if(r!==null){let i=t.index+1;for(let o=0;onull;function hh(e,t,n,r,i,o){if(e.type&3){let s=ue(e,t);r=o!=null?o(r,e.value||"",n):r,i.setProperty(s,n,r)}else e.type&12}function ph(e,t,n){let r=n.directiveStart,i=n.directiveEnd;wn(n)&&Bf(t,n,e.data[r+n.componentOffset]),e.firstCreatePass||un(n,t);let o=n.initialInputs;for(let s=r;s{Tt(e.lView)},consumerOnSignalRead(){this.lView[q]=this}});function Dh(e){let t=e[q]??Object.create(Mh);return t.lView=e,t}var Mh=Q($({},St),{consumerIsAlwaysLive:!0,kind:"template",consumerMarkedDirty:e=>{let t=Ae(e.lView);for(;t&&!Jl(t[g]);)t=Ae(t);t&&Vs(t)},consumerOnSignalRead(){this.lView[q]=this}});function Jl(e){return e.type!==2}function ea(e){if(e[De]===null)return;let t=!0;for(;t;){let n=!1;for(let r of e[De])r.dirty&&(n=!0,r.zone===null||Zone.current===r.zone?r.run():r.zone.run(()=>r.run()));t=n&&!!(e[v]&8192)}}var Ph=100;function ta(e,t=0){let n=e[ie].rendererFactory,r=!1;r||n.begin?.();try{Nh(e,t)}finally{r||n.end?.()}}function Nh(e,t){let n=zs();try{rn(!0),Or(e,t);let r=0;for(;nn(e);){if(r===Ph)throw new w(103,!1);r++,Or(e,1)}}finally{rn(n)}}function Ah(e,t,n,r){if(it(t))return;let i=t[v],o=!1,s=!1;hi(t);let l=!0,a=null,u=null;o||(Jl(e)?(u=Eh(t),a=Gt(u)):Wa()===null?(l=!1,u=Dh(t),a=Gt(u)):t[q]&&(yn(t[q]),t[q]=null));try{As(t),Mc(e.bindingStartIndex),n!==null&&Gl(e,t,n,2,r);let c=(i&3)===3;if(!o)if(c){let f=e.preOrderCheckHooks;f!==null&&qt(t,f,null)}else{let f=e.preOrderHooks;f!==null&&Ut(t,f,0,null),Qn(t,0)}if(s||Vh(t),ea(t),na(t,0),e.contentQueries!==null&&Ol(e,t),!o)if(c){let f=e.contentCheckHooks;f!==null&&qt(t,f)}else{let f=e.contentHooks;f!==null&&Ut(t,f,1),Qn(t,1)}Lh(e,t);let d=e.components;d!==null&&ia(t,d,0);let h=e.viewQuery;if(h!==null&&kr(2,h,r),!o)if(c){let f=e.viewCheckHooks;f!==null&&qt(t,f)}else{let f=e.viewHooks;f!==null&&Ut(t,f,2),Qn(t,2)}if(e.firstUpdatePass===!0&&(e.firstUpdatePass=!1),t[Un]){for(let f of t[Un])f();t[Un]=null}o||(Kl(t),t[v]&=-73)}catch(c){throw o||Tt(t),c}finally{u!==null&&(Gr(u,a),l&&Th(u)),pi()}}function na(e,t){for(let n=bl(e);n!==null;n=wl(n))for(let r=T;r0&&(e[n-1][Z]=r[Z]);let o=Xt(e,T+t);Gf(r[g],r);let s=o[oe];s!==null&&s.detachView(o[g]),r[A]=null,r[Z]=null,r[v]&=-129}return r}function jh(e,t,n,r){let i=T+r,o=n.length;r>0&&(n[i-1][Z]=t),r-1&&(xt(e,n),Xt(t,n))}this._attachedToViewContainer=!1}Mn(this._lView[g],this._lView)}onDestroy(e){Rs(this._lView,e)}markForCheck(){Di(this._cdRefInjectingView||this._lView,4)}detach(){this._lView[v]&=-129}reattach(){gr(this._lView),this._lView[v]|=128}detectChanges(){this._lView[v]|=1024,ta(this._lView)}checkNoChanges(){}attachToViewContainerRef(){if(this._appRef)throw new w(902,!1);this._attachedToViewContainer=!0}detachFromAppRef(){this._appRef=null;let e=tn(this._lView),t=this._lView[Oe];t!==null&&!e&&Ei(t,this._lView),Ql(this._lView[g],this._lView)}attachToAppRef(e){if(this._attachedToViewContainer)throw new w(902,!1);this._appRef=e;let t=tn(this._lView),n=this._lView[Oe];n!==null&&!t&&aa(n,this._lView),gr(this._lView)}},fn=(()=>{class e{_declarationLView;_declarationTContainer;elementRef;static __NG_ELEMENT_ID__=Fh;constructor(n,r,i){this._declarationLView=n,this._declarationTContainer=r,this.elementRef=i}get ssrId(){return this._declarationTContainer.tView?.ssrId||null}createEmbeddedView(n,r){return this.createEmbeddedViewImpl(n,r)}createEmbeddedViewImpl(n,r,i){let o=Nn(this._declarationLView,this._declarationTContainer,n,{embeddedViewInjector:r,dehydratedView:i});return new Mi(o)}}return e})();function Fh(){return Pi(ce(),S())}function Pi(e,t){return e.type&4?new fn(t,e,ot(e,t)):null}function Vn(e,t,n,r,i){let o=e.data[t];if(o===null)o=Hh(e,t,n,r,i),Nc()&&(o.flags|=32);else if(o.type&64){o.type=n,o.value=r,o.attrs=i;let s=Oc();o.injectorIndex=s===null?-1:s.injectorIndex}return Ot(o,!0),o}function Hh(e,t,n,r,i){let o=Fs(),s=Hs(),l=s?o:o&&o.parent,a=e.data[t]=Bh(e,l,n,t,r,i);return zh(e,a,o,s),a}function zh(e,t,n,r){e.firstChild===null&&(e.firstChild=t),n!==null&&(r?n.child==null&&t.parent!==null&&(n.child=t):n.next===null&&(n.next=t,t.prev=n))}function Bh(e,t,n,r,i,o){let s=t?t.injectorIndex:-1,l=0;return Ec()&&(l|=128),{type:n,index:r,insertBeforeIndex:null,injectorIndex:s,directiveStart:-1,directiveEnd:-1,directiveStylingLast:-1,componentOffset:-1,controlDirectiveIndex:-1,customControlIndex:-1,propertyBindings:null,flags:l,providerIndexes:0,value:i,attrs:o,mergedAttrs:null,localNames:null,initialInputs:null,inputs:null,hostDirectiveInputs:null,outputs:null,hostDirectiveOutputs:null,directiveToIndex:null,tView:null,next:null,prev:null,projectionNext:null,child:null,parent:t,projection:null,styles:null,stylesWithoutHost:null,residualStyles:void 0,classes:null,classesWithoutHost:null,residualClasses:void 0,classBindings:0,styleBindings:0}}function qh(e){let t=e[eo]??[],n=e[A][V],r=[];for(let i of t)i.data[qd]!==void 0?r.push(i):Uh(i,n);e[eo]=r}function Uh(e,t){let n=0,r=e.firstChild;if(r){let i=e.data[Bd];for(;nnull,$h=()=>null;function Dr(e,t){return Zh(e,t)}function ua(e,t,n){return $h(e,t,n)}var Qh=class{},ca=class{},Wh=class{resolveComponentFactory(e){throw new w(917,!1)}},Ni=class{static NULL=new Wh},Ai=class{},Gh=(()=>{class e{static \u0275prov=D({token:e,providedIn:"root",factory:()=>null})}return e})(),Yn={},Yh=class{injector;parentInjector;constructor(e,t){this.injector=e,this.parentInjector=t}get(e,t,n){let r=this.injector.get(e,Yn,n);return r!==Yn||t===Yn?r:this.parentInjector.get(e,t,n)}};function hn(e,t,n){let r=n?e.styles:null,i=n?e.classes:null,o=0;if(t!==null)for(let s=0;s0&&(n.directiveToIndex=new Map);for(let h=0;h0;){let n=e[--t];if(typeof n=="number"&&n<0)return n}return 0}function op(e,t,n){if(n){if(t.exportAs)for(let r=0;rr(X(M[e.index])):e.index;pp(p,t,n,o,l,f,!1)}}return u}function fp(e){return e.startsWith("animation")||e.startsWith("transition")}function hp(e,t,n,r){let i=e.cleanup;if(i!=null)for(let o=0;oa?l[a]:null}typeof s=="string"&&(o+=2)}return null}function pp(e,t,n,r,i,o,s){let l=t.firstCreatePass?js(t):null,a=Ls(n),u=a.length;a.push(i,o),l&&l.push(r,e,u,(u+1)*(s?-1:1))}var Mr=Symbol("BINDING");function gp(e){return e.debugInfo?.className||e.type.name||null}var mp=class extends Ni{ngModule;constructor(e){super(),this.ngModule=e}resolveComponentFactory(e){let t=gt(e);return new fa(t,this.ngModule)}};function vp(e){return Object.keys(e).map(t=>{let[n,r,i]=e[t],o={propName:n,templateName:t,isSignal:(r&On.SignalBased)!==0};return i&&(o.transform=i),o})}function yp(e){return Object.keys(e).map(t=>({propName:e[t],templateName:t}))}function bp(e,t,n){let r=t instanceof ve?t:t?.injector;return r&&e.getStandaloneInjector!==null&&(r=e.getStandaloneInjector(r)||r),r?new Yh(n,r):n}function wp(e){let t=e.get(Ai,null);if(t===null)throw new w(407,!1);let n=e.get(Gh,null),r=e.get(bi,null),i=e.get(Dn,null,{optional:!0});return{rendererFactory:t,sanitizer:n,changeDetectionScheduler:r,ngReflect:!1,tracingService:i}}function _p(e,t){let n=Cp(e);return Ll(t,n,n==="svg"?mc:n==="math"?vc:null)}function Cp(e){return(e.selectors[0][0]||"div").toLowerCase()}var fa=class extends ca{componentDef;ngModule;selector;componentType;ngContentSelectors;isBoundToModule;cachedInputs=null;cachedOutputs=null;get inputs(){return this.cachedInputs??=vp(this.componentDef.inputs),this.cachedInputs}get outputs(){return this.cachedOutputs??=yp(this.componentDef.outputs),this.cachedOutputs}constructor(e,t){super(),this.componentDef=e,this.ngModule=t,this.componentType=e.type,this.selector=jf(e.selectors),this.ngContentSelectors=e.ngContentSelectors??[],this.isBoundToModule=!!t}create(e,t,n,r,i,o){x(_.DynamicComponentStart);let s=m(null);try{let l=this.componentDef,a=bp(l,r||this.ngModule,e),u=wp(a),c=u.tracingService;return c&&c.componentCreate?c.componentCreate(gp(l),()=>this.createComponentRef(u,a,t,n,i,o)):this.createComponentRef(u,a,t,n,i,o)}finally{m(s)}}createComponentRef(e,t,n,r,i,o){let s=this.componentDef,l=xp(r,s,o,i),a=e.rendererFactory.createRenderer(null,s),u=r?ch(a,r,s.encapsulation,t):_p(s,a),c=o?.some(To)||i?.some(f=>typeof f!="function"&&f.bindings.some(To)),d=xi(null,l,null,512|zl(s),null,null,e,a,t,null,Tl(u,t,!0));d[H]=u,hi(d);let h=null;try{let f=lp(H,d,2,"#host",()=>l.directiveRegistry,!0,0);Hl(a,u,f),Je(u,d),uh(l,d,f),Gd(l,f,d),ap(l,f),n!==void 0&&Ep(f,this.ngContentSelectors,n),h=Ne(f.index,d),d[O]=h[O],Oi(l,d,null)}catch(f){throw h!==null&&Cr(h),Cr(d),f}finally{x(_.DynamicComponentEnd),pi()}return new Sp(this.componentType,d,!!c)}};function xp(e,t,n,r){let i=e?["ng-version","21.2.11"]:Ff(t.selectors[0]),o=null,s=null,l=0;if(n)for(let u of n)l+=u[Mr].requiredVars,u.create&&(u.targetIdx=0,(o??=[]).push(u)),u.update&&(u.targetIdx=0,(s??=[]).push(u));if(r)for(let u=0;u{if(n&1&&e)for(let r of e)r.create();if(n&2&&t)for(let r of t)r.update()}}function To(e){let t=e[Mr].kind;return t==="input"||t==="twoWay"}var Sp=class extends Qh{_rootLView;_hasInputBindings;instance;hostView;changeDetectorRef;componentType;location;previousInputValues=null;_tNode;constructor(e,t,n){super(),this._rootLView=t,this._hasInputBindings=n,this._tNode=ci(t[g],H),this.location=ot(this._tNode,t),this.instance=Ne(this._tNode.index,t)[O],this.hostView=this.changeDetectorRef=new Mi(t,void 0),this.componentType=e}setInput(e,t){this._hasInputBindings;let n=this._tNode;if(this.previousInputValues??=new Map,this.previousInputValues.has(e)&&Object.is(this.previousInputValues.get(e),t))return;let r=this._rootLView,i=_h(n,r[g],r,e,t);this.previousInputValues.set(e,t);let o=Ne(n.index,r);Di(o,1)}get injector(){return new pt(this._tNode,this._rootLView)}destroy(){this.hostView.destroy()}onDestroy(e){this.hostView.onDestroy(e)}};function Ep(e,t,n){let r=e.projection=[];for(let i=0;i{class e{static __NG_ELEMENT_ID__=Ip}return e})();function Ip(){let e=ce();return pa(e,S())}var Tp=class ha extends Ri{_lContainer;_hostTNode;_hostLView;constructor(t,n,r){super(),this._lContainer=t,this._hostTNode=n,this._hostLView=r}get element(){return ot(this._hostTNode,this._hostLView)}get injector(){return new pt(this._hostTNode,this._hostLView)}get parentInjector(){let t=_i(this._hostTNode,this._hostLView);if(ll(t)){let n=an(t,this._hostLView),r=ln(t),i=n[g].data[r+8];return new pt(i,n)}else return new pt(null,this._hostLView)}clear(){for(;this.length>0;)this.remove(this.length-1)}get(t){let n=Oo(this._lContainer);return n!==null&&n[t]||null}get length(){return this._lContainer.length-T}createEmbeddedView(t,n,r){let i,o;typeof r=="number"?i=r:r!=null&&(i=r.index,o=r.injector);let s=Dr(this._lContainer,t.ssrId),l=t.createEmbeddedViewImpl(n||{},o,s);return this.insertImpl(l,i,Ct(this._hostTNode,s)),l}createComponent(t,n,r,i,o,s,l){let a=t&&!hd(t),u;if(a)u=n;else{let I=n||{};u=I.index,r=I.injector,i=I.projectableNodes,o=I.environmentInjector||I.ngModuleRef,s=I.directives,l=I.bindings}let c=a?t:new fa(gt(t)),d=r||this.parentInjector;if(!o&&c.ngModule==null){let I=(a?d:this.parentInjector).get(ve,null);I&&(o=I)}let h=gt(c.componentType??{}),f=Dr(this._lContainer,h?.id??null),p=f?.firstChild??null,M=c.create(d,i,p,o,s,l);return this.insertImpl(M.hostView,u,Ct(this._hostTNode,f)),M}insert(t,n){return this.insertImpl(t,n,!0)}insertImpl(t,n,r){let i=t._lView;if(bc(i)){let l=this.indexOf(t);if(l!==-1)this.detach(l);else{let a=i[A],u=new ha(a,a[W],a[A]);u.detach(u.indexOf(t))}}let o=this._adjustIndex(n),s=this._lContainer;return An(s,i,o,r),t.attachToViewContainerRef(),ws(Kn(s),o,t),t}move(t,n){return this.insert(t,n)}indexOf(t){let n=Oo(this._lContainer);return n!==null?n.indexOf(t):-1}remove(t){let n=this._adjustIndex(t,-1),r=xt(this._lContainer,n);r&&(Xt(Kn(this._lContainer),n),Mn(r[g],r))}detach(t){let n=this._adjustIndex(t,-1),r=xt(this._lContainer,n);return r&&Xt(Kn(this._lContainer),n)!=null?new Mi(r):null}_adjustIndex(t,n=0){return t??this.length+n}};function Oo(e){return e[en]}function Kn(e){return e[en]||(e[en]=[])}function pa(e,t){let n,r=t[e.index];return ae(r)?n=r:(n=oa(r,t,null,e),t[e.index]=n,ki(t,n)),Dp(n,t,e,r),new Tp(n,e,t)}function Op(e,t){let n=e[V],r=n.createComment(""),i=ue(t,e),o=n.parentNode(i);return cn(n,o,r,n.nextSibling(i),!1),r}var Dp=Np,Mp=()=>!1;function Pp(e,t,n){return Mp(e,t,n)}function Np(e,t,n,r){if(e[Pe])return;let i;n.type&8?i=X(r):i=Op(t,n),e[Pe]=i}var Ap=class ga{queryList;matches=null;constructor(t){this.queryList=t}clone(){return new ga(this.queryList)}setDirty(){this.queryList.setDirty()}},Vp=class ma{queries;constructor(t=[]){this.queries=t}createEmbeddedView(t){let n=t.queries;if(n!==null){let r=t.contentQueries!==null?t.contentQueries[0]:n.length,i=[];for(let o=0;o0)r.push(s[l/2]);else{let u=o[l+1],c=t[-a];for(let d=T;dt.trim())}function Qp(e,t,n){e.queries===null&&(e.queries=new Lp),e.queries.track(new jp(t,n))}function Li(e,t){return e.queries.getByIndex(t)}function Wp(e,t){let n=e[g],r=Li(n,t);return r.crossesNgTemplate?Pr(n,e,t,[]):ba(n,e,r,t)}var Nr=class{},wa=class extends Nr{injector;componentFactoryResolver=new mp(this);instance=null;constructor(e){super();let t=new ui([...e.providers,{provide:Nr,useValue:this},{provide:Ni,useValue:this.componentFactoryResolver}],e.parent||ai(),e.debugName,new Set(["environment"]));this.injector=t,e.runEnvironmentInitializers&&t.resolveInjectorInitializers()}destroy(){this.injector.destroy()}onDestroy(e){this.injector.onDestroy(e)}};function Gp(e,t,n=null){return new wa({providers:e,parent:t,debugName:n,runEnvironmentInitializers:!0}).injector}var Yp=(()=>{class e{_injector;cachedInjectors=new Map;constructor(n){this._injector=n}getOrCreateStandaloneInjector(n){if(!n.standalone)return null;if(!this.cachedInjectors.has(n)){let r=ks(!1,n.type),i=r.length>0?Gp([r],this._injector,""):null;this.cachedInjectors.set(n,i)}return this.cachedInjectors.get(n)}ngOnDestroy(){try{for(let n of this.cachedInjectors.values())n!==null&&n.destroy()}finally{this.cachedInjectors.clear()}}static \u0275prov=D({token:e,providedIn:"environment",factory:()=>new e(C(ve))})}return e})();function Kp(e){return fd(()=>{let t=tg(e),n=Q($({},t),{decls:e.decls,vars:e.vars,template:e.template,consts:e.consts||null,ngContentSelectors:e.ngContentSelectors,onPush:e.changeDetection===vl.OnPush,directiveDefs:null,pipeDefs:null,dependencies:t.standalone&&e.dependencies||null,getStandaloneInjector:t.standalone?i=>i.get(Yp).getOrCreateStandaloneInjector(n):null,getExternalStyles:null,signals:e.signals??!1,data:e.data||{},encapsulation:e.encapsulation||re.Emulated,styles:e.styles||Te,_:null,schemas:e.schemas||null,tView:null,id:""});t.standalone&&st("NgStandalone"),ng(n);let r=e.dependencies;return n.directiveDefs=Do(r,Xp),n.pipeDefs=Do(r,qu),n.id=rg(n),n})}function Xp(e){return gt(e)||hs(e)}function Jp(e,t){if(e==null)return Ye;let n={};for(let r in e)if(e.hasOwnProperty(r)){let i=e[r],o,s,l,a;Array.isArray(i)?(l=i[0],o=i[1],s=i[2]??o,a=i[3]||null):(o=i,s=i,l=On.None,a=null),n[o]=[r,l,a],t[o]=s}return n}function eg(e){if(e==null)return Ye;let t={};for(let n in e)e.hasOwnProperty(n)&&(t[e[n]]=n);return t}function tg(e){let t={};return{type:e.type,providersResolver:null,viewProvidersResolver:null,factory:null,hostBindings:e.hostBindings||null,hostVars:e.hostVars||0,hostAttrs:e.hostAttrs||null,contentQueries:e.contentQueries||null,declaredInputs:t,inputConfig:e.inputs||Ye,exportAs:e.exportAs||null,standalone:e.standalone??!0,signals:e.signals===!0,selectors:e.selectors||Te,viewQuery:e.viewQuery||null,features:e.features||null,setInput:null,resolveHostDirectives:null,hostDirectives:null,controlDef:null,inputs:Jp(e.inputs,t),outputs:eg(e.outputs),debugInfo:null}}function ng(e){e.features?.forEach(t=>t(e))}function Do(e,t){return e?()=>{let n=typeof e=="function"?e():e,r=[];for(let i of n){let o=t(i);o!==null&&r.push(o)}return r}:null}function rg(e){let t=0,n=typeof e.consts=="function"?"":e.consts,r=[e.selectors,e.ngContentSelectors,e.hostVars,e.hostAttrs,n,e.vars,e.decls,e.encapsulation,e.standalone,e.signals,e.exportAs,JSON.stringify(e.inputs),JSON.stringify(e.outputs),Object.getOwnPropertyNames(e.type.prototype),!!e.contentQueries,!!e.viewQuery];for(let i of r.join("|"))t=Math.imul(31,t)+i.charCodeAt(0)<<0;return t+=2147483648,"c"+t}function ig(e,t,n,r,i,o,s,l){if(n.firstCreatePass){e.mergedAttrs=En(e.mergedAttrs,e.attrs);let c=e.tView=Ci(2,e,i,o,s,n.directiveRegistry,n.pipeRegistry,null,n.schemas,n.consts,null);n.queries!==null&&(n.queries.template(n,e),c.queries=n.queries.embeddedTView(e))}l&&(e.flags|=l),Ot(e,!1);let a=og(n,t,e,r);gi()&&Ii(n,t,a,e),Je(a,t);let u=oa(a,t,a,e);t[r+H]=u,ki(t,u),Pp(u,e,t)}function pn(e,t,n,r,i,o,s,l,a,u,c){let d=n+H,h;if(t.firstCreatePass){if(h=Vn(t,d,4,s||null,l||null),u!=null){let f=se(t.consts,u);h.localNames=[];for(let p=0;p{class e{resolve;reject;initialized=!1;done=!1;donePromise=new Promise((n,r)=>{this.resolve=n,this.reject=r});appInits=b(ug,{optional:!0})??[];injector=b(Cn);constructor(){}runInitializers(){if(this.initialized)return;let n=[];for(let i of this.appInits){let o=Ts(this.injector,i);if(_a(o))n.push(o);else if(ag(o)){let s=new Promise((l,a)=>{o.subscribe({complete:l,error:a})});n.push(s)}}let r=()=>{this.done=!0,this.resolve()};Promise.all(n).then(()=>{r()}).catch(i=>{this.reject(i)}),n.length===0&&r(),this.initialized=!0}static \u0275fac=function(n){return new(n||e)};static \u0275prov=D({token:e,factory:e.\u0275fac,providedIn:"root"})}return e})(),cg=new E("");function dg(){ru(()=>{let e="";throw new w(600,e)})}function fg(e){return e.isBoundToModule}var hg=10,Ar=(()=>{class e{_runningTick=!1;_destroyed=!1;_destroyListeners=[];_views=[];internalErrorHandler=b(Dt);afterRenderManager=b($f);zonelessEnabled=b(wi);rootEffectScheduler=b(el);dirtyFlags=0;tracingSnapshot=null;allTestViews=new Set;autoDetectTestViews=new Set;includeAllTestViews=!1;afterTick=new Et;get allViews(){return[...(this.includeAllTestViews?this.allTestViews:this.autoDetectTestViews).keys(),...this._views]}get destroyed(){return this._destroyed}componentTypes=[];components=[];internalPendingTask=b(kn);get isStable(){return this.internalPendingTask.hasPendingTasksObservable.pipe(Pu(n=>!n))}constructor(){b(Dn,{optional:!0})}whenStable(){let n;return new Promise(r=>{n=this.isStable.subscribe({next:i=>{i&&r()}})}).finally(()=>{n.unsubscribe()})}_injector=b(ve);_rendererFactory=null;get injector(){return this._injector}bootstrap(n,r){return this.bootstrapImpl(n,r)}bootstrapImpl(n,r,i=Cn.NULL){return this._injector.get(He).run(()=>{x(_.BootstrapComponentStart);let o=n instanceof ca;if(!this._injector.get(Ca).done){let h="";throw new w(405,h)}let s;o?s=n:s=this._injector.get(Ni).resolveComponentFactory(n),this.componentTypes.push(s.componentType);let l=fg(s)?void 0:this._injector.get(Nr),a=r||s.selector,u=s.create(i,[],a,l),c=u.location.nativeElement,d=u.injector.get(lg,null);return d?.registerApplication(c),u.onDestroy(()=>{this.detachView(u.hostView),$t(this.components,u),d?.unregisterApplication(c)}),this._loadComponent(u),x(_.BootstrapComponentEnd,u),u})}tick(){this.zonelessEnabled||(this.dirtyFlags|=1),this._tick()}_tick(){x(_.ChangeDetectionStart),this.tracingSnapshot!==null?this.tracingSnapshot.run(Ul.CHANGE_DETECTION,this.tickImpl):this.tickImpl()}tickImpl=()=>{if(this._runningTick)throw x(_.ChangeDetectionEnd),new w(101,!1);let n=m(null);try{this._runningTick=!0,this.synchronize()}finally{this._runningTick=!1,this.tracingSnapshot?.dispose(),this.tracingSnapshot=null,m(n),this.afterTick.next(),x(_.ChangeDetectionEnd)}};synchronize(){this._rendererFactory===null&&!this._injector.destroyed&&(this._rendererFactory=this._injector.get(Ai,null,{optional:!0}));let n=0;for(;this.dirtyFlags!==0&&n++nn(n))){this.dirtyFlags|=2;return}else this.dirtyFlags&=-8}attachView(n){let r=n;this._views.push(r),r.attachToAppRef(this)}detachView(n){let r=n;$t(this._views,r),r.detachFromAppRef()}_loadComponent(n){this.attachView(n.hostView);try{this.tick()}catch(r){this.internalErrorHandler(r)}this.components.push(n),this._injector.get(cg,[]).forEach(r=>r(n))}ngOnDestroy(){if(!this._destroyed)try{this._destroyListeners.forEach(n=>n()),this._views.slice().forEach(n=>n.destroy())}finally{this._destroyed=!0,this._views=[],this._destroyListeners=[]}}onDestroy(n){return this._destroyListeners.push(n),()=>$t(this._destroyListeners,n)}destroy(){if(this._destroyed)throw new w(406,!1);let n=this._injector;n.destroy&&!n.destroyed&&n.destroy()}get viewCount(){return this._views.length}static \u0275fac=function(n){return new(n||e)};static \u0275prov=D({token:e,factory:e.\u0275fac,providedIn:"root"})}return e})();function $t(e,t){let n=e.indexOf(t);n>-1&&e.splice(n,1)}var pg=class{destroy(e){}updateValue(e,t){}swap(e,t){let n=Math.min(e,t),r=Math.max(e,t),i=this.detach(r);if(r-n>1){let o=this.detach(n);this.attach(n,i),this.attach(r,o)}else this.attach(n,i)}move(e,t){this.attach(t,this.detach(e))}};function Xn(e,t,n,r,i){return e===n&&Object.is(t,r)?1:Object.is(i(e,t),i(n,r))?-1:0}function gg(e,t,n,r){let i,o,s=0,l=e.length-1,a;if(Array.isArray(t)){m(r);let u=t.length-1;for(m(null);s<=l&&s<=u;){let c=e.at(s),d=t[s],h=Xn(s,c,s,d,n);if(h!==0){h<0&&e.updateValue(s,d),s++;continue}let f=e.at(l),p=t[u],M=Xn(l,f,u,p,n);if(M!==0){M<0&&e.updateValue(l,p),l--,u--;continue}let I=n(s,c),J=n(l,f),lt=n(s,d);if(Object.is(lt,J)){let Rn=n(u,p);Object.is(Rn,I)?(e.swap(s,l),e.updateValue(l,p),u--,l--):e.move(l,s),e.updateValue(s,d),s++;continue}if(i??=new No,o??=Po(e,s,l,n),Vr(e,i,s,lt))e.updateValue(s,d),s++,l++;else if(o.has(lt))i.set(I,e.detach(s)),l--;else{let Rn=e.create(s,t[s]);e.attach(s,Rn),s++,l++}}for(;s<=u;)Mo(e,i,n,s,t[s]),s++}else if(t!=null){m(r);let u=t[Symbol.iterator]();m(null);let c=u.next();for(;!c.done&&s<=l;){let d=e.at(s),h=c.value,f=Xn(s,d,s,h,n);if(f!==0)f<0&&e.updateValue(s,h),s++,c=u.next();else{i??=new No,o??=Po(e,s,l,n);let p=n(s,h);if(Vr(e,i,s,p))e.updateValue(s,h),s++,l++,c=u.next();else if(!o.has(p))e.attach(s,e.create(s,h)),s++,l++,c=u.next();else{let M=n(s,d);i.set(M,e.detach(s)),l--}}}for(;!c.done;)Mo(e,i,n,e.length,c.value),c=u.next()}for(;s<=l;)e.destroy(e.detach(l--));i?.forEach(u=>{e.destroy(u)})}function Vr(e,t,n,r){return t!==void 0&&t.has(r)?(e.attach(n,t.get(r)),t.delete(r),!0):!1}function Mo(e,t,n,r,i){if(Vr(e,t,r,n(r,i)))e.updateValue(r,i);else{let o=e.create(r,i);e.attach(r,o)}}function Po(e,t,n,r){let i=new Set;for(let o=t;o<=n;o++)i.add(r(o,e.at(o)));return i}var No=class{kvMap=new Map;_vMap=void 0;has(e){return this.kvMap.has(e)}delete(e){if(!this.has(e))return!1;let t=this.kvMap.get(e);return this._vMap!==void 0&&this._vMap.has(t)?(this.kvMap.set(e,this._vMap.get(t)),this._vMap.delete(t)):this.kvMap.delete(e),!0}get(e){return this.kvMap.get(e)}set(e,t){if(this.kvMap.has(e)){let n=this.kvMap.get(e);this._vMap===void 0&&(this._vMap=new Map);let r=this._vMap;for(;r.has(n);)n=r.get(n);r.set(n,t)}else this.kvMap.set(e,t)}forEach(e){for(let[t,n]of this.kvMap)if(e(n,t),this._vMap!==void 0){let r=this._vMap;for(;r.has(n);)n=r.get(n),e(n,t)}}};function Rr(e,t,n,r,i,o,s,l){st("NgControlFlow");let a=S(),u=U(),c=se(u.consts,o);return pn(a,u,e,t,n,r,i,c,256,s,l),xa}function xa(e,t,n,r,i,o,s,l){st("NgControlFlow");let a=S(),u=U(),c=se(u.consts,o);return pn(a,u,e,t,n,r,i,c,512,s,l),xa}function Lr(e,t){st("NgControlFlow");let n=S(),r=_n(),i=n[r]!==_e?n[r]:-1,o=i!==-1?gn(n,H+i):void 0,s=0;if(Nt(n,r,e)){let l=m(null);try{if(o!==void 0&&la(o,s),e!==-1){let a=H+e,u=gn(n,a),c=jr(n[g],a),d=ua(u,c,n),h=Nn(n,c,t,{dehydratedView:d});An(u,h,s,Ct(c,d))}}finally{m(l)}}else if(o!==void 0){let l=sa(o,s);l!==void 0&&(l[O]=t)}}var mg=class{lContainer;$implicit;$index;constructor(e,t,n){this.lContainer=e,this.$implicit=t,this.$index=n}get $count(){return this.lContainer.length-T}};function Ao(e,t){return t}var vg=class{hasEmptyBlock;trackByFn;liveCollection;constructor(e,t,n){this.hasEmptyBlock=e,this.trackByFn=t,this.liveCollection=n}};function Vo(e,t,n,r,i,o,s,l,a,u,c,d,h){st("NgControlFlow");let f=S(),p=U(),M=a!==void 0,I=S(),J=l?s.bind(I[K][O]):s,lt=new vg(M,J);I[H+e]=lt,pn(f,p,e+1,t,n,r,i,se(p.consts,o),256),M&&pn(f,p,e+2,a,u,c,d,se(p.consts,h),512)}var yg=class extends pg{lContainer;hostLView;templateTNode;operationsCounter=void 0;needsIndexUpdate=!1;constructor(e,t,n){super(),this.lContainer=e,this.hostLView=t,this.templateTNode=n}get length(){return this.lContainer.length-T}at(e){return this.getLView(e)[O].$implicit}attach(e,t){let n=t[yt];this.needsIndexUpdate||=e!==this.length,An(this.lContainer,t,e,Ct(this.templateTNode,n)),bg(this.lContainer,e)}detach(e){return this.needsIndexUpdate||=e!==this.length-1,wg(this.lContainer,e),_g(this.lContainer,e)}create(e,t){let n=Dr(this.lContainer,this.templateTNode.tView.ssrId);return Nn(this.hostLView,this.templateTNode,new mg(this.lContainer,t,e),{dehydratedView:n})}destroy(e){Mn(e[g],e)}updateValue(e,t){this.getLView(e)[O].$implicit=t}reset(){this.needsIndexUpdate=!1}updateIndexes(){if(this.needsIndexUpdate)for(let e=0;e0){let o=r[ye];Qf(o,i),_t.delete(r[be]),i.detachedLeaveAnimationFns=void 0}}function wg(e,t){if(e.length<=T)return;let n=T+t,r=e[n],i=r?r[Me]:void 0;i&&i.leave&&i.leave.size>0&&(i.detachedLeaveAnimationFns=[])}function _g(e,t){return xt(e,t)}function Cg(e,t){return sa(e,t)}function jr(e,t){return ci(e,t)}function P(e,t,n,r){let i=S(),o=i[g],s=e+H,l=o.firstCreatePass?up(s,o,2,t,n,r):o.data[s];return yh(l,i,e,t,xg),r!=null&&Yl(i,l),P}function N(){let e=ce(),t=bh(e);return Ic(t)&&Tc(),Sc(),N}function ka(e,t,n,r){return P(e,t,n,r),N(),ka}var xg=(e,t,n,r,i)=>(mi(!0),Ll(t[V],r,Hc()));function Sa(){return S()}function te(e,t,n){let r=S(),i=_n();if(Nt(r,i,t)){let o=U(),s=Fc();hh(s,r,e,t,r[V],n)}return te}var mn="en-US",kg=mn;function Sg(e){typeof e=="string"&&(kg=e.toLowerCase().replace(/_/g,"-"))}function xe(e,t,n){let r=S(),i=U(),o=ce();return(o.type&3||n)&&dp(o,i,r,n,r[V],e,t,cp(o,r,t)),xe}function ee(e=1){return jc(e)}function Ea(e,t,n){return Zp(e,t,n),Ea}function Eg(e){let t=S(),n=U(),r=Bs();fi(r+1);let i=Li(n,r);if(e.dirty&&yc(t)===((i.metadata.flags&2)===2)){if(i.matches===null)e.reset([]);else{let o=Wp(t,r);e.reset(o,Nd),e.notifyOnChanges()}return!0}return!1}function Ig(){return qp(S(),Bs())}function jt(e,t){return e<<17|t<<2}function Re(e){return e>>17&32767}function Tg(e){return(e&2)==2}function Og(e,t){return e&131071|t<<17}function Fr(e){return e|2}function et(e){return(e&131068)>>2}function Jn(e,t){return e&-131069|t<<2}function Dg(e){return(e&1)===1}function Hr(e){return e|1}function Mg(e,t,n,r,i,o){let s=o?t.classBindings:t.styleBindings,l=Re(s),a=et(s);e[r]=n;let u=!1,c;if(Array.isArray(n)){let d=n;c=d[1],(c===null||It(d,c)>0)&&(u=!0)}else c=n;if(i)if(a!==0){let d=Re(e[l+1]);e[r+1]=jt(d,l),d!==0&&(e[d+1]=Jn(e[d+1],r)),e[l+1]=Og(e[l+1],r)}else e[r+1]=jt(l,0),l!==0&&(e[l+1]=Jn(e[l+1],r)),l=r;else e[r+1]=jt(a,0),l===0?l=r:e[a+1]=Jn(e[a+1],r),a=r;u&&(e[r+1]=Fr(e[r+1])),Lo(e,c,r,!0),Lo(e,c,r,!1),Pg(t,c,e,r,o),s=jt(l,a),o?t.classBindings=s:t.styleBindings=s}function Pg(e,t,n,r,i){let o=i?e.residualClasses:e.residualStyles;o!=null&&typeof t=="string"&&It(o,t)>=0&&(n[r+1]=Hr(n[r+1]))}function Lo(e,t,n,r){let i=e[n+1],o=t===null,s=r?Re(i):et(i),l=!1;for(;s!==0&&(l===!1||o);){let a=e[s],u=e[s+1];Ng(a,t)&&(l=!0,e[s+1]=r?Hr(u):Fr(u)),s=r?Re(u):et(u)}l&&(e[n+1]=r?Fr(i):Hr(i))}function Ng(e,t){return e===null||t==null||(Array.isArray(e)?e[1]:e)===t?!0:Array.isArray(e)&&typeof t=="string"?It(e,t)>=0:!1}function Ia(e,t){return Ag(e,t,null,!0),Ia}function Ag(e,t,n,r){let i=S(),o=U(),s=Pc(2);if(o.firstUpdatePass&&Rg(o,e,s,r),t!==_e&&Nt(i,s,t)){let l=o.data[Fe()];zg(o,l,i,i[V],e,i[s+1]=Bg(t,n),r,s)}}function Vg(e,t){return t>=e.expandoStartIndex}function Rg(e,t,n,r){let i=e.data;if(i[n+1]===null){let o=i[Fe()],s=Vg(e,n);qg(o,r)&&t===null&&!s&&(t=!1),t=Lg(i,o,t,r),Mg(i,o,t,n,s,r)}}function Lg(e,t,n,r){let i=Rc(e),o=r?t.residualClasses:t.residualStyles;if(i===null)(r?t.classBindings:t.styleBindings)===0&&(n=er(null,e,t,n,r),n=kt(n,t.attrs,r),o=null);else{let s=t.directiveStylingLast;if(s===-1||e[s]!==i)if(n=er(i,e,t,n,r),o===null){let l=jg(e,t,r);l!==void 0&&Array.isArray(l)&&(l=er(null,e,t,l[1],r),l=kt(l,t.attrs,r),Fg(e,t,r,l))}else o=Hg(e,t,r)}return o!==void 0&&(r?t.residualClasses=o:t.residualStyles=o),n}function jg(e,t,n){let r=n?t.classBindings:t.styleBindings;if(et(r)!==0)return e[Re(r)]}function Fg(e,t,n,r){let i=n?t.classBindings:t.styleBindings;e[Re(i)]=r}function Hg(e,t,n){let r,i=t.directiveEnd;for(let o=1+t.directiveStylingLast;o0;){let a=e[i],u=Array.isArray(a),c=u?a[1]:a,d=c===null,h=n[i+1];h===_e&&(h=d?Te:void 0);let f=d?Bn(h,r):c===r?h:void 0;if(u&&!vn(f)&&(f=Bn(a,r)),vn(f)&&(l=f,s))return l;let p=e[i+1];i=s?Re(p):et(p)}if(t!==null){let a=o?t.residualClasses:t.residualStyles;a!=null&&(l=Bn(a,r))}return l}function vn(e){return e!==void 0}function Bg(e,t){return e==null||e===""||(typeof t=="string"?e=e+t:typeof e=="object"&&(e=us(Ce(e)))),e}function qg(e,t){return(e.flags&(t?8:16))!==0}function B(e,t=""){let n=S(),r=U(),i=e+H,o=r.firstCreatePass?Vn(r,i,1,t,null):r.data[i],s=Ug(r,n,o,t);n[i]=s,gi()&&Ii(r,n,s,o),Ot(o,!1)}var Ug=(e,t,n,r)=>(mi(!0),Of(t[V],r));function Zg(e,t,n,r=""){return Nt(e,_n(),n)?t+ps(n)+r:_e}function Le(e){return Ta("",e),Le}function Ta(e,t,n){let r=S(),i=Zg(r,e,t,n);return i!==_e&&$g(r,Fe(),i),Ta}function $g(e,t,n){let r=Ns(t,e);Df(e[V],r,n)}function Fo(e,t,n){let r=U();r.firstCreatePass&&Oa(t,r.data,r.blueprint,rt(e),n)}function Oa(e,t,n,r,i){if(e=F(e),Array.isArray(e))for(let o=0;o>20;if(Ke(e)||!e.multi){let f=new Mt(u,i,Vi,null),p=nr(a,t,i?c:c+h,d);p===-1?(_r(un(l,s),o,a),tr(o,e,t.length),t.push(a),l.directiveStart++,l.directiveEnd++,i&&(l.providerIndexes+=1048576),n.push(f),s.push(f)):(n[p]=f,s[p]=f)}else{let f=nr(a,t,c+h,d),p=nr(a,t,c,c+h),M=f>=0&&n[f],I=p>=0&&n[p];if(i&&!I||!i&&!M){_r(un(l,s),o,a);let J=Gg(i?Wg:Qg,n.length,i,r,u,e);!i&&I&&(n[p].providerFactory=J),tr(o,e,t.length,0),t.push(a),l.directiveStart++,l.directiveEnd++,i&&(l.providerIndexes+=1048576),n.push(J),s.push(J)}else{let J=Da(n[i?p:f],u,!i&&r);tr(o,e,f>-1?f:p,J)}!i&&r&&I&&n[p].componentProviders++}}}function tr(e,t,n,r){let i=Ke(t),o=uc(t);if(i||o){let s=(o?F(t.useClass):t).prototype.ngOnDestroy;if(s){let l=e.destroyHooks||(e.destroyHooks=[]);if(!i&&t.multi){let a=l.indexOf(n);a===-1?l.push(n,[r,s]):l[a+1].push(r,s)}else l.push(n,s)}}}function Da(e,t,n){return n&&e.componentProviders++,e.multi.push(t)-1}function nr(e,t,n,r){for(let i=n;i{n.providersResolver=(r,i)=>Fo(r,i?i(e):e,!1),t&&(n.viewProvidersResolver=(r,i)=>Fo(r,i?i(t):t,!0))}}var Kg=(()=>{class e{applicationErrorHandler=b(Dt);appRef=b(Ar);taskService=b(kn);ngZone=b(He);zonelessEnabled=b(wi);tracing=b(Dn,{optional:!0});zoneIsDefined=typeof Zone<"u"&&!!Zone.root.run;schedulerTickApplyArgs=[{data:{__scheduler_tick__:!0}}];subscriptions=new me;angularZoneId=this.zoneIsDefined?this.ngZone._inner?.get(sn):null;scheduleInRootZone=!this.zonelessEnabled&&this.zoneIsDefined&&(b(od,{optional:!0})??!1);cancelScheduledCallback=null;useMicrotaskScheduler=!1;runningTick=!1;pendingRenderTaskId=null;constructor(){this.subscriptions.add(this.appRef.afterTick.subscribe(()=>{let n=this.taskService.add();if(!this.runningTick&&(this.cleanup(),!this.zonelessEnabled||this.appRef.includeAllTestViews)){this.taskService.remove(n);return}this.switchToMicrotaskScheduler(),this.taskService.remove(n)})),this.subscriptions.add(this.ngZone.onUnstable.subscribe(()=>{this.runningTick||this.cleanup()}))}switchToMicrotaskScheduler(){this.ngZone.runOutsideAngular(()=>{let n=this.taskService.add();this.useMicrotaskScheduler=!0,queueMicrotask(()=>{this.useMicrotaskScheduler=!1,this.taskService.remove(n)})})}notify(n){if(!this.zonelessEnabled&&n===5)return;switch(n){case 0:{this.appRef.dirtyFlags|=2;break}case 3:case 2:case 4:case 5:case 1:{this.appRef.dirtyFlags|=4;break}case 6:{this.appRef.dirtyFlags|=2;break}case 12:{this.appRef.dirtyFlags|=16;break}case 13:{this.appRef.dirtyFlags|=2;break}case 11:break;default:this.appRef.dirtyFlags|=8}if(this.appRef.tracingSnapshot=this.tracing?.snapshot(this.appRef.tracingSnapshot)??null,!this.shouldScheduleTick())return;let r=this.useMicrotaskScheduler?$c:Ks;this.pendingRenderTaskId=this.taskService.add(),this.scheduleInRootZone?this.cancelScheduledCallback=Zone.root.run(()=>r(()=>this.tick())):this.cancelScheduledCallback=this.ngZone.runOutsideAngular(()=>r(()=>this.tick()))}shouldScheduleTick(){return!(this.appRef.destroyed||this.pendingRenderTaskId!==null||this.runningTick||this.appRef._runningTick||!this.zonelessEnabled&&this.zoneIsDefined&&Zone.current.get(sn+this.angularZoneId))}tick(){if(this.runningTick||this.appRef.destroyed)return;if(this.appRef.dirtyFlags===0){this.cleanup();return}!this.zonelessEnabled&&this.appRef.dirtyFlags&7&&(this.appRef.dirtyFlags|=1);let n=this.taskService.add();try{this.ngZone.run(()=>{this.runningTick=!0,this.appRef._tick()},void 0,this.schedulerTickApplyArgs)}catch(r){this.applicationErrorHandler(r)}finally{this.taskService.remove(n),this.cleanup()}}ngOnDestroy(){this.subscriptions.unsubscribe(),this.cleanup()}cleanup(){if(this.runningTick=!1,this.cancelScheduledCallback?.(),this.cancelScheduledCallback=null,this.pendingRenderTaskId!==null){let n=this.pendingRenderTaskId;this.pendingRenderTaskId=null,this.taskService.remove(n)}}static \u0275fac=function(n){return new(n||e)};static \u0275prov=D({token:e,factory:e.\u0275fac,providedIn:"root"})}return e})();function Xg(){return st("NgZoneless"),oi([...Ma(),[]])}function Ma(){return[{provide:bi,useExisting:Kg},{provide:He,useClass:Kc},{provide:wi,useValue:!0}]}function Jg(){return typeof $localize<"u"&&$localize.locale||mn}var Pa=new E("",{factory:()=>b(Pa,{optional:!0,skipSelf:!0})||Jg()});function Be(e,t){return eu(e,t?.equal)}var Br=new E(""),em=new E("");function at(e){return!e.moduleRef}function tm(e){let t=at(e)?e.r3Injector:e.moduleRef.injector,n=t.get(He);return n.run(()=>{at(e)?e.r3Injector.resolveInjectorInitializers():e.moduleRef.resolveInjectorInitializers();let r=t.get(Dt),i;if(n.runOutsideAngular(()=>{i=n.onError.subscribe({next:r})}),at(e)){let o=()=>t.destroy(),s=e.platformInjector.get(Br);s.add(o),t.onDestroy(()=>{i.unsubscribe(),s.delete(o)})}else{let o=()=>e.moduleRef.destroy(),s=e.platformInjector.get(Br);s.add(o),e.moduleRef.onDestroy(()=>{$t(e.allPlatformModules,e.moduleRef),i.unsubscribe(),s.delete(o)})}return rm(r,n,()=>{let o=t.get(kn),s=o.add(),l=t.get(Ca);return l.runInitializers(),l.donePromise.then(()=>{let a=t.get(Pa,mn);if(Sg(a||mn),!t.get(em,!0))return at(e)?t.get(Ar):(e.allPlatformModules.push(e.moduleRef),e.moduleRef);if(at(e)){let u=t.get(Ar);return e.rootComponent!==void 0&&u.bootstrap(e.rootComponent),u}else return nm?.(e.moduleRef,e.allPlatformModules),e.moduleRef}).finally(()=>{o.remove(s)})})})}var nm;function rm(e,t,n){try{let r=n();return _a(r)?r.catch(i=>{throw t.runOutsideAngular(()=>e(i)),i}):r}catch(r){throw t.runOutsideAngular(()=>e(r)),r}}var Qt=null;function im(e=[],t){return Cn.create({name:t,providers:[{provide:li,useValue:"platform"},{provide:Br,useValue:new Set([()=>Qt=null])},...e]})}function om(e=[]){if(Qt)return Qt;let t=im(e);return Qt=t,dg(),sm(t),t}function sm(e){let t=e.get(xl,null);Ts(e,()=>{t?.forEach(n=>n())})}var lm=1e4,Wm=lm-1e3;function am(e){let{rootComponent:t,appProviders:n,platformProviders:r,platformRef:i}=e;x(_.BootstrapApplicationStart);try{let o=i?.injector??om(r),s=[Ma(),ed,...n||[]],l=new wa({providers:s,parent:o,debugName:"",runEnvironmentInitializers:!1});return tm({r3Injector:l.injector,platformInjector:o,rootComponent:t})}catch(o){return Promise.reject(o)}finally{x(_.BootstrapApplicationEnd)}}var Na=null;function Aa(){return Na}function um(e){Na??=e}var cm=class{};function dm(e,t){t=encodeURIComponent(t);for(let n of e.split(";")){let r=n.indexOf("="),[i,o]=r==-1?[n,""]:[n.slice(0,r),n.slice(r+1)];if(i.trim()===t)return decodeURIComponent(o)}return null}var fm=class{},hm="browser",Va=class{_doc;constructor(e){this._doc=e}manager},qr=(()=>{class e extends Va{constructor(n){super(n)}supports(n){return!0}addEventListener(n,r,i,o){return n.addEventListener(r,i,o),()=>this.removeEventListener(n,r,i,o)}removeEventListener(n,r,i,o){return n.removeEventListener(r,i,o)}static \u0275fac=function(n){return new(n||e)(C(we))};static \u0275prov=D({token:e,factory:e.\u0275fac})}return e})(),Ur=new E(""),Ra=(()=>{class e{_zone;_plugins;_eventNameToPlugin=new Map;constructor(n,r){this._zone=r,n.forEach(s=>{s.manager=this});let i=n.filter(s=>!(s instanceof qr));this._plugins=i.slice().reverse();let o=n.find(s=>s instanceof qr);o&&this._plugins.push(o)}addEventListener(n,r,i,o){return this._findPluginFor(r).addEventListener(n,r,i,o)}getZone(){return this._zone}_findPluginFor(n){let r=this._eventNameToPlugin.get(n);if(r)return r;if(r=this._plugins.find(i=>i.supports(n)),!r)throw new w(5101,!1);return this._eventNameToPlugin.set(n,r),r}static \u0275fac=function(n){return new(n||e)(C(Ur),C(He))};static \u0275prov=D({token:e,factory:e.\u0275fac})}return e})(),rr="ng-app-id";function Ho(e){for(let t of e)t.remove()}function zo(e,t){let n=t.createElement("style");return n.textContent=e,n}function pm(e,t,n,r){let i=e.head?.querySelectorAll(`style[${rr}="${t}"],link[${rr}="${t}"]`);if(i)for(let o of i)o.removeAttribute(rr),o instanceof HTMLLinkElement?r.set(o.href.slice(o.href.lastIndexOf("/")+1),{usage:0,elements:[o]}):o.textContent&&n.set(o.textContent,{usage:0,elements:[o]})}function Zr(e,t){let n=t.createElement("link");return n.setAttribute("rel","stylesheet"),n.setAttribute("href",e),n}var La=(()=>{class e{doc;appId;nonce;inline=new Map;external=new Map;hosts=new Set;constructor(n,r,i,o={}){this.doc=n,this.appId=r,this.nonce=i,pm(n,r,this.inline,this.external),this.hosts.add(n.head)}addStyles(n,r){for(let i of n)this.addUsage(i,this.inline,zo);r?.forEach(i=>this.addUsage(i,this.external,Zr))}removeStyles(n,r){for(let i of n)this.removeUsage(i,this.inline);r?.forEach(i=>this.removeUsage(i,this.external))}addUsage(n,r,i){let o=r.get(n);o?o.usage++:r.set(n,{usage:1,elements:[...this.hosts].map(s=>this.addElement(s,i(n,this.doc)))})}removeUsage(n,r){let i=r.get(n);i&&(i.usage--,i.usage<=0&&(Ho(i.elements),r.delete(n)))}ngOnDestroy(){for(let[,{elements:n}]of[...this.inline,...this.external])Ho(n);this.hosts.clear()}addHost(n){this.hosts.add(n);for(let[r,{elements:i}]of this.inline)i.push(this.addElement(n,zo(r,this.doc)));for(let[r,{elements:i}]of this.external)i.push(this.addElement(n,Zr(r,this.doc)))}removeHost(n){this.hosts.delete(n)}addElement(n,r){return this.nonce&&r.setAttribute("nonce",this.nonce),n.appendChild(r)}static \u0275fac=function(n){return new(n||e)(C(we),C(Cl),C(Sl,8),C(kl))};static \u0275prov=D({token:e,factory:e.\u0275fac})}return e})(),ir={svg:"http://www.w3.org/2000/svg",xhtml:"http://www.w3.org/1999/xhtml",xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/",math:"http://www.w3.org/1998/Math/MathML"},ji=/%COMP%/g,ja="%COMP%",gm=`_nghost-${ja}`,mm=`_ngcontent-${ja}`,vm=!0,ym=new E("",{factory:()=>vm});function bm(e){return mm.replace(ji,e)}function wm(e){return gm.replace(ji,e)}function Fa(e,t){return t.map(n=>n.replace(ji,e))}var Bo=(()=>{class e{eventManager;sharedStylesHost;appId;removeStylesOnCompDestroy;doc;ngZone;nonce;tracingService;rendererByCompId=new Map;defaultRenderer;constructor(n,r,i,o,s,l,a=null,u=null){this.eventManager=n,this.sharedStylesHost=r,this.appId=i,this.removeStylesOnCompDestroy=o,this.doc=s,this.ngZone=l,this.nonce=a,this.tracingService=u,this.defaultRenderer=new Fi(n,s,l,this.tracingService)}createRenderer(n,r){if(!n||!r)return this.defaultRenderer;let i=this.getOrCreateRenderer(n,r);return i instanceof Zo?i.applyToHost(n):i instanceof $r&&i.applyStyles(),i}getOrCreateRenderer(n,r){let i=this.rendererByCompId,o=i.get(r.id);if(!o){let s=this.doc,l=this.ngZone,a=this.eventManager,u=this.sharedStylesHost,c=this.removeStylesOnCompDestroy,d=this.tracingService;switch(r.encapsulation){case re.Emulated:o=new Zo(a,u,r,this.appId,c,s,l,d);break;case re.ShadowDom:return new Uo(a,n,r,s,l,this.nonce,d,u);case re.ExperimentalIsolatedShadowDom:return new Uo(a,n,r,s,l,this.nonce,d);default:o=new $r(a,u,r,c,s,l,d);break}i.set(r.id,o)}return o}ngOnDestroy(){this.rendererByCompId.clear()}componentReplaced(n){this.rendererByCompId.delete(n)}static \u0275fac=function(n){return new(n||e)(C(Ra),C(La),C(Cl),C(ym),C(we),C(He),C(Sl),C(Dn,8))};static \u0275prov=D({token:e,factory:e.\u0275fac})}return e})(),Fi=class{eventManager;doc;ngZone;tracingService;data=Object.create(null);throwOnSyntheticProps=!0;constructor(e,t,n,r){this.eventManager=e,this.doc=t,this.ngZone=n,this.tracingService=r}destroy(){}destroyNode=null;createElement(e,t){return t?this.doc.createElementNS(ir[t]||t,e):this.doc.createElement(e)}createComment(e){return this.doc.createComment(e)}createText(e){return this.doc.createTextNode(e)}appendChild(e,t){(qo(e)?e.content:e).appendChild(t)}insertBefore(e,t,n){e&&(qo(e)?e.content:e).insertBefore(t,n)}removeChild(e,t){t.remove()}selectRootElement(e,t){let n=typeof e=="string"?this.doc.querySelector(e):e;if(!n)throw new w(-5104,!1);return t||(n.textContent=""),n}parentNode(e){return e.parentNode}nextSibling(e){return e.nextSibling}setAttribute(e,t,n,r){if(r){t=r+":"+t;let i=ir[r];i?e.setAttributeNS(i,t,n):e.setAttribute(t,n)}else e.setAttribute(t,n)}removeAttribute(e,t,n){if(n){let r=ir[n];r?e.removeAttributeNS(r,t):e.removeAttribute(`${n}:${t}`)}else e.removeAttribute(t)}addClass(e,t){e.classList.add(t)}removeClass(e,t){e.classList.remove(t)}setStyle(e,t,n,r){r&(Se.DashCase|Se.Important)?e.style.setProperty(t,n,r&Se.Important?"important":""):e.style[t]=n}removeStyle(e,t,n){n&Se.DashCase?e.style.removeProperty(t):e.style[t]=""}setProperty(e,t,n){e!=null&&(e[t]=n)}setValue(e,t){e.nodeValue=t}listen(e,t,n,r){if(typeof e=="string"&&(e=Aa().getGlobalEventTarget(this.doc,e),!e))throw new w(5102,!1);let i=this.decoratePreventDefault(n);return this.tracingService?.wrapEventListener&&(i=this.tracingService.wrapEventListener(e,t,i)),this.eventManager.addEventListener(e,t,i,r)}decoratePreventDefault(e){return t=>{if(t==="__ngUnwrap__")return e;e(t)===!1&&t.preventDefault()}}};function qo(e){return e.tagName==="TEMPLATE"&&e.content!==void 0}var Uo=class extends Fi{hostEl;sharedStylesHost;shadowRoot;constructor(e,t,n,r,i,o,s,l){super(e,r,i,s),this.hostEl=t,this.sharedStylesHost=l,this.shadowRoot=t.attachShadow({mode:"open"}),this.sharedStylesHost&&this.sharedStylesHost.addHost(this.shadowRoot);let a=n.styles;a=Fa(n.id,a);for(let c of a){let d=document.createElement("style");o&&d.setAttribute("nonce",o),d.textContent=c,this.shadowRoot.appendChild(d)}let u=n.getExternalStyles?.();if(u)for(let c of u){let d=Zr(c,r);o&&d.setAttribute("nonce",o),this.shadowRoot.appendChild(d)}}nodeOrShadowRoot(e){return e===this.hostEl?this.shadowRoot:e}appendChild(e,t){return super.appendChild(this.nodeOrShadowRoot(e),t)}insertBefore(e,t,n){return super.insertBefore(this.nodeOrShadowRoot(e),t,n)}removeChild(e,t){return super.removeChild(null,t)}parentNode(e){return this.nodeOrShadowRoot(super.parentNode(this.nodeOrShadowRoot(e)))}destroy(){this.sharedStylesHost&&this.sharedStylesHost.removeHost(this.shadowRoot)}},$r=class extends Fi{sharedStylesHost;removeStylesOnCompDestroy;styles;styleUrls;constructor(e,t,n,r,i,o,s,l){super(e,i,o,s),this.sharedStylesHost=t,this.removeStylesOnCompDestroy=r;let a=n.styles;this.styles=l?Fa(l,a):a,this.styleUrls=n.getExternalStyles?.(l)}applyStyles(){this.sharedStylesHost.addStyles(this.styles,this.styleUrls)}destroy(){this.removeStylesOnCompDestroy&&_t.size===0&&this.sharedStylesHost.removeStyles(this.styles,this.styleUrls)}},Zo=class extends $r{contentAttr;hostAttr;constructor(e,t,n,r,i,o,s,l){let a=r+"-"+n.id;super(e,t,n,i,o,s,l,a),this.contentAttr=bm(a),this.hostAttr=wm(a)}applyToHost(e){this.applyStyles(),this.setAttribute(e,this.hostAttr,"")}createElement(e,t){let n=super.createElement(e,t);return super.setAttribute(n,this.contentAttr,""),n}},_m=class Ha extends cm{supportsDOMEvents=!0;static makeCurrent(){um(new Ha)}onAndCancel(t,n,r,i){return t.addEventListener(n,r,i),()=>{t.removeEventListener(n,r,i)}}dispatchEvent(t,n){t.dispatchEvent(n)}remove(t){t.remove()}createElement(t,n){return n=n||this.getDefaultDocument(),n.createElement(t)}createHtmlDocument(){return document.implementation.createHTMLDocument("fakeTitle")}getDefaultDocument(){return document}isElementNode(t){return t.nodeType===Node.ELEMENT_NODE}isShadowRoot(t){return t instanceof DocumentFragment}getGlobalEventTarget(t,n){return n==="window"?window:n==="document"?t:n==="body"?t.body:null}getBaseHref(t){let n=Cm();return n==null?null:xm(n)}resetBaseElement(){ft=null}getUserAgent(){return window.navigator.userAgent}getCookie(t){return dm(document.cookie,t)}},ft=null;function Cm(){return ft=ft||document.head.querySelector("base"),ft?ft.getAttribute("href"):null}function xm(e){return new URL(e,document.baseURI).pathname}var km=(()=>{class e{build(){return new XMLHttpRequest}static \u0275fac=function(n){return new(n||e)};static \u0275prov=D({token:e,factory:e.\u0275fac})}return e})(),$o=["alt","control","meta","shift"],Sm={"\b":"Backspace"," ":"Tab","\x7F":"Delete","\x1B":"Escape",Del:"Delete",Esc:"Escape",Left:"ArrowLeft",Right:"ArrowRight",Up:"ArrowUp",Down:"ArrowDown",Menu:"ContextMenu",Scroll:"ScrollLock",Win:"OS"},Em={alt:e=>e.altKey,control:e=>e.ctrlKey,meta:e=>e.metaKey,shift:e=>e.shiftKey},Im=(()=>{class e extends Va{constructor(n){super(n)}supports(n){return e.parseEventName(n)!=null}addEventListener(n,r,i,o){let s=e.parseEventName(r),l=e.eventCallback(s.fullKey,i,this.manager.getZone());return this.manager.getZone().runOutsideAngular(()=>Aa().onAndCancel(n,s.domEventName,l,o))}static parseEventName(n){let r=n.toLowerCase().split("."),i=r.shift();if(r.length===0||!(i==="keydown"||i==="keyup"))return null;let o=e._normalizeKey(r.pop()),s="",l=r.indexOf("code");if(l>-1&&(r.splice(l,1),s="code."),$o.forEach(u=>{let c=r.indexOf(u);c>-1&&(r.splice(c,1),s+=u+".")}),s+=o,r.length!=0||o.length===0)return null;let a={};return a.domEventName=i,a.fullKey=s,a}static matchEventFullKeyCode(n,r){let i=Sm[n.key]||n.key,o="";return r.indexOf("code.")>-1&&(i=n.code,o="code."),i==null||!i?!1:(i=i.toLowerCase(),i===" "?i="space":i==="."&&(i="dot"),$o.forEach(s=>{if(s!==i){let l=Em[s];l(n)&&(o+=s+".")}}),o+=i,o===r)}static eventCallback(n,r,i){return o=>{e.matchEventFullKeyCode(o,n)&&i.runGuarded(()=>r(o))}}static _normalizeKey(n){return n==="esc"?"escape":n}static \u0275fac=function(n){return new(n||e)(C(we))};static \u0275prov=D({token:e,factory:e.\u0275fac})}return e})();async function Tm(e,t){return am(Om(e,t))}function Om(e,t){return{platformRef:t?.platformRef,appProviders:[...Am,...e?.providers??[]],platformProviders:Nm}}function Dm(){_m.makeCurrent()}function Mm(){return new Sn}function Pm(){return Fd(document),document}var Nm=[{provide:kl,useValue:hm},{provide:xl,useValue:Dm,multi:!0},{provide:we,useFactory:Pm}],Am=[{provide:li,useValue:"root"},{provide:Sn,useFactory:Mm},{provide:Ur,useClass:qr,multi:!0},{provide:Ur,useClass:Im,multi:!0},Bo,La,Ra,{provide:Ai,useExisting:Bo},{provide:fm,useClass:km},[]],za=(()=>{class e{static \u0275fac=function(n){return new(n||e)};static \u0275prov=D({token:e,factory:function(n){let r=null;return n?r=new(n||e):r=C(Vm),r},providedIn:"root"})}return e})(),Vm=(()=>{class e extends za{_doc;constructor(n){super(),this._doc=n}sanitize(n,r){if(r==null)return null;switch(n){case he.NONE:return r;case he.HTML:return Ze(r,"HTML")?Ce(r):Rl(this._doc,String(r)).toString();case he.STYLE:return Ze(r,"Style")?Ce(r):r;case he.SCRIPT:if(Ze(r,"Script"))return Ce(r);throw new w(5200,!1);case he.URL:return Ze(r,"URL")?Ce(r):Dl(String(r));case he.RESOURCE_URL:if(Ze(r,"ResourceURL"))return Ce(r);throw new w(5201,!1);default:throw new w(5202,!1)}}bypassSecurityTrustHtml(n){return of(n)}bypassSecurityTrustStyle(n){return sf(n)}bypassSecurityTrustScript(n){return lf(n)}bypassSecurityTrustUrl(n){return af(n)}bypassSecurityTrustResourceUrl(n){return uf(n)}static \u0275fac=function(n){return new(n||e)(C(we))};static \u0275prov=D({token:e,factory:e.\u0275fac,providedIn:"root"})}return e})(),Qo=class Wt{constructor(t){if(this.model=t,t){this.page.set(t.get("page")??0),this.pageSize.set(t.get("page_size")??10),this.maxColumns.set(t.get("max_columns")??0),this.rowCount.set(t.get("row_count")??null),this.tableHtml.set(t.get("table_html")??""),this.sortContext.set(t.get("sort_context")??[]),this.orderableColumns.set(t.get("orderable_columns")??[]);let n=t.get("error_message")??t.get("_error_message")??null;this.errorMessage.set(n),this.startExecution.set(t.get("start_execution")??!1),this.isDeferredMode.set(t.get("is_deferred_mode")??!1),this.dryRunInfo.set(t.get("dry_run_info")??""),this.ping.set(t.get("ping")??0),t.on("change:page",()=>{this.page.set(t.get("page"))}),t.on("change:page_size",()=>{this.pageSize.set(t.get("page_size"))}),t.on("change:max_columns",()=>{this.maxColumns.set(t.get("max_columns"))}),t.on("change:row_count",()=>{this.rowCount.set(t.get("row_count"))}),t.on("change:table_html",()=>{this.tableHtml.set(t.get("table_html"))}),t.on("change:sort_context",()=>{this.sortContext.set(t.get("sort_context"))}),t.on("change:orderable_columns",()=>{this.orderableColumns.set(t.get("orderable_columns"))}),t.on("change:start_execution",()=>{this.startExecution.set(t.get("start_execution")??!1)}),t.on("change:is_deferred_mode",()=>{this.isDeferredMode.set(t.get("is_deferred_mode")??!1)}),t.on("change:dry_run_info",()=>{this.dryRunInfo.set(t.get("dry_run_info")??"")}),t.on("change:ping",()=>{this.ping.set(t.get("ping")??0)});let r=()=>{let i=t.get("error_message")??t.get("_error_message")??null;this.errorMessage.set(i)};t.on("change:error_message",r),t.on("change:_error_message",r)}}page=j(0);pageSize=j(10);maxColumns=j(0);rowCount=j(null);tableHtml=j("");sortContext=j([]);orderableColumns=j([]);errorMessage=j(null);startExecution=j(!1);isDeferredMode=j(!1);dryRunInfo=j("");ping=j(0);setPage(t){this.page.set(t),this.model&&(this.model.set("page",t),this.model.save_changes())}setPageSize(t){this.pageSize.set(t),this.page.set(0),this.model&&(this.model.set("page_size",t),this.model.set("page",0),this.model.save_changes())}setMaxColumns(t){this.maxColumns.set(t),this.model&&(this.model.set("max_columns",t),this.model.save_changes())}setSortContext(t){this.sortContext.set(t),this.model&&(this.model.set("sort_context",t),this.model.save_changes())}setStartExecution(t){this.startExecution.set(t),this.model&&(this.model.set("start_execution",t),this.model.save_changes())}setPing(t){this.ping.set(t),this.model&&(this.model.set("ping",t),this.model.save_changes())}static \u0275fac=function(t){return new(t||Wt)(C("ANYWIDGET_MODEL"))};static \u0275prov=D({token:Wt,factory:Wt.\u0275fac})},Rm=["tableContainer"],Lm=["app-root",""];function jm(e,t){if(e&1&&(P(0,"div",2),B(1),N()),e&2){let n=ee();L(),Le(n.errorMessage())}}function Fm(e,t){e&1&&(ka(0,"span",7),B(1," Run Query "))}function Hm(e,t){e&1&&B(0," Run Query ")}function zm(e,t){if(e&1){let n=Sa();P(0,"div",3)(1,"div",4)(2,"p",5),B(3),N(),P(4,"button",6),xe("click",function(){qe(n);let r=ee();return Ue(r.handleRunQuery())}),Rr(5,Fm,2,0)(6,Hm,1,0),N()()()}if(e&2){let n=ee();L(3),Le(n.dryRunInfo()),L(),te("disabled",n.isLoading()),L(),Lr(n.isLoading()?5:6)}}function Bm(e,t){if(e&1&&(P(0,"option",18),B(1),N()),e&2){let n=t.$implicit;te("value",n),L(),Le(n===0?"All":n)}}function qm(e,t){if(e&1&&(P(0,"option",18),B(1),N()),e&2){let n=t.$implicit;te("value",n),L(),Le(n)}}function Um(e,t){if(e&1){let n=Sa();P(0,"div",8,0),xe("click",function(r){qe(n);let i=ee();return Ue(i.handleTableClick(r))}),N(),P(2,"footer",9)(3,"span",10),B(4),N(),P(5,"div",11)(6,"button",12),xe("click",function(){qe(n);let r=ee();return Ue(r.handlePageChange(-1))}),B(7,"<"),N(),P(8,"span",13),B(9),N(),P(10,"button",12),xe("click",function(){qe(n);let r=ee();return Ue(r.handlePageChange(1))}),B(11,">"),N()(),P(12,"div",14)(13,"div",15)(14,"label",16),B(15,"Max columns:"),N(),P(16,"select",17),xe("change",function(r){qe(n);let i=ee();return Ue(i.handleMaxColumnsChange(r))}),Vo(17,Bm,2,2,"option",18,Ao),N()(),P(19,"div",19)(20,"label",20),B(21,"Page size:"),N(),P(22,"select",21),xe("change",function(r){qe(n);let i=ee();return Ue(i.handlePageSizeChange(r))}),Vo(23,qm,2,2,"option",18,Ao),N()()()()}if(e&2){let n=ee();te("innerHTML",n.sanitizedHtml(),Nf),L(4),Le(n.rowCountText()),L(2),te("disabled",n.prevPageDisabled()),L(3),Le(n.pageIndicatorText()),L(),te("disabled",n.nextPageDisabled()),L(6),te("value",n.maxColumns()),L(),Ro(n.maxColumnOptions),L(5),te("value",n.pageSize()),L(),Ro(n.pageSizeOptions)}}var Zm=class Qr{state=b(Qo);sanitizer=b(za);maxColumnOptions=[5,10,15,20,0];pageSizeOptions=[10,25,50,100];errorMessage=this.state.errorMessage;maxColumns=this.state.maxColumns;pageSize=this.state.pageSize;page=this.state.page;rowCount=this.state.rowCount;isDeferredMode=this.state.isDeferredMode;dryRunInfo=this.state.dryRunInfo;isLoading=j(!1);sanitizedHtml=Be(()=>this.sanitizer.bypassSecurityTrustHtml(this.state.tableHtml()));totalPages=Be(()=>{let t=this.rowCount(),n=this.pageSize();return t!==null&&n>0?Math.ceil(t/n):null});pageIndicatorText=Be(()=>{let t=this.page(),n=this.rowCount(),r=this.totalPages(),i=(t+1).toLocaleString(),o=(r??1).toLocaleString();return`Page ${i} of ${o}`});rowCountText=Be(()=>{let t=this.rowCount();return t===null?"Total rows unknown":t===0?"0 total rows":`${t.toLocaleString()} total rows`});prevPageDisabled=Be(()=>this.page()===0);nextPageDisabled=Be(()=>{let t=this.page(),n=this.rowCount(),r=this.totalPages();return n===null?!1:n===0?!0:r!==null&&t>=r-1});isDarkMode=j(!1);themeObserver=null;tableContainerRef;isHeightInitialized=!1;constructor(){$n(()=>{let t=this.state.tableHtml(),n=this.state.sortContext(),r=this.state.orderableColumns();this.isDeferredMode()&&(this.isHeightInitialized=!1),setTimeout(()=>{this.applySortIndicators(),this.lockInitialHeight()},0)}),$n(()=>{this.state.startExecution()||this.isLoading.set(!1)}),$n(t=>{if(this.state.startExecution()){let n=setInterval(()=>{if(this.state.startExecution()){let r=this.state.ping();this.state.setPing(r+1)}else clearInterval(n)},500);t(()=>{clearInterval(n)})}})}ngOnInit(){this.initThemeDetection()}ngOnDestroy(){this.themeObserver?.disconnect()}handleRunQuery(){this.isLoading.set(!0),this.state.setStartExecution(!0)}handlePageChange(t){let n=this.page()+t;this.state.setPage(n)}handlePageSizeChange(t){let n=t.target,r=Number(n.value);r&&this.state.setPageSize(r)}handleMaxColumnsChange(t){let n=t.target,r=Number(n.value);this.state.setMaxColumns(r)}handleTableClick(t){let n=t.target.closest("th");if(!n)return;let r=n.querySelector("div.bf-header-content");if(!r)return;let i=this.getColumnName(r),o=this.state.orderableColumns();if(!i||!o.includes(i))return;let s=[...this.state.sortContext()],l=s.findIndex(u=>u.column===i),a=[...s];t.shiftKey?l!==-1?a[l].ascending?a[l]=Q($({},a[l]),{ascending:!1}):a.splice(l,1):a.push({column:i,ascending:!0}):l!==-1&&a.length===1?a[l].ascending?a[l]=Q($({},a[l]),{ascending:!1}):a=[]:a=[{column:i,ascending:!0}],this.state.setSortContext(a)}getColumnName(t){let n=t.cloneNode(!0);return n.querySelector(".sort-indicator")?.remove(),n.textContent?.trim()||""}applySortIndicators(){let t=this.tableContainerRef?.nativeElement;if(!t)return;let n=this.state.orderableColumns(),r=this.state.sortContext()||[],i=o=>r.findIndex(s=>s.column===o);t.querySelectorAll("th").forEach(o=>{let s=o.querySelector("div.bf-header-content");if(!s)return;let l=this.getColumnName(s);if(l&&n.includes(l)){let a=s.querySelector(".sort-indicator");a||(a=document.createElement("span"),a.classList.add("sort-indicator"),a.style.paddingLeft="5px",s.appendChild(a));let u=i(l);if(u!==-1){let c=r[u].ascending;a.textContent=c?"\u25B2":"\u25BC",a.style.visibility="visible"}else a.textContent="\u25CF",a.style.visibility="hidden"}})}lockInitialHeight(){if(this.isHeightInitialized)return;let t=this.tableContainerRef?.nativeElement;if(!t)return;let n=t.querySelector("table");if(n&&n.offsetHeight>0){let r=t.offsetHeight;r>0&&(t.style.height=`${r}px`,this.isHeightInitialized=!0)}}initThemeDetection(){this.updateTheme();let t=new MutationObserver(()=>this.updateTheme());t.observe(document.body,{attributes:!0,attributeFilter:["class","data-theme","data-vscode-theme-kind"]}),this.themeObserver=t}updateTheme(){let t=document.body,n=t.classList.contains("vscode-dark")||t.classList.contains("theme-dark")||t.dataset.theme==="dark"||t.getAttribute("data-vscode-theme-kind")==="vscode-dark";this.isDarkMode.set(n)}static \u0275fac=function(t){return new(t||Qr)};static \u0275cmp=Kp({type:Qr,selectors:[["","app-root",""]],viewQuery:function(t,n){if(t&1&&Ea(Rm,5),t&2){let r;Eg(r=Ig())&&(n.tableContainerRef=r.first)}},features:[Yg([Qo])],attrs:Lm,decls:4,vars:4,consts:[["tableContainer",""],[1,"bigframes-widget"],[1,"bigframes-error-message"],[1,"deferred-container"],[1,"deferred-card"],[1,"deferred-estimate"],[1,"run-query-button",3,"click","disabled"],[1,"spinner"],[1,"table-container",3,"click","innerHTML"],[1,"footer"],[1,"row-count"],[1,"pagination"],[3,"click","disabled"],[1,"page-indicator"],[1,"settings"],[1,"max-columns"],["for","max-cols-select"],["id","max-cols-select",3,"change","value"],[3,"value"],[1,"page-size"],["for","page-size-select"],["id","page-size-select",3,"change","value"]],template:function(t,n){t&1&&(P(0,"div",1),Rr(1,jm,2,1,"div",2),Rr(2,zm,7,3,"div",3)(3,Um,25,7),N()),t&2&&(Ia("bigframes-dark-mode",n.isDarkMode()),L(),Lr(n.errorMessage()?1:-1),L(),Lr(n.isDeferredMode()?2:3))},styles:[".bigframes-widget.bigframes-widget[_ngcontent-%COMP%]{--bf-bg: white;--bf-border-color: #ccc;--bf-error-bg: #fbe;--bf-error-border: red;--bf-error-fg: black;--bf-fg: black;--bf-header-bg: #f5f5f5;--bf-null-fg: gray;--bf-row-even-bg: #f5f5f5;--bf-row-odd-bg: white;background-color:var(--bf-bg);box-sizing:border-box;color:var(--bf-fg);display:flex;flex-direction:column;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif;margin:0;padding:0;width:100%}.bigframes-widget[_ngcontent-%COMP%] *[_ngcontent-%COMP%]{box-sizing:border-box}@media(prefers-color-scheme:dark){.bigframes-widget.bigframes-widget[_ngcontent-%COMP%]{--bf-bg: var(--vscode-editor-background, #202124);--bf-border-color: #444;--bf-error-bg: #511;--bf-error-border: #f88;--bf-error-fg: #fcc;--bf-fg: white;--bf-header-bg: var(--vscode-editor-background, black);--bf-null-fg: #aaa;--bf-row-even-bg: #202124;--bf-row-odd-bg: #383838}}.bigframes-widget.bigframes-dark-mode.bigframes-dark-mode[_ngcontent-%COMP%]{--bf-bg: var(--vscode-editor-background, #202124);--bf-border-color: #444;--bf-error-bg: #511;--bf-error-border: #f88;--bf-error-fg: #fcc;--bf-fg: white;--bf-header-bg: var(--vscode-editor-background, black);--bf-null-fg: #aaa;--bf-row-even-bg: #202124;--bf-row-odd-bg: #383838}.bigframes-widget[_ngcontent-%COMP%] .table-container[_ngcontent-%COMP%]{background-color:var(--bf-bg);margin:0;overflow:auto;padding:0}.bigframes-widget[_ngcontent-%COMP%] .footer[_ngcontent-%COMP%]{align-items:center;background-color:var(--bf-bg);color:var(--bf-fg);display:flex;font-size:.8rem;justify-content:space-between;padding:8px}.bigframes-widget[_ngcontent-%COMP%] .footer[_ngcontent-%COMP%] > *[_ngcontent-%COMP%]{flex:1}.bigframes-widget[_ngcontent-%COMP%] .pagination[_ngcontent-%COMP%]{align-items:center;display:flex;flex-direction:row;gap:4px;justify-content:center;padding:4px}.bigframes-widget[_ngcontent-%COMP%] .page-indicator[_ngcontent-%COMP%], .bigframes-widget[_ngcontent-%COMP%] .row-count[_ngcontent-%COMP%]{margin:0 8px}.bigframes-widget[_ngcontent-%COMP%] .settings[_ngcontent-%COMP%]{align-items:center;display:flex;flex-direction:row;gap:16px;justify-content:end}.bigframes-widget[_ngcontent-%COMP%] .page-size[_ngcontent-%COMP%], .bigframes-widget[_ngcontent-%COMP%] .max-columns[_ngcontent-%COMP%]{align-items:center;display:flex;flex-direction:row;gap:4px}.bigframes-widget[_ngcontent-%COMP%] .page-size[_ngcontent-%COMP%] label[_ngcontent-%COMP%], .bigframes-widget[_ngcontent-%COMP%] .max-columns[_ngcontent-%COMP%] label[_ngcontent-%COMP%]{margin-right:8px}.bigframes-widget[_ngcontent-%COMP%] table.bigframes-widget-table, .bigframes-widget[_ngcontent-%COMP%] table.dataframe{background-color:var(--bf-bg);border:1px solid var(--bf-border-color);border-collapse:collapse;border-spacing:0;box-shadow:none;color:var(--bf-fg);margin:0;outline:none;text-align:left;width:auto}.bigframes-widget[_ngcontent-%COMP%] tr{border:none}.bigframes-widget[_ngcontent-%COMP%] th{background-color:var(--bf-header-bg);border:1px solid var(--bf-border-color);color:var(--bf-fg);padding:0;position:sticky;text-align:left;top:0;z-index:1}.bigframes-widget[_ngcontent-%COMP%] td{border:1px solid var(--bf-border-color);color:var(--bf-fg);padding:.5em}.bigframes-widget[_ngcontent-%COMP%] table tbody tr:nth-child(odd), .bigframes-widget[_ngcontent-%COMP%] table tbody tr:nth-child(odd) td{background-color:var(--bf-row-odd-bg)}.bigframes-widget[_ngcontent-%COMP%] table tbody tr:nth-child(2n), .bigframes-widget[_ngcontent-%COMP%] table tbody tr:nth-child(2n) td{background-color:var(--bf-row-even-bg)}.bigframes-widget[_ngcontent-%COMP%] .bf-header-content{box-sizing:border-box;height:100%;overflow:auto;padding:.5em;resize:horizontal;width:100%}.bigframes-widget[_ngcontent-%COMP%] th .sort-indicator{padding-left:4px;visibility:hidden}.bigframes-widget[_ngcontent-%COMP%] th:hover .sort-indicator{visibility:visible}.bigframes-widget[_ngcontent-%COMP%] button[_ngcontent-%COMP%]{background-color:transparent;border:1px solid currentColor;border-radius:4px;color:inherit;cursor:pointer;display:inline-block;padding:2px 8px;text-align:center;text-decoration:none;-webkit-user-select:none;user-select:none;vertical-align:middle}.bigframes-widget[_ngcontent-%COMP%] button[_ngcontent-%COMP%]:disabled{opacity:.65;pointer-events:none}.bigframes-widget[_ngcontent-%COMP%] .bigframes-error-message[_ngcontent-%COMP%]{background-color:var(--bf-error-bg);border:1px solid var(--bf-error-border);border-radius:4px;color:var(--bf-error-fg);font-size:14px;margin-bottom:8px;padding:8px}.bigframes-widget[_ngcontent-%COMP%] .cell-align-right{text-align:right}.bigframes-widget[_ngcontent-%COMP%] .cell-align-left{text-align:left}.bigframes-widget[_ngcontent-%COMP%] .null-value{color:var(--bf-null-fg)}.bigframes-widget[_ngcontent-%COMP%] .debug-info{border-top:1px solid var(--bf-border-color)}.bigframes-widget[_ngcontent-%COMP%] .deferred-container[_ngcontent-%COMP%]{align-items:center;display:flex;justify-content:center;min-height:220px;padding:24px;width:100%}.bigframes-widget[_ngcontent-%COMP%] .deferred-card[_ngcontent-%COMP%]{background:linear-gradient(135deg,#fff9,#ffffff4d);border:1px solid rgba(255,255,255,.4);border-radius:16px;box-shadow:0 8px 32px #1f268712;display:flex;flex-direction:column;gap:16px;max-width:500px;padding:32px;text-align:center;transition:all .3s ease-in-out}.bigframes-widget.bigframes-dark-mode[_ngcontent-%COMP%] .deferred-card[_ngcontent-%COMP%]{background:linear-gradient(135deg,#20212499,#2021244d);border:1px solid rgba(255,255,255,.1);box-shadow:0 8px 32px #0000004d}@media(prefers-color-scheme:dark){.bigframes-widget[_ngcontent-%COMP%] .deferred-card[_ngcontent-%COMP%]{background:linear-gradient(135deg,#20212499,#2021244d);border:1px solid rgba(255,255,255,.1);box-shadow:0 8px 32px #0000004d}}.bigframes-widget[_ngcontent-%COMP%] .deferred-title[_ngcontent-%COMP%]{font-size:1.1rem;font-weight:600;margin:0}.bigframes-widget[_ngcontent-%COMP%] .deferred-estimate[_ngcontent-%COMP%]{color:var(--bf-null-fg);font-size:.9rem;margin:0}.bigframes-widget[_ngcontent-%COMP%] .run-query-button[_ngcontent-%COMP%]{align-items:center;background-color:var(--bf-fg);border:1px solid var(--bf-fg);border-radius:8px;color:var(--bf-bg);cursor:pointer;display:inline-flex;font-size:14px;font-weight:600;gap:8px;justify-content:center;padding:10px 20px;transition:transform .2s ease,opacity .2s ease}.bigframes-widget[_ngcontent-%COMP%] .run-query-button[_ngcontent-%COMP%]:hover{opacity:.9;transform:translateY(-1px)}.bigframes-widget[_ngcontent-%COMP%] .run-query-button[_ngcontent-%COMP%]:active{transform:translateY(0)}.bigframes-widget[_ngcontent-%COMP%] .run-query-button[_ngcontent-%COMP%]:disabled{cursor:not-allowed;opacity:.6}.bigframes-widget[_ngcontent-%COMP%] .spinner[_ngcontent-%COMP%]{animation:_ngcontent-%COMP%_spin 1s linear infinite;border:2px solid currentColor;border-radius:50%;border-top-color:transparent;display:inline-block;height:12px;width:12px}@keyframes _ngcontent-%COMP%_spin{to{transform:rotate(360deg)}}"]})};function $m({model:e,el:t}){let n=document.createElement("div");n.setAttribute("app-root",""),t.appendChild(n);let r={providers:[nd(),Xg(),{provide:"ANYWIDGET_MODEL",useValue:e}]};Tm(r).then(i=>{i.bootstrap(Zm,n),n.removeAttribute("app-root")}).catch(i=>console.error(i))}var Gm={render:$m};export{Gm as default}; diff --git a/bigframes/display/table_widget_angular/.editorconfig b/bigframes/display/table_widget_angular/.editorconfig deleted file mode 100644 index f166060da1c..00000000000 --- a/bigframes/display/table_widget_angular/.editorconfig +++ /dev/null @@ -1,17 +0,0 @@ -# Editor configuration, see https://editorconfig.org -root = true - -[*] -charset = utf-8 -indent_style = space -indent_size = 2 -insert_final_newline = true -trim_trailing_whitespace = true - -[*.ts] -quote_type = single -ij_typescript_use_double_quotes = false - -[*.md] -max_line_length = off -trim_trailing_whitespace = false diff --git a/bigframes/display/table_widget_angular/.gitignore b/bigframes/display/table_widget_angular/.gitignore deleted file mode 100644 index 854acd5fc03..00000000000 --- a/bigframes/display/table_widget_angular/.gitignore +++ /dev/null @@ -1,44 +0,0 @@ -# See https://docs.github.com/get-started/getting-started-with-git/ignoring-files for more about ignoring files. - -# Compiled output -/dist -/tmp -/out-tsc -/bazel-out - -# Node -/node_modules -npm-debug.log -yarn-error.log - -# IDEs and editors -.idea/ -.project -.classpath -.c9/ -*.launch -.settings/ -*.sublime-workspace - -# Visual Studio Code -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -!.vscode/mcp.json -.history/* - -# Miscellaneous -/.angular/cache -.sass-cache/ -/connect.lock -/coverage -/libpeerconnection.log -testem.log -/typings -__screenshots__/ - -# System files -.DS_Store -Thumbs.db diff --git a/bigframes/display/table_widget_angular/.prettierrc b/bigframes/display/table_widget_angular/.prettierrc deleted file mode 100644 index d6c16d7ee77..00000000000 --- a/bigframes/display/table_widget_angular/.prettierrc +++ /dev/null @@ -1,12 +0,0 @@ -{ - "printWidth": 100, - "singleQuote": true, - "overrides": [ - { - "files": "*.html", - "options": { - "parser": "angular" - } - } - ] -} diff --git a/bigframes/display/table_widget_angular/README.md b/bigframes/display/table_widget_angular/README.md deleted file mode 100644 index db09b5b9f56..00000000000 --- a/bigframes/display/table_widget_angular/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# TableWidgetAngular - -This project is the Angular-based interactive Table Widget frontend for BigQuery DataFrames (``bigframes``). It is integrated into the Python backend using ``anywidget``. - -This project was generated using [Angular CLI](https://github.com/angular/angular-cli) version 21.2.9. - -## Getting Started - -Ensure you have [Node.js](https://nodejs.org/) installed. - -1. Install dependencies: - ```bash - npm install - ``` - -2. Start the local development server: - ```bash - npm run start - ``` - Navigate to `http://localhost:4200/`. The application will automatically reload when you modify the source files under `src/`. - -## Development & Code Scaffolding - -To generate a new component, directive, or service: -```bash -ng generate component component-name -``` - -For a complete list of available schematics (such as `components`, `directives`, or `pipes`), run: -```bash -ng generate --help -``` - -## Running Tests - -To execute unit tests: -```bash -npm run test -``` - -## Packaging for Python - -Before testing the widget inside a Jupyter notebook or committing changes, compile the Angular app and bundle it so that the Python backend can load it: -```bash -npm run build:widget -``` - -This command compiles the project in production mode and then triggers `bundle.js` (via `esbuild`) to bundle the browser artifacts into a single unified ES module file at `../table_widget_angular.js`. diff --git a/bigframes/display/table_widget_angular/angular.json b/bigframes/display/table_widget_angular/angular.json deleted file mode 100644 index 497168c4c95..00000000000 --- a/bigframes/display/table_widget_angular/angular.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "$schema": "./node_modules/@angular/cli/lib/config/schema.json", - "version": 1, - "cli": { - "packageManager": "npm" - }, - "newProjectRoot": "projects", - "projects": { - "table-widget-angular": { - "projectType": "application", - "schematics": {}, - "root": "", - "sourceRoot": "src", - "prefix": "app", - "architect": { - "build": { - "builder": "@angular/build:application", - "options": { - "browser": "src/main.ts", - "tsConfig": "tsconfig.app.json", - "assets": [ - { - "glob": "**/*", - "input": "public" - } - ], - "styles": [ - "src/styles.css" - ] - }, - "configurations": { - "production": { - "budgets": [ - { - "type": "initial", - "maximumWarning": "500kB", - "maximumError": "1MB" - }, - { - "type": "anyComponentStyle", - "maximumWarning": "4kB", - "maximumError": "8kB" - } - ], - "outputHashing": "all" - }, - "development": { - "optimization": false, - "extractLicenses": false, - "sourceMap": true - } - }, - "defaultConfiguration": "production" - }, - "serve": { - "builder": "@angular/build:dev-server", - "configurations": { - "production": { - "buildTarget": "table-widget-angular:build:production" - }, - "development": { - "buildTarget": "table-widget-angular:build:development" - } - }, - "defaultConfiguration": "development" - }, - "test": { - "builder": "@angular/build:unit-test" - } - } - } - } -} diff --git a/bigframes/display/table_widget_angular/bundle.js b/bigframes/display/table_widget_angular/bundle.js deleted file mode 100644 index fb97ab8a376..00000000000 --- a/bigframes/display/table_widget_angular/bundle.js +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const esbuild = require('esbuild'); -const path = require('path'); - -const banner = `/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -`; - -esbuild.build({ - entryPoints: [path.resolve(__dirname, 'dist/table-widget-angular/browser/main.js')], - bundle: true, - outfile: path.resolve(__dirname, '../table_widget_angular.js'), - format: 'esm', - logLevel: 'info', - minify: true, - banner: { - js: banner, - }, -}).catch(() => process.exit(1)); diff --git a/bigframes/display/table_widget_angular/package-lock.json b/bigframes/display/table_widget_angular/package-lock.json deleted file mode 100644 index 33540ab8512..00000000000 --- a/bigframes/display/table_widget_angular/package-lock.json +++ /dev/null @@ -1,9591 +0,0 @@ -{ - "name": "table-widget-angular", - "version": "0.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "table-widget-angular", - "version": "0.0.0", - "dependencies": { - "@angular/common": "^22.1.0", - "@angular/compiler": "^22.1.0", - "@angular/core": "^22.1.0", - "@angular/forms": "^22.1.0", - "@angular/platform-browser": "^22.1.0", - "@angular/router": "^22.1.0", - "rxjs": "~7.8.0", - "tslib": "^2.3.0" - }, - "devDependencies": { - "@angular/build": "^22.1.2", - "@angular/cli": "^21.2.16", - "@angular/compiler-cli": "^22.1.0", - "esbuild": "^0.28.0", - "jsdom": "^28.0.0", - "prettier": "^3.8.1", - "typescript": "~5.9.2", - "vitest": "^4.0.8" - } - }, - "node_modules/@acemir/cssom": { - "version": "0.9.31", - "resolved": "https://registry.npmjs.org/@acemir/cssom/-/cssom-0.9.31.tgz", - "integrity": "sha512-ZnR3GSaH+/vJ0YlHau21FjfLYjMpYVIzTD8M8vIEQvIGxeOXyXdzCI140rrCY862p/C/BbzWsjc1dgnM9mkoTA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@algolia/abtesting": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/@algolia/abtesting/-/abtesting-1.14.1.tgz", - "integrity": "sha512-Dkj0BgPiLAaim9sbQ97UKDFHJE/880wgStAM18U++NaJ/2Cws34J5731ovJifr6E3Pv4T2CqvMXf8qLCC417Ew==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1", - "@algolia/requester-browser-xhr": "5.48.1", - "@algolia/requester-fetch": "5.48.1", - "@algolia/requester-node-http": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/client-abtesting": { - "version": "5.48.1", - "resolved": "https://registry.npmjs.org/@algolia/client-abtesting/-/client-abtesting-5.48.1.tgz", - "integrity": "sha512-LV5qCJdj+/m9I+Aj91o+glYszrzd7CX6NgKaYdTOj4+tUYfbS62pwYgUfZprYNayhkQpVFcrW8x8ZlIHpS23Vw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1", - "@algolia/requester-browser-xhr": "5.48.1", - "@algolia/requester-fetch": "5.48.1", - "@algolia/requester-node-http": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/client-analytics": { - "version": "5.48.1", - "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-5.48.1.tgz", - "integrity": "sha512-/AVoMqHhPm14CcHq7mwB+bUJbfCv+jrxlNvRjXAuO+TQa+V37N8k1b0ijaRBPdmSjULMd8KtJbQyUyabXOu6Kg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1", - "@algolia/requester-browser-xhr": "5.48.1", - "@algolia/requester-fetch": "5.48.1", - "@algolia/requester-node-http": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/client-common": { - "version": "5.48.1", - "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-5.48.1.tgz", - "integrity": "sha512-VXO+qu2Ep6ota28ktvBm3sG53wUHS2n7bgLWmce5jTskdlCD0/JrV4tnBm1l7qpla1CeoQb8D7ShFhad+UoSOw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/client-insights": { - "version": "5.48.1", - "resolved": "https://registry.npmjs.org/@algolia/client-insights/-/client-insights-5.48.1.tgz", - "integrity": "sha512-zl+Qyb0nLg+Y5YvKp1Ij+u9OaPaKg2/EPzTwKNiVyOHnQJlFxmXyUZL1EInczAZsEY8hVpPCLtNfhMhfxluXKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1", - "@algolia/requester-browser-xhr": "5.48.1", - "@algolia/requester-fetch": "5.48.1", - "@algolia/requester-node-http": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/client-personalization": { - "version": "5.48.1", - "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-5.48.1.tgz", - "integrity": "sha512-r89Qf9Oo9mKWQXumRu/1LtvVJAmEDpn8mHZMc485pRfQUMAwSSrsnaw1tQ3sszqzEgAr1c7rw6fjBI+zrAXTOw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1", - "@algolia/requester-browser-xhr": "5.48.1", - "@algolia/requester-fetch": "5.48.1", - "@algolia/requester-node-http": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/client-query-suggestions": { - "version": "5.48.1", - "resolved": "https://registry.npmjs.org/@algolia/client-query-suggestions/-/client-query-suggestions-5.48.1.tgz", - "integrity": "sha512-TPKNPKfghKG/bMSc7mQYD9HxHRUkBZA4q1PEmHgICaSeHQscGqL4wBrKkhfPlDV1uYBKW02pbFMUhsOt7p4ZpA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1", - "@algolia/requester-browser-xhr": "5.48.1", - "@algolia/requester-fetch": "5.48.1", - "@algolia/requester-node-http": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/client-search": { - "version": "5.48.1", - "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-5.48.1.tgz", - "integrity": "sha512-4Fu7dnzQyQmMFknYwTiN/HxPbH4DyxvQ1m+IxpPp5oslOgz8m6PG5qhiGbqJzH4HiT1I58ecDiCAC716UyVA8Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1", - "@algolia/requester-browser-xhr": "5.48.1", - "@algolia/requester-fetch": "5.48.1", - "@algolia/requester-node-http": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/ingestion": { - "version": "1.48.1", - "resolved": "https://registry.npmjs.org/@algolia/ingestion/-/ingestion-1.48.1.tgz", - "integrity": "sha512-/RFq3TqtXDUUawwic/A9xylA2P3LDMO8dNhphHAUOU51b1ZLHrmZ6YYJm3df1APz7xLY1aht6okCQf+/vmrV9w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1", - "@algolia/requester-browser-xhr": "5.48.1", - "@algolia/requester-fetch": "5.48.1", - "@algolia/requester-node-http": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/monitoring": { - "version": "1.48.1", - "resolved": "https://registry.npmjs.org/@algolia/monitoring/-/monitoring-1.48.1.tgz", - "integrity": "sha512-Of0jTeAZRyRhC7XzDSjJef0aBkgRcvRAaw0ooYRlOw57APii7lZdq+layuNdeL72BRq1snaJhoMMwkmLIpJScw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1", - "@algolia/requester-browser-xhr": "5.48.1", - "@algolia/requester-fetch": "5.48.1", - "@algolia/requester-node-http": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/recommend": { - "version": "5.48.1", - "resolved": "https://registry.npmjs.org/@algolia/recommend/-/recommend-5.48.1.tgz", - "integrity": "sha512-bE7JcpFXzxF5zHwj/vkl2eiCBvyR1zQ7aoUdO+GDXxGp0DGw7nI0p8Xj6u8VmRQ+RDuPcICFQcCwRIJT5tDJFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1", - "@algolia/requester-browser-xhr": "5.48.1", - "@algolia/requester-fetch": "5.48.1", - "@algolia/requester-node-http": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/requester-browser-xhr": { - "version": "5.48.1", - "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-5.48.1.tgz", - "integrity": "sha512-MK3wZ2koLDnvH/AmqIF1EKbJlhRS5j74OZGkLpxI4rYvNi9Jn/C7vb5DytBnQ4KUWts7QsmbdwHkxY5txQHXVw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/requester-fetch": { - "version": "5.48.1", - "resolved": "https://registry.npmjs.org/@algolia/requester-fetch/-/requester-fetch-5.48.1.tgz", - "integrity": "sha512-2oDT43Y5HWRSIQMPQI4tA/W+TN/N2tjggZCUsqQV440kxzzoPGsvv9QP1GhQ4CoDa+yn6ygUsGp6Dr+a9sPPSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@algolia/requester-node-http": { - "version": "5.48.1", - "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-5.48.1.tgz", - "integrity": "sha512-xcaCqbhupVWhuBP1nwbk1XNvwrGljozutEiLx06mvqDf3o8cHyEgQSHS4fKJM+UAggaWVnnFW+Nne5aQ8SUJXg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/client-common": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@ampproject/remapping": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", - "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@angular-devkit/architect": { - "version": "0.2102.16", - "resolved": "https://registry.npmjs.org/@angular-devkit/architect/-/architect-0.2102.16.tgz", - "integrity": "sha512-FDUKPpq70nJwGK4CICPD31XmesBEGv57Z+JBCPWrTa5mVZIXCQkeo5waIaNfzAnLdbpd74ULJJ3MDNVt4iaGZg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@angular-devkit/core": "21.2.16", - "rxjs": "7.8.2" - }, - "bin": { - "architect": "bin/cli.js" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0", - "npm": "^6.11.0 || ^7.5.6 || >=8.0.0", - "yarn": ">= 1.13.0" - } - }, - "node_modules/@angular-devkit/core": { - "version": "21.2.16", - "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-21.2.16.tgz", - "integrity": "sha512-bRot0dqonxdSuGzXyOYtVJis/u9CJycrfC/aaxLeMF37gKtWIyCR2KFkMRXAoiV/AKk5/NuuqDNqcQS9w5G3Fg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ajv": "8.18.0", - "ajv-formats": "3.0.1", - "jsonc-parser": "3.3.1", - "picomatch": "4.0.4", - "rxjs": "7.8.2", - "source-map": "0.7.6" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0", - "npm": "^6.11.0 || ^7.5.6 || >=8.0.0", - "yarn": ">= 1.13.0" - }, - "peerDependencies": { - "chokidar": "^5.0.0" - }, - "peerDependenciesMeta": { - "chokidar": { - "optional": true - } - } - }, - "node_modules/@angular-devkit/schematics": { - "version": "21.2.16", - "resolved": "https://registry.npmjs.org/@angular-devkit/schematics/-/schematics-21.2.16.tgz", - "integrity": "sha512-3wTn2N6iWxYLrRaFDk3J3a6P3OxL+yvYGoDA7pNKfI+Nu0PpTK8BBwhNQD8L5P3US/QGWTkMNbzZ7XxBBfFP/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@angular-devkit/core": "21.2.16", - "jsonc-parser": "3.3.1", - "magic-string": "0.30.21", - "ora": "9.3.0", - "rxjs": "7.8.2" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0", - "npm": "^6.11.0 || ^7.5.6 || >=8.0.0", - "yarn": ">= 1.13.0" - } - }, - "node_modules/@angular/build": { - "version": "22.1.2", - "resolved": "https://registry.npmjs.org/@angular/build/-/build-22.1.2.tgz", - "integrity": "sha512-DE/3o17JTel4EBt2BA4DqJYeBBuz5Ef/kf1jL9YZTyJu4SrLr/HI79K14jFr0VRIxzcqG92FdIzfLDxbOesQsg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@ampproject/remapping": "2.3.0", - "@angular-devkit/architect": "0.2201.2", - "@babel/core": "8.0.1", - "@babel/helper-annotate-as-pure": "8.0.0", - "@babel/helper-split-export-declaration": "7.24.7", - "@inquirer/confirm": "6.1.1", - "@vitejs/plugin-basic-ssl": "2.3.0", - "beasties": "0.4.3", - "browserslist": "^4.26.0", - "esbuild": "0.28.1", - "https-proxy-agent": "9.1.0", - "jsonc-parser": "3.3.1", - "listr2": "10.2.2", - "magic-string": "1.0.0", - "mrmime": "2.0.1", - "oxc-parser": "0.142.0", - "parse5-html-rewriting-stream": "8.0.1", - "picomatch": "4.0.5", - "piscina": "5.2.0", - "rolldown": "1.2.0", - "sass": "1.101.0", - "semver": "7.8.5", - "source-map-support": "0.5.21", - "tinyglobby": "0.2.17", - "vite": "8.1.5", - "watchpack": "2.5.2" - }, - "engines": { - "node": "^22.22.3 || ^24.15.0 || >=26.0.0", - "npm": "^6.11.0 || ^7.5.6 || >=8.0.0", - "yarn": ">= 1.13.0" - }, - "optionalDependencies": { - "lmdb": "3.5.6" - }, - "peerDependencies": { - "@angular/compiler": "^22.0.0", - "@angular/compiler-cli": "^22.0.0", - "@angular/core": "^22.0.0", - "@angular/localize": "^22.0.0", - "@angular/platform-browser": "^22.0.0", - "@angular/platform-server": "^22.0.0", - "@angular/service-worker": "^22.0.0", - "@angular/ssr": "^22.1.2", - "istanbul-lib-instrument": "^6.0.0", - "karma": "^6.4.0", - "less": "^4.2.0", - "ng-packagr": "^22.0.0", - "postcss": "^8.4.0", - "rollup": "^4.0.0", - "tailwindcss": "^2.0.0 || ^3.0.0 || ^4.0.0", - "tslib": "^2.3.0", - "typescript": ">=6.0 <6.1", - "vitest": "^4.0.8" - }, - "peerDependenciesMeta": { - "@angular/core": { - "optional": true - }, - "@angular/localize": { - "optional": true - }, - "@angular/platform-browser": { - "optional": true - }, - "@angular/platform-server": { - "optional": true - }, - "@angular/service-worker": { - "optional": true - }, - "@angular/ssr": { - "optional": true - }, - "istanbul-lib-instrument": { - "optional": true - }, - "karma": { - "optional": true - }, - "less": { - "optional": true - }, - "ng-packagr": { - "optional": true - }, - "postcss": { - "optional": true - }, - "rollup": { - "optional": true - }, - "tailwindcss": { - "optional": true - }, - "vitest": { - "optional": true - } - } - }, - "node_modules/@angular/build/node_modules/@angular-devkit/architect": { - "version": "0.2201.2", - "resolved": "https://registry.npmjs.org/@angular-devkit/architect/-/architect-0.2201.2.tgz", - "integrity": "sha512-RRG3JA3hPH0ypbDIyquZt9DDTP5pOMPgqQ/iLSkok1MZdKiOgpk6FGfXCa1ei72SwlX7lnJdq94d6WWdqpbyKg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@angular-devkit/core": "22.1.2", - "rxjs": "7.8.2" - }, - "bin": { - "architect": "bin/cli.js" - }, - "engines": { - "node": "^22.22.3 || ^24.15.0 || >=26.0.0", - "npm": "^6.11.0 || ^7.5.6 || >=8.0.0", - "yarn": ">= 1.13.0" - } - }, - "node_modules/@angular/build/node_modules/@angular-devkit/core": { - "version": "22.1.2", - "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-22.1.2.tgz", - "integrity": "sha512-tF1oEE7KPs8I08HJQmH5e4GkLUB3+MXXy8t6gMJULaLFxZYP9K1oXRFLappMpdm9OIbEXOChk23hrho0By9aYg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ajv": "8.20.0", - "ajv-formats": "3.0.1", - "jsonc-parser": "3.3.1", - "picomatch": "4.0.5", - "rxjs": "7.8.2", - "source-map": "0.7.6" - }, - "engines": { - "node": "^22.22.3 || ^24.15.0 || >=26.0.0", - "npm": "^6.11.0 || ^7.5.6 || >=8.0.0", - "yarn": ">= 1.13.0" - }, - "peerDependencies": { - "chokidar": "^5.0.0" - }, - "peerDependenciesMeta": { - "chokidar": { - "optional": true - } - } - }, - "node_modules/@angular/build/node_modules/@inquirer/ansi": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-2.0.7.tgz", - "integrity": "sha512-3eTuUO1vH2cZm2ZKHeQxnOqlTi9EfZDGgIe3BL3I4u+rJHocr9Fz86M4fjYABPvFnQG/gGK551HqDiIcETwU6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" - } - }, - "node_modules/@angular/build/node_modules/@inquirer/confirm": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-6.1.1.tgz", - "integrity": "sha512-eb8DBZcz/2qHWQda4rk2JiQk5h9QV/cVHi1yjt0f69WFZMRFn0sJTye3EAP8icut8UDMjQPsaH5KbcOogefrFQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/core": "^11.2.1", - "@inquirer/type": "^4.0.7" - }, - "engines": { - "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@angular/build/node_modules/@inquirer/core": { - "version": "11.2.1", - "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-11.2.1.tgz", - "integrity": "sha512-Qd6GJT1yVyrZZCfN8W2qKF5ApmqryXRhRKCuip8h01x2w/esJQ2XIYc6f9abMIHgKQdBfFTSOdbHRLAhuM09UA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/ansi": "^2.0.7", - "@inquirer/figures": "^2.0.7", - "@inquirer/type": "^4.0.7", - "cli-width": "^4.1.0", - "fast-wrap-ansi": "^0.2.0", - "mute-stream": "^3.0.0", - "signal-exit": "^4.1.0" - }, - "engines": { - "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@angular/build/node_modules/@inquirer/figures": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-2.0.7.tgz", - "integrity": "sha512-aJ8TBPOGB6f/2qziPfElISTCEd5XOYTFckA2SGjhNmiKzfK/u4ot3v0DUzGVdUnKjN10EqnnEPck36BkyfLnJw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" - } - }, - "node_modules/@angular/build/node_modules/@inquirer/type": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-4.0.7.tgz", - "integrity": "sha512-t28inv14nMQ1PhKpsJPY+kEs/c00qzeCOS2gTNRyTjG5d6qsVA2fItxW4hkvGZ5lvanGLdtCzVIx5dwdRpN1+g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@angular/build/node_modules/agent-base": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-9.0.0.tgz", - "integrity": "sha512-TQf59BsZnytt8GdJKLPfUZ54g/iaUL2OWDSFCCvMOhsHduDQxO8xC4PNeyIkVcA5KwL2phPSv0douC0fgWzmnA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 20" - } - }, - "node_modules/@angular/build/node_modules/ajv": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", - "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/@angular/build/node_modules/entities": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz", - "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=20.19.0" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/@angular/build/node_modules/https-proxy-agent": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-9.1.0.tgz", - "integrity": "sha512-ag87y7cJJ9/3+GxFr8Oy4O5faDsGRGnBGsJj/YjOSsSx/5eadKLYTMPlzuR6obgoCDDm0abAAZitXXQkMOPSpA==", - "dev": true, - "license": "MIT", - "dependencies": { - "agent-base": "9.0.0", - "debug": "^4.3.4", - "proxy-agent-negotiate": "1.1.0" - }, - "engines": { - "node": ">= 20" - } - }, - "node_modules/@angular/build/node_modules/listr2": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/listr2/-/listr2-10.2.2.tgz", - "integrity": "sha512-JtNtbZj8q5BnDMR7trpwvwk3RIrANtIVzEUm8w7amp6xelLgyuq+4WZoTH913XaQAoH/cNdYhaNzBPA2U3xbDw==", - "dev": true, - "license": "MIT", - "dependencies": { - "cli-truncate": "^5.2.0", - "eventemitter3": "^5.0.4", - "log-update": "^6.1.0", - "rfdc": "^1.4.1", - "wrap-ansi": "^10.0.0" - }, - "engines": { - "node": ">=22.13.0" - } - }, - "node_modules/@angular/build/node_modules/magic-string": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.0.0.tgz", - "integrity": "sha512-CGvjzMN08iv6w1mm4/x3Gh1hLb4VnyRUA15FFpl6CsCIGGoe36k7kY5KNz9QDbSBN5I/fWHM6ZlIkUTa5xdUEA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.5" - } - }, - "node_modules/@angular/build/node_modules/mute-stream": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-3.0.0.tgz", - "integrity": "sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw==", - "dev": true, - "license": "ISC", - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@angular/build/node_modules/parse5-html-rewriting-stream": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/parse5-html-rewriting-stream/-/parse5-html-rewriting-stream-8.0.1.tgz", - "integrity": "sha512-NaRku2aMpUN1Sh1Gyk1KWUh2A7EJx2c6qYzvwsPtqhoHoaURshdrceYK3LunVCm3WHhm6FS7Vcczbvdh3/UIVw==", - "dev": true, - "license": "MIT", - "dependencies": { - "entities": "^8.0.0", - "parse5": "^8.0.0", - "parse5-sax-parser": "^8.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/@angular/build/node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/@angular/build/node_modules/semver": { - "version": "7.8.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", - "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@angular/build/node_modules/wrap-ansi": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-10.0.0.tgz", - "integrity": "sha512-SGcvg80f0wUy2/fXES19feHMz8E0JoXv2uNgHOu4Dgi2OrCy1lqwFYEJz1BLbDI0exjPMe/ZdzZ/YpGECBG/aQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.2.3", - "string-width": "^8.2.0", - "strip-ansi": "^7.1.2" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/@angular/cli": { - "version": "21.2.16", - "resolved": "https://registry.npmjs.org/@angular/cli/-/cli-21.2.16.tgz", - "integrity": "sha512-/O2Bsy4jae/op06ejyfsL6K4cD4yo7TEH9iesD4UPEvcWTnV8lCdmE2oxbc1WGT3DIsZ00yBQhURSbetDPGFCg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@angular-devkit/architect": "0.2102.16", - "@angular-devkit/core": "21.2.16", - "@angular-devkit/schematics": "21.2.16", - "@inquirer/prompts": "7.10.1", - "@listr2/prompt-adapter-inquirer": "3.0.5", - "@modelcontextprotocol/sdk": "1.26.0", - "@schematics/angular": "21.2.16", - "@yarnpkg/lockfile": "1.1.0", - "algoliasearch": "5.48.1", - "ini": "6.0.0", - "jsonc-parser": "3.3.1", - "listr2": "9.0.5", - "npm-package-arg": "13.0.2", - "pacote": "21.5.1", - "parse5-html-rewriting-stream": "8.0.0", - "semver": "7.7.4", - "yargs": "18.0.0", - "zod": "4.3.6" - }, - "bin": { - "ng": "bin/ng.js" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0", - "npm": "^6.11.0 || ^7.5.6 || >=8.0.0", - "yarn": ">= 1.13.0" - } - }, - "node_modules/@angular/common": { - "version": "22.1.0", - "resolved": "https://registry.npmjs.org/@angular/common/-/common-22.1.0.tgz", - "integrity": "sha512-67L8AS00egxwEKnoMhNDxy+TY+eKOwvwa+os0Odq8nLm7+Qh7JnMVeub8hfncpenOFqlC/RUjO2W9H7Gd2veNA==", - "license": "MIT", - "dependencies": { - "tslib": "^2.3.0" - }, - "engines": { - "node": "^22.22.3 || ^24.15.0 || >=26.0.0" - }, - "peerDependencies": { - "@angular/core": "22.1.0", - "rxjs": "^6.5.3 || ^7.4.0" - } - }, - "node_modules/@angular/compiler": { - "version": "22.1.0", - "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-22.1.0.tgz", - "integrity": "sha512-WCmuPnuXgqnqrkbrwqQRyldi1k3rlzNLVDl8ntINF7XWuJh0KfQLEkRK0FCCmBztWJkbGug4RBVnKTWlKhRzCQ==", - "license": "MIT", - "dependencies": { - "tslib": "^2.3.0" - }, - "engines": { - "node": "^22.22.3 || ^24.15.0 || >=26.0.0" - } - }, - "node_modules/@angular/compiler-cli": { - "version": "22.1.0", - "resolved": "https://registry.npmjs.org/@angular/compiler-cli/-/compiler-cli-22.1.0.tgz", - "integrity": "sha512-jL89dbzkrV8AeLaxedBgT7ErMnbfi2dvwDJuCrUgm3eCNfbcOpGbNxzH+wDgvbRg2Lhj11/u3JPbW50xA6rvvg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "8.0.1", - "@jridgewell/sourcemap-codec": "^1.4.14", - "chokidar": "^5.0.0", - "convert-source-map": "^1.5.1", - "reflect-metadata": "^0.2.0", - "semver": "^7.0.0", - "tslib": "^2.3.0", - "yargs": "^18.0.0" - }, - "bin": { - "ng-xi18n": "bundles/src/bin/ng_xi18n.js", - "ngc": "bundles/src/bin/ngc.js" - }, - "engines": { - "node": "^22.22.3 || ^24.15.0 || >=26.0.0" - }, - "peerDependencies": { - "@angular/compiler": "22.1.0", - "typescript": ">=6.0 <6.1" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@angular/core": { - "version": "22.1.0", - "resolved": "https://registry.npmjs.org/@angular/core/-/core-22.1.0.tgz", - "integrity": "sha512-X5UaMuOCI4HAvSQIs3QtM+5e0Cni16DRaHUIL3BIBd4ZQNnSH3pZ25TsKQ8Jlu/3hAQ9rzV278kNQcecooGJ7g==", - "license": "MIT", - "dependencies": { - "tslib": "^2.3.0" - }, - "engines": { - "node": "^22.22.3 || ^24.15.0 || >=26.0.0" - }, - "peerDependencies": { - "@angular/compiler": "22.1.0", - "rxjs": "^6.5.3 || ^7.4.0", - "zone.js": "~0.15.0 || ~0.16.0" - }, - "peerDependenciesMeta": { - "@angular/compiler": { - "optional": true - }, - "zone.js": { - "optional": true - } - } - }, - "node_modules/@angular/forms": { - "version": "22.1.0", - "resolved": "https://registry.npmjs.org/@angular/forms/-/forms-22.1.0.tgz", - "integrity": "sha512-nWlSM/pPp78Sx/fBM/tFEgZxdfZe50LkCE2/hkO22Fi1UM2maGc43LDsu/s6l0q9hFep4Wj+xa30KXDBS7Cn8A==", - "license": "MIT", - "dependencies": { - "@standard-schema/spec": "^1.0.0", - "tslib": "^2.3.0", - "zod": "^4.0.10" - }, - "engines": { - "node": "^22.22.3 || ^24.15.0 || >=26.0.0" - }, - "peerDependencies": { - "@angular/common": "22.1.0", - "@angular/core": "22.1.0", - "@angular/platform-browser": "22.1.0", - "rxjs": "^6.5.3 || ^7.4.0" - } - }, - "node_modules/@angular/platform-browser": { - "version": "22.1.0", - "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-22.1.0.tgz", - "integrity": "sha512-gqUYDUiPfwbaLYdH8WLnOLl3feo3OcNpnMO08HBHaUdi4TLNkC28xwa9fC6ANyYD22QZ5A3abSg8fmR6upWMwg==", - "license": "MIT", - "dependencies": { - "tslib": "^2.3.0" - }, - "engines": { - "node": "^22.22.3 || ^24.15.0 || >=26.0.0" - }, - "peerDependencies": { - "@angular/animations": "22.1.0", - "@angular/common": "22.1.0", - "@angular/core": "22.1.0" - }, - "peerDependenciesMeta": { - "@angular/animations": { - "optional": true - } - } - }, - "node_modules/@angular/router": { - "version": "22.1.0", - "resolved": "https://registry.npmjs.org/@angular/router/-/router-22.1.0.tgz", - "integrity": "sha512-42Bs0g+tV2gE70Lqnt+VD/+DWbvWwQcg8QgXkTIu3A504tYknrZG/wmvki2AJGyZhmyQ46B4pfXLG4WDP8MFSA==", - "license": "MIT", - "dependencies": { - "tslib": "^2.3.0" - }, - "engines": { - "node": "^22.22.3 || ^24.15.0 || >=26.0.0" - }, - "peerDependencies": { - "@angular/common": "22.1.0", - "@angular/core": "22.1.0", - "@angular/platform-browser": "22.1.0", - "rxjs": "^6.5.3 || ^7.4.0" - } - }, - "node_modules/@asamuzakjp/css-color": { - "version": "5.1.11", - "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.11.tgz", - "integrity": "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@asamuzakjp/generational-cache": "^1.0.1", - "@csstools/css-calc": "^3.2.0", - "@csstools/css-color-parser": "^4.1.0", - "@csstools/css-parser-algorithms": "^4.0.0", - "@csstools/css-tokenizer": "^4.0.0" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - } - }, - "node_modules/@asamuzakjp/dom-selector": { - "version": "6.8.1", - "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-6.8.1.tgz", - "integrity": "sha512-MvRz1nCqW0fsy8Qz4dnLIvhOlMzqDVBabZx6lH+YywFDdjXhMY37SmpV1XFX3JzG5GWHn63j6HX6QPr3lZXHvQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@asamuzakjp/nwsapi": "^2.3.9", - "bidi-js": "^1.0.3", - "css-tree": "^3.1.0", - "is-potential-custom-element-name": "^1.0.1", - "lru-cache": "^11.2.6" - } - }, - "node_modules/@asamuzakjp/generational-cache": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@asamuzakjp/generational-cache/-/generational-cache-1.0.1.tgz", - "integrity": "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - } - }, - "node_modules/@asamuzakjp/nwsapi": { - "version": "2.3.9", - "resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz", - "integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/@babel/code-frame": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-8.0.0.tgz", - "integrity": "sha512-dYYg153EyN2Ekbqw2zAsbd6/JR+9N2SEoC7YV2GyyqMM7x9bLDTjBD6XBhSMLH0wtIVyJj03jWNriQhaN+eoCw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^8.0.0", - "js-tokens": "^10.0.0" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/code-frame/node_modules/@babel/helper-validator-identifier": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.4.tgz", - "integrity": "sha512-4wFaiLd0bVo4cIoTXI3zKI038NIWE/cr3jvBjejOVYVxV/m8Ltav1USiGzG1fmS5J2RhgEOgXNNK46cRPnRsrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/compat-data": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-8.0.0.tgz", - "integrity": "sha512-DOjnob/cXOUgDOozCDeq/aK2p5y8dUIVdf6tNhEV1HQRd6I8aQ4f4fbtHRVEvb6lP3BGomrKHiS8ICAASSVQSw==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/core": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-8.0.1.tgz", - "integrity": "sha512-5FgxM4dLQpMJHSiVATk8foW263dVHQHBVpXYiimNECVWG01f4nFyEbQixeT6Mwvg7TayREJ2gpKl3o2RoMdnqw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^8.0.0", - "@babel/generator": "^8.0.0", - "@babel/helper-compilation-targets": "^8.0.0", - "@babel/helpers": "^8.0.0", - "@babel/parser": "^8.0.0", - "@babel/template": "^8.0.0", - "@babel/traverse": "^8.0.0", - "@babel/types": "^8.0.0", - "@types/gensync": "^1.0.5", - "convert-source-map": "^2.0.0", - "empathic": "^2.0.1", - "gensync": "^1.0.0-beta.2", - "import-meta-resolve": "^4.2.0", - "json5": "^2.2.3", - "obug": "^2.1.1", - "semver": "^7.7.3" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/core/node_modules/@babel/helper-string-parser": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.0.tgz", - "integrity": "sha512-6mJgmFFFIIO82vvoLt9XtRC7/TkzXfts1t/SpRX4IHSzMgqoPYCWesVu1udUPUWioAE/2fcG6WuI8zrkE1gwrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/core/node_modules/@babel/helper-validator-identifier": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.4.tgz", - "integrity": "sha512-4wFaiLd0bVo4cIoTXI3zKI038NIWE/cr3jvBjejOVYVxV/m8Ltav1USiGzG1fmS5J2RhgEOgXNNK46cRPnRsrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/core/node_modules/@babel/types": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.4.tgz", - "integrity": "sha512-eY+Yn3dCqTGmyiq2QRU66lA5FL8lqqqvecHt0fF3uHONIa7ToYsaCiWV8lOKqAs0Rb2SjixiKFROngnulPtt2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^8.0.0", - "@babel/helper-validator-identifier": "^8.0.4" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/core/node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@babel/generator": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-8.0.0.tgz", - "integrity": "sha512-NT9NrVwJsbSV6Y2FSstWa71EETOnzrjkL5/wX3D2mYHtKM+qvqB1DvR4D0Setb/gDBsHzRICifwEWMO8CnTF6g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^8.0.0", - "@babel/types": "^8.0.0", - "@jridgewell/gen-mapping": "^0.3.12", - "@jridgewell/trace-mapping": "^0.3.28", - "@types/jsesc": "^2.5.0", - "jsesc": "^3.0.2" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/generator/node_modules/@babel/helper-string-parser": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.0.tgz", - "integrity": "sha512-6mJgmFFFIIO82vvoLt9XtRC7/TkzXfts1t/SpRX4IHSzMgqoPYCWesVu1udUPUWioAE/2fcG6WuI8zrkE1gwrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/generator/node_modules/@babel/helper-validator-identifier": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.4.tgz", - "integrity": "sha512-4wFaiLd0bVo4cIoTXI3zKI038NIWE/cr3jvBjejOVYVxV/m8Ltav1USiGzG1fmS5J2RhgEOgXNNK46cRPnRsrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/generator/node_modules/@babel/types": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.4.tgz", - "integrity": "sha512-eY+Yn3dCqTGmyiq2QRU66lA5FL8lqqqvecHt0fF3uHONIa7ToYsaCiWV8lOKqAs0Rb2SjixiKFROngnulPtt2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^8.0.0", - "@babel/helper-validator-identifier": "^8.0.4" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/helper-annotate-as-pure": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-8.0.0.tgz", - "integrity": "sha512-NSpMkMsvvZqzThJ0p1B02cbtA2ObEyfBvq950bmNkyxsxvcxwhvvCB036rKhlEnuBBo30bOrk13u3FzlKSoRrw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^8.0.0" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/helper-annotate-as-pure/node_modules/@babel/helper-string-parser": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.0.tgz", - "integrity": "sha512-6mJgmFFFIIO82vvoLt9XtRC7/TkzXfts1t/SpRX4IHSzMgqoPYCWesVu1udUPUWioAE/2fcG6WuI8zrkE1gwrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/helper-annotate-as-pure/node_modules/@babel/helper-validator-identifier": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.4.tgz", - "integrity": "sha512-4wFaiLd0bVo4cIoTXI3zKI038NIWE/cr3jvBjejOVYVxV/m8Ltav1USiGzG1fmS5J2RhgEOgXNNK46cRPnRsrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/helper-annotate-as-pure/node_modules/@babel/types": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.4.tgz", - "integrity": "sha512-eY+Yn3dCqTGmyiq2QRU66lA5FL8lqqqvecHt0fF3uHONIa7ToYsaCiWV8lOKqAs0Rb2SjixiKFROngnulPtt2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^8.0.0", - "@babel/helper-validator-identifier": "^8.0.4" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-8.0.0.tgz", - "integrity": "sha512-JwculLABZvyPvyLBpwU/E/IbH2uM3mnxNtIJpxnIfb24y1PrdVxK5Dqjle4DpgqpGRnwgC7G8IkzPdSXZrO1Ew==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/compat-data": "^8.0.0", - "@babel/helper-validator-option": "^8.0.0", - "browserslist": "^4.24.0", - "lru-cache": "^11.0.0", - "semver": "^7.7.3" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/helper-globals": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-8.0.0.tgz", - "integrity": "sha512-lLozHOM6sWWlxNo8CYqHy4MBZeTvHXNgVPBfPOGsjPKUzHC2Az9QwB6gxdQmpwHl6GlQtbGgS+lj5887guDiLw==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/helper-split-export-declaration": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.24.7.tgz", - "integrity": "sha512-oy5V7pD+UvfkEATUKvIjvIAH/xCzfsFVw7ygW2SI6NClZzquT+mwdTfgfdbUiceh6iQO0CHtCPsyze/MZ2YbAA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.24.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-option": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-8.0.0.tgz", - "integrity": "sha512-U4Dybxh4WESWHt5XhBeExi4DrY0/DNK1aHpQbsrQXCUbFHuMweT0TpLEWKvaraV2Y6fS+ZXunsZ8zIuZIgvF2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/helpers": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-8.0.0.tgz", - "integrity": "sha512-wfbi91pM3py96oIiJEz7qIpyXDytgr9zQC1HEWwlGNVRAEmItuU/0a41ZUKu1sJGyhhOIpc4t5vk4PYzt8wpsg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^8.0.0", - "@babel/types": "^8.0.0" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/helpers/node_modules/@babel/helper-string-parser": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.0.tgz", - "integrity": "sha512-6mJgmFFFIIO82vvoLt9XtRC7/TkzXfts1t/SpRX4IHSzMgqoPYCWesVu1udUPUWioAE/2fcG6WuI8zrkE1gwrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/helpers/node_modules/@babel/helper-validator-identifier": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.4.tgz", - "integrity": "sha512-4wFaiLd0bVo4cIoTXI3zKI038NIWE/cr3jvBjejOVYVxV/m8Ltav1USiGzG1fmS5J2RhgEOgXNNK46cRPnRsrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/helpers/node_modules/@babel/types": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.4.tgz", - "integrity": "sha512-eY+Yn3dCqTGmyiq2QRU66lA5FL8lqqqvecHt0fF3uHONIa7ToYsaCiWV8lOKqAs0Rb2SjixiKFROngnulPtt2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^8.0.0", - "@babel/helper-validator-identifier": "^8.0.4" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/parser": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-8.0.4.tgz", - "integrity": "sha512-srpptsAkEbbNIC/q8nT7o+m6CQe8CJUTV/t7MYc9NnWlgYVtHOb7JH6SorxMhN0kuRJjVqXbKClG6xSbPtzz+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^8.0.4" - }, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/parser/node_modules/@babel/helper-string-parser": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.0.tgz", - "integrity": "sha512-6mJgmFFFIIO82vvoLt9XtRC7/TkzXfts1t/SpRX4IHSzMgqoPYCWesVu1udUPUWioAE/2fcG6WuI8zrkE1gwrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/parser/node_modules/@babel/helper-validator-identifier": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.4.tgz", - "integrity": "sha512-4wFaiLd0bVo4cIoTXI3zKI038NIWE/cr3jvBjejOVYVxV/m8Ltav1USiGzG1fmS5J2RhgEOgXNNK46cRPnRsrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/parser/node_modules/@babel/types": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.4.tgz", - "integrity": "sha512-eY+Yn3dCqTGmyiq2QRU66lA5FL8lqqqvecHt0fF3uHONIa7ToYsaCiWV8lOKqAs0Rb2SjixiKFROngnulPtt2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^8.0.0", - "@babel/helper-validator-identifier": "^8.0.4" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/template": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-8.0.0.tgz", - "integrity": "sha512-eAD0QW/AlbamBbw0FeGiwasbCVPq5ncW0HNVyLP3B9czqLyh4gvw+5JTSNt6le9+ziAU7mqDZsKTHf3jTb4chQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^8.0.0", - "@babel/parser": "^8.0.0", - "@babel/types": "^8.0.0" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/template/node_modules/@babel/helper-string-parser": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.0.tgz", - "integrity": "sha512-6mJgmFFFIIO82vvoLt9XtRC7/TkzXfts1t/SpRX4IHSzMgqoPYCWesVu1udUPUWioAE/2fcG6WuI8zrkE1gwrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/template/node_modules/@babel/helper-validator-identifier": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.4.tgz", - "integrity": "sha512-4wFaiLd0bVo4cIoTXI3zKI038NIWE/cr3jvBjejOVYVxV/m8Ltav1USiGzG1fmS5J2RhgEOgXNNK46cRPnRsrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/template/node_modules/@babel/types": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.4.tgz", - "integrity": "sha512-eY+Yn3dCqTGmyiq2QRU66lA5FL8lqqqvecHt0fF3uHONIa7ToYsaCiWV8lOKqAs0Rb2SjixiKFROngnulPtt2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^8.0.0", - "@babel/helper-validator-identifier": "^8.0.4" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/traverse": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-8.0.4.tgz", - "integrity": "sha512-bZnmqzGG8UZneG1lLxBoWIH0G6Gr1D846Yu4/3XnY6FhCndMR49u26nTY08u/dAxWmLWF9vGQOuC+84FfIUoeg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^8.0.0", - "@babel/generator": "^8.0.0", - "@babel/helper-globals": "^8.0.0", - "@babel/parser": "^8.0.4", - "@babel/template": "^8.0.0", - "@babel/types": "^8.0.4", - "obug": "^2.1.1" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/traverse/node_modules/@babel/helper-string-parser": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.0.tgz", - "integrity": "sha512-6mJgmFFFIIO82vvoLt9XtRC7/TkzXfts1t/SpRX4IHSzMgqoPYCWesVu1udUPUWioAE/2fcG6WuI8zrkE1gwrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/traverse/node_modules/@babel/helper-validator-identifier": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.4.tgz", - "integrity": "sha512-4wFaiLd0bVo4cIoTXI3zKI038NIWE/cr3jvBjejOVYVxV/m8Ltav1USiGzG1fmS5J2RhgEOgXNNK46cRPnRsrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/traverse/node_modules/@babel/types": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.4.tgz", - "integrity": "sha512-eY+Yn3dCqTGmyiq2QRU66lA5FL8lqqqvecHt0fF3uHONIa7ToYsaCiWV8lOKqAs0Rb2SjixiKFROngnulPtt2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^8.0.0", - "@babel/helper-validator-identifier": "^8.0.4" - }, - "engines": { - "node": "^22.18.0 || >=24.11.0" - } - }, - "node_modules/@babel/types": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", - "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.28.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@bramus/specificity": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/@bramus/specificity/-/specificity-2.4.2.tgz", - "integrity": "sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==", - "dev": true, - "license": "MIT", - "dependencies": { - "css-tree": "^3.0.0" - }, - "bin": { - "specificity": "bin/cli.js" - } - }, - "node_modules/@csstools/color-helpers": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.0.2.tgz", - "integrity": "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT-0", - "engines": { - "node": ">=20.19.0" - } - }, - "node_modules/@csstools/css-calc": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.2.0.tgz", - "integrity": "sha512-bR9e6o2BDB12jzN/gIbjHa5wLJ4UjD1CB9pM7ehlc0ddk6EBz+yYS1EV2MF55/HUxrHcB/hehAyt5vhsA3hx7w==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT", - "engines": { - "node": ">=20.19.0" - }, - "peerDependencies": { - "@csstools/css-parser-algorithms": "^4.0.0", - "@csstools/css-tokenizer": "^4.0.0" - } - }, - "node_modules/@csstools/css-color-parser": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.1.0.tgz", - "integrity": "sha512-U0KhLYmy2GVj6q4T3WaAe6NPuFYCPQoE3b0dRGxejWDgcPp8TP7S5rVdM5ZrFaqu4N67X8YaPBw14dQSYx3IyQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT", - "dependencies": { - "@csstools/color-helpers": "^6.0.2", - "@csstools/css-calc": "^3.2.0" - }, - "engines": { - "node": ">=20.19.0" - }, - "peerDependencies": { - "@csstools/css-parser-algorithms": "^4.0.0", - "@csstools/css-tokenizer": "^4.0.0" - } - }, - "node_modules/@csstools/css-parser-algorithms": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.0.tgz", - "integrity": "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT", - "engines": { - "node": ">=20.19.0" - }, - "peerDependencies": { - "@csstools/css-tokenizer": "^4.0.0" - } - }, - "node_modules/@csstools/css-syntax-patches-for-csstree": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.3.tgz", - "integrity": "sha512-SH60bMfrRCJF3morcdk57WklujF4Jr/EsQUzqkarfHXEFcAR1gg7fS/chAE922Sehgzc1/+Tz5H3Ypa1HiEKrg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT-0", - "peerDependencies": { - "css-tree": "^3.2.1" - }, - "peerDependenciesMeta": { - "css-tree": { - "optional": true - } - } - }, - "node_modules/@csstools/css-tokenizer": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-4.0.0.tgz", - "integrity": "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT", - "engines": { - "node": ">=20.19.0" - } - }, - "node_modules/@emnapi/core": { - "version": "1.11.2", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.2.tgz", - "integrity": "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.2", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.11.2", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", - "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", - "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", - "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", - "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", - "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", - "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", - "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", - "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", - "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", - "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", - "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", - "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", - "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", - "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", - "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", - "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", - "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", - "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", - "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", - "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", - "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", - "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", - "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", - "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", - "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@exodus/bytes": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.0.tgz", - "integrity": "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - }, - "peerDependencies": { - "@noble/hashes": "^1.8.0 || ^2.0.0" - }, - "peerDependenciesMeta": { - "@noble/hashes": { - "optional": true - } - } - }, - "node_modules/@gar/promise-retry": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@gar/promise-retry/-/promise-retry-1.0.3.tgz", - "integrity": "sha512-GmzA9ckNokPypTg10pgpeHNQe7ph+iIKKmhKu3Ob9ANkswreCx7R3cKmY781K8QK3AqVL3xVh9A42JvIAbkkSA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@harperfast/extended-iterable": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@harperfast/extended-iterable/-/extended-iterable-1.0.3.tgz", - "integrity": "sha512-sSAYhQca3rDWtQUHSAPeO7axFIUJOI6hn1gjRC5APVE1a90tuyT8f5WIgRsFhhWA7htNkju2veB9eWL6YHi/Lw==", - "dev": true, - "license": "Apache-2.0", - "optional": true - }, - "node_modules/@hono/node-server": { - "version": "1.19.14", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", - "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18.14.1" - }, - "peerDependencies": { - "hono": "^4" - } - }, - "node_modules/@inquirer/ansi": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-1.0.2.tgz", - "integrity": "sha512-S8qNSZiYzFd0wAcyG5AXCvUHC5Sr7xpZ9wZ2py9XR88jUz8wooStVx5M6dRzczbBWjic9NP7+rY0Xi7qqK/aMQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@inquirer/checkbox": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.3.2.tgz", - "integrity": "sha512-VXukHf0RR1doGe6Sm4F0Em7SWYLTHSsbGfJdS9Ja2bX5/D5uwVOEjr07cncLROdBvmnvCATYEWlHqYmXv2IlQA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/ansi": "^1.0.2", - "@inquirer/core": "^10.3.2", - "@inquirer/figures": "^1.0.15", - "@inquirer/type": "^3.0.10", - "yoctocolors-cjs": "^2.1.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/confirm": { - "version": "5.1.21", - "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.21.tgz", - "integrity": "sha512-KR8edRkIsUayMXV+o3Gv+q4jlhENF9nMYUZs9PA2HzrXeHI8M5uDag70U7RJn9yyiMZSbtF5/UexBtAVtZGSbQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/type": "^3.0.10" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/core": { - "version": "10.3.2", - "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.3.2.tgz", - "integrity": "sha512-43RTuEbfP8MbKzedNqBrlhhNKVwoK//vUFNW3Q3vZ88BLcrs4kYpGg+B2mm5p2K/HfygoCxuKwJJiv8PbGmE0A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/ansi": "^1.0.2", - "@inquirer/figures": "^1.0.15", - "@inquirer/type": "^3.0.10", - "cli-width": "^4.1.0", - "mute-stream": "^2.0.0", - "signal-exit": "^4.1.0", - "wrap-ansi": "^6.2.0", - "yoctocolors-cjs": "^2.1.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/editor": { - "version": "4.2.23", - "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.23.tgz", - "integrity": "sha512-aLSROkEwirotxZ1pBaP8tugXRFCxW94gwrQLxXfrZsKkfjOYC1aRvAZuhpJOb5cu4IBTJdsCigUlf2iCOu4ZDQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/external-editor": "^1.0.3", - "@inquirer/type": "^3.0.10" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/expand": { - "version": "4.0.23", - "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.23.tgz", - "integrity": "sha512-nRzdOyFYnpeYTTR2qFwEVmIWypzdAx/sIkCMeTNTcflFOovfqUk+HcFhQQVBftAh9gmGrpFj6QcGEqrDMDOiew==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/type": "^3.0.10", - "yoctocolors-cjs": "^2.1.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/external-editor": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.3.tgz", - "integrity": "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==", - "dev": true, - "license": "MIT", - "dependencies": { - "chardet": "^2.1.1", - "iconv-lite": "^0.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/figures": { - "version": "1.0.15", - "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.15.tgz", - "integrity": "sha512-t2IEY+unGHOzAaVM5Xx6DEWKeXlDDcNPeDyUpsRc6CUhBfU3VQOEl+Vssh7VNp1dR8MdUJBWhuObjXCsVpjN5g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@inquirer/input": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.3.1.tgz", - "integrity": "sha512-kN0pAM4yPrLjJ1XJBjDxyfDduXOuQHrBB8aLDMueuwUGn+vNpF7Gq7TvyVxx8u4SHlFFj4trmj+a2cbpG4Jn1g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/type": "^3.0.10" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/number": { - "version": "3.0.23", - "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.23.tgz", - "integrity": "sha512-5Smv0OK7K0KUzUfYUXDXQc9jrf8OHo4ktlEayFlelCjwMXz0299Y8OrI+lj7i4gCBY15UObk76q0QtxjzFcFcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/type": "^3.0.10" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/password": { - "version": "4.0.23", - "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.23.tgz", - "integrity": "sha512-zREJHjhT5vJBMZX/IUbyI9zVtVfOLiTO66MrF/3GFZYZ7T4YILW5MSkEYHceSii/KtRk+4i3RE7E1CUXA2jHcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/ansi": "^1.0.2", - "@inquirer/core": "^10.3.2", - "@inquirer/type": "^3.0.10" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/prompts": { - "version": "7.10.1", - "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.10.1.tgz", - "integrity": "sha512-Dx/y9bCQcXLI5ooQ5KyvA4FTgeo2jYj/7plWfV5Ak5wDPKQZgudKez2ixyfz7tKXzcJciTxqLeK7R9HItwiByg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/checkbox": "^4.3.2", - "@inquirer/confirm": "^5.1.21", - "@inquirer/editor": "^4.2.23", - "@inquirer/expand": "^4.0.23", - "@inquirer/input": "^4.3.1", - "@inquirer/number": "^3.0.23", - "@inquirer/password": "^4.0.23", - "@inquirer/rawlist": "^4.1.11", - "@inquirer/search": "^3.2.2", - "@inquirer/select": "^4.4.2" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/rawlist": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.1.11.tgz", - "integrity": "sha512-+LLQB8XGr3I5LZN/GuAHo+GpDJegQwuPARLChlMICNdwW7OwV2izlCSCxN6cqpL0sMXmbKbFcItJgdQq5EBXTw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/type": "^3.0.10", - "yoctocolors-cjs": "^2.1.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/search": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.2.2.tgz", - "integrity": "sha512-p2bvRfENXCZdWF/U2BXvnSI9h+tuA8iNqtUKb9UWbmLYCRQxd8WkvwWvYn+3NgYaNwdUkHytJMGG4MMLucI1kA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/figures": "^1.0.15", - "@inquirer/type": "^3.0.10", - "yoctocolors-cjs": "^2.1.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/select": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.4.2.tgz", - "integrity": "sha512-l4xMuJo55MAe+N7Qr4rX90vypFwCajSakx59qe/tMaC1aEHWLyw68wF4o0A4SLAY4E0nd+Vt+EyskeDIqu1M6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/ansi": "^1.0.2", - "@inquirer/core": "^10.3.2", - "@inquirer/figures": "^1.0.15", - "@inquirer/type": "^3.0.10", - "yoctocolors-cjs": "^2.1.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/type": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.10.tgz", - "integrity": "sha512-BvziSRxfz5Ov8ch0z/n3oijRSEcEsHnhggm4xFZe93DHcUCTlutlq9Ox4SVENAfcRD22UQq7T/atg9Wr3k09eA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@isaacs/fs-minipass": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", - "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", - "dev": true, - "license": "ISC", - "dependencies": { - "minipass": "^7.0.4" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.31", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", - "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, - "node_modules/@listr2/prompt-adapter-inquirer": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/@listr2/prompt-adapter-inquirer/-/prompt-adapter-inquirer-3.0.5.tgz", - "integrity": "sha512-WELs+hj6xcilkloBXYf9XXK8tYEnKsgLj01Xl5ONUJpKjmT5hGVUzNUS5tooUxs7pGMrw+jFD/41WpqW4V3LDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@inquirer/type": "^3.0.8" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@inquirer/prompts": ">= 3 < 8", - "listr2": "9.0.5" - } - }, - "node_modules/@lmdb/lmdb-darwin-arm64": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-darwin-arm64/-/lmdb-darwin-arm64-3.5.6.tgz", - "integrity": "sha512-mY5FG4TjPAkY4P0w+OhHaUka5mDh2TX2WKYIwuKzJ1zeW3VvRgxdam/lGJTquI+bthTx5CSHDW+BAQCnNAzkEA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@lmdb/lmdb-darwin-x64": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-darwin-x64/-/lmdb-darwin-x64-3.5.6.tgz", - "integrity": "sha512-foa+pwitysO8k+xhs7psBFfTKnVgR69NlZRRTHaFVDqphh7AdGpLeyRzKw/ofatr/sN6TiHRRW6mmop0ZrrppQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@lmdb/lmdb-linux-arm": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-arm/-/lmdb-linux-arm-3.5.6.tgz", - "integrity": "sha512-QR4YRyR5h5Z8eGXrNQjiyo2NNDfqi3tCc9dQG5Is1blCt+qWw1ZoBWhlWAr5d+jshkifMIJjVHzHGKbkKzF8Tw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@lmdb/lmdb-linux-arm64": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-arm64/-/lmdb-linux-arm64-3.5.6.tgz", - "integrity": "sha512-HmiyFFdJa38s1heCMSooSPaBSFTHJ3C+ERPp28xAPlDX1YiALJVOgbry065nXd8Y7KISWjnw05zpG1RX8IfftA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@lmdb/lmdb-linux-x64": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-x64/-/lmdb-linux-x64-3.5.6.tgz", - "integrity": "sha512-ADzCuCF2cTNiX9kDScqcz1fjnAkxPpQNneV3KFTdV3wWtVlI2sTGzySoMTgDpinkMMFj1NTJlxA6XR8fwc4hlA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@lmdb/lmdb-win32-arm64": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-win32-arm64/-/lmdb-win32-arm64-3.5.6.tgz", - "integrity": "sha512-J7A9aEQsQiv0TYtBGL7NDIPp2lOS8nnl+zm4sWZm1xlsTTaQ4PgD096Adzdrk27rw3UxCkDXdCUa4ax41oztBQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@lmdb/lmdb-win32-x64": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-win32-x64/-/lmdb-win32-x64-3.5.6.tgz", - "integrity": "sha512-1g7G0knRX2iV/voDu54yxrGqw5Dk0w2oIYb7dgJq8IkOi+m7wbD8Q3QpPFjh0C01G58S88dqGn03len6UPCXsg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@modelcontextprotocol/sdk": { - "version": "1.26.0", - "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.26.0.tgz", - "integrity": "sha512-Y5RmPncpiDtTXDbLKswIJzTqu2hyBKxTNsgKqKclDbhIgg1wgtf1fRuvxgTnRfcnxtvvgbIEcqUOzZrJ6iSReg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@hono/node-server": "^1.19.9", - "ajv": "^8.17.1", - "ajv-formats": "^3.0.1", - "content-type": "^1.0.5", - "cors": "^2.8.5", - "cross-spawn": "^7.0.5", - "eventsource": "^3.0.2", - "eventsource-parser": "^3.0.0", - "express": "^5.2.1", - "express-rate-limit": "^8.2.1", - "hono": "^4.11.4", - "jose": "^6.1.3", - "json-schema-typed": "^8.0.2", - "pkce-challenge": "^5.0.0", - "raw-body": "^3.0.0", - "zod": "^3.25 || ^4.0", - "zod-to-json-schema": "^3.25.1" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@cfworker/json-schema": "^4.1.1", - "zod": "^3.25 || ^4.0" - }, - "peerDependenciesMeta": { - "@cfworker/json-schema": { - "optional": true - }, - "zod": { - "optional": false - } - } - }, - "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.4.tgz", - "integrity": "sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.4.tgz", - "integrity": "sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.4.tgz", - "integrity": "sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.4.tgz", - "integrity": "sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.4.tgz", - "integrity": "sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.4.tgz", - "integrity": "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@napi-rs/nice": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice/-/nice-1.1.1.tgz", - "integrity": "sha512-xJIPs+bYuc9ASBl+cvGsKbGrJmS6fAKaSZCnT0lhahT5rhA2VVy9/EcIgd2JhtEuFOJNx7UHNn/qiTPTY4nrQw==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">= 10" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - }, - "optionalDependencies": { - "@napi-rs/nice-android-arm-eabi": "1.1.1", - "@napi-rs/nice-android-arm64": "1.1.1", - "@napi-rs/nice-darwin-arm64": "1.1.1", - "@napi-rs/nice-darwin-x64": "1.1.1", - "@napi-rs/nice-freebsd-x64": "1.1.1", - "@napi-rs/nice-linux-arm-gnueabihf": "1.1.1", - "@napi-rs/nice-linux-arm64-gnu": "1.1.1", - "@napi-rs/nice-linux-arm64-musl": "1.1.1", - "@napi-rs/nice-linux-ppc64-gnu": "1.1.1", - "@napi-rs/nice-linux-riscv64-gnu": "1.1.1", - "@napi-rs/nice-linux-s390x-gnu": "1.1.1", - "@napi-rs/nice-linux-x64-gnu": "1.1.1", - "@napi-rs/nice-linux-x64-musl": "1.1.1", - "@napi-rs/nice-openharmony-arm64": "1.1.1", - "@napi-rs/nice-win32-arm64-msvc": "1.1.1", - "@napi-rs/nice-win32-ia32-msvc": "1.1.1", - "@napi-rs/nice-win32-x64-msvc": "1.1.1" - } - }, - "node_modules/@napi-rs/nice-android-arm-eabi": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-android-arm-eabi/-/nice-android-arm-eabi-1.1.1.tgz", - "integrity": "sha512-kjirL3N6TnRPv5iuHw36wnucNqXAO46dzK9oPb0wj076R5Xm8PfUVA9nAFB5ZNMmfJQJVKACAPd/Z2KYMppthw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-android-arm64": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-android-arm64/-/nice-android-arm64-1.1.1.tgz", - "integrity": "sha512-blG0i7dXgbInN5urONoUCNf+DUEAavRffrO7fZSeoRMJc5qD+BJeNcpr54msPF6qfDD6kzs9AQJogZvT2KD5nw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-darwin-arm64": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-darwin-arm64/-/nice-darwin-arm64-1.1.1.tgz", - "integrity": "sha512-s/E7w45NaLqTGuOjC2p96pct4jRfo61xb9bU1unM/MJ/RFkKlJyJDx7OJI/O0ll/hrfpqKopuAFDV8yo0hfT7A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-darwin-x64": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-darwin-x64/-/nice-darwin-x64-1.1.1.tgz", - "integrity": "sha512-dGoEBnVpsdcC+oHHmW1LRK5eiyzLwdgNQq3BmZIav+9/5WTZwBYX7r5ZkQC07Nxd3KHOCkgbHSh4wPkH1N1LiQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-freebsd-x64": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-freebsd-x64/-/nice-freebsd-x64-1.1.1.tgz", - "integrity": "sha512-kHv4kEHAylMYmlNwcQcDtXjklYp4FCf0b05E+0h6nDHsZ+F0bDe04U/tXNOqrx5CmIAth4vwfkjjUmp4c4JktQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-linux-arm-gnueabihf": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-arm-gnueabihf/-/nice-linux-arm-gnueabihf-1.1.1.tgz", - "integrity": "sha512-E1t7K0efyKXZDoZg1LzCOLxgolxV58HCkaEkEvIYQx12ht2pa8hoBo+4OB3qh7e+QiBlp1SRf+voWUZFxyhyqg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-linux-arm64-gnu": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-arm64-gnu/-/nice-linux-arm64-gnu-1.1.1.tgz", - "integrity": "sha512-CIKLA12DTIZlmTaaKhQP88R3Xao+gyJxNWEn04wZwC2wmRapNnxCUZkVwggInMJvtVElA+D4ZzOU5sX4jV+SmQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-linux-arm64-musl": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-arm64-musl/-/nice-linux-arm64-musl-1.1.1.tgz", - "integrity": "sha512-+2Rzdb3nTIYZ0YJF43qf2twhqOCkiSrHx2Pg6DJaCPYhhaxbLcdlV8hCRMHghQ+EtZQWGNcS2xF4KxBhSGeutg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-linux-ppc64-gnu": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-ppc64-gnu/-/nice-linux-ppc64-gnu-1.1.1.tgz", - "integrity": "sha512-4FS8oc0GeHpwvv4tKciKkw3Y4jKsL7FRhaOeiPei0X9T4Jd619wHNe4xCLmN2EMgZoeGg+Q7GY7BsvwKpL22Tg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-linux-riscv64-gnu": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-riscv64-gnu/-/nice-linux-riscv64-gnu-1.1.1.tgz", - "integrity": "sha512-HU0nw9uD4FO/oGCCk409tCi5IzIZpH2agE6nN4fqpwVlCn5BOq0MS1dXGjXaG17JaAvrlpV5ZeyZwSon10XOXw==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-linux-s390x-gnu": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-s390x-gnu/-/nice-linux-s390x-gnu-1.1.1.tgz", - "integrity": "sha512-2YqKJWWl24EwrX0DzCQgPLKQBxYDdBxOHot1KWEq7aY2uYeX+Uvtv4I8xFVVygJDgf6/92h9N3Y43WPx8+PAgQ==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-linux-x64-gnu": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-x64-gnu/-/nice-linux-x64-gnu-1.1.1.tgz", - "integrity": "sha512-/gaNz3R92t+dcrfCw/96pDopcmec7oCcAQ3l/M+Zxr82KT4DljD37CpgrnXV+pJC263JkW572pdbP3hP+KjcIg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-linux-x64-musl": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-x64-musl/-/nice-linux-x64-musl-1.1.1.tgz", - "integrity": "sha512-xScCGnyj/oppsNPMnevsBe3pvNaoK7FGvMjT35riz9YdhB2WtTG47ZlbxtOLpjeO9SqqQ2J2igCmz6IJOD5JYw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-openharmony-arm64": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-openharmony-arm64/-/nice-openharmony-arm64-1.1.1.tgz", - "integrity": "sha512-6uJPRVwVCLDeoOaNyeiW0gp2kFIM4r7PL2MczdZQHkFi9gVlgm+Vn+V6nTWRcu856mJ2WjYJiumEajfSm7arPQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-win32-arm64-msvc": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-win32-arm64-msvc/-/nice-win32-arm64-msvc-1.1.1.tgz", - "integrity": "sha512-uoTb4eAvM5B2aj/z8j+Nv8OttPf2m+HVx3UjA5jcFxASvNhQriyCQF1OB1lHL43ZhW+VwZlgvjmP5qF3+59atA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-win32-ia32-msvc": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-win32-ia32-msvc/-/nice-win32-ia32-msvc-1.1.1.tgz", - "integrity": "sha512-CNQqlQT9MwuCsg1Vd/oKXiuH+TcsSPJmlAFc5frFyX/KkOh0UpBLEj7aoY656d5UKZQMQFP7vJNa1DNUNORvug==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/nice-win32-x64-msvc": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/nice-win32-x64-msvc/-/nice-win32-x64-msvc-1.1.1.tgz", - "integrity": "sha512-vB+4G/jBQCAh0jelMTY3+kgFy00Hlx2f2/1zjMoH821IbplbWZOkLiTYXQkygNTzQJTq5cvwBDgn2ppHD+bglQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.2.tgz", - "integrity": "sha512-JfB4kuJQjaoHuCTseIINHtHWeJnvgEcxjwA5t/Y00ZgaOO1Crz3fjT/p8kT28zA/Caz7oiUMn3d6H2yOVCVwuw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@tybys/wasm-util": "^0.10.3" - }, - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=23.5.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - }, - "peerDependencies": { - "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.3", - "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.3" - } - }, - "node_modules/@npmcli/agent": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-4.0.2.tgz", - "integrity": "sha512-EUEuWAxnL07Sp5/iC/1X6Xj+XThUvnbei9zfRWZdEXa7lss9RTHMhAHBeg+MZ5To9s/gGaSI+UwZTPdYMvKSeg==", - "dev": true, - "license": "ISC", - "dependencies": { - "agent-base": "^7.1.0", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.1", - "lru-cache": "^11.2.1", - "socks-proxy-agent": "^8.0.3" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@npmcli/fs": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-5.0.0.tgz", - "integrity": "sha512-7OsC1gNORBEawOa5+j2pXN9vsicaIOH5cPXxoR6fJOmH6/EXpJB2CajXOu1fPRFun2m1lktEFX11+P89hqO/og==", - "dev": true, - "license": "ISC", - "dependencies": { - "semver": "^7.3.5" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@npmcli/git": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@npmcli/git/-/git-7.0.2.tgz", - "integrity": "sha512-oeolHDjExNAJAnlYP2qzNjMX/Xi9bmu78C9dIGr4xjobrSKbuMYCph8lTzn4vnW3NjIqVmw/f8BCfouqyJXlRg==", - "dev": true, - "license": "ISC", - "dependencies": { - "@gar/promise-retry": "^1.0.0", - "@npmcli/promise-spawn": "^9.0.0", - "ini": "^6.0.0", - "lru-cache": "^11.2.1", - "npm-pick-manifest": "^11.0.1", - "proc-log": "^6.0.0", - "semver": "^7.3.5", - "which": "^6.0.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@npmcli/git/node_modules/isexe": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", - "integrity": "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=20" - } - }, - "node_modules/@npmcli/git/node_modules/which": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/which/-/which-6.0.1.tgz", - "integrity": "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^4.0.0" - }, - "bin": { - "node-which": "bin/which.js" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@npmcli/installed-package-contents": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@npmcli/installed-package-contents/-/installed-package-contents-4.0.0.tgz", - "integrity": "sha512-yNyAdkBxB72gtZ4GrwXCM0ZUedo9nIbOMKfGjt6Cu6DXf0p8y1PViZAKDC8q8kv/fufx0WTjRBdSlyrvnP7hmA==", - "dev": true, - "license": "ISC", - "dependencies": { - "npm-bundled": "^5.0.0", - "npm-normalize-package-bin": "^5.0.0" - }, - "bin": { - "installed-package-contents": "bin/index.js" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@npmcli/node-gyp": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@npmcli/node-gyp/-/node-gyp-5.0.0.tgz", - "integrity": "sha512-uuG5HZFXLfyFKqg8QypsmgLQW7smiRjVc45bqD/ofZZcR/uxEjgQU8qDPv0s9TEeMUiAAU/GC5bR6++UdTirIQ==", - "dev": true, - "license": "ISC", - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@npmcli/package-json": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/@npmcli/package-json/-/package-json-7.0.5.tgz", - "integrity": "sha512-iVuTlG3ORq2iaVa1IWUxAO/jIp77tUKBhoMjuzYW2kL4MLN1bi/ofqkZ7D7OOwh8coAx1/S2ge0rMdGv8sLSOQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "@npmcli/git": "^7.0.0", - "glob": "^13.0.0", - "hosted-git-info": "^9.0.0", - "json-parse-even-better-errors": "^5.0.0", - "proc-log": "^6.0.0", - "semver": "^7.5.3", - "spdx-expression-parse": "^4.0.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@npmcli/promise-spawn": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/@npmcli/promise-spawn/-/promise-spawn-9.0.1.tgz", - "integrity": "sha512-OLUaoqBuyxeTqUvjA3FZFiXUfYC1alp3Sa99gW3EUDz3tZ3CbXDdcZ7qWKBzicrJleIgucoWamWH1saAmH/l2Q==", - "dev": true, - "license": "ISC", - "dependencies": { - "which": "^6.0.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@npmcli/promise-spawn/node_modules/isexe": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", - "integrity": "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=20" - } - }, - "node_modules/@npmcli/promise-spawn/node_modules/which": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/which/-/which-6.0.1.tgz", - "integrity": "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^4.0.0" - }, - "bin": { - "node-which": "bin/which.js" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@npmcli/redact": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@npmcli/redact/-/redact-4.0.0.tgz", - "integrity": "sha512-gOBg5YHMfZy+TfHArfVogwgfBeQnKbbGo3pSUyK/gSI0AVu+pEiDVcKlQb0D8Mg1LNRZILZ6XG8I5dJ4KuAd9Q==", - "dev": true, - "license": "ISC", - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@npmcli/run-script": { - "version": "10.0.4", - "resolved": "https://registry.npmjs.org/@npmcli/run-script/-/run-script-10.0.4.tgz", - "integrity": "sha512-mGUWr1uMnf0le2TwfOZY4SFxZGXGfm4Jtay/nwAa2FLNAKXUoUwaGwBMNH36UHPtinWfTSJ3nqFQr0091CxVGg==", - "dev": true, - "license": "ISC", - "dependencies": { - "@npmcli/node-gyp": "^5.0.0", - "@npmcli/package-json": "^7.0.0", - "@npmcli/promise-spawn": "^9.0.0", - "node-gyp": "^12.1.0", - "proc-log": "^6.0.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@oxc-parser/binding-android-arm-eabi": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.142.0.tgz", - "integrity": "sha512-ZiRGDutGsv1G6bL/ozy/koC0Sv39T1DqyoC4KD1DOy9ZoACm1O5UWhEK2c02Qdk+4lfLVkvFa/mQ0fm/4h1BtQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-android-arm64": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm64/-/binding-android-arm64-0.142.0.tgz", - "integrity": "sha512-WZkvGRLNQTz8lR9zP5nLjUdlroRCopBu3g9zF1p/laE6DzT1UbQo8Rdz5MWhaJUPYg/6gp+jo7HUgsyKaN1FtQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-darwin-arm64": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-arm64/-/binding-darwin-arm64-0.142.0.tgz", - "integrity": "sha512-l4khS8LQOOVYsGRVARo1gSaCT/aBSceUVXgtovWc2+drnxVuDr082WA3OCHVdVzIz5JIrP/y9CWsSKxBDNmYGg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-darwin-x64": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-x64/-/binding-darwin-x64-0.142.0.tgz", - "integrity": "sha512-QBsNF3nqlXmcH2B1YOPqQYmCJoy4HuIjUxGbBO/k5JAJUl68ghU2psRY2zPk+RyBaWqKP/qfL4oaFgEMCdwskA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-freebsd-x64": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-freebsd-x64/-/binding-freebsd-x64-0.142.0.tgz", - "integrity": "sha512-b7Q7m4Cqc6XqNhri3R+QhU+GVy646Pn+bkdhrDdWym/Fdi0ZUa+d73H9dm5H91JtbtAQ/z1d8XKMW3oOV8a4tQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-linux-arm-gnueabihf": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.142.0.tgz", - "integrity": "sha512-3riVS5IhdH3uCZj1Y9ftDQlR0dvLsIlw/edrRqk8JhgNd5K0XSs+UBtgh50N13CAlW9/TXj6sVGXaKNBocd0Yg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-linux-arm-musleabihf": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.142.0.tgz", - "integrity": "sha512-NmXUOpgpTSkhl795TiXmWppTwmSJ92RC1qvD6e4XOF+slgmo3e6Ah+kEu+6AN8s7NAOEwqGmir58MgSQSWmBSA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-linux-arm64-gnu": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.142.0.tgz", - "integrity": "sha512-gc0EXsKtXgerujmU2Bql3u1L1HsSQ2774R83idq/FoNMPVV/RY/1ErFsvnit7KoiP/sLvzQixeUo4Ut0ic0wmw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-linux-arm64-musl": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.142.0.tgz", - "integrity": "sha512-F2XvmWSE0uWpie+jHKKIFgdVOe9ypGhkEZxKx5DuW215K6cbAC274yYaPkcM7EqY4Df3Weyhpcz3lsURyH2LVg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-linux-ppc64-gnu": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.142.0.tgz", - "integrity": "sha512-wLMbT21U/QxknQsk+VvNF0b9D2/aGWhcaQQQ+VYlE8FwD5+GoWZIPPXNzyHmkYyhm0KB3itL+TBavjMatqNnYA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-linux-riscv64-gnu": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.142.0.tgz", - "integrity": "sha512-+G8F/4ckwT7FCJV4H2bt09xEzJbjNCfuL4Sp1AYNaFtFMVtgIGMuJlteT82U+K0UIZ/DzAR/LDlMFnEuajG7Kw==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-linux-riscv64-musl": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.142.0.tgz", - "integrity": "sha512-hTsHtTLxMAfCo+rpF5K3qZJKW2NpPN/CHd4mYB3y7XlSdspHkd2gehDIofP64AacA9nWQw2tY3O7wR6UY8IVOA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-linux-s390x-gnu": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.142.0.tgz", - "integrity": "sha512-6y7qYY3TCUDYjqswImdTGl92y+KA/80twALegQPN27kfY+bG7Ib1+L3jbmrCZQx6wrVnai9IPsEZp07I0hx7JQ==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-linux-x64-gnu": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.142.0.tgz", - "integrity": "sha512-i69kAWU+2LgoH5bR+zWiiu+UzAw7Oxkwv7COeJTeY19pn4e70nKQcr9Pm6cL2Z0Z54d+gl9qADlK/0yyuCPiBA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-linux-x64-musl": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-musl/-/binding-linux-x64-musl-0.142.0.tgz", - "integrity": "sha512-4SQs678MmjYVrmhAgCWD4o0vpaFszXw9xLX5p2Z9MMFcltxiLkA88wQjh80YHjPrXtpyZ2CWI5m+1yNKM0m2Pw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-openharmony-arm64": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-openharmony-arm64/-/binding-openharmony-arm64-0.142.0.tgz", - "integrity": "sha512-YHpx9N7Ln3a++Tc8rv+H7mrK1zyJQOAwCFg8LZ3lTs1T5afGWeZrLPhPT9HLnIwSjCyJqPWVMIrMxbjcmBr2oQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-wasm32-wasi": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-wasm32-wasi/-/binding-wasm32-wasi-0.142.0.tgz", - "integrity": "sha512-3pLDyY3+oogW73RM5uehNgAiR/Xfb7fvO2Q1Z1gIqZ2+50XDVQmBVlRkHXZTU4gKnQHpwETNsYQVsJ3joVB2iA==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "1.11.2", - "@emnapi/runtime": "1.11.2", - "@napi-rs/wasm-runtime": "^1.1.6" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-win32-arm64-msvc": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.142.0.tgz", - "integrity": "sha512-Had/VeVY28Oyb0K+Q4FV8KCzoBycIh93oDK6pCbya9lkzdq+ikMHMgBubsdqqlybjJmQRawCQRrnBRHyQwYvcQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-win32-ia32-msvc": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.142.0.tgz", - "integrity": "sha512-GGi3+YphVHavvgs6gum2UXoNCqzHAmPt/nXkn8ZQZstV2Q1qZD1Mn8fz/nWrDkefHQtrG/+1/XrbMxsBTo6Svw==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-parser/binding-win32-x64-msvc": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.142.0.tgz", - "integrity": "sha512-Ny/Wv4Us1LGC/ljwNTp+Hx3r/pH15EFfeDF0p+n898gt+TtRd6C9SccHcuUhDiNTb8s5tt7jdeAMDRQZ4Vq6hg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-project/types": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.142.0.tgz", - "integrity": "sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/Boshen" - } - }, - "node_modules/@parcel/watcher": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.6.tgz", - "integrity": "sha512-tmmZ3lQxAe/k/+rNnXQRawJ4NjxO2hqiOLTHvWchtGZULp4RyFeh6aU4XdOYBFe2KE1oShQTv4AblOs2iOrNnQ==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "dependencies": { - "detect-libc": "^2.0.3", - "is-glob": "^4.0.3", - "node-addon-api": "^7.0.0", - "picomatch": "^4.0.3" - }, - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "optionalDependencies": { - "@parcel/watcher-android-arm64": "2.5.6", - "@parcel/watcher-darwin-arm64": "2.5.6", - "@parcel/watcher-darwin-x64": "2.5.6", - "@parcel/watcher-freebsd-x64": "2.5.6", - "@parcel/watcher-linux-arm-glibc": "2.5.6", - "@parcel/watcher-linux-arm-musl": "2.5.6", - "@parcel/watcher-linux-arm64-glibc": "2.5.6", - "@parcel/watcher-linux-arm64-musl": "2.5.6", - "@parcel/watcher-linux-x64-glibc": "2.5.6", - "@parcel/watcher-linux-x64-musl": "2.5.6", - "@parcel/watcher-win32-arm64": "2.5.6", - "@parcel/watcher-win32-ia32": "2.5.6", - "@parcel/watcher-win32-x64": "2.5.6" - } - }, - "node_modules/@parcel/watcher-android-arm64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-android-arm64/-/watcher-android-arm64-2.5.6.tgz", - "integrity": "sha512-YQxSS34tPF/6ZG7r/Ih9xy+kP/WwediEUsqmtf0cuCV5TPPKw/PQHRhueUo6JdeFJaqV3pyjm0GdYjZotbRt/A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-darwin-arm64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-darwin-arm64/-/watcher-darwin-arm64-2.5.6.tgz", - "integrity": "sha512-Z2ZdrnwyXvvvdtRHLmM4knydIdU9adO3D4n/0cVipF3rRiwP+3/sfzpAwA/qKFL6i1ModaabkU7IbpeMBgiVEA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-darwin-x64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-darwin-x64/-/watcher-darwin-x64-2.5.6.tgz", - "integrity": "sha512-HgvOf3W9dhithcwOWX9uDZyn1lW9R+7tPZ4sug+NGrGIo4Rk1hAXLEbcH1TQSqxts0NYXXlOWqVpvS1SFS4fRg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-freebsd-x64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-freebsd-x64/-/watcher-freebsd-x64-2.5.6.tgz", - "integrity": "sha512-vJVi8yd/qzJxEKHkeemh7w3YAn6RJCtYlE4HPMoVnCpIXEzSrxErBW5SJBgKLbXU3WdIpkjBTeUNtyBVn8TRng==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-linux-arm-glibc": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm-glibc/-/watcher-linux-arm-glibc-2.5.6.tgz", - "integrity": "sha512-9JiYfB6h6BgV50CCfasfLf/uvOcJskMSwcdH1PHH9rvS1IrNy8zad6IUVPVUfmXr+u+Km9IxcfMLzgdOudz9EQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-linux-arm-musl": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm-musl/-/watcher-linux-arm-musl-2.5.6.tgz", - "integrity": "sha512-Ve3gUCG57nuUUSyjBq/MAM0CzArtuIOxsBdQ+ftz6ho8n7s1i9E1Nmk/xmP323r2YL0SONs1EuwqBp2u1k5fxg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-linux-arm64-glibc": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm64-glibc/-/watcher-linux-arm64-glibc-2.5.6.tgz", - "integrity": "sha512-f2g/DT3NhGPdBmMWYoxixqYr3v/UXcmLOYy16Bx0TM20Tchduwr4EaCbmxh1321TABqPGDpS8D/ggOTaljijOA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-linux-arm64-musl": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm64-musl/-/watcher-linux-arm64-musl-2.5.6.tgz", - "integrity": "sha512-qb6naMDGlbCwdhLj6hgoVKJl2odL34z2sqkC7Z6kzir8b5W65WYDpLB6R06KabvZdgoHI/zxke4b3zR0wAbDTA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-linux-x64-glibc": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-glibc/-/watcher-linux-x64-glibc-2.5.6.tgz", - "integrity": "sha512-kbT5wvNQlx7NaGjzPFu8nVIW1rWqV780O7ZtkjuWaPUgpv2NMFpjYERVi0UYj1msZNyCzGlaCWEtzc+exjMGbQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-linux-x64-musl": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-musl/-/watcher-linux-x64-musl-2.5.6.tgz", - "integrity": "sha512-1JRFeC+h7RdXwldHzTsmdtYR/Ku8SylLgTU/reMuqdVD7CtLwf0VR1FqeprZ0eHQkO0vqsbvFLXUmYm/uNKJBg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-win32-arm64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-arm64/-/watcher-win32-arm64-2.5.6.tgz", - "integrity": "sha512-3ukyebjc6eGlw9yRt678DxVF7rjXatWiHvTXqphZLvo7aC5NdEgFufVwjFfY51ijYEWpXbqF5jtrK275z52D4Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-win32-ia32": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-ia32/-/watcher-win32-ia32-2.5.6.tgz", - "integrity": "sha512-k35yLp1ZMwwee3Ez/pxBi5cf4AoBKYXj00CZ80jUz5h8prpiaQsiRPKQMxoLstNuqe2vR4RNPEAEcjEFzhEz/g==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-win32-x64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-x64/-/watcher-win32-x64-2.5.6.tgz", - "integrity": "sha512-hbQlYcCq5dlAX9Qx+kFb0FHue6vbjlf0FrNzSKdYK2APUf7tGfGxQCk2ihEREmbR6ZMc0MVAD5RIX/41gpUzTw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher/node_modules/node-addon-api": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", - "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/@rolldown/binding-android-arm64": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.0.tgz", - "integrity": "sha512-9yB1l95IrJuNGDFdOYe79vdApdz6WWBCObE+rQ2LUliYUlcyFwSYIb2xb5/Ifw7dAtMy2ZqNyd8QTSOc7duAKw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.0.tgz", - "integrity": "sha512-pexNaW9ACLUOaBITOpU6qVu4VrsOFIjTv6bzgu0YUATo4eUJx0V605PxwZfndpPOn0ilqGqvGQ0M8UW0IE24jg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.0.tgz", - "integrity": "sha512-NqKYaq0355ZmNMG4QGpxtEDxsc7tGDhjhCm4PpE0cwnBW+5Il95LJyq414niEiaKLVjnVHBEjSo1wngKxJNiFw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.0.tgz", - "integrity": "sha512-3vPoHzh6eBTz9IbB0/qZdSr0Qeks2echn+I4cHu2joV74VriPDdldswksEDzrl1mBB+oPRi+67+3Ib59paxIPQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.0.tgz", - "integrity": "sha512-E6NNefZ1bUVmKJq2tJkf45J4Zyczj7qm9rUT7NY+Xo2474Y13qWAwc2tvBt0BAVbmtXR1llkxXg0Ou1jbDf2SQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.0.tgz", - "integrity": "sha512-D+TgkdgM1vu+7/Fpf8+v0ARW+RXEP9Ccazgm8zQ4JFFd9Q7SrYQ2TakU5S5ihazQDgpKyAgZDOcIFsvoHmTZ8w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.0.tgz", - "integrity": "sha512-wUqdwJBbAv0APN87GecstdMUtLjjNTs0hBALpxETD73mccFxdmt/XeizXDtN5RAlBwNKmI+Tg+blect2G+8IeQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.0.tgz", - "integrity": "sha512-9DtF35qR9/NrfhM4oxLplCzVVjE+KKm8Pjemi0i/sdhAWkUasjmSo8WTTubNJClhSHCfyk2yeyoXDQEDPtDAAw==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.0.tgz", - "integrity": "sha512-RzuHrBh8X8Hntd2N4VR02QGEciq/9JhcZoTpR/Cee6otRrlILGCf3cg2ygHuih+ZebUnWmMrDX6ITI85btO6rQ==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.0.tgz", - "integrity": "sha512-MK7L0018jjh1jR3mh21G2j1zAVcpscJBlPo2z19pRjv2XOYGRhaV4LyiD8HO6nCDdZln9IFgCMIV5yt4E3klGQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.0.tgz", - "integrity": "sha512-gyrxLQ9NfGb/9LoVnC4kb9miUghw1mghnkfYvNHSnVIXriabnfgGPUP4RLcJm87q3KgYz4FYUG8IDiWUT+CpSw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.0.tgz", - "integrity": "sha512-/6VFMQGRmrhP77KXDC+StIxGzcNp5JOIyYtw0CQ8gPlzhpiIRucYfoM5FaFamHd5BJYIdH86yfP46l1p3WdrFA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.2.0.tgz", - "integrity": "sha512-rwdbUL465kisF24WEJLvP3JrEG6E5GRuIHt5wpMwHGERtHe4Wm2CIvtf5gTBgr2tGOHKh5NdKEAFS2VkOPE91g==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "1.11.2", - "@emnapi/runtime": "1.11.2", - "@napi-rs/wasm-runtime": "^1.1.6" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.0.tgz", - "integrity": "sha512-+5suHwRiKGmhwyUaNT8a5QbrBvLFh2DbO910TEmGRH1aSxwrCezodvGQnulv4uiWEIv1Kq4ypRsJ5+O+ry1DiA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.0.tgz", - "integrity": "sha512-WfFv6/qGufotqBSBzBYwgpCkJBk8Nj7697LL9vTz/XWc67e0r3oewu8iMRwQj3AUL45GVD7wVsPjCsAAtW66Wg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/pluginutils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", - "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@schematics/angular": { - "version": "21.2.16", - "resolved": "https://registry.npmjs.org/@schematics/angular/-/angular-21.2.16.tgz", - "integrity": "sha512-ctvsRartACu77VAM416VlNV3mag7FhU08I/734f4+sS/UZmnhuTM5a4tTTWEI1U7iPeJoBtjreh6LgeP+QZLbQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@angular-devkit/core": "21.2.16", - "@angular-devkit/schematics": "21.2.16", - "jsonc-parser": "3.3.1" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0", - "npm": "^6.11.0 || ^7.5.6 || >=8.0.0", - "yarn": ">= 1.13.0" - } - }, - "node_modules/@sigstore/bundle": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@sigstore/bundle/-/bundle-4.0.0.tgz", - "integrity": "sha512-NwCl5Y0V6Di0NexvkTqdoVfmjTaQwoLM236r89KEojGmq/jMls8S+zb7yOwAPdXvbwfKDlP+lmXgAL4vKSQT+A==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@sigstore/protobuf-specs": "^0.5.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@sigstore/core": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/@sigstore/core/-/core-3.2.1.tgz", - "integrity": "sha512-qRsxPnCrbC/puegGxKuynfnxgLiHqWStrSjxkoB4YKqq3Z3s4cyZyj42ZdWFAEblNP65C+rBH8EuREHIXoi83g==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@sigstore/protobuf-specs": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@sigstore/protobuf-specs/-/protobuf-specs-0.5.1.tgz", - "integrity": "sha512-/ScWUhhoFasJsSRGTVBwId1loQjjnjAfE4djL6ZhrXRpNCmPTnUKF5Jokd58ILseOMjzET3UrMOtJPS9sYeI0g==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^18.17.0 || >=20.5.0" - } - }, - "node_modules/@sigstore/sign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/@sigstore/sign/-/sign-4.1.1.tgz", - "integrity": "sha512-Hf4xglukg0XXQ2RiD5vSoLjdPe8OBUPA8XeVjUObheuDcWdYWrnH/BNmxZCzkAy68MzmNCxXLeurJvs6hcP2OQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@gar/promise-retry": "^1.0.2", - "@sigstore/bundle": "^4.0.0", - "@sigstore/core": "^3.2.0", - "@sigstore/protobuf-specs": "^0.5.0", - "make-fetch-happen": "^15.0.4", - "proc-log": "^6.1.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@sigstore/tuf": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@sigstore/tuf/-/tuf-4.0.2.tgz", - "integrity": "sha512-TCAzTy0xzdP79EnxSjq9KQ3eaR7+FmudLC6eRKknVKZbV7ZNlGLClAAQb/HMNJ5n2OBNk2GT1tEmU0xuPr+SLQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@sigstore/protobuf-specs": "^0.5.0", - "tuf-js": "^4.1.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@sigstore/verify": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@sigstore/verify/-/verify-3.1.1.tgz", - "integrity": "sha512-qv7+G3J2cc6wwFj3yKvXOamzqhMwSk1ogPGmhpS8iXllcPrJaIIBA+4HbttlHVu1pqWTdmaCH/WE7UOC51kdoA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@sigstore/bundle": "^4.0.0", - "@sigstore/core": "^3.2.1", - "@sigstore/protobuf-specs": "^0.5.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@standard-schema/spec": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", - "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", - "license": "MIT" - }, - "node_modules/@tufjs/canonical-json": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tufjs/canonical-json/-/canonical-json-2.0.0.tgz", - "integrity": "sha512-yVtV8zsdo8qFHe+/3kw81dSLyF7D576A5cCFCi4X7B39tWT7SekaEFUnvnWJHz+9qO7qJTah1JbrDjWKqFtdWA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^16.14.0 || >=18.0.0" - } - }, - "node_modules/@tufjs/models": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@tufjs/models/-/models-4.1.0.tgz", - "integrity": "sha512-Y8cK9aggNRsqJVaKUlEYs4s7CvQ1b1ta2DVPyAimb0I2qhzjNk+A+mxvll/klL0RlfuIUei8BF7YWiua4kQqww==", - "dev": true, - "license": "MIT", - "dependencies": { - "@tufjs/canonical-json": "2.0.0", - "minimatch": "^10.1.1" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@tybys/wasm-util": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", - "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@types/chai": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", - "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/deep-eql": "*", - "assertion-error": "^2.0.1" - } - }, - "node_modules/@types/deep-eql": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", - "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/estree": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", - "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/gensync": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@types/gensync/-/gensync-1.0.5.tgz", - "integrity": "sha512-MbsRCT7mTikHwKZ0X+LVUTLRrZZRLipTuXEO9qOYO+zmjMVk81axyClMROf6uoPD9MRVu46bx8zoR0Ad9q3NAg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/jsesc": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/@types/jsesc/-/jsesc-2.5.1.tgz", - "integrity": "sha512-9VN+6yxLOPLOav+7PwjZbxiID2bVaeq0ED4qSQmdQTdjnXJSaCVKTR58t15oqH1H5t8Ng2ZX1SabJVoN9Q34bw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@vitejs/plugin-basic-ssl": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-basic-ssl/-/plugin-basic-ssl-2.3.0.tgz", - "integrity": "sha512-bdyo8rB3NnQbikdMpHaML9Z1OZPBu6fFOBo+OtxsBlvMJtysWskmBcnbIDhUqgC8tcxNv/a+BcV5U+2nQMm1OQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.0.0 || ^20.0.0 || >=22.0.0" - }, - "peerDependencies": { - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - } - }, - "node_modules/@vitest/expect": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.5.tgz", - "integrity": "sha512-PWBaRY5JoKuRnHlUHfpV/KohFylaDZTupcXN1H9vYryNLOnitSw60Mw9IAE2r67NbwwzBw/Cc/8q9BK3kIX8Kw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@standard-schema/spec": "^1.1.0", - "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.5", - "@vitest/utils": "4.1.5", - "chai": "^6.2.2", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/mocker": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.5.tgz", - "integrity": "sha512-/x2EmFC4mT4NNzqvC3fmesuV97w5FC903KPmey4gsnJiMQ3Be1IlDKVaDaG8iqaLFHqJ2FVEkxZk5VmeLjIItw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/spy": "4.1.5", - "estree-walker": "^3.0.3", - "magic-string": "^0.30.21" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "msw": { - "optional": true - }, - "vite": { - "optional": true - } - } - }, - "node_modules/@vitest/pretty-format": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.5.tgz", - "integrity": "sha512-7I3q6l5qr03dVfMX2wCo9FxwSJbPdwKjy2uu/YPpU3wfHvIL4QHwVRp57OfGrDFeUJ8/8QdfBKIV12FTtLn00g==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/runner": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.5.tgz", - "integrity": "sha512-2D+o7Pr82IEO46YPpoA/YU0neeyr6FTerQb5Ro7BUnBuv6NQtT/kmVnczngiMEBhzgqz2UZYl5gArejsyERDSQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/utils": "4.1.5", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/snapshot": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.5.tgz", - "integrity": "sha512-zypXEt4KH/XgKGPUz4eC2AvErYx0My5hfL8oDb1HzGFpEk1P62bxSohdyOmvz+d9UJwanI68MKwr2EquOaOgMQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.5", - "@vitest/utils": "4.1.5", - "magic-string": "^0.30.21", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/spy": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.5.tgz", - "integrity": "sha512-2lNOsh6+R2Idnf1TCZqSwYlKN2E/iDlD8sgU59kYVl+OMDmvldO1VDk39smRfpUNwYpNRVn3w4YfuC7KfbBnkQ==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/utils": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.5.tgz", - "integrity": "sha512-76wdkrmfXfqGjueGgnb45ITPyUi1ycZ4IHgC2bhPDUfWHklY/q3MdLOAB+TF1e6xfl8NxNY0ZYaPCFNWSsw3Ug==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.5", - "convert-source-map": "^2.0.0", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/utils/node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@yarnpkg/lockfile": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz", - "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==", - "dev": true, - "license": "BSD-2-Clause" - }, - "node_modules/abbrev": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-4.0.0.tgz", - "integrity": "sha512-a1wflyaL0tHtJSmLSOVybYhy22vRih4eduhhrkcjgrWGnRfrZtovJ2FRjxuTtkkj47O/baf0R86QU5OuYpz8fA==", - "dev": true, - "license": "ISC", - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/accepts": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", - "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-types": "^3.0.0", - "negotiator": "^1.0.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/ajv": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", - "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ajv-formats": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", - "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/algoliasearch": { - "version": "5.48.1", - "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-5.48.1.tgz", - "integrity": "sha512-Rf7xmeuIo7nb6S4mp4abW2faW8DauZyE2faBIKFaUfP3wnpOvNSbiI5AwVhqBNj0jPgBWEvhyCu0sLjN2q77Rg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@algolia/abtesting": "1.14.1", - "@algolia/client-abtesting": "5.48.1", - "@algolia/client-analytics": "5.48.1", - "@algolia/client-common": "5.48.1", - "@algolia/client-insights": "5.48.1", - "@algolia/client-personalization": "5.48.1", - "@algolia/client-query-suggestions": "5.48.1", - "@algolia/client-search": "5.48.1", - "@algolia/ingestion": "1.48.1", - "@algolia/monitoring": "1.48.1", - "@algolia/recommend": "5.48.1", - "@algolia/requester-browser-xhr": "5.48.1", - "@algolia/requester-fetch": "5.48.1", - "@algolia/requester-node-http": "5.48.1" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/ansi-escapes": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz", - "integrity": "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==", - "dev": true, - "license": "MIT", - "dependencies": { - "environment": "^1.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/assertion-error": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", - "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - } - }, - "node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/baseline-browser-mapping": { - "version": "2.10.27", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.27.tgz", - "integrity": "sha512-zEs/ufmZoUd7WftKpKyXaT6RFxpQ5Qm9xytKRHvJfxFV9DFJkZph9RvJ1LcOUi0Z1ZVijMte65JbILeV+8QQEA==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "baseline-browser-mapping": "dist/cli.cjs" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/beasties": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/beasties/-/beasties-0.4.3.tgz", - "integrity": "sha512-fIIeLOcbAB/K1kb1HBVJoiq1alHL4RCYBSo5e7HzrNkkgMggXR1Vqt/Z9JWnkfe/qdCo66Ux3QRwZioAIBdWRA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "css-select": "^6.0.0", - "css-what": "^7.0.0", - "dom-serializer": "^2.0.0", - "domhandler": "^5.0.3", - "htmlparser2": "^10.0.0", - "picocolors": "^1.1.1", - "postcss": "^8.4.49", - "postcss-media-query-parser": "^0.2.3", - "postcss-safe-parser": "^7.0.1" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/bidi-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", - "integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==", - "dev": true, - "license": "MIT", - "dependencies": { - "require-from-string": "^2.0.2" - } - }, - "node_modules/body-parser": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", - "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", - "dev": true, - "license": "MIT", - "dependencies": { - "bytes": "^3.1.2", - "content-type": "^1.0.5", - "debug": "^4.4.3", - "http-errors": "^2.0.0", - "iconv-lite": "^0.7.0", - "on-finished": "^2.4.1", - "qs": "^6.14.1", - "raw-body": "^3.0.1", - "type-is": "^2.0.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/boolbase": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", - "dev": true, - "license": "ISC" - }, - "node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/browserslist": { - "version": "4.28.2", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", - "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "baseline-browser-mapping": "^2.10.12", - "caniuse-lite": "^1.0.30001782", - "electron-to-chromium": "^1.5.328", - "node-releases": "^2.0.36", - "update-browserslist-db": "^1.2.3" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, - "node_modules/buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/cacache": { - "version": "20.0.4", - "resolved": "https://registry.npmjs.org/cacache/-/cacache-20.0.4.tgz", - "integrity": "sha512-M3Lab8NPYlZU2exsL3bMVvMrMqgwCnMWfdZbK28bn3pK6APT/Te/I8hjRPNu1uwORY9a1eEQoifXbKPQMfMTOA==", - "dev": true, - "license": "ISC", - "dependencies": { - "@npmcli/fs": "^5.0.0", - "fs-minipass": "^3.0.0", - "glob": "^13.0.0", - "lru-cache": "^11.1.0", - "minipass": "^7.0.3", - "minipass-collect": "^2.0.1", - "minipass-flush": "^1.0.5", - "minipass-pipeline": "^1.2.4", - "p-map": "^7.0.2", - "ssri": "^13.0.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/call-bound": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", - "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "get-intrinsic": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/caniuse-lite": { - "version": "1.0.30001791", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001791.tgz", - "integrity": "sha512-yk0l/YSrOnFZk3UROpDLQD9+kC1l4meK/wed583AXrzoarMGJcbRi2Q4RaUYbKxYAsZ8sWmaSa/DsLmdBeI1vQ==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "CC-BY-4.0" - }, - "node_modules/chai": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", - "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/chalk": { - "version": "5.6.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", - "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/chardet": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", - "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/chokidar": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", - "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", - "dev": true, - "license": "MIT", - "dependencies": { - "readdirp": "^5.0.0" - }, - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/chownr": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", - "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=18" - } - }, - "node_modules/cli-cursor": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", - "integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==", - "dev": true, - "license": "MIT", - "dependencies": { - "restore-cursor": "^5.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/cli-spinners": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-3.4.0.tgz", - "integrity": "sha512-bXfOC4QcT1tKXGorxL3wbJm6XJPDqEnij2gQ2m7ESQuE+/z9YFIWnl/5RpTiKWbMq3EVKR4fRLJGn6DVfu0mpw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18.20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/cli-truncate": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-5.2.0.tgz", - "integrity": "sha512-xRwvIOMGrfOAnM1JYtqQImuaNtDEv9v6oIYAs4LIHwTiKee8uwvIi363igssOC0O5U04i4AlENs79LQLu9tEMw==", - "dev": true, - "license": "MIT", - "dependencies": { - "slice-ansi": "^8.0.0", - "string-width": "^8.2.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/cli-width": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", - "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">= 12" - } - }, - "node_modules/cliui": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", - "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^7.2.0", - "strip-ansi": "^7.1.0", - "wrap-ansi": "^9.0.0" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/cliui/node_modules/string-width": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", - "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^10.3.0", - "get-east-asian-width": "^1.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/cliui/node_modules/wrap-ansi": { - "version": "9.0.2", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", - "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.2.1", - "string-width": "^7.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, - "license": "MIT" - }, - "node_modules/colorette": { - "version": "2.0.20", - "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", - "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", - "dev": true, - "license": "MIT" - }, - "node_modules/content-disposition": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", - "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/content-type": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", - "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/convert-source-map": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", - "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==", - "dev": true, - "license": "MIT" - }, - "node_modules/cookie": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", - "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie-signature": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", - "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.6.0" - } - }, - "node_modules/cors": { - "version": "2.8.6", - "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", - "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", - "dev": true, - "license": "MIT", - "dependencies": { - "object-assign": "^4", - "vary": "^1" - }, - "engines": { - "node": ">= 0.10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/css-select": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-6.0.0.tgz", - "integrity": "sha512-rZZVSLle8v0+EY8QAkDWrKhpgt6SA5OtHsgBnsj6ZaLb5dmDVOWUDtQitd9ydxxvEjhewNudS6eTVU7uOyzvXw==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0", - "css-what": "^7.0.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "nth-check": "^2.1.1" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/css-tree": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", - "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", - "dev": true, - "license": "MIT", - "dependencies": { - "mdn-data": "2.27.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" - } - }, - "node_modules/css-what": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-7.0.0.tgz", - "integrity": "sha512-wD5oz5xibMOPHzy13CyGmogB3phdvcDaB5t0W/Nr5Z2O/agcB8YwOz6e2Lsp10pNDzBoDO9nVa3RGs/2BttpHQ==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">= 6" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/cssstyle": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-6.2.0.tgz", - "integrity": "sha512-Fm5NvhYathRnXNVndkUsCCuR63DCLVVwGOOwQw782coXFi5HhkXdu289l59HlXZBawsyNccXfWRYvLzcDCdDig==", - "dev": true, - "license": "MIT", - "dependencies": { - "@asamuzakjp/css-color": "^5.0.1", - "@csstools/css-syntax-patches-for-csstree": "^1.0.28", - "css-tree": "^3.1.0", - "lru-cache": "^11.2.6" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/data-urls": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", - "integrity": "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==", - "dev": true, - "license": "MIT", - "dependencies": { - "whatwg-mimetype": "^5.0.0", - "whatwg-url": "^16.0.0" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/decimal.js": { - "version": "10.6.0", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", - "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", - "dev": true, - "license": "MIT" - }, - "node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, - "node_modules/dom-serializer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", - "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", - "dev": true, - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.2", - "entities": "^4.2.0" - }, - "funding": { - "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" - } - }, - "node_modules/domelementtype": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", - "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "BSD-2-Clause" - }, - "node_modules/domhandler": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", - "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "domelementtype": "^2.3.0" - }, - "engines": { - "node": ">= 4" - }, - "funding": { - "url": "https://github.com/fb55/domhandler?sponsor=1" - } - }, - "node_modules/domutils": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", - "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "dom-serializer": "^2.0.0", - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3" - }, - "funding": { - "url": "https://github.com/fb55/domutils?sponsor=1" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", - "dev": true, - "license": "MIT" - }, - "node_modules/electron-to-chromium": { - "version": "1.5.349", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.349.tgz", - "integrity": "sha512-QsWVGyRuY07Aqb234QytTfwd5d9AJlfNIQ5wIOl1L+PZDzI9d9+Fn0FRale/QYlFxt/bUnB0/nLd1jFPGxGK1A==", - "dev": true, - "license": "ISC" - }, - "node_modules/emoji-regex": { - "version": "10.6.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", - "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", - "dev": true, - "license": "MIT" - }, - "node_modules/empathic": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/empathic/-/empathic-2.0.1.tgz", - "integrity": "sha512-YGRs8knHhKHVShLkFET/rWAU8kmHbOV5LwN938RHI0pljAJ1Gf6SzXsSmRaEzcXTtOOmVqJ5+WtQPL5uigY50Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14" - } - }, - "node_modules/encodeurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", - "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/env-paths": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", - "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/environment": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", - "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-module-lexer": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.1.0.tgz", - "integrity": "sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/esbuild": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", - "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.28.1", - "@esbuild/android-arm": "0.28.1", - "@esbuild/android-arm64": "0.28.1", - "@esbuild/android-x64": "0.28.1", - "@esbuild/darwin-arm64": "0.28.1", - "@esbuild/darwin-x64": "0.28.1", - "@esbuild/freebsd-arm64": "0.28.1", - "@esbuild/freebsd-x64": "0.28.1", - "@esbuild/linux-arm": "0.28.1", - "@esbuild/linux-arm64": "0.28.1", - "@esbuild/linux-ia32": "0.28.1", - "@esbuild/linux-loong64": "0.28.1", - "@esbuild/linux-mips64el": "0.28.1", - "@esbuild/linux-ppc64": "0.28.1", - "@esbuild/linux-riscv64": "0.28.1", - "@esbuild/linux-s390x": "0.28.1", - "@esbuild/linux-x64": "0.28.1", - "@esbuild/netbsd-arm64": "0.28.1", - "@esbuild/netbsd-x64": "0.28.1", - "@esbuild/openbsd-arm64": "0.28.1", - "@esbuild/openbsd-x64": "0.28.1", - "@esbuild/openharmony-arm64": "0.28.1", - "@esbuild/sunos-x64": "0.28.1", - "@esbuild/win32-arm64": "0.28.1", - "@esbuild/win32-ia32": "0.28.1", - "@esbuild/win32-x64": "0.28.1" - } - }, - "node_modules/esbuild/node_modules/@esbuild/netbsd-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", - "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/esbuild/node_modules/@esbuild/openbsd-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", - "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/esbuild/node_modules/@esbuild/openharmony-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", - "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", - "dev": true, - "license": "MIT" - }, - "node_modules/estree-walker": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", - "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0" - } - }, - "node_modules/etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/eventemitter3": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", - "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", - "dev": true, - "license": "MIT" - }, - "node_modules/eventsource": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", - "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", - "dev": true, - "license": "MIT", - "dependencies": { - "eventsource-parser": "^3.0.1" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/eventsource-parser": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.8.tgz", - "integrity": "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/expect-type": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", - "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/exponential-backoff": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.3.tgz", - "integrity": "sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==", - "dev": true, - "license": "Apache-2.0" - }, - "node_modules/express": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", - "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", - "dev": true, - "license": "MIT", - "dependencies": { - "accepts": "^2.0.0", - "body-parser": "^2.2.1", - "content-disposition": "^1.0.0", - "content-type": "^1.0.5", - "cookie": "^0.7.1", - "cookie-signature": "^1.2.1", - "debug": "^4.4.0", - "depd": "^2.0.0", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "etag": "^1.8.1", - "finalhandler": "^2.1.0", - "fresh": "^2.0.0", - "http-errors": "^2.0.0", - "merge-descriptors": "^2.0.0", - "mime-types": "^3.0.0", - "on-finished": "^2.4.1", - "once": "^1.4.0", - "parseurl": "^1.3.3", - "proxy-addr": "^2.0.7", - "qs": "^6.14.0", - "range-parser": "^1.2.1", - "router": "^2.2.0", - "send": "^1.1.0", - "serve-static": "^2.2.0", - "statuses": "^2.0.1", - "type-is": "^2.0.1", - "vary": "^1.1.2" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/express-rate-limit": { - "version": "8.6.1", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.6.1.tgz", - "integrity": "sha512-0D493aP61w0TJ2A0wy27riRsO7FMQ7FK+KUHOKCSfPvYo0R55aiC6emCVgFUeShH0fq0ICPVzNcgoS+BsbXQCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.4.3", - "ip-address": "^10.2.0" - }, - "engines": { - "node": ">= 16" - }, - "funding": { - "url": "https://github.com/sponsors/express-rate-limit" - }, - "peerDependencies": { - "express": ">= 4.11" - } - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-string-truncated-width": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-3.0.3.tgz", - "integrity": "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-string-width": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/fast-string-width/-/fast-string-width-3.0.2.tgz", - "integrity": "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-string-truncated-width": "^3.0.2" - } - }, - "node_modules/fast-uri": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", - "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/fast-wrap-ansi": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.2.tgz", - "integrity": "sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-string-width": "^3.0.2" - } - }, - "node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, - "node_modules/finalhandler": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", - "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.4.0", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "on-finished": "^2.4.1", - "parseurl": "^1.3.3", - "statuses": "^2.0.1" - }, - "engines": { - "node": ">= 18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/forwarded": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", - "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/fresh": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", - "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/fs-minipass": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz", - "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==", - "dev": true, - "license": "ISC", - "dependencies": { - "minipass": "^7.0.3" - }, - "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", - "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-east-asian-width": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", - "integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "dev": true, - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/glob": { - "version": "13.0.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", - "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "minimatch": "^10.2.2", - "minipass": "^7.1.3", - "path-scurry": "^2.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", - "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/hono": { - "version": "4.13.1", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.1.tgz", - "integrity": "sha512-kdJoFVv2xmayw6cY09H7AbMJMt8Jn5jdlEdXsP7AGBdF2DIptVlKlOLKXP41yPip4/a3yQPv9gVcJYI8YY04dw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=16.9.0" - } - }, - "node_modules/hosted-git-info": { - "version": "9.0.3", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.3.tgz", - "integrity": "sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg==", - "dev": true, - "license": "ISC", - "dependencies": { - "lru-cache": "^11.1.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/html-encoding-sniffer": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz", - "integrity": "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@exodus/bytes": "^1.6.0" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - } - }, - "node_modules/htmlparser2": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", - "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", - "dev": true, - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "entities": "^7.0.1" - } - }, - "node_modules/htmlparser2/node_modules/entities": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", - "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/http-cache-semantics": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", - "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", - "dev": true, - "license": "BSD-2-Clause" - }, - "node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "dev": true, - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "dev": true, - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/iconv-lite": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", - "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", - "dev": true, - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/ignore-walk": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-8.0.0.tgz", - "integrity": "sha512-FCeMZT4NiRQGh+YkeKMtWrOmBgWjHjMJ26WQWrRQyoyzqevdaGSakUaJW5xQYmjLlUVk2qUnCjYVBax9EKKg8A==", - "dev": true, - "license": "ISC", - "dependencies": { - "minimatch": "^10.0.3" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/immutable": { - "version": "5.1.9", - "resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.9.tgz", - "integrity": "sha512-m8nVez3rwrgmWxtLMt1ZYXB2Lv7OKYn/disyxAlSDYAlKSlFoPPfIAmAM/M5xqL4m4C/wAPw7S2/CNaUii1Hxg==", - "dev": true, - "license": "MIT" - }, - "node_modules/import-meta-resolve": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz", - "integrity": "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==", - "dev": true, - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/ini": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/ini/-/ini-6.0.0.tgz", - "integrity": "sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ==", - "dev": true, - "license": "ISC", - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/ip-address": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz", - "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, - "node_modules/ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", - "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "get-east-asian-width": "^1.3.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-interactive": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-2.0.0.tgz", - "integrity": "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-potential-custom-element-name": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", - "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/is-promise": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", - "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/is-unicode-supported": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", - "integrity": "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, - "license": "ISC" - }, - "node_modules/jose": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.3.tgz", - "integrity": "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/panva" - } - }, - "node_modules/js-tokens": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", - "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/jsdom": { - "version": "28.1.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-28.1.0.tgz", - "integrity": "sha512-0+MoQNYyr2rBHqO1xilltfDjV9G7ymYGlAUazgcDLQaUf8JDHbuGwsxN6U9qWaElZ4w1B2r7yEGIL3GdeW3Rug==", - "dev": true, - "license": "MIT", - "dependencies": { - "@acemir/cssom": "^0.9.31", - "@asamuzakjp/dom-selector": "^6.8.1", - "@bramus/specificity": "^2.4.2", - "@exodus/bytes": "^1.11.0", - "cssstyle": "^6.0.1", - "data-urls": "^7.0.0", - "decimal.js": "^10.6.0", - "html-encoding-sniffer": "^6.0.0", - "http-proxy-agent": "^7.0.2", - "https-proxy-agent": "^7.0.6", - "is-potential-custom-element-name": "^1.0.1", - "parse5": "^8.0.0", - "saxes": "^6.0.0", - "symbol-tree": "^3.2.4", - "tough-cookie": "^6.0.0", - "undici": "^7.21.0", - "w3c-xmlserializer": "^5.0.0", - "webidl-conversions": "^8.0.1", - "whatwg-mimetype": "^5.0.0", - "whatwg-url": "^16.0.0", - "xml-name-validator": "^5.0.0" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - }, - "peerDependencies": { - "canvas": "^3.0.0" - }, - "peerDependenciesMeta": { - "canvas": { - "optional": true - } - } - }, - "node_modules/jsesc": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", - "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", - "dev": true, - "license": "MIT", - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/json-parse-even-better-errors": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-5.0.0.tgz", - "integrity": "sha512-ZF1nxZ28VhQouRWhUcVlUIN3qwSgPuswK05s/HIaoetAoE/9tngVmCHjSxmSQPav1nd+lPtTL0YZ/2AFdR/iYQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, - "license": "MIT" - }, - "node_modules/json-schema-typed": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", - "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", - "dev": true, - "license": "BSD-2-Clause" - }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/jsonc-parser": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", - "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/jsonparse": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", - "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", - "dev": true, - "engines": [ - "node >= 0.2.0" - ], - "license": "MIT" - }, - "node_modules/lightningcss": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", - "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", - "dev": true, - "license": "MPL-2.0", - "dependencies": { - "detect-libc": "^2.0.3" - }, - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "optionalDependencies": { - "lightningcss-android-arm64": "1.33.0", - "lightningcss-darwin-arm64": "1.33.0", - "lightningcss-darwin-x64": "1.33.0", - "lightningcss-freebsd-x64": "1.33.0", - "lightningcss-linux-arm-gnueabihf": "1.33.0", - "lightningcss-linux-arm64-gnu": "1.33.0", - "lightningcss-linux-arm64-musl": "1.33.0", - "lightningcss-linux-x64-gnu": "1.33.0", - "lightningcss-linux-x64-musl": "1.33.0", - "lightningcss-win32-arm64-msvc": "1.33.0", - "lightningcss-win32-x64-msvc": "1.33.0" - } - }, - "node_modules/lightningcss-android-arm64": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", - "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-darwin-arm64": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", - "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-darwin-x64": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", - "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-freebsd-x64": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", - "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm-gnueabihf": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", - "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm64-gnu": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", - "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm64-musl": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", - "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-x64-gnu": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", - "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-x64-musl": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", - "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-win32-arm64-msvc": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", - "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-win32-x64-msvc": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", - "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/listr2": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/listr2/-/listr2-9.0.5.tgz", - "integrity": "sha512-ME4Fb83LgEgwNw96RKNvKV4VTLuXfoKudAmm2lP8Kk87KaMK0/Xrx/aAkMWmT8mDb+3MlFDspfbCs7adjRxA2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "cli-truncate": "^5.0.0", - "colorette": "^2.0.20", - "eventemitter3": "^5.0.1", - "log-update": "^6.1.0", - "rfdc": "^1.4.1", - "wrap-ansi": "^9.0.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/listr2/node_modules/string-width": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", - "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^10.3.0", - "get-east-asian-width": "^1.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/listr2/node_modules/wrap-ansi": { - "version": "9.0.2", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", - "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.2.1", - "string-width": "^7.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/lmdb": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/lmdb/-/lmdb-3.5.6.tgz", - "integrity": "sha512-j3uE8ReKNyUWDjhfEFSJqE/1DLtfTR5Z8yFzVHvBjAk37wNg7HdScjcv8ttPHRvrdgPQMPWxFFI0SsdBzI5lBw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@harperfast/extended-iterable": "^1.0.3", - "msgpackr": "^1.11.2", - "node-addon-api": "^6.1.0", - "node-gyp-build-optional-packages": "5.2.2", - "ordered-binary": "^1.5.3", - "weak-lru-cache": "^1.2.2" - }, - "bin": { - "download-lmdb-prebuilds": "bin/download-prebuilds.js" - }, - "optionalDependencies": { - "@lmdb/lmdb-darwin-arm64": "3.5.6", - "@lmdb/lmdb-darwin-x64": "3.5.6", - "@lmdb/lmdb-linux-arm": "3.5.6", - "@lmdb/lmdb-linux-arm64": "3.5.6", - "@lmdb/lmdb-linux-x64": "3.5.6", - "@lmdb/lmdb-win32-arm64": "3.5.6", - "@lmdb/lmdb-win32-x64": "3.5.6" - } - }, - "node_modules/log-symbols": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-7.0.1.tgz", - "integrity": "sha512-ja1E3yCr9i/0hmBVaM0bfwDjnGy8I/s6PP4DFp+yP+a+mrHO4Rm7DtmnqROTUkHIkqffC84YY7AeqX6oFk0WFg==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-unicode-supported": "^2.0.0", - "yoctocolors": "^2.1.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/log-update": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/log-update/-/log-update-6.1.0.tgz", - "integrity": "sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-escapes": "^7.0.0", - "cli-cursor": "^5.0.0", - "slice-ansi": "^7.1.0", - "strip-ansi": "^7.1.0", - "wrap-ansi": "^9.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/log-update/node_modules/slice-ansi": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-7.1.2.tgz", - "integrity": "sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.2.1", - "is-fullwidth-code-point": "^5.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/chalk/slice-ansi?sponsor=1" - } - }, - "node_modules/log-update/node_modules/string-width": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", - "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^10.3.0", - "get-east-asian-width": "^1.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/log-update/node_modules/wrap-ansi": { - "version": "9.0.2", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", - "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.2.1", - "string-width": "^7.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/lru-cache": { - "version": "11.5.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", - "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/magic-string": { - "version": "0.30.21", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", - "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.5" - } - }, - "node_modules/make-fetch-happen": { - "version": "15.0.6", - "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-15.0.6.tgz", - "integrity": "sha512-Je0fLJ0F5atA7F+eIlLzk+Wkcl57JDf4kf+EW8xiP5E31xOQxkIxTbgf1Oi1Lw9tRI9UEMRdI5Vz2xTzoNU1Jw==", - "dev": true, - "license": "ISC", - "dependencies": { - "@gar/promise-retry": "^1.0.0", - "@npmcli/agent": "^4.0.0", - "@npmcli/redact": "^4.0.0", - "cacache": "^20.0.1", - "http-cache-semantics": "^4.1.1", - "minipass": "^7.0.2", - "minipass-fetch": "^5.0.0", - "minipass-flush": "^1.0.5", - "minipass-pipeline": "^1.2.4", - "negotiator": "^1.0.0", - "proc-log": "^6.0.0", - "ssri": "^13.0.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/mdn-data": { - "version": "2.27.1", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", - "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", - "dev": true, - "license": "CC0-1.0" - }, - "node_modules/media-typer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", - "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/merge-descriptors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", - "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/mimic-function": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", - "integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minipass": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", - "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/minipass-collect": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-2.0.1.tgz", - "integrity": "sha512-D7V8PO9oaz7PWGLbCACuI1qEOsq7UKfLotx/C0Aet43fCUB/wfQ7DYeq2oR/svFJGYDHPr38SHATeaj/ZoKHKw==", - "dev": true, - "license": "ISC", - "dependencies": { - "minipass": "^7.0.3" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/minipass-fetch": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-5.0.2.tgz", - "integrity": "sha512-2d0q2a8eCi2IRg/IGubCNRJoYbA1+YPXAzQVRFmB45gdGZafyivnZ5YSEfo3JikbjGxOdntGFvBQGqaSMXlAFQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "minipass": "^7.0.3", - "minipass-sized": "^2.0.0", - "minizlib": "^3.0.1" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - }, - "optionalDependencies": { - "iconv-lite": "^0.7.2" - } - }, - "node_modules/minipass-flush": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.7.tgz", - "integrity": "sha512-TbqTz9cUwWyHS2Dy89P3ocAGUGxKjjLuR9z8w4WUTGAVgEj17/4nhgo2Du56i0Fm3Pm30g4iA8Lcqctc76jCzA==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/minipass-flush/node_modules/minipass": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", - "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/minipass-flush/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, - "license": "ISC" - }, - "node_modules/minipass-pipeline": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", - "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", - "dev": true, - "license": "ISC", - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/minipass-pipeline/node_modules/minipass": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", - "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/minipass-pipeline/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, - "license": "ISC" - }, - "node_modules/minipass-sized": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-2.0.0.tgz", - "integrity": "sha512-zSsHhto5BcUVM2m1LurnXY6M//cGhVaegT71OfOXoprxT6o780GZd792ea6FfrQkuU4usHZIUczAQMRUE2plzA==", - "dev": true, - "license": "ISC", - "dependencies": { - "minipass": "^7.1.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/minizlib": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", - "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "minipass": "^7.1.2" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/mrmime": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", - "integrity": "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/msgpackr": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.12.1.tgz", - "integrity": "sha512-4EUH9tQHnMmEgzW/MdAP0KIfa1T9AF+htl0ffe2n5vb2EKn9y2co8ccpgWko6S52Jy1PQZKwRnx5/KkYjtd9MQ==", - "dev": true, - "license": "MIT", - "optional": true, - "optionalDependencies": { - "msgpackr-extract": "^3.0.2" - } - }, - "node_modules/msgpackr-extract": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.4.tgz", - "integrity": "sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "dependencies": { - "node-gyp-build-optional-packages": "5.2.2" - }, - "bin": { - "download-msgpackr-prebuilds": "bin/download-prebuilds.js" - }, - "optionalDependencies": { - "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.4", - "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.4", - "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.4", - "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.4", - "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.4", - "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.4" - } - }, - "node_modules/mute-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz", - "integrity": "sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==", - "dev": true, - "license": "ISC", - "engines": { - "node": "^18.17.0 || >=20.5.0" - } - }, - "node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "node_modules/negotiator": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", - "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/node-addon-api": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", - "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/node-gyp": { - "version": "12.4.0", - "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-12.4.0.tgz", - "integrity": "sha512-OMcPNvqTCFUnNaBlmdgq+lfNqY7gTiSmNRDjY3uAXRyudeKZEZxu3CLtjMQrx4zZxCX2b/mpNqTtwuCJgXhHkw==", - "dev": true, - "license": "MIT", - "dependencies": { - "env-paths": "^2.2.0", - "exponential-backoff": "^3.1.1", - "graceful-fs": "^4.2.6", - "nopt": "^9.0.0", - "proc-log": "^6.0.0", - "semver": "^7.3.5", - "tar": "^7.5.4", - "tinyglobby": "^0.2.12", - "undici": "^6.25.0", - "which": "^6.0.0" - }, - "bin": { - "node-gyp": "bin/node-gyp.js" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/node-gyp-build-optional-packages": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", - "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "detect-libc": "^2.0.1" - }, - "bin": { - "node-gyp-build-optional-packages": "bin.js", - "node-gyp-build-optional-packages-optional": "optional.js", - "node-gyp-build-optional-packages-test": "build-test.js" - } - }, - "node_modules/node-gyp/node_modules/isexe": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", - "integrity": "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=20" - } - }, - "node_modules/node-gyp/node_modules/undici": { - "version": "6.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.0.tgz", - "integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18.17" - } - }, - "node_modules/node-gyp/node_modules/which": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/which/-/which-6.0.1.tgz", - "integrity": "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^4.0.0" - }, - "bin": { - "node-which": "bin/which.js" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/node-releases": { - "version": "2.0.38", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.38.tgz", - "integrity": "sha512-3qT/88Y3FbH/Kx4szpQQ4HzUbVrHPKTLVpVocKiLfoYvw9XSGOX2FmD2d6DrXbVYyAQTF2HeF6My8jmzx7/CRw==", - "dev": true, - "license": "MIT" - }, - "node_modules/nopt": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-9.0.0.tgz", - "integrity": "sha512-Zhq3a+yFKrYwSBluL4H9XP3m3y5uvQkB/09CwDruCiRmR/UJYnn9W4R48ry0uGC70aeTPKLynBtscP9efFFcPw==", - "dev": true, - "license": "ISC", - "dependencies": { - "abbrev": "^4.0.0" - }, - "bin": { - "nopt": "bin/nopt.js" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/npm-bundled": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-5.0.0.tgz", - "integrity": "sha512-JLSpbzh6UUXIEoqPsYBvVNVmyrjVZ1fzEFbqxKkTJQkWBO3xFzFT+KDnSKQWwOQNbuWRwt5LSD6HOTLGIWzfrw==", - "dev": true, - "license": "ISC", - "dependencies": { - "npm-normalize-package-bin": "^5.0.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/npm-install-checks": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/npm-install-checks/-/npm-install-checks-8.0.0.tgz", - "integrity": "sha512-ScAUdMpyzkbpxoNekQ3tNRdFI8SJ86wgKZSQZdUxT+bj0wVFpsEMWnkXP0twVe1gJyNF5apBWDJhhIbgrIViRA==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "semver": "^7.1.1" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/npm-normalize-package-bin": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-5.0.0.tgz", - "integrity": "sha512-CJi3OS4JLsNMmr2u07OJlhcrPxCeOeP/4xq67aWNai6TNWWbTrlNDgl8NcFKVlcBKp18GPj+EzbNIgrBfZhsag==", - "dev": true, - "license": "ISC", - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/npm-package-arg": { - "version": "13.0.2", - "resolved": "https://registry.npmjs.org/npm-package-arg/-/npm-package-arg-13.0.2.tgz", - "integrity": "sha512-IciCE3SY3uE84Ld8WZU23gAPPV9rIYod4F+rc+vJ7h7cwAJt9Vk6TVsK60ry7Uj3SRS3bqRRIGuTp9YVlk6WNA==", - "dev": true, - "license": "ISC", - "dependencies": { - "hosted-git-info": "^9.0.0", - "proc-log": "^6.0.0", - "semver": "^7.3.5", - "validate-npm-package-name": "^7.0.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/npm-packlist": { - "version": "10.0.4", - "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-10.0.4.tgz", - "integrity": "sha512-uMW73iajD8hiH4ZBxEV3HC+eTnppIqwakjOYuvgddnalIw2lJguKviK1pcUJDlIWm1wSJkchpDZDSVVsZEYRng==", - "dev": true, - "license": "ISC", - "dependencies": { - "ignore-walk": "^8.0.0", - "proc-log": "^6.0.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/npm-pick-manifest": { - "version": "11.0.3", - "resolved": "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-11.0.3.tgz", - "integrity": "sha512-buzyCfeoGY/PxKqmBqn1IUJrZnUi1VVJTdSSRPGI60tJdUhUoSQFhs0zycJokDdOznQentgrpf8LayEHyyYlqQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "npm-install-checks": "^8.0.0", - "npm-normalize-package-bin": "^5.0.0", - "npm-package-arg": "^13.0.0", - "semver": "^7.3.5" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/npm-registry-fetch": { - "version": "19.1.1", - "resolved": "https://registry.npmjs.org/npm-registry-fetch/-/npm-registry-fetch-19.1.1.tgz", - "integrity": "sha512-TakBap6OM1w0H73VZVDf44iFXsOS3h+L4wVMXmbWOQroZgFhMch0juN6XSzBNlD965yIKvWg2dfu7NSiaYLxtw==", - "dev": true, - "license": "ISC", - "dependencies": { - "@npmcli/redact": "^4.0.0", - "jsonparse": "^1.3.1", - "make-fetch-happen": "^15.0.0", - "minipass": "^7.0.2", - "minipass-fetch": "^5.0.0", - "minizlib": "^3.0.1", - "npm-package-arg": "^13.0.0", - "proc-log": "^6.0.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/nth-check": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", - "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0" - }, - "funding": { - "url": "https://github.com/fb55/nth-check?sponsor=1" - } - }, - "node_modules/object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/object-inspect": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", - "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/obug": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", - "integrity": "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==", - "dev": true, - "funding": [ - "https://github.com/sponsors/sxzz", - "https://opencollective.com/debug" - ], - "license": "MIT" - }, - "node_modules/on-finished": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", - "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/onetime": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-7.0.0.tgz", - "integrity": "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "mimic-function": "^5.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ora": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/ora/-/ora-9.3.0.tgz", - "integrity": "sha512-lBX72MWFduWEf7v7uWf5DHp9Jn5BI8bNPGuFgtXMmr2uDz2Gz2749y3am3agSDdkhHPHYmmxEGSKH85ZLGzgXw==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^5.6.2", - "cli-cursor": "^5.0.0", - "cli-spinners": "^3.2.0", - "is-interactive": "^2.0.0", - "is-unicode-supported": "^2.1.0", - "log-symbols": "^7.0.1", - "stdin-discarder": "^0.3.1", - "string-width": "^8.1.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ordered-binary": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/ordered-binary/-/ordered-binary-1.6.1.tgz", - "integrity": "sha512-QkCdPooczexPLiXIrbVOPYkR3VO3T6v2OyKRkR1Xbhpy7/LAVXwahnRCgRp78Oe/Ehf0C/HATAxfSr6eA1oX+w==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/oxc-parser": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/oxc-parser/-/oxc-parser-0.142.0.tgz", - "integrity": "sha512-kKR+jPiRJYJDexVoziIg/FVGvr1fT1FZSSJOk6tVoMKKSlsf1Cso+cgGCJkOEDWOP174vRntCPFKg+AS7InWvw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@oxc-project/types": "^0.142.0" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "funding": { - "url": "https://github.com/sponsors/Boshen" - }, - "optionalDependencies": { - "@oxc-parser/binding-android-arm-eabi": "0.142.0", - "@oxc-parser/binding-android-arm64": "0.142.0", - "@oxc-parser/binding-darwin-arm64": "0.142.0", - "@oxc-parser/binding-darwin-x64": "0.142.0", - "@oxc-parser/binding-freebsd-x64": "0.142.0", - "@oxc-parser/binding-linux-arm-gnueabihf": "0.142.0", - "@oxc-parser/binding-linux-arm-musleabihf": "0.142.0", - "@oxc-parser/binding-linux-arm64-gnu": "0.142.0", - "@oxc-parser/binding-linux-arm64-musl": "0.142.0", - "@oxc-parser/binding-linux-ppc64-gnu": "0.142.0", - "@oxc-parser/binding-linux-riscv64-gnu": "0.142.0", - "@oxc-parser/binding-linux-riscv64-musl": "0.142.0", - "@oxc-parser/binding-linux-s390x-gnu": "0.142.0", - "@oxc-parser/binding-linux-x64-gnu": "0.142.0", - "@oxc-parser/binding-linux-x64-musl": "0.142.0", - "@oxc-parser/binding-openharmony-arm64": "0.142.0", - "@oxc-parser/binding-wasm32-wasi": "0.142.0", - "@oxc-parser/binding-win32-arm64-msvc": "0.142.0", - "@oxc-parser/binding-win32-ia32-msvc": "0.142.0", - "@oxc-parser/binding-win32-x64-msvc": "0.142.0" - } - }, - "node_modules/p-map": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.4.tgz", - "integrity": "sha512-tkAQEw8ysMzmkhgw8k+1U/iPhWNhykKnSk4Rd5zLoPJCuJaGRPo6YposrZgaxHKzDHdDWWZvE/Sk7hsL2X/CpQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/pacote": { - "version": "21.5.1", - "resolved": "https://registry.npmjs.org/pacote/-/pacote-21.5.1.tgz", - "integrity": "sha512-KvcJ9iy3crysCsgqc4+PknH/w6jkrp8JN36mpZBPwNaDRwTfMZD37YzRazNstiZUOhuF5pno9f78n9mEJBavwg==", - "dev": true, - "license": "ISC", - "dependencies": { - "@gar/promise-retry": "^1.0.0", - "@npmcli/git": "^7.0.0", - "@npmcli/installed-package-contents": "^4.0.0", - "@npmcli/package-json": "^7.0.0", - "@npmcli/promise-spawn": "^9.0.0", - "@npmcli/run-script": "^10.0.0", - "cacache": "^20.0.0", - "fs-minipass": "^3.0.0", - "minipass": "^7.0.2", - "npm-package-arg": "^13.0.0", - "npm-packlist": "^10.0.1", - "npm-pick-manifest": "^11.0.1", - "npm-registry-fetch": "^19.0.0", - "proc-log": "^6.0.0", - "sigstore": "^4.0.0", - "ssri": "^13.0.0", - "tar": "^7.4.3" - }, - "bin": { - "pacote": "bin/index.js" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/parse5": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", - "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", - "dev": true, - "license": "MIT", - "dependencies": { - "entities": "^8.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/parse5-html-rewriting-stream": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/parse5-html-rewriting-stream/-/parse5-html-rewriting-stream-8.0.0.tgz", - "integrity": "sha512-wzh11mj8KKkno1pZEu+l2EVeWsuKDfR5KNWZOTsslfUX8lPDZx77m9T0kIoAVkFtD1nx6YF8oh4BnPHvxMtNMw==", - "dev": true, - "license": "MIT", - "dependencies": { - "entities": "^6.0.0", - "parse5": "^8.0.0", - "parse5-sax-parser": "^8.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/parse5-html-rewriting-stream/node_modules/entities": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", - "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/parse5-sax-parser": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/parse5-sax-parser/-/parse5-sax-parser-8.0.0.tgz", - "integrity": "sha512-/dQ8UzHZwnrzs3EvDj6IkKrD/jIZyTlB+8XrHJvcjNgRdmWruNdN9i9RK/JtxakmlUdPwKubKPTCqvbTgzGhrw==", - "dev": true, - "license": "MIT", - "dependencies": { - "parse5": "^8.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/parse5/node_modules/entities": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz", - "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=20.19.0" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/parseurl": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-scurry": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", - "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/path-to-regexp": { - "version": "8.4.2", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", - "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", - "dev": true, - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/piscina": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/piscina/-/piscina-5.2.0.tgz", - "integrity": "sha512-DszUCKeVN/5G5QKo6jAVHL8fmKnkJvQ0ACiVgY7YGCq3TUB2oznAOayvZPIAdEThvhczkXR+qm3IHsNXpFCYfA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.x" - }, - "optionalDependencies": { - "@napi-rs/nice": "^1.0.4" - } - }, - "node_modules/pkce-challenge": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", - "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/postcss": { - "version": "8.5.23", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", - "integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "nanoid": "^3.3.16", - "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } - }, - "node_modules/postcss-media-query-parser": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/postcss-media-query-parser/-/postcss-media-query-parser-0.2.3.tgz", - "integrity": "sha512-3sOlxmbKcSHMjlUXQZKQ06jOswE7oVkXPxmZdoB1r5l0q6gTFTQSHxNxOrCccElbW7dxNytifNEo8qidX2Vsig==", - "dev": true, - "license": "MIT" - }, - "node_modules/postcss-safe-parser": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/postcss-safe-parser/-/postcss-safe-parser-7.0.1.tgz", - "integrity": "sha512-0AioNCJZ2DPYz5ABT6bddIqlhgwhpHZ/l65YAYo0BCIn0xiDpsnTHz0gnoTGk0OXZW0JRs+cDwL8u/teRdz+8A==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss-safe-parser" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "engines": { - "node": ">=18.0" - }, - "peerDependencies": { - "postcss": "^8.4.31" - } - }, - "node_modules/prettier": { - "version": "3.8.3", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.3.tgz", - "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==", - "dev": true, - "license": "MIT", - "bin": { - "prettier": "bin/prettier.cjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" - } - }, - "node_modules/proc-log": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", - "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", - "dev": true, - "license": "ISC", - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", - "dev": true, - "license": "MIT", - "dependencies": { - "forwarded": "0.2.0", - "ipaddr.js": "1.9.1" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/proxy-agent-negotiate": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-agent-negotiate/-/proxy-agent-negotiate-1.1.0.tgz", - "integrity": "sha512-N8IBcM3UgCVzz2L2Lqv8DVntDnnC8/hiV4nEDUPkqq72TPUgYWjQc+bdZlBPZK9LzPAvOY//gAt0S0DApoOXWQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 20" - }, - "peerDependencies": { - "kerberos": "^2.0.0" - }, - "peerDependenciesMeta": { - "kerberos": { - "optional": true - } - } - }, - "node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/qs": { - "version": "6.15.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.1.tgz", - "integrity": "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "side-channel": "^1.1.0" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/raw-body": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", - "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", - "dev": true, - "license": "MIT", - "dependencies": { - "bytes": "~3.1.2", - "http-errors": "~2.0.1", - "iconv-lite": "~0.7.0", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/readdirp": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", - "integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/reflect-metadata": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", - "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", - "dev": true, - "license": "Apache-2.0" - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/restore-cursor": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz", - "integrity": "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==", - "dev": true, - "license": "MIT", - "dependencies": { - "onetime": "^7.0.0", - "signal-exit": "^4.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/rfdc": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz", - "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", - "dev": true, - "license": "MIT" - }, - "node_modules/rolldown": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.0.tgz", - "integrity": "sha512-u7tgm5l4Yw1iTqUL4EcYOAt7fFvCgQMLeidrnD4GALlC6aOznCjezYajgxeyKw27u0Q5N7fwgCzjVyPIWzwuBA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@oxc-project/types": "=0.140.0", - "@rolldown/pluginutils": "^1.0.0" - }, - "bin": { - "rolldown": "bin/cli.mjs" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.2.0", - "@rolldown/binding-darwin-arm64": "1.2.0", - "@rolldown/binding-darwin-x64": "1.2.0", - "@rolldown/binding-freebsd-x64": "1.2.0", - "@rolldown/binding-linux-arm-gnueabihf": "1.2.0", - "@rolldown/binding-linux-arm64-gnu": "1.2.0", - "@rolldown/binding-linux-arm64-musl": "1.2.0", - "@rolldown/binding-linux-ppc64-gnu": "1.2.0", - "@rolldown/binding-linux-s390x-gnu": "1.2.0", - "@rolldown/binding-linux-x64-gnu": "1.2.0", - "@rolldown/binding-linux-x64-musl": "1.2.0", - "@rolldown/binding-openharmony-arm64": "1.2.0", - "@rolldown/binding-wasm32-wasi": "1.2.0", - "@rolldown/binding-win32-arm64-msvc": "1.2.0", - "@rolldown/binding-win32-x64-msvc": "1.2.0" - } - }, - "node_modules/rolldown/node_modules/@oxc-project/types": { - "version": "0.140.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.140.0.tgz", - "integrity": "sha512-h5LUOzGArYemnW1NMz/DuuQhBi96J6JL2Bk8zE4kvqxB5Sg3jxmCiH4uyOWHDkiKSt5vWlG4FIwCR/DbstcNRQ==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/Boshen" - } - }, - "node_modules/router": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", - "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.4.0", - "depd": "^2.0.0", - "is-promise": "^4.0.0", - "parseurl": "^1.3.3", - "path-to-regexp": "^8.0.0" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/rxjs": { - "version": "7.8.2", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", - "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.1.0" - } - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true, - "license": "MIT" - }, - "node_modules/sass": { - "version": "1.101.0", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.101.0.tgz", - "integrity": "sha512-OL3GoQyoUdDt843DpVmDO6y2k1sc5IhUDSpu8XucEI+35neq5QivZ1iuegnpraEVTJXlQGK1gl27zKcTLEPbQw==", - "dev": true, - "license": "MIT", - "dependencies": { - "chokidar": "^5.0.0", - "immutable": "^5.1.5", - "source-map-js": ">=0.6.2 <2.0.0" - }, - "bin": { - "sass": "sass.js" - }, - "engines": { - "node": ">=20.19.0" - }, - "optionalDependencies": { - "@parcel/watcher": "^2.4.1" - } - }, - "node_modules/saxes": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", - "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", - "dev": true, - "license": "ISC", - "dependencies": { - "xmlchars": "^2.2.0" - }, - "engines": { - "node": ">=v12.22.7" - } - }, - "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/send": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", - "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.4.3", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "etag": "^1.8.1", - "fresh": "^2.0.0", - "http-errors": "^2.0.1", - "mime-types": "^3.0.2", - "ms": "^2.1.3", - "on-finished": "^2.4.1", - "range-parser": "^1.2.1", - "statuses": "^2.0.2" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/serve-static": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", - "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", - "dev": true, - "license": "MIT", - "dependencies": { - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "parseurl": "^1.3.3", - "send": "^1.2.0" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/setprototypeof": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", - "dev": true, - "license": "ISC" - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/side-channel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", - "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.3", - "side-channel-list": "^1.0.0", - "side-channel-map": "^1.0.1", - "side-channel-weakmap": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-list": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", - "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.4" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-map": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", - "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-weakmap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", - "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3", - "side-channel-map": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/siginfo": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", - "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", - "dev": true, - "license": "ISC" - }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/sigstore": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/sigstore/-/sigstore-4.1.1.tgz", - "integrity": "sha512-endqECJkfhozrXMK5ngu/UAA0xVcVEFdnHJCElGaExypjW+HK5i6zu3NteLoaX/iFbRUbC3+DjttQs0GARr+5w==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@sigstore/bundle": "^4.0.0", - "@sigstore/core": "^3.2.1", - "@sigstore/protobuf-specs": "^0.5.0", - "@sigstore/sign": "^4.1.1", - "@sigstore/tuf": "^4.0.2", - "@sigstore/verify": "^3.1.1" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/slice-ansi": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-8.0.0.tgz", - "integrity": "sha512-stxByr12oeeOyY2BlviTNQlYV5xOj47GirPr4yA1hE9JCtxfQN0+tVbkxwCtYDQWhEKWFHsEK48ORg5jrouCAg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.2.3", - "is-fullwidth-code-point": "^5.1.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/chalk/slice-ansi?sponsor=1" - } - }, - "node_modules/smart-buffer": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", - "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 6.0.0", - "npm": ">= 3.0.0" - } - }, - "node_modules/socks": { - "version": "2.8.9", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz", - "integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==", - "dev": true, - "license": "MIT", - "dependencies": { - "ip-address": "^10.1.1", - "smart-buffer": "^4.2.0" - }, - "engines": { - "node": ">= 10.0.0", - "npm": ">= 3.0.0" - } - }, - "node_modules/socks-proxy-agent": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", - "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", - "dev": true, - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "socks": "^2.8.3" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/source-map": { - "version": "0.7.6", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", - "integrity": "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">= 12" - } - }, - "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/source-map-support": { - "version": "0.5.21", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", - "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, - "node_modules/source-map-support/node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/spdx-exceptions": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", - "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", - "dev": true, - "license": "CC-BY-3.0" - }, - "node_modules/spdx-expression-parse": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-4.0.0.tgz", - "integrity": "sha512-Clya5JIij/7C6bRR22+tnGXbc4VKlibKSVj2iHvVeX5iMW7s1SIQlqu699JkODJJIhh/pUu8L0/VLh8xflD+LQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "spdx-exceptions": "^2.1.0", - "spdx-license-ids": "^3.0.0" - } - }, - "node_modules/spdx-license-ids": { - "version": "3.0.23", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.23.tgz", - "integrity": "sha512-CWLcCCH7VLu13TgOH+r8p1O/Znwhqv/dbb6lqWy67G+pT1kHmeD/+V36AVb/vq8QMIQwVShJ6Ssl5FPh0fuSdw==", - "dev": true, - "license": "CC0-1.0" - }, - "node_modules/ssri": { - "version": "13.0.1", - "resolved": "https://registry.npmjs.org/ssri/-/ssri-13.0.1.tgz", - "integrity": "sha512-QUiRf1+u9wPTL/76GTYlKttDEBWV1ga9ZXW8BG6kfdeyyM8LGPix9gROyg9V2+P0xNyF3X2Go526xKFdMZrHSQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "minipass": "^7.0.3" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/stackback": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", - "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", - "dev": true, - "license": "MIT" - }, - "node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/std-env": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz", - "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/stdin-discarder": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/stdin-discarder/-/stdin-discarder-0.3.2.tgz", - "integrity": "sha512-eCPu1qRxPVkl5605OTWF8Wz40b4Mf45NY5LQmVPQ599knfs5QhASUm9GbJ5BDMDOXgrnh0wyEdvzmL//YMlw0A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/string-width": { - "version": "8.2.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.1.tgz", - "integrity": "sha512-IIaP0g3iy9Cyy18w3M9YcaDudujEAVHKt3a3QJg1+sr/oX96TbaGUubG0hJyCjCBThFH+tFpcIyoUHUn1ogaLA==", - "dev": true, - "license": "MIT", - "dependencies": { - "get-east-asian-width": "^1.5.0", - "strip-ansi": "^7.1.2" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/strip-ansi": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", - "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.2.2" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/symbol-tree": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", - "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", - "dev": true, - "license": "MIT" - }, - "node_modules/tar": { - "version": "7.5.20", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.20.tgz", - "integrity": "sha512-9FcyK4PA6+WbzlTM9WhQm6vB5W7cP7dUiPsv1g7YDwEQnQ1CGpK3MGlKk/ITVWMk05kHZuBhmVhiv8LZoy/PFQ==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/fs-minipass": "^4.0.0", - "chownr": "^3.0.0", - "minipass": "^7.1.2", - "minizlib": "^3.1.0", - "yallist": "^5.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/tar/node_modules/yallist": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", - "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=18" - } - }, - "node_modules/tinybench": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", - "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", - "dev": true, - "license": "MIT" - }, - "node_modules/tinyexec": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.1.2.tgz", - "integrity": "sha512-dAqSqE/RabpBKI8+h26GfLq6Vb3JVXs30XYQjdMjaj/c2tS8IYYMbIzP599KtRj7c57/wYApb3QjgRgXmrCukA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/tinyglobby": { - "version": "0.2.17", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", - "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "fdir": "^6.5.0", - "picomatch": "^4.0.4" - }, - "engines": { - "node": ">=12.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/SuperchupuDev" - } - }, - "node_modules/tinyrainbow": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", - "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/tldts": { - "version": "7.0.30", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.30.tgz", - "integrity": "sha512-ELrFxuqsDdHUwoh0XxDbxuLD3Wnz49Z57IFvTtvWy1hJdcMZjXLIuonjilCiWHlT2GbE4Wlv1wKVTzDFnXH1aw==", - "dev": true, - "license": "MIT", - "dependencies": { - "tldts-core": "^7.0.30" - }, - "bin": { - "tldts": "bin/cli.js" - } - }, - "node_modules/tldts-core": { - "version": "7.0.30", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.30.tgz", - "integrity": "sha512-uiHN8PIB1VmWyS98eZYja4xzlYqeFZVjb4OuYlJQnZAuJhMw4PbKQOKgHKhBdJR3FE/t5mUQ1Kd80++B+qhD1Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/toidentifier": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/tough-cookie": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.1.tgz", - "integrity": "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "tldts": "^7.0.5" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/tr46": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz", - "integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==", - "dev": true, - "license": "MIT", - "dependencies": { - "punycode": "^2.3.1" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" - }, - "node_modules/tuf-js": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/tuf-js/-/tuf-js-4.1.0.tgz", - "integrity": "sha512-50QV99kCKH5P/Vs4E2Gzp7BopNV+KzTXqWeaxrfu5IQJBOULRsTIS9seSsOVT8ZnGXzCyx55nYWAi4qJzpZKEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@tufjs/models": "4.1.0", - "debug": "^4.4.3", - "make-fetch-happen": "^15.0.1" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/type-is": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", - "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", - "dev": true, - "license": "MIT", - "dependencies": { - "content-type": "^1.0.5", - "media-typer": "^1.1.0", - "mime-types": "^3.0.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", - "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, - "node_modules/unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "escalade": "^3.2.0", - "picocolors": "^1.1.1" - }, - "bin": { - "update-browserslist-db": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, - "node_modules/validate-npm-package-name": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/validate-npm-package-name/-/validate-npm-package-name-7.0.2.tgz", - "integrity": "sha512-hVDIBwsRruT73PbK7uP5ebUt+ezEtCmzZz3F59BSr2F6OVFnJ/6h8liuvdLrQ88Xmnk6/+xGGuq+pG9WwTuy3A==", - "dev": true, - "license": "ISC", - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/vite": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.5.tgz", - "integrity": "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw==", - "dev": true, - "license": "MIT", - "dependencies": { - "lightningcss": "^1.32.0", - "picomatch": "^4.0.5", - "postcss": "^8.5.17", - "rolldown": "~1.1.5", - "tinyglobby": "^0.2.17" - }, - "bin": { - "vite": "bin/vite.js" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "funding": { - "url": "https://github.com/vitejs/vite?sponsor=1" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - }, - "peerDependencies": { - "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.3.0", - "esbuild": "^0.27.0 || ^0.28.0", - "jiti": ">=1.21.0", - "less": "^4.0.0", - "sass": "^1.70.0", - "sass-embedded": "^1.70.0", - "stylus": ">=0.54.8", - "sugarss": "^5.0.0", - "terser": "^5.16.0", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "@vitejs/devtools": { - "optional": true - }, - "esbuild": { - "optional": true - }, - "jiti": { - "optional": true - }, - "less": { - "optional": true - }, - "sass": { - "optional": true - }, - "sass-embedded": { - "optional": true - }, - "stylus": { - "optional": true - }, - "sugarss": { - "optional": true - }, - "terser": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } - } - }, - "node_modules/vite/node_modules/@emnapi/core": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", - "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.2", - "tslib": "^2.4.0" - } - }, - "node_modules/vite/node_modules/@emnapi/runtime": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", - "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/vite/node_modules/@oxc-project/types": { - "version": "0.139.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz", - "integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/Boshen" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-android-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", - "integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz", - "integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-darwin-x64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz", - "integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz", - "integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz", - "integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz", - "integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz", - "integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz", - "integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz", - "integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz", - "integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz", - "integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz", - "integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz", - "integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "1.11.1", - "@emnapi/runtime": "1.11.1", - "@napi-rs/wasm-runtime": "^1.1.6" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", - "integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz", - "integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/vite/node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/vite/node_modules/rolldown": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz", - "integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@oxc-project/types": "=0.139.0", - "@rolldown/pluginutils": "^1.0.0" - }, - "bin": { - "rolldown": "bin/cli.mjs" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.1.5", - "@rolldown/binding-darwin-arm64": "1.1.5", - "@rolldown/binding-darwin-x64": "1.1.5", - "@rolldown/binding-freebsd-x64": "1.1.5", - "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", - "@rolldown/binding-linux-arm64-gnu": "1.1.5", - "@rolldown/binding-linux-arm64-musl": "1.1.5", - "@rolldown/binding-linux-ppc64-gnu": "1.1.5", - "@rolldown/binding-linux-s390x-gnu": "1.1.5", - "@rolldown/binding-linux-x64-gnu": "1.1.5", - "@rolldown/binding-linux-x64-musl": "1.1.5", - "@rolldown/binding-openharmony-arm64": "1.1.5", - "@rolldown/binding-wasm32-wasi": "1.1.5", - "@rolldown/binding-win32-arm64-msvc": "1.1.5", - "@rolldown/binding-win32-x64-msvc": "1.1.5" - } - }, - "node_modules/vitest": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.5.tgz", - "integrity": "sha512-9Xx1v3/ih3m9hN+SbfkUyy0JAs72ap3r7joc87XL6jwF0jGg6mFBvQ1SrwaX+h8BlkX6Hz9shdd1uo6AF+ZGpg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/expect": "4.1.5", - "@vitest/mocker": "4.1.5", - "@vitest/pretty-format": "4.1.5", - "@vitest/runner": "4.1.5", - "@vitest/snapshot": "4.1.5", - "@vitest/spy": "4.1.5", - "@vitest/utils": "4.1.5", - "es-module-lexer": "^2.0.0", - "expect-type": "^1.3.0", - "magic-string": "^0.30.21", - "obug": "^2.1.1", - "pathe": "^2.0.3", - "picomatch": "^4.0.3", - "std-env": "^4.0.0-rc.1", - "tinybench": "^2.9.0", - "tinyexec": "^1.0.2", - "tinyglobby": "^0.2.15", - "tinyrainbow": "^3.1.0", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", - "why-is-node-running": "^2.3.0" - }, - "bin": { - "vitest": "vitest.mjs" - }, - "engines": { - "node": "^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@edge-runtime/vm": "*", - "@opentelemetry/api": "^1.9.0", - "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.5", - "@vitest/browser-preview": "4.1.5", - "@vitest/browser-webdriverio": "4.1.5", - "@vitest/coverage-istanbul": "4.1.5", - "@vitest/coverage-v8": "4.1.5", - "@vitest/ui": "4.1.5", - "happy-dom": "*", - "jsdom": "*", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "@edge-runtime/vm": { - "optional": true - }, - "@opentelemetry/api": { - "optional": true - }, - "@types/node": { - "optional": true - }, - "@vitest/browser-playwright": { - "optional": true - }, - "@vitest/browser-preview": { - "optional": true - }, - "@vitest/browser-webdriverio": { - "optional": true - }, - "@vitest/coverage-istanbul": { - "optional": true - }, - "@vitest/coverage-v8": { - "optional": true - }, - "@vitest/ui": { - "optional": true - }, - "happy-dom": { - "optional": true - }, - "jsdom": { - "optional": true - }, - "vite": { - "optional": false - } - } - }, - "node_modules/w3c-xmlserializer": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", - "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", - "dev": true, - "license": "MIT", - "dependencies": { - "xml-name-validator": "^5.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/watchpack": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.2.tgz", - "integrity": "sha512-6i/00NBjP4yGPs+caKSyRfpTF/8Torsu0MOW3mMzIbhgISFder8i7xbqgHlLMwJrdiN8ndBV3UA1/AfzPSr+jg==", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.1.2" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/weak-lru-cache": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz", - "integrity": "sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/webidl-conversions": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz", - "integrity": "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=20" - } - }, - "node_modules/whatwg-mimetype": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz", - "integrity": "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20" - } - }, - "node_modules/whatwg-url": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", - "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@exodus/bytes": "^1.11.0", - "tr46": "^6.0.0", - "webidl-conversions": "^8.0.1" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/why-is-node-running": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", - "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", - "dev": true, - "license": "MIT", - "dependencies": { - "siginfo": "^2.0.0", - "stackback": "0.0.2" - }, - "bin": { - "why-is-node-running": "cli.js" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/wrap-ansi/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/wrap-ansi/node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/xml-name-validator": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", - "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18" - } - }, - "node_modules/xmlchars": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", - "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", - "dev": true, - "license": "MIT" - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs": { - "version": "18.0.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.0.0.tgz", - "integrity": "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^9.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "string-width": "^7.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^22.0.0" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=23" - } - }, - "node_modules/yargs-parser": { - "version": "22.0.0", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", - "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", - "dev": true, - "license": "ISC", - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=23" - } - }, - "node_modules/yargs/node_modules/string-width": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", - "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^10.3.0", - "get-east-asian-width": "^1.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/yoctocolors": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yoctocolors/-/yoctocolors-2.1.2.tgz", - "integrity": "sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/yoctocolors-cjs": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.3.tgz", - "integrity": "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/zod": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", - "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/zod-to-json-schema": { - "version": "3.25.2", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", - "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", - "dev": true, - "license": "ISC", - "peerDependencies": { - "zod": "^3.25.28 || ^4" - } - } - } -} diff --git a/bigframes/display/table_widget_angular/package.json b/bigframes/display/table_widget_angular/package.json deleted file mode 100644 index 5008d5829e6..00000000000 --- a/bigframes/display/table_widget_angular/package.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "table-widget-angular", - "version": "0.0.0", - "scripts": { - "ng": "ng", - "start": "ng serve", - "build": "ng build", - "watch": "ng build --watch --configuration development", - "test": "ng test", - "build:widget": "ng build --output-hashing none && node bundle.js" - }, - "private": true, - "packageManager": "npm@11.7.0", - "dependencies": { - "@angular/common": "^22.1.0", - "@angular/compiler": "^22.1.0", - "@angular/core": "^22.1.0", - "@angular/forms": "^22.1.0", - "@angular/platform-browser": "^22.1.0", - "@angular/router": "^22.1.0", - "rxjs": "~7.8.0", - "tslib": "^2.3.0" - }, - "devDependencies": { - "@angular/build": "^22.1.2", - "@angular/cli": "^21.2.16", - "@angular/compiler-cli": "^22.1.0", - "esbuild": "^0.28.0", - "jsdom": "^28.0.0", - "prettier": "^3.8.1", - "typescript": "~5.9.2", - "vitest": "^4.0.8" - } -} diff --git a/bigframes/display/table_widget_angular/public/favicon.ico b/bigframes/display/table_widget_angular/public/favicon.ico deleted file mode 100644 index 57614f9c967..00000000000 Binary files a/bigframes/display/table_widget_angular/public/favicon.ico and /dev/null differ diff --git a/bigframes/display/table_widget_angular/src/app/app.spec.ts b/bigframes/display/table_widget_angular/src/app/app.spec.ts deleted file mode 100644 index 75ccf03e436..00000000000 --- a/bigframes/display/table_widget_angular/src/app/app.spec.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { TestBed } from '@angular/core/testing'; -import { App } from './app'; - -describe('App', () => { - beforeEach(async () => { - await TestBed.configureTestingModule({ - imports: [App], - providers: [{ provide: 'ANYWIDGET_MODEL', useValue: null }] - }).compileComponents(); - }); - - it('should create the app', () => { - const fixture = TestBed.createComponent(App); - const app = fixture.componentInstance; - expect(app).toBeTruthy(); - }); - - it('should render the table container', async () => { - const fixture = TestBed.createComponent(App); - fixture.detectChanges(); - const compiled = fixture.nativeElement as HTMLElement; - expect(compiled.querySelector('.table-container')).toBeTruthy(); - }); -}); diff --git a/bigframes/display/table_widget_angular/src/app/app.ts b/bigframes/display/table_widget_angular/src/app/app.ts deleted file mode 100644 index 60b94d30e78..00000000000 --- a/bigframes/display/table_widget_angular/src/app/app.ts +++ /dev/null @@ -1,704 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Component, ElementRef, ViewChild, computed, effect, inject, signal } from '@angular/core'; -import { DomSanitizer } from '@angular/platform-browser'; -import { WidgetStateService } from './widget-state.service'; - -@Component({ - selector: '[app-root]', - standalone: true, - imports: [], - providers: [WidgetStateService], - template: ` -
- @if (errorMessage()) { -
{{ errorMessage() }}
- } - - @if (isDeferredMode()) { -
-
-

{{ dryRunInfo() }}

- -
-
- } @else { -
-
- -
- {{ rowCountText() }} - - - -
-
- - -
- -
- - -
-
-
- } -
- `, - styles: [` - /* Increase specificity to override framework styles without !important */ - .bigframes-widget.bigframes-widget { - /* Default Light Mode Variables */ - --bf-bg: white; - --bf-border-color: #ccc; - --bf-error-bg: #fbe; - --bf-error-border: red; - --bf-error-fg: black; - --bf-fg: black; - --bf-header-bg: #f5f5f5; - --bf-null-fg: gray; - --bf-row-even-bg: #f5f5f5; - --bf-row-odd-bg: white; - - background-color: var(--bf-bg); - box-sizing: border-box; - color: var(--bf-fg); - display: flex; - flex-direction: column; - font-family: - '-apple-system', 'BlinkMacSystemFont', 'Segoe UI', 'Roboto', sans-serif; - margin: 0; - padding: 0; - width: 100%; - } - - .bigframes-widget * { - box-sizing: border-box; - } - - /* Dark Mode Overrides */ - @media (prefers-color-scheme: dark) { - .bigframes-widget.bigframes-widget { - --bf-bg: var(--vscode-editor-background, #202124); - --bf-border-color: #444; - --bf-error-bg: #511; - --bf-error-border: #f88; - --bf-error-fg: #fcc; - --bf-fg: white; - --bf-header-bg: var(--vscode-editor-background, black); - --bf-null-fg: #aaa; - --bf-row-even-bg: #202124; - --bf-row-odd-bg: #383838; - } - } - - .bigframes-widget.bigframes-dark-mode.bigframes-dark-mode { - --bf-bg: var(--vscode-editor-background, #202124); - --bf-border-color: #444; - --bf-error-bg: #511; - --bf-error-border: #f88; - --bf-error-fg: #fcc; - --bf-fg: white; - --bf-header-bg: var(--vscode-editor-background, black); - --bf-null-fg: #aaa; - --bf-row-even-bg: #202124; - --bf-row-odd-bg: #383838; - } - - .bigframes-widget .table-container { - background-color: var(--bf-bg); - margin: 0; - overflow: auto; - padding: 0; - } - - .bigframes-widget .footer { - align-items: center; - background-color: var(--bf-bg); - color: var(--bf-fg); - display: flex; - font-size: 0.8rem; - justify-content: space-between; - padding: 8px; - } - - .bigframes-widget .footer > * { - flex: 1; - } - - .bigframes-widget .pagination { - align-items: center; - display: flex; - flex-direction: row; - gap: 4px; - justify-content: center; - padding: 4px; - } - - .bigframes-widget .page-indicator { - margin: 0 8px; - } - - .bigframes-widget .row-count { - margin: 0 8px; - } - - .bigframes-widget .settings { - align-items: center; - display: flex; - flex-direction: row; - gap: 16px; - justify-content: end; - } - - .bigframes-widget .page-size, - .bigframes-widget .max-columns { - align-items: center; - display: flex; - flex-direction: row; - gap: 4px; - } - - .bigframes-widget .page-size label, - .bigframes-widget .max-columns label { - margin-right: 8px; - } - - /* Dynamic internal elements styles */ - .bigframes-widget ::ng-deep table.bigframes-widget-table, - .bigframes-widget ::ng-deep table.dataframe { - background-color: var(--bf-bg); - border: 1px solid var(--bf-border-color); - border-collapse: collapse; - border-spacing: 0; - box-shadow: none; - color: var(--bf-fg); - margin: 0; - outline: none; - text-align: left; - width: auto; - } - - .bigframes-widget ::ng-deep tr { - border: none; - } - - .bigframes-widget ::ng-deep th { - background-color: var(--bf-header-bg); - border: 1px solid var(--bf-border-color); - color: var(--bf-fg); - padding: 0; - position: sticky; - text-align: left; - top: 0; - z-index: 1; - } - - .bigframes-widget ::ng-deep td { - border: 1px solid var(--bf-border-color); - color: var(--bf-fg); - padding: 0.5em; - } - - .bigframes-widget ::ng-deep table tbody tr:nth-child(odd), - .bigframes-widget ::ng-deep table tbody tr:nth-child(odd) td { - background-color: var(--bf-row-odd-bg); - } - - .bigframes-widget ::ng-deep table tbody tr:nth-child(even), - .bigframes-widget ::ng-deep table tbody tr:nth-child(even) td { - background-color: var(--bf-row-even-bg); - } - - .bigframes-widget ::ng-deep .bf-header-content { - box-sizing: border-box; - height: 100%; - overflow: auto; - padding: 0.5em; - resize: horizontal; - width: 100%; - } - - .bigframes-widget ::ng-deep th .sort-indicator { - padding-left: 4px; - visibility: hidden; - } - - .bigframes-widget ::ng-deep th:hover .sort-indicator { - visibility: visible; - } - - .bigframes-widget button { - background-color: transparent; - border: 1px solid currentColor; - border-radius: 4px; - color: inherit; - cursor: pointer; - display: inline-block; - padding: 2px 8px; - text-align: center; - text-decoration: none; - user-select: none; - vertical-align: middle; - } - - .bigframes-widget button:disabled { - opacity: 0.65; - pointer-events: none; - } - - .bigframes-widget .bigframes-error-message { - background-color: var(--bf-error-bg); - border: 1px solid var(--bf-error-border); - border-radius: 4px; - color: var(--bf-error-fg); - font-size: 14px; - margin-bottom: 8px; - padding: 8px; - } - - .bigframes-widget ::ng-deep .cell-align-right { - text-align: right; - } - - .bigframes-widget ::ng-deep .cell-align-left { - text-align: left; - } - - .bigframes-widget ::ng-deep .null-value { - color: var(--bf-null-fg); - } - - .bigframes-widget ::ng-deep .debug-info { - border-top: 1px solid var(--bf-border-color); - } - - .bigframes-widget .deferred-container { - align-items: center; - display: flex; - justify-content: center; - min-height: 220px; - padding: 24px; - width: 100%; - } - - .bigframes-widget .deferred-card { - background: linear-gradient( - 135deg, - rgba(255, 255, 255, 0.6), - rgba(255, 255, 255, 0.3) - ); - border: 1px solid rgba(255, 255, 255, 0.4); - border-radius: 16px; - box-shadow: 0 8px 32px 0 rgba(31, 38, 135, 0.07); - display: flex; - flex-direction: column; - gap: 16px; - max-width: 500px; - padding: 32px; - text-align: center; - transition: all 0.3s ease-in-out; - } - - .bigframes-widget.bigframes-dark-mode .deferred-card { - background: linear-gradient( - 135deg, - rgba(32, 33, 36, 0.6), - rgba(32, 33, 36, 0.3) - ); - border: 1px solid rgba(255, 255, 255, 0.1); - box-shadow: 0 8px 32px 0 rgba(0, 0, 0, 0.3); - } - - @media (prefers-color-scheme: dark) { - .bigframes-widget .deferred-card { - background: linear-gradient( - 135deg, - rgba(32, 33, 36, 0.6), - rgba(32, 33, 36, 0.3) - ); - border: 1px solid rgba(255, 255, 255, 0.1); - box-shadow: 0 8px 32px 0 rgba(0, 0, 0, 0.3); - } - } - - .bigframes-widget .deferred-title { - font-size: 1.1rem; - font-weight: 600; - margin: 0; - } - - .bigframes-widget .deferred-estimate { - color: var(--bf-null-fg); - font-size: 0.9rem; - margin: 0; - } - - .bigframes-widget .run-query-button { - align-items: center; - background-color: var(--bf-fg); - border: 1px solid var(--bf-fg); - border-radius: 8px; - color: var(--bf-bg); - cursor: pointer; - display: inline-flex; - font-size: 14px; - font-weight: 600; - gap: 8px; - justify-content: center; - padding: 10px 20px; - transition: transform 0.20s ease, opacity 0.20s ease; - } - - .bigframes-widget .run-query-button:hover { - opacity: 0.90; - transform: translateY(-1px); - } - - .bigframes-widget .run-query-button:active { - transform: translateY(0); - } - - .bigframes-widget .run-query-button:disabled { - cursor: not-allowed; - opacity: 0.60; - } - - .bigframes-widget .spinner { - animation: spin 1s linear infinite; - border: 2px solid currentColor; - border-radius: 50%; - border-top-color: transparent; - display: inline-block; - height: 12px; - width: 12px; - } - - @keyframes spin { - to { - transform: rotate(360deg); - } - } - `] -}) -export class App { - protected readonly state = inject(WidgetStateService); - private readonly sanitizer = inject(DomSanitizer); - - protected readonly maxColumnOptions = [5, 10, 15, 20, 0]; - protected readonly pageSizeOptions = [10, 25, 50, 100]; - - // State signals - protected readonly errorMessage = this.state.errorMessage; - protected readonly maxColumns = this.state.maxColumns; - protected readonly pageSize = this.state.pageSize; - protected readonly page = this.state.page; - protected readonly rowCount = this.state.rowCount; - protected readonly isDeferredMode = this.state.isDeferredMode; - protected readonly dryRunInfo = this.state.dryRunInfo; - protected readonly isLoading = signal(false); - - // Computed properties for formatting and display states - protected readonly sanitizedHtml = computed(() => - this.sanitizer.bypassSecurityTrustHtml(this.state.tableHtml()) - ); - - protected readonly totalPages = computed(() => { - const count = this.rowCount(); - const size = this.pageSize(); - return count !== null && size > 0 ? Math.ceil(count / size) : null; - }); - - protected readonly pageIndicatorText = computed(() => { - const currentPage = this.page(); - const count = this.rowCount(); - const total = this.totalPages(); - const currentStr = (currentPage + 1).toLocaleString(); - const totalStr = (total ?? 1).toLocaleString(); - return `Page ${currentStr} of ${totalStr}`; - }); - - protected readonly rowCountText = computed(() => { - const count = this.rowCount(); - if (count === null) { - return 'Total rows unknown'; - } - if (count === 0) { - return '0 total rows'; - } - return `${count.toLocaleString()} total rows`; - }); - - protected readonly prevPageDisabled = computed(() => this.page() === 0); - - protected readonly nextPageDisabled = computed(() => { - const currentPage = this.page(); - const count = this.rowCount(); - const total = this.totalPages(); - if (count === null) { - return false; - } - if (count === 0) { - return true; - } - return total !== null && currentPage >= total - 1; - }); - - protected readonly isDarkMode = signal(false); - private themeObserver: MutationObserver | null = null; - - @ViewChild('tableContainer') - tableContainerRef!: ElementRef; - - private isHeightInitialized = false; - - constructor() { - effect(() => { - // Setup dependencies for reactive effect - const _html = this.state.tableHtml(); - const _sort = this.state.sortContext(); - const _orderable = this.state.orderableColumns(); - const deferred = this.isDeferredMode(); - if (deferred) { - this.isHeightInitialized = false; - } - - // Schedule DOM post-processing once the innerHTML render completes - setTimeout(() => { - this.applySortIndicators(); - this.lockInitialHeight(); - }, 0); - }); - - effect(() => { - if (!this.state.startExecution()) { - this.isLoading.set(false); - } - }); - - effect((onCleanup) => { - const executing = this.state.startExecution(); - if (executing) { - const intervalId = setInterval(() => { - if (this.state.startExecution()) { - const currentPing = this.state.ping(); - this.state.setPing(currentPing + 1); - } else { - clearInterval(intervalId); - } - }, 500); - onCleanup(() => { - clearInterval(intervalId); - }); - } - }); - } - - ngOnInit() { - this.initThemeDetection(); - } - - ngOnDestroy() { - this.themeObserver?.disconnect(); - } - - protected handleRunQuery() { - this.isLoading.set(true); - this.state.setStartExecution(true); - } - - protected handlePageChange(direction: number) { - const nextPage = this.page() + direction; - this.state.setPage(nextPage); - } - - protected handlePageSizeChange(event: Event) { - const select = event.target as HTMLSelectElement; - const newSize = Number(select.value); - if (newSize) { - this.state.setPageSize(newSize); - } - } - - protected handleMaxColumnsChange(event: Event) { - const select = event.target as HTMLSelectElement; - const maxCols = Number(select.value); - this.state.setMaxColumns(maxCols); - } - - protected handleTableClick(event: MouseEvent) { - const target = event.target as HTMLElement; - const header = target.closest('th'); - if (!header) return; - - const headerDiv = header.querySelector( - 'div.bf-header-content' - ) as HTMLElement | null; - if (!headerDiv) return; - - const columnName = this.getColumnName(headerDiv); - const sortableColumns = this.state.orderableColumns(); - if (!columnName || !sortableColumns.includes(columnName)) return; - - const currentSortContext = [...this.state.sortContext()]; - const sortIndex = currentSortContext.findIndex( - (item) => item.column === columnName - ); - let newContext = [...currentSortContext]; - - if (event.shiftKey) { - if (sortIndex !== -1) { - // Toggle: Asc -> Desc -> Unsorted - if (newContext[sortIndex].ascending) { - newContext[sortIndex] = { - ...newContext[sortIndex], - ascending: false - }; - } else { - newContext.splice(sortIndex, 1); - } - } else { - newContext.push({ column: columnName, ascending: true }); - } - } else { - // Single column sort mode - if (sortIndex !== -1 && newContext.length === 1) { - // Toggle: Asc -> Desc -> Unsorted - if (newContext[sortIndex].ascending) { - newContext[sortIndex] = { - ...newContext[sortIndex], - ascending: false - }; - } else { - newContext = []; - } - } else { - newContext = [{ column: columnName, ascending: true }]; - } - } - - this.state.setSortContext(newContext); - } - - private getColumnName(headerDiv: HTMLElement): string { - const clone = headerDiv.cloneNode(true) as HTMLElement; - clone.querySelector('.sort-indicator')?.remove(); - return clone.textContent?.trim() || ''; - } - - private applySortIndicators() { - const container = this.tableContainerRef?.nativeElement; - if (!container) return; - - const sortableColumns = this.state.orderableColumns(); - const currentSortContext = this.state.sortContext() || []; - - const getSortIndex = (colName: string) => - currentSortContext.findIndex((item) => item.column === colName); - - const headers = container.querySelectorAll('th'); - headers.forEach((header: HTMLElement) => { - const headerDiv = header.querySelector( - 'div.bf-header-content' - ) as HTMLElement | null; - if (!headerDiv) return; - - const columnName = this.getColumnName(headerDiv); - if (columnName && sortableColumns.includes(columnName)) { - - let indicatorSpan = headerDiv.querySelector( - '.sort-indicator' - ) as HTMLElement; - if (!indicatorSpan) { - indicatorSpan = document.createElement('span'); - indicatorSpan.classList.add('sort-indicator'); - indicatorSpan.style.paddingLeft = '5px'; - headerDiv.appendChild(indicatorSpan); - } - - const sortIndex = getSortIndex(columnName); - if (sortIndex !== -1) { - const isAscending = currentSortContext[sortIndex].ascending; - indicatorSpan.textContent = isAscending ? '▲' : '▼'; - indicatorSpan.style.visibility = 'visible'; - } else { - indicatorSpan.textContent = '●'; - indicatorSpan.style.visibility = 'hidden'; - } - } - }); - } - - private lockInitialHeight() { - if (this.isHeightInitialized) return; - const container = this.tableContainerRef?.nativeElement; - if (!container) return; - - const table = container.querySelector('table'); - if (table && (table as HTMLElement).offsetHeight > 0) { - const currentHeight = container.offsetHeight; - if (currentHeight > 0) { - container.style.height = `${currentHeight}px`; - this.isHeightInitialized = true; - } - } - } - - private initThemeDetection() { - this.updateTheme(); - const observer = new MutationObserver(() => this.updateTheme()); - observer.observe(document.body, { - attributes: true, - attributeFilter: ['class', 'data-theme', 'data-vscode-theme-kind'], - }); - this.themeObserver = observer; - } - - private updateTheme() { - const body = document.body; - const isDark = - body.classList.contains('vscode-dark') || - body.classList.contains('theme-dark') || - body.dataset['theme'] === 'dark' || - body.getAttribute('data-vscode-theme-kind') === 'vscode-dark'; - this.isDarkMode.set(isDark); - } -} diff --git a/bigframes/display/table_widget_angular/src/app/widget-state.service.spec.ts b/bigframes/display/table_widget_angular/src/app/widget-state.service.spec.ts deleted file mode 100644 index 563f9fa75a5..00000000000 --- a/bigframes/display/table_widget_angular/src/app/widget-state.service.spec.ts +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { TestBed } from '@angular/core/testing'; -import { vi } from 'vitest'; -import { WidgetStateService } from './widget-state.service'; - -describe('WidgetStateService', () => { - let service: WidgetStateService; - let mockModel: any; - let mockListeners: { [key: string]: Function }; - - beforeEach(() => { - mockListeners = {}; - mockModel = { - get: vi.fn().mockImplementation((prop: string) => { - if (prop === 'page') return 2; - if (prop === 'page_size') return 25; - if (prop === 'max_columns') return 10; - if (prop === 'row_count') return 150; - if (prop === 'table_html') return '
'; - if (prop === 'sort_context') { - return [{ column: 'col1', ascending: true }]; - } - if (prop === 'orderable_columns') { - return ['col1', 'col2']; - } - if (prop === 'error_message') return 'initial error'; - return null; - }), - set: vi.fn(), - save_changes: vi.fn(), - on: vi.fn().mockImplementation( - (event: string, callback: Function) => { - mockListeners[event] = callback; - } - ) - }; - - TestBed.configureTestingModule({ - providers: [ - WidgetStateService, - { provide: 'ANYWIDGET_MODEL', useValue: mockModel } - ] - }); - service = TestBed.inject(WidgetStateService); - }); - - it('should be created', () => { - expect(service).toBeTruthy(); - }); - - it('should initialize signals from model values', () => { - expect(service.page()).toBe(2); - expect(service.pageSize()).toBe(25); - expect(service.maxColumns()).toBe(10); - expect(service.rowCount()).toBe(150); - expect(service.tableHtml()).toBe('
'); - expect(service.sortContext()).toEqual([ - { column: 'col1', ascending: true } - ]); - expect(service.orderableColumns()).toEqual(['col1', 'col2']); - expect(service.errorMessage()).toBe('initial error'); - }); - - it('should update signals when model triggers change events', () => { - mockModel.get.mockImplementation((prop: string) => { - if (prop === 'page') return 5; - if (prop === 'page_size') return 50; - return null; - }); - - mockListeners['change:page'](); - mockListeners['change:page_size'](); - - expect(service.page()).toBe(5); - expect(service.pageSize()).toBe(50); - }); - - it('should support dual-listen pattern for error messages', () => { - // 1. Check error_message change - mockModel.get.mockImplementation((prop: string) => { - if (prop === 'error_message') return 'new error'; - return null; - }); - mockListeners['change:error_message'](); - expect(service.errorMessage()).toBe('new error'); - - // 2. Check _error_message change - mockModel.get.mockImplementation((prop: string) => { - if (prop === '_error_message') return 'new private error'; - return null; - }); - mockListeners['change:_error_message'](); - expect(service.errorMessage()).toBe('new private error'); - }); - - it('should write updates back to model on setter methods', () => { - service.setPage(4); - expect(mockModel.set).toHaveBeenCalledWith('page', 4); - expect(mockModel.save_changes).toHaveBeenCalled(); - - service.setPageSize(100); - expect(mockModel.set).toHaveBeenCalledWith('page_size', 100); - expect(mockModel.set).toHaveBeenCalledWith('page', 0); - - service.setMaxColumns(15); - expect(mockModel.set).toHaveBeenCalledWith('max_columns', 15); - - service.setSortContext([{ column: 'col2', ascending: false }]); - expect(mockModel.set).toHaveBeenCalledWith( - 'sort_context', - [{ column: 'col2', ascending: false }] - ); - }); -}); diff --git a/bigframes/display/table_widget_angular/src/app/widget-state.service.ts b/bigframes/display/table_widget_angular/src/app/widget-state.service.ts deleted file mode 100644 index 54eff6eb948..00000000000 --- a/bigframes/display/table_widget_angular/src/app/widget-state.service.ts +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Injectable, Inject, signal } from '@angular/core'; - -export interface SortItem { - column: string; - ascending: boolean; -} - -@Injectable() -export class WidgetStateService { - readonly page = signal(0); - readonly pageSize = signal(10); - readonly maxColumns = signal(0); - readonly rowCount = signal(null); - readonly tableHtml = signal(''); - readonly sortContext = signal([]); - readonly orderableColumns = signal([]); - readonly errorMessage = signal(null); - readonly startExecution = signal(false); - readonly isDeferredMode = signal(false); - readonly dryRunInfo = signal(''); - readonly ping = signal(0); - - constructor(@Inject('ANYWIDGET_MODEL') private model: any) { - if (model) { - // Initialize from the model - this.page.set(model.get('page') ?? 0); - this.pageSize.set(model.get('page_size') ?? 10); - this.maxColumns.set(model.get('max_columns') ?? 0); - this.rowCount.set(model.get('row_count') ?? null); - this.tableHtml.set(model.get('table_html') ?? ''); - this.sortContext.set(model.get('sort_context') ?? []); - this.orderableColumns.set(model.get('orderable_columns') ?? []); - const initialError = - model.get('error_message') ?? - model.get('_error_message') ?? - null; - this.errorMessage.set(initialError); - this.startExecution.set(model.get('start_execution') ?? false); - this.isDeferredMode.set(model.get('is_deferred_mode') ?? false); - this.dryRunInfo.set(model.get('dry_run_info') ?? ''); - this.ping.set(model.get('ping') ?? 0); - - // Register event listeners for anywidget updates - model.on('change:page', () => { - this.page.set(model.get('page')); - }); - model.on('change:page_size', () => { - this.pageSize.set(model.get('page_size')); - }); - model.on('change:max_columns', () => { - this.maxColumns.set(model.get('max_columns')); - }); - model.on('change:row_count', () => { - this.rowCount.set(model.get('row_count')); - }); - model.on('change:table_html', () => { - this.tableHtml.set(model.get('table_html')); - }); - model.on('change:sort_context', () => { - this.sortContext.set(model.get('sort_context')); - }); - model.on('change:orderable_columns', () => { - this.orderableColumns.set(model.get('orderable_columns')); - }); - model.on('change:start_execution', () => { - this.startExecution.set(model.get('start_execution') ?? false); - }); - model.on('change:is_deferred_mode', () => { - this.isDeferredMode.set(model.get('is_deferred_mode') ?? false); - }); - model.on('change:dry_run_info', () => { - this.dryRunInfo.set(model.get('dry_run_info') ?? ''); - }); - model.on('change:ping', () => { - this.ping.set(model.get('ping') ?? 0); - }); - - // Robust dual-listen pattern for error messages (with/without underscore) - const handleErrorChange = () => { - const err = - model.get('error_message') ?? - model.get('_error_message') ?? - null; - this.errorMessage.set(err); - }; - model.on('change:error_message', handleErrorChange); - model.on('change:_error_message', handleErrorChange); - } - } - - setPage(page: number) { - this.page.set(page); - if (this.model) { - this.model.set('page', page); - this.model.save_changes(); - } - } - - setPageSize(pageSize: number) { - this.pageSize.set(pageSize); - this.page.set(0); - if (this.model) { - this.model.set('page_size', pageSize); - // Reset to page 0 on page size change - this.model.set('page', 0); - this.model.save_changes(); - } - } - - setMaxColumns(maxColumns: number) { - this.maxColumns.set(maxColumns); - if (this.model) { - this.model.set('max_columns', maxColumns); - this.model.save_changes(); - } - } - - setSortContext(context: SortItem[]) { - this.sortContext.set(context); - if (this.model) { - this.model.set('sort_context', context); - this.model.save_changes(); - } - } - - setStartExecution(startExecution: boolean) { - this.startExecution.set(startExecution); - if (this.model) { - this.model.set('start_execution', startExecution); - this.model.save_changes(); - } - } - - setPing(ping: number) { - this.ping.set(ping); - if (this.model) { - this.model.set('ping', ping); - this.model.save_changes(); - } - } -} diff --git a/bigframes/display/table_widget_angular/src/index.html b/bigframes/display/table_widget_angular/src/index.html deleted file mode 100644 index f5dda01b48a..00000000000 --- a/bigframes/display/table_widget_angular/src/index.html +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - TableWidgetAngular - - - - - -
- - diff --git a/bigframes/display/table_widget_angular/src/main.ts b/bigframes/display/table_widget_angular/src/main.ts deleted file mode 100644 index 3d515bb3d34..00000000000 --- a/bigframes/display/table_widget_angular/src/main.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { createApplication } from '@angular/platform-browser'; -import { App } from './app/app'; -import { ApplicationConfig, provideBrowserGlobalErrorListeners, provideZonelessChangeDetection } from '@angular/core'; - -function render({ model, el }: { model: any, el: HTMLElement }) { - // Create a container for the Angular app - const appRoot = document.createElement('div'); - appRoot.setAttribute('app-root', ''); - el.appendChild(appRoot); - - const appConfig: ApplicationConfig = { - providers: [ - provideBrowserGlobalErrorListeners(), - provideZonelessChangeDetection(), - { provide: 'ANYWIDGET_MODEL', useValue: model } - ] - }; - - createApplication(appConfig) - .then((appRef) => { - appRef.bootstrap(App, appRoot); - appRoot.removeAttribute('app-root'); - }) - .catch((err) => console.error(err)); -} - -export default { render }; diff --git a/bigframes/display/table_widget_angular/src/styles.css b/bigframes/display/table_widget_angular/src/styles.css deleted file mode 100644 index 95b248dae0a..00000000000 --- a/bigframes/display/table_widget_angular/src/styles.css +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* You can add global styles to this file, and also import other style files */ diff --git a/bigframes/display/table_widget_angular/tsconfig.app.json b/bigframes/display/table_widget_angular/tsconfig.app.json deleted file mode 100644 index 264f459bf87..00000000000 --- a/bigframes/display/table_widget_angular/tsconfig.app.json +++ /dev/null @@ -1,15 +0,0 @@ -/* To learn more about Typescript configuration file: https://www.typescriptlang.org/docs/handbook/tsconfig-json.html. */ -/* To learn more about Angular compiler options: https://angular.dev/reference/configs/angular-compiler-options. */ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "outDir": "./out-tsc/app", - "types": [] - }, - "include": [ - "src/**/*.ts" - ], - "exclude": [ - "src/**/*.spec.ts" - ] -} diff --git a/bigframes/display/table_widget_angular/tsconfig.json b/bigframes/display/table_widget_angular/tsconfig.json deleted file mode 100644 index 2ab7442758f..00000000000 --- a/bigframes/display/table_widget_angular/tsconfig.json +++ /dev/null @@ -1,33 +0,0 @@ -/* To learn more about Typescript configuration file: https://www.typescriptlang.org/docs/handbook/tsconfig-json.html. */ -/* To learn more about Angular compiler options: https://angular.dev/reference/configs/angular-compiler-options. */ -{ - "compileOnSave": false, - "compilerOptions": { - "strict": true, - "noImplicitOverride": true, - "noPropertyAccessFromIndexSignature": true, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": true, - "skipLibCheck": true, - "isolatedModules": true, - "experimentalDecorators": true, - "importHelpers": true, - "target": "ES2022", - "module": "preserve" - }, - "angularCompilerOptions": { - "enableI18nLegacyMessageIdFormat": false, - "strictInjectionParameters": true, - "strictInputAccessModifiers": true, - "strictTemplates": true - }, - "files": [], - "references": [ - { - "path": "./tsconfig.app.json" - }, - { - "path": "./tsconfig.spec.json" - } - ] -} diff --git a/bigframes/display/table_widget_angular/tsconfig.spec.json b/bigframes/display/table_widget_angular/tsconfig.spec.json deleted file mode 100644 index d38370633f6..00000000000 --- a/bigframes/display/table_widget_angular/tsconfig.spec.json +++ /dev/null @@ -1,15 +0,0 @@ -/* To learn more about Typescript configuration file: https://www.typescriptlang.org/docs/handbook/tsconfig-json.html. */ -/* To learn more about Angular compiler options: https://angular.dev/reference/configs/angular-compiler-options. */ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "outDir": "./out-tsc/spec", - "types": [ - "vitest/globals" - ] - }, - "include": [ - "src/**/*.d.ts", - "src/**/*.spec.ts" - ] -} diff --git a/bigframes/dtypes.py b/bigframes/dtypes.py index 3cc7e918aa0..29e1be1acea 100644 --- a/bigframes/dtypes.py +++ b/bigframes/dtypes.py @@ -14,13 +14,13 @@ """Mappings for Pandas dtypes supported by BigQuery DataFrames package""" +from dataclasses import dataclass import datetime import decimal import textwrap import typing -import warnings -from dataclasses import dataclass from typing import Any, Dict, List, Literal, Sequence, Union +import warnings import bigframes_vendored.constants as constants import db_dtypes # type: ignore @@ -31,6 +31,7 @@ import pyarrow as pa import shapely.geometry # type: ignore +import bigframes.core.backports import bigframes.exceptions # Type hints for Pandas dtypes supported by BigQuery DataFrame @@ -117,21 +118,6 @@ ] LOCAL_SCALAR_TYPES = typing.get_args(LOCAL_SCALAR_TYPE) -SUPPORTED_LITERAL_TYPE = typing.Union[ - bytes, - str, - int, - bool, - float, - datetime.datetime, - datetime.date, - datetime.time, - decimal.Decimal, - list, - shapely.geometry.base.BaseGeometry, -] -SUPPORTED_LITERAL_TYPES = typing.get_args(SUPPORTED_LITERAL_TYPE) - # Will have a few dtype variants: simple(eg. int, string, bool), complex (eg. list, struct), and virtual (eg. micro intervals, categorical) @dataclass(frozen=True) @@ -364,30 +350,10 @@ def is_json_like(type_: ExpressionType) -> bool: return type_ == JSON_DTYPE or type_ == STRING_DTYPE # Including JSON string -def is_json_encoding_type(type_: ExpressionType, strict: bool = False) -> bool: +def is_json_encoding_type(type_: ExpressionType) -> bool: # Types can be converted into JSON. # https://cloud.google.com/bigquery/docs/reference/standard-sql/json_functions#json_encodings - if is_array_like(type_): - return is_json_encoding_type(get_array_inner_type(type_), strict=strict) - if is_struct_like(type_): - return all( - is_json_encoding_type(field_type, strict=strict) - for field_type in get_struct_fields(type_).values() - ) - - if strict: - # Strict are the types (mostly) defined by json spec, with no/minimal - # encoding/decoding involved. So no temporal types. - return type_ in ( - INT_DTYPE, - FLOAT_DTYPE, - BOOL_DTYPE, - STRING_DTYPE, - JSON_DTYPE, - ) - else: - # GoogleSQL implementation handles anything but GEO - return type_ != GEO_DTYPE + return type_ != GEO_DTYPE def is_numeric(type_: ExpressionType, include_bool: bool = True) -> bool: @@ -404,11 +370,8 @@ def is_comparable(type_: ExpressionType) -> bool: def can_compare(type1: ExpressionType, type2: ExpressionType) -> bool: - try: - coerced_type = coerce_to_common(type1, type2) - return is_comparable(coerced_type) - except TypeError: - return False + coerced_type = coerce_to_common(type1, type2) + return is_comparable(coerced_type) def get_struct_fields(type_: ExpressionType) -> dict[str, Dtype]: @@ -416,10 +379,6 @@ def get_struct_fields(type_: ExpressionType) -> dict[str, Dtype]: assert isinstance(type_.pyarrow_dtype, pa.StructType) struct_type = type_.pyarrow_dtype result: dict[str, Dtype] = {} - - # Local import to break circular dependency with core.backports - import bigframes.core.backports - for field in bigframes.core.backports.pyarrow_struct_type_fields(struct_type): result[field.name] = arrow_dtype_to_bigframes_dtype(field.type) return result @@ -468,12 +427,7 @@ def is_clusterable(type_: ExpressionType) -> bool: def is_bool_coercable(type_: ExpressionType) -> bool: # TODO: Implement more bool coercions - return ( - (type_ is None) - or is_numeric(type_) - or is_string_like(type_) - or is_array_like(type_) - ) + return (type_ is None) or is_numeric(type_) or is_string_like(type_) BIGFRAMES_STRING_TO_BIGFRAMES: Dict[DtypeString, Dtype] = { @@ -610,10 +564,6 @@ def to_storage_type( return pa.list_(to_storage_type(arrow_type.value_type)) if pa.types.is_struct(arrow_type): assert isinstance(arrow_type, pa.StructType) - - # Local import to break circular dependency with core.backports - import bigframes.core.backports - return pa.struct( field.with_type(to_storage_type(field.type)) for field in bigframes.core.backports.pyarrow_struct_type_fields(arrow_type) @@ -627,10 +577,6 @@ def arrow_type_to_literal( """Create a representative literal value for an arrow type.""" if pa.types.is_list(arrow_type): return [arrow_type_to_literal(arrow_type.value_type)] - - # Local import to break circular dependency with core.backports - import bigframes.core.backports - if pa.types.is_struct(arrow_type): return { field.name: arrow_type_to_literal(field.type) @@ -760,6 +706,10 @@ def infer_literal_type(literal) -> typing.Optional[Dtype]: # Maybe also normalize literal to canonical python representation to remove this burden from compilers? if isinstance(literal, pa.Scalar): return arrow_dtype_to_bigframes_dtype(literal.type) + if pd.api.types.is_list_like(literal): + element_types = [infer_literal_type(i) for i in literal] + common_type = lcd_type(*element_types) + return list_type(common_type) if pd.api.types.is_dict_like(literal): fields = [] for key in literal.keys(): @@ -770,10 +720,6 @@ def infer_literal_type(literal) -> typing.Optional[Dtype]: pa.field(key, field_type, nullable=(not pa.types.is_list(field_type))) ) return pd.ArrowDtype(pa.struct(fields)) - if pd.api.types.is_list_like(literal): - element_types = [infer_literal_type(i) for i in literal] - common_type = lcd_type(*element_types) - return list_type(common_type) if pd.isna(literal): return None # Null value without a definite type # Make sure to check datetime before date as datetimes are also dates @@ -808,13 +754,6 @@ def convert_schema_field( ) -> typing.Tuple[str, Dtype]: is_repeated = field.mode == "REPEATED" if field.field_type == "RECORD": - if field.description == OBJ_REF_DESCRIPTION_TAG: - bf_dtype = OBJ_REF_DTYPE # type: ignore - if is_repeated: - pa_type = pa.list_(bigframes_dtype_to_arrow_dtype(bf_dtype)) - bf_dtype = pd.ArrowDtype(pa_type) - return field.name, bf_dtype - mapped_fields = map(convert_schema_field, field.fields) fields = [] for name, dtype in mapped_fields: @@ -858,14 +797,10 @@ def convert_to_schema_field( ) inner_field = convert_to_schema_field(name, inner_type, overrides) return google.cloud.bigquery.SchemaField( - name, - inner_field.field_type, - mode="REPEATED", - fields=inner_field.fields, - description=inner_field.description, + name, inner_field.field_type, mode="REPEATED", fields=inner_field.fields ) if pa.types.is_struct(bigframes_dtype.pyarrow_dtype): - inner_fields: list[google.cloud.bigquery.SchemaField] = [] + inner_fields: list[pa.Field] = [] struct_type = typing.cast(pa.StructType, bigframes_dtype.pyarrow_dtype) for i in range(struct_type.num_fields): field = struct_type.field(i) @@ -874,14 +809,6 @@ def convert_to_schema_field( convert_to_schema_field(field.name, inner_bf_type, overrides) ) - if bigframes_dtype == OBJ_REF_DTYPE: - return google.cloud.bigquery.SchemaField( - name, - "RECORD", - fields=inner_fields, - description=OBJ_REF_DESCRIPTION_TAG, - ) - return google.cloud.bigquery.SchemaField( name, "RECORD", fields=inner_fields ) @@ -896,7 +823,7 @@ def convert_to_schema_field( def bf_type_from_type_kind( - bq_schema: Sequence[google.cloud.bigquery.SchemaField], + bq_schema: list[google.cloud.bigquery.SchemaField], ) -> typing.Dict[str, Dtype]: """Converts bigquery sql type to the default bigframes dtype.""" return {name: dtype for name, dtype in map(convert_schema_field, bq_schema)} @@ -970,16 +897,11 @@ def is_compatible(scalar: typing.Any, dtype: Dtype) -> typing.Optional[Dtype]: def lcd_type(*dtypes: Dtype) -> Dtype: if len(dtypes) < 1: raise ValueError("at least one dypes should be provided") - + if len(dtypes) == 1: + return dtypes[0] unique_dtypes = set(dtypes) - if None in unique_dtypes: - unique_dtypes.remove(None) - - if len(unique_dtypes) == 0: - return None if len(unique_dtypes) == 1: - return next(iter(unique_dtypes)) - + return unique_dtypes.pop() # Implicit conversion currently only supported for numeric types hierarchy: list[Dtype] = [ BOOL_DTYPE, @@ -988,9 +910,9 @@ def lcd_type(*dtypes: Dtype) -> Dtype: BIGNUMERIC_DTYPE, FLOAT_DTYPE, ] - if any([dtype not in hierarchy for dtype in unique_dtypes]): + if any([dtype not in hierarchy for dtype in dtypes]): return None - lcd_index = max([hierarchy.index(dtype) for dtype in unique_dtypes]) + lcd_index = max([hierarchy.index(dtype) for dtype in dtypes]) return hierarchy[lcd_index] @@ -1026,7 +948,6 @@ def lcd_type_or_throw(dtype1: Dtype, dtype2: Dtype) -> Dtype: TIMEDELTA_DESCRIPTION_TAG = "#microseconds" -OBJ_REF_DESCRIPTION_TAG = "bigframes_dtype: OBJ_REF_DTYPE" def contains_db_dtypes_json_arrow_type(type_): @@ -1037,9 +958,6 @@ def contains_db_dtypes_json_arrow_type(type_): return contains_db_dtypes_json_arrow_type(type_.value_type) if isinstance(type_, pa.StructType): - # Local import to break circular dependency with core.backports - import bigframes.core.backports - return any( contains_db_dtypes_json_arrow_type(field.type) for field in bigframes.core.backports.pyarrow_struct_type_fields(type_) diff --git a/bigframes/enums.py b/bigframes/enums.py index 3aaf6020206..aa5e1c830fc 100644 --- a/bigframes/enums.py +++ b/bigframes/enums.py @@ -16,6 +16,7 @@ # NOTE: This module should not depend on any others in the package. + import enum diff --git a/bigframes/exceptions.py b/bigframes/exceptions.py index dea8a55f9b5..9facb40e8ea 100644 --- a/bigframes/exceptions.py +++ b/bigframes/exceptions.py @@ -75,10 +75,6 @@ class MaximumResultRowsExceeded(RuntimeError): """Maximum number of rows in the result was exceeded.""" -class TranspilationError(RuntimeError): - """Failed to transpile a Python function to BigFrames Expression.""" - - class TimeTravelDisabledWarning(Warning): """A query was reattempted without time travel.""" @@ -130,10 +126,6 @@ class FunctionPackageVersionWarning(PreviewWarning): """ -class PythonTranspilerPreviewWarning(PreviewWarning): - """Python Transpiler is a preview feature.""" - - def format_message(message: str, fill: bool = True): """[Private] Formats a warning message. diff --git a/bigframes/extensions/bigframes/dataframe_accessor.py b/bigframes/extensions/bigframes/dataframe_accessor.py deleted file mode 100644 index f706c19ef2d..00000000000 --- a/bigframes/extensions/bigframes/dataframe_accessor.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from typing import TypeVar, cast - -import bigframes.dataframe -import bigframes.extensions.core.dataframe_accessor as core_accessor -import bigframes.series -from bigframes.core.logging import log_adapter - -T = TypeVar("T", bound="bigframes.dataframe.DataFrame") -S = TypeVar("S", bound="bigframes.series.Series") - - -@log_adapter.class_logger -class BigframesAIAccessor(core_accessor.AIAccessor[T, S]): - """ - BigFrames DataFrame accessor for BigQuery AI functions. - """ - - def __init__(self, bf_obj: T): - super().__init__(bf_obj) - - def _bf_from_dataframe( - self, session: bigframes.session.Session | None - ) -> bigframes.dataframe.DataFrame: - return self._obj - - def _to_dataframe(self, bf_df: bigframes.dataframe.DataFrame) -> T: - return cast(T, bf_df) - - def _to_series(self, bf_series: bigframes.series.Series) -> S: - return cast(S, bf_series) - - -@log_adapter.class_logger -class BigframesBigQueryDataFrameAccessor(core_accessor.BigQueryDataFrameAccessor[T, S]): - """ - BigFrames DataFrame accessor for BigQuery DataFrames functionality. - """ - - def __init__(self, bf_obj: T): - super().__init__(bf_obj) - - @property - def ai(self) -> BigframesAIAccessor: - return BigframesAIAccessor(self._obj) - - def _bf_from_dataframe( - self, session: bigframes.session.Session | None - ) -> bigframes.dataframe.DataFrame: - return self._obj - - def _to_dataframe(self, bf_df: bigframes.dataframe.DataFrame) -> T: - return cast(T, bf_df) - - def _to_series(self, bf_series: bigframes.series.Series) -> S: - return cast(S, bf_series) diff --git a/bigframes/extensions/bigframes/series_accessor.py b/bigframes/extensions/bigframes/series_accessor.py deleted file mode 100644 index c9026595d97..00000000000 --- a/bigframes/extensions/bigframes/series_accessor.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated by the script: scripts/generate_bigframes_bigquery.py -# - -from __future__ import annotations - -from typing import Optional, TypeVar, cast - -from bigframes import dataframe, series, session -from bigframes.core.logging import log_adapter -from bigframes.extensions.core import series_accessor as core_accessor - -T = TypeVar("T", bound="dataframe.DataFrame") -S = TypeVar("S", bound="series.Series") - - -@log_adapter.class_logger -class BigframesBigQuerySeriesAccessor(core_accessor.BigQuerySeriesAccessor[T, S]): - def __init__(self, bf_obj: S): - super().__init__(bf_obj) - - def _bf_from_series( - self, session: Optional[session.Session] = None - ) -> series.Series: - return self._obj - - def _to_dataframe(self, bf_df: dataframe.DataFrame) -> T: - return cast(T, bf_df) - - def _to_series(self, bf_series: series.Series) -> S: - return cast(S, bf_series) - - @property - def aead(self) -> BigframesAeadSeriesAccessor[T, S]: - return BigframesAeadSeriesAccessor(self._obj) - - @property - def ai(self) -> BigframesAiSeriesAccessor[T, S]: - return BigframesAiSeriesAccessor(self._obj) - - -@log_adapter.class_logger -class BigframesAeadSeriesAccessor(core_accessor.AeadSeriesAccessor[T, S]): - def __init__(self, bf_obj: S): - super().__init__(bf_obj) - - def _bf_from_series( - self, session: Optional[session.Session] = None - ) -> series.Series: - return self._obj - - def _to_dataframe(self, bf_df: dataframe.DataFrame) -> T: - return cast(T, bf_df) - - def _to_series(self, bf_series: series.Series) -> S: - return cast(S, bf_series) - - -@log_adapter.class_logger -class BigframesAiSeriesAccessor(core_accessor.AiSeriesAccessor[T, S]): - def __init__(self, bf_obj: S): - super().__init__(bf_obj) - - def _bf_from_series( - self, session: Optional[session.Session] = None - ) -> series.Series: - return self._obj - - def _to_dataframe(self, bf_df: dataframe.DataFrame) -> T: - return cast(T, bf_df) - - def _to_series(self, bf_series: series.Series) -> S: - return cast(S, bf_series) diff --git a/bigframes/extensions/core/abstract_series_accessor.py b/bigframes/extensions/core/abstract_series_accessor.py deleted file mode 100644 index 22d09861877..00000000000 --- a/bigframes/extensions/core/abstract_series_accessor.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated by the script: scripts/generate_bigframes_bigquery.py -# - -from __future__ import annotations - -import abc -from typing import ( - Generic, - Optional, - TypeVar, -) - -from bigframes import dataframe, series, session - -T = TypeVar("T") -S = TypeVar("S") - - -class AbstractBigQuerySeriesAccessor(abc.ABC, Generic[T, S]): - def __init__(self, obj: S): - self._obj = obj - - @abc.abstractmethod - def _bf_from_series( - self, session: Optional[session.Session] = None - ) -> series.Series: - """Convert the accessor's object to a BigFrames Series.""" - - @abc.abstractmethod - def _to_dataframe(self, bf_df: dataframe.DataFrame) -> T: - """Convert a BigFrames DataFrame to the accessor's object type.""" - - @abc.abstractmethod - def _to_series(self, bf_series: series.Series) -> S: - """Convert a BigFrames Series to the accessor's object type.""" diff --git a/bigframes/extensions/core/dataframe_accessor.py b/bigframes/extensions/core/dataframe_accessor.py deleted file mode 100644 index e490aa907dc..00000000000 --- a/bigframes/extensions/core/dataframe_accessor.py +++ /dev/null @@ -1,340 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import abc -from typing import ( - TYPE_CHECKING, - Any, - Generic, - Iterable, - List, - Literal, - Mapping, - Tuple, - TypeVar, - Union, -) - -if TYPE_CHECKING: - import pandas as pd - - import bigframes.dataframe - import bigframes.series - import bigframes.session - - PROMPT_TYPE = Union[ - str, - bigframes.series.Series, - pd.Series, - List[Union[str, bigframes.series.Series, pd.Series]], - Tuple[Union[str, bigframes.series.Series, pd.Series], ...], - ] -else: - PROMPT_TYPE = Any - -T = TypeVar("T") -S = TypeVar("S") - - -class AbstractBigQueryDataFrameAccessor(abc.ABC, Generic[T, S]): - @abc.abstractmethod - def _bf_from_dataframe( - self, session: bigframes.session.Session | None - ) -> bigframes.dataframe.DataFrame: - """Convert the accessor's object to a BigFrames DataFrame.""" - - @abc.abstractmethod - def _to_dataframe(self, bf_df: bigframes.dataframe.DataFrame) -> T: - """Convert a BigFrames DataFrame to the accessor's object type.""" - - @abc.abstractmethod - def _to_series(self, bf_series: bigframes.series.Series) -> S: - """Convert a BigFrames Series to the accessor's object type.""" - - -class AIAccessor(AbstractBigQueryDataFrameAccessor[T, S]): - """ - DataFrame accessor for BigQuery AI functions. - """ - - def __init__(self, obj: T): - self._obj = obj - - def forecast( - self, - *, - data_col: str, - timestamp_col: str, - model: str = "TimesFM 2.0", - id_cols: Iterable[str] | None = None, - horizon: int = 10, - confidence_level: float = 0.95, - context_window: int | None = None, - output_historical_time_series: bool = False, - session: bigframes.session.Session | None = None, - ) -> T: - """ - Forecast time series at future horizon using BigQuery AI.FORECAST. - - This is an accessor for :func:`bigframes.bigquery.ai.forecast`. See that - function's documentation for detailed parameter descriptions and examples. - """ - import bigframes.bigquery.ai - - bf_df = self._bf_from_dataframe(session) - result = bigframes.bigquery.ai.forecast( - bf_df, - data_col=data_col, - timestamp_col=timestamp_col, - model=model, - id_cols=id_cols, - horizon=horizon, - confidence_level=confidence_level, - context_window=context_window, - output_historical_time_series=output_historical_time_series, - ) - return self._to_dataframe(result) - - def generate( - self, - prompt: PROMPT_TYPE, - *, - connection_id: str | None = None, - endpoint: str | None = None, - request_type: Literal["dedicated", "shared", "unspecified"] | None = None, - model_params: Mapping[Any, Any] | None = None, - output_schema: Mapping[str, str] | None = None, - ) -> S: - """ - Returns the AI analysis based on the prompt, which can be any combination of text and unstructured data. - - This is an accessor for :func:`bigframes.bigquery.ai.generate`. See that - function's documentation for detailed parameter descriptions and examples. - """ - import bigframes.bigquery.ai - - result = bigframes.bigquery.ai.generate( - prompt, - connection_id=connection_id, - endpoint=endpoint, - request_type=request_type, - model_params=model_params, - output_schema=output_schema, - ) - return self._to_series(result) - - def generate_bool( - self, - prompt: PROMPT_TYPE, - *, - connection_id: str | None = None, - endpoint: str | None = None, - request_type: Literal["dedicated", "shared", "unspecified"] | None = None, - model_params: Mapping[Any, Any] | None = None, - ) -> S: - """ - Returns the AI analysis based on the prompt, which can be any combination of text and unstructured data. - - This is an accessor for :func:`bigframes.bigquery.ai.generate_bool`. See that - function's documentation for detailed parameter descriptions and examples. - """ - import bigframes.bigquery.ai - - result = bigframes.bigquery.ai.generate_bool( - prompt, - connection_id=connection_id, - endpoint=endpoint, - request_type=request_type, - model_params=model_params, - ) - return self._to_series(result) - - def generate_int( - self, - prompt: PROMPT_TYPE, - *, - connection_id: str | None = None, - endpoint: str | None = None, - request_type: Literal["dedicated", "shared", "unspecified"] | None = None, - model_params: Mapping[Any, Any] | None = None, - ) -> S: - """ - Returns the AI analysis based on the prompt, which can be any combination of text and unstructured data. - - This is an accessor for :func:`bigframes.bigquery.ai.generate_int`. See that - function's documentation for detailed parameter descriptions and examples. - """ - import bigframes.bigquery.ai - - result = bigframes.bigquery.ai.generate_int( - prompt, - connection_id=connection_id, - endpoint=endpoint, - request_type=request_type, - model_params=model_params, - ) - return self._to_series(result) - - def generate_double( - self, - prompt: PROMPT_TYPE, - *, - connection_id: str | None = None, - endpoint: str | None = None, - request_type: Literal["dedicated", "shared", "unspecified"] | None = None, - model_params: Mapping[Any, Any] | None = None, - ) -> S: - """ - Returns the AI analysis based on the prompt, which can be any combination of text and unstructured data. - - This is an accessor for :func:`bigframes.bigquery.ai.generate_double`. See that - function's documentation for detailed parameter descriptions and examples. - """ - import bigframes.bigquery.ai - - result = bigframes.bigquery.ai.generate_double( - prompt, - connection_id=connection_id, - endpoint=endpoint, - request_type=request_type, - model_params=model_params, - ) - return self._to_series(result) - - def classify( - self, - input: PROMPT_TYPE, - categories: tuple[str, ...] | list[str], - *, - examples: list[tuple[str, str]] - | list[tuple[str, list[str] | tuple[str, ...]]] - | None = None, - connection_id: str | None = None, - endpoint: str | None = None, - output_mode: Literal["single", "multi"] | None = None, - optimization_mode: Literal["minimize_cost", "maximize_quality"] | None = None, - max_error_ratio: float | None = None, - ) -> S: - """ - Classifies a given input into one of the specified categories. It will always return one of the provided categories best fit the prompt input. - - This is an accessor for :func:`bigframes.bigquery.ai.classify`. See that - function's documentation for detailed parameter descriptions and examples. - """ - import bigframes.bigquery.ai - - result = bigframes.bigquery.ai.classify( - input, - categories, - examples=examples, - connection_id=connection_id, - endpoint=endpoint, - output_mode=output_mode, - optimization_mode=optimization_mode, - max_error_ratio=max_error_ratio, - ) - return self._to_series(result) - - def if_( - self, - prompt: PROMPT_TYPE, - *, - connection_id: str | None = None, - endpoint: str | None = None, - optimization_mode: Literal["minimize_cost", "maximize_quality"] | None = None, - max_error_ratio: float | None = None, - ) -> S: - """ - Evaluates the prompt to True or False. Compared to ``ai.generate_bool()``, this function - provides optimization such that not all rows are evaluated with the LLM. - - This is an accessor for :func:`bigframes.bigquery.ai.if_`. See that - function's documentation for detailed parameter descriptions and examples. - """ - import bigframes.bigquery.ai - - result = bigframes.bigquery.ai.if_( - prompt, - connection_id=connection_id, - endpoint=endpoint, - optimization_mode=optimization_mode, - max_error_ratio=max_error_ratio, - ) - return self._to_series(result) - - def score( - self, - prompt: PROMPT_TYPE, - *, - connection_id: str | None = None, - endpoint: str | None = None, - max_error_ratio: float | None = None, - ) -> S: - """ - Computes a score based on rubrics described in natural language. It will return a double value. - - This is an accessor for :func:`bigframes.bigquery.ai.score`. See that - function's documentation for detailed parameter descriptions and examples. - """ - import bigframes.bigquery.ai - - result = bigframes.bigquery.ai.score( - prompt, - connection_id=connection_id, - endpoint=endpoint, - max_error_ratio=max_error_ratio, - ) - return self._to_series(result) - - -class BigQueryDataFrameAccessor(AbstractBigQueryDataFrameAccessor[T, S]): - """ - DataFrame accessor for BigQuery DataFrames functionality. - """ - - def __init__(self, obj: T): - self._obj = obj - - @property - @abc.abstractmethod - def ai(self) -> AIAccessor: - """ - Accessor for BigQuery AI functions. - - Returns: - AIAccessor: Accessor for BigQuery AI functions. - """ - - def sql_scalar( - self, - sql_template: str, - *, - output_dtype=None, - session: bigframes.session.Session | None = None, - ) -> S: - """ - Compute a new Series by applying a SQL scalar function to the DataFrame. - - This is an accessor for :func:`bigframes.bigquery.sql_scalar`. See that - function's documentation for detailed parameter descriptions and examples. - """ - import bigframes.bigquery - - bf_df = self._bf_from_dataframe(session) - result = bigframes.bigquery.sql_scalar( - sql_template, bf_df, output_dtype=output_dtype - ) - return self._to_series(result) diff --git a/bigframes/extensions/core/series_accessor.py b/bigframes/extensions/core/series_accessor.py deleted file mode 100644 index 5aa50905c8f..00000000000 --- a/bigframes/extensions/core/series_accessor.py +++ /dev/null @@ -1,1229 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated by the script: scripts/generate_bigframes_bigquery.py -# - -from __future__ import annotations - -import abc -import datetime -from typing import ( - Any, - Literal, - Optional, - TypeVar, - Union, - cast, -) - -from bigframes import series, session -from bigframes.core import col, sentinels -from bigframes.extensions.core import abstract_series_accessor, series_mixins - -T = TypeVar("T") -S = TypeVar("S") - - -class BigQuerySeriesAccessor( - abstract_series_accessor.AbstractBigQuerySeriesAccessor[T, S] -): - """Series accessor for BigQuery functions.""" - - @property - @abc.abstractmethod - def aead(self) -> AeadSeriesAccessor[T, S]: - """Accessor for BigQuery aead functions.""" - - @property - @abc.abstractmethod - def ai(self) -> AiSeriesAccessor[T, S]: - """Accessor for BigQuery ai functions.""" - - def deterministic_decrypt_bytes( - self, - ciphertext: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes], - ], - additional_data: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Uses the matching key from `keyset` to decrypt `ciphertext` and verifies the integrity of the data using `additional_data`. Returns an error if decryption fails.""" - from bigframes.operations.googlesql.global_namespace.aead_encryption import ( - deterministic_decrypt_bytes as deterministic_decrypt_bytes_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - ciphertext, - additional_data, - ) - - bf_series = self._bf_from_series(session) - result = deterministic_decrypt_bytes_impl( - bf_series, - ciphertext, - additional_data, - ) - return self._to_series(cast(series.Series, result)) - - def deterministic_decrypt_string( - self, - ciphertext: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes], - ], - additional_data: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Like `DETERMINISTIC_DECRYPT_BYTES`, but where plaintext is of type STRING.""" - from bigframes.operations.googlesql.global_namespace.aead_encryption import ( - deterministic_decrypt_string as deterministic_decrypt_string_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - ciphertext, - additional_data, - ) - - bf_series = self._bf_from_series(session) - result = deterministic_decrypt_string_impl( - bf_series, - ciphertext, - additional_data, - ) - return self._to_series(cast(series.Series, result)) - - def deterministic_encrypt( - self, - plaintext: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, str], - ], - additional_data: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, str], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Encrypts `plaintext` using the primary cryptographic key in `keyset` using deterministic AEAD. The algorithm of the primary key must be `DETERMINISTIC_AEAD_AES_SIV_CMAC_256`. Binds the ciphertext to the context defined by `additional_data`. Returns `NULL` if any input is `NULL`.""" - from bigframes.operations.googlesql.global_namespace.aead_encryption import ( - deterministic_encrypt as deterministic_encrypt_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - plaintext, - additional_data, - ) - - bf_series = self._bf_from_series(session) - result = deterministic_encrypt_impl( - bf_series, - plaintext, - additional_data, - ) - return self._to_series(cast(series.Series, result)) - - def array_concat( - self, - array_expression_2: Union[ - series.Series, - col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Concatenates one or more arrays with the same element type into a single array.""" - from bigframes.operations.googlesql.global_namespace.array import ( - array_concat as array_concat_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - array_expression_2, - ) - - bf_series = self._bf_from_series(session) - result = array_concat_impl( - bf_series, - array_expression_2, - ) - return self._to_series(cast(series.Series, result)) - - def array_first( - self, - *, - session: Optional[session.Session] = None, - ) -> S: - """Takes an array and returns the first element in the array.""" - from bigframes.operations.googlesql.global_namespace.array import ( - array_first as array_first_impl, - ) - - bf_series = self._bf_from_series(session) - result = array_first_impl( - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def array_first_n( - self, - n: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Returns a prefix of `input_array` consisting of the first `n` elements.""" - from bigframes.operations.googlesql.global_namespace.array import ( - array_first_n as array_first_n_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - n, - ) - - bf_series = self._bf_from_series(session) - result = array_first_n_impl( - bf_series, - n, - ) - return self._to_series(cast(series.Series, result)) - - def array_includes( - self, - search_value: Union[ - series.Series, - col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Takes an array and returns `TRUE` if there is an element in the array that is equal to the search_value.""" - from bigframes.operations.googlesql.global_namespace.array import ( - array_includes as array_includes_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - search_value, - ) - - bf_series = self._bf_from_series(session) - result = array_includes_impl( - bf_series, - search_value, - ) - return self._to_series(cast(series.Series, result)) - - def array_includes_all( - self, - search_values: Union[ - series.Series, - col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Takes an array to search and an array of search values. Returns `TRUE` if all search values are in the array to search, otherwise returns `FALSE`.""" - from bigframes.operations.googlesql.global_namespace.array import ( - array_includes_all as array_includes_all_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - search_values, - ) - - bf_series = self._bf_from_series(session) - result = array_includes_all_impl( - bf_series, - search_values, - ) - return self._to_series(cast(series.Series, result)) - - def array_includes_any( - self, - search_values: Union[ - series.Series, - col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Takes an array to search and an array of search values. Returns `TRUE` if any search values are in the array to search, otherwise returns `FALSE`.""" - from bigframes.operations.googlesql.global_namespace.array import ( - array_includes_any as array_includes_any_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - search_values, - ) - - bf_series = self._bf_from_series(session) - result = array_includes_any_impl( - bf_series, - search_values, - ) - return self._to_series(cast(series.Series, result)) - - def array_is_distinct( - self, - *, - session: Optional[session.Session] = None, - ) -> S: - """Returns `TRUE` if the array contains no repeated elements, using the same equality comparison logic as `SELECT DISTINCT`.""" - from bigframes.operations.googlesql.global_namespace.array import ( - array_is_distinct as array_is_distinct_impl, - ) - - bf_series = self._bf_from_series(session) - result = array_is_distinct_impl( - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def array_last( - self, - *, - session: Optional[session.Session] = None, - ) -> S: - """Takes an array and returns the last element in the array.""" - from bigframes.operations.googlesql.global_namespace.array import ( - array_last as array_last_impl, - ) - - bf_series = self._bf_from_series(session) - result = array_last_impl( - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def array_length( - self, - *, - session: Optional[session.Session] = None, - ) -> S: - """Compute the length of each array element in the Series. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - - >>> s = bpd.Series([[1, 2, 8, 3], [], [3, 4]]) - >>> bbq.array_length(s) - 0 4 - 1 0 - 2 2 - dtype: Int64 - - You can call this function using the Series `bigquery` accessor. - - >>> s.bigquery.array_length() - 0 4 - 1 0 - 2 2 - dtype: Int64 - - You can also use this accessor on a pandas Series after importing bigframes. - - >>> import bigframes - >>> import pandas as pd - >>> ps = pd.Series([[1, 2, 8, 3], [], [3, 4]]) - >>> ps.bigquery.array_length() - 0 4 - 1 0 - 2 2 - dtype: Int64 - - You can also apply this function directly to Series using `apply`. - - >>> s.apply(bbq.array_length, by_row=False) - 0 4 - 1 0 - 2 2 - dtype: Int64 - - Args: - series (bigframes.series.Series): A Series with array columns. - - Returns: - bigframes.series.Series: A Series of integer values indicating - the length of each element in the Series. - """ - from bigframes.operations.googlesql.global_namespace.array import ( - array_length as array_length_impl, - ) - - bf_series = self._bf_from_series(session) - result = array_length_impl( - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def array_reverse( - self, - *, - session: Optional[session.Session] = None, - ) -> S: - """Returns the input `ARRAY` with elements in reverse order.""" - from bigframes.operations.googlesql.global_namespace.array import ( - array_reverse as array_reverse_impl, - ) - - bf_series = self._bf_from_series(session) - result = array_reverse_impl( - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def array_slice( - self, - start_offset: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ], - end_offset: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Returns an array containing zero or more consecutive elements from the input array.""" - from bigframes.operations.googlesql.global_namespace.array import ( - array_slice as array_slice_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - start_offset, - end_offset, - ) - - bf_series = self._bf_from_series(session) - result = array_slice_impl( - bf_series, - start_offset, - end_offset, - ) - return self._to_series(cast(series.Series, result)) - - def array_to_string( - self, - delimiter: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, str], - ], - null_text: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - *, - session: Optional[session.Session] = None, - ) -> S: - """Converts array elements within a Series into delimited strings. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - - >>> s = bpd.Series([["H", "i", "!"], ["Hello", "World"], np.nan, [], ["Hi"]]) - >>> bbq.array_to_string(s, delimiter=", ") - 0 H, i, ! - 1 Hello, World - 2 - 3 - 4 Hi - dtype: string - - You can call this function using the Series `bigquery` accessor. - - >>> s.bigquery.array_to_string(delimiter=", ") - 0 H, i, ! - 1 Hello, World - 2 - 3 - 4 Hi - dtype: string - - You can also use this accessor on a pandas Series after importing bigframes. - - >>> import bigframes - >>> import pandas as pd - >>> ps = pd.Series([["H", "i", "!"], ["Hello", "World"], None, [], ["Hi"]]) - >>> ps.bigquery.array_to_string(delimiter=", ") - 0 H, i, ! - 1 Hello, World - 2 - 3 - 4 Hi - dtype: string - - Args: - series (bigframes.series.Series): A Series containing arrays. - delimiter (str): The string used to separate array elements. - null_text (str, optional): The string to replace any NULL values in the array with. - - Returns: - bigframes.series.Series: A Series containing delimited strings. - """ - from bigframes.operations.googlesql.global_namespace.array import ( - array_to_string as array_to_string_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - delimiter, - null_text, - ) - - bf_series = self._bf_from_series(session) - result = array_to_string_impl( - bf_series, - delimiter, - null_text, - ) - return self._to_series(cast(series.Series, result)) - - def flatten( - self, - depth: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - *, - session: Optional[session.Session] = None, - ) -> S: - """Takes an array of nested data and flattens a specific part of it into a single, flat array with the [array elements field access operator][array-el-field-operator]. Returns `NULL` if the input value is `NULL`.""" - from bigframes.operations.googlesql.global_namespace.array import ( - flatten as flatten_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - depth, - ) - - bf_series = self._bf_from_series(session) - result = flatten_impl( - bf_series, - depth, - ) - return self._to_series(cast(series.Series, result)) - - def bit_count( - self, - *, - session: Optional[session.Session] = None, - ) -> S: - """The input, `expression`, must be an integer or `BYTES`. Returns the number of bits that are set in the input expression. For signed integers, this is the number of bits in two's complement form.""" - from bigframes.operations.googlesql.global_namespace.bit import ( - bit_count as bit_count_impl, - ) - - bf_series = self._bf_from_series(session) - result = bit_count_impl( - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def bool_( - self, - *, - session: Optional[session.Session] = None, - ) -> S: - """Converts a JSON boolean to a SQL BOOL value.""" - from bigframes.operations.googlesql.global_namespace.conversion import ( - bool_ as bool__impl, - ) - - bf_series = self._bf_from_series(session) - result = bool__impl( - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def double( - self, - wide_number_mode: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - *, - session: Optional[session.Session] = None, - ) -> S: - """Converts a JSON number to a SQL FLOAT64 value.""" - from bigframes.operations.googlesql.global_namespace.conversion import ( - double as double_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - wide_number_mode, - ) - - bf_series = self._bf_from_series(session) - result = double_impl( - bf_series, - wide_number_mode, - ) - return self._to_series(cast(series.Series, result)) - - def float64( - self, - wide_number_mode: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - *, - session: Optional[session.Session] = None, - ) -> S: - """Converts a JSON number to a SQL FLOAT64 value.""" - from bigframes.operations.googlesql.global_namespace.conversion import ( - float64 as float64_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - wide_number_mode, - ) - - bf_series = self._bf_from_series(session) - result = float64_impl( - bf_series, - wide_number_mode, - ) - return self._to_series(cast(series.Series, result)) - - def int64( - self, - *, - session: Optional[session.Session] = None, - ) -> S: - """Converts a JSON number to a SQL INT64 value.""" - from bigframes.operations.googlesql.global_namespace.conversion import ( - int64 as int64_impl, - ) - - bf_series = self._bf_from_series(session) - result = int64_impl( - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def parse_bignumeric( - self, - *, - session: Optional[session.Session] = None, - ) -> S: - """Converts a STRING to a BIGNUMERIC value.""" - from bigframes.operations.googlesql.global_namespace.conversion import ( - parse_bignumeric as parse_bignumeric_impl, - ) - - bf_series = self._bf_from_series(session) - result = parse_bignumeric_impl( - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def parse_numeric( - self, - *, - session: Optional[session.Session] = None, - ) -> S: - """Converts a STRING to a NUMERIC value.""" - from bigframes.operations.googlesql.global_namespace.conversion import ( - parse_numeric as parse_numeric_impl, - ) - - bf_series = self._bf_from_series(session) - result = parse_numeric_impl( - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def string( - self, - timezone: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - *, - session: Optional[session.Session] = None, - ) -> S: - """Converts a value to a STRING value.""" - from bigframes.operations.googlesql.global_namespace.conversion import ( - string as string_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - timezone, - ) - - bf_series = self._bf_from_series(session) - result = string_impl( - bf_series, - timezone, - ) - return self._to_series(cast(series.Series, result)) - - def date( - self, - time_zone_expression: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - year: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - month: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - day: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - *, - session: Optional[session.Session] = None, - ) -> S: - """Constructs or extracts a date.""" - from bigframes.operations.googlesql.global_namespace.date import ( - date as date_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - time_zone_expression, - year, - month, - day, - ) - - bf_series = self._bf_from_series(session) - result = date_impl( - bf_series, - time_zone_expression, - year, - month, - day, - ) - return self._to_series(cast(series.Series, result)) - - def date_add( - self, - int64_expression: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ], - date_part: Union[ - series.Series, - col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Adds a specified time interval to a DATE.""" - from bigframes.operations.googlesql.global_namespace.date import ( - date_add as date_add_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - int64_expression, - date_part, - ) - - bf_series = self._bf_from_series(session) - result = date_add_impl( - bf_series, - int64_expression, - date_part, - ) - return self._to_series(cast(series.Series, result)) - - def date_diff( - self, - start_date: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], datetime.date], - ], - granularity: Union[ - series.Series, - col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Gets the number of unit boundaries between two DATE values (end_date - start_date) at a particular time granularity.""" - from bigframes.operations.googlesql.global_namespace.date import ( - date_diff as date_diff_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - start_date, - granularity, - ) - - bf_series = self._bf_from_series(session) - result = date_diff_impl( - bf_series, - start_date, - granularity, - ) - return self._to_series(cast(series.Series, result)) - - def date_from_unix_date( - self, - *, - session: Optional[session.Session] = None, - ) -> S: - """Interprets an INT64 expression as the number of days since 1970-01-01.""" - from bigframes.operations.googlesql.global_namespace.date import ( - date_from_unix_date as date_from_unix_date_impl, - ) - - bf_series = self._bf_from_series(session) - result = date_from_unix_date_impl( - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def date_sub( - self, - int64_expression: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ], - date_part: Union[ - series.Series, - col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Subtracts a specified time interval from a DATE.""" - from bigframes.operations.googlesql.global_namespace.date import ( - date_sub as date_sub_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - int64_expression, - date_part, - ) - - bf_series = self._bf_from_series(session) - result = date_sub_impl( - bf_series, - int64_expression, - date_part, - ) - return self._to_series(cast(series.Series, result)) - - def date_trunc( - self, - granularity: Union[ - series.Series, - col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Truncates a DATE, DATETIME, or TIMESTAMP value at a particular granularity.""" - from bigframes.operations.googlesql.global_namespace.date import ( - date_trunc as date_trunc_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - granularity, - ) - - bf_series = self._bf_from_series(session) - result = date_trunc_impl( - bf_series, - granularity, - ) - return self._to_series(cast(series.Series, result)) - - def extract( - self, - part: Union[ - series.Series, - col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - time_zone: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - *, - session: Optional[session.Session] = None, - ) -> S: - """Returns the value corresponding to the specified date part.""" - from bigframes.operations.googlesql.global_namespace.date import ( - extract as extract_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - part, - time_zone, - ) - - bf_series = self._bf_from_series(session) - result = extract_impl( - bf_series, - part, - time_zone, - ) - return self._to_series(cast(series.Series, result)) - - def format_date( - self, - format_string: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Formats a DATE value according to a specified format string.""" - from bigframes.operations.googlesql.global_namespace.date import ( - format_date as format_date_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - format_string, - ) - - bf_series = self._bf_from_series(session) - result = format_date_impl( - format_string, - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def last_day( - self, - date_part: Union[ - series.Series, - col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - *, - session: Optional[session.Session] = None, - ) -> S: - """Returns the last day from a date expression. This is commonly used to return the last day of the month.""" - from bigframes.operations.googlesql.global_namespace.date import ( - last_day as last_day_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - date_part, - ) - - bf_series = self._bf_from_series(session) - result = last_day_impl( - bf_series, - date_part, - ) - return self._to_series(cast(series.Series, result)) - - def parse_date( - self, - format_string: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Converts a STRING value to a DATE value.""" - from bigframes.operations.googlesql.global_namespace.date import ( - parse_date as parse_date_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - format_string, - ) - - bf_series = self._bf_from_series(session) - result = parse_date_impl( - format_string, - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - def unix_date( - self, - *, - session: Optional[session.Session] = None, - ) -> S: - """Returns the number of days since 1970-01-01.""" - from bigframes.operations.googlesql.global_namespace.date import ( - unix_date as unix_date_impl, - ) - - bf_series = self._bf_from_series(session) - result = unix_date_impl( - bf_series, - ) - return self._to_series(cast(series.Series, result)) - - -class AeadSeriesAccessor(abstract_series_accessor.AbstractBigQuerySeriesAccessor[T, S]): - """Series accessor for BigQuery aead functions.""" - - def decrypt_bytes( - self, - ciphertext: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes], - ], - additional_data: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Uses the matching key from keyset to decrypt ciphertext and verifies the integrity of the data using additional_data. Returns an error if decryption or verification fails.""" - from bigframes.operations.googlesql.aead import ( - decrypt_bytes as decrypt_bytes_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - ciphertext, - additional_data, - ) - - bf_series = self._bf_from_series(session) - result = decrypt_bytes_impl( - bf_series, - ciphertext, - additional_data, - ) - return self._to_series(cast(series.Series, result)) - - def decrypt_string( - self, - ciphertext: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes], - ], - additional_data: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Like AEAD.DECRYPT_BYTES, but where additional_data is of type STRING.""" - from bigframes.operations.googlesql.aead import ( - decrypt_string as decrypt_string_impl, - ) - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - ciphertext, - additional_data, - ) - - bf_series = self._bf_from_series(session) - result = decrypt_string_impl( - bf_series, - ciphertext, - additional_data, - ) - return self._to_series(cast(series.Series, result)) - - def encrypt( - self, - plaintext: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, str], - ], - additional_data: Union[ - series.Series, - col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, str], - ], - *, - session: Optional[session.Session] = None, - ) -> S: - """Encrypts plaintext using the primary cryptographic key in keyset. The algorithm of the primary key must be AEAD_AES_GCM_256. Binds the ciphertext to the context defined by additional_data. Returns NULL if any input is NULL.""" - from bigframes.operations.googlesql.aead import encrypt as encrypt_impl - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - - session = googlesql._find_session( - plaintext, - additional_data, - ) - - bf_series = self._bf_from_series(session) - result = encrypt_impl( - bf_series, - plaintext, - additional_data, - ) - return self._to_series(cast(series.Series, result)) - - -class AiSeriesAccessor(series_mixins.AIMixin[T, S]): - """Series accessor for BigQuery ai functions.""" diff --git a/bigframes/extensions/core/series_mixins.py b/bigframes/extensions/core/series_mixins.py deleted file mode 100644 index 4d1b61ecb0c..00000000000 --- a/bigframes/extensions/core/series_mixins.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from typing import Any, List, Literal, Mapping, TypeVar - -import pandas as pd - -from bigframes import series -from bigframes import session as bf_session -from bigframes.bigquery import ai -from bigframes.extensions.core import abstract_series_accessor -from bigframes.ml import base as ml_base - -T = TypeVar("T") -S = TypeVar("S") - - -class AIMixin(abstract_series_accessor.AbstractBigQuerySeriesAccessor[T, S]): - def generate_embedding( - self, - model: ml_base.BaseEstimator | str | pd.Series, - *, - output_dimensionality: int | None = None, - task_type: str | None = None, - start_second: float | None = None, - end_second: float | None = None, - interval_seconds: float | None = None, - trial_id: int | None = None, - session: bf_session.Session | None = None, - ) -> T: - """ - Creates embeddings that describe an entity — for example, a piece of text or an image. - - This is an accessor for :func:`bigframes.bigquery.ai.generate_embedding`. See that - function's documentation for detailed parameter descriptions and examples. - """ - - bf_series = self._bf_from_series(session) - result = ai.generate_embedding( - model, - bf_series, - output_dimensionality=output_dimensionality, - task_type=task_type, - start_second=start_second, - end_second=end_second, - interval_seconds=interval_seconds, - trial_id=trial_id, - ) - return self._to_dataframe(result) - - def generate_text( - self, - model: ml_base.BaseEstimator | str | pd.Series, - *, - temperature: float | None = None, - max_output_tokens: int | None = None, - top_k: int | None = None, - top_p: float | None = None, - stop_sequences: List[str] | None = None, - ground_with_google_search: bool | None = None, - request_type: str | None = None, - session: bf_session.Session | None = None, - ) -> T: - """ - Generates text using a BigQuery ML model. - - This is an accessor for :func:`bigframes.bigquery.ai.generate_text`. See that - function's documentation for detailed parameter descriptions and examples. - """ - bf_series = self._bf_from_series(session) - result = ai.generate_text( - model, - bf_series, - temperature=temperature, - max_output_tokens=max_output_tokens, - top_k=top_k, - top_p=top_p, - stop_sequences=stop_sequences, - ground_with_google_search=ground_with_google_search, - request_type=request_type, - ) - return self._to_dataframe(result) - - def generate_table( - self, - model: ml_base.BaseEstimator | str | pd.Series, - *, - output_schema: str | Mapping[str, str], - temperature: float | None = None, - top_p: float | None = None, - max_output_tokens: int | None = None, - stop_sequences: List[str] | None = None, - request_type: str | None = None, - session: bf_session.Session | None = None, - ) -> T: - """ - Generates a table using a BigQuery ML model. - - This is an accessor for :func:`bigframes.bigquery.ai.generate_table`. See that - function's documentation for detailed parameter descriptions and examples. - """ - bf_series = self._bf_from_series(session) - result = ai.generate_table( - model, - bf_series, - output_schema=output_schema, - temperature=temperature, - top_p=top_p, - max_output_tokens=max_output_tokens, - stop_sequences=stop_sequences, - request_type=request_type, - ) - return self._to_dataframe(result) - - def embed( - self, - *, - endpoint: str | None = None, - model: str | None = None, - task_type: ( - Literal[ - "retrieval_query", - "retrieval_document", - "semantic_similarity", - "classification", - "clustering", - "question_answering", - "fact_verification", - "code_retrieval_query", - ] - | None - ) = None, - title: str | None = None, - model_params: Mapping[Any, Any] | None = None, - connection_id: str | None = None, - session: bf_session.Session | None = None, - ) -> S: - """ - Creates embeddings from text or image data in BigQuery. - - This is an accessor for :func:`bigframes.bigquery.ai.embed`. See that - function's documentation for detailed parameter descriptions and examples. - """ - - bf_series = self._bf_from_series(session) - result = ai.embed( - bf_series, - endpoint=endpoint, - model=model, - task_type=task_type, - title=title, - model_params=model_params, - connection_id=connection_id, - ) - return self._to_series(result) - - def similarity( - self, - other: str | series.Series | pd.Series, - *, - endpoint: str | None = None, - model: str | None = None, - model_params: Mapping[Any, Any] | None = None, - connection_id: str | None = None, - session: bf_session.Session | None = None, - ) -> S: - """ - Returns a FLOAT64 value that represents the cosine similarity between the two inputs. - - This is an accessor for :func:`bigframes.bigquery.ai.similarity`. See that - function's documentation for detailed parameter descriptions and examples. - """ - - bf_series = self._bf_from_series(session) - result = ai.similarity( - bf_series, - other, - endpoint=endpoint, - model=model, - model_params=model_params, - connection_id=connection_id, - ) - return self._to_series(result) diff --git a/bigframes/extensions/pandas/__init__.py b/bigframes/extensions/pandas/__init__.py deleted file mode 100644 index 6af1f769b5b..00000000000 --- a/bigframes/extensions/pandas/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -BigQuery DataFrames automatically registers a pandas extenstion when imported. -This allows you to use the power of the BigQuery engine with pandas objects -directly. -""" - -from bigframes.extensions.pandas.dataframe_accessor import ( - PandasBigQueryDataFrameAccessor, -) -from bigframes.extensions.pandas.series_accessor import ( - PandasBigQuerySeriesAccessor, -) - -__all__ = [ - "PandasBigQueryDataFrameAccessor", - "PandasBigQuerySeriesAccessor", -] diff --git a/bigframes/extensions/pandas/dataframe_accessor.py b/bigframes/extensions/pandas/dataframe_accessor.py deleted file mode 100644 index 512134cac03..00000000000 --- a/bigframes/extensions/pandas/dataframe_accessor.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import TypeVar, cast - -import pandas -import pandas.api.extensions - -import bigframes.core.global_session as bf_session -import bigframes.dataframe -import bigframes.pandas as bpd -from bigframes.core.logging import log_adapter -from bigframes.extensions.core.dataframe_accessor import ( - AIAccessor, - BigQueryDataFrameAccessor, -) - -T = TypeVar("T", bound="pandas.DataFrame") -S = TypeVar("S", bound="pandas.Series") - - -@log_adapter.class_logger -class PandasAIAccessor(AIAccessor[T, S]): - """ - Pandas DataFrame accessor for BigQuery AI functions. - """ - - def __init__(self, pandas_obj: T): - super().__init__(pandas_obj) - - def _bf_from_dataframe( - self, session: bigframes.session.Session | None - ) -> bigframes.dataframe.DataFrame: - if session is None: - session = bf_session.get_global_session() - - return cast(bpd.DataFrame, session.read_pandas(self._obj)) - - def _to_dataframe(self, bf_df: bigframes.dataframe.DataFrame) -> T: - return cast(T, bf_df.to_pandas(ordered=True)) - - def _to_series(self, bf_series: bigframes.series.Series) -> S: - return cast(S, bf_series.to_pandas(ordered=True)) - - -@pandas.api.extensions.register_dataframe_accessor("bigquery") -@log_adapter.class_logger -class PandasBigQueryDataFrameAccessor(BigQueryDataFrameAccessor[T, S]): - """ - Pandas DataFrame accessor for BigQuery DataFrames functionality. - - This accessor is registered under the ``bigquery`` namespace on pandas DataFrame objects. - """ - - def __init__(self, pandas_obj: T): - super().__init__(pandas_obj) - - @property - def ai(self) -> PandasAIAccessor: - return PandasAIAccessor(self._obj) - - def _bf_from_dataframe(self, session) -> bigframes.dataframe.DataFrame: - if session is None: - session = bf_session.get_global_session() - - return cast(bpd.DataFrame, session.read_pandas(self._obj)) - - def _to_dataframe(self, bf_df: bigframes.dataframe.DataFrame) -> T: - return cast(T, bf_df.to_pandas(ordered=True)) - - def _to_series(self, bf_series: bigframes.series.Series) -> S: - return cast(S, bf_series.to_pandas(ordered=True)) diff --git a/bigframes/extensions/pandas/series_accessor.py b/bigframes/extensions/pandas/series_accessor.py deleted file mode 100644 index 9c33996c421..00000000000 --- a/bigframes/extensions/pandas/series_accessor.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated by the script: scripts/generate_bigframes_bigquery.py -# - -from __future__ import annotations - -from typing import Optional, TypeVar, cast - -import pandas -import pandas.api.extensions - -from bigframes import dataframe, series, session -from bigframes.core import global_session as bf_session -from bigframes.core.logging import log_adapter -from bigframes.extensions.core import series_accessor as core_accessor - -T = TypeVar("T", bound="pandas.DataFrame") -S = TypeVar("S", bound="pandas.Series") - - -@pandas.api.extensions.register_series_accessor("bigquery") -@log_adapter.class_logger -class PandasBigQuerySeriesAccessor(core_accessor.BigQuerySeriesAccessor[T, S]): - def __init__(self, pandas_obj: S): - super().__init__(pandas_obj) - - def _bf_from_series( - self, session: Optional[session.Session] = None - ) -> series.Series: - if session is None: - session = bf_session.get_global_session() - return cast(series.Series, session.read_pandas(self._obj)) - - def _to_dataframe(self, bf_df: dataframe.DataFrame) -> T: - return cast(T, bf_df.to_pandas(ordered=True)) - - def _to_series(self, bf_series: series.Series) -> S: - return cast(S, bf_series.to_pandas(ordered=True)) - - @property - def aead(self) -> PandasAeadSeriesAccessor[T, S]: - return PandasAeadSeriesAccessor(self._obj) - - @property - def ai(self) -> PandasAiSeriesAccessor[T, S]: - return PandasAiSeriesAccessor(self._obj) - - -@log_adapter.class_logger -class PandasAeadSeriesAccessor(core_accessor.AeadSeriesAccessor[T, S]): - def __init__(self, pandas_obj: S): - super().__init__(pandas_obj) - - def _bf_from_series( - self, session: Optional[session.Session] = None - ) -> series.Series: - if session is None: - session = bf_session.get_global_session() - return cast(series.Series, session.read_pandas(self._obj)) - - def _to_dataframe(self, bf_df: dataframe.DataFrame) -> T: - return cast(T, bf_df.to_pandas(ordered=True)) - - def _to_series(self, bf_series: series.Series) -> S: - return cast(S, bf_series.to_pandas(ordered=True)) - - -@log_adapter.class_logger -class PandasAiSeriesAccessor(core_accessor.AiSeriesAccessor[T, S]): - def __init__(self, pandas_obj: S): - super().__init__(pandas_obj) - - def _bf_from_series( - self, session: Optional[session.Session] = None - ) -> series.Series: - if session is None: - session = bf_session.get_global_session() - return cast(series.Series, session.read_pandas(self._obj)) - - def _to_dataframe(self, bf_df: dataframe.DataFrame) -> T: - return cast(T, bf_df.to_pandas(ordered=True)) - - def _to_series(self, bf_series: series.Series) -> S: - return cast(S, bf_series.to_pandas(ordered=True)) diff --git a/bigframes/formatting_helpers.py b/bigframes/formatting_helpers.py index 9ab25951932..3c37a3470d5 100644 --- a/bigframes/formatting_helpers.py +++ b/bigframes/formatting_helpers.py @@ -19,14 +19,14 @@ import datetime import html import random -from typing import TYPE_CHECKING, Any, Optional, Type, Union +from typing import Any, Optional, Type, TYPE_CHECKING, Union import bigframes_vendored.constants as constants import google.api_core.exceptions as api_core_exceptions import google.cloud.bigquery as bigquery import humanize - -import bigframes._config +import IPython +import IPython.display as display if TYPE_CHECKING: import bigframes.core.events @@ -47,7 +47,7 @@ def add_feedback_link( exception: Union[ api_core_exceptions.RetryError, api_core_exceptions.GoogleAPICallError - ], + ] ): exception.message = exception.message + f" {constants.FEEDBACK_LINK}" @@ -73,23 +73,18 @@ def repr_query_job(query_job: Optional[bigquery.QueryJob]): if query_job is None: return "No job information available" if query_job.dry_run: - return ( - f"Computation deferred. Computation will process " - f"{get_formatted_bytes(query_job.total_bytes_processed)}" - ) + return f"Computation deferred. Computation will process {get_formatted_bytes(query_job.total_bytes_processed)}" res = "Query Job Info" for key, value in query_job_prop_pairs.items(): job_val = getattr(query_job, value) if job_val is not None: res += "\n" if key == "Job Id": # add link to job - res += f"""Job url: { - get_job_url( - project_id=query_job.project, - location=query_job.location, - job_id=query_job.job_id, - ) - }""" + res += f"""Job url: {get_job_url( + project_id=query_job.project, + location=query_job.location, + job_id=query_job.job_id, + )}""" elif key == "Slot Time": res += f"""{key}: {get_formatted_time(job_val)}""" elif key == "Bytes Processed": @@ -110,15 +105,11 @@ def repr_query_job_html(query_job: Optional[bigquery.QueryJob]): if query_job is None: return "No job information available" if query_job.dry_run: - return ( - f"Computation deferred. Computation will process " - f"{get_formatted_bytes(query_job.total_bytes_processed)}" - ) + return f"Computation deferred. Computation will process {get_formatted_bytes(query_job.total_bytes_processed)}" # We can reuse the plaintext repr for now or make a nicer table. - # For deferred mode consistency, let's just wrap the text in a pre - # block or similar, but the request implies we want a distinct HTML - # representation if possible. + # For deferred mode consistency, let's just wrap the text in a pre block or similar, + # but the request implies we want a distinct HTML representation if possible. # However, existing repr_query_job returns a simple string. # Let's format it as a simple table or list. @@ -132,10 +123,7 @@ def repr_query_job_html(query_job: Optional[bigquery.QueryJob]): location=query_job.location, job_id=query_job.job_id, ) - res += ( - f'
  • Job: ' - f"{query_job.job_id}
    • " - ) + res += f'
  • Job: {query_job.job_id}
    • ' elif key == "Slot Time": res += f"
  • {key}: {get_formatted_time(job_val)}
    • " elif key == "Bytes Processed": @@ -146,95 +134,96 @@ def repr_query_job_html(query_job: Optional[bigquery.QueryJob]): return res +current_display: Optional[display.HTML] = None current_display_id: Optional[str] = None +previous_display_html: str = "" -def create_progress_callback(): - # bind potentially thread-local config to the callback so that it uses the user thread - # config even if callback is invoked from a worker thread. - display_opts = bigframes._config.options.display - - def progress_callback( - envelope: Any, - ): - """Displays a progress bar while the query is running""" - global current_display_id - - try: - import bigframes._config - import bigframes.core.events - except ImportError: - # Since this gets called from __del__, skip if the import fails to avoid - # ImportError: sys.meta_path is None, Python is likely shutting down. - # This will allow cleanup to continue. - return - - # Publisher.publish automatically wraps raw Event objects in an - # EventEnvelope, ensuring subscribers receive a consistent contract. - assert isinstance(envelope, bigframes.core.events.EventEnvelope) - event = envelope.event - progress_bar = envelope.progress_bar - - if progress_bar == bigframes.core.events._DEFAULT: - progress_bar = display_opts.progress_bar - - if progress_bar == "auto": - progress_bar = "notebook" if in_ipython() else "terminal" - - if progress_bar == "notebook": - import IPython.display as display - - display_html = None - - if isinstance(event, bigframes.core.events.ExecutionStarted): - # Start a new context for progress output. - current_display_id = None - - elif isinstance(event, bigframes.core.events.BigQuerySentEvent): - display_html = render_bqquery_sent_event_html(event) - - elif isinstance(event, bigframes.core.events.BigQueryRetryEvent): - display_html = render_bqquery_retry_event_html(event) - - elif isinstance(event, bigframes.core.events.BigQueryReceivedEvent): - display_html = render_bqquery_received_event_html(event) - - elif isinstance(event, bigframes.core.events.BigQueryFinishedEvent): - display_html = render_bqquery_finished_event_html(event) +def progress_callback( + event: bigframes.core.events.Event, +): + """Displays a progress bar while the query is running""" + global current_display, current_display_id, previous_display_html - elif isinstance(event, bigframes.core.events.SessionClosed): - display_html = f"Session {event.session_id} closed." + try: + import bigframes._config + import bigframes.core.events + except ImportError: + # Since this gets called from __del__, skip if the import fails to avoid + # ImportError: sys.meta_path is None, Python is likely shutting down. + # This will allow cleanup to continue. + return - if display_html: - if current_display_id: - display.update_display( - display.HTML(display_html), - display_id=current_display_id, - ) - else: - current_display_id = str(random.random()) - display.display( - display.HTML(display_html), - display_id=current_display_id, - ) + progress_bar = bigframes._config.options.display.progress_bar - elif progress_bar == "terminal": - message = None + if progress_bar == "auto": + progress_bar = "notebook" if in_ipython() else "terminal" - if isinstance(event, bigframes.core.events.BigQuerySentEvent): - message = render_bqquery_sent_event_plaintext(event) - print(message) - elif isinstance(event, bigframes.core.events.BigQueryRetryEvent): - message = render_bqquery_retry_event_plaintext(event) - print(message) - elif isinstance(event, bigframes.core.events.BigQueryReceivedEvent): - message = render_bqquery_received_event_plaintext(event) - print(message) - elif isinstance(event, bigframes.core.events.BigQueryFinishedEvent): - message = render_bqquery_finished_event_plaintext(event) - print(message) + if progress_bar == "notebook": + if ( + isinstance(event, bigframes.core.events.ExecutionStarted) + or current_display is None + or current_display_id is None + ): + previous_display_html = "" + current_display_id = str(random.random()) + current_display = display.HTML("Starting.") + display.display( + current_display, + display_id=current_display_id, + ) - return progress_callback + if isinstance(event, bigframes.core.events.BigQuerySentEvent): + previous_display_html = render_bqquery_sent_event_html(event) + display.update_display( + display.HTML(previous_display_html), + display_id=current_display_id, + ) + elif isinstance(event, bigframes.core.events.BigQueryRetryEvent): + previous_display_html = render_bqquery_retry_event_html(event) + display.update_display( + display.HTML(previous_display_html), + display_id=current_display_id, + ) + elif isinstance(event, bigframes.core.events.BigQueryReceivedEvent): + previous_display_html = render_bqquery_received_event_html(event) + display.update_display( + display.HTML(previous_display_html), + display_id=current_display_id, + ) + elif isinstance(event, bigframes.core.events.BigQueryFinishedEvent): + previous_display_html = render_bqquery_finished_event_html(event) + display.update_display( + display.HTML(previous_display_html), + display_id=current_display_id, + ) + elif isinstance(event, bigframes.core.events.ExecutionFinished): + display.update_display( + display.HTML(f"✅ Completed. {previous_display_html}"), + display_id=current_display_id, + ) + elif isinstance(event, bigframes.core.events.SessionClosed): + display.update_display( + display.HTML(f"Session {event.session_id} closed."), + display_id=current_display_id, + ) + elif progress_bar == "terminal": + if isinstance(event, bigframes.core.events.ExecutionStarted): + print("Starting execution.") + elif isinstance(event, bigframes.core.events.BigQuerySentEvent): + message = render_bqquery_sent_event_plaintext(event) + print(message) + elif isinstance(event, bigframes.core.events.BigQueryRetryEvent): + message = render_bqquery_retry_event_plaintext(event) + print(message) + elif isinstance(event, bigframes.core.events.BigQueryReceivedEvent): + message = render_bqquery_received_event_plaintext(event) + print(message) + elif isinstance(event, bigframes.core.events.BigQueryFinishedEvent): + message = render_bqquery_finished_event_plaintext(event) + print(message) + elif isinstance(event, bigframes.core.events.ExecutionFinished): + print("Execution done.") def wait_for_job(job: GenericJob, progress_bar: Optional[str] = None): @@ -250,16 +239,13 @@ def wait_for_job(job: GenericJob, progress_bar: Optional[str] = None): try: if progress_bar == "notebook": - import IPython.display as display - display_id = str(random.random()) loading_bar = display.HTML(get_base_job_loading_html(job)) display.display(loading_bar, display_id=display_id) job.result() job.reload() display.update_display( - display.HTML(get_base_job_loading_html(job)), - display_id=display_id, + display.HTML(get_base_job_loading_html(job)), display_id=display_id ) elif progress_bar == "terminal": inital_loading_bar = get_base_job_loading_string(job) @@ -313,10 +299,7 @@ def render_job_link_html( job_id=job_id, ) if job_url: - job_link = ( - f' [' - f"Job {project_id}:{location}.{job_id} details]" - ) + job_link = f' [Job {project_id}:{location}.{job_id} details]' else: job_link = "" return job_link @@ -353,10 +336,7 @@ def get_job_url( """ if project_id is None or location is None or job_id is None: return None - return ( - f"https://console.cloud.google.com/bigquery?project={project_id}" - f"&j=bq:{location}:{job_id}&page=queryresults" - ) + return f"""https://console.cloud. google.com/bigquery?project={project_id}&j=bq:{location}:{job_id}&page=queryresults""" def render_bqquery_sent_event_html( @@ -381,10 +361,7 @@ def render_bqquery_sent_event_html( job_id=event.job_id, request_id=event.request_id, ) - query_text_details = ( - f"
    SQL
    "
-        f"{html.escape(event.query)}
    " - ) + query_text_details = f"
    SQL
    {html.escape(event.query)}
    " return f""" Query started{query_id}.{job_link}{query_text_details} @@ -433,10 +410,7 @@ def render_bqquery_retry_event_html( job_id=event.job_id, request_id=event.request_id, ) - query_text_details = ( - f"
    SQL
    "
-        f"{html.escape(event.query)}
    " - ) + query_text_details = f"
    SQL
    {html.escape(event.query)}
    " return f""" Retrying query{query_id}.{job_link}{query_text_details} @@ -482,10 +456,7 @@ def render_bqquery_received_event_html( query_plan_details = "" if event.query_plan: plan_str = "\n".join([str(entry) for entry in event.query_plan]) - query_plan_details = ( - f"
    Query Plan
    "
-            f"{html.escape(plan_str)}
    " - ) + query_plan_details = f"
    Query Plan
    {html.escape(plan_str)}
    " return f""" Query{query_id} is {event.state}.{job_link}{query_plan_details} @@ -548,8 +519,7 @@ def render_bqquery_finished_event_plaintext( bytes_str = "" if event.total_bytes_processed is not None: - size_str = humanize.naturalsize(event.total_bytes_processed) - bytes_str = f" {size_str} processed." + bytes_str = f" {humanize.naturalsize(event.total_bytes_processed)} processed." slot_time_str = "" if event.slot_millis is not None: @@ -578,15 +548,11 @@ def get_base_job_loading_html(job: GenericJob): Returns: Html string. """ - return f"""{job.job_type.capitalize()} job {job.job_id} is { - job.state - }. Open Job""" + return f"""{job.job_type.capitalize()} job {job.job_id} is {job.state}. Open Job""" def get_base_job_loading_string(job: GenericJob): @@ -597,13 +563,11 @@ def get_base_job_loading_string(job: GenericJob): Returns: String """ - return f"""{job.job_type.capitalize()} job {job.job_id} is {job.state}. \n{ - get_job_url( - project_id=job.job_id, - location=job.location, - job_id=job.job_id, - ) - }""" + return f"""{job.job_type.capitalize()} job {job.job_id} is {job.state}. \n{get_job_url( + project_id=job.job_id, + location=job.location, + job_id=job.job_id, + )}""" def get_formatted_time(val): @@ -615,8 +579,7 @@ def get_formatted_time(val): Duration string """ try: - delta = datetime.timedelta(milliseconds=float(val)) - return humanize.naturaldelta(delta) + return humanize.naturaldelta(datetime.timedelta(milliseconds=float(val))) except Exception: return val @@ -635,10 +598,7 @@ def get_formatted_bytes(val): def get_bytes_processed_string(val: Any): - """Try to get bytes processed string. - - Return empty if passed non int value. - """ + """Try to get bytes processed string. Return empty if passed non int value""" bytes_processed_string = "" if isinstance(val, int): bytes_processed_string = f"""{get_formatted_bytes(val)} processed. """ @@ -647,8 +607,4 @@ def get_bytes_processed_string(val: Any): def in_ipython(): """Return True iff we're in a colab-like IPython.""" - try: - import IPython - except (ImportError, NameError): - return False return hasattr(IPython.get_ipython(), "kernel") diff --git a/bigframes/functions/__init__.py b/bigframes/functions/__init__.py index 86119717d7b..5f87956a611 100644 --- a/bigframes/functions/__init__.py +++ b/bigframes/functions/__init__.py @@ -11,8 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from bigframes.functions.function import Udf +from bigframes.functions.function import ( + BigqueryCallableRoutine, + BigqueryCallableRowRoutine, +) __all__ = [ - "Udf", + "BigqueryCallableRoutine", + "BigqueryCallableRowRoutine", ] diff --git a/bigframes/functions/_function_client.py b/bigframes/functions/_function_client.py index 69f99b50276..a82217da035 100644 --- a/bigframes/functions/_function_client.py +++ b/bigframes/functions/_function_client.py @@ -15,26 +15,32 @@ from __future__ import annotations +import inspect import logging import os -import re +import random import shutil +import string import tempfile import textwrap import types +from typing import Any, cast, Optional, Sequence, Tuple, TYPE_CHECKING import warnings -from typing import Any, cast -import google.api_core.exceptions -import google.api_core.retry import requests -from google.cloud import bigquery, functions_v2 import bigframes.exceptions as bfe import bigframes.formatting_helpers as bf_formatting import bigframes.functions.function_template as bff_template -import bigframes.functions.udf_def as udf_def -from bigframes.functions import _utils + +if TYPE_CHECKING: + from bigframes.session import Session + +import google.api_core.exceptions +import google.api_core.retry +from google.cloud import bigquery, functions_v2 + +from . import _utils logger = logging.getLogger(__name__) @@ -61,83 +67,92 @@ class FunctionClient: + # Wait time (in seconds) for an IAM binding to take effect after creation. + _iam_wait_seconds = 120 + # TODO(b/392707725): Convert all necessary parameters for cloud function # deployment into method parameters. def __init__( self, - gcp_project_id: str, - bq_location: str, - bq_client: bigquery.Client, + gcp_project_id, + bq_location, + bq_dataset, + bq_client, + bq_connection_id, bq_connection_manager, - cloud_functions_client: functions_v2.FunctionServiceClient, - publisher, + cloud_function_region=None, + cloud_functions_client=None, + cloud_function_service_account=None, + cloud_function_kms_key_name=None, + cloud_function_docker_repository=None, + cloud_build_service_account=None, + *, + session: Session, ): self._gcp_project_id = gcp_project_id self._bq_location = bq_location + self._bq_dataset = bq_dataset self._bq_client = bq_client + self._bq_connection_id = bq_connection_id self._bq_connection_manager = bq_connection_manager - self._publisher = publisher - self._cloud_functions_client = cloud_functions_client - - self._cf_location = _utils.gcf_location_from_bq_location(bq_location) - - @property - def cloudfunctions_region(self) -> str: - return self._cf_location + self._session = session - def _create_bq_connection( - self, - connection_id: str, - bq_project_id: str, - ) -> None: - self._bq_connection_manager.create_bq_connection( - bq_project_id, - self._bq_location, - connection_id, - "run.invoker", - ) + # Optional attributes only for remote functions. + self._cloud_function_region = cloud_function_region + self._cloud_functions_client = cloud_functions_client + self._cloud_function_service_account = cloud_function_service_account + self._cloud_function_kms_key_name = cloud_function_kms_key_name + self._cloud_function_docker_repository = cloud_function_docker_repository + self._cloud_build_service_account = cloud_build_service_account + + def _create_bq_connection(self) -> None: + if self._bq_connection_manager: + self._bq_connection_manager.create_bq_connection( + self._gcp_project_id, + self._bq_location, + self._bq_connection_id, + "run.invoker", + ) - def _ensure_dataset_exists(self, dataset_ref: bigquery.DatasetReference) -> None: + def _ensure_dataset_exists(self) -> None: # Make sure the dataset exists, i.e. if it doesn't exist, go ahead and # create it. + dataset = bigquery.Dataset( + bigquery.DatasetReference.from_string( + self._bq_dataset, default_project=self._gcp_project_id + ) + ) + dataset.location = self._bq_location try: # This check does not require bigquery.datasets.create IAM # permission. So, if the data set already exists, then user can work # without having that permission. - self._bq_client.get_dataset(dataset_ref) + self._bq_client.get_dataset(dataset) except google.api_core.exceptions.NotFound: # This requires bigquery.datasets.create IAM permission. - dataset = bigquery.Dataset(dataset_ref) - dataset.location = self._bq_location self._bq_client.create_dataset(dataset, exists_ok=True) def _create_bq_function(self, create_function_ddl: str) -> None: # TODO(swast): plumb through the original, user-facing api_name. import bigframes.session._io.bigquery - _, query_job = bigframes.session._io.bigquery.start_query_with_job( - self._bq_client, + _, query_job = bigframes.session._io.bigquery.start_query_with_client( + cast(bigquery.Client, self._session.bqclient), create_function_ddl, job_config=bigquery.QueryJobConfig(), - location=self._bq_location, + location=None, project=None, timeout=None, metrics=None, - publisher=self._publisher, + query_with_job=True, + publisher=self._session._publisher, ) logger.info(f"Created bigframes function {query_job.ddl_target_routine}") def _format_function_options(self, function_options: dict) -> str: - def format_val(val): - if isinstance(val, str): - return f"'{val}'" - if isinstance(val, (list, tuple)): - return str(list(val)) - return str(val) - return ", ".join( [ - f"{key}={format_val(val)}" + f"{key}='{val}'" if isinstance(val, str) else f"{key}={val}" for key, val in function_options.items() if val is not None ] @@ -145,70 +160,73 @@ def format_val(val): def create_bq_remote_function( self, - routine_ref: bigquery.RoutineReference, - udf_def: udf_def.RemoteFunctionConfig, - maybe_reuse: bool, - try_create_connection: bool, + input_args: Sequence[str], + input_types: Sequence[str], + output_type: str, + endpoint: str, + bq_function_name: str, + max_batching_rows: int, + metadata: str, ): """Create a BigQuery remote function given the artifacts of a user defined function and the http endpoint of a corresponding cloud function.""" - - if maybe_reuse: - existing_rf_spec = self.get_remote_function_specs(routine_ref) - if existing_rf_spec and existing_rf_spec == udf_def: - logger.info(f"Remote function {str(routine_ref)} already exists.") - return - - if try_create_connection: - self._create_bq_connection(udf_def.connection_id, routine_ref.project) + self._create_bq_connection() # Create BQ function # https://cloud.google.com/bigquery/docs/reference/standard-sql/remote-functions#create_a_remote_function_2 + bq_function_args = [] + bq_function_return_type = output_type + + # We are expecting the input type annotations to be 1:1 with the input args + for name, type_ in zip(input_args, input_types): + bq_function_args.append(f"{name} {type_}") remote_function_options = { - "endpoint": udf_def.endpoint, - "max_batching_rows": udf_def.max_batching_rows, + "endpoint": endpoint, + "max_batching_rows": max_batching_rows, } - if udf_def.bq_metadata: + if metadata: # We are using the description field to store this structured # bigframes specific metadata for the lack of a better option - remote_function_options["description"] = udf_def.bq_metadata + remote_function_options["description"] = metadata remote_function_options_str = self._format_function_options( remote_function_options ) - import bigframes.core.sql - import bigframes.core.utils - - # removes anything that isn't letter, number or underscore - _validate_routine_name(routine_ref.routine_id) - bq_function_name_escaped = bigframes.core.sql.identifier(routine_ref.routine_id) create_function_ddl = f""" - CREATE OR REPLACE FUNCTION `{routine_ref.project}.{routine_ref.dataset_id}`.{bq_function_name_escaped}({udf_def.signature.to_sql_input_signature()}) - RETURNS {udf_def.signature.with_devirtualize().output.sql_type} - REMOTE WITH CONNECTION `{routine_ref.project}.{self._bq_location}.{udf_def.connection_id}` + CREATE OR REPLACE FUNCTION `{self._gcp_project_id}.{self._bq_dataset}`.{bq_function_name}({','.join(bq_function_args)}) + RETURNS {bq_function_return_type} + REMOTE WITH CONNECTION `{self._gcp_project_id}.{self._bq_location}.{self._bq_connection_id}` OPTIONS ({remote_function_options_str})""" logger.info(f"Creating BQ remote function: {create_function_ddl}") - self._ensure_dataset_exists( - bigquery.DatasetReference(routine_ref.project, routine_ref.dataset_id) - ) + self._ensure_dataset_exists() self._create_bq_function(create_function_ddl) def provision_bq_managed_function( self, - routine_ref: bigquery.RoutineReference, - config: udf_def.ManagedFunctionConfig, + func, + input_types: Sequence[str], + output_type: str, + name: Optional[str], + packages: Optional[Sequence[str]], + max_batching_rows: Optional[int], + container_cpu: Optional[float], + container_memory: Optional[str], + is_row_processor: bool, + bq_connection_id, + *, + capture_references: bool = False, ): """Create a BigQuery managed function.""" # TODO(b/406283812): Expose the capability to pass down # capture_references=True in the public udf API. if ( - config.capture_references + capture_references and (python_version := _utils.get_python_version()) != _MANAGED_FUNC_PYTHON_VERSION ): @@ -218,27 +236,29 @@ def provision_bq_managed_function( ) # Create BQ managed function. - bq_function_args = config.signature.to_sql_input_signature() - bq_function_return_type = config.signature.with_devirtualize().output.sql_type + bq_function_args = [] + bq_function_return_type = output_type + + input_args = inspect.getargs(func.__code__).args + # We expect the input type annotations to be 1:1 with the input args. + for name_, type_ in zip(input_args, input_types): + bq_function_args.append(f"{name_} {type_}") managed_function_options: dict[str, Any] = { "runtime_version": _MANAGED_FUNC_PYTHON_VERSION, "entry_point": "bigframes_handler", } - if config.max_batching_rows: - managed_function_options["max_batching_rows"] = config.max_batching_rows - if config.container_cpu: - managed_function_options["container_cpu"] = config.container_cpu - if config.container_memory: - managed_function_options["container_memory"] = config.container_memory + if max_batching_rows: + managed_function_options["max_batching_rows"] = max_batching_rows + if container_cpu: + managed_function_options["container_cpu"] = container_cpu + if container_memory: + managed_function_options["container_memory"] = container_memory # Augment user package requirements with any internal package # requirements. packages = _utils.get_updated_package_requirements( - config.code.package_requirements or [], - config.signature.is_row_processor, - config.capture_references, - ignore_package_version=True, + packages, is_row_processor, capture_references, ignore_package_version=True ) if packages: managed_function_options["packages"] = packages @@ -246,15 +266,27 @@ def provision_bq_managed_function( managed_function_options ) + session_id = None if name else self._session.session_id + bq_function_name = name + if not bq_function_name: + # Compute a unique hash representing the user code. + function_hash = _utils.get_hash(func, packages) + bq_function_name = _utils.get_bigframes_function_name( + function_hash, + session_id, + ) + persistent_func_id = ( - f"`{routine_ref.project}.{routine_ref.dataset_id}.{routine_ref.routine_id}`" + f"`{self._gcp_project_id}.{self._bq_dataset}`.{bq_function_name}" ) + udf_name = func.__name__ + with_connection_clause = ( ( - f"WITH CONNECTION `{routine_ref.project}.{self._bq_location}.{config.bq_connection_id}`" + f"WITH CONNECTION `{self._gcp_project_id}.{self._bq_location}.{self._bq_connection_id}`" ) - if config.bq_connection_id + if bq_connection_id else "" ) @@ -262,13 +294,13 @@ def provision_bq_managed_function( # including the user's function, necessary imports, and the BigQuery # handler wrapper. python_code_block = bff_template.generate_managed_function_code( - config.code, config.signature, config.capture_references + func, udf_name, is_row_processor, capture_references ) create_function_ddl = ( textwrap.dedent( f""" - CREATE OR REPLACE FUNCTION {persistent_func_id}({bq_function_args}) + CREATE OR REPLACE FUNCTION {persistent_func_id}({','.join(bq_function_args)}) RETURNS {bq_function_return_type} LANGUAGE python {with_connection_clause} @@ -282,24 +314,28 @@ def provision_bq_managed_function( .replace("__UDF_PLACE_HOLDER__", python_code_block) ) - self._ensure_dataset_exists( - bigquery.DatasetReference(routine_ref.project, routine_ref.dataset_id) - ) + self._ensure_dataset_exists() self._create_bq_function(create_function_ddl) + return bq_function_name + def get_cloud_function_fully_qualified_parent(self): "Get the fully qualilfied parent for a cloud function." return self._cloud_functions_client.common_location_path( - self._gcp_project_id, self._cf_location + self._gcp_project_id, self._cloud_function_region ) def get_cloud_function_fully_qualified_name(self, name): "Get the fully qualilfied name for a cloud function." return self._cloud_functions_client.function_path( - self._gcp_project_id, self._cf_location, name + self._gcp_project_id, self._cloud_function_region, name ) - def get_cloud_function_endpoint(self, name) -> str | None: + def get_remote_function_fully_qualilfied_name(self, name): + "Get the fully qualilfied name for a BQ remote function." + return f"{self._gcp_project_id}.{self._bq_dataset}.{name}" + + def get_cloud_function_endpoint(self, name): """Get the http endpoint of a cloud function if it exists.""" fully_qualified_name = self.get_cloud_function_fully_qualified_name(name) try: @@ -311,26 +347,31 @@ def get_cloud_function_endpoint(self, name) -> str | None: pass return None - def _generate_cloud_function_code( + def generate_cloud_function_code( self, - code_def: udf_def.CodeDef, + def_, directory, *, - udf_signature: udf_def.UdfSignature, + input_types: Tuple[str], + output_type: str, + package_requirements=None, + is_row_processor=False, ): """Generate the cloud function code for a given user defined function.""" # requirements.txt - if code_def.package_requirements: + if package_requirements: requirements_txt = os.path.join(directory, "requirements.txt") with open(requirements_txt, "w") as f: - f.write("\n".join(code_def.package_requirements)) + f.write("\n".join(package_requirements)) # main.py entry_point = bff_template.generate_cloud_function_main_code( - code_def, + def_, directory, - udf_signature=udf_signature, + input_types=input_types, + output_type=output_type, + is_row_processor=is_row_processor, ) return entry_point @@ -350,30 +391,33 @@ def _get_cloud_function_endpoint_with_retry(self, name): def create_cloud_function( self, - name: str, - func_def: udf_def.CloudRunFunctionConfig, - ) -> str: + def_, + *, + random_name, + input_types: Tuple[str], + output_type: str, + package_requirements=None, + timeout_seconds=600, + max_instance_count=None, + is_row_processor=False, + vpc_connector=None, + vpc_connector_egress_settings="private-ranges-only", + memory_mib=1024, + ingress_settings="internal-only", + ): """Create a cloud function from the given user defined function.""" - config = func_def - # Build and deploy folder structure containing cloud function - with tempfile.TemporaryDirectory() as scratch_dir: - # Keep the generated sources in a subdirectory so the archive can be - # written inside the 0700 TemporaryDirectory. shutil.make_archive - # appends ".zip" to base_name, so archiving `directory` into itself - # would leave a world-readable copy of the (pickled) user code as a - # sibling of the temp dir that also survives the cleanup. - directory = os.path.join(scratch_dir, "src") - os.mkdir(directory) - entry_point = self._generate_cloud_function_code( - config.code, + with tempfile.TemporaryDirectory() as directory: + entry_point = self.generate_cloud_function_code( + def_, directory, - udf_signature=config.signature, - ) - archive_path = shutil.make_archive( - os.path.join(scratch_dir, "source"), "zip", directory + package_requirements=package_requirements, + input_types=input_types, + output_type=output_type, + is_row_processor=is_row_processor, ) + archive_path = shutil.make_archive(directory, "zip", directory) # We are creating cloud function source code from the currently running # python version. Use the same version to deploy. This is necessary @@ -386,7 +430,7 @@ def create_cloud_function( # Determine an upload URL for user code upload_url_request = functions_v2.GenerateUploadUrlRequest( - kms_key_name=config.kms_key_name + kms_key_name=self._cloud_function_kms_key_name ) upload_url_request.parent = self.get_cloud_function_fully_qualified_parent() upload_url_response = self._cloud_functions_client.generate_upload_url( @@ -411,9 +455,9 @@ def create_cloud_function( create_function_request.parent = ( self.get_cloud_function_fully_qualified_parent() ) - create_function_request.function_id = name + create_function_request.function_id = random_name function = functions_v2.Function() - function.name = self.get_cloud_function_fully_qualified_name(name) + function.name = self.get_cloud_function_fully_qualified_name(random_name) function.build_config = functions_v2.BuildConfig() function.build_config.runtime = python_version function.build_config.entry_point = entry_point @@ -425,84 +469,64 @@ def create_cloud_function( function.build_config.source.storage_source.object_ = ( upload_url_response.storage_source.object_ ) - if config.docker_repository is not None: - function.build_config.docker_repository = config.docker_repository + function.build_config.docker_repository = ( + self._cloud_function_docker_repository + ) - if config.cloud_build_service_account is not None: + if self._cloud_build_service_account: canonical_cloud_build_service_account = ( - config.cloud_build_service_account - if "/" in config.cloud_build_service_account - else f"projects/{self._gcp_project_id}/serviceAccounts/{config.cloud_build_service_account}" + self._cloud_build_service_account + if "/" in self._cloud_build_service_account + else f"projects/{self._gcp_project_id}/serviceAccounts/{self._cloud_build_service_account}" ) function.build_config.service_account = ( canonical_cloud_build_service_account ) function.service_config = functions_v2.ServiceConfig() - if config.memory_mib is not None: - function.service_config.available_memory = f"{config.memory_mib}Mi" - if config.cpus is not None: - function.service_config.available_cpu = str(config.cpus) - if config.timeout_seconds is not None: - if config.timeout_seconds > 1200: + if memory_mib is not None: + function.service_config.available_memory = f"{memory_mib}Mi" + if timeout_seconds is not None: + if timeout_seconds > 1200: raise bf_formatting.create_exception_with_feedback_link( ValueError, "BigQuery remote function can wait only up to 20 minutes" ", see for more details " "https://cloud.google.com/bigquery/quotas#remote_function_limits.", ) - function.service_config.timeout_seconds = config.timeout_seconds - if config.max_instance_count is not None: - function.service_config.max_instance_count = config.max_instance_count - if config.vpc_connector is not None: - function.service_config.vpc_connector = config.vpc_connector - vpc_connector_egress_settings = config.vpc_connector_egress_settings - if config.vpc_connector_egress_settings is None: + function.service_config.timeout_seconds = timeout_seconds + if max_instance_count is not None: + function.service_config.max_instance_count = max_instance_count + if vpc_connector is not None: + function.service_config.vpc_connector = vpc_connector + if vpc_connector_egress_settings is None: msg = bfe.format_message( "The 'vpc_connector_egress_settings' was not specified. Defaulting to 'private-ranges-only'.", ) warnings.warn(msg, category=UserWarning) vpc_connector_egress_settings = "private-ranges-only" - if config.vpc_connector_egress_settings not in _VPC_EGRESS_SETTINGS_MAP: + if vpc_connector_egress_settings not in _VPC_EGRESS_SETTINGS_MAP: raise bf_formatting.create_exception_with_feedback_link( ValueError, - f"'{config.vpc_connector_egress_settings}' is not one of the supported vpc egress settings values: {list(_VPC_EGRESS_SETTINGS_MAP)}", + f"'{vpc_connector_egress_settings}' is not one of the supported vpc egress settings values: {list(_VPC_EGRESS_SETTINGS_MAP)}", ) function.service_config.vpc_connector_egress_settings = cast( functions_v2.ServiceConfig.VpcConnectorEgressSettings, _VPC_EGRESS_SETTINGS_MAP[vpc_connector_egress_settings], ) - if config.cloud_run_service_account: - function.service_config.service_account_email = ( - config.cloud_run_service_account - ) - if config.concurrency: - function.service_config.max_instance_request_concurrency = ( - config.concurrency - ) - - # Functions framework use environment variables to pass config to gunicorn - # See https://github.com/GoogleCloudPlatform/functions-framework-python/issues/241 - # Code: https://github.com/GoogleCloudPlatform/functions-framework-python/blob/v3.10.1/src/functions_framework/_http/gunicorn.py#L37-L43 - env_vars = {} - if config.workers: - env_vars["WORKERS"] = str(config.workers) - if config.threads: - env_vars["THREADS"] = str(config.threads) - if env_vars: - function.service_config.environment_variables = env_vars - - if config.ingress_settings not in _INGRESS_SETTINGS_MAP: + function.service_config.service_account_email = ( + self._cloud_function_service_account + ) + if ingress_settings not in _INGRESS_SETTINGS_MAP: raise bf_formatting.create_exception_with_feedback_link( ValueError, - f"'{config.ingress_settings}' not one of the supported ingress settings values: {list(_INGRESS_SETTINGS_MAP)}", + f"'{ingress_settings}' not one of the supported ingress settings values: {list(_INGRESS_SETTINGS_MAP)}", ) function.service_config.ingress_settings = cast( functions_v2.ServiceConfig.IngressSettings, - _INGRESS_SETTINGS_MAP[config.ingress_settings], + _INGRESS_SETTINGS_MAP[ingress_settings], ) - if config.kms_key_name: - function.kms_key_name = config.kms_key_name + function.kms_key_name = self._cloud_function_kms_key_name create_function_request.function = function # Create the cloud function and wait for it to be ready to use @@ -531,52 +555,144 @@ def create_cloud_function( # Fetch the endpoint with retries if it wasn't returned by the operation if not endpoint: try: - endpoint = self._get_cloud_function_endpoint_with_retry(name) + endpoint = self._get_cloud_function_endpoint_with_retry(random_name) except Exception as e: raise bf_formatting.create_exception_with_feedback_link( ValueError, f"Couldn't fetch the http endpoint: {e}" ) - logger.info(f"Successfully created cloud function {name} with uri ({endpoint})") + logger.info( + f"Successfully created cloud function {random_name} with uri ({endpoint})" + ) return endpoint - def get_remote_function_specs( - self, remote_function_name: bigquery.RoutineReference - ) -> udf_def.RemoteFunctionConfig | None: - """Check whether a remote function already exists for the udf.""" - try: - routine = self._bq_client.get_routine(str(remote_function_name)) - if routine.reference == remote_function_name: - try: - return udf_def.RemoteFunctionConfig.from_bq_routine(routine) - except udf_def.ReturnTypeMissingError: - # The remote function exists, but it's missing a return type. - # Something is wrong with the function, so we should replace it. - return None - except google.api_core.exceptions.NotFound: - # The dataset might not exist, in which case the remote function doesn't, either. - # Note: list_routines doesn't make an API request until we iterate on the response object. - pass - return None + def provision_bq_remote_function( + self, + def_, + input_types, + output_type, + reuse, + name, + package_requirements, + max_batching_rows, + cloud_function_timeout, + cloud_function_max_instance_count, + is_row_processor, + cloud_function_vpc_connector, + cloud_function_vpc_connector_egress_settings, + cloud_function_memory_mib, + cloud_function_ingress_settings, + bq_metadata, + ): + """Provision a BigQuery remote function.""" + # Augment user package requirements with any internal package + # requirements + package_requirements = _utils.get_updated_package_requirements( + package_requirements, is_row_processor + ) - def delete_routine(self, routine_name: bigquery.RoutineReference) -> None: - self._bq_client.delete_routine(str(routine_name), not_found_ok=True) + # Compute a unique hash representing the user code + function_hash = _utils.get_hash(def_, package_requirements) + + # If reuse of any existing function with the same name (indicated by the + # same hash of its source code) is not intended, then attach a unique + # suffix to the intended function name to make it unique. + uniq_suffix = None + if not reuse: + # use 4 digits as a unique suffix which should suffice for + # uniqueness per session + uniq_suffix = "".join( + random.choices(string.ascii_lowercase + string.digits, k=4) + ) - def delete_cloud_function(self, cloud_function_name: str) -> None: - try: - self._cloud_functions_client.delete_function( - name=self.get_cloud_function_fully_qualified_name(cloud_function_name) + # Derive the name of the cloud function underlying the intended BQ + # remote function. Use the session id to identify the GCF for unnamed + # functions. The named remote functions are treated as a persistant + # artifacts, so let's keep them independent of session id, which also + # makes their naming more stable for the same udf code + session_id = None if name else self._session.session_id + cloud_function_name = _utils.get_cloud_function_name( + function_hash, session_id, uniq_suffix + ) + cf_endpoint = self.get_cloud_function_endpoint(cloud_function_name) + + # Create the cloud function if it does not exist + if not cf_endpoint: + cf_endpoint = self.create_cloud_function( + def_, + random_name=cloud_function_name, + input_types=input_types, + output_type=output_type, + package_requirements=package_requirements, + timeout_seconds=cloud_function_timeout, + max_instance_count=cloud_function_max_instance_count, + is_row_processor=is_row_processor, + vpc_connector=cloud_function_vpc_connector, + vpc_connector_egress_settings=cloud_function_vpc_connector_egress_settings, + memory_mib=cloud_function_memory_mib, + ingress_settings=cloud_function_ingress_settings, + ) + else: + logger.info(f"Cloud function {cloud_function_name} already exists.") + + # Derive the name of the remote function + remote_function_name = name + if not remote_function_name: + remote_function_name = _utils.get_bigframes_function_name( + function_hash, self._session.session_id, uniq_suffix + ) + rf_endpoint, rf_conn = self.get_remote_function_specs(remote_function_name) + + # Create the BQ remote function in following circumstances: + # 1. It does not exist + # 2. It exists but the existing remote function has different + # configuration than intended + created_new = False + if not rf_endpoint or ( + rf_endpoint != cf_endpoint or rf_conn != self._bq_connection_id + ): + input_args = inspect.getargs(def_.__code__).args + if len(input_args) != len(input_types): + raise bf_formatting.create_exception_with_feedback_link( + ValueError, + "Exactly one type should be provided for every input arg.", + ) + self.create_bq_remote_function( + input_args, + input_types, + output_type, + cf_endpoint, + remote_function_name, + max_batching_rows, + bq_metadata, ) - except google.api_core.exceptions.NotFound: - # The dataset might not exist, in which case the remote function doesn't, either. - pass + created_new = True + else: + logger.info(f"Remote function {remote_function_name} already exists.") -def _validate_routine_name(name: str) -> None: - """Validate that the given name is a valid BigQuery routine name.""" - # Routine IDs can contain only letters (a-z, A-Z), numbers (0-9), or underscores (_) - # must also start with a letter or underscore only - if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", name): - raise ValueError( - "Routine ID can contain only letters (a-z, A-Z), numbers (0-9), or underscores (_)" + return remote_function_name, cloud_function_name, created_new + + def get_remote_function_specs(self, remote_function_name): + """Check whether a remote function already exists for the udf.""" + http_endpoint = None + bq_connection = None + routines = self._bq_client.list_routines( + f"{self._gcp_project_id}.{self._bq_dataset}" ) + try: + for routine in routines: + routine = cast(bigquery.Routine, routine) + if routine.reference.routine_id == remote_function_name: + rf_options = routine.remote_function_options + if rf_options: + http_endpoint = rf_options.endpoint + bq_connection = rf_options.connection + if bq_connection: + bq_connection = os.path.basename(bq_connection) + break + except google.api_core.exceptions.NotFound: + # The dataset might not exist, in which case the http_endpoint doesn't, either. + # Note: list_routines doesn't make an API request until we iterate on the response object. + pass + return (http_endpoint, bq_connection) diff --git a/bigframes/functions/_function_session.py b/bigframes/functions/_function_session.py index 2bc2b597372..a456f054170 100644 --- a/bigframes/functions/_function_session.py +++ b/bigframes/functions/_function_session.py @@ -15,123 +15,159 @@ from __future__ import annotations +import collections.abc import functools -import logging -import random -import string +import inspect +import sys import threading -import time -import warnings from typing import ( - TYPE_CHECKING, + Any, + cast, + Dict, + get_origin, Literal, + Mapping, Optional, Sequence, + TYPE_CHECKING, Union, ) +import warnings +import google.api_core.exceptions from google.cloud import ( bigquery, + bigquery_connection_v1, + functions_v2, + resourcemanager_v3, ) +from bigframes import clients import bigframes.exceptions as bfe import bigframes.formatting_helpers as bf_formatting -from bigframes import clients -from bigframes.functions import _function_client, _utils, udf_def from bigframes.functions import function as bq_functions -from bigframes.functions._utils import ( - _BIGFRAMES_FUNCTION_PREFIX, - _BQ_FUNCTION_NAME_SEPERATOR, - _GCF_FUNCTION_NAME_SEPERATOR, -) +from bigframes.functions import udf_def if TYPE_CHECKING: - from bigframes.session import anonymous_dataset - - -_DEFAULT_FUNCTION_MEMORY_MIB = 1024 + from bigframes.session import Session +import pandas -logger = logging.getLogger(__name__) +from bigframes.functions import _function_client, _utils class FunctionSession: """Session to manage bigframes functions.""" - def __init__( - self, - functions_client: _function_client.FunctionClient, - dataset_manager: anonymous_dataset.AnonymousDatasetManager, - default_connection: str, - location: str, - session_id: str, - manage_connections: bool, - ): - self._temp_cloud_functions: set[str] = set() - self._temp_remote_functions: set[bigquery.RoutineReference] = set() + def __init__(self): + # Session level mapping of function artifacts + self._temp_artifacts: Dict[str, str] = dict() # Lock to synchronize the update of the session artifacts self._artifacts_lock = threading.Lock() - self._deployed_routines: set[bytes] = set() - self._deploying_routines: set[bytes] = set() - - self._function_client: _function_client.FunctionClient = functions_client - self._dataset_manager: anonymous_dataset.AnonymousDatasetManager = ( - dataset_manager - ) - self._default_connection: str = default_connection - self._location: str = location - self._session_id: str = session_id - self._manage_connections: bool = manage_connections + def _resolve_session(self, session: Optional[Session]) -> Session: + """Resolves the BigFrames session.""" + import bigframes.pandas as bpd + import bigframes.session + + # Using the global session if none is provided. + return cast(bigframes.session.Session, session or bpd.get_global_session()) + + def _resolve_bigquery_client( + self, session: Session, bigquery_client: Optional[bigquery.Client] + ) -> bigquery.Client: + """Resolves the BigQuery client.""" + if not bigquery_client: + bigquery_client = session.bqclient + if not bigquery_client: + raise bf_formatting.create_exception_with_feedback_link( + ValueError, + "A bigquery client must be provided, either directly or via " + "session.", + ) + return bigquery_client - @property - def session_id(self) -> str: - return self._session_id + def _resolve_bigquery_connection_client( + self, + session: Session, + bigquery_connection_client: Optional[ + bigquery_connection_v1.ConnectionServiceClient + ], + ) -> bigquery_connection_v1.ConnectionServiceClient: + """Resolves the BigQuery connection client.""" + if not bigquery_connection_client: + bigquery_connection_client = session.bqconnectionclient + if not bigquery_connection_client: + raise bf_formatting.create_exception_with_feedback_link( + ValueError, + "A bigquery connection client must be provided, either " + "directly or via session.", + ) + return bigquery_connection_client - @property - def default_dataset(self) -> bigquery.DatasetReference: - # We defer this as a property since this can actually take a query to determine - # which dataset it is. - return self._dataset_manager.dataset + def _resolve_resource_manager_client( + self, + session: Session, + resource_manager_client: Optional[resourcemanager_v3.ProjectsClient], + ) -> resourcemanager_v3.ProjectsClient: + """Resolves the resource manager client.""" + if not resource_manager_client: + resource_manager_client = session.resourcemanagerclient + if not resource_manager_client: + raise bf_formatting.create_exception_with_feedback_link( + ValueError, + "A resource manager client must be provided, either directly " + "or via session.", + ) + return resource_manager_client def _resolve_dataset_reference( self, + session: Session, + bigquery_client: bigquery.Client, dataset: Optional[str], ) -> bigquery.DatasetReference: - """ - Resolves the dataset reference for the bigframes function. - """ - return ( - bigquery.DatasetReference.from_string( - dataset, default_project=self.default_dataset.project + """Resolves the dataset reference for the bigframes function.""" + if dataset: + dataset_ref = bigquery.DatasetReference.from_string( + dataset, default_project=bigquery_client.project ) - if dataset - else self.default_dataset - ) + else: + dataset_ref = session._anonymous_dataset + return dataset_ref - def _resolve_routine_reference( + def _resolve_cloud_functions_client( self, - function_name: str, - dataset: Optional[bigquery.DatasetReference] = None, - ) -> bigquery.RoutineReference: - """Resolves the routine reference for a BQ routine.""" - dataset_ref = dataset if dataset else self.default_dataset - return dataset_ref.routine(function_name) + session: Session, + cloud_functions_client: Optional[functions_v2.FunctionServiceClient], + ) -> Optional[functions_v2.FunctionServiceClient]: + """Resolves the Cloud Functions client.""" + if not cloud_functions_client: + cloud_functions_client = session.cloudfunctionsclient + if not cloud_functions_client: + raise bf_formatting.create_exception_with_feedback_link( + ValueError, + "A cloud functions client must be provided, either directly " + "or via session.", + ) + return cloud_functions_client def _resolve_bigquery_connection_id( self, + session: Session, dataset_ref: bigquery.DatasetReference, + bq_location: str, bigquery_connection: Optional[str] = None, ) -> str: """Resolves BigQuery connection id.""" if not bigquery_connection: - bigquery_connection = self._default_connection + bigquery_connection = session._bq_connection # type: ignore bigquery_connection = clients.get_canonical_bq_connection_id( bigquery_connection, default_project=dataset_ref.project, - default_location=self._location, + default_location=bq_location, ) # Guaranteed to be the form of .. ( @@ -145,90 +181,41 @@ def _resolve_bigquery_connection_id( "The project_id does not match BigQuery connection " f"gcp_project_id: {dataset_ref.project}.", ) - if bq_connection_location.casefold() != self._location.casefold(): + if bq_connection_location.casefold() != bq_location.casefold(): raise bf_formatting.create_exception_with_feedback_link( ValueError, "The location does not match BigQuery connection location: " - f"{self._location}.", + f"{bq_location}.", ) return bq_connection_id - def _add_temp_cloud_function(self, gcf_path: str): - with self._artifacts_lock: - self._temp_cloud_functions.add(gcf_path) - - def _add_temp_remote_function(self, bqrf_routine: bigquery.RoutineReference): + def _update_temp_artifacts(self, bqrf_routine: str, gcf_path: str): + """Update function artifacts in the current session.""" with self._artifacts_lock: - self._temp_remote_functions.add(bqrf_routine) + self._temp_artifacts[bqrf_routine] = gcf_path - def _deploy_managed_function( + def clean_up( self, - config: udf_def.ManagedFunctionConfig, - name: str, - temp: bool, - dataset: Optional[bigquery.DatasetReference] = None, - ) -> udf_def.BigqueryUdf: - routine_ref = self._resolve_routine_reference(name, dataset=dataset) - if temp: - self._add_temp_remote_function(routine_ref) - self._function_client.provision_bq_managed_function( - routine_ref=routine_ref, config=config - ) - return udf_def.BigqueryUdf( - routine_ref=routine_ref, - signature=config.signature, - ) - - def _deploy_udf( - self, - bq_udf: udf_def.PythonUdf, - ) -> udf_def.BigqueryUdf: - """Deploys a UDF to BigQuery if not already deployed.""" - udf_hash = bq_udf.stable_hash() - - config = bq_udf.to_managed_function_config() - bq_function_name = get_managed_function_name(config, self.session_id) - routine_ref = self._resolve_routine_reference(bq_function_name) - while True: - with self._artifacts_lock: - if udf_hash in self._deployed_routines: - return udf_def.BigqueryUdf( - routine_ref=routine_ref, - signature=bq_udf.signature, - ) - - if udf_hash not in self._deploying_routines: - self._deploying_routines.add(udf_hash) - break - - time.sleep(0.1) - try: - self._function_client.provision_bq_managed_function( - routine_ref=routine_ref, config=config - ) - except Exception: - with self._artifacts_lock: - self._deploying_routines.discard(udf_hash) - raise - self._add_temp_remote_function(routine_ref) - with self._artifacts_lock: - self._deploying_routines.discard(udf_hash) - self._deployed_routines.add(udf_hash) - return udf_def.BigqueryUdf( - routine_ref=routine_ref, - signature=bq_udf.signature, - ) - - def clean_up(self): + bqclient: bigquery.Client, + gcfclient: functions_v2.FunctionServiceClient, + session_id: str, + ): """Delete function artifacts in the current session.""" with self._artifacts_lock: - for bqrf_routine in self._temp_remote_functions: - self._function_client.delete_routine(bqrf_routine) - for gcf_name in self._temp_cloud_functions: - self._function_client.delete_cloud_function(gcf_name) + for bqrf_routine, gcf_path in self._temp_artifacts.items(): + # Let's accept the possibility that the function may have been + # deleted directly by the user + bqclient.delete_routine(bqrf_routine, not_found_ok=True) + + if gcf_path: + # Let's accept the possibility that the cloud function may + # have been deleted directly by the user + try: + gcfclient.delete_function(name=gcf_path) + except google.api_core.exceptions.NotFound: + pass - self._temp_remote_functions.clear() - self._temp_cloud_functions.clear() + self._temp_artifacts.clear() # Inspired by @udf decorator implemented in ibis-bigquery package # https://github.com/ibis-project/ibis-bigquery/blob/main/ibis_bigquery/udf/__init__.py @@ -239,6 +226,13 @@ def remote_function( *, input_types: Union[None, type, Sequence[type]] = None, output_type: Optional[type] = None, + session: Optional[Session] = None, + bigquery_client: Optional[bigquery.Client] = None, + bigquery_connection_client: Optional[ + bigquery_connection_v1.ConnectionServiceClient + ] = None, + cloud_functions_client: Optional[functions_v2.FunctionServiceClient] = None, + resource_manager_client: Optional[resourcemanager_v3.ProjectsClient] = None, dataset: Optional[str] = None, bigquery_connection: Optional[str] = None, reuse: bool = True, @@ -247,15 +241,14 @@ def remote_function( cloud_function_service_account: str, cloud_function_kms_key_name: Optional[str] = None, cloud_function_docker_repository: Optional[str] = None, - max_batching_rows: Optional[int] = None, + max_batching_rows: Optional[int] = 1000, cloud_function_timeout: Optional[int] = 600, cloud_function_max_instances: Optional[int] = None, cloud_function_vpc_connector: Optional[str] = None, cloud_function_vpc_connector_egress_settings: Optional[ Literal["all", "private-ranges-only", "unspecified"] ] = None, - cloud_function_memory_mib: Optional[int] = None, - cloud_function_cpus: Optional[float] = None, + cloud_function_memory_mib: Optional[int] = 1024, cloud_function_ingress_settings: Literal[ "all", "internal-only", "internal-and-gclb" ] = "internal-only", @@ -324,6 +317,24 @@ def remote_function( be specified. The supported output types are `bool`, `bytes`, `float`, `int`, `str`, `list[bool]`, `list[float]`, `list[int]` and `list[str]`. + session (bigframes.Session, Optional): + BigQuery DataFrames session to use for getting default project, + dataset and BigQuery connection. + bigquery_client (google.cloud.bigquery.Client, Optional): + Client to use for BigQuery operations. If this param is not provided + then bigquery client from the session would be used. + bigquery_connection_client (google.cloud.bigquery_connection_v1.ConnectionServiceClient, Optional): + Client to use for BigQuery connection operations. If this param is + not provided then bigquery connection client from the session would + be used. + cloud_functions_client (google.cloud.functions_v2.FunctionServiceClient, Optional): + Client to use for cloud functions operations. If this param is not + provided then the functions client from the session would be used. + resource_manager_client (google.cloud.resourcemanager_v3.ProjectsClient, Optional): + Client to use for cloud resource management operations, e.g. for + getting and setting IAM roles on cloud resources. If this param is + not provided then resource manager client from the session would be + used. dataset (str, Optional): Dataset in which to create a BigQuery remote function. It should be in `.` or `` format. If this @@ -433,10 +444,6 @@ def remote_function( default memory of cloud functions be allocated, pass `None`. See for more details https://cloud.google.com/functions/docs/configuring/memory. - cloud_function_cpus (float, Optional): - The number of cpus to allocate for the cloud - function (2nd gen) created. - https://docs.cloud.google.com/run/docs/configuring/services/cpu. cloud_function_ingress_settings (str, Optional): Ingress settings controls dictating what traffic can reach the function. Options are: `all`, `internal-only`, or `internal-and-gclb`. @@ -454,6 +461,9 @@ def remote_function( https://cloud.google.com/build/docs/cloud-build-service-account for more details. """ + # Some defaults may be used from the session if not provided otherwise. + session = self._resolve_session(session) + # If the user forces the cloud function service argument to None, throw # an exception if cloud_function_service_account is None: @@ -461,13 +471,36 @@ def remote_function( 'You must provide a user managed cloud_function_service_account, or "default" if you would like to let the default service account be used.' ) + # A BigQuery client is required to perform BQ operations. + bigquery_client = self._resolve_bigquery_client(session, bigquery_client) + + # A BigQuery connection client is required for BQ connection operations. + bigquery_connection_client = self._resolve_bigquery_connection_client( + session, bigquery_connection_client + ) + + # A resource manager client is required to get/set IAM operations. + resource_manager_client = self._resolve_resource_manager_client( + session, resource_manager_client + ) + # BQ remote function must be persisted, for which we need a dataset. # https://cloud.google.com/bigquery/docs/reference/standard-sql/remote-functions#:~:text=You%20cannot%20create%20temporary%20remote%20functions. - dataset_ref = self._resolve_dataset_reference(dataset) + dataset_ref = self._resolve_dataset_reference(session, bigquery_client, dataset) + + # A cloud functions client is required for cloud functions operations. + cloud_functions_client = self._resolve_cloud_functions_client( + session, cloud_functions_client + ) + + bq_location, cloud_function_region = _utils.get_remote_function_locations( + bigquery_client.location + ) + # A connection is required for BQ remote function. # https://cloud.google.com/bigquery/docs/reference/standard-sql/remote-functions#create_a_remote_function bq_connection_id = self._resolve_bigquery_connection_id( - dataset_ref, bigquery_connection + session, dataset_ref, bq_location, bigquery_connection ) # If any CMEK is intended then check that a docker repository is also specified. @@ -498,138 +531,167 @@ def remote_function( ) warnings.warn(msg, category=UserWarning, stacklevel=2) + bq_connection_manager = session.bqconnectionmanager + def wrapper(func): nonlocal input_types, output_type - ### Step 1: Validate inputs and package into cloud run function, remote function defs. ### if not callable(func): raise bf_formatting.create_exception_with_feedback_link( TypeError, f"func must be a callable, got {func}" ) - udf_sig = _utils.get_func_signature( - func, - input_types, - output_type, - ).to_remote_function_compatible() + if sys.version_info >= (3, 10): + # Add `eval_str = True` so that deferred annotations are turned into their + # corresponding type objects. Need Python 3.10 for eval_str parameter. + # https://docs.python.org/3/library/inspect.html#inspect.signature + signature_kwargs: Mapping[str, Any] = {"eval_str": True} + else: + signature_kwargs = {} # type: ignore - full_package_requirements = _utils.get_updated_package_requirements( - packages or [], udf_sig.is_row_processor - ) - memory_mib = cloud_function_memory_mib or _DEFAULT_FUNCTION_MEMORY_MIB - - # assumption is most bigframes functions are cpu bound, single-threaded and many won't release GIL - # therefore, want to allocate a worker for each cpu, and allow a concurrent request per worker - expected_milli_cpus = ( - int(cloud_function_cpus * 1000) - if (cloud_function_cpus is not None) - else _infer_milli_cpus_from_memory(memory_mib) + py_sig = inspect.signature( + func, + **signature_kwargs, ) - workers = -( - expected_milli_cpus // -1000 - ) # ceil(cpus) without invoking floats - threads = 4 # (per worker) - # max concurrency==1 for vcpus < 1 hard limit from cloud run - concurrency = (workers * threads) if (expected_milli_cpus >= 1000) else 1 - - ### Step 1: Create resources or fetch existing matching resources. ### - cloud_func_spec = udf_def.CloudRunFunctionConfig( - code=udf_def.CodeDef.from_func(func, full_package_requirements), - signature=udf_sig, - timeout_seconds=cloud_function_timeout, - max_instance_count=cloud_function_max_instances, - vpc_connector=cloud_function_vpc_connector, - vpc_connector_egress_settings=cloud_function_vpc_connector_egress_settings - or "private-ranges-only", - memory_mib=memory_mib, - cpus=cloud_function_cpus, - ingress_settings=cloud_function_ingress_settings, - workers=workers, - threads=threads, - concurrency=concurrency, - kms_key_name=cloud_function_kms_key_name, - docker_repository=cloud_function_docker_repository, + if input_types is not None: + if not isinstance(input_types, collections.abc.Sequence): + input_types = [input_types] + if _utils.has_conflict_input_type(py_sig, input_types): + msg = bfe.format_message( + "Conflicting input types detected, using the one from the decorator." + ) + warnings.warn(msg, category=bfe.FunctionConflictTypeHintWarning) + py_sig = py_sig.replace( + parameters=[ + par.replace(annotation=itype) + for par, itype in zip(py_sig.parameters.values(), input_types) + ] + ) + if output_type: + if _utils.has_conflict_output_type(py_sig, output_type): + msg = bfe.format_message( + "Conflicting return type detected, using the one from the decorator." + ) + warnings.warn(msg, category=bfe.FunctionConflictTypeHintWarning) + py_sig = py_sig.replace(return_annotation=output_type) + + # The function will actually be receiving a pandas Series, but allow + # both BigQuery DataFrames and pandas object types for compatibility. + is_row_processor = False + if new_sig := _convert_row_processor_sig(py_sig): + py_sig = new_sig + is_row_processor = True + + remote_function_client = _function_client.FunctionClient( + dataset_ref.project, + bq_location, + dataset_ref.dataset_id, + bigquery_client, + bq_connection_id, + bq_connection_manager, + cloud_function_region, + cloud_functions_client, + None + if cloud_function_service_account == "default" + else cloud_function_service_account, + cloud_function_kms_key_name, + cloud_function_docker_repository, cloud_build_service_account=cloud_build_service_account, - cloud_run_service_account=( - None - if (cloud_function_service_account == "default") - else cloud_function_service_account - ), + session=session, # type: ignore ) - uniq_suffix = None - if not reuse: - uniq_suffix = "".join( - random.choices(string.ascii_lowercase + string.digits, k=4) + + # resolve the output type that can be supported in the bigframes, + # ibis, BQ remote functions and cloud functions integration. + bqrf_metadata = None + post_process_routine = None + if get_origin(py_sig.return_annotation) is list: + # TODO(b/284515241): remove this special handling to support + # array output types once BQ remote functions support ARRAY. + # Until then, use json serialized strings at the cloud function + # and BQ level, and parse that to the intended output type at + # the bigframes level. + bqrf_metadata = _utils.get_bigframes_metadata( + python_output_type=py_sig.return_annotation ) - cf_name = get_cloud_function_name( - cloud_func_spec, - # only session scope a temp unnamed function - session_id=self.session_id if (name is None) else None, - uniq_suffix=uniq_suffix, - ) - if not name: - self._add_temp_cloud_function(cf_name) - - # Create remote function that points at the cloud function - cf_endpoint = None - if reuse is not None: - cf_endpoint = self._function_client.get_cloud_function_endpoint(cf_name) - - # If the endpoint is empty, the function might exist but the URL propagation is pending. - # Running create_cloud_function will handle AlreadyExists and retry endpoint fetching. - if not cf_endpoint: - cf_endpoint = self._function_client.create_cloud_function( - cf_name, cloud_func_spec + post_process_routine = _utils.build_unnest_post_routine( + py_sig.return_annotation ) - else: - logger.info(f"Cloud function {cf_name} already exists.") + py_sig = py_sig.replace(return_annotation=str) - remote_function_config = udf_def.RemoteFunctionConfig( - endpoint=cf_endpoint, - connection_id=bq_connection_id, - max_batching_rows=max_batching_rows or 1000, - signature=udf_sig, - bq_metadata=udf_sig.protocol_metadata, + udf_sig = udf_def.UdfSignature.from_py_signature(py_sig) + + ( + rf_name, + cf_name, + created_new, + ) = remote_function_client.provision_bq_remote_function( + func, + input_types=udf_sig.sql_input_types, + output_type=udf_sig.sql_output_type, + reuse=reuse, + name=name, + package_requirements=packages, + max_batching_rows=max_batching_rows, + cloud_function_timeout=cloud_function_timeout, + cloud_function_max_instance_count=cloud_function_max_instances, + is_row_processor=is_row_processor, + cloud_function_vpc_connector=cloud_function_vpc_connector, + cloud_function_vpc_connector_egress_settings=cloud_function_vpc_connector_egress_settings, + cloud_function_memory_mib=cloud_function_memory_mib, + cloud_function_ingress_settings=cloud_function_ingress_settings, + bq_metadata=bqrf_metadata, ) - remote_function_name = name or get_bigframes_function_name( - remote_function_config, - session_id=self.session_id, - uniq_suffix=uniq_suffix, + + bigframes_cloud_function = ( + remote_function_client.get_cloud_function_fully_qualified_name(cf_name) ) - routine_ref = self._resolve_routine_reference( - remote_function_name, dataset=dataset_ref + bigframes_bigquery_function = ( + remote_function_client.get_remote_function_fully_qualilfied_name( + rf_name + ) ) - if not name: - self._add_temp_remote_function(routine_ref) - self._function_client.create_bq_remote_function( - udf_def=remote_function_config, - routine_ref=routine_ref, - maybe_reuse=reuse, - try_create_connection=self._manage_connections, - ) + # If a new remote function was created, update the cloud artifacts + # created in the session. This would be used to clean up any + # resources in the session. Note that we need to do this only for + # the case where an explicit name was not provided by the user and + # we used an internal name. For the cases where the user provided an + # explicit name, we are assuming that the user wants to persist them + # with that name and would directly manage their lifecycle. + if created_new and (not name): + self._update_temp_artifacts( + bigframes_bigquery_function, bigframes_cloud_function + ) udf_definition = udf_def.BigqueryUdf( - routine_ref=routine_ref, + routine_ref=bigquery.RoutineReference.from_string( + bigframes_bigquery_function + ), signature=udf_sig, ) decorator = functools.wraps(func) - if udf_sig.is_row_processor: - msg = bfe.format_message("input_types=Series is in preview.") - warnings.warn(msg, stacklevel=1, category=bfe.PreviewWarning) - - cf_full_path = ( - self._function_client.get_cloud_function_fully_qualified_name(cf_name) - ) - return decorator( - bq_functions.BigqueryCallableRoutine( - udf_definition, - self._function_client._bq_client, - cloud_function_ref=cf_full_path, - local_func=func, - is_managed=False, + if is_row_processor: + return decorator( + bq_functions.BigqueryCallableRowRoutine( + udf_definition, + session, + post_routine=post_process_routine, + cloud_function_ref=bigframes_cloud_function, + local_func=func, + is_managed=False, + ) + ) + else: + return decorator( + bq_functions.BigqueryCallableRoutine( + udf_definition, + session, + post_routine=post_process_routine, + cloud_function_ref=bigframes_cloud_function, + local_func=func, + is_managed=False, + ) ) - ) return wrapper @@ -659,17 +721,17 @@ def deploy_remote_function( def udf( self, - input_types: type | Sequence[type] | None = None, - output_type: type | None = None, - dataset: str | None = None, - bigquery_connection: str | None = None, - name: str | None = None, - packages: Sequence[str] | None = None, - max_batching_rows: int | None = None, + input_types: Union[None, type, Sequence[type]] = None, + output_type: Optional[type] = None, + session: Optional[Session] = None, + bigquery_client: Optional[bigquery.Client] = None, + dataset: Optional[str] = None, + bigquery_connection: Optional[str] = None, + name: Optional[str] = None, + packages: Optional[Sequence[str]] = None, + max_batching_rows: Optional[int] = None, container_cpu: Optional[float] = None, container_memory: Optional[str] = None, - *, - _force_deploy: bool = False, ): """Decorator to turn a Python user defined function (udf) into a BigQuery managed function. @@ -698,6 +760,12 @@ def udf( be specified. The supported output types are `bool`, `bytes`, `float`, `int`, `str`, `list[bool]`, `list[float]`, `list[int]` and `list[str]`. + session (bigframes.Session, Optional): + BigQuery DataFrames session to use for getting default project, + dataset and BigQuery connection. + bigquery_client (google.cloud.bigquery.Client, Optional): + Client to use for BigQuery operations. If this param is not + provided, then bigquery client from the session would be used. dataset (str, Optional): Dataset in which to create a BigQuery managed function. It should be in `.` or `` @@ -749,16 +817,29 @@ def udf( """ warnings.warn("udf is in preview.", category=bfe.PreviewWarning, stacklevel=5) + + # Some defaults may be used from the session if not provided otherwise. + session = self._resolve_session(session) + + # A BigQuery client is required to perform BQ operations. + bigquery_client = self._resolve_bigquery_client(session, bigquery_client) + # BQ managed function must be persisted, for which we need a dataset. - dataset_ref = self._resolve_dataset_reference(dataset) + dataset_ref = self._resolve_dataset_reference(session, bigquery_client, dataset) + + bq_location, _ = _utils.get_remote_function_locations(bigquery_client.location) # A connection is optional for BQ managed function. bq_connection_id = ( - self._resolve_bigquery_connection_id(dataset_ref, bigquery_connection) + self._resolve_bigquery_connection_id( + session, dataset_ref, bq_location, bigquery_connection + ) if bigquery_connection else None ) + bq_connection_manager = session.bqconnectionmanager + # TODO(b/399129906): Write a method for the repeated part in the wrapper # for both managed function and remote function. def wrapper(func): @@ -769,57 +850,100 @@ def wrapper(func): TypeError, f"func must be a callable, got {func}" ) - udf_sig = _utils.get_func_signature( + if sys.version_info >= (3, 10): + # Add `eval_str = True` so that deferred annotations are turned into their + # corresponding type objects. Need Python 3.10 for eval_str parameter. + # https://docs.python.org/3/library/inspect.html#inspect.signature + signature_kwargs: Mapping[str, Any] = {"eval_str": True} + else: + signature_kwargs = {} # type: ignore + + py_sig = inspect.signature( func, - input_types, - output_type, + **signature_kwargs, + ) + if input_types is not None: + if not isinstance(input_types, collections.abc.Sequence): + input_types = [input_types] + if _utils.has_conflict_input_type(py_sig, input_types): + msg = bfe.format_message( + "Conflicting input types detected, using the one from the decorator." + ) + warnings.warn(msg, category=bfe.FunctionConflictTypeHintWarning) + py_sig = py_sig.replace( + parameters=[ + par.replace(annotation=itype) + for par, itype in zip(py_sig.parameters.values(), input_types) + ] + ) + if output_type: + if _utils.has_conflict_output_type(py_sig, output_type): + msg = bfe.format_message( + "Conflicting return type detected, using the one from the decorator." + ) + warnings.warn(msg, category=bfe.FunctionConflictTypeHintWarning) + py_sig = py_sig.replace(return_annotation=output_type) + + # The function will actually be receiving a pandas Series, but allow + # both BigQuery DataFrames and pandas object types for compatibility. + is_row_processor = False + if new_sig := _convert_row_processor_sig(py_sig): + py_sig = new_sig + is_row_processor = True + + udf_sig = udf_def.UdfSignature.from_py_signature(py_sig) + + managed_function_client = _function_client.FunctionClient( + dataset_ref.project, + bq_location, + dataset_ref.dataset_id, + bigquery_client, + bq_connection_id, + bq_connection_manager, + session=session, # type: ignore ) - code_def = udf_def.CodeDef.from_func(func, package_requirements=packages) - requirements = udf_def.RuntimeRequirements( + bq_function_name = managed_function_client.provision_bq_managed_function( + func=func, + input_types=udf_sig.sql_input_types, + output_type=udf_sig.sql_output_type, + name=name, + packages=packages, + max_batching_rows=max_batching_rows, container_cpu=container_cpu, container_memory=container_memory, + is_row_processor=is_row_processor, bq_connection_id=bq_connection_id, - max_batching_rows=max_batching_rows, - packages=tuple(packages) if packages else (), ) - if udf_sig.is_row_processor: - msg = bfe.format_message("input_types=Series is in preview.") - warnings.warn(msg, stacklevel=1, category=bfe.PreviewWarning) - - if ( - not name and not dataset and not _force_deploy - ): # session-owned resource - deferred deployment - udf_definition = udf_def.PythonUdf( - signature=udf_sig, - code=code_def, - requirements=requirements, - ) - return bq_functions.UdfRoutine(func=func, _udf_def=udf_definition) - else: # deploy immediately - config = udf_def.ManagedFunctionConfig( - code=code_def, - signature=udf_sig, - max_batching_rows=max_batching_rows, - container_cpu=container_cpu, - container_memory=container_memory, - bq_connection_id=bq_connection_id, - capture_references=False, + full_rf_name = ( + managed_function_client.get_remote_function_fully_qualilfied_name( + bq_function_name ) - function_name = name or get_managed_function_name( - config, self.session_id - ) - rf_def = self._deploy_managed_function( - config, - name=function_name, - temp=(name is None), - dataset=dataset_ref, + ) + + udf_definition = udf_def.BigqueryUdf( + routine_ref=bigquery.RoutineReference.from_string(full_rf_name), + signature=udf_sig, + ) + + if not name: + self._update_temp_artifacts(full_rf_name, "") + + decorator = functools.wraps(func) + if is_row_processor: + return decorator( + bq_functions.BigqueryCallableRowRoutine( + udf_definition, session, local_func=func, is_managed=True + ) ) - return bq_functions.BigqueryCallableRoutine( - rf_def, - self._function_client._bq_client, - local_func=func, - is_managed=True, + else: + return decorator( + bq_functions.BigqueryCallableRoutine( + udf_definition, + session, + local_func=func, + is_managed=True, + ) ) return wrapper @@ -846,67 +970,36 @@ def deploy_udf( A wrapped Python user defined function, usable in :meth:`~bigframes.series.Series.apply`. """ - return self.udf(_force_deploy=True, **kwargs)(func) - - -def get_cloud_function_name( - function_def: udf_def.CloudRunFunctionConfig, session_id=None, uniq_suffix=False -): - """ - Get a name for the cloud function for the given user defined function. - - If make_unique is True, append a random suffix to the name. - """ - parts = [_BIGFRAMES_FUNCTION_PREFIX] - if session_id: - parts.append(session_id) - parts.append(function_def.stable_hash().hex()) - if uniq_suffix: - parts.append(uniq_suffix) - return _GCF_FUNCTION_NAME_SEPERATOR.join(parts) - - -def get_bigframes_function_name( - function: udf_def.RemoteFunctionConfig, session_id, uniq_suffix=None -): - """Get a name for the bigframes function for the given user defined function.""" - parts = [_BIGFRAMES_FUNCTION_PREFIX, session_id, function.stable_hash().hex()] - if uniq_suffix: - parts.append(uniq_suffix) - return _BQ_FUNCTION_NAME_SEPERATOR.join(parts) - - -def get_managed_function_name( - function_def: udf_def.ManagedFunctionConfig, - session_id: str | None = None, -): - """Get a name for the bigframes managed function for the given user defined function.""" - parts = [_BIGFRAMES_FUNCTION_PREFIX] - if session_id: - parts.append(session_id) - parts.append(function_def.stable_hash().hex()) - return _BQ_FUNCTION_NAME_SEPERATOR.join(parts) - - -def _infer_milli_cpus_from_memory(memory_mib: int) -> int: - # observed values, not formally documented by cloud run functions - if memory_mib < 128: - raise ValueError("Cloud run supports at minimum 128MiB per instance") - elif memory_mib == 128: - return 83 - elif memory_mib <= 256: - return 167 - elif memory_mib <= 512: - return 333 - elif memory_mib <= 1024: - return 583 - elif memory_mib <= 2048: - return 1000 - elif memory_mib <= 8192: - return 2000 - elif memory_mib <= 16384: - return 4000 - elif memory_mib <= 32768: - return 8000 - else: - raise ValueError("Cloud run supports at most 32768MiB per instance") + # TODO(tswast): If we update udf to defer deployment, update this method + # to deploy immediately. + return self.udf(**kwargs)(func) + + +def _convert_row_processor_sig( + signature: inspect.Signature, +) -> Optional[inspect.Signature]: + import bigframes.series as bf_series + + if len(signature.parameters) >= 1: + first_param = next(iter(signature.parameters.values())) + param_type = first_param.annotation + # Type hints for Series inputs should use pandas.Series because the + # underlying serialization process converts the input to a string + # representation of a pandas Series (not bigframes Series). Using + # bigframes Series will lead to TypeError when creating the function + # remotely. See more from b/445182819. + if param_type == bf_series.Series: + raise bf_formatting.create_exception_with_feedback_link( + TypeError, + "Argument type hint must be Pandas Series, not BigFrames Series.", + ) + if param_type == pandas.Series: + msg = bfe.format_message("input_types=Series is in preview.") + warnings.warn(msg, stacklevel=1, category=bfe.PreviewWarning) + return signature.replace( + parameters=[ + p.replace(annotation=str) if i == 0 else p + for i, p in enumerate(signature.parameters.values()) + ] + ) + return None diff --git a/bigframes/functions/_utils.py b/bigframes/functions/_utils.py index 358f20b2ab4..b6dedeac504 100644 --- a/bigframes/functions/_utils.py +++ b/bigframes/functions/_utils.py @@ -13,26 +13,25 @@ # limitations under the License. -import collections import hashlib import inspect import json import sys import typing +from typing import Any, cast, Optional, Sequence, Set import warnings -from typing import Any, Mapping, Optional, Sequence, Set, cast import cloudpickle import google.api_core.exceptions +from google.cloud import bigquery, functions_v2 import numpy +from packaging.requirements import Requirement import pandas import pyarrow -from google.cloud import bigquery, functions_v2 -from packaging.requirements import Requirement import bigframes.exceptions as bfe import bigframes.formatting_helpers as bf_formatting -from bigframes.functions import function_typing, udf_def +from bigframes.functions import function_typing # Naming convention for the function artifacts _BIGFRAMES_FUNCTION_PREFIX = "bigframes" @@ -44,19 +43,25 @@ _pickle_protocol_version = 4 -def gcf_location_from_bq_location(bq_location: str) -> str: - """Get the cloud functions region that corresponds to a BQ location.""" - bq_location = bq_location.lower() +def get_remote_function_locations(bq_location): + """Get BQ location and cloud functions region given a BQ client.""" + # TODO(shobs, b/274647164): Find the best way to determine default location. + # For now let's assume that if no BQ location is set in the client then it + # defaults to US multi region + bq_location = bq_location.lower() if bq_location else "us" + + # Cloud function should be in the same region as the bigquery remote function + cloud_function_region = bq_location # BigQuery has multi region but cloud functions does not. # Any region in the multi region that supports cloud functions should work # https://cloud.google.com/functions/docs/locations if bq_location == "us": - return "us-central1" + cloud_function_region = "us-central1" elif bq_location == "eu": - return "europe-west1" + cloud_function_region = "europe-west1" - return bq_location + return bq_location, cloud_function_region def _package_existed(package_requirements: list[str], package: str) -> bool: @@ -70,12 +75,12 @@ def _package_existed(package_requirements: list[str], package: str) -> bool: def get_updated_package_requirements( - package_requirements: Sequence[str] = (), - is_row_processor: bool = False, - capture_references: bool = True, - ignore_package_version: bool = False, -) -> Sequence[str]: - requirements: list[str] = [] + package_requirements=None, + is_row_processor=False, + capture_references=True, + ignore_package_version=False, +): + requirements = [] if capture_references: requirements.append(f"cloudpickle=={cloudpickle.__version__}") @@ -103,14 +108,15 @@ def get_updated_package_requirements( requirements.append(f"numpy=={numpy.__version__}") if not requirements: - return list(package_requirements) + return package_requirements - result = list(package_requirements) + if not package_requirements: + package_requirements = [] for package in requirements: - if not _package_existed(result, package): - result.append(package) + if not _package_existed(package_requirements, package): + package_requirements.append(package) - return sorted(result) + return sorted(package_requirements) def clean_up_by_session_id( @@ -159,7 +165,7 @@ def clean_up_by_session_id( # Now clean up the cloud functions bq_location = bqclient.get_dataset(dataset).location - gcf_location = gcf_location_from_bq_location(bq_location) + bq_location, gcf_location = get_remote_function_locations(bq_location) parent_path = gcfclient.common_location_path( project=dataset.project, location=gcf_location ) @@ -177,11 +183,6 @@ def clean_up_by_session_id( pass -def routine_ref_to_string_for_query(routine_ref: bigquery.RoutineReference) -> str: - return f"`{routine_ref.project}.{routine_ref.dataset_id}`.{routine_ref.routine_id}" - - -# Deprecated: Use CodeDef.stable_hash() instead. def get_hash(def_, package_requirements=None): "Get hash (32 digits alphanumeric) of a function." # There is a known cell-id sensitivity of the cloudpickle serialization in @@ -207,28 +208,46 @@ def get_hash(def_, package_requirements=None): return hashlib.md5(def_repr).hexdigest() -def get_python_output_type_str_from_bigframes_metadata( +def routine_ref_to_string_for_query(routine_ref: bigquery.RoutineReference) -> str: + return f"`{routine_ref.project}.{routine_ref.dataset_id}`.{routine_ref.routine_id}" + + +def get_cloud_function_name(function_hash, session_id=None, uniq_suffix=None): + "Get a name for the cloud function for the given user defined function." + parts = [_BIGFRAMES_FUNCTION_PREFIX] + if session_id: + parts.append(session_id) + parts.append(function_hash) + if uniq_suffix: + parts.append(uniq_suffix) + return _GCF_FUNCTION_NAME_SEPERATOR.join(parts) + + +def get_bigframes_function_name(function_hash, session_id, uniq_suffix=None): + "Get a name for the bigframes function for the given user defined function." + parts = [_BIGFRAMES_FUNCTION_PREFIX, session_id, function_hash] + if uniq_suffix: + parts.append(uniq_suffix) + return _BQ_FUNCTION_NAME_SEPERATOR.join(parts) + + +def get_python_output_type_from_bigframes_metadata( metadata_text: str, -) -> Optional[str]: +) -> Optional[type]: try: metadata_dict = json.loads(metadata_text) except (TypeError, json.decoder.JSONDecodeError): return None + try: - return metadata_dict["value"]["python_array_output_type"] + output_type = metadata_dict["value"]["python_array_output_type"] except KeyError: return None - -def get_python_output_type_from_bigframes_metadata( - metadata_text: str, -) -> Optional[type]: - output_type_str = get_python_output_type_str_from_bigframes_metadata(metadata_text) - for ( python_output_array_type ) in function_typing.RF_SUPPORTED_ARRAY_OUTPUT_PYTHON_TYPES: - if python_output_array_type.__name__ == output_type_str: + if python_output_array_type.__name__ == output_type: return list[python_output_array_type] # type: ignore return None @@ -246,9 +265,9 @@ def get_bigframes_metadata(*, python_output_type: Optional[type] = None) -> str: python_output_array_type in function_typing.RF_SUPPORTED_ARRAY_OUTPUT_PYTHON_TYPES ): - inner_metadata["python_array_output_type"] = ( - python_output_array_type.__name__ - ) + inner_metadata[ + "python_array_output_type" + ] = python_output_array_type.__name__ metadata = {"value": inner_metadata} metadata_ser = json.dumps(metadata) @@ -274,6 +293,20 @@ def get_python_version(is_compat: bool = False) -> str: return f"python{major}{minor}" if is_compat else f"python-{major}.{minor}" +def build_unnest_post_routine(py_list_type: type[list]): + sdk_type = function_typing.sdk_array_output_type_from_python_type(py_list_type) + assert sdk_type.array_element_type is not None + inner_sdk_type = sdk_type.array_element_type + result_dtype = function_typing.sdk_type_to_bf_type(inner_sdk_type) + + def post_process(input): + import bigframes.bigquery as bbq + + return bbq.json_extract_string_array(input, value_dtype=result_dtype) + + return post_process + + def has_conflict_input_type( signature: inspect.Signature, input_types: Sequence[Any], @@ -305,54 +338,3 @@ def has_conflict_output_type( return False return return_annotation != output_type - - -def get_func_signature( - func, - input_types: type | Sequence[type] | None = None, - output_type: type | None = None, -) -> udf_def.UdfSignature: - if sys.version_info >= (3, 10): - # Add `eval_str = True` so that deferred annotations are turned into their - # corresponding type objects. Need Python 3.10 for eval_str parameter. - # https://docs.python.org/3/library/inspect.html#inspect.signature - signature_kwargs: Mapping[str, Any] = {"eval_str": True} - else: - signature_kwargs = {} # type: ignore - - py_sig = resolve_signature( - inspect.signature(func, **signature_kwargs), - input_types, - output_type, - ) - return udf_def.UdfSignature.from_py_signature(py_sig) - - -def resolve_signature( - py_sig: inspect.Signature, - input_types: type | Sequence[type] | None = None, - output_type: type | None = None, -) -> inspect.Signature: - if input_types is not None: - if not isinstance(input_types, collections.abc.Sequence): - input_types = [input_types] - if has_conflict_input_type(py_sig, input_types): - msg = bfe.format_message( - "Conflicting input types detected, using the one from the decorator." - ) - warnings.warn(msg, category=bfe.FunctionConflictTypeHintWarning) - py_sig = py_sig.replace( - parameters=[ - par.replace(annotation=itype) - for par, itype in zip(py_sig.parameters.values(), input_types) - ] - ) - if output_type: - if has_conflict_output_type(py_sig, output_type): - msg = bfe.format_message( - "Conflicting return type detected, using the one from the decorator." - ) - warnings.warn(msg, category=bfe.FunctionConflictTypeHintWarning) - py_sig = py_sig.replace(return_annotation=output_type) - - return py_sig diff --git a/bigframes/functions/function.py b/bigframes/functions/function.py index b3a56dafcef..242daf7525d 100644 --- a/bigframes/functions/function.py +++ b/bigframes/functions/function.py @@ -14,19 +14,19 @@ from __future__ import annotations -import dataclasses import logging -from typing import TYPE_CHECKING, Callable, Optional, Protocol, Union, runtime_checkable +from typing import Callable, cast, get_origin, Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from bigframes.session import Session + import bigframes.series import google.api_core.exceptions from google.cloud import bigquery import bigframes.formatting_helpers as bf_formatting from bigframes.functions import _function_session as bff_session -from bigframes.functions import function_typing, udf_def - -if TYPE_CHECKING: - from bigframes.session import Session +from bigframes.functions import _utils, function_typing, udf_def logger = logging.getLogger(__name__) @@ -63,9 +63,7 @@ def get_routine_reference( def remote_function(*args, **kwargs): - import bigframes - - function_session = bigframes.get_global_session()._function_session + function_session = bff_session.FunctionSession() return function_session.remote_function(*args, **kwargs) @@ -73,9 +71,7 @@ def remote_function(*args, **kwargs): def udf(*args, **kwargs): - import bigframes - - function_session = bigframes.get_global_session()._function_session + function_session = bff_session.FunctionSession() return function_session.udf(*args, **kwargs) @@ -83,33 +79,42 @@ def udf(*args, **kwargs): def _try_import_routine( - routine: bigquery.Routine, bq_client: bigquery.Client + routine: bigquery.Routine, session: bigframes.Session ) -> BigqueryCallableRoutine: udf_def = _routine_as_udf_def(routine) + override_type = _get_output_type_override(routine) is_remote = ( hasattr(routine, "remote_function_options") and routine.remote_function_options ) - return BigqueryCallableRoutine(udf_def, bq_client, is_managed=not is_remote) + if override_type is not None: + return BigqueryCallableRoutine( + udf_def, + session, + post_routine=_utils.build_unnest_post_routine(override_type), + ) + return BigqueryCallableRoutine(udf_def, session, is_managed=not is_remote) def _try_import_row_routine( - routine: bigquery.Routine, bq_client: bigquery.Client -) -> BigqueryCallableRoutine: - udf_def = _routine_as_udf_def(routine, is_row_processor=True) - + routine: bigquery.Routine, session: bigframes.Session +) -> BigqueryCallableRowRoutine: + udf_def = _routine_as_udf_def(routine) + override_type = _get_output_type_override(routine) is_remote = ( hasattr(routine, "remote_function_options") and routine.remote_function_options ) - return BigqueryCallableRoutine(udf_def, bq_client, is_managed=not is_remote) + if override_type is not None: + return BigqueryCallableRowRoutine( + udf_def, + session, + post_routine=_utils.build_unnest_post_routine(override_type), + ) + return BigqueryCallableRowRoutine(udf_def, session, is_managed=not is_remote) -def _routine_as_udf_def( - routine: bigquery.Routine, is_row_processor: bool = False -) -> udf_def.BigqueryUdf: +def _routine_as_udf_def(routine: bigquery.Routine) -> udf_def.BigqueryUdf: try: - return udf_def.BigqueryUdf.from_routine( - routine, is_row_processor=is_row_processor - ) + return udf_def.BigqueryUdf.from_routine(routine) except udf_def.ReturnTypeMissingError: raise bf_formatting.create_exception_with_feedback_link( ValueError, "Function return type must be specified." @@ -121,6 +126,31 @@ def _routine_as_udf_def( ) +def _get_output_type_override(routine: bigquery.Routine) -> Optional[type[list]]: + if routine.description is not None and isinstance(routine.description, str): + if python_output_type := _utils.get_python_output_type_from_bigframes_metadata( + routine.description + ): + bq_return_type = cast(bigquery.StandardSqlDataType, routine.return_type) + + if bq_return_type is None or bq_return_type.type_kind != "STRING": + raise bf_formatting.create_exception_with_feedback_link( + TypeError, + "An explicit output_type should be provided only for a BigQuery function with STRING output.", + ) + if get_origin(python_output_type) is list: + return python_output_type + else: + raise bf_formatting.create_exception_with_feedback_link( + TypeError, + "Currently only list of " + "a type is supported as python output type.", + ) + + return None + + +# TODO(b/399894805): Support managed function. def read_gbq_function( function_name: str, *, @@ -148,23 +178,10 @@ def read_gbq_function( ValueError, f"Unknown function '{routine_ref}'." ) - # TODO(493293086): Deprecate is_row_processor. if is_row_processor: - return _try_import_row_routine(routine, bigquery_client) + return _try_import_row_routine(routine, session) else: - return _try_import_routine(routine, bigquery_client) - - -@runtime_checkable -class Udf(Protocol): - """ - Protocol for all BigFrames user-defined functions. - - Has @runtime_checkable so functions like df.apply() can dispatch UDFs with isinstance() checks. - """ - - @property - def udf_def(self) -> Union[udf_def.BigqueryUdf, udf_def.PythonUdf]: ... + return _try_import_routine(routine, session) class BigqueryCallableRoutine: @@ -177,14 +194,18 @@ class BigqueryCallableRoutine: def __init__( self, udf_def: udf_def.BigqueryUdf, - bq_client: bigquery.Client, + session: bigframes.Session, *, local_func: Optional[Callable] = None, cloud_function_ref: Optional[str] = None, + post_routine: Optional[ + Callable[[bigframes.series.Series], bigframes.series.Series] + ] = None, is_managed: bool = False, ): self._udf_def = udf_def - self._bq_client = bq_client + self._session = session + self._post_routine = post_routine self._local_fun = local_func self._cloud_function = cloud_function_ref self._is_managed = is_managed @@ -193,17 +214,19 @@ def __call__(self, *args, **kwargs): if self._local_fun: return self._local_fun(*args, **kwargs) # avoid circular imports + import bigframes.core.sql as bf_sql import bigframes.session._io.bigquery as bf_io_bigquery - from bigframes.core.compile.sqlglot import sql as sg_sql - args_string = ", ".join([sg_sql.to_sql(sg_sql.literal(v)) for v in args]) + args_string = ", ".join(map(bf_sql.simple_literal, args)) sql = f"SELECT `{str(self._udf_def.routine_ref)}`({args_string})" - row_iterator = bf_io_bigquery.start_query_job_optional( - self._bq_client, + iter, job = bf_io_bigquery.start_query_with_client( + self._session.bqclient, sql=sql, + query_with_job=True, job_config=bigquery.QueryJobConfig(), + publisher=self._session._publisher, ) # type: ignore - return list(row_iterator.to_arrow().to_pydict().values())[0][0] + return list(iter.to_arrow().to_pydict().values())[0][0] @property def bigframes_bigquery_function(self) -> str: @@ -215,7 +238,7 @@ def bigframes_remote_function(self): @property def is_row_processor(self) -> bool: - return self.udf_def.signature.is_row_processor + return False @property def udf_def(self) -> udf_def.BigqueryUdf: @@ -227,27 +250,103 @@ def bigframes_cloud_function(self) -> Optional[str]: @property def input_dtypes(self): - return tuple(arg.bf_type for arg in self.udf_def.signature.inputs) + return self.udf_def.signature.bf_input_types @property def output_dtype(self): - return self.udf_def.signature.output.bf_type + return self.udf_def.signature.bf_output_type @property def bigframes_bigquery_function_output_dtype(self): - return self.udf_def.signature.output.emulating_type.bf_type + return self.output_dtype + def _post_process_series( + self, series: bigframes.series.Series + ) -> bigframes.series.Series: + if self._post_routine is not None: + return self._post_routine(series) + return series -@dataclasses.dataclass(frozen=True) -class UdfRoutine: - func: Callable - # Try not to depend on this, bq managed function creation will be deferred later - # And this ref will be replaced with requirements rather to support lazy creation - _udf_def: Union[udf_def.BigqueryUdf, udf_def.PythonUdf] + +class BigqueryCallableRowRoutine: + """ + A reference to a routine in the context of a session. + + Can be used both directly as a callable, or as an input to dataframe ops that take a callable. + """ + + def __init__( + self, + udf_def: udf_def.BigqueryUdf, + session: bigframes.Session, + *, + local_func: Optional[Callable] = None, + cloud_function_ref: Optional[str] = None, + post_routine: Optional[ + Callable[[bigframes.series.Series], bigframes.series.Series] + ] = None, + is_managed: bool = False, + ): + self._udf_def = udf_def + self._session = session + self._post_routine = post_routine + self._local_fun = local_func + self._cloud_function = cloud_function_ref + self._is_managed = is_managed def __call__(self, *args, **kwargs): - return self.func(*args, **kwargs) + if self._local_fun: + return self._local_fun(*args, **kwargs) + # avoid circular imports + import bigframes.core.sql as bf_sql + import bigframes.session._io.bigquery as bf_io_bigquery + + args_string = ", ".join(map(bf_sql.simple_literal, args)) + sql = f"SELECT `{str(self._udf_def.routine_ref)}`({args_string})" + iter, job = bf_io_bigquery.start_query_with_client( + self._session.bqclient, + sql=sql, + query_with_job=True, + job_config=bigquery.QueryJobConfig(), + publisher=self._session._publisher, + ) # type: ignore + return list(iter.to_arrow().to_pydict().values())[0][0] + + @property + def bigframes_bigquery_function(self) -> str: + return str(self._udf_def.routine_ref) @property - def udf_def(self) -> Union[udf_def.BigqueryUdf, udf_def.PythonUdf]: + def bigframes_remote_function(self): + return None if self._is_managed else str(self._udf_def.routine_ref) + + @property + def is_row_processor(self) -> bool: + return True + + @property + def udf_def(self) -> udf_def.BigqueryUdf: return self._udf_def + + @property + def bigframes_cloud_function(self) -> Optional[str]: + return self._cloud_function + + @property + def input_dtypes(self): + return self.udf_def.signature.bf_input_types + + @property + def output_dtype(self): + return self.udf_def.signature.bf_output_type + + @property + def bigframes_bigquery_function_output_dtype(self): + return self.output_dtype + + def _post_process_series( + self, series: bigframes.series.Series + ) -> bigframes.series.Series: + if self._post_routine is not None: + return self._post_routine(series) + return series diff --git a/bigframes/functions/function_template.py b/bigframes/functions/function_template.py index 598de7c853d..a3680a7a88c 100644 --- a/bigframes/functions/function_template.py +++ b/bigframes/functions/function_template.py @@ -19,12 +19,18 @@ import os import re import textwrap +from typing import Tuple -from bigframes.functions import udf_def +import cloudpickle logger = logging.getLogger(__name__) +# Protocol version 4 is available in python version 3.4 and above +# https://docs.python.org/3/library/pickle.html#data-stream-format +_pickle_protocol_version = 4 + + # Placeholder variables for testing. input_types = ("STRING",) output_type = "STRING" @@ -43,7 +49,7 @@ def convert_from_bq_json(type_, arg): import base64 import collections - converters = collections.defaultdict(lambda: lambda value: value) # type: ignore + converters = collections.defaultdict(lambda: (lambda value: value)) # type: ignore converters["BYTES"] = base64.b64decode converter = converters[type_] return converter(arg) if arg is not None else None @@ -53,7 +59,7 @@ def convert_to_bq_json(type_, arg): import base64 import collections - converters = collections.defaultdict(lambda: lambda value: value) # type: ignore + converters = collections.defaultdict(lambda: (lambda value: value)) # type: ignore converters["BYTES"] = lambda value: base64.b64encode(value).decode("utf-8") converter = converters[type_] return converter(arg) if arg is not None else None @@ -181,8 +187,8 @@ def udf_http_row_processor(request): import math import traceback - import pandas as pd from flask import jsonify + import pandas as pd try: request_json = request.get_json(silent=True) @@ -222,39 +228,38 @@ def udf_http_row_processor(request): return jsonify({"errorMessage": traceback.format_exc()}), 400 -def generate_udf_code(code_def: udf_def.CodeDef, directory: str): +def generate_udf_code(def_, directory): """Generate serialized code using cloudpickle given a udf.""" udf_code_file_name = "udf.py" udf_pickle_file_name = "udf.cloudpickle" # original code, only for debugging purpose - if code_def.function_source: - udf_code_file_path = os.path.join(directory, udf_code_file_name) - with open(udf_code_file_path, "w") as f: - f.write(code_def.function_source) + udf_code = textwrap.dedent(inspect.getsource(def_)) + udf_code_file_path = os.path.join(directory, udf_code_file_name) + with open(udf_code_file_path, "w") as f: + f.write(udf_code) # serialized udf udf_pickle_file_path = os.path.join(directory, udf_pickle_file_name) # TODO(b/345433300): try io.BytesIO to avoid writing to the file system with open(udf_pickle_file_path, "wb") as f: - f.write(code_def.pickled_code) + cloudpickle.dump(def_, f, protocol=_pickle_protocol_version) return udf_code_file_name, udf_pickle_file_name def generate_cloud_function_main_code( - code_def: udf_def.CodeDef, - directory: str, + def_, + directory, *, - udf_signature: udf_def.UdfSignature, + input_types: Tuple[str], + output_type: str, + is_row_processor=False, ): """Get main.py code for the cloud function for the given user defined function.""" # Pickle the udf with all its dependencies - udf_code_file, udf_pickle_file = generate_udf_code(code_def, directory) - - input_types = tuple(arg.sql_type for arg in udf_signature.inputs) - output_type = udf_signature.output.sql_type + udf_code_file, udf_pickle_file = generate_udf_code(def_, directory) code_blocks = [ f"""\ @@ -273,7 +278,7 @@ def generate_cloud_function_main_code( # For converting scalar outputs to the correct type. code_blocks.append(inspect.getsource(convert_to_bq_json)) - if udf_signature.is_row_processor: + if is_row_processor: code_blocks.append(inspect.getsource(get_pd_series)) handler_func_name = "udf_http_row_processor" code_blocks.append(inspect.getsource(udf_http_row_processor)) @@ -292,87 +297,74 @@ def generate_cloud_function_main_code( def generate_managed_function_code( - code_def: udf_def.CodeDef, - signature: udf_def.UdfSignature, + def_, + udf_name: str, + is_row_processor: bool, capture_references: bool, ) -> str: """Generates the Python code block for managed Python UDF.""" - udf_name = "unpickled_udf" if capture_references: # This code path ensures that if the udf body contains any # references to variables and/or imports outside the body, they are # captured as well. + import cloudpickle + + pickled = cloudpickle.dumps(def_) func_code = textwrap.dedent( f""" import cloudpickle - {udf_name} = cloudpickle.loads({code_def.pickled_code!r}) + {udf_name} = cloudpickle.loads({pickled}) """ ) else: # This code path ensures that if the udf body is self contained, # i.e. there are no references to variables or imports outside the # body. - assert code_def.function_source is not None - assert code_def.entry_point is not None - func_code = code_def.function_source - udf_name = code_def.entry_point + func_code = textwrap.dedent(inspect.getsource(def_)) match = re.search(r"^def ", func_code, flags=re.MULTILINE) if match is None: raise ValueError("The UDF is not defined correctly.") func_code = func_code[match.start() :] - if signature.is_row_processor: + if is_row_processor: udf_code = textwrap.dedent(inspect.getsource(get_pd_series)) udf_code = udf_code[udf_code.index("def") :] bigframes_handler_code = textwrap.dedent( - f""" - def bigframes_handler(str_arg): - return {udf_name}({get_pd_series.__name__}(str_arg)) - """ + f"""def bigframes_handler(str_arg): + return {udf_name}({get_pd_series.__name__}(str_arg))""" ) - params = list(arg.name for arg in signature.inputs) + sig = inspect.signature(def_) + params = list(sig.parameters.values()) additional_params = params[1:] # Build the parameter list for the new handler function definition. # e.g., "str_arg, y: bool, z" handler_def_parts = ["str_arg"] - handler_def_parts.extend(additional_params) + handler_def_parts.extend(str(p) for p in additional_params) handler_def_str = ", ".join(handler_def_parts) # Build the argument list for the call to the original UDF. # e.g., "get_pd_series(str_arg), y, z" udf_call_parts = [f"{get_pd_series.__name__}(str_arg)"] - udf_call_parts.extend(additional_params) + udf_call_parts.extend(p.name for p in additional_params) udf_call_str = ", ".join(udf_call_parts) bigframes_handler_code = textwrap.dedent( - f""" - def bigframes_handler({handler_def_str}): - return {udf_name}({udf_call_str}) - """ + f"""def bigframes_handler({handler_def_str}): + return {udf_name}({udf_call_str})""" ) else: udf_code = "" bigframes_handler_code = textwrap.dedent( - f""" - def bigframes_handler(*args): - return {udf_name}(*args) - """ + f"""def bigframes_handler(*args): + return {udf_name}(*args)""" ) udf_code_block = [] - if code_def.package_requirements: - # Include package requirements as comments to help force a new - # BigQuery UDF definition when only package requirements change. - packages_comment = "# Packages: " + ", ".join( - sorted(code_def.package_requirements) - ) - udf_code_block.append(packages_comment) - - if not capture_references and signature.is_row_processor: + if not capture_references and is_row_processor: # Enable postponed evaluation of type annotations. This converts all # type hints to strings at runtime, which is necessary for correctly # handling the type annotation of pandas.Series after the UDF code is diff --git a/bigframes/functions/function_typing.py b/bigframes/functions/function_typing.py index a64b3992b68..30804f317c4 100644 --- a/bigframes/functions/function_typing.py +++ b/bigframes/functions/function_typing.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Type, get_args, get_origin +from typing import Any, get_args, get_origin, Type from google.cloud import bigquery @@ -81,7 +81,7 @@ def __init__(self, type_, supported_types): def sdk_type_from_python_type( - t: type, allow_lists: bool = True + t: type, allow_lists: bool = False ) -> bigquery.StandardSqlDataType: if (get_origin(t) is list) and allow_lists: return sdk_array_output_type_from_python_type(t) diff --git a/bigframes/functions/udf_def.py b/bigframes/functions/udf_def.py index 70e0406a6f6..078e45f32d4 100644 --- a/bigframes/functions/udf_def.py +++ b/bigframes/functions/udf_def.py @@ -14,17 +14,10 @@ from __future__ import annotations import dataclasses -import functools import inspect -import io -import os -import textwrap +from typing import cast, Optional import warnings -from typing import Any, Optional, Sequence, Type, cast, get_args, get_origin -import cloudpickle -import google_crc32c -import pandas as pd from google.cloud import bigquery import bigframes.dtypes @@ -32,586 +25,149 @@ import bigframes.formatting_helpers as bf_formatting from bigframes.functions import function_typing -# Protocol version 4 is available in python version 3.4 and above -# https://docs.python.org/3/library/pickle.html#data-stream-format -_pickle_protocol_version = 4 - class ReturnTypeMissingError(ValueError): pass @dataclasses.dataclass(frozen=True) -class UdfArg: +class UdfField: name: str = dataclasses.field() - dtype: DirectScalarType | RowSeriesInputFieldV1 - - def __post_init__(self): - assert isinstance(self.name, str) - assert isinstance(self.dtype, (DirectScalarType, RowSeriesInputFieldV1)) + dtype: bigquery.StandardSqlDataType = dataclasses.field(hash=False, compare=False) @classmethod - def from_py_param(cls, param: inspect.Parameter) -> UdfArg: - if param.annotation == pd.Series: - return cls(param.name, RowSeriesInputFieldV1()) - return cls(param.name, DirectScalarType(param.annotation)) - - @classmethod - def from_sdk(cls, arg: bigquery.RoutineArgument) -> UdfArg: + def from_sdk(cls, arg: bigquery.RoutineArgument) -> UdfField: assert arg.name is not None - - if arg.data_type is None: - msg = bfe.format_message( - "The function has one or more missing input data types. BigQuery DataFrames " - f"will assume default data type {function_typing.DEFAULT_RF_TYPE} for them." - ) - warnings.warn(msg, category=bfe.UnknownDataTypeWarning) - sdk_type = function_typing.DEFAULT_RF_TYPE - else: - sdk_type = arg.data_type - return cls(arg.name, DirectScalarType.from_sdk_type(sdk_type)) - - @property - def py_type(self) -> type: - return self.dtype.py_type - - @property - def bf_type(self) -> bigframes.dtypes.Dtype: - return self.dtype.bf_type - - @property - def sql_type(self) -> str: - return self.dtype.sql_type - - def stable_hash(self) -> bytes: - hash_val = google_crc32c.Checksum() - hash_val.update(self.name.encode()) - hash_val.update(self.dtype.stable_hash()) - return hash_val.digest() + assert arg.data_type is not None + return cls(arg.name, arg.data_type) @dataclasses.dataclass(frozen=True) -class DirectScalarType: - """ - Represents a scalar value that is passed directly to the remote function. - - For these values, BigQuery handles the serialization and deserialization without any additional processing. - """ - - _py_type: type - - @property - def py_type(self) -> type: - return self._py_type +class UdfSignature: + input_types: tuple[UdfField, ...] = dataclasses.field() + output_bq_type: bigquery.StandardSqlDataType = dataclasses.field( + hash=False, compare=False + ) @property - def bf_type(self) -> bigframes.dtypes.Dtype: - return function_typing.sdk_type_to_bf_type( - function_typing.sdk_type_from_python_type(self._py_type) + def bf_input_types(self) -> tuple[bigframes.dtypes.Dtype, ...]: + return tuple( + function_typing.sdk_type_to_bf_type(arg.dtype) for arg in self.input_types ) @property - def sql_type(self) -> str: - sdk_type = function_typing.sdk_type_from_python_type(self._py_type) - return function_typing.sdk_type_to_sql_string(sdk_type) - - def stable_hash(self) -> bytes: - hash_val = google_crc32c.Checksum() - hash_val.update(self._py_type.__name__.encode()) - return hash_val.digest() - - @classmethod - def from_sdk_type(cls, sdk_type: bigquery.StandardSqlDataType) -> DirectScalarType: - return cls(function_typing.sdk_type_to_py_type(sdk_type)) - - @property - def emulating_type(self) -> DirectScalarType: - return self - - -@dataclasses.dataclass(frozen=True) -class VirtualListTypeV1: - """ - Represents a list of scalar values that is emulated as a JSON array string in the remote function. - - Only works as output paramter right now where array -> string in function runtime, and then string -> array in SQL post-processing (defined in out_expr()). - """ - - _PROTOCOL_ID = "virtual_list_v1" - - inner_dtype: DirectScalarType + def bf_output_type(self) -> bigframes.dtypes.Dtype: + return function_typing.sdk_type_to_bf_type(self.output_bq_type) @property - def py_type(self) -> Type[list[Any]]: - return list[self.inner_dtype.py_type] # type: ignore - - @property - def bf_type(self) -> bigframes.dtypes.Dtype: - return bigframes.dtypes.list_type(self.inner_dtype.bf_type) - - @property - def emulating_type(self) -> DirectScalarType: - # Regardless of list inner type, string is used to emulate the list in the remote function. - return DirectScalarType(str) - - def out_expr( - self, expr: bigframes.core.expression.Expression - ) -> bigframes.core.expression.Expression: - # essentially we are undoing json.dumps in sql - import bigframes.operations as ops - - as_str_list = ops.JSONValueArray(json_path="$").as_expr(expr) - if self.inner_dtype.py_type is str: - return as_str_list - elif self.inner_dtype.py_type is bool: - # hack so we don't need to make ArrayMap support general expressions yet - # with b/495513753 we can map the equality operator instead - return ops.ArrayMapOp(ops.IsInOp(values=("true",))).as_expr(as_str_list) - else: - return ops.ArrayMapOp(ops.AsTypeOp(self.inner_dtype.bf_type)).as_expr( - as_str_list - ) - - @property - def sql_type(self) -> str: - return f"ARRAY<{self.inner_dtype.sql_type}>" - - def stable_hash(self) -> bytes: - hash_val = google_crc32c.Checksum() - hash_val.update(self._PROTOCOL_ID.encode()) - hash_val.update(self.inner_dtype.stable_hash()) - return hash_val.digest() - - -@dataclasses.dataclass(frozen=True) -class RowSeriesInputFieldV1: - """ - Used to handle functions that logically take a series as an input, but handled via a string protocol in the remote function. - - For these, the serialization is dependent on index metadata, which must be provided by the caller. - """ - - _PROTOCOL_ID = "row_series_input_v1" - - @property - def py_type(self) -> type: - return pd.Series - - @property - def bf_type(self) -> bigframes.dtypes.Dtype: - # Code paths shouldn't hit this. - raise ValueError("Series does not have a corresponding BigFrames type.") - - @property - def sql_type(self) -> str: - return "STRING" - - @property - def emulating_type(self) -> DirectScalarType: - # Regardless of list inner type, string is used to emulate the list in the remote function. - return DirectScalarType(str) - - def stable_hash(self) -> bytes: - hash_val = google_crc32c.Checksum() - hash_val.update(self._PROTOCOL_ID.encode()) - return hash_val.digest() - - -@dataclasses.dataclass(frozen=True) -class UdfSignature: - """ - Represents the mapping of input types from bigframes to sql to python and back. - """ - - inputs: tuple[UdfArg, ...] = dataclasses.field() - output: DirectScalarType | VirtualListTypeV1 - - def __post_init__(self): - # Validate inputs and outputs are of the correct types. - assert all(isinstance(arg, UdfArg) for arg in self.inputs) - assert isinstance(self.output, (DirectScalarType, VirtualListTypeV1)) - - def to_sql_input_signature(self) -> str: - return ",".join( - f"{field.name} {field.sql_type}" - for field in self.with_devirtualize().inputs + def py_input_types(self) -> tuple[type, ...]: + return tuple( + function_typing.sdk_type_to_py_type(arg.dtype) for arg in self.input_types ) @property - def protocol_metadata(self) -> str | None: - import bigframes.functions._utils - - if isinstance(self.output, VirtualListTypeV1): - return bigframes.functions._utils.get_bigframes_metadata( - python_output_type=self.output.py_type - ) - return None + def py_output_type(self) -> type: + return function_typing.sdk_type_to_py_type(self.output_bq_type) @property - def is_virtual(self) -> bool: - dtypes = (self.output,) + tuple(arg.dtype for arg in self.inputs) - return not all(isinstance(dtype, DirectScalarType) for dtype in dtypes) + def sql_input_types(self) -> tuple[str, ...]: + return tuple( + function_typing.sdk_type_to_sql_string(arg.dtype) + for arg in self.input_types + ) @property - def is_row_processor(self) -> bool: - return any(isinstance(arg.dtype, RowSeriesInputFieldV1) for arg in self.inputs) + def sql_output_type(self) -> str: + return function_typing.sdk_type_to_sql_string(self.output_bq_type) - def with_devirtualize(self) -> UdfSignature: - return UdfSignature( - inputs=tuple( - UdfArg(arg.name, arg.dtype.emulating_type) for arg in self.inputs - ), - output=self.output.emulating_type, - ) - - # TODO(493293086): Deprecate is_row_processor. @classmethod - def from_routine( - cls, routine: bigquery.Routine, is_row_processor: bool = False - ) -> UdfSignature: - import bigframes.functions._utils - - ## Handle return type + def from_routine(cls, routine: bigquery.Routine) -> UdfSignature: if routine.return_type is None: - raise ReturnTypeMissingError( - f"Routine {routine} has no return type. Routine properties: {routine._properties}" - ) - + raise ReturnTypeMissingError bq_return_type = cast(bigquery.StandardSqlDataType, routine.return_type) - return_type: DirectScalarType | VirtualListTypeV1 = ( - DirectScalarType.from_sdk_type(bq_return_type) - ) if ( - python_output_type - := bigframes.functions._utils.get_python_output_type_from_bigframes_metadata( - routine.description - ) + bq_return_type.type_kind is None + or bq_return_type.type_kind + not in function_typing.RF_SUPPORTED_IO_BIGQUERY_TYPEKINDS ): - if bq_return_type.type_kind != "STRING": - raise bf_formatting.create_exception_with_feedback_link( - TypeError, - "An explicit output_type should be provided only for a BigQuery function with STRING output.", - ) - - if get_origin(python_output_type) is list: - inner_type = get_args(python_output_type)[0] - return_type = VirtualListTypeV1(DirectScalarType(inner_type)) - else: - raise bf_formatting.create_exception_with_feedback_link( - TypeError, - "Currently only list of a type is supported as python output type.", - ) + raise ValueError( + f"Remote function must have one of the following supported output types: {function_typing.RF_SUPPORTED_IO_BIGQUERY_TYPEKINDS}" + ) - ## Handle input types udf_fields = [] - - for i, argument in enumerate(routine.arguments): - if is_row_processor and i == 0: - if argument.data_type.type_kind == "STRING": - udf_fields.append(UdfArg(argument.name, RowSeriesInputFieldV1())) - else: - raise ValueError( - "Row processor functions must have STRING input type as first argument." - ) - udf_fields.append(UdfArg.from_sdk(argument)) + for argument in routine.arguments: + if argument.data_type is None: + msg = bfe.format_message( + "The function has one or more missing input data types. BigQuery DataFrames " + f"will assume default data type {function_typing.DEFAULT_RF_TYPE} for them." + ) + warnings.warn(msg, category=bfe.UnknownDataTypeWarning) + assert argument.name is not None + udf_fields.append( + UdfField(argument.name, function_typing.DEFAULT_RF_TYPE) + ) + else: + udf_fields.append(UdfField.from_sdk(argument)) return cls( - inputs=tuple(udf_fields), - output=return_type, + input_types=tuple(udf_fields), + output_bq_type=bq_return_type, ) @classmethod def from_py_signature(cls, signature: inspect.Signature): - import bigframes.series - - input_types: list[UdfArg] = [] + input_types: list[UdfField] = [] for parameter in signature.parameters.values(): if parameter.annotation is inspect.Signature.empty: raise bf_formatting.create_exception_with_feedback_link( ValueError, "'input_types' was not set and parameter " f"'{parameter.name}' is missing a type annotation. " - "Types are required to use udfs.", - ) - if parameter.annotation is bigframes.series.Series: - raise TypeError( - "Argument type hint must be Pandas Series, not BigFrames Series." + "Types are required to use @remote_function.", ) - - input_types.append(UdfArg.from_py_param(parameter)) + bq_type = function_typing.sdk_type_from_python_type(parameter.annotation) + input_types.append(UdfField(parameter.name, bq_type)) if signature.return_annotation is inspect.Signature.empty: raise bf_formatting.create_exception_with_feedback_link( ValueError, "'output_type' was not set and function is missing a " "return type annotation. Types are required to use " - "udfs.", + "@remote_function.", ) - - output_type = DirectScalarType(signature.return_annotation) - return cls(tuple(input_types), output_type) - - def to_remote_function_compatible(self) -> UdfSignature: - # need to virtualize list outputs - if isinstance(self.output, DirectScalarType): - if get_origin(self.output.py_type) is list: - inner_py_type = get_args(self.output.py_type)[0] - return UdfSignature( - inputs=self.inputs, - output=VirtualListTypeV1(DirectScalarType(inner_py_type)), - ) - return self - - def stable_hash(self) -> bytes: - hash_val = google_crc32c.Checksum() - for input_type in self.inputs: - hash_val.update(input_type.stable_hash()) - hash_val.update(self.output.stable_hash()) - return hash_val.digest() - - -@dataclasses.dataclass(frozen=True) -class RuntimeRequirements: - container_cpu: Optional[float] = None - container_memory: Optional[str] = None - bq_connection_id: Optional[str] = None - max_batching_rows: Optional[int] = None - packages: tuple[str, ...] = () - - def stable_hash(self) -> bytes: - hash_val = google_crc32c.Checksum() - if self.container_cpu is not None: - hash_val.update(str(self.container_cpu).encode()) - if self.container_memory is not None: - hash_val.update(str(self.container_memory).encode()) - if self.bq_connection_id is not None: - hash_val.update(str(self.bq_connection_id).encode()) - if self.max_batching_rows is not None: - hash_val.update(str(self.max_batching_rows).encode()) - if self.packages: - for p in sorted(self.packages): - hash_val.update(p.encode()) - return hash_val.digest() + output_bq_type = function_typing.sdk_type_from_python_type( + signature.return_annotation, + allow_lists=True, + ) + return cls(tuple(input_types), output_bq_type) @dataclasses.dataclass(frozen=True) class BigqueryUdf: - """ - Represents the information needed to call a BigQuery remote function - not a full spec. - """ - routine_ref: bigquery.RoutineReference = dataclasses.field() signature: UdfSignature - - def with_devirtualize(self) -> BigqueryUdf: - if not self.signature.is_virtual: - return self - return BigqueryUdf( - routine_ref=self.routine_ref, - signature=self.signature.with_devirtualize(), - ) - - @classmethod - def from_routine( - cls, routine: bigquery.Routine, is_row_processor: bool = False - ) -> BigqueryUdf: - signature = UdfSignature.from_routine( - routine, is_row_processor=is_row_processor - ) - return cls(routine.reference, signature=signature) - - -@dataclasses.dataclass(frozen=True) -class PythonUdf: - """ - Represents user-requested Python UDF semantics, including the code and runtime requirements. - """ - - signature: UdfSignature - code: CodeDef - requirements: RuntimeRequirements = dataclasses.field( - default_factory=RuntimeRequirements + # Used to provide alternative interpretations of output bq type, eg interpret int as timestamp + output_type_override: Optional[bigframes.dtypes.Dtype] = dataclasses.field( + default=None ) - def stable_hash(self) -> bytes: - hash_val = google_crc32c.Checksum() - hash_val.update(self.code.stable_hash()) - hash_val.update(self.signature.stable_hash()) - hash_val.update(self.requirements.stable_hash()) - return hash_val.digest() - - def to_managed_function_config(self) -> ManagedFunctionConfig: - return ManagedFunctionConfig( - code=self.code, - signature=self.signature, - max_batching_rows=self.requirements.max_batching_rows, - container_cpu=self.requirements.container_cpu, - container_memory=self.requirements.container_memory, - bq_connection_id=self.requirements.bq_connection_id, - capture_references=False, - ) - - -@dataclasses.dataclass(frozen=True) -class CodeDef: - # Produced by cloudpickle, not compatible across python versions - pickled_code: bytes - # This is just the function itself, and does not include referenced objects/functions/modules - function_source: Optional[str] - entry_point: Optional[str] - package_requirements: tuple[str, ...] - - @classmethod - def from_func(cls, func, package_requirements: Sequence[str] | None = None): - bytes_io = io.BytesIO() - cloudpickle.dump(func, bytes_io, protocol=_pickle_protocol_version) - source = None - entry_point = None - try: - # dedent is hacky, but works for some nested functions - source = textwrap.dedent(inspect.getsource(func)) - entry_point = func.__name__ - except OSError: - pass - return cls( - pickled_code=bytes_io.getvalue(), - function_source=source, - entry_point=entry_point, - package_requirements=tuple(package_requirements or []), - ) - - @functools.cache - def stable_hash(self) -> bytes: - # There is a known cell-id sensitivity of the cloudpickle serialization in - # notebooks https://github.com/cloudpipe/cloudpickle/issues/538. Because of - # this, if a cell contains a udf decorated with @remote_function, a unique - # cloudpickle code is generated every time the cell is run, creating new - # cloud artifacts every time. This is slow and wasteful. - # A workaround of the same can be achieved by replacing the filename in the - # code object to a static value - # https://github.com/cloudpipe/cloudpickle/issues/120#issuecomment-338510661. - # - # To respect the user code/environment let's make this modification on a - # copy of the udf, not on the original udf itself. - def_copy = cloudpickle.loads(self.pickled_code) - def_copy.__code__ = def_copy.__code__.replace( - co_filename="bigframes_place_holder_filename" - ) - - normalized_pickled_code = cloudpickle.dumps( - def_copy, protocol=_pickle_protocol_version + @property + def bigframes_output_type(self) -> bigframes.dtypes.Dtype: + return self.output_type_override or function_typing.sdk_type_to_bf_type( + self.signature.output_bq_type ) - hash_val = google_crc32c.Checksum() - hash_val.update(normalized_pickled_code) - - if self.package_requirements: - for p in sorted(self.package_requirements): - hash_val.update(p.encode()) - - return hash_val.digest() - - def to_callable(self): - """ - Reconstructs the python callable from the pickled code. - - Assumption: package_requirements match local environment - """ - return cloudpickle.loads(self.pickled_code) - - -@dataclasses.dataclass(frozen=True) -class ManagedFunctionConfig: - code: CodeDef - signature: UdfSignature - max_batching_rows: Optional[int] - container_cpu: Optional[float] - container_memory: Optional[str] - bq_connection_id: Optional[str] - # capture_refernces=True -> deploy as cloudpickle - # capture_references=False -> deploy as source - capture_references: bool = False - - def stable_hash(self) -> bytes: - hash_val = google_crc32c.Checksum() - hash_val.update(self.code.stable_hash()) - hash_val.update(self.signature.stable_hash()) - hash_val.update(str(self.max_batching_rows).encode()) - hash_val.update(str(self.container_cpu).encode()) - hash_val.update(str(self.container_memory).encode()) - hash_val.update(str(self.bq_connection_id).encode()) - hash_val.update(str(self.capture_references).encode()) - return hash_val.digest() - - -@dataclasses.dataclass(frozen=True) -class CloudRunFunctionConfig: - code: CodeDef - signature: UdfSignature - timeout_seconds: int | None - max_instance_count: int | None - vpc_connector: str | None - vpc_connector_egress_settings: str - memory_mib: int | None - cpus: float | None - ingress_settings: str - workers: int | None - threads: int | None - concurrency: int | None - kms_key_name: str | None - docker_repository: str | None - cloud_build_service_account: str | None - cloud_run_service_account: str | None - - def stable_hash(self) -> bytes: - hash_val = google_crc32c.Checksum() - hash_val.update(self.code.stable_hash()) - hash_val.update(self.signature.stable_hash()) - hash_val.update(str(self.timeout_seconds).encode()) - hash_val.update(str(self.max_instance_count).encode()) - hash_val.update(str(self.vpc_connector).encode()) - hash_val.update(str(self.vpc_connector_egress_settings).encode()) - hash_val.update(str(self.memory_mib).encode()) - hash_val.update(str(self.cpus).encode()) - hash_val.update(str(self.ingress_settings).encode()) - hash_val.update(str(self.workers).encode()) - hash_val.update(str(self.threads).encode()) - hash_val.update(str(self.concurrency).encode()) - hash_val.update(str(self.kms_key_name).encode()) - hash_val.update(str(self.docker_repository).encode()) - hash_val.update(str(self.cloud_build_service_account).encode()) - hash_val.update(str(self.cloud_run_service_account).encode()) - return hash_val.digest() - - -@dataclasses.dataclass(frozen=True) -class RemoteFunctionConfig: - """ - Represents the information needed to create a BigQuery remote function. - """ - - endpoint: str - signature: UdfSignature - connection_id: str - max_batching_rows: int - bq_metadata: str | None = None - @classmethod - def from_bq_routine(cls, routine: bigquery.Routine) -> RemoteFunctionConfig: - return cls( - endpoint=routine.remote_function_options.endpoint, - connection_id=os.path.basename(routine.remote_function_options.connection), - signature=UdfSignature.from_routine(routine), - max_batching_rows=routine.remote_function_options.max_batching_rows, - bq_metadata=routine.description, - ) + def from_routine(cls, routine: bigquery.Routine) -> BigqueryUdf: + signature = UdfSignature.from_routine(routine) - def stable_hash(self) -> bytes: - hash_val = google_crc32c.Checksum() - hash_val.update(self.endpoint.encode()) - hash_val.update(self.signature.stable_hash()) - hash_val.update(self.connection_id.encode()) - hash_val.update(str(self.max_batching_rows).encode()) - hash_val.update(str(self.bq_metadata).encode()) - return hash_val.digest() + if ( + signature.output_bq_type.type_kind is None + or signature.output_bq_type.type_kind + not in function_typing.RF_SUPPORTED_IO_BIGQUERY_TYPEKINDS + ): + raise ValueError( + f"Remote function must have one of the following supported output types: {function_typing.RF_SUPPORTED_IO_BIGQUERY_TYPEKINDS}" + ) + return cls(routine.reference, signature=signature) diff --git a/bigframes/geopandas/geoseries.py b/bigframes/geopandas/geoseries.py index dc373216b65..660f1939a94 100644 --- a/bigframes/geopandas/geoseries.py +++ b/bigframes/geopandas/geoseries.py @@ -22,11 +22,11 @@ import bigframes.operations as ops import bigframes.series import bigframes.session -from bigframes._tools import docs -@docs.inherit_docs(vendored_geoseries.GeoSeries) -class GeoSeries(bigframes.series.Series): +class GeoSeries(vendored_geoseries.GeoSeries, bigframes.series.Series): + __doc__ = vendored_geoseries.GeoSeries.__doc__ + def __init__(self, data=None, index=None, **kwargs): super().__init__( data=data, index=index, dtype=geopandas.array.GeometryDtype(), **kwargs @@ -107,7 +107,7 @@ def buffer(self: GeoSeries, distance: float) -> bigframes.series.Series: # type @property def centroid(self: GeoSeries) -> bigframes.series.Series: # type: ignore - return self._apply_nary_op(ops.googlesql.ST_CENTROID, []) + return self._apply_unary_op(ops.geo_st_centroid_op) @property def convex_hull(self: GeoSeries) -> bigframes.series.Series: # type: ignore diff --git a/bigframes/ml/base.py b/bigframes/ml/base.py index fbfaf6b537c..9b38702ccea 100644 --- a/bigframes/ml/base.py +++ b/bigframes/ml/base.py @@ -24,21 +24,18 @@ """ import abc -import typing +from typing import cast, Optional, TypeVar, Union import warnings -from typing import Optional, TypeVar, Union import bigframes_vendored.sklearn.base import bigframes.exceptions as bfe +from bigframes.ml import core import bigframes.ml.utils as utils import bigframes.pandas as bpd -from bigframes._tools import docs -from bigframes.ml import core -@docs.inherit_docs(bigframes_vendored.sklearn.base.BaseEstimator) -class BaseEstimator(abc.ABC): +class BaseEstimator(bigframes_vendored.sklearn.base.BaseEstimator, abc.ABC): """ A BigQuery DataFrames machine learning component follows sklearn API design Ref: https://bit.ly/3NyhKjN @@ -136,7 +133,7 @@ def register(self: _T, vertex_ai_model_id: Optional[str] = None) -> _T: self._bqml_model = self._create_bqml_model() # type: ignore except AttributeError: raise RuntimeError("A model must be trained before register.") - self._bqml_model = typing.cast(core.BqmlModel, self._bqml_model) + self._bqml_model = cast(core.BqmlModel, self._bqml_model) self._bqml_model.register(vertex_ai_model_id) return self @@ -289,7 +286,7 @@ def _predict_and_retry( bpd.concat([df_result, df_succ]) if df_result is not None else df_succ ) - df_result = typing.cast( + df_result = cast( bpd.DataFrame, bpd.concat([df_result, df_fail]) if df_result is not None else df_fail, ) @@ -309,7 +306,7 @@ def _extract_output_names(self): output_names = [] for transform_col in self._bqml_model._model._properties["transformColumns"]: - transform_col_dict = typing.cast(dict, transform_col) + transform_col_dict = cast(dict, transform_col) # pass the columns that are not transformed if "transformSql" not in transform_col_dict: continue diff --git a/bigframes/ml/cluster.py b/bigframes/ml/cluster.py index f7a84a57e97..f371be0cf38 100644 --- a/bigframes/ml/cluster.py +++ b/bigframes/ml/cluster.py @@ -20,13 +20,13 @@ from typing import List, Literal, Optional, Union import bigframes_vendored.sklearn.cluster._kmeans -import pandas as pd from google.cloud import bigquery +import pandas as pd import bigframes -import bigframes.pandas as bpd from bigframes.core.logging import log_adapter from bigframes.ml import base, core, globals, utils +import bigframes.pandas as bpd _BQML_PARAMS_MAPPING = { "n_clusters": "numClusters", @@ -44,6 +44,7 @@ class KMeans( base.UnsupervisedTrainablePredictor, bigframes_vendored.sklearn.cluster._kmeans.KMeans, ): + __doc__ = bigframes_vendored.sklearn.cluster._kmeans.KMeans.__doc__ def __init__( diff --git a/bigframes/ml/compose.py b/bigframes/ml/compose.py index 0d6c58897ac..d638e026e45 100644 --- a/bigframes/ml/compose.py +++ b/bigframes/ml/compose.py @@ -21,17 +21,17 @@ import re import types import typing -from typing import Iterable, List, Optional, Set, Tuple, Union +from typing import cast, Iterable, List, Optional, Set, Tuple, Union -import bigframes_vendored.sklearn.compose._column_transformer from bigframes_vendored import constants +import bigframes_vendored.sklearn.compose._column_transformer from google.cloud import bigquery -import bigframes.core.utils as core_utils -import bigframes.pandas as bpd -from bigframes.core.compile.sqlglot import sql as sg_sql +import bigframes.core.compile.googlesql as sql_utils from bigframes.core.logging import log_adapter +import bigframes.core.utils as core_utils from bigframes.ml import base, core, globals, impute, preprocessing, utils +import bigframes.pandas as bpd _BQML_TRANSFROM_TYPE_MAPPING = types.MappingProxyType( { @@ -111,9 +111,9 @@ def _compile_to_sql( columns, _ = core_utils.get_standardized_ids(columns) result = [] for column in columns: - current_sql = self._sql.format(sg_sql.to_sql(sg_sql.identifier(column))) - current_target_column = sg_sql.to_sql( - sg_sql.identifier(self._target_column.format(column)) + current_sql = self._sql.format(sql_utils.identifier(column)) + current_target_column = sql_utils.identifier( + self._target_column.format(column) ) result.append(f"{current_sql} AS {current_target_column}") return result @@ -171,13 +171,7 @@ def _keys(self): @property def transformers_( self, - ) -> List[ - Tuple[ - str, - SingleColTransformer, - str, - ] - ]: + ) -> List[Tuple[str, SingleColTransformer, str,]]: """The collection of transformers as tuples of (name, transformer, column).""" result: List[ Tuple[ @@ -224,7 +218,7 @@ def camel_to_snake(name): output_names = [] for transform_col in bq_model._properties["transformColumns"]: - transform_col_dict = typing.cast(dict, transform_col) + transform_col_dict = cast(dict, transform_col) # pass the columns that are not transformed if "transformSql" not in transform_col_dict: continue @@ -288,7 +282,7 @@ def _merge( return self # SQLScalarColumnTransformer only work inside ColumnTransformer feature_columns_sorted = sorted( [ - typing.cast(str, feature_column.name) + cast(str, feature_column.name) for feature_column in bq_model.feature_columns ] ) diff --git a/bigframes/ml/core.py b/bigframes/ml/core.py index 5a096f305bd..4dbc1a5fa30 100644 --- a/bigframes/ml/core.py +++ b/bigframes/ml/core.py @@ -18,17 +18,16 @@ import dataclasses import datetime -import typing +from typing import Callable, cast, Iterable, Mapping, Optional, Union import uuid -from typing import Callable, Iterable, Mapping, Optional, Union from google.cloud import bigquery import bigframes.constants as constants import bigframes.formatting_helpers as formatting_helpers +from bigframes.ml import sql as ml_sql import bigframes.pandas as bpd import bigframes.session -from bigframes.ml import sql as ml_sql class BaseBqml: @@ -377,7 +376,7 @@ def copy(self, new_model_name: str, replace: bool = False) -> BqmlModel: def register(self, vertex_ai_model_id: Optional[str] = None) -> BqmlModel: if vertex_ai_model_id is None: # vertex id needs to start with letters. https://cloud.google.com/vertex-ai/docs/general/resource-naming - vertex_ai_model_id = "bigframes_" + typing.cast(str, self._model.model_id) + vertex_ai_model_id = "bigframes_" + cast(str, self._model.model_id) # truncate as Vertex ID only accepts 63 characters, easily exceeding the limit for temp models. # The possibility of conflicts should be low. @@ -509,15 +508,15 @@ def create_time_series_model( transforms: Optional[Iterable[str]] = None, options: Mapping[str, Union[str, int, float, Iterable[str]]] = {}, ) -> BqmlModel: - assert X_train.columns.size == 1, ( - "Time series timestamp input must only contain 1 column." - ) - assert y_train.columns.size == 1, ( - "Time stamp data input must only contain 1 column." - ) - assert id_col is None or (id_col is not None and id_col.columns.size == 1), ( - "Time series id input is either None or must only contain 1 column." - ) + assert ( + X_train.columns.size == 1 + ), "Time series timestamp input must only contain 1 column." + assert ( + y_train.columns.size == 1 + ), "Time stamp data input must only contain 1 column." + assert id_col is None or ( + id_col is not None and id_col.columns.size == 1 + ), "Time series id input is either None or must only contain 1 column." options = dict(options) # Cache dataframes to make sure base table is not a snapshot diff --git a/bigframes/ml/decomposition.py b/bigframes/ml/decomposition.py index eedf6c09170..ca5ff102b44 100644 --- a/bigframes/ml/decomposition.py +++ b/bigframes/ml/decomposition.py @@ -23,10 +23,10 @@ import bigframes_vendored.sklearn.decomposition._pca from google.cloud import bigquery -import bigframes.pandas as bpd -import bigframes.session from bigframes.core.logging import log_adapter from bigframes.ml import base, core, globals, utils +import bigframes.pandas as bpd +import bigframes.session _BQML_PARAMS_MAPPING = { "svd_solver": "pcaSolver", @@ -226,6 +226,7 @@ def __init__( # TODO: Add support for hyperparameter tuning. l2_reg: float = 1.0, ): + feedback_type = feedback_type.lower() # type: ignore if feedback_type not in ("explicit", "implicit"): raise ValueError("Expected feedback_type to be `explicit` or `implicit`.") diff --git a/bigframes/ml/ensemble.py b/bigframes/ml/ensemble.py index 5d2b130d7ae..7cd7079dfbd 100644 --- a/bigframes/ml/ensemble.py +++ b/bigframes/ml/ensemble.py @@ -23,10 +23,10 @@ import bigframes_vendored.xgboost.sklearn from google.cloud import bigquery -import bigframes.dataframe -import bigframes.session from bigframes.core.logging import log_adapter +import bigframes.dataframe from bigframes.ml import base, core, globals, utils +import bigframes.session _BQML_PARAMS_MAPPING = { "booster": "boosterType", @@ -213,6 +213,7 @@ class XGBClassifier( base.SupervisedTrainableWithEvaluationPredictor, bigframes_vendored.xgboost.sklearn.XGBClassifier, ): + __doc__ = bigframes_vendored.xgboost.sklearn.XGBClassifier.__doc__ def __init__( @@ -369,6 +370,7 @@ class RandomForestRegressor( base.SupervisedTrainableWithEvaluationPredictor, bigframes_vendored.sklearn.ensemble._forest.RandomForestRegressor, ): + __doc__ = bigframes_vendored.sklearn.ensemble._forest.RandomForestRegressor.__doc__ def __init__( @@ -534,6 +536,7 @@ class RandomForestClassifier( base.SupervisedTrainableWithEvaluationPredictor, bigframes_vendored.sklearn.ensemble._forest.RandomForestClassifier, ): + __doc__ = bigframes_vendored.sklearn.ensemble._forest.RandomForestClassifier.__doc__ def __init__( diff --git a/bigframes/ml/forecasting.py b/bigframes/ml/forecasting.py index bfdd736f855..99a7b1743d3 100644 --- a/bigframes/ml/forecasting.py +++ b/bigframes/ml/forecasting.py @@ -20,10 +20,10 @@ from google.cloud import bigquery -import bigframes.pandas as bpd -import bigframes.session from bigframes.core.logging import log_adapter from bigframes.ml import base, core, globals, utils +import bigframes.pandas as bpd +import bigframes.session _BQML_PARAMS_MAPPING = { "horizon": "horizon", diff --git a/bigframes/ml/imported.py b/bigframes/ml/imported.py index ca83b0ee568..295649ed7f5 100644 --- a/bigframes/ml/imported.py +++ b/bigframes/ml/imported.py @@ -16,15 +16,14 @@ from __future__ import annotations -import typing -from typing import Mapping, Optional +from typing import cast, Mapping, Optional from google.cloud import bigquery -import bigframes.pandas as bpd -import bigframes.session from bigframes.core.logging import log_adapter from bigframes.ml import base, core, globals, utils +import bigframes.pandas as bpd +import bigframes.session @log_adapter.class_logger @@ -73,14 +72,13 @@ def predict(self, X: utils.ArrayType) -> bpd.DataFrame: Input DataFrame. Schema is defined by the model. Returns: - bigframes.dataframe.DataFrame: Output DataFrame. Schema is defined by the model. - """ + bigframes.dataframe.DataFrame: Output DataFrame. Schema is defined by the model.""" if not self._bqml_model: if self.model_path is None: raise ValueError("Model GCS path must be provided.") self._bqml_model = self._create_bqml_model() - self._bqml_model = typing.cast(core.BqmlModel, self._bqml_model) + self._bqml_model = cast(core.BqmlModel, self._bqml_model) (X,) = utils.batch_convert_to_dataframe(X) @@ -101,7 +99,7 @@ def to_gbq(self, model_name: str, replace: bool = False) -> TensorFlowModel: if self.model_path is None: raise ValueError("Model GCS path must be provided.") self._bqml_model = self._create_bqml_model() - self._bqml_model = typing.cast(core.BqmlModel, self._bqml_model) + self._bqml_model = cast(core.BqmlModel, self._bqml_model) new_model = self._bqml_model.copy(model_name, replace) return new_model.session.read_gbq_model(model_name) @@ -153,14 +151,13 @@ def predict(self, X: utils.ArrayType) -> bpd.DataFrame: Input DataFrame or Series. Schema is defined by the model. Returns: - bigframes.dataframe.DataFrame: Output DataFrame, schema is defined by the model. - """ + bigframes.dataframe.DataFrame: Output DataFrame, schema is defined by the model.""" if not self._bqml_model: if self.model_path is None: raise ValueError("Model GCS path must be provided.") self._bqml_model = self._create_bqml_model() - self._bqml_model = typing.cast(core.BqmlModel, self._bqml_model) + self._bqml_model = cast(core.BqmlModel, self._bqml_model) (X,) = utils.batch_convert_to_dataframe(X, session=self._bqml_model.session) @@ -181,7 +178,7 @@ def to_gbq(self, model_name: str, replace: bool = False) -> ONNXModel: if self.model_path is None: raise ValueError("Model GCS path must be provided.") self._bqml_model = self._create_bqml_model() - self._bqml_model = typing.cast(core.BqmlModel, self._bqml_model) + self._bqml_model = cast(core.BqmlModel, self._bqml_model) new_model = self._bqml_model.copy(model_name, replace) return new_model.session.read_gbq_model(model_name) @@ -273,14 +270,13 @@ def predict(self, X: utils.ArrayType) -> bpd.DataFrame: Input DataFrame or Series. Schema is defined by the model. Returns: - bigframes.dataframe.DataFrame: Output DataFrame. Schema is defined by the model. - """ + bigframes.dataframe.DataFrame: Output DataFrame. Schema is defined by the model.""" if not self._bqml_model: if self.model_path is None: raise ValueError("Model GCS path must be provided.") self._bqml_model = self._create_bqml_model() - self._bqml_model = typing.cast(core.BqmlModel, self._bqml_model) + self._bqml_model = cast(core.BqmlModel, self._bqml_model) (X,) = utils.batch_convert_to_dataframe(X, session=self._bqml_model.session) @@ -301,7 +297,7 @@ def to_gbq(self, model_name: str, replace: bool = False) -> XGBoostModel: if self.model_path is None: raise ValueError("Model GCS path must be provided.") self._bqml_model = self._create_bqml_model() - self._bqml_model = typing.cast(core.BqmlModel, self._bqml_model) + self._bqml_model = cast(core.BqmlModel, self._bqml_model) new_model = self._bqml_model.copy(model_name, replace) return new_model.session.read_gbq_model(model_name) diff --git a/bigframes/ml/impute.py b/bigframes/ml/impute.py index 77314c360ad..b3da895201d 100644 --- a/bigframes/ml/impute.py +++ b/bigframes/ml/impute.py @@ -22,10 +22,10 @@ import bigframes_vendored.sklearn.impute._base -import bigframes.core.utils as core_utils -import bigframes.pandas as bpd from bigframes.core.logging import log_adapter +import bigframes.core.utils as core_utils from bigframes.ml import base, core, globals, utils +import bigframes.pandas as bpd @log_adapter.class_logger @@ -33,6 +33,7 @@ class SimpleImputer( base.Transformer, bigframes_vendored.sklearn.impute._base.SimpleImputer, ): + __doc__ = bigframes_vendored.sklearn.impute._base.SimpleImputer.__doc__ def __init__( diff --git a/bigframes/ml/linear_model.py b/bigframes/ml/linear_model.py index d35a2d45ecb..df054eb3062 100644 --- a/bigframes/ml/linear_model.py +++ b/bigframes/ml/linear_model.py @@ -24,10 +24,10 @@ import bigframes_vendored.sklearn.linear_model._logistic from google.cloud import bigquery -import bigframes.pandas as bpd -import bigframes.session from bigframes.core.logging import log_adapter from bigframes.ml import base, core, globals, utils +import bigframes.pandas as bpd +import bigframes.session _BQML_PARAMS_MAPPING = { "optimize_strategy": "optimizationStrategy", diff --git a/bigframes/ml/llm.py b/bigframes/ml/llm.py index 9e5ceb95d0e..f4e60f3f9d4 100644 --- a/bigframes/ml/llm.py +++ b/bigframes/ml/llm.py @@ -16,20 +16,19 @@ from __future__ import annotations -import typing +from typing import cast, Iterable, Literal, Mapping, Optional, Union import warnings -from typing import Iterable, Literal, Mapping, Optional, Union import bigframes_vendored.constants as constants from google.cloud import bigquery -import bigframes.bigquery as bbq -import bigframes.dataframe -import bigframes.series from bigframes import dtypes, exceptions +import bigframes.bigquery as bbq from bigframes.core import blocks, global_session from bigframes.core.logging import log_adapter +import bigframes.dataframe from bigframes.ml import base, core, globals, utils +import bigframes.series _BQML_PARAMS_MAPPING = { "max_iterations": "maxIterations", @@ -59,8 +58,6 @@ _GEMINI_2P5_PRO_ENDPOINT = "gemini-2.5-pro" _GEMINI_2P5_FLASH_ENDPOINT = "gemini-2.5-flash" _GEMINI_2P5_FLASH_LITE_ENDPOINT = "gemini-2.5-flash-lite" -_GEMINI_3P1_FLASH_LITE_ENDPOINT = "gemini-3.1-flash-lite" -_GEMINI_3P5_FLASH_ENDPOINT = "gemini-3.5-flash" _GEMINI_ENDPOINTS = ( _GEMINI_1P5_PRO_PREVIEW_ENDPOINT, @@ -75,8 +72,6 @@ _GEMINI_2P5_PRO_ENDPOINT, _GEMINI_2P5_FLASH_ENDPOINT, _GEMINI_2P5_FLASH_LITE_ENDPOINT, - _GEMINI_3P1_FLASH_LITE_ENDPOINT, - _GEMINI_3P5_FLASH_ENDPOINT, ) _GEMINI_PREVIEW_ENDPOINTS = ( _GEMINI_1P5_PRO_PREVIEW_ENDPOINT, @@ -100,8 +95,6 @@ _GEMINI_2P5_PRO_ENDPOINT, _GEMINI_2P5_FLASH_ENDPOINT, _GEMINI_2P5_FLASH_LITE_ENDPOINT, - _GEMINI_3P1_FLASH_LITE_ENDPOINT, - _GEMINI_3P5_FLASH_ENDPOINT, ) _CLAUDE_3_SONNET_ENDPOINT = "claude-3-sonnet" @@ -259,7 +252,7 @@ def predict( if len(X.columns) == 1: # BQML identified the column by name - col_label = typing.cast(blocks.Label, X.columns[0]) + col_label = cast(blocks.Label, X.columns[0]) X = X.rename(columns={col_label: "content"}) options: dict = {} @@ -292,7 +285,7 @@ class MultimodalEmbeddingGenerator(base.RetriableRemotePredictor): """Multimodal embedding generator LLM model. .. note:: - BigFrames ObjectRef is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + BigFrames Blob is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" and might have limited support. For more information, see the launch stage descriptions (https://cloud.google.com/products#product-launch-stages). @@ -380,7 +373,7 @@ def predict( Args: X (bigframes.dataframe.DataFrame or bigframes.series.Series or pandas.core.frame.DataFrame or pandas.core.series.Series): Input DataFrame or Series, can contain one or more columns. If multiple columns are in the DataFrame, it must contain a "content" column for prediction. - The content column must be of string type or BigFrames `ObjectRef `_ of image or video. + The content column must be of string type or BigFrames Blob of image or video. max_retries (int, default 0): Max number of retries if the prediction for any rows failed. Each try needs to make progress (i.e. has successfully predicted rows) to continue the retry. @@ -398,12 +391,12 @@ def predict( if len(X.columns) == 1: # BQML identified the column by name - col_label = typing.cast(blocks.Label, X.columns[0]) + col_label = cast(blocks.Label, X.columns[0]) X = X.rename(columns={col_label: "content"}) # TODO(garrettwu): remove transform to ObjRefRuntime when BQML supports ObjRef as input if X["content"].dtype == dtypes.OBJ_REF_DTYPE: - X["content"] = bbq.obj.get_access_url(X["content"], mode="r") + X["content"] = X["content"].blob._get_runtime("R", with_metadata=True) options: dict = {} @@ -440,15 +433,14 @@ class GeminiTextGenerator(base.RetriableRemotePredictor): gemini-1.5-X are going to be deprecated. Use gemini-2.5-X (https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.ml.llm.GeminiTextGenerator) instead. Args: - model_name (str, Default to "gemini-2.5-flash"): + model_name (str, Default to "gemini-2.0-flash-001"): The model for natural language tasks. Accepted values are "gemini-1.5-pro-preview-0514", "gemini-1.5-flash-preview-0514", "gemini-1.5-pro-001", "gemini-1.5-pro-002", "gemini-1.5-flash-001", "gemini-1.5-flash-002", "gemini-2.0-flash-exp", "gemini-2.0-flash-lite-001", "gemini-2.0-flash-001", - "gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite", - "gemini-3.1-flash-lite" and "gemini-3.5-flash". - If no setting is provided, "gemini-2.5-flash" will be used by + "gemini-2.5-pro", "gemini-2.5-flash" and "gemini-2.5-flash-lite". + If no setting is provided, "gemini-2.0-flash-001" will be used by default and a warning will be issued. .. note:: @@ -485,8 +477,6 @@ def __init__( "gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite", - "gemini-3.1-flash-lite", - "gemini-3.5-flash", ] ] = None, session: Optional[bigframes.Session] = None, @@ -505,7 +495,7 @@ def __init__( warnings.warn(msg, category=exceptions.PreviewWarning) if model_name is None: - model_name = "gemini-2.5-flash" + model_name = "gemini-2.0-flash-001" msg = exceptions.format_message(_REMOVE_DEFAULT_MODEL_WARNING) warnings.warn(msg, category=FutureWarning, stacklevel=2) @@ -614,10 +604,7 @@ def fit( options["prompt_col"] = X.columns.tolist()[0] self._bqml_model = self._bqml_model_factory.create_llm_remote_model( - X, - y, - options=options, - connection_name=typing.cast(str, self.connection_name), + X, y, options=options, connection_name=cast(str, self.connection_name) ) return self @@ -677,13 +664,13 @@ def predict( prompt (Iterable of str or bigframes.series.Series, or None, default None): .. note:: - BigFrames ObjectRef is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + BigFrames Blob is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" and might have limited support. For more information, see the launch stage descriptions (https://cloud.google.com/products#product-launch-stages). Construct a prompt struct column for prediction based on the input. The input must be an Iterable that can take string literals, - such as "summarize", string column(s) of X, such as X["str_col"], or `ObjectRef column(s) `_ of X, such as X["objectref_col"]. + such as "summarize", string column(s) of X, such as X["str_col"], or blob column(s) of X, such as X["blob_col"]. It creates a struct column of the items of the iterable, and use the concatenated result as the input prompt. No-op if set to None. output_schema (Mapping[str, str] or None, default None): The schema used to generate structured output as a bigframes DataFrame. The schema is a string key-value pair of :. @@ -740,7 +727,7 @@ def predict( isinstance(item, bigframes.series.Series) and item.dtype == dtypes.OBJ_REF_DTYPE ): - item = bbq.obj.get_access_url(item, mode="r") + item = item.blob._get_runtime("R", with_metadata=True) df_prompt[label] = item df_prompt = df_prompt.drop(columns="bigframes_placeholder_col") @@ -748,7 +735,7 @@ def predict( if len(X.columns) == 1: # BQML identified the column by name - col_label = typing.cast(blocks.Label, X.columns[0]) + col_label = cast(blocks.Label, X.columns[0]) X = X.rename(columns={col_label: "prompt"}) options: dict = { @@ -833,8 +820,8 @@ def score( ) # BQML identified the column by name - X_col_label = typing.cast(blocks.Label, X.columns[0]) - y_col_label = typing.cast(blocks.Label, y.columns[0]) + X_col_label = cast(blocks.Label, X.columns[0]) + y_col_label = cast(blocks.Label, y.columns[0]) X = X.rename(columns={X_col_label: "input_text"}) y = y.rename(columns={y_col_label: "output_text"}) @@ -886,7 +873,7 @@ class Claude3TextGenerator(base.RetriableRemotePredictor): The model for natural language tasks. Possible values are "claude-3-sonnet", "claude-3-haiku", "claude-3-5-sonnet" and "claude-3-opus". "claude-3-sonnet" (deprecated) is Anthropic's dependable combination of skills and speed. It is engineered to be dependable for scaled AI deployments across a variety of use cases. "claude-3-haiku" is Anthropic's fastest, most compact vision and text model for near-instant responses to simple queries, meant for seamless AI experiences mimicking human interactions. - "claude-3-5-sonnet" (deprecated) is Anthropic's most powerful AI model and maintains the speed and cost of Claude 3 Sonnet, which is a mid-tier model. + "claude-3-5-sonnet" is Anthropic's most powerful AI model and maintains the speed and cost of Claude 3 Sonnet, which is a mid-tier model. "claude-3-opus" (deprecated) is Anthropic's second-most powerful AI model, with strong performance on highly complex tasks. https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude#available-claude-models If no setting is provided, "claude-3-sonnet" will be used by default @@ -1046,7 +1033,7 @@ def predict( if len(X.columns) == 1: # BQML identified the column by name - col_label = typing.cast(blocks.Label, X.columns[0]) + col_label = cast(blocks.Label, X.columns[0]) X = X.rename(columns={col_label: "prompt"}) options = { diff --git a/bigframes/ml/loader.py b/bigframes/ml/loader.py index 76975752457..f6b5e4e2dc7 100644 --- a/bigframes/ml/loader.py +++ b/bigframes/ml/loader.py @@ -20,7 +20,6 @@ import bigframes_vendored.constants as constants from google.cloud import bigquery -import bigframes.session from bigframes.ml import ( cluster, compose, @@ -36,6 +35,7 @@ preprocessing, utils, ) +import bigframes.session _BQML_MODEL_TYPE_MAPPING = MappingProxyType( { @@ -70,8 +70,6 @@ llm._GEMINI_2P5_FLASH_ENDPOINT: llm.GeminiTextGenerator, llm._GEMINI_2P5_FLASH_LITE_ENDPOINT: llm.GeminiTextGenerator, llm._GEMINI_2P5_PRO_ENDPOINT: llm.GeminiTextGenerator, - llm._GEMINI_3P1_FLASH_LITE_ENDPOINT: llm.GeminiTextGenerator, - llm._GEMINI_3P5_FLASH_ENDPOINT: llm.GeminiTextGenerator, llm._CLAUDE_3_HAIKU_ENDPOINT: llm.Claude3TextGenerator, llm._CLAUDE_3_SONNET_ENDPOINT: llm.Claude3TextGenerator, llm._CLAUDE_3_5_SONNET_ENDPOINT: llm.Claude3TextGenerator, diff --git a/bigframes/ml/metrics/_metrics.py b/bigframes/ml/metrics/_metrics.py index 1f69d60e317..8787a68c58b 100644 --- a/bigframes/ml/metrics/_metrics.py +++ b/bigframes/ml/metrics/_metrics.py @@ -19,7 +19,7 @@ import inspect import typing -from typing import Literal, Tuple, Union, overload +from typing import Literal, overload, Tuple, Union import bigframes_vendored.constants as constants import bigframes_vendored.sklearn.metrics._classification as vendored_metrics_classification @@ -28,8 +28,8 @@ import numpy as np import pandas as pd -import bigframes.pandas as bpd from bigframes.ml import utils +import bigframes.pandas as bpd def r2_score( @@ -214,7 +214,7 @@ def confusion_matrix( y_true = row["y_true"] y_pred = row["y_pred"] count = row["dummy"] - confusion_matrix.at[y_true, y_pred] = count + confusion_matrix[y_pred][y_true] = count return confusion_matrix @@ -251,7 +251,7 @@ def recall_score( / is_accurate.groupby(y_true_series).count() ).to_pandas() - recall_score = pd.Series(0.0, index=index) + recall_score = pd.Series(0, index=index) for i in recall_score.index: recall_score.loc[i] = recall.loc[i] @@ -268,7 +268,8 @@ def precision_score( *, pos_label: int | float | bool | str = ..., average: Literal["binary"] = ..., -) -> float: ... +) -> float: + ... @overload @@ -278,7 +279,8 @@ def precision_score( *, pos_label: int | float | bool | str = ..., average: None = ..., -) -> pd.Series: ... +) -> pd.Series: + ... def precision_score( @@ -319,7 +321,7 @@ def _precision_score_per_label(y_true: bpd.Series, y_pred: bpd.Series) -> pd.Ser is_accurate.groupby(y_pred).sum() / is_accurate.groupby(y_pred).count() ).to_pandas() - precision_score = pd.Series(0.0, index=index) + precision_score = pd.Series(0, index=index) for i in precision.index: precision_score.loc[i] = precision.loc[i] @@ -364,7 +366,7 @@ def f1_score( recall = recall_score(y_true_series, y_pred_series, average=None) precision = precision_score(y_true_series, y_pred_series, average=None) - f1_score = pd.Series(0.0, index=recall.index) + f1_score = pd.Series(0, index=recall.index) for index in recall.index: if precision[index] + recall[index] != 0: f1_score[index] = ( diff --git a/bigframes/ml/metrics/pairwise.py b/bigframes/ml/metrics/pairwise.py index 41785a8462d..658eef15aa5 100644 --- a/bigframes/ml/metrics/pairwise.py +++ b/bigframes/ml/metrics/pairwise.py @@ -17,9 +17,9 @@ import bigframes_vendored.sklearn.metrics.pairwise as vendored_metrics_pairwise +from bigframes.ml import utils import bigframes.operations as ops import bigframes.pandas as bpd -from bigframes.ml import utils def paired_cosine_distances( diff --git a/bigframes/ml/model_selection.py b/bigframes/ml/model_selection.py index 57e07d89301..5adfb03b7f5 100644 --- a/bigframes/ml/model_selection.py +++ b/bigframes/ml/model_selection.py @@ -16,20 +16,19 @@ scikit-learn's model_selection module: https://scikit-learn.org/stable/modules/classes.html#module-sklearn.model_selection.""" + import inspect +from itertools import chain import time -import typing -from itertools import chain as _chain -from typing import Generator, List, Optional, Union +from typing import cast, Generator, List, Optional, Union import bigframes_vendored.sklearn.model_selection._split as vendored_model_selection_split import bigframes_vendored.sklearn.model_selection._validation as vendored_model_selection_validation import pandas as pd -import bigframes.pandas as bpd -from bigframes._tools import docs from bigframes.core.logging import log_adapter from bigframes.ml import utils +import bigframes.pandas as bpd def train_test_split( @@ -40,6 +39,7 @@ def train_test_split( stratify: Union[bpd.Series, None] = None, shuffle: bool = True, ) -> List[Union[bpd.DataFrame, bpd.Series]]: + if test_size is None: if train_size is None: test_size = 0.25 @@ -71,7 +71,7 @@ def train_test_split( test_rows = total_rows - train_rows return list( - _chain.from_iterable( + chain.from_iterable( [ [bf_array.head(train_rows), bf_array.tail(test_rows)] for bf_array in bf_arrays @@ -99,10 +99,10 @@ def _stratify_split(df: bpd.DataFrame, stratify: bpd.Series) -> List[bpd.DataFra train_dfs.append(train) test_dfs.append(test) - train_df = typing.cast( + train_df = cast( bpd.DataFrame, bpd.concat(train_dfs).drop(columns="bigframes_stratify_col") ) - test_df = typing.cast( + test_df = cast( bpd.DataFrame, bpd.concat(test_dfs).drop(columns="bigframes_stratify_col") ) return [train_df, test_df] @@ -132,8 +132,9 @@ def _stratify_split(df: bpd.DataFrame, stratify: bpd.Series) -> List[bpd.DataFra @log_adapter.class_logger -@docs.inherit_docs(vendored_model_selection_split.KFold) -class KFold: +class KFold(vendored_model_selection_split.KFold): + __doc__ = inspect.getdoc(vendored_model_selection_split.KFold) + def __init__(self, n_splits: int = 5, *, random_state: Union[int, None] = None): if n_splits < 2: raise ValueError(f"n_splits must be at least 2. Got {n_splits}") diff --git a/bigframes/ml/pipeline.py b/bigframes/ml/pipeline.py index 59057fb2faf..8d692176940 100644 --- a/bigframes/ml/pipeline.py +++ b/bigframes/ml/pipeline.py @@ -15,6 +15,7 @@ """For composing estimators together. This module is styled after scikit-learn's pipeline module: https://scikit-learn.org/stable/modules/pipeline.html.""" + from __future__ import annotations from typing import List, Optional, Tuple @@ -23,9 +24,8 @@ import bigframes_vendored.sklearn.pipeline from google.cloud import bigquery -import bigframes.dataframe -import bigframes.session from bigframes.core.logging import log_adapter +import bigframes.dataframe from bigframes.ml import ( base, compose, @@ -35,6 +35,7 @@ preprocessing, utils, ) +import bigframes.session @log_adapter.class_logger diff --git a/bigframes/ml/preprocessing.py b/bigframes/ml/preprocessing.py index 28272fd6a02..8bf89b08387 100644 --- a/bigframes/ml/preprocessing.py +++ b/bigframes/ml/preprocessing.py @@ -18,7 +18,7 @@ from __future__ import annotations import typing -from typing import Iterable, List, Literal, Optional, Union +from typing import cast, Iterable, List, Literal, Optional, Union import bigframes_vendored.sklearn.preprocessing._data import bigframes_vendored.sklearn.preprocessing._discretization @@ -26,10 +26,10 @@ import bigframes_vendored.sklearn.preprocessing._label import bigframes_vendored.sklearn.preprocessing._polynomial -import bigframes.core.utils as core_utils -import bigframes.pandas as bpd from bigframes.core.logging import log_adapter +import bigframes.core.utils as core_utils from bigframes.ml import base, core, globals, utils +import bigframes.pandas as bpd @log_adapter.class_logger @@ -273,7 +273,9 @@ class KBinsDiscretizer( base.Transformer, bigframes_vendored.sklearn.preprocessing._discretization.KBinsDiscretizer, ): - __doc__ = bigframes_vendored.sklearn.preprocessing._discretization.KBinsDiscretizer.__doc__ + __doc__ = ( + bigframes_vendored.sklearn.preprocessing._discretization.KBinsDiscretizer.__doc__ + ) def __init__( self, @@ -326,6 +328,7 @@ def _compile_to_sql( ] elif self.strategy == "quantile": + return [ self._base_sql_generator.ml_quantile_bucketize( column, self.n_bins, f"kbinsdiscretizer_{column}" @@ -467,7 +470,7 @@ def _parse_from_sql(cls, sql: str) -> tuple[OneHotEncoder, str]: s = sql[sql.find("(") + 1 : sql.find(")")] col_label, drop_str, top_k, frequency_threshold = s.split(", ") drop = ( - typing.cast(Literal["most_frequent"], "most_frequent") + cast(Literal["most_frequent"], "most_frequent") if drop_str.lower() == "'most_frequent'" else None ) diff --git a/bigframes/ml/remote.py b/bigframes/ml/remote.py index f53ea645e92..24083bd4e88 100644 --- a/bigframes/ml/remote.py +++ b/bigframes/ml/remote.py @@ -16,15 +16,15 @@ from __future__ import annotations -import warnings from typing import Mapping, Optional +import warnings -import bigframes.dataframe -import bigframes.exceptions as bfe -import bigframes.session from bigframes.core import global_session from bigframes.core.logging import log_adapter +import bigframes.dataframe +import bigframes.exceptions as bfe from bigframes.ml import base, core, globals, utils +import bigframes.session _REMOTE_MODEL_STATUS = "remote_model_status" diff --git a/bigframes/ml/sql.py b/bigframes/ml/sql.py index 894fc44b1b3..2937368c92c 100644 --- a/bigframes/ml/sql.py +++ b/bigframes/ml/sql.py @@ -21,7 +21,8 @@ import bigframes_vendored.constants as constants import google.cloud.bigquery -from bigframes.core.compile.sqlglot import sql as sg_sql +import bigframes.core.compile.googlesql as sql_utils +import bigframes.core.sql as sql_vals INDENT_STR = " " @@ -34,7 +35,7 @@ class BaseSqlGenerator: def encode_value(self, v: Union[str, int, float, Iterable[str]]) -> str: """Encode a parameter value for SQL""" if isinstance(v, (str, int, float)): - return sg_sql.to_sql(sg_sql.literal(v)) + return sql_vals.simple_literal(v) elif isinstance(v, Iterable): inner = ", ".join([self.encode_value(x) for x in v]) return f"[{inner}]" @@ -61,7 +62,7 @@ def build_structs(self, **kwargs: Union[int, float, str, Mapping]) -> str: v_trans = self.build_schema(**v) if isinstance(v, Mapping) else v param_strs.append( - f"{sg_sql.to_sql(sg_sql.literal(v_trans))} AS {sg_sql.to_sql(sg_sql.identifier(k))}" + f"{sql_vals.simple_literal(v_trans)} AS {sql_utils.identifier(k)}" ) return "\n" + INDENT_STR + f",\n{INDENT_STR}".join(param_strs) @@ -72,9 +73,7 @@ def build_expressions(self, *expr_sqls: str) -> str: def build_schema(self, **kwargs: str) -> str: """Encode a dict of values into a formatted schema type items for SQL""" - param_strs = [ - f"{sg_sql.to_sql(sg_sql.identifier(k))} {v}" for k, v in kwargs.items() - ] + param_strs = [f"{sql_utils.identifier(k)} {v}" for k, v in kwargs.items()] return "\n" + INDENT_STR + f",\n{INDENT_STR}".join(param_strs) def options(self, **kwargs: Union[str, int, float, Iterable[str]]) -> str: @@ -87,9 +86,7 @@ def struct_options(self, **kwargs: Union[int, float, Mapping]) -> str: def struct_columns(self, columns: Iterable[str]) -> str: """Encode a BQ Table columns to a STRUCT.""" - columns_str = ", ".join( - map(lambda x: sg_sql.to_sql(sg_sql.identifier(x)), columns) - ) + columns_str = ", ".join(map(sql_utils.identifier, columns)) return f"STRUCT({columns_str})" def input(self, **kwargs: str) -> str: @@ -112,15 +109,15 @@ def transform(self, *expr_sqls: str) -> str: def ml_standard_scaler(self, numeric_expr_sql: str, name: str) -> str: """Encode ML.STANDARD_SCALER for BQML""" - return f"""ML.STANDARD_SCALER({sg_sql.to_sql(sg_sql.identifier(numeric_expr_sql))}) OVER() AS {sg_sql.to_sql(sg_sql.identifier(name))}""" + return f"""ML.STANDARD_SCALER({sql_utils.identifier(numeric_expr_sql)}) OVER() AS {sql_utils.identifier(name)}""" def ml_max_abs_scaler(self, numeric_expr_sql: str, name: str) -> str: """Encode ML.MAX_ABS_SCALER for BQML""" - return f"""ML.MAX_ABS_SCALER({sg_sql.to_sql(sg_sql.identifier(numeric_expr_sql))}) OVER() AS {sg_sql.to_sql(sg_sql.identifier(name))}""" + return f"""ML.MAX_ABS_SCALER({sql_utils.identifier(numeric_expr_sql)}) OVER() AS {sql_utils.identifier(name)}""" def ml_min_max_scaler(self, numeric_expr_sql: str, name: str) -> str: """Encode ML.MIN_MAX_SCALER for BQML""" - return f"""ML.MIN_MAX_SCALER({sg_sql.to_sql(sg_sql.identifier(numeric_expr_sql))}) OVER() AS {sg_sql.to_sql(sg_sql.identifier(name))}""" + return f"""ML.MIN_MAX_SCALER({sql_utils.identifier(numeric_expr_sql)}) OVER() AS {sql_utils.identifier(name)}""" def ml_imputer( self, @@ -129,7 +126,7 @@ def ml_imputer( name: str, ) -> str: """Encode ML.IMPUTER for BQML""" - return f"""ML.IMPUTER({sg_sql.to_sql(sg_sql.identifier(col_name))}, '{strategy}') OVER() AS {sg_sql.to_sql(sg_sql.identifier(name))}""" + return f"""ML.IMPUTER({sql_utils.identifier(col_name)}, '{strategy}') OVER() AS {sql_utils.identifier(name)}""" def ml_bucketize( self, @@ -143,7 +140,7 @@ def ml_bucketize( point.item() if hasattr(point, "item") else point for point in array_split_points ] - return f"""ML.BUCKETIZE({sg_sql.to_sql(sg_sql.identifier(input_id))}, {points}, FALSE) AS {sg_sql.to_sql(sg_sql.identifier(output_id))}""" + return f"""ML.BUCKETIZE({sql_utils.identifier(input_id)}, {points}, FALSE) AS {sql_utils.identifier(output_id)}""" def ml_quantile_bucketize( self, @@ -152,7 +149,7 @@ def ml_quantile_bucketize( name: str, ) -> str: """Encode ML.QUANTILE_BUCKETIZE for BQML""" - return f"""ML.QUANTILE_BUCKETIZE({sg_sql.to_sql(sg_sql.identifier(numeric_expr_sql))}, {num_bucket}) OVER() AS {sg_sql.to_sql(sg_sql.identifier(name))}""" + return f"""ML.QUANTILE_BUCKETIZE({sql_utils.identifier(numeric_expr_sql)}, {num_bucket}) OVER() AS {sql_utils.identifier(name)}""" def ml_one_hot_encoder( self, @@ -163,9 +160,8 @@ def ml_one_hot_encoder( name: str, ) -> str: """Encode ML.ONE_HOT_ENCODER for BQML. - https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-one-hot-encoder for params. - """ - return f"""ML.ONE_HOT_ENCODER({sg_sql.to_sql(sg_sql.identifier(numeric_expr_sql))}, '{drop}', {top_k}, {frequency_threshold}) OVER() AS {sg_sql.to_sql(sg_sql.identifier(name))}""" + https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-one-hot-encoder for params.""" + return f"""ML.ONE_HOT_ENCODER({sql_utils.identifier(numeric_expr_sql)}, '{drop}', {top_k}, {frequency_threshold}) OVER() AS {sql_utils.identifier(name)}""" def ml_label_encoder( self, @@ -175,17 +171,15 @@ def ml_label_encoder( name: str, ) -> str: """Encode ML.LABEL_ENCODER for BQML. - https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-label-encoder for params. - """ - return f"""ML.LABEL_ENCODER({sg_sql.to_sql(sg_sql.identifier(numeric_expr_sql))}, {top_k}, {frequency_threshold}) OVER() AS {sg_sql.to_sql(sg_sql.identifier(name))}""" + https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-label-encoder for params.""" + return f"""ML.LABEL_ENCODER({sql_utils.identifier(numeric_expr_sql)}, {top_k}, {frequency_threshold}) OVER() AS {sql_utils.identifier(name)}""" def ml_polynomial_expand( self, columns: Iterable[str], degree: int, name: str ) -> str: """Encode ML.POLYNOMIAL_EXPAND. - https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-polynomial-expand - """ - return f"""ML.POLYNOMIAL_EXPAND({self.struct_columns(columns)}, {degree}) AS {sg_sql.to_sql(sg_sql.identifier(name))}""" + https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-polynomial-expand""" + return f"""ML.POLYNOMIAL_EXPAND({self.struct_columns(columns)}, {degree}) AS {sql_utils.identifier(name)}""" def ml_distance( self, @@ -196,9 +190,8 @@ def ml_distance( name: str, ) -> str: """Encode ML.DISTANCE for BQML. - https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-distance - """ - return f"""SELECT *, ML.DISTANCE({sg_sql.to_sql(sg_sql.identifier(col_x))}, {sg_sql.to_sql(sg_sql.identifier(col_y))}, '{type}') AS {sg_sql.to_sql(sg_sql.identifier(name))} FROM ({source_sql})""" + https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-distance""" + return f"""SELECT *, ML.DISTANCE({sql_utils.identifier(col_x)}, {sql_utils.identifier(col_y)}, '{type}') AS {sql_utils.identifier(name)} FROM ({source_sql})""" def ai_forecast( self, @@ -206,8 +199,7 @@ def ai_forecast( options: Mapping[str, Union[int, float, bool, Iterable[str]]], ): """Encode AI.FORECAST. - https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-ai-forecast - """ + https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-ai-forecast""" named_parameters_sql = self.build_named_parameters(**options) return f"""SELECT * FROM AI.FORECAST(({source_sql}),{named_parameters_sql})""" @@ -220,7 +212,7 @@ def _model_id_sql( self, model_ref: google.cloud.bigquery.ModelReference, ): - return f"{sg_sql.to_sql(sg_sql.identifier(model_ref.project))}.{sg_sql.to_sql(sg_sql.identifier(model_ref.dataset_id))}.{sg_sql.to_sql(sg_sql.identifier(model_ref.model_id))}" + return f"{sql_utils.identifier(model_ref.project)}.{sql_utils.identifier(model_ref.dataset_id)}.{sql_utils.identifier(model_ref.model_id)}" # Model create and alter def create_model( @@ -311,7 +303,7 @@ def __init__(self, model_ref: google.cloud.bigquery.ModelReference): self._model_ref = model_ref def _model_ref_sql(self) -> str: - return f"{sg_sql.to_sql(sg_sql.identifier(self._model_ref.project))}.{sg_sql.to_sql(sg_sql.identifier(self._model_ref.dataset_id))}.{sg_sql.to_sql(sg_sql.identifier(self._model_ref.model_id))}" + return f"{sql_utils.identifier(self._model_ref.project)}.{sql_utils.identifier(self._model_ref.dataset_id)}.{sql_utils.identifier(self._model_ref.model_id)}" # Alter model def alter_model( diff --git a/bigframes/ml/utils.py b/bigframes/ml/utils.py index 134020b7167..f97dd561be0 100644 --- a/bigframes/ml/utils.py +++ b/bigframes/ml/utils.py @@ -26,11 +26,11 @@ ) import bigframes_vendored.constants as constants -import pandas as pd from google.cloud import bigquery +import pandas as pd -import bigframes.pandas as bpd from bigframes.core import convert, guid +import bigframes.pandas as bpd from bigframes.session import Session # Internal type alias diff --git a/bigframes/operations/__init__.py b/bigframes/operations/__init__.py index 6df8da69b11..a1c7754ab5c 100644 --- a/bigframes/operations/__init__.py +++ b/bigframes/operations/__init__.py @@ -16,17 +16,15 @@ from bigframes.operations.ai_ops import ( AIClassify, - AIEmbed, AIGenerate, AIGenerateBool, AIGenerateDouble, AIGenerateInt, AIIf, AIScore, - AISimilarity, ) from bigframes.operations.array_ops import ( - ArrayMapOp, + ArrayIndexOp, ArrayReduceOp, ArraySliceOp, ArrayToStringOp, @@ -41,10 +39,10 @@ UnaryOp, ) from bigframes.operations.blob_ops import ( - ObjGetAccessUrl, obj_fetch_metadata_op, obj_make_ref_json_op, obj_make_ref_op, + ObjGetAccessUrl, ) from bigframes.operations.bool_ops import and_op, or_op, xor_op from bigframes.operations.comparison_ops import ( @@ -69,15 +67,15 @@ year_op, ) from bigframes.operations.datetime_ops import ( + date_op, StrftimeOp, + time_op, + timestamp_diff_op, ToDatetimeOp, ToTimestampOp, UnixMicros, UnixMillis, UnixSeconds, - date_op, - time_op, - timestamp_diff_op, ) from bigframes.operations.distance_ops import ( cosine_distance_op, @@ -91,35 +89,28 @@ ) from bigframes.operations.generic_ops import ( AsTypeOp, - CaseWhenOp, - CoerceToBoolOp, - DynamicGetItemOp, - GetItemOp, - IsInOp, - MapOp, - RowKey, - SqlScalarOp, case_when_op, + CaseWhenOp, clip_op, coalesce_op, - coerce_to_bool_op, fillna_op, hash_op, invert_op, + IsInOp, isnull_op, + MapOp, maximum_op, minimum_op, notnull_op, + RowKey, + SqlScalarOp, where_op, ) from bigframes.operations.geo_ops import ( - GeoStBufferOp, - GeoStDistanceOp, - GeoStLengthOp, - GeoStRegionStatsOp, - GeoStSimplifyOp, + geo_area_op, geo_st_astext_op, geo_st_boundary_op, + geo_st_centroid_op, geo_st_convexhull_op, geo_st_difference_op, geo_st_geogfromtext_op, @@ -128,10 +119,13 @@ geo_st_isclosed_op, geo_x_op, geo_y_op, + GeoStBufferOp, + GeoStDistanceOp, + GeoStLengthOp, + GeoStRegionStatsOp, + GeoStSimplifyOp, ) -from bigframes.operations.googlesql import GoogleSqlScalarOp from bigframes.operations.json_ops import ( - JSONDecode, JSONExtract, JSONExtractArray, JSONExtractStringArray, @@ -146,13 +140,9 @@ ToJSONString, ) from bigframes.operations.numeric_ops import ( - AddOp, - DivOp, - FloorDivOp, - MulOp, - SubOp, abs_op, add_op, + AddOp, arccos_op, arccosh_op, arcsin_op, @@ -164,15 +154,18 @@ cos_op, cosh_op, div_op, + DivOp, exp_op, expm1_op, floor_op, floordiv_op, + FloorDivOp, ln_op, log1p_op, log10_op, mod_op, mul_op, + MulOp, neg_op, pos_op, pow_op, @@ -181,24 +174,40 @@ sinh_op, sqrt_op, sub_op, + SubOp, tan_op, tanh_op, unsafe_pow_op, ) from bigframes.operations.numpy_op_maps import NUMPY_TO_BINOP, NUMPY_TO_OP from bigframes.operations.remote_function_ops import ( - PythonUdfOp, + BinaryRemoteFunctionOp, + NaryRemoteFunctionOp, RemoteFunctionOp, ) from bigframes.operations.string_ops import ( + capitalize_op, EndsWithOp, + isalnum_op, + isalpha_op, + isdecimal_op, + isdigit_op, + islower_op, + isnumeric_op, + isspace_op, + isupper_op, + len_op, + lower_op, RegexReplaceStrOp, ReplaceStrOp, + reverse_op, StartsWithOp, + strconcat_op, StrContainsOp, StrContainsRegexOp, StrExtractOp, StrFindOp, + StrGetOp, StringSplitOp, StrLstripOp, StrPadOp, @@ -206,33 +215,19 @@ StrRstripOp, StrSliceOp, StrStripOp, - ZfillOp, - capitalize_op, - isalnum_op, - isalpha_op, - isdecimal_op, - isdigit_op, - islower_op, - isnumeric_op, - isspace_op, - isupper_op, - len_op, - lower_op, - reverse_op, - strconcat_op, upper_op, + ZfillOp, ) from bigframes.operations.struct_ops import StructFieldOp, StructOp from bigframes.operations.time_ops import hour_op, minute_op, normalize_op, second_op from bigframes.operations.timedelta_ops import ( - ToTimedeltaOp, date_add_op, date_sub_op, timedelta_floor_op, timestamp_add_op, timestamp_sub_op, + ToTimedeltaOp, ) -from bigframes.operations.to_op import func_to_expr __all__ = [ # Base ops @@ -249,8 +244,6 @@ "clip_op", "coalesce_op", "fillna_op", - "DynamicGetItemOp", - "GetItemOp", "hash_op", "invert_op", "IsInOp", @@ -259,8 +252,6 @@ "maximum_op", "minimum_op", "notnull_op", - "CoerceToBoolOp", - "coerce_to_bool_op", "RowKey", "SqlScalarOp", "where_op", @@ -286,6 +277,7 @@ "StrContainsRegexOp", "StrExtractOp", "StrFindOp", + "StrGetOp", "StrLstripOp", "StringSplitOp", "strip_op", @@ -369,6 +361,7 @@ "tanh_op", "unsafe_pow_op", # Array ops + "ArrayIndexOp", "ArraySliceOp", "ArrayToStringOp", # Blob ops @@ -380,14 +373,14 @@ "StructFieldOp", "StructOp", # Remote Functions ops + "BinaryRemoteFunctionOp", + "NaryRemoteFunctionOp", "RemoteFunctionOp", - "PythonUdfOp", # Frequency ops "DatetimeToIntegerLabelOp", "FloorDtOp", "IntegerLabelToDatetimeOp", # JSON ops - "JSONDecode", "JSONExtract", "JSONExtractArray", "JSONExtractStringArray", @@ -417,7 +410,9 @@ "euclidean_distance_op", "manhattan_distance_op", # Geo ops + "geo_area_op", "geo_st_boundary_op", + "geo_st_centroid_op", "geo_st_convexhull_op", "geo_st_difference_op", "geo_st_astext_op", @@ -438,18 +433,11 @@ "AIGenerateBool", "AIGenerateDouble", "AIGenerateInt", - "AIEmbed", "AIIf", "AIScore", - "AISimilarity", - # Helper functions - "func_to_expr", # Numpy ops mapping "NUMPY_TO_BINOP", "NUMPY_TO_OP", "ToArrayOp", "ArrayReduceOp", - "ArrayMapOp", - # GoogleSql - "GoogleSqlScalarOp", ] diff --git a/bigframes/operations/_op_converters.py b/bigframes/operations/_op_converters.py index 14417a24f6e..3ebf22bcb6a 100644 --- a/bigframes/operations/_op_converters.py +++ b/bigframes/operations/_op_converters.py @@ -15,10 +15,10 @@ import bigframes.operations as ops -def convert_index(key: int) -> ops.GetItemOp: +def convert_index(key: int) -> ops.ArrayIndexOp: if key < 0: raise NotImplementedError("Negative indexing is not supported.") - return ops.GetItemOp(key=key) + return ops.ArrayIndexOp(index=key) def convert_slice(key: slice) -> ops.ArraySliceOp: diff --git a/bigframes/operations/aggregations.py b/bigframes/operations/aggregations.py index f7b89b949a8..5fe83302638 100644 --- a/bigframes/operations/aggregations.py +++ b/bigframes/operations/aggregations.py @@ -17,15 +17,15 @@ import abc import dataclasses import typing -from typing import TYPE_CHECKING, Callable, ClassVar, Iterable, Optional +from typing import Callable, ClassVar, Iterable, Optional, TYPE_CHECKING import numpy as np import pandas as pd import pyarrow as pa +from bigframes.core import agg_expressions import bigframes.dtypes as dtypes import bigframes.operations.type as signatures -from bigframes.core import agg_expressions if TYPE_CHECKING: from bigframes.core import expression @@ -65,9 +65,8 @@ def order_independent(self): return False @abc.abstractmethod - def output_type( - self, *input_types: dtypes.ExpressionType - ) -> dtypes.ExpressionType: ... + def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: + ... @property def can_be_windowized(self): @@ -98,11 +97,13 @@ class AggregateOp(WindowOp): @property @abc.abstractmethod - def name(self) -> str: ... + def name(self) -> str: + ... @property @abc.abstractmethod - def arguments(self) -> int: ... + def arguments(self) -> int: + ... @property def order_independent(self): @@ -204,7 +205,7 @@ def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionT return dtypes.TIMEDELTA_DTYPE if dtypes.is_numeric(input_types[0]): - if pd.api.types.is_bool_dtype(input_types[0]): # type: ignore + if pd.api.types.is_bool_dtype(input_types[0]): return dtypes.INT_DTYPE return input_types[0] @@ -223,7 +224,7 @@ def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionT # These will change if median is changed to exact implementation. if not dtypes.is_orderable(input_types[0]): raise TypeError(f"Type {input_types[0]} is not orderable") - if pd.api.types.is_bool_dtype(input_types[0]): # type: ignore + if pd.api.types.is_bool_dtype(input_types[0]): return dtypes.INT_DTYPE else: return input_types[0] diff --git a/bigframes/operations/ai.py b/bigframes/operations/ai.py new file mode 100644 index 00000000000..6921299acd8 --- /dev/null +++ b/bigframes/operations/ai.py @@ -0,0 +1,846 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import re +import typing +from typing import Dict, Iterable, List, Optional, Sequence, Union +import warnings + +from bigframes import dtypes, exceptions, options +from bigframes.core import guid +from bigframes.core.logging import log_adapter + + +@log_adapter.class_logger +class AIAccessor: + def __init__(self, df, base_bqml=None) -> None: + import bigframes # Import in the function body to avoid circular imports. + import bigframes.dataframe + from bigframes.ml import core as ml_core + + self._df: bigframes.dataframe.DataFrame = df + self._base_bqml: ml_core.BaseBqml = base_bqml or ml_core.BaseBqml(df._session) + + def filter( + self, + instruction: str, + model, + ground_with_google_search: bool = False, + ): + """ + Filters the DataFrame with the semantics of the user instruction. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.ai_operators = True + >>> bpd.options.compute.ai_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.GeminiTextGenerator(model_name="gemini-2.0-flash-001") + + >>> df = bpd.DataFrame({"country": ["USA", "Germany"], "city": ["Seattle", "Berlin"]}) + >>> df.ai.filter("{city} is the capital of {country}", model) + country city + 1 Germany Berlin + + [1 rows x 2 columns] + + Args: + instruction (str): + An instruction on how to filter the data. This value must contain + column references by name, which should be wrapped in a pair of braces. + For example, if you have a column "food", you can refer to this column + in the instructions like: + "The {food} is healthy." + + model (bigframes.ml.llm.GeminiTextGenerator): + A GeminiTextGenerator provided by Bigframes ML package. + + ground_with_google_search (bool, default False): + Enables Grounding with Google Search for the GeminiTextGenerator model. + When set to True, the model incorporates relevant information from Google + Search results into its responses, enhancing their accuracy and factualness. + Note: Using this feature may impact billing costs. Refer to the pricing + page for details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models + The default is `False`. + + Returns: + bigframes.pandas.DataFrame: DataFrame filtered by the instruction. + + Raises: + NotImplementedError: when the AI operator experiment is off. + ValueError: when the instruction refers to a non-existing column, or when no + columns are referred to. + """ + if not options.experiments.ai_operators: + raise NotImplementedError() + + answer_col = "answer" + + output_schema = {answer_col: "bool"} + result = self.map( + instruction, + model, + output_schema, + ground_with_google_search, + ) + + return result[result[answer_col]].drop(answer_col, axis=1) + + def map( + self, + instruction: str, + model, + output_schema: Dict[str, str] | None = None, + ground_with_google_search: bool = False, + ): + """ + Maps the DataFrame with the semantics of the user instruction. The name of the keys in the output_schema parameter carry + semantic meaning, and can be used for information extraction. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.ai_operators = True + >>> bpd.options.compute.ai_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.GeminiTextGenerator(model_name="gemini-2.0-flash-001") + + >>> df = bpd.DataFrame({"ingredient_1": ["Burger Bun", "Soy Bean"], "ingredient_2": ["Beef Patty", "Bittern"]}) + >>> df.ai.map("What is the food made from {ingredient_1} and {ingredient_2}? One word only.", model=model, output_schema={"food": "string"}) + ingredient_1 ingredient_2 food + 0 Burger Bun Beef Patty Burger + + 1 Soy Bean Bittern Tofu + + + [2 rows x 3 columns] + + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.ai_operators = True + >>> bpd.options.compute.ai_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.GeminiTextGenerator(model_name="gemini-2.0-flash-001") + + >>> df = bpd.DataFrame({"text": ["Elmo lives at 123 Sesame Street."]}) + >>> df.ai.map("{text}", model=model, output_schema={"person": "string", "address": "string"}) + text person address + 0 Elmo lives at 123 Sesame Street. Elmo 123 Sesame Street + + [1 rows x 3 columns] + + Args: + instruction (str): + An instruction on how to map the data. This value must contain + column references by name, which should be wrapped in a pair of braces. + For example, if you have a column "food", you can refer to this column + in the instructions like: + "Get the ingredients of {food}." + + model (bigframes.ml.llm.GeminiTextGenerator): + A GeminiTextGenerator provided by Bigframes ML package. + + output_schema (Dict[str, str] or None, default None): + The schema used to generate structured output as a bigframes DataFrame. The schema is a string key-value pair of :. + Supported types are int64, float64, bool, string, array and struct. If None, generate string result under the column + "ml_generate_text_llm_result". + + ground_with_google_search (bool, default False): + Enables Grounding with Google Search for the GeminiTextGenerator model. + When set to True, the model incorporates relevant information from Google + Search results into its responses, enhancing their accuracy and factualness. + Note: Using this feature may impact billing costs. Refer to the pricing + page for details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models + The default is `False`. + + Returns: + bigframes.pandas.DataFrame: DataFrame with attached mapping results. + + Raises: + NotImplementedError: when the AI operator experiment is off. + ValueError: when the instruction refers to a non-existing column, or when no + columns are referred to. + """ + if not options.experiments.ai_operators: + raise NotImplementedError() + + import bigframes.dataframe + import bigframes.series + + self._validate_model(model) + columns = self._parse_columns(instruction) + for column in columns: + if column not in self._df.columns: + raise ValueError(f"Column {column} not found.") + + if ground_with_google_search: + msg = exceptions.format_message( + "Enables Grounding with Google Search may impact billing cost. See pricing " + "details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models" + ) + warnings.warn(msg, category=UserWarning) + + self._confirm_operation(len(self._df)) + + df: bigframes.dataframe.DataFrame = self._df[columns].copy() + has_blob_column = False + for column in columns: + if df[column].dtype == dtypes.OBJ_REF_DTYPE: + # Don't cast blob columns to string + has_blob_column = True + continue + + if df[column].dtype != dtypes.STRING_DTYPE: + df[column] = df[column].astype(dtypes.STRING_DTYPE) + + user_instruction = self._format_instruction(instruction, columns) + output_instruction = ( + "Based on the provided contenxt, answer the following instruction:" + ) + + if output_schema is None: + output_schema = {"ml_generate_text_llm_result": "string"} + + if has_blob_column: + results = typing.cast( + bigframes.series.Series, + model.predict( + df, + prompt=self._make_multimodel_prompt( + df, columns, user_instruction, output_instruction + ), + temperature=0.0, + ground_with_google_search=ground_with_google_search, + output_schema=output_schema, + ), + ) + else: + results = typing.cast( + bigframes.series.Series, + model.predict( + self._make_text_prompt( + df, columns, user_instruction, output_instruction + ), + temperature=0.0, + ground_with_google_search=ground_with_google_search, + output_schema=output_schema, + ), + ) + + attach_columns = [results[col] for col, _ in output_schema.items()] + + from bigframes.core.reshape.api import concat + + return concat([self._df, *attach_columns], axis=1) + + def classify( + self, + instruction: str, + model, + labels: Sequence[str], + output_column: str = "result", + ground_with_google_search: bool = False, + ): + """ + Classifies the rows of dataframes based on user instruction into the provided labels. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.ai_operators = True + >>> bpd.options.compute.ai_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.GeminiTextGenerator(model_name="gemini-2.0-flash-001") + + >>> df = bpd.DataFrame({ + ... "feedback_text": [ + ... "The product is amazing, but the shipping was slow.", + ... "I had an issue with my recent bill.", + ... "The user interface is very intuitive." + ... ], + ... }) + >>> df.ai.classify("{feedback_text}", model=model, labels=["Shipping", "Billing", "UI"]) + feedback_text result + 0 The product is amazing, but the shipping was s... Shipping + 1 I had an issue with my recent bill. Billing + 2 The user interface is very intuitive. UI + + [3 rows x 2 columns] + + Args: + instruction (str): + An instruction on how to classify the data. This value must contain + column references by name, which should be wrapped in a pair of braces. + For example, if you have a column "feedback", you can refer to this column + with"{food}". + + model (bigframes.ml.llm.GeminiTextGenerator): + A GeminiTextGenerator provided by Bigframes ML package. + + labels (Sequence[str]): + A collection of labels (categories). It must contain at least two and at most 20 elements. + Labels are case sensitive. Duplicated labels are not allowed. + + output_column (str, default "result"): + The name of column for the output. + + ground_with_google_search (bool, default False): + Enables Grounding with Google Search for the GeminiTextGenerator model. + When set to True, the model incorporates relevant information from Google + Search results into its responses, enhancing their accuracy and factualness. + Note: Using this feature may impact billing costs. Refer to the pricing + page for details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models + The default is `False`. + + Returns: + bigframes.pandas.DataFrame: DataFrame with classification result. + + Raises: + NotImplementedError: when the AI operator experiment is off. + ValueError: when the instruction refers to a non-existing column, when no + columns are referred to, or when the count of labels does not meet the + requirement. + """ + if not options.experiments.ai_operators: + raise NotImplementedError() + + if len(labels) < 2 or len(labels) > 20: + raise ValueError( + f"The number of labels should be between 2 and 20 (inclusive), but {len(labels)} labels are provided." + ) + + if len(set(labels)) != len(labels): + raise ValueError("There are duplicate labels.") + + updated_instruction = f"Based on the user instruction {instruction}, you must provide an answer that must exist in the following list of labels: {labels}" + + return self.map( + updated_instruction, + model, + output_schema={output_column: "string"}, + ground_with_google_search=ground_with_google_search, + ) + + def join( + self, + other, + instruction: str, + model, + ground_with_google_search: bool = False, + ): + """ + Joines two dataframes by applying the instruction over each pair of rows from + the left and right table. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.ai_operators = True + >>> bpd.options.compute.ai_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.GeminiTextGenerator(model_name="gemini-2.0-flash-001") + + >>> cities = bpd.DataFrame({'city': ['Seattle', 'Ottawa', 'Berlin', 'Shanghai', 'New Delhi']}) + >>> continents = bpd.DataFrame({'continent': ['North America', 'Africa', 'Asia']}) + + >>> cities.ai.join(continents, "{city} is in {continent}", model) + city continent + 0 Seattle North America + 1 Ottawa North America + 2 Shanghai Asia + 3 New Delhi Asia + + [4 rows x 2 columns] + + Args: + other (bigframes.pandas.DataFrame): + The other dataframe. + + instruction (str): + An instruction on how left and right rows can be joined. This value must contain + column references by name. which should be wrapped in a pair of braces. + For example: "The {city} belongs to the {country}". + For column names that are shared between two dataframes, you need to add "left." + and "right." prefix for differentiation. This is especially important when you do + self joins. For example: "The {left.employee_name} reports to {right.employee_name}" + For unique column names, this prefix is optional. + + model (bigframes.ml.llm.GeminiTextGenerator): + A GeminiTextGenerator provided by Bigframes ML package. + + ground_with_google_search (bool, default False): + Enables Grounding with Google Search for the GeminiTextGenerator model. + When set to True, the model incorporates relevant information from Google + Search results into its responses, enhancing their accuracy and factualness. + Note: Using this feature may impact billing costs. Refer to the pricing + page for details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models + The default is `False`. + + Returns: + bigframes.pandas.DataFrame: The joined dataframe. + + Raises: + ValueError if the amount of data that will be sent for LLM processing is larger than max_rows. + """ + if not options.experiments.ai_operators: + raise NotImplementedError() + + self._validate_model(model) + columns = self._parse_columns(instruction) + + if ground_with_google_search: + msg = exceptions.format_message( + "Enables Grounding with Google Search may impact billing cost. See pricing " + "details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models" + ) + warnings.warn(msg, category=UserWarning) + + work_estimate = len(self._df) * len(other) + self._confirm_operation(work_estimate) + + left_columns = [] + right_columns = [] + + for col in columns: + if col in self._df.columns and col in other.columns: + raise ValueError(f"Ambiguous column reference: {col}") + + elif col in self._df.columns: + left_columns.append(col) + + elif col in other.columns: + right_columns.append(col) + + elif col.startswith("left."): + original_col_name = col[len("left.") :] + if ( + original_col_name in self._df.columns + and original_col_name in other.columns + ): + left_columns.append(col) + elif original_col_name in self._df.columns: + left_columns.append(col) + instruction = instruction.replace(col, original_col_name) + else: + raise ValueError(f"Column {col} not found") + + elif col.startswith("right."): + original_col_name = col[len("right.") :] + if ( + original_col_name in self._df.columns + and original_col_name in other.columns + ): + right_columns.append(col) + elif original_col_name in other.columns: + right_columns.append(col) + instruction = instruction.replace(col, original_col_name) + else: + raise ValueError(f"Column {col} not found") + + else: + raise ValueError(f"Column {col} not found") + + if not left_columns: + raise ValueError("No left column references.") + + if not right_columns: + raise ValueError("No right column references.") + + # Update column references to be compatible with internal naming scheme. + # That is, "left.col" -> "col_left" and "right.col" -> "col_right" + instruction = re.sub(r"(?>> import bigframes.pandas as bpd + + >>> import bigframes + >>> bigframes.options.experiments.ai_operators = True + >>> bpd.options.compute.ai_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.TextEmbeddingGenerator(model_name="text-embedding-005") + + >>> df = bpd.DataFrame({"creatures": ["salmon", "sea urchin", "frog", "chimpanzee"]}) + >>> df.ai.search("creatures", "monkey", top_k=1, model=model, score_column='distance') + creatures distance + 3 chimpanzee 0.635844 + + [1 rows x 2 columns] + + Args: + search_column: + The name of the column to search from. + query (str): + The search query. + top_k (int): + The number of nearest neighbors to return. + model (TextEmbeddingGenerator): + A TextEmbeddingGenerator provided by Bigframes ML package. + score_column (Optional[str], default None): + The name of the the additional column containning the similarity scores. If None, + this column won't be attached to the result. + + Returns: + DataFrame: the DataFrame with the search result. + + Raises: + ValueError: when the search_column is not found from the the data frame. + TypeError: when the provided model is not TextEmbeddingGenerator. + """ + if not options.experiments.ai_operators: + raise NotImplementedError() + + if search_column not in self._df.columns: + raise ValueError(f"Column `{search_column}` not found") + + self._confirm_operation(len(self._df)) + + import bigframes.ml.llm as llm + + if not isinstance(model, llm.TextEmbeddingGenerator): + raise TypeError(f"Expect a text embedding model, but got: {type(model)}") + + if top_k < 1: + raise ValueError("top_k must be an integer greater than or equal to 1.") + + embedded_df = model.predict(self._df[search_column]) + embedded_table = embedded_df.reset_index().to_gbq() + + import bigframes.pandas as bpd + + embedding_result_column = "ml_generate_embedding_result" + query_df = model.predict(bpd.DataFrame({"query_id": [query]})).rename( + columns={"content": "query_id", embedding_result_column: "embedding"} + ) + + import bigframes.bigquery as bbq + + search_result = ( + bbq.vector_search( + base_table=embedded_table, + column_to_search=embedding_result_column, + query=query_df, + top_k=top_k, + # TODO(tswast): set allow_large_results based on Series size. + # If we expect small results, it could be faster to set + # allow_large_results to False. + allow_large_results=True, + ) + .rename(columns={"content": search_column}) + .set_index("index") + ) + + search_result.index.name = self._df.index.name + + if score_column is not None: + search_result = search_result.rename(columns={"distance": score_column})[ + [search_column, score_column] + ] + else: + search_result = search_result[[search_column]] + + import bigframes.dataframe + + return typing.cast(bigframes.dataframe.DataFrame, search_result) + + def sim_join( + self, + other, + left_on: str, + right_on: str, + model, + top_k: int = 3, + score_column: Optional[str] = None, + max_rows: int = 1000, + ): + """ + Joins two dataframes based on the similarity of the specified columns. + + This method uses BigQuery's VECTOR_SEARCH function to match rows on the left side with the rows that have + nearest embedding vectors on the right. In the worst case scenario, the complexity is around O(M * N * log K). + Therefore, this is a potentially expensive operation. + + ** Examples: ** + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.ai_operators = True + >>> bpd.options.compute.ai_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.TextEmbeddingGenerator(model_name="text-embedding-005") + + >>> df1 = bpd.DataFrame({'animal': ['monkey', 'spider']}) + >>> df2 = bpd.DataFrame({'animal': ['scorpion', 'baboon']}) + + >>> df1.ai.sim_join(df2, left_on='animal', right_on='animal', model=model, top_k=1) + animal animal_1 + 0 monkey baboon + 1 spider scorpion + + [2 rows x 2 columns] + + Args: + other (DataFrame): + The other data frame to join with. + left_on (str): + The name of the column on left side for the join. + right_on (str): + The name of the column on the right side for the join. + top_k (int, default 3): + The number of nearest neighbors to return. + model (TextEmbeddingGenerator): + A TextEmbeddingGenerator provided by Bigframes ML package. + score_column (Optional[str], default None): + The name of the the additional column containning the similarity scores. If None, + this column won't be attached to the result. + max_rows: + The maximum number of rows allowed to be processed per call. If the result is too large, the method + call will end early with an error. + + Returns: + DataFrame: the data frame with the join result. + + Raises: + ValueError: when the amount of data to be processed exceeds the specified max_rows. + """ + if not options.experiments.ai_operators: + raise NotImplementedError() + + if left_on not in self._df.columns: + raise ValueError(f"Left column {left_on} not found") + if right_on not in self._df.columns: + raise ValueError(f"Right column {right_on} not found") + + import bigframes.ml.llm as llm + + if not isinstance(model, llm.TextEmbeddingGenerator): + raise TypeError(f"Expect a text embedding model, but got: {type(model)}") + + joined_table_rows = len(self._df) * len(other) + if joined_table_rows > max_rows: + raise ValueError( + f"Number of rows that need processing is {joined_table_rows}, which exceeds row limit {max_rows}." + ) + + if top_k < 1: + raise ValueError("top_k must be an integer greater than or equal to 1.") + + work_estimate = len(self._df) * len(other) + self._confirm_operation(work_estimate) + + base_table_embedding_column = guid.generate_guid() + base_table = self._attach_embedding( + other, right_on, base_table_embedding_column, model + ).to_gbq() + query_table = self._attach_embedding(self._df, left_on, "embedding", model) + + import bigframes.bigquery as bbq + + join_result = bbq.vector_search( + base_table=base_table, + column_to_search=base_table_embedding_column, + query=query_table, + top_k=top_k, + ) + + join_result = join_result.drop( + ["embedding", base_table_embedding_column], axis=1 + ) + + if score_column is not None: + join_result = join_result.rename(columns={"distance": score_column}) + else: + del join_result["distance"] + + return join_result + + def forecast( + self, + timestamp_column: str, + data_column: str, + *, + model: str = "TimesFM 2.0", + id_columns: Optional[Iterable[str]] = None, + horizon: int = 10, + confidence_level: float = 0.95, + ): + """ + Forecast time series at future horizon. Using Google Research's open source TimesFM(https://github.com/google-research/timesfm) model. + + .. note:: + + This product or feature is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). + + Args: + timestamp_column (str): + A str value that specified the name of the time points column. + The time points column provides the time points used to generate the forecast. + The time points column must use one of the following data types: TIMESTAMP, DATE and DATETIME + data_column (str): + A str value that specifies the name of the data column. The data column contains the data to forecast. + The data column must use one of the following data types: INT64, NUMERIC and FLOAT64 + model (str, default "TimesFM 2.0"): + A str value that specifies the name of the model. TimesFM 2.0 is the only supported value, and is the default value. + id_columns (Iterable[str] or None, default None): + An iterable of str value that specifies the names of one or more ID columns. Each ID identifies a unique time series to forecast. + Specify one or more values for this argument in order to forecast multiple time series using a single query. + The columns that you specify must use one of the following data types: STRING, INT64, ARRAY and ARRAY + horizon (int, default 10): + An int value that specifies the number of time points to forecast. The default value is 10. The valid input range is [1, 10,000]. + confidence_level (float, default 0.95): + A FLOAT64 value that specifies the percentage of the future values that fall in the prediction interval. + The default value is 0.95. The valid input range is [0, 1). + + Returns: + DataFrame: + The forecast dataframe matches that of the BigQuery AI.FORECAST function. + See: https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-ai-forecast + + Raises: + ValueError: when referring to a non-existing column. + """ + columns = [timestamp_column, data_column] + if id_columns: + columns += id_columns + for column in columns: + if column not in self._df.columns: + raise ValueError(f"Column `{column}` not found") + + options: dict[str, Union[int, float, str, Iterable[str]]] = { + "data_col": data_column, + "timestamp_col": timestamp_column, + "model": model, + "horizon": horizon, + "confidence_level": confidence_level, + } + if id_columns: + options["id_cols"] = id_columns + + return self._base_bqml.ai_forecast(input_data=self._df, options=options) + + @staticmethod + def _attach_embedding(dataframe, source_column: str, embedding_column: str, model): + result_df = dataframe.copy() + embeddings = model.predict(dataframe[source_column])[ + "ml_generate_embedding_result" + ] + result_df[embedding_column] = embeddings + return result_df + + @staticmethod + def _make_multimodel_prompt( + prompt_df, columns, user_instruction: str, output_instruction: str + ): + prompt = [f"{output_instruction}\n{user_instruction}\nContext: "] + for col in columns: + prompt.extend([f"{col} is ", prompt_df[col]]) + + return prompt + + @staticmethod + def _make_text_prompt( + prompt_df, columns, user_instruction: str, output_instruction: str + ): + prompt_df["prompt"] = f"{output_instruction}\n{user_instruction}\nContext: " + + # Combine context from multiple columns. + for col in columns: + prompt_df["prompt"] += f"{col} is `" + prompt_df[col] + "`\n" + + return prompt_df["prompt"] + + @staticmethod + def _parse_columns(instruction: str) -> List[str]: + """Extracts column names enclosed in curly braces from the user instruction. + For example, _parse_columns("{city} is in {continent}") == ["city", "continent"] + """ + columns = re.findall(r"(? str: + """Extracts column names enclosed in curly braces from the user instruction. + For example, `_format_instruction(["city", "continent"], "{city} is in {continent}") + == "city is in continent"` + """ + return instruction.format(**{col: col for col in columns}) + + @staticmethod + def _validate_model(model): + from bigframes.ml.llm import GeminiTextGenerator + + if not isinstance(model, GeminiTextGenerator): + raise TypeError("Model is not GeminiText Generator") + + @staticmethod + def _confirm_operation(row_count: int): + """Raises OperationAbortedError when the confirmation fails""" + import bigframes # Import in the function body to avoid circular imports. + + threshold = bigframes.options.compute.ai_ops_confirmation_threshold + + if threshold is None or row_count <= threshold: + return + + if bigframes.options.compute.ai_ops_threshold_autofail: + raise exceptions.OperationAbortedError( + f"Operation was cancelled because your work estimate is {row_count} rows, which exceeds the threshold {threshold} rows." + ) + + # Separate the prompt out. In IDE such VS Code, leaving prompt in the + # input function makes it less visible to the end user. + print(f"This operation will process about {row_count} rows.") + print( + "You can raise the confirmation threshold by setting `bigframes.options.compute.ai_ops_confirmation_threshold` to a higher value. To completely turn off the confirmation check, set the threshold to `None`." + ) + print("Proceed? [Y/n]") + reply = input().casefold() + if reply not in {"y", "yes", ""}: + raise exceptions.OperationAbortedError("Operation was cancelled.") diff --git a/bigframes/operations/ai_ops.py b/bigframes/operations/ai_ops.py index ad2b9850577..8dc8c2ffab4 100644 --- a/bigframes/operations/ai_ops.py +++ b/bigframes/operations/ai_ops.py @@ -15,7 +15,7 @@ from __future__ import annotations import dataclasses -from typing import ClassVar, Tuple +from typing import ClassVar, Literal, Tuple import pandas as pd import pyarrow as pa @@ -29,11 +29,11 @@ class AIGenerate(base_ops.NaryOp): name: ClassVar[str] = "ai_generate" prompt_context: Tuple[str | None, ...] - connection_id: str | None = None - endpoint: str | None = None - request_type: str | None = None - model_params: str | None = None - output_schema: str | None = None + connection_id: str | None + endpoint: str | None + request_type: Literal["dedicated", "shared", "unspecified"] + model_params: str | None + output_schema: str | None def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: if self.output_schema is None: @@ -57,10 +57,10 @@ class AIGenerateBool(base_ops.NaryOp): name: ClassVar[str] = "ai_generate_bool" prompt_context: Tuple[str | None, ...] - connection_id: str | None = None - endpoint: str | None = None - request_type: str | None = None - model_params: str | None = None + connection_id: str | None + endpoint: str | None + request_type: Literal["dedicated", "shared", "unspecified"] + model_params: str | None def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: return pd.ArrowDtype( @@ -79,10 +79,10 @@ class AIGenerateInt(base_ops.NaryOp): name: ClassVar[str] = "ai_generate_int" prompt_context: Tuple[str | None, ...] - connection_id: str | None = None - endpoint: str | None = None - request_type: str | None = None - model_params: str | None = None + connection_id: str | None + endpoint: str | None + request_type: Literal["dedicated", "shared", "unspecified"] + model_params: str | None def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: return pd.ArrowDtype( @@ -101,10 +101,10 @@ class AIGenerateDouble(base_ops.NaryOp): name: ClassVar[str] = "ai_generate_double" prompt_context: Tuple[str | None, ...] - connection_id: str | None = None - endpoint: str | None = None - request_type: str | None = None - model_params: str | None = None + connection_id: str | None + endpoint: str | None + request_type: Literal["dedicated", "shared", "unspecified"] + model_params: str | None def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: return pd.ArrowDtype( @@ -118,37 +118,12 @@ def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionT ) -@dataclasses.dataclass(frozen=True) -class AIEmbed(base_ops.UnaryOp): - name: ClassVar[str] = "ai_embed" - - endpoint: str | None = None - model: str | None = None - task_type: str | None = None - title: str | None = None - model_params: str | None = None - connection_id: str | None = None - - def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: - return pd.ArrowDtype( - pa.struct( - ( - pa.field("result", pa.list_(pa.float64())), - pa.field("status", pa.string()), - ) - ) - ) - - @dataclasses.dataclass(frozen=True) class AIIf(base_ops.NaryOp): name: ClassVar[str] = "ai_if" prompt_context: Tuple[str | None, ...] - connection_id: str | None = None - endpoint: str | None = None - optimization_mode: str | None = None - max_error_ratio: float | None = None + connection_id: str def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: return dtypes.BOOL_DTYPE @@ -160,18 +135,9 @@ class AIClassify(base_ops.NaryOp): prompt_context: Tuple[str | None, ...] categories: tuple[str, ...] - examples: ( - tuple[tuple[str, str], ...] | tuple[tuple[str, tuple[str, ...]], ...] | None - ) = None - connection_id: str | None = None - endpoint: str | None = None - output_mode: str | None = None - optimization_mode: str | None = None - max_error_ratio: float | None = None + connection_id: str def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: - if self.output_mode is not None: - return dtypes.list_type(dtypes.STRING_DTYPE) return dtypes.STRING_DTYPE @@ -180,22 +146,7 @@ class AIScore(base_ops.NaryOp): name: ClassVar[str] = "ai_score" prompt_context: Tuple[str | None, ...] - connection_id: str | None = None - endpoint: str | None = None - max_error_ratio: float | None = None - - def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: - return dtypes.FLOAT_DTYPE - - -@dataclasses.dataclass(frozen=True) -class AISimilarity(base_ops.BinaryOp): - name: ClassVar[str] = "ai_similarity" - - endpoint: str | None = None - model: str | None = None - model_params: str | None = None - connection_id: str | None = None + connection_id: str def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: return dtypes.FLOAT_DTYPE diff --git a/bigframes/operations/array_ops.py b/bigframes/operations/array_ops.py index e6f5743989b..61ada59cc7b 100644 --- a/bigframes/operations/array_ops.py +++ b/bigframes/operations/array_ops.py @@ -32,6 +32,23 @@ def output_type(self, *input_types): return dtypes.STRING_DTYPE +@dataclasses.dataclass(frozen=True) +class ArrayIndexOp(base_ops.UnaryOp): + name: typing.ClassVar[str] = "array_index" + index: int + + def output_type(self, *input_types): + input_type = input_types[0] + if dtypes.is_string_like(input_type): + return dtypes.STRING_DTYPE + elif dtypes.is_array_like(input_type): + return dtypes.arrow_dtype_to_bigframes_dtype( + input_type.pyarrow_dtype.value_type + ) + else: + raise TypeError("Input type must be an array or string-like type.") + + @dataclasses.dataclass(frozen=True) class ArraySliceOp(base_ops.UnaryOp): name: typing.ClassVar[str] = "array_slice" @@ -71,17 +88,3 @@ def output_type(self, *input_types): assert dtypes.is_array_like(input_type) inner_type = dtypes.get_array_inner_type(input_type) return self.aggregation.output_type(inner_type) - - -@dataclasses.dataclass(frozen=True) -class ArrayMapOp(base_ops.UnaryOp): - name: typing.ClassVar[str] = "array_map" - # TODO(b/495513753): Generalize to chained expressions - map_op: base_ops.UnaryOp - - def output_type(self, *input_types): - input_type = input_types[0] - assert dtypes.is_array_like(input_type) - inner_type = dtypes.get_array_inner_type(input_type) - out_inner_type = self.map_op.output_type(inner_type) - return dtypes.list_type(out_inner_type) diff --git a/bigframes/operations/base_ops.py b/bigframes/operations/base_ops.py index a3a0187d2d1..c0145a6711e 100644 --- a/bigframes/operations/base_ops.py +++ b/bigframes/operations/base_ops.py @@ -17,8 +17,8 @@ import dataclasses import typing -import bigframes.operations.type as op_typing from bigframes import dtypes +import bigframes.operations.type as op_typing if typing.TYPE_CHECKING: # Avoids circular dependency @@ -27,11 +27,11 @@ class RowOp(typing.Protocol): @property - def name(self) -> str: ... + def name(self) -> str: + ... - def output_type( - self, *input_types: dtypes.ExpressionType - ) -> dtypes.ExpressionType: ... + def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: + ... @property def is_monotonic(self) -> bool: @@ -168,7 +168,7 @@ def as_expr( def _convert_expr_input( - input: typing.Union[str, bigframes.core.expression.Expression], + input: typing.Union[str, bigframes.core.expression.Expression] ) -> bigframes.core.expression.Expression: """Allows creating column references with just a string""" import bigframes.core.expression diff --git a/bigframes/operations/blob.py b/bigframes/operations/blob.py index 3666ee66602..29f720b3ebc 100644 --- a/bigframes/operations/blob.py +++ b/bigframes/operations/blob.py @@ -14,27 +14,153 @@ from __future__ import annotations +import os +from typing import cast, Literal, Optional, Union +import warnings + +import IPython.display as ipy_display +import pandas as pd +import requests + +from bigframes import clients, dtypes +from bigframes.core.logging import log_adapter import bigframes.dataframe +import bigframes.exceptions as bfe import bigframes.operations as ops import bigframes.series -from bigframes.core.logging import log_adapter FILE_FOLDER_REGEX = r"^.*\/(.*)$" FILE_EXT_REGEX = r"(\.[0-9a-zA-Z]+$)" @log_adapter.class_logger -class _BlobAccessor: +class BlobAccessor: """ - Internal blob functions for Series and Index. + Blob functions for Series and Index. + + .. note:: + BigFrames Blob is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). """ def __init__(self, data: bigframes.series.Series): self._data = data + def uri(self) -> bigframes.series.Series: + """URIs of the Blob. + + Returns: + bigframes.series.Series: URIs as string.""" + s = bigframes.series.Series(self._data._block) + + return s.struct.field("uri") + + def authorizer(self) -> bigframes.series.Series: + """Authorizers of the Blob. + + Returns: + bigframes.series.Series: Autorithers(connection) as string.""" + s = bigframes.series.Series(self._data._block) + + return s.struct.field("authorizer") + + def version(self) -> bigframes.series.Series: + """Versions of the Blob. + + Returns: + bigframes.series.Series: Version as string.""" + # version must be retrieved after fetching metadata + return self._data._apply_unary_op(ops.obj_fetch_metadata_op).struct.field( + "version" + ) + + def metadata(self) -> bigframes.series.Series: + """Retrieve the metadata of the Blob. + + Returns: + bigframes.series.Series: JSON metadata of the Blob. Contains fields: content_type, md5_hash, size and updated(time).""" + series_to_check = bigframes.series.Series(self._data._block) + # Check if it's a struct series from a verbose operation + if dtypes.is_struct_like(series_to_check.dtype): + pyarrow_dtype = series_to_check.dtype.pyarrow_dtype + if "content" in [field.name for field in pyarrow_dtype]: + content_field_type = pyarrow_dtype.field("content").type + content_bf_type = dtypes.arrow_dtype_to_bigframes_dtype( + content_field_type + ) + if content_bf_type == dtypes.OBJ_REF_DTYPE: + series_to_check = series_to_check.struct.field("content") + details_json = series_to_check._apply_unary_op( + ops.obj_fetch_metadata_op + ).struct.field("details") + import bigframes.bigquery as bbq + + return bbq.json_extract(details_json, "$.gcs_metadata").rename("metadata") + + def content_type(self) -> bigframes.series.Series: + """Retrieve the content type of the Blob. + + Returns: + bigframes.series.Series: string of the content type.""" + return ( + self.metadata() + ._apply_unary_op(ops.JSONValue(json_path="$.content_type")) + .rename("content_type") + ) + + def md5_hash(self) -> bigframes.series.Series: + """Retrieve the md5 hash of the Blob. + + Returns: + bigframes.series.Series: string of the md5 hash.""" + return ( + self.metadata() + ._apply_unary_op(ops.JSONValue(json_path="$.md5_hash")) + .rename("md5_hash") + ) + + def size(self) -> bigframes.series.Series: + """Retrieve the file size of the Blob. + + Returns: + bigframes.series.Series: file size in bytes.""" + return ( + self.metadata() + ._apply_unary_op(ops.JSONValue(json_path="$.size")) + .rename("size") + .astype("Int64") + ) + + def updated(self) -> bigframes.series.Series: + """Retrieve the updated time of the Blob. + + Returns: + bigframes.series.Series: updated time as UTC datetime.""" + import bigframes.pandas as bpd + + updated = ( + self.metadata() + ._apply_unary_op(ops.JSONValue(json_path="$.updated")) + .rename("updated") + .astype("Int64") + ) + + return bpd.to_datetime(updated, unit="us", utc=True) + def _get_runtime( self, mode: str, with_metadata: bool = False ) -> bigframes.series.Series: + """Retrieve the ObjectRefRuntime as JSON. + + Args: + mode (str): mode for the URLs, "R" for read, "RW" for read & write. + metadata (bool, default False): whether to fetch the metadata in the ObjectRefRuntime. + + Returns: + bigframes.series.Series: ObjectRefRuntime JSON. + """ s = ( self._data._apply_unary_op(ops.obj_fetch_metadata_op) if with_metadata @@ -43,7 +169,911 @@ def _get_runtime( return s._apply_unary_op(ops.ObjGetAccessUrl(mode=mode)) - def _read_url(self) -> bigframes.series.Series: + def _df_apply_udf( + self, df: bigframes.dataframe.DataFrame, udf + ) -> bigframes.series.Series: + # Catch and rethrow function axis=1 warning to be more user-friendly. + with warnings.catch_warnings(record=True) as catched_warnings: + s = df.apply(udf, axis=1) + for w in catched_warnings: + if isinstance(w.message, bfe.FunctionAxisOnePreviewWarning): + warnings.warn( + "Blob Functions use bigframes DataFrame Managed function with axis=1 senario, which is a preview feature.", + category=w.category, + stacklevel=2, + ) + else: + warnings.warn_explicit( + message=w.message, + category=w.category, + filename=w.filename, + lineno=w.lineno, + source=w.source, + ) + + return s + + def _apply_udf_or_raise_error( + self, df: bigframes.dataframe.DataFrame, udf, operation_name: str + ) -> bigframes.series.Series: + """Helper to apply UDF with consistent error handling.""" + try: + res = self._df_apply_udf(df, udf) + except Exception as e: + raise RuntimeError(f"{operation_name} UDF execution failed: {e}") from e + + if res is None: + raise RuntimeError(f"{operation_name} returned None result") + + return res + + def read_url(self) -> bigframes.series.Series: + """Retrieve the read URL of the Blob. + + Returns: + bigframes.series.Series: Read only URLs.""" return self._get_runtime(mode="R")._apply_unary_op( ops.JSONValue(json_path="$.access_urls.read_url") ) + + def write_url(self) -> bigframes.series.Series: + """Retrieve the write URL of the Blob. + + Returns: + bigframes.series.Series: Writable URLs.""" + return self._get_runtime(mode="RW")._apply_unary_op( + ops.JSONValue(json_path="$.access_urls.write_url") + ) + + def display( + self, + n: int = 3, + *, + content_type: str = "", + width: Optional[int] = None, + height: Optional[int] = None, + ): + """Display the blob content in the IPython Notebook environment. Only works for image type now. + + Args: + n (int, default 3): number of sample blob objects to display. + content_type (str, default ""): content type of the blob. If unset, use the blob metadata of the storage. Possible values are "image", "audio" and "video". + width (int or None, default None): width in pixels that the image/video are constrained to. If unset, use the global setting in bigframes.options.display.blob_display_width, otherwise image/video's original size or ratio is used. No-op for other content types. + height (int or None, default None): height in pixels that the image/video are constrained to. If unset, use the global setting in bigframes.options.display.blob_display_height, otherwise image/video's original size or ratio is used. No-op for other content types. + """ + width = width or bigframes.options.display.blob_display_width + height = height or bigframes.options.display.blob_display_height + + # col name doesn't matter here. Rename to avoid column name conflicts + df = bigframes.series.Series(self._data._block).rename("blob_col").to_frame() + + df["read_url"] = df["blob_col"].blob.read_url() + + if content_type: + df["content_type"] = content_type + else: + df["content_type"] = df["blob_col"].blob.content_type() + + pandas_df, _, query_job = df._block.retrieve_repr_request_results(n) + df._set_internal_query_job(query_job) + + def display_single_url( + read_url: Union[str, pd._libs.missing.NAType], + content_type: Union[str, pd._libs.missing.NAType], + ): + if pd.isna(read_url): + ipy_display.display("") + return + + if pd.isna(content_type): # display as raw data or error + response = requests.get(read_url) + ipy_display.display(response.content) + return + + content_type = cast(str, content_type).casefold() + + if content_type.startswith("image"): + ipy_display.display( + ipy_display.Image(url=read_url, width=width, height=height) + ) + elif content_type.startswith("audio"): + # using url somehow doesn't work with audios + response = requests.get(read_url) + ipy_display.display(ipy_display.Audio(response.content)) + elif content_type.startswith("video"): + ipy_display.display( + ipy_display.Video(read_url, width=width, height=height) + ) + else: # display as raw data + response = requests.get(read_url) + ipy_display.display(response.content) + + for _, row in pandas_df.iterrows(): + display_single_url(row["read_url"], row["content_type"]) + + @property + def session(self): + return self._data._block.session + + def _resolve_connection(self, connection: Optional[str] = None) -> str: + """Resovle the BigQuery connection. + + Args: + connection (str or None, default None): BQ connection used for + function internet transactions, and the output blob if "dst" is + str. If None, uses default connection of the session. + + Returns: + str: the resolved BigQuery connection string in the format: + "project.location.connection_id". + + Raises: + ValueError: If the connection cannot be resolved to a valid string. + """ + connection = connection or self._data._block.session._bq_connection + return clients.get_canonical_bq_connection_id( + connection, + default_project=self._data._block.session._project, + default_location=self._data._block.session._location, + ) + + def get_runtime_json_str( + self, mode: str = "R", *, with_metadata: bool = False + ) -> bigframes.series.Series: + """Get the runtime (contains signed URL to access gcs data) and apply the ToJSONSTring transformation. + + Args: + mode(str or str, default "R"): the mode for accessing the runtime. + Default to "R". Possible values are "R" (read-only) and + "RW" (read-write) + with_metadata (bool, default False): whether to include metadata + in the JSON string. Default to False. + + Returns: + str: the runtime object in the JSON string. + """ + runtime = self._get_runtime(mode=mode, with_metadata=with_metadata) + return runtime._apply_unary_op(ops.ToJSONString()) + + def exif( + self, + *, + engine: Literal[None, "pillow"] = None, + connection: Optional[str] = None, + max_batching_rows: int = 8192, + container_cpu: Union[float, int] = 0.33, + container_memory: str = "512Mi", + verbose: bool = False, + ) -> bigframes.series.Series: + """Extract EXIF data. Now only support image types. + + Args: + engine ('pillow' or None, default None): The engine (bigquery or third party library) used for the function. The value must be specified. + connection (str or None, default None): BQ connection used for function internet transactions, and the output blob if "dst" is str. If None, uses default connection of the session. + max_batching_rows (int, default 8,192): Max number of rows per batch send to cloud run to execute the function. + container_cpu (int or float, default 0.33): number of container CPUs. Possible values are [0.33, 8]. Floats larger than 1 are cast to intergers. + container_memory (str, default "512Mi"): container memory size. String of the format . Possible values are from 512Mi to 32Gi. + verbose (bool, default False): If True, returns a struct with status and content fields. If False, returns only the content. + + Returns: + bigframes.series.Series: JSON series of key-value pairs if verbose=False, or struct with status and content if verbose=True. + + Raises: + ValueError: If engine is not 'pillow'. + RuntimeError: If EXIF extraction fails or returns invalid structure. + """ + if engine is None or engine.casefold() != "pillow": + raise ValueError("Must specify the engine, supported value is 'pillow'.") + + import bigframes.bigquery as bbq + import bigframes.blob._functions as blob_func + import bigframes.pandas as bpd + + connection = self._resolve_connection(connection) + df = self.get_runtime_json_str(mode="R").to_frame() + df["verbose"] = verbose + + exif_udf = blob_func.TransformFunction( + blob_func.exif_func_def, + session=self._data._block.session, + connection=connection, + max_batching_rows=max_batching_rows, + container_cpu=container_cpu, + container_memory=container_memory, + ).udf() + + res = self._apply_udf_or_raise_error(df, exif_udf, "EXIF extraction") + + if verbose: + try: + exif_content_series = bbq.parse_json( + res._apply_unary_op(ops.JSONValue(json_path="$.content")) + ).rename("exif_content") + exif_status_series = res._apply_unary_op( + ops.JSONValue(json_path="$.status") + ) + except Exception as e: + raise RuntimeError(f"Failed to parse EXIF JSON result: {e}") from e + results_df = bpd.DataFrame( + {"status": exif_status_series, "content": exif_content_series} + ) + results_struct = bbq.struct(results_df).rename("exif_results") + return results_struct + else: + try: + return bbq.parse_json(res) + except Exception as e: + raise RuntimeError(f"Failed to parse EXIF JSON result: {e}") from e + + def image_blur( + self, + ksize: tuple[int, int], + *, + engine: Literal[None, "opencv"] = None, + dst: Optional[Union[str, bigframes.series.Series]] = None, + connection: Optional[str] = None, + max_batching_rows: int = 8192, + container_cpu: Union[float, int] = 0.33, + container_memory: str = "512Mi", + verbose: bool = False, + ) -> bigframes.series.Series: + """Blurs images. + + Args: + ksize (tuple(int, int)): Kernel size. + engine ('opencv' or None, default None): The engine (bigquery or third party library) used for the function. The value must be specified. + dst (str or bigframes.series.Series or None, default None): Output destination. Can be one of: + str: GCS folder str. The output filenames are the same as the input files. + blob Series: The output file paths are determined by the uris of the blob Series. + None: Output to BQ as bytes. + Encoding is determined by the extension of the output filenames (or input filenames if doesn't have output filenames). If filename doesn't have an extension, use ".jpeg" for encoding. + connection (str or None, default None): BQ connection used for function internet transactions, and the output blob if "dst" is str. If None, uses default connection of the session. + max_batching_rows (int, default 8,192): Max number of rows per batch send to cloud run to execute the function. + container_cpu (int or float, default 0.33): number of container CPUs. Possible values are [0.33, 8]. Floats larger than 1 are cast to intergers. + container_memory (str, default "512Mi"): container memory size. String of the format . Possible values are from 512Mi to 32Gi. + verbose (bool, default False): If True, returns a struct with status and content fields. If False, returns only the content. + + Returns: + bigframes.series.Series: blob Series if destination is GCS. Or bytes Series if destination is BQ. If verbose=True, returns struct with status and content. + + Raises: + ValueError: If engine is not 'opencv' or parameters are invalid. + RuntimeError: If image blur operation fails. + """ + if engine is None or engine.casefold() != "opencv": + raise ValueError("Must specify the engine, supported value is 'opencv'.") + + import bigframes.bigquery as bbq + import bigframes.blob._functions as blob_func + import bigframes.pandas as bpd + + connection = self._resolve_connection(connection) + df = self.get_runtime_json_str(mode="R").to_frame() + + if dst is None: + ext = self.uri().str.extract(FILE_EXT_REGEX) + + image_blur_udf = blob_func.TransformFunction( + blob_func.image_blur_to_bytes_def, + session=self._data._block.session, + connection=connection, + max_batching_rows=max_batching_rows, + container_cpu=container_cpu, + container_memory=container_memory, + ).udf() + + df["ksize_x"], df["ksize_y"] = ksize + df["ext"] = ext # type: ignore + df["verbose"] = verbose + res = self._apply_udf_or_raise_error(df, image_blur_udf, "Image blur") + + if verbose: + blurred_content_b64_series = res._apply_unary_op( + ops.JSONValue(json_path="$.content") + ) + blurred_content_series = bbq.sql_scalar( + "FROM_BASE64({0})", columns=[blurred_content_b64_series] + ) + blurred_status_series = res._apply_unary_op( + ops.JSONValue(json_path="$.status") + ) + results_df = bpd.DataFrame( + {"status": blurred_status_series, "content": blurred_content_series} + ) + results_struct = bbq.struct(results_df).rename("blurred_results") + return results_struct + else: + blurred_bytes = bbq.sql_scalar( + "FROM_BASE64({0})", columns=[res] + ).rename("blurred_bytes") + return blurred_bytes + + if isinstance(dst, str): + dst = os.path.join(dst, "") + # Replace src folder with dst folder, keep the file names. + dst_uri = self.uri().str.replace(FILE_FOLDER_REGEX, rf"{dst}\1", regex=True) + dst = cast( + bigframes.series.Series, dst_uri.str.to_blob(connection=connection) + ) + + ext = dst.blob.uri().str.extract(FILE_EXT_REGEX) + + image_blur_udf = blob_func.TransformFunction( + blob_func.image_blur_def, + session=self._data._block.session, + connection=connection, + max_batching_rows=max_batching_rows, + container_cpu=container_cpu, + container_memory=container_memory, + ).udf() + + dst_rt = dst.blob.get_runtime_json_str(mode="RW") + + df = df.join(dst_rt, how="outer") + df["ksize_x"], df["ksize_y"] = ksize + df["ext"] = ext # type: ignore + df["verbose"] = verbose + + res = self._apply_udf_or_raise_error(df, image_blur_udf, "Image blur") + res.cache() # to execute the udf + + if verbose: + blurred_status_series = res._apply_unary_op( + ops.JSONValue(json_path="$.status") + ) + results_df = bpd.DataFrame( + { + "status": blurred_status_series, + "content": dst.blob.uri().str.to_blob( + connection=self._resolve_connection(connection) + ), + } + ) + results_struct = bbq.struct(results_df).rename("blurred_results") + return results_struct + else: + return dst + + def image_resize( + self, + dsize: tuple[int, int] = (0, 0), + *, + engine: Literal[None, "opencv"] = None, + fx: float = 0.0, + fy: float = 0.0, + dst: Optional[Union[str, bigframes.series.Series]] = None, + connection: Optional[str] = None, + max_batching_rows: int = 8192, + container_cpu: Union[float, int] = 0.33, + container_memory: str = "512Mi", + verbose: bool = False, + ): + """Resize images. + + Args: + dsize (tuple(int, int), default (0, 0)): Destination size. If set to 0, fx and fy parameters determine the size. + engine ('opencv' or None, default None): The engine (bigquery or third party library) used for the function. The value must be specified. + fx (float, default 0.0): scale factor along the horizontal axis. If set to 0.0, dsize parameter determines the output size. + fy (float, defalut 0.0): scale factor along the vertical axis. If set to 0.0, dsize parameter determines the output size. + dst (str or bigframes.series.Series or None, default None): Output destination. Can be one of: + str: GCS folder str. The output filenames are the same as the input files. + blob Series: The output file paths are determined by the uris of the blob Series. + None: Output to BQ as bytes. + Encoding is determined by the extension of the output filenames (or input filenames if doesn't have output filenames). If filename doesn't have an extension, use ".jpeg" for encoding. + connection (str or None, default None): BQ connection used for function internet transactions, and the output blob if "dst" is str. If None, uses default connection of the session. + max_batching_rows (int, default 8,192): Max number of rows per batch send to cloud run to execute the function. + container_cpu (int or float, default 0.33): number of container CPUs. Possible values are [0.33, 8]. Floats larger than 1 are cast to intergers. + container_memory (str, default "512Mi"): container memory size. String of the format . Possible values are from 512Mi to 32Gi. + verbose (bool, default False): If True, returns a struct with status and content fields. If False, returns only the content. + + Returns: + bigframes.series.Series: blob Series if destination is GCS. Or bytes Series if destination is BQ. If verbose=True, returns struct with status and content. + + Raises: + ValueError: If engine is not 'opencv' or parameters are invalid. + RuntimeError: If image resize operation fails. + """ + if engine is None or engine.casefold() != "opencv": + raise ValueError("Must specify the engine, supported value is 'opencv'.") + + dsize_set = dsize[0] > 0 and dsize[1] > 0 + fsize_set = fx > 0.0 and fy > 0.0 + if not dsize_set ^ fsize_set: + raise ValueError( + "Only one of dsize or (fx, fy) parameters must be set. And the set values must be positive. " + ) + + import bigframes.bigquery as bbq + import bigframes.blob._functions as blob_func + import bigframes.pandas as bpd + + connection = self._resolve_connection(connection) + df = self.get_runtime_json_str(mode="R").to_frame() + + if dst is None: + ext = self.uri().str.extract(FILE_EXT_REGEX) + + image_resize_udf = blob_func.TransformFunction( + blob_func.image_resize_to_bytes_def, + session=self._data._block.session, + connection=connection, + max_batching_rows=max_batching_rows, + container_cpu=container_cpu, + container_memory=container_memory, + ).udf() + + df["dsize_x"], df["dsize_y"] = dsize + df["fx"], df["fy"] = fx, fy + df["ext"] = ext # type: ignore + df["verbose"] = verbose + res = self._apply_udf_or_raise_error(df, image_resize_udf, "Image resize") + + if verbose: + resized_content_b64_series = res._apply_unary_op( + ops.JSONValue(json_path="$.content") + ) + resized_content_series = bbq.sql_scalar( + "FROM_BASE64({0})", columns=[resized_content_b64_series] + ) + + resized_status_series = res._apply_unary_op( + ops.JSONValue(json_path="$.status") + ) + results_df = bpd.DataFrame( + {"status": resized_status_series, "content": resized_content_series} + ) + results_struct = bbq.struct(results_df).rename("resized_results") + return results_struct + else: + resized_bytes = bbq.sql_scalar( + "FROM_BASE64({0})", columns=[res] + ).rename("resized_bytes") + return resized_bytes + + if isinstance(dst, str): + dst = os.path.join(dst, "") + # Replace src folder with dst folder, keep the file names. + dst_uri = self.uri().str.replace(FILE_FOLDER_REGEX, rf"{dst}\1", regex=True) + dst = cast( + bigframes.series.Series, dst_uri.str.to_blob(connection=connection) + ) + + ext = dst.blob.uri().str.extract(FILE_EXT_REGEX) + + image_resize_udf = blob_func.TransformFunction( + blob_func.image_resize_def, + session=self._data._block.session, + connection=connection, + max_batching_rows=max_batching_rows, + container_cpu=container_cpu, + container_memory=container_memory, + ).udf() + + dst_rt = dst.blob.get_runtime_json_str(mode="RW") + + df = df.join(dst_rt, how="outer") + df["dsize_x"], df["dsize_y"] = dsize + df["fx"], df["fy"] = fx, fy + df["ext"] = ext # type: ignore + df["verbose"] = verbose + + res = self._apply_udf_or_raise_error(df, image_resize_udf, "Image resize") + res.cache() # to execute the udf + + if verbose: + resized_status_series = res._apply_unary_op( + ops.JSONValue(json_path="$.status") + ) + results_df = bpd.DataFrame( + { + "status": resized_status_series, + "content": dst.blob.uri().str.to_blob( + connection=self._resolve_connection(connection) + ), + } + ) + results_struct = bbq.struct(results_df).rename("resized_results") + return results_struct + else: + return dst + + def image_normalize( + self, + *, + engine: Literal[None, "opencv"] = None, + alpha: float = 1.0, + beta: float = 0.0, + norm_type: str = "l2", + dst: Optional[Union[str, bigframes.series.Series]] = None, + connection: Optional[str] = None, + max_batching_rows: int = 8192, + container_cpu: Union[float, int] = 0.33, + container_memory: str = "512Mi", + verbose: bool = False, + ) -> bigframes.series.Series: + """Normalize images. + + Args: + engine ('opencv' or None, default None): The engine (bigquery or third party library) used for the function. The value must be specified. + alpha (float, default 1.0): Norm value to normalize to or the lower range boundary in case of the range normalization. + beta (float, default 0.0): Upper range boundary in case of the range normalization; it is not used for the norm normalization. + norm_type (str, default "l2"): Normalization type. Accepted values are "inf", "l1", "l2" and "minmax". + dst (str or bigframes.series.Series or None, default None): Output destination. Can be one of: + str: GCS folder str. The output filenames are the same as the input files. + blob Series: The output file paths are determined by the uris of the blob Series. + None: Output to BQ as bytes. + Encoding is determined by the extension of the output filenames (or input filenames if doesn't have output filenames). If filename doesn't have an extension, use ".jpeg" for encoding. + connection (str or None, default None): BQ connection used for function internet transactions, and the output blob if "dst" is str. If None, uses default connection of the session. + max_batching_rows (int, default 8,192): Max number of rows per batch send to cloud run to execute the function. + container_cpu (int or float, default 0.33): number of container CPUs. Possible values are [0.33, 8]. Floats larger than 1 are cast to intergers. + container_memory (str, default "512Mi"): container memory size. String of the format . Possible values are from 512Mi to 32Gi. + verbose (bool, default False): If True, returns a struct with status and content fields. If False, returns only the content. + + Returns: + bigframes.series.Series: blob Series if destination is GCS. Or bytes Series if destination is BQ. If verbose=True, returns struct with status and content. + + Raises: + ValueError: If engine is not 'opencv' or parameters are invalid. + RuntimeError: If image normalize operation fails. + """ + if engine is None or engine.casefold() != "opencv": + raise ValueError("Must specify the engine, supported value is 'opencv'.") + + import bigframes.bigquery as bbq + import bigframes.blob._functions as blob_func + import bigframes.pandas as bpd + + connection = self._resolve_connection(connection) + df = self.get_runtime_json_str(mode="R").to_frame() + + if dst is None: + ext = self.uri().str.extract(FILE_EXT_REGEX) + + image_normalize_udf = blob_func.TransformFunction( + blob_func.image_normalize_to_bytes_def, + session=self._data._block.session, + connection=connection, + max_batching_rows=max_batching_rows, + container_cpu=container_cpu, + container_memory=container_memory, + ).udf() + + df["alpha"] = alpha + df["beta"] = beta + df["norm_type"] = norm_type + df["ext"] = ext # type: ignore + df["verbose"] = verbose + res = self._apply_udf_or_raise_error( + df, image_normalize_udf, "Image normalize" + ) + + if verbose: + normalized_content_b64_series = res._apply_unary_op( + ops.JSONValue(json_path="$.content") + ) + normalized_bytes = bbq.sql_scalar( + "FROM_BASE64({0})", columns=[normalized_content_b64_series] + ) + normalized_status_series = res._apply_unary_op( + ops.JSONValue(json_path="$.status") + ) + results_df = bpd.DataFrame( + {"status": normalized_status_series, "content": normalized_bytes} + ) + results_struct = bbq.struct(results_df).rename("normalized_results") + return results_struct + else: + normalized_bytes = bbq.sql_scalar( + "FROM_BASE64({0})", columns=[res] + ).rename("normalized_bytes") + return normalized_bytes + + if isinstance(dst, str): + dst = os.path.join(dst, "") + # Replace src folder with dst folder, keep the file names. + dst_uri = self.uri().str.replace(FILE_FOLDER_REGEX, rf"{dst}\1", regex=True) + dst = cast( + bigframes.series.Series, dst_uri.str.to_blob(connection=connection) + ) + + ext = dst.blob.uri().str.extract(FILE_EXT_REGEX) + + image_normalize_udf = blob_func.TransformFunction( + blob_func.image_normalize_def, + session=self._data._block.session, + connection=connection, + max_batching_rows=max_batching_rows, + container_cpu=container_cpu, + container_memory=container_memory, + ).udf() + + dst_rt = dst.blob.get_runtime_json_str(mode="RW") + + df = df.join(dst_rt, how="outer") + df["alpha"] = alpha + df["beta"] = beta + df["norm_type"] = norm_type + df["ext"] = ext # type: ignore + df["verbose"] = verbose + + res = self._apply_udf_or_raise_error(df, image_normalize_udf, "Image normalize") + res.cache() # to execute the udf + + if verbose: + normalized_status_series = res._apply_unary_op( + ops.JSONValue(json_path="$.status") + ) + results_df = bpd.DataFrame( + { + "status": normalized_status_series, + "content": dst.blob.uri().str.to_blob( + connection=self._resolve_connection(connection) + ), + } + ) + results_struct = bbq.struct(results_df).rename("normalized_results") + return results_struct + else: + return dst + + def pdf_extract( + self, + *, + engine: Literal[None, "pypdf"] = None, + connection: Optional[str] = None, + max_batching_rows: int = 1, + container_cpu: Union[float, int] = 2, + container_memory: str = "1Gi", + verbose: bool = False, + ) -> bigframes.series.Series: + """Extracts text from PDF URLs and saves the text as string. + + Args: + engine ('pypdf' or None, default None): The engine (bigquery or third party library) used for the function. The value must be specified. + connection (str or None, default None): BQ connection used for + function internet transactions, and the output blob if "dst" + is str. If None, uses default connection of the session. + max_batching_rows (int, default 1): Max number of rows per batch + send to cloud run to execute the function. + container_cpu (int or float, default 2): number of container CPUs. Possible values are [0.33, 8]. Floats larger than 1 are cast to intergers. + container_memory (str, default "1Gi"): container memory size. String of the format . Possible values are from 512Mi to 32Gi. + verbose (bool, default "False"): controls the verbosity of the output. + When set to True, both error messages and the extracted content + are displayed. Conversely, when set to False, only the extracted + content is presented, suppressing error messages. + + Returns: + bigframes.series.Series: str or struct[str, str], + depend on the "verbose" parameter. + Contains the extracted text from the PDF file. + Includes error messages if verbosity is enabled. + + Raises: + ValueError: If engine is not 'pypdf'. + RuntimeError: If PDF extraction fails or returns invalid structure. + """ + if engine is None or engine.casefold() != "pypdf": + raise ValueError("Must specify the engine, supported value is 'pypdf'.") + + import bigframes.bigquery as bbq + import bigframes.blob._functions as blob_func + import bigframes.pandas as bpd + + connection = self._resolve_connection(connection) + + pdf_extract_udf = blob_func.TransformFunction( + blob_func.pdf_extract_def, + session=self._data._block.session, + connection=connection, + max_batching_rows=max_batching_rows, + container_cpu=container_cpu, + container_memory=container_memory, + ).udf() + + df = self.get_runtime_json_str(mode="R").to_frame() + df["verbose"] = verbose + + res = self._apply_udf_or_raise_error(df, pdf_extract_udf, "PDF extraction") + + if verbose: + # Extract content with error handling + try: + content_series = res._apply_unary_op( + ops.JSONValue(json_path="$.content") + ) + except Exception as e: + raise RuntimeError( + f"Failed to extract content field from PDF result: {e}" + ) from e + try: + status_series = res._apply_unary_op(ops.JSONValue(json_path="$.status")) + except Exception as e: + raise RuntimeError( + f"Failed to extract status field from PDF result: {e}" + ) from e + + res_df = bpd.DataFrame({"status": status_series, "content": content_series}) + struct_series = bbq.struct(res_df).rename("extracted_results") + return struct_series + else: + return res.rename("extracted_content") + + def pdf_chunk( + self, + *, + engine: Literal[None, "pypdf"] = None, + connection: Optional[str] = None, + chunk_size: int = 2000, + overlap_size: int = 200, + max_batching_rows: int = 1, + container_cpu: Union[float, int] = 2, + container_memory: str = "1Gi", + verbose: bool = False, + ) -> bigframes.series.Series: + """Extracts and chunks text from PDF URLs and saves the text as + arrays of strings. + + Args: + engine ('pypdf' or None, default None): The engine (bigquery or third party library) used for the function. The value must be specified. + connection (str or None, default None): BQ connection used for + function internet transactions, and the output blob if "dst" + is str. If None, uses default connection of the session. + chunk_size (int, default 2000): the desired size of each text chunk + (number of characters). + overlap_size (int, default 200): the number of overlapping characters + between consective chunks. The helps to ensure context is + perserved across chunk boundaries. + max_batching_rows (int, default 1): Max number of rows per batch + send to cloud run to execute the function. + container_cpu (int or float, default 2): number of container CPUs. Possible values are [0.33, 8]. Floats larger than 1 are cast to intergers. + container_memory (str, default "1Gi"): container memory size. String of the format . Possible values are from 512Mi to 32Gi. + verbose (bool, default "False"): controls the verbosity of the output. + When set to True, both error messages and the extracted content + are displayed. Conversely, when set to False, only the extracted + content is presented, suppressing error messages. + + Returns: + bigframe.series.Series: array[str] or struct[str, array[str]], + depend on the "verbose" parameter. + where each string is a chunk of text extracted from PDF. + Includes error messages if verbosity is enabled. + + Raises: + ValueError: If engine is not 'pypdf'. + RuntimeError: If PDF chunking fails or returns invalid structure. + """ + if engine is None or engine.casefold() != "pypdf": + raise ValueError("Must specify the engine, supported value is 'pypdf'.") + + import bigframes.bigquery as bbq + import bigframes.blob._functions as blob_func + import bigframes.pandas as bpd + + connection = self._resolve_connection(connection) + + if chunk_size <= 0: + raise ValueError("chunk_size must be a positive integer.") + if overlap_size < 0: + raise ValueError("overlap_size must be a non-negative integer.") + if overlap_size >= chunk_size: + raise ValueError("overlap_size must be smaller than chunk_size.") + + pdf_chunk_udf = blob_func.TransformFunction( + blob_func.pdf_chunk_def, + session=self._data._block.session, + connection=connection, + max_batching_rows=max_batching_rows, + container_cpu=container_cpu, + container_memory=container_memory, + ).udf() + + df = self.get_runtime_json_str(mode="R").to_frame() + df["chunk_size"] = chunk_size + df["overlap_size"] = overlap_size + df["verbose"] = verbose + + res = self._apply_udf_or_raise_error(df, pdf_chunk_udf, "PDF chunking") + + try: + content_series = bbq.json_extract_string_array(res, "$.content") + except Exception as e: + raise RuntimeError( + f"Failed to extract content array from PDF chunk result: {e}" + ) from e + + if verbose: + try: + status_series = res._apply_unary_op(ops.JSONValue(json_path="$.status")) + except Exception as e: + raise RuntimeError( + f"Failed to extract status field from PDF chunk result: {e}" + ) from e + + results_df = bpd.DataFrame( + {"status": status_series, "content": content_series} + ) + resultes_struct = bbq.struct(results_df).rename("chunked_results") + return resultes_struct + else: + return bbq.json_extract_string_array(res, "$").rename("chunked_content") + + def audio_transcribe( + self, + *, + engine: Literal["bigquery"] = "bigquery", + connection: Optional[str] = None, + model_name: Optional[ + Literal[ + "gemini-2.0-flash-001", + "gemini-2.0-flash-lite-001", + ] + ] = None, + verbose: bool = False, + ) -> bigframes.series.Series: + """ + Transcribe audio content using a Gemini multimodal model. + + Args: + engine ('bigquery'): The engine (bigquery or third party library) used for the function. + connection (str or None, default None): BQ connection used for + function internet transactions, and the output blob if "dst" + is str. If None, uses default connection of the session. + model_name (str): The model for natural language tasks. Accepted + values are "gemini-2.0-flash-lite-001", and "gemini-2.0-flash-001". + See "https://ai.google.dev/gemini-api/docs/models" for model choices. + verbose (bool, default "False"): controls the verbosity of the output. + When set to True, both error messages and the transcribed content + are displayed. Conversely, when set to False, only the transcribed + content is presented, suppressing error messages. + + Returns: + bigframes.series.Series: str or struct[str, str], + depend on the "verbose" parameter. + Contains the transcribed text from the audio file. + Includes error messages if verbosity is enabled. + + Raises: + ValueError: If engine is not 'bigquery'. + RuntimeError: If the transcription result structure is invalid. + """ + if engine.casefold() != "bigquery": + raise ValueError("Must specify the engine, supported value is 'bigquery'.") + + import bigframes.bigquery as bbq + import bigframes.pandas as bpd + + # col name doesn't matter here. Rename to avoid column name conflicts + audio_series = bigframes.series.Series(self._data._block) + + prompt_text = "**Task:** Transcribe the provided audio. **Instructions:** - Your response must contain only the verbatim transcription of the audio. - Do not include any introductory text, summaries, or conversational filler in your response. The output should begin directly with the first word of the audio." + + # Convert the audio series to the runtime representation required by the model. + audio_runtime = audio_series.blob._get_runtime("R", with_metadata=True) + + transcribed_results = bbq.ai.generate( + prompt=(prompt_text, audio_runtime), + connection_id=connection, + endpoint=model_name, + model_params={"generationConfig": {"temperature": 0.0}}, + ) + + # Validate that the result is not None + if transcribed_results is None: + raise RuntimeError("Transcription returned None result") + + transcribed_content_series = transcribed_results.struct.field("result").rename( + "transcribed_content" + ) + + if verbose: + transcribed_status_series = transcribed_results.struct.field("status") + results_df = bpd.DataFrame( + { + "status": transcribed_status_series, + "content": transcribed_content_series, + } + ) + results_struct = bbq.struct(results_df).rename("transcription_results") + return results_struct + else: + return transcribed_content_series.rename("transcribed_content") diff --git a/bigframes/operations/blob_ops.py b/bigframes/operations/blob_ops.py index 21d645a2fee..d1e2764eb45 100644 --- a/bigframes/operations/blob_ops.py +++ b/bigframes/operations/blob_ops.py @@ -15,9 +15,9 @@ import dataclasses import typing -import bigframes.operations.type as op_typing from bigframes import dtypes from bigframes.operations import base_ops +import bigframes.operations.type as op_typing ObjFetchMetadataOp = base_ops.create_unary_op( name="obj_fetch_metadata", type_signature=op_typing.BLOB_TRANSFORM diff --git a/bigframes/operations/bool_ops.py b/bigframes/operations/bool_ops.py index ce4406d8f70..003318f8226 100644 --- a/bigframes/operations/bool_ops.py +++ b/bigframes/operations/bool_ops.py @@ -13,8 +13,8 @@ # limitations under the License. -import bigframes.operations.type as op_typing from bigframes.operations import base_ops +import bigframes.operations.type as op_typing AndOp = base_ops.create_binary_op(name="and", type_signature=op_typing.LOGICAL) and_op = AndOp() diff --git a/bigframes/operations/comparison_ops.py b/bigframes/operations/comparison_ops.py index f3c01a3536b..4c2911808db 100644 --- a/bigframes/operations/comparison_ops.py +++ b/bigframes/operations/comparison_ops.py @@ -13,8 +13,8 @@ # limitations under the License. -import bigframes.operations.type as op_typing from bigframes.operations import base_ops +import bigframes.operations.type as op_typing EqOp = base_ops.create_binary_op(name="eq", type_signature=op_typing.COMPARISON) eq_op = EqOp() diff --git a/bigframes/operations/date_ops.py b/bigframes/operations/date_ops.py index 1dbb244afbc..352bc9f93e9 100644 --- a/bigframes/operations/date_ops.py +++ b/bigframes/operations/date_ops.py @@ -15,9 +15,9 @@ import dataclasses import typing -import bigframes.operations.type as op_typing from bigframes import dtypes from bigframes.operations import base_ops +import bigframes.operations.type as op_typing DayOp = base_ops.create_unary_op( name="day", diff --git a/bigframes/operations/datetime_ops.py b/bigframes/operations/datetime_ops.py index 702466c4f35..9988e8ed7b9 100644 --- a/bigframes/operations/datetime_ops.py +++ b/bigframes/operations/datetime_ops.py @@ -18,9 +18,9 @@ import pandas as pd import pyarrow as pa -import bigframes.operations.type as op_typing from bigframes import dtypes from bigframes.operations import base_ops +import bigframes.operations.type as op_typing DateOp = base_ops.create_unary_op( name="date", @@ -73,8 +73,6 @@ def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionT dtypes.INT_DTYPE, dtypes.STRING_DTYPE, dtypes.DATE_DTYPE, - dtypes.TIMESTAMP_DTYPE, - dtypes.DATETIME_DTYPE, ): raise TypeError("expected string or numeric input") return pd.ArrowDtype(pa.timestamp("us", tz=None)) @@ -88,14 +86,11 @@ class ToTimestampOp(base_ops.UnaryOp): def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: # Must be numeric or string - if input_types[0] == dtypes.TIMESTAMP_DTYPE: - raise TypeError("Already tz-aware.") if input_types[0] not in ( dtypes.FLOAT_DTYPE, dtypes.INT_DTYPE, dtypes.STRING_DTYPE, dtypes.DATE_DTYPE, - dtypes.DATETIME_DTYPE, ): raise TypeError("expected string or numeric input") return pd.ArrowDtype(pa.timestamp("us", tz="UTC")) diff --git a/bigframes/operations/datetimes.py b/bigframes/operations/datetimes.py index b16c596120a..2eedb96b43e 100644 --- a/bigframes/operations/datetimes.py +++ b/bigframes/operations/datetimes.py @@ -15,18 +15,15 @@ from __future__ import annotations import datetime as dt -from typing import Generic, Literal, Optional, TypeVar +from typing import Optional import bigframes_vendored.pandas.core.arrays.datetimelike as vendored_pandas_datetimelike import bigframes_vendored.pandas.core.indexes.accessor as vendordt import pandas -import bigframes.core.col -import bigframes.core.indexes.base as indices -import bigframes.operations as ops from bigframes import dataframe, dtypes, series -from bigframes._tools import docs from bigframes.core.logging import log_adapter +import bigframes.operations as ops _ONE_DAY = pandas.Timedelta("1D") _ONE_SECOND = pandas.Timedelta("1s") @@ -34,135 +31,110 @@ _SUPPORTED_FREQS = ("Y", "Q", "M", "W", "D", "h", "min", "s", "ms", "us") -T = TypeVar("T", series.Series, indices.Index, bigframes.core.col.Expression) - +@log_adapter.class_logger +class DatetimeMethods( + vendordt.DatetimeProperties, + vendored_pandas_datetimelike.DatelikeOps, +): + __doc__ = vendordt.DatetimeProperties.__doc__ -# Simpler base class for datetime properties, excludes isocalendar, unit, tz -class DatetimeSimpleMethods(Generic[T]): - def __init__(self, data: T): - self._data: T = data + def __init__(self, data: series.Series): + self._data = data # Date accessors @property - def day(self) -> T: + def day(self) -> series.Series: return self._data._apply_unary_op(ops.day_op) @property - def dayofweek(self) -> T: + def dayofweek(self) -> series.Series: return self._data._apply_unary_op(ops.dayofweek_op) @property - def day_of_week(self) -> T: + def day_of_week(self) -> series.Series: return self.dayofweek @property - def weekday(self) -> T: + def weekday(self) -> series.Series: return self.dayofweek @property - def dayofyear(self) -> T: + def dayofyear(self) -> series.Series: return self._data._apply_unary_op(ops.dayofyear_op) @property - def day_of_year(self) -> T: + def day_of_year(self) -> series.Series: return self.dayofyear @property - def date(self) -> T: + def date(self) -> series.Series: return self._data._apply_unary_op(ops.date_op) @property - def quarter(self) -> T: + def quarter(self) -> series.Series: return self._data._apply_unary_op(ops.quarter_op) @property - def year(self) -> T: + def year(self) -> series.Series: return self._data._apply_unary_op(ops.year_op) @property - def month(self) -> T: + def month(self) -> series.Series: return self._data._apply_unary_op(ops.month_op) + def isocalendar(self) -> dataframe.DataFrame: + iso_ops = [ops.iso_year_op, ops.iso_week_op, ops.iso_day_op] + labels = pandas.Index(["year", "week", "day"]) + block = self._data._block.project_exprs( + [op.as_expr(self._data._value_column) for op in iso_ops], labels, drop=True + ) + return dataframe.DataFrame(block) + # Time accessors @property - def hour(self) -> T: + def hour(self) -> series.Series: return self._data._apply_unary_op(ops.hour_op) @property - def minute(self) -> T: + def minute(self) -> series.Series: return self._data._apply_unary_op(ops.minute_op) @property - def second(self) -> T: + def second(self) -> series.Series: return self._data._apply_unary_op(ops.second_op) @property - def time(self) -> T: + def time(self) -> series.Series: return self._data._apply_unary_op(ops.time_op) # Timedelta accessors @property - def days(self) -> T: + def days(self) -> series.Series: self._check_dtype(dtypes.TIMEDELTA_DTYPE) return self._data._apply_binary_op(_ONE_DAY, ops.floordiv_op) @property - def seconds(self) -> T: + def seconds(self) -> series.Series: self._check_dtype(dtypes.TIMEDELTA_DTYPE) return self._data._apply_binary_op(_ONE_DAY, ops.mod_op) // _ONE_SECOND # type: ignore @property - def microseconds(self) -> T: + def microseconds(self) -> series.Series: self._check_dtype(dtypes.TIMEDELTA_DTYPE) return self._data._apply_binary_op(_ONE_SECOND, ops.mod_op) // _ONE_MICRO # type: ignore - def total_seconds(self) -> T: + def total_seconds(self) -> series.Series: self._check_dtype(dtypes.TIMEDELTA_DTYPE) return self._data._apply_binary_op(_ONE_SECOND, ops.div_op) def _check_dtype(self, target_dtype: dtypes.Dtype): - if isinstance(self._data, (indices.Index, series.Series)): - if self._data.dtype != target_dtype: - raise TypeError( - f"Expect dtype: {target_dtype}, but got {self._data.dtype}" - ) - return - - def tz_localize(self, tz: Literal["UTC"] | None) -> T: - if tz == "UTC": - return self._data._apply_unary_op(ops.ToTimestampOp()) - - if tz is None: - return self._data._apply_unary_op(ops.ToDatetimeOp()) - - raise ValueError(f"Unsupported timezone {tz}") - - def day_name(self) -> T: - return self.strftime("%A") - - def strftime(self, date_format: str) -> T: - return self._data._apply_unary_op(ops.StrftimeOp(date_format=date_format)) - - def normalize(self) -> T: - return self._data._apply_unary_op(ops.normalize_op) - - def floor(self, freq: str) -> T: - if freq not in _SUPPORTED_FREQS: - raise ValueError(f"freq must be one of {_SUPPORTED_FREQS}") - return self._data._apply_unary_op(ops.FloorDtOp(freq=freq)) # type: ignore - - -# this is the version used by series.dt, and the one that shows up in reference docs -@log_adapter.class_logger -@docs.inherit_docs(vendordt.DatetimeProperties) -@docs.inherit_docs(vendored_pandas_datetimelike.DatelikeOps) -class DatetimeMethods(DatetimeSimpleMethods[bigframes.series.Series]): - def __init__(self, data: series.Series): - super().__init__(data) + if self._data._dtype == target_dtype: + return + raise TypeError(f"Expect dtype: {target_dtype}, but got {self._data._dtype}") @property def tz(self) -> Optional[dt.timezone]: @@ -180,10 +152,16 @@ def unit(self) -> str: # Assumption: pyarrow dtype return self._data._dtype.pyarrow_dtype.unit - def isocalendar(self) -> dataframe.DataFrame: - iso_ops = [ops.iso_year_op, ops.iso_week_op, ops.iso_day_op] - labels = pandas.Index(["year", "week", "day"]) - block = self._data._block.project_exprs( - [op.as_expr(self._data._value_column) for op in iso_ops], labels, drop=True - ) - return dataframe.DataFrame(block) + def day_name(self) -> series.Series: + return self.strftime("%A") + + def strftime(self, date_format: str) -> series.Series: + return self._data._apply_unary_op(ops.StrftimeOp(date_format=date_format)) + + def normalize(self) -> series.Series: + return self._data._apply_unary_op(ops.normalize_op) + + def floor(self, freq: str) -> series.Series: + if freq not in _SUPPORTED_FREQS: + raise ValueError(f"freq must be one of {_SUPPORTED_FREQS}") + return self._data._apply_unary_op(ops.FloorDtOp(freq=freq)) # type: ignore diff --git a/bigframes/operations/distance_ops.py b/bigframes/operations/distance_ops.py index 435308f9c40..ac0863b9e6b 100644 --- a/bigframes/operations/distance_ops.py +++ b/bigframes/operations/distance_ops.py @@ -13,8 +13,8 @@ # limitations under the License. -import bigframes.operations.type as op_typing from bigframes.operations import base_ops +import bigframes.operations.type as op_typing CosineDistanceOp = base_ops.create_binary_op( name="ml_cosine_distance", type_signature=op_typing.VECTOR_METRIC diff --git a/bigframes/operations/generic_ops.py b/bigframes/operations/generic_ops.py index 9b226ad28d8..d6155a770c1 100644 --- a/bigframes/operations/generic_ops.py +++ b/bigframes/operations/generic_ops.py @@ -16,9 +16,9 @@ import functools import typing -import bigframes.operations.type as op_typing from bigframes import dtypes from bigframes.operations import base_ops +import bigframes.operations.type as op_typing InvertOp = base_ops.create_unary_op( name="invert", @@ -45,21 +45,6 @@ ) notnull_op = NotNullOp() - -# Semantics match Python's truth value testing (truthy and falsey objects). -# See https://docs.python.org/3/library/stdtypes.html#truth-value-testing -CoerceToBoolOp = base_ops.create_unary_op( - name="coerce_to_bool", - type_signature=op_typing.FixedOutputType( - dtypes.is_bool_coercable, dtypes.BOOL_DTYPE, description="coercable to bool" - ), -) -CoerceToBoolOp.__doc__ = ( - "Coerce a value to a boolean, matching Python's truth value testing semantics " - "(truthy/falsey). See https://docs.python.org/3/library/stdtypes.html#truth-value-testing" -) -coerce_to_bool_op = CoerceToBoolOp() - HashOp = base_ops.create_unary_op( name="hash", type_signature=op_typing.FixedOutputType( @@ -108,6 +93,10 @@ dtypes.STRING_DTYPE, dtypes.INT_DTYPE, ), + ( + dtypes.JSON_DTYPE, + dtypes.INT_DTYPE, + ), # Float casts ( dtypes.BOOL_DTYPE, @@ -129,6 +118,10 @@ dtypes.STRING_DTYPE, dtypes.FLOAT_DTYPE, ), + ( + dtypes.JSON_DTYPE, + dtypes.FLOAT_DTYPE, + ), # Bool casts ( dtypes.INT_DTYPE, @@ -138,6 +131,10 @@ dtypes.FLOAT_DTYPE, dtypes.BOOL_DTYPE, ), + ( + dtypes.JSON_DTYPE, + dtypes.BOOL_DTYPE, + ), # String casts ( dtypes.BYTES_DTYPE, @@ -171,6 +168,10 @@ dtypes.DATE_DTYPE, dtypes.STRING_DTYPE, ), + ( + dtypes.JSON_DTYPE, + dtypes.STRING_DTYPE, + ), # bytes casts ( dtypes.STRING_DTYPE, @@ -275,6 +276,23 @@ dtypes.INT_DTYPE, dtypes.TIMEDELTA_DTYPE, ), + # json casts + ( + dtypes.BOOL_DTYPE, + dtypes.JSON_DTYPE, + ), + ( + dtypes.FLOAT_DTYPE, + dtypes.JSON_DTYPE, + ), + ( + dtypes.STRING_DTYPE, + dtypes.JSON_DTYPE, + ), + ( + dtypes.INT_DTYPE, + dtypes.JSON_DTYPE, + ), ) ) @@ -425,15 +443,10 @@ class SqlScalarOp(base_ops.NaryOp): name: typing.ClassVar[str] = "sql_scalar" _output_type: dtypes.ExpressionType sql_template: str - is_deterministic: bool = True def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: return self._output_type - @property - def deterministic(self) -> bool: - return self.is_deterministic - @dataclasses.dataclass(frozen=True) class PyUdfOp(base_ops.NaryOp): @@ -445,66 +458,3 @@ class PyUdfOp(base_ops.NaryOp): def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: return self._output_type - - -@dataclasses.dataclass(frozen=True) -class GetItemOp(base_ops.UnaryOp): - """Represents subscripting with a statically-known key (e.g. `obj[1]` or `obj["field"]`). - - We must keep this static UnaryOp separate from DynamicGetItemOp (a BinaryOp) - primarily to support Struct field subscripting. Because the return type of a Struct - field lookup depends on the specific field being accessed, and type resolution - (output_type) only has access to input types rather than input values, we must store - the static key inside the operation instance to infer the correct output type. - """ - - name: typing.ClassVar[str] = "getitem" - key: typing.Union[str, int] - - def output_type(self, *input_types): - input_type = input_types[0] - if dtypes.is_struct_like(input_type): - pa_type = input_type.pyarrow_dtype - pa_result_type = pa_type[self.key].type - return dtypes.arrow_dtype_to_bigframes_dtype(pa_result_type) - elif dtypes.is_array_like(input_type): - if not isinstance(self.key, int): - raise TypeError("Array index must be an integer") - return dtypes.arrow_dtype_to_bigframes_dtype( - input_type.pyarrow_dtype.value_type - ) - elif dtypes.is_string_like(input_type): - if not isinstance(self.key, int): - raise TypeError("String index must be an integer") - return dtypes.STRING_DTYPE - else: - raise TypeError(f"Cannot subscript input of type {input_type}") - - -@dataclasses.dataclass(frozen=True) -class DynamicGetItemOp(base_ops.BinaryOp): - """Represents subscripting with a dynamic key expression (e.g. `obj[expr]`). - - Unlike GetItemOp, this operates on 2 dynamic inputs (the container and the key). - Because SQL/BigQuery does not support dynamic struct field access (struct paths must - be statically declared), this operation is only supported for array and string - subscripting, where output type inference does not require knowing the runtime - index value. - """ - - name: typing.ClassVar[str] = "dynamic_getitem" - - def output_type(self, *input_types): - left_type = input_types[0] - right_type = input_types[1] - if not dtypes.is_numeric(right_type): - raise TypeError(f"Subscript index must be numeric type, got {right_type}") - - if dtypes.is_array_like(left_type): - return dtypes.arrow_dtype_to_bigframes_dtype( - left_type.pyarrow_dtype.value_type - ) - elif dtypes.is_string_like(left_type): - return dtypes.STRING_DTYPE - else: - raise TypeError(f"Cannot dynamically subscript input of type {left_type}") diff --git a/bigframes/operations/geo_ops.py b/bigframes/operations/geo_ops.py index a965ddca2b9..75fef1b832e 100644 --- a/bigframes/operations/geo_ops.py +++ b/bigframes/operations/geo_ops.py @@ -15,9 +15,17 @@ import dataclasses from typing import Optional -import bigframes.operations.type as op_typing from bigframes import dtypes from bigframes.operations import base_ops +import bigframes.operations.type as op_typing + +GeoAreaOp = base_ops.create_unary_op( + name="geo_area", + type_signature=op_typing.FixedOutputType( + dtypes.is_geo_like, dtypes.FLOAT_DTYPE, description="geo-like" + ), +) +geo_area_op = GeoAreaOp() GeoStAstextOp = base_ops.create_unary_op( name="geo_st_astext", diff --git a/bigframes/operations/googlesql/__init__.py b/bigframes/operations/googlesql/__init__.py deleted file mode 100644 index edec5b84f8e..00000000000 --- a/bigframes/operations/googlesql/__init__.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from __future__ import annotations - -import dataclasses -import typing - -import bigframes.operations as ops -from bigframes import dtypes - - -@dataclasses.dataclass(frozen=True) -class ArgSpec: - arg_name: str | None = None - optional: bool = False - is_vararg: bool = False - const_only: bool = False - - -@dataclasses.dataclass(frozen=True) -class OpSignature: - # Detailed specs for each parameter. This is particularly relevant for ren - arg_specs: typing.Sequence[ArgSpec] - resolve_return_type: typing.Any - has_varargs: bool = False - - -# Eventually we should migrate every op over to this that can be directly emitted 1:1 as a sql op -# This will allow us to fully lower to pure SQL dialect expressions and emitting sql text is trivial. -@dataclasses.dataclass(frozen=True) -class GoogleSqlScalarOp(ops.NaryOp): - name: typing.ClassVar[str] = "googlesql_scalar" - - # syntax - sql_name: str - args: tuple[ArgSpec, ...] - # typing - signature: typing.Callable[..., dtypes.ExpressionType] - - # semantics - is_deterministic: bool = True - - @property - def deterministic(self) -> bool: - return self.is_deterministic - - def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType: - return self.signature(*input_types) - - -RAND = GoogleSqlScalarOp( - "RAND", args=(), is_deterministic=False, signature=lambda: dtypes.FLOAT_DTYPE -) - - -def _check_geo_input( - t: dtypes.ExpressionType, out: dtypes.ExpressionType -) -> dtypes.ExpressionType: - if t is not None and not dtypes.is_geo_like(t): - raise TypeError(f"Type {t} is not supported. Type must be geo-like") - return out - - -def _check_simplify_inputs( - geo: dtypes.ExpressionType, tol: dtypes.ExpressionType -) -> dtypes.ExpressionType: - if geo is not None and not dtypes.is_geo_like(geo): - raise TypeError(f"Type {geo} is not supported. Type must be geo-like") - if tol is not None and not dtypes.is_numeric(tol): - raise TypeError(f"Type {tol} is not supported. Type must be numeric") - return dtypes.GEO_DTYPE - - -ST_AREA = GoogleSqlScalarOp( - "ST_AREA", - args=(ArgSpec(),), - is_deterministic=True, - signature=lambda geo: _check_geo_input(geo, dtypes.FLOAT_DTYPE), -) - -ST_CENTROID = GoogleSqlScalarOp( - "ST_CENTROID", - args=(ArgSpec(),), - is_deterministic=True, - signature=lambda geo: _check_geo_input(geo, dtypes.GEO_DTYPE), -) - -ST_SIMPLIFY = GoogleSqlScalarOp( - "ST_SIMPLIFY", - args=(ArgSpec(), ArgSpec()), - is_deterministic=True, - signature=_check_simplify_inputs, -) diff --git a/bigframes/operations/googlesql/aead.py b/bigframes/operations/googlesql/aead.py deleted file mode 100644 index f719d7d6989..00000000000 --- a/bigframes/operations/googlesql/aead.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: scripts/data/sql-functions/aead.yaml -# by the script: scripts/generate_bigframes_bigquery.py - -from __future__ import annotations - -from typing import Literal, Union - -import bigframes.core.col -import bigframes.core.googlesql -import bigframes.core.sentinels as sentinels -import bigframes.series as series -from bigframes import dtypes -from bigframes.operations import googlesql - -_DECRYPT_BYTES_OP = googlesql.GoogleSqlScalarOp( - "AEAD.DECRYPT_BYTES", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.BYTES_DTYPE, -) -_DECRYPT_STRING_OP = googlesql.GoogleSqlScalarOp( - "AEAD.DECRYPT_STRING", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.STRING_DTYPE, -) -_ENCRYPT_OP = googlesql.GoogleSqlScalarOp( - "AEAD.ENCRYPT", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.BYTES_DTYPE, -) - - -def decrypt_bytes( - keyset: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, dict], - ], - ciphertext: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes], - ], - additional_data: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Uses the matching key from keyset to decrypt ciphertext and verifies the integrity of the data using additional_data. Returns an error if decryption or verification fails.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _DECRYPT_BYTES_OP, - keyset, - ciphertext, - additional_data, - ) - - -def decrypt_string( - keyset: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, dict], - ], - ciphertext: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes], - ], - additional_data: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Like AEAD.DECRYPT_BYTES, but where additional_data is of type STRING.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _DECRYPT_STRING_OP, - keyset, - ciphertext, - additional_data, - ) - - -def encrypt( - keyset: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, dict], - ], - plaintext: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, str], - ], - additional_data: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, str], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Encrypts plaintext using the primary cryptographic key in keyset. The algorithm of the primary key must be AEAD_AES_GCM_256. Binds the ciphertext to the context defined by additional_data. Returns NULL if any input is NULL.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ENCRYPT_OP, - keyset, - plaintext, - additional_data, - ) diff --git a/bigframes/operations/googlesql/global_namespace/aead_encryption.py b/bigframes/operations/googlesql/global_namespace/aead_encryption.py deleted file mode 100644 index 4613ddd7e6d..00000000000 --- a/bigframes/operations/googlesql/global_namespace/aead_encryption.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: scripts/data/sql-functions/global_namespace/aead_encryption.yaml -# by the script: scripts/generate_bigframes_bigquery.py - -from __future__ import annotations - -from typing import Literal, Union - -import bigframes.core.col -import bigframes.core.googlesql -import bigframes.core.sentinels as sentinels -import bigframes.series as series -from bigframes import dtypes -from bigframes.operations import googlesql - -_DETERMINISTIC_DECRYPT_BYTES_OP = googlesql.GoogleSqlScalarOp( - "DETERMINISTIC_DECRYPT_BYTES", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.BYTES_DTYPE, -) -_DETERMINISTIC_DECRYPT_STRING_OP = googlesql.GoogleSqlScalarOp( - "DETERMINISTIC_DECRYPT_STRING", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.STRING_DTYPE, -) -_DETERMINISTIC_ENCRYPT_OP = googlesql.GoogleSqlScalarOp( - "DETERMINISTIC_ENCRYPT", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.BYTES_DTYPE, -) - - -def deterministic_decrypt_bytes( - keyset: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, dict], - ], - ciphertext: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes], - ], - additional_data: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Uses the matching key from `keyset` to decrypt `ciphertext` and verifies the integrity of the data using `additional_data`. Returns an error if decryption fails.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _DETERMINISTIC_DECRYPT_BYTES_OP, - keyset, - ciphertext, - additional_data, - ) - - -def deterministic_decrypt_string( - keyset: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, dict], - ], - ciphertext: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes], - ], - additional_data: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Like `DETERMINISTIC_DECRYPT_BYTES`, but where plaintext is of type STRING.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _DETERMINISTIC_DECRYPT_STRING_OP, - keyset, - ciphertext, - additional_data, - ) - - -def deterministic_encrypt( - keyset: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, dict], - ], - plaintext: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, str], - ], - additional_data: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, str], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Encrypts `plaintext` using the primary cryptographic key in `keyset` using deterministic AEAD. The algorithm of the primary key must be `DETERMINISTIC_AEAD_AES_SIV_CMAC_256`. Binds the ciphertext to the context defined by `additional_data`. Returns `NULL` if any input is `NULL`.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _DETERMINISTIC_ENCRYPT_OP, - keyset, - plaintext, - additional_data, - ) diff --git a/bigframes/operations/googlesql/global_namespace/array.py b/bigframes/operations/googlesql/global_namespace/array.py deleted file mode 100644 index 94adbad1839..00000000000 --- a/bigframes/operations/googlesql/global_namespace/array.py +++ /dev/null @@ -1,892 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: scripts/data/sql-functions/global_namespace/array.yaml -# by the script: scripts/generate_bigframes_bigquery.py - -from __future__ import annotations - -import decimal -from typing import Any, Literal, Union - -import bigframes.core.col -import bigframes.core.googlesql -import bigframes.core.sentinels as sentinels -import bigframes.series as series -from bigframes import dtypes -from bigframes.operations import googlesql - - -def _ARRAY_CONCAT_SIG(*args): - # Pad args with None to match max expected args - args = args + (None,) * (2 - len(args)) - # Try matching impl 0 - any1_val = None - match_ok = True - if match_ok and args[0] is not None: - if not dtypes.is_array_like(args[0]): - match_ok = False - else: - inner = dtypes.get_array_inner_type(args[0]) - if any1_val is not None: - try: - any1_val = dtypes.coerce_to_common(any1_val, inner) - except TypeError: - match_ok = False - else: - any1_val = inner - if match_ok and args[1] is not None: - if not dtypes.is_array_like(args[1]): - match_ok = False - else: - inner = dtypes.get_array_inner_type(args[1]) - if any1_val is not None: - try: - any1_val = dtypes.coerce_to_common(any1_val, inner) - except TypeError: - match_ok = False - else: - any1_val = inner - if match_ok: - if any1_val is not None: - return dtypes.list_type(any1_val) - else: - return None - - raise TypeError( - f"Could not find matching signature for array_concat with argument types: {[str(t) for t in args]}" - ) - - -_ARRAY_CONCAT_OP = googlesql.GoogleSqlScalarOp( - "ARRAY_CONCAT", - args=(googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=_ARRAY_CONCAT_SIG, -) - - -def _ARRAY_FIRST_SIG(*args): - # Pad args with None to match max expected args - args = args + (None,) * (1 - len(args)) - # Try matching impl 0 - any1_val = None - match_ok = True - if match_ok and args[0] is not None: - if not dtypes.is_array_like(args[0]): - match_ok = False - else: - inner = dtypes.get_array_inner_type(args[0]) - if any1_val is not None: - try: - any1_val = dtypes.coerce_to_common(any1_val, inner) - except TypeError: - match_ok = False - else: - any1_val = inner - if match_ok: - return any1_val - - raise TypeError( - f"Could not find matching signature for array_first with argument types: {[str(t) for t in args]}" - ) - - -_ARRAY_FIRST_OP = googlesql.GoogleSqlScalarOp( - "ARRAY_FIRST", - args=(googlesql.ArgSpec(),), - signature=_ARRAY_FIRST_SIG, -) - - -def _ARRAY_FIRST_N_SIG(*args): - # Pad args with None to match max expected args - args = args + (None,) * (2 - len(args)) - # Try matching impl 0 - any1_val = None - match_ok = True - if match_ok and args[0] is not None: - if not dtypes.is_array_like(args[0]): - match_ok = False - else: - inner = dtypes.get_array_inner_type(args[0]) - if any1_val is not None: - try: - any1_val = dtypes.coerce_to_common(any1_val, inner) - except TypeError: - match_ok = False - else: - any1_val = inner - if match_ok and args[1] is not None: - try: - if dtypes.coerce_to_common(args[1], dtypes.INT_DTYPE) != dtypes.INT_DTYPE: - match_ok = False - except TypeError: - match_ok = False - if match_ok: - if any1_val is not None: - return dtypes.list_type(any1_val) - else: - return None - - raise TypeError( - f"Could not find matching signature for array_first_n with argument types: {[str(t) for t in args]}" - ) - - -_ARRAY_FIRST_N_OP = googlesql.GoogleSqlScalarOp( - "ARRAY_FIRST_N", - args=(googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=_ARRAY_FIRST_N_SIG, -) -_ARRAY_INCLUDES_OP = googlesql.GoogleSqlScalarOp( - "ARRAY_INCLUDES", - args=(googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.BOOL_DTYPE, -) -_ARRAY_INCLUDES_ALL_OP = googlesql.GoogleSqlScalarOp( - "ARRAY_INCLUDES_ALL", - args=(googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.BOOL_DTYPE, -) -_ARRAY_INCLUDES_ANY_OP = googlesql.GoogleSqlScalarOp( - "ARRAY_INCLUDES_ANY", - args=(googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.BOOL_DTYPE, -) -_ARRAY_IS_DISTINCT_OP = googlesql.GoogleSqlScalarOp( - "ARRAY_IS_DISTINCT", - args=(googlesql.ArgSpec(),), - signature=lambda *args: dtypes.BOOL_DTYPE, -) - - -def _ARRAY_LAST_SIG(*args): - # Pad args with None to match max expected args - args = args + (None,) * (1 - len(args)) - # Try matching impl 0 - any1_val = None - match_ok = True - if match_ok and args[0] is not None: - if not dtypes.is_array_like(args[0]): - match_ok = False - else: - inner = dtypes.get_array_inner_type(args[0]) - if any1_val is not None: - try: - any1_val = dtypes.coerce_to_common(any1_val, inner) - except TypeError: - match_ok = False - else: - any1_val = inner - if match_ok: - return any1_val - - raise TypeError( - f"Could not find matching signature for array_last with argument types: {[str(t) for t in args]}" - ) - - -_ARRAY_LAST_OP = googlesql.GoogleSqlScalarOp( - "ARRAY_LAST", - args=(googlesql.ArgSpec(),), - signature=_ARRAY_LAST_SIG, -) -_ARRAY_LENGTH_OP = googlesql.GoogleSqlScalarOp( - "ARRAY_LENGTH", - args=(googlesql.ArgSpec(),), - signature=lambda *args: dtypes.INT_DTYPE, -) - - -def _ARRAY_REVERSE_SIG(*args): - # Pad args with None to match max expected args - args = args + (None,) * (1 - len(args)) - # Try matching impl 0 - any1_val = None - match_ok = True - if match_ok and args[0] is not None: - if not dtypes.is_array_like(args[0]): - match_ok = False - else: - inner = dtypes.get_array_inner_type(args[0]) - if any1_val is not None: - try: - any1_val = dtypes.coerce_to_common(any1_val, inner) - except TypeError: - match_ok = False - else: - any1_val = inner - if match_ok: - if any1_val is not None: - return dtypes.list_type(any1_val) - else: - return None - - raise TypeError( - f"Could not find matching signature for array_reverse with argument types: {[str(t) for t in args]}" - ) - - -_ARRAY_REVERSE_OP = googlesql.GoogleSqlScalarOp( - "ARRAY_REVERSE", - args=(googlesql.ArgSpec(),), - signature=_ARRAY_REVERSE_SIG, -) - - -def _ARRAY_SLICE_SIG(*args): - # Pad args with None to match max expected args - args = args + (None,) * (3 - len(args)) - # Try matching impl 0 - any1_val = None - match_ok = True - if match_ok and args[0] is not None: - if not dtypes.is_array_like(args[0]): - match_ok = False - else: - inner = dtypes.get_array_inner_type(args[0]) - if any1_val is not None: - try: - any1_val = dtypes.coerce_to_common(any1_val, inner) - except TypeError: - match_ok = False - else: - any1_val = inner - if match_ok and args[1] is not None: - try: - if dtypes.coerce_to_common(args[1], dtypes.INT_DTYPE) != dtypes.INT_DTYPE: - match_ok = False - except TypeError: - match_ok = False - if match_ok and args[2] is not None: - try: - if dtypes.coerce_to_common(args[2], dtypes.INT_DTYPE) != dtypes.INT_DTYPE: - match_ok = False - except TypeError: - match_ok = False - if match_ok: - if any1_val is not None: - return dtypes.list_type(any1_val) - else: - return None - - raise TypeError( - f"Could not find matching signature for array_slice with argument types: {[str(t) for t in args]}" - ) - - -_ARRAY_SLICE_OP = googlesql.GoogleSqlScalarOp( - "ARRAY_SLICE", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=_ARRAY_SLICE_SIG, -) - - -def _ARRAY_TO_STRING_SIG(*args): - # Pad args with None to match max expected args - args = args + (None,) * (3 - len(args)) - # Try matching impl 0 - match_ok = True - if match_ok and args[0] is not None: - if not dtypes.is_array_like(args[0]): - match_ok = False - else: - inner = dtypes.get_array_inner_type(args[0]) - try: - if ( - dtypes.coerce_to_common(inner, dtypes.STRING_DTYPE) - != dtypes.STRING_DTYPE - ): - match_ok = False - except TypeError: - match_ok = False - if match_ok and args[1] is not None: - try: - if ( - dtypes.coerce_to_common(args[1], dtypes.STRING_DTYPE) - != dtypes.STRING_DTYPE - ): - match_ok = False - except TypeError: - match_ok = False - if match_ok and args[2] is not None: - try: - if ( - dtypes.coerce_to_common(args[2], dtypes.STRING_DTYPE) - != dtypes.STRING_DTYPE - ): - match_ok = False - except TypeError: - match_ok = False - if match_ok: - return dtypes.STRING_DTYPE - - # Try matching impl 1 - match_ok = True - if match_ok and args[0] is not None: - if not dtypes.is_array_like(args[0]): - match_ok = False - else: - inner = dtypes.get_array_inner_type(args[0]) - try: - if ( - dtypes.coerce_to_common(inner, dtypes.BYTES_DTYPE) - != dtypes.BYTES_DTYPE - ): - match_ok = False - except TypeError: - match_ok = False - if match_ok and args[1] is not None: - try: - if ( - dtypes.coerce_to_common(args[1], dtypes.BYTES_DTYPE) - != dtypes.BYTES_DTYPE - ): - match_ok = False - except TypeError: - match_ok = False - if match_ok and args[2] is not None: - try: - if ( - dtypes.coerce_to_common(args[2], dtypes.BYTES_DTYPE) - != dtypes.BYTES_DTYPE - ): - match_ok = False - except TypeError: - match_ok = False - if match_ok: - return dtypes.BYTES_DTYPE - - raise TypeError( - f"Could not find matching signature for array_to_string with argument types: {[str(t) for t in args]}" - ) - - -_ARRAY_TO_STRING_OP = googlesql.GoogleSqlScalarOp( - "ARRAY_TO_STRING", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec(optional=True)), - signature=_ARRAY_TO_STRING_SIG, -) - - -def _FLATTEN_SIG(*args): - # Pad args with None to match max expected args - args = args + (None,) * (2 - len(args)) - # Try matching impl 0 - any1_val = None - match_ok = True - if match_ok and args[0] is not None: - if not dtypes.is_array_like(args[0]): - match_ok = False - else: - inner = dtypes.get_array_inner_type(args[0]) - if any1_val is not None: - try: - any1_val = dtypes.coerce_to_common(any1_val, inner) - except TypeError: - match_ok = False - else: - any1_val = inner - if match_ok and args[1] is not None: - try: - if dtypes.coerce_to_common(args[1], dtypes.INT_DTYPE) != dtypes.INT_DTYPE: - match_ok = False - except TypeError: - match_ok = False - if match_ok: - if any1_val is not None: - return dtypes.list_type(any1_val) - else: - return None - - raise TypeError( - f"Could not find matching signature for flatten with argument types: {[str(t) for t in args]}" - ) - - -_FLATTEN_OP = googlesql.GoogleSqlScalarOp( - "FLATTEN", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(arg_name="depth", optional=True)), - signature=_FLATTEN_SIG, -) - - -def _GENERATE_ARRAY_SIG(*args): - # Pad args with None to match max expected args - args = args + (None,) * (3 - len(args)) - # Try matching impl 0 - match_ok = True - if match_ok and args[0] is not None: - try: - if dtypes.coerce_to_common(args[0], dtypes.INT_DTYPE) != dtypes.INT_DTYPE: - match_ok = False - except TypeError: - match_ok = False - if match_ok and args[1] is not None: - try: - if dtypes.coerce_to_common(args[1], dtypes.INT_DTYPE) != dtypes.INT_DTYPE: - match_ok = False - except TypeError: - match_ok = False - if match_ok and args[2] is not None: - try: - if dtypes.coerce_to_common(args[2], dtypes.INT_DTYPE) != dtypes.INT_DTYPE: - match_ok = False - except TypeError: - match_ok = False - if match_ok: - return dtypes.list_type(dtypes.INT_DTYPE) - - # Try matching impl 1 - match_ok = True - if match_ok and args[0] is not None: - try: - if ( - dtypes.coerce_to_common(args[0], dtypes.NUMERIC_DTYPE) - != dtypes.NUMERIC_DTYPE - ): - match_ok = False - except TypeError: - match_ok = False - if match_ok and args[1] is not None: - try: - if ( - dtypes.coerce_to_common(args[1], dtypes.NUMERIC_DTYPE) - != dtypes.NUMERIC_DTYPE - ): - match_ok = False - except TypeError: - match_ok = False - if match_ok and args[2] is not None: - try: - if ( - dtypes.coerce_to_common(args[2], dtypes.NUMERIC_DTYPE) - != dtypes.NUMERIC_DTYPE - ): - match_ok = False - except TypeError: - match_ok = False - if match_ok: - return dtypes.list_type(dtypes.NUMERIC_DTYPE) - - # Try matching impl 2 - match_ok = True - if match_ok and args[0] is not None: - try: - if ( - dtypes.coerce_to_common(args[0], dtypes.FLOAT_DTYPE) - != dtypes.FLOAT_DTYPE - ): - match_ok = False - except TypeError: - match_ok = False - if match_ok and args[1] is not None: - try: - if ( - dtypes.coerce_to_common(args[1], dtypes.FLOAT_DTYPE) - != dtypes.FLOAT_DTYPE - ): - match_ok = False - except TypeError: - match_ok = False - if match_ok and args[2] is not None: - try: - if ( - dtypes.coerce_to_common(args[2], dtypes.FLOAT_DTYPE) - != dtypes.FLOAT_DTYPE - ): - match_ok = False - except TypeError: - match_ok = False - if match_ok: - return dtypes.list_type(dtypes.FLOAT_DTYPE) - - raise TypeError( - f"Could not find matching signature for generate_array with argument types: {[str(t) for t in args]}" - ) - - -_GENERATE_ARRAY_OP = googlesql.GoogleSqlScalarOp( - "GENERATE_ARRAY", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec(optional=True)), - signature=_GENERATE_ARRAY_SIG, -) - - -def array_concat( - array_expression_1: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - array_expression_2: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Concatenates one or more arrays with the same element type into a single array.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ARRAY_CONCAT_OP, - array_expression_1, - array_expression_2, - ) - - -def array_first( - array_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Takes an array and returns the first element in the array.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ARRAY_FIRST_OP, - array_expression, - ) - - -def array_first_n( - input_array: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - n: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Returns a prefix of `input_array` consisting of the first `n` elements.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ARRAY_FIRST_N_OP, - input_array, - n, - ) - - -def array_includes( - array_to_search: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - search_value: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Takes an array and returns `TRUE` if there is an element in the array that is equal to the search_value.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ARRAY_INCLUDES_OP, - array_to_search, - search_value, - ) - - -def array_includes_all( - array_to_search: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - search_values: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Takes an array to search and an array of search values. Returns `TRUE` if all search values are in the array to search, otherwise returns `FALSE`.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ARRAY_INCLUDES_ALL_OP, - array_to_search, - search_values, - ) - - -def array_includes_any( - array_to_search: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - search_values: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Takes an array to search and an array of search values. Returns `TRUE` if any search values are in the array to search, otherwise returns `FALSE`.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ARRAY_INCLUDES_ANY_OP, - array_to_search, - search_values, - ) - - -def array_is_distinct( - array_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Returns `TRUE` if the array contains no repeated elements, using the same equality comparison logic as `SELECT DISTINCT`.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ARRAY_IS_DISTINCT_OP, - array_expression, - ) - - -def array_last( - array_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Takes an array and returns the last element in the array.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ARRAY_LAST_OP, - array_expression, - ) - - -def array_length( - series: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Compute the length of each array element in the Series. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - - >>> s = bpd.Series([[1, 2, 8, 3], [], [3, 4]]) - >>> bbq.array_length(s) - 0 4 - 1 0 - 2 2 - dtype: Int64 - - You can call this function using the Series `bigquery` accessor. - - >>> s.bigquery.array_length() - 0 4 - 1 0 - 2 2 - dtype: Int64 - - You can also use this accessor on a pandas Series after importing bigframes. - - >>> import bigframes - >>> import pandas as pd - >>> ps = pd.Series([[1, 2, 8, 3], [], [3, 4]]) - >>> ps.bigquery.array_length() - 0 4 - 1 0 - 2 2 - dtype: Int64 - - You can also apply this function directly to Series using `apply`. - - >>> s.apply(bbq.array_length, by_row=False) - 0 4 - 1 0 - 2 2 - dtype: Int64 - - Args: - series (bigframes.series.Series): A Series with array columns. - - Returns: - bigframes.series.Series: A Series of integer values indicating - the length of each element in the Series. - """ - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ARRAY_LENGTH_OP, - series, - ) - - -def array_reverse( - value: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Returns the input `ARRAY` with elements in reverse order.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ARRAY_REVERSE_OP, - value, - ) - - -def array_slice( - array_to_slice: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - start_offset: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ], - end_offset: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Returns an array containing zero or more consecutive elements from the input array.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ARRAY_SLICE_OP, - array_to_slice, - start_offset, - end_offset, - ) - - -def array_to_string( - series: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - delimiter: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, str], - ], - null_text: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, -) -> Union[series.Series, bigframes.core.col.Expression]: - """Converts array elements within a Series into delimited strings. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - - >>> s = bpd.Series([["H", "i", "!"], ["Hello", "World"], np.nan, [], ["Hi"]]) - >>> bbq.array_to_string(s, delimiter=", ") - 0 H, i, ! - 1 Hello, World - 2 - 3 - 4 Hi - dtype: string - - You can call this function using the Series `bigquery` accessor. - - >>> s.bigquery.array_to_string(delimiter=", ") - 0 H, i, ! - 1 Hello, World - 2 - 3 - 4 Hi - dtype: string - - You can also use this accessor on a pandas Series after importing bigframes. - - >>> import bigframes - >>> import pandas as pd - >>> ps = pd.Series([["H", "i", "!"], ["Hello", "World"], None, [], ["Hi"]]) - >>> ps.bigquery.array_to_string(delimiter=", ") - 0 H, i, ! - 1 Hello, World - 2 - 3 - 4 Hi - dtype: string - - Args: - series (bigframes.series.Series): A Series containing arrays. - delimiter (str): The string used to separate array elements. - null_text (str, optional): The string to replace any NULL values in the array with. - - Returns: - bigframes.series.Series: A Series containing delimited strings. - """ - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _ARRAY_TO_STRING_OP, - series, - delimiter, - null_text, - ) - - -def flatten( - array_to_flatten: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - depth: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, -) -> Union[series.Series, bigframes.core.col.Expression]: - """Takes an array of nested data and flattens a specific part of it into a single, flat array with the [array elements field access operator][array-el-field-operator]. Returns `NULL` if the input value is `NULL`.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _FLATTEN_OP, - array_to_flatten, - depth, - ) - - -def generate_array( - start_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[ - Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], decimal.Decimal, float, int - ], - ], - end_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[ - Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], decimal.Decimal, float, int - ], - ], - step_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[ - Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], decimal.Decimal, float, int - ], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, -) -> Union[series.Series, bigframes.core.col.Expression]: - """Returns an array of values. The `start_expression` and `end_expression` parameters determine the inclusive start and end of the array.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _GENERATE_ARRAY_OP, - start_expression, - end_expression, - step_expression, - ) diff --git a/bigframes/operations/googlesql/global_namespace/bit.py b/bigframes/operations/googlesql/global_namespace/bit.py deleted file mode 100644 index e0c22dfc299..00000000000 --- a/bigframes/operations/googlesql/global_namespace/bit.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: scripts/data/sql-functions/global_namespace/bit.yaml -# by the script: scripts/generate_bigframes_bigquery.py - -from __future__ import annotations - -from typing import Any, Literal, Union - -import bigframes.core.col -import bigframes.core.googlesql -import bigframes.core.sentinels as sentinels -import bigframes.series as series -from bigframes import dtypes -from bigframes.operations import googlesql - -_BIT_COUNT_OP = googlesql.GoogleSqlScalarOp( - "BIT_COUNT", - args=(googlesql.ArgSpec(),), - signature=lambda *args: dtypes.INT_DTYPE, -) - - -def bit_count( - expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], bytes, int], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """The input, `expression`, must be an integer or `BYTES`. Returns the number of bits that are set in the input expression. For signed integers, this is the number of bits in two's complement form.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _BIT_COUNT_OP, - expression, - ) diff --git a/bigframes/operations/googlesql/global_namespace/conversion.py b/bigframes/operations/googlesql/global_namespace/conversion.py deleted file mode 100644 index cea4e45d836..00000000000 --- a/bigframes/operations/googlesql/global_namespace/conversion.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: scripts/data/sql-functions/global_namespace/conversion.yaml -# by the script: scripts/generate_bigframes_bigquery.py - -from __future__ import annotations - -import datetime -from typing import Literal, Union - -import bigframes.core.col -import bigframes.core.googlesql -import bigframes.core.sentinels as sentinels -import bigframes.series as series -from bigframes import dtypes -from bigframes.operations import googlesql - -_BOOL_OP = googlesql.GoogleSqlScalarOp( - "BOOL", - args=(googlesql.ArgSpec(),), - signature=lambda *args: dtypes.BOOL_DTYPE, -) -_DOUBLE_OP = googlesql.GoogleSqlScalarOp( - "DOUBLE", - args=( - googlesql.ArgSpec(), - googlesql.ArgSpec(arg_name="wide_number_mode", optional=True), - ), - signature=lambda *args: dtypes.FLOAT_DTYPE, -) -_FLOAT64_OP = googlesql.GoogleSqlScalarOp( - "FLOAT64", - args=( - googlesql.ArgSpec(), - googlesql.ArgSpec(arg_name="wide_number_mode", optional=True), - ), - signature=lambda *args: dtypes.FLOAT_DTYPE, -) -_INT64_OP = googlesql.GoogleSqlScalarOp( - "INT64", - args=(googlesql.ArgSpec(),), - signature=lambda *args: dtypes.INT_DTYPE, -) -_PARSE_BIGNUMERIC_OP = googlesql.GoogleSqlScalarOp( - "PARSE_BIGNUMERIC", - args=(googlesql.ArgSpec(),), - signature=lambda *args: dtypes.BIGNUMERIC_DTYPE, -) -_PARSE_NUMERIC_OP = googlesql.GoogleSqlScalarOp( - "PARSE_NUMERIC", - args=(googlesql.ArgSpec(),), - signature=lambda *args: dtypes.NUMERIC_DTYPE, -) -_STRING_OP = googlesql.GoogleSqlScalarOp( - "STRING", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(optional=True)), - signature=lambda *args: dtypes.STRING_DTYPE, -) - - -def bool_( - json_string_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Converts a JSON boolean to a SQL BOOL value.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _BOOL_OP, - json_string_expression, - ) - - -def double( - json_string_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], - wide_number_mode: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, -) -> Union[series.Series, bigframes.core.col.Expression]: - """Converts a JSON number to a SQL FLOAT64 value.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _DOUBLE_OP, - json_string_expression, - wide_number_mode, - ) - - -def float64( - json_string_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], - wide_number_mode: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, -) -> Union[series.Series, bigframes.core.col.Expression]: - """Converts a JSON number to a SQL FLOAT64 value.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _FLOAT64_OP, - json_string_expression, - wide_number_mode, - ) - - -def int64( - json_string_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Converts a JSON number to a SQL INT64 value.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _INT64_OP, - json_string_expression, - ) - - -def parse_bignumeric( - string_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Converts a STRING to a BIGNUMERIC value.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _PARSE_BIGNUMERIC_OP, - string_expression, - ) - - -def parse_numeric( - string_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Converts a STRING to a NUMERIC value.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _PARSE_NUMERIC_OP, - string_expression, - ) - - -def string( - expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[ - Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], - datetime.date, - datetime.datetime, - datetime.time, - str, - ], - ], - timezone: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, -) -> Union[series.Series, bigframes.core.col.Expression]: - """Converts a value to a STRING value.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _STRING_OP, - expression, - timezone, - ) diff --git a/bigframes/operations/googlesql/global_namespace/date.py b/bigframes/operations/googlesql/global_namespace/date.py deleted file mode 100644 index b6cfc9722b5..00000000000 --- a/bigframes/operations/googlesql/global_namespace/date.py +++ /dev/null @@ -1,412 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: scripts/data/sql-functions/global_namespace/date.yaml -# by the script: scripts/generate_bigframes_bigquery.py - -from __future__ import annotations - -import datetime -from typing import Any, Literal, Union - -import bigframes.core.col -import bigframes.core.googlesql -import bigframes.core.sentinels as sentinels -import bigframes.series as series -from bigframes import dtypes -from bigframes.operations import googlesql - -_CURRENT_DATE_OP = googlesql.GoogleSqlScalarOp( - "CURRENT_DATE", - args=(googlesql.ArgSpec(optional=True),), - signature=lambda *args: dtypes.DATE_DTYPE, -) -_DATE_OP = googlesql.GoogleSqlScalarOp( - "DATE", - args=( - googlesql.ArgSpec(optional=True), - googlesql.ArgSpec(optional=True), - googlesql.ArgSpec(optional=True), - googlesql.ArgSpec(optional=True), - googlesql.ArgSpec(optional=True), - ), - signature=lambda *args: dtypes.DATE_DTYPE, -) -_DATE_ADD_OP = googlesql.GoogleSqlScalarOp( - "DATE_ADD", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.DATE_DTYPE, -) -_DATE_DIFF_OP = googlesql.GoogleSqlScalarOp( - "DATE_DIFF", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.INT_DTYPE, -) -_DATE_FROM_UNIX_DATE_OP = googlesql.GoogleSqlScalarOp( - "DATE_FROM_UNIX_DATE", - args=(googlesql.ArgSpec(),), - signature=lambda *args: dtypes.DATE_DTYPE, -) -_DATE_SUB_OP = googlesql.GoogleSqlScalarOp( - "DATE_SUB", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.DATE_DTYPE, -) -_DATE_TRUNC_OP = googlesql.GoogleSqlScalarOp( - "DATE_TRUNC", - args=(googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.DATE_DTYPE, -) -_EXTRACT_OP = googlesql.GoogleSqlScalarOp( - "EXTRACT", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(), googlesql.ArgSpec(optional=True)), - signature=lambda *args: dtypes.INT_DTYPE, -) -_FORMAT_DATE_OP = googlesql.GoogleSqlScalarOp( - "FORMAT_DATE", - args=(googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.STRING_DTYPE, -) -_GENERATE_DATE_ARRAY_OP = googlesql.GoogleSqlScalarOp( - "GENERATE_DATE_ARRAY", - args=( - googlesql.ArgSpec(), - googlesql.ArgSpec(), - googlesql.ArgSpec(optional=True), - googlesql.ArgSpec(optional=True), - ), - signature=lambda *args: dtypes.list_type(dtypes.DATE_DTYPE), -) -_LAST_DAY_OP = googlesql.GoogleSqlScalarOp( - "LAST_DAY", - args=(googlesql.ArgSpec(), googlesql.ArgSpec(optional=True)), - signature=lambda *args: dtypes.DATE_DTYPE, -) -_PARSE_DATE_OP = googlesql.GoogleSqlScalarOp( - "PARSE_DATE", - args=(googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: dtypes.DATE_DTYPE, -) -_UNIX_DATE_OP = googlesql.GoogleSqlScalarOp( - "UNIX_DATE", - args=(googlesql.ArgSpec(),), - signature=lambda *args: dtypes.INT_DTYPE, -) - - -def current_date( - time_zone_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, -) -> Union[series.Series, bigframes.core.col.Expression]: - """Returns the current date as a DATE object. Parentheses are optional when called with no arguments.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _CURRENT_DATE_OP, - time_zone_expression, - ) - - -def date( - expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[ - Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], - datetime.date, - datetime.datetime, - str, - ], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - time_zone_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - year: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - month: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - day: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, -) -> Union[series.Series, bigframes.core.col.Expression]: - """Constructs or extracts a date.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _DATE_OP, - expression, - time_zone_expression, - year, - month, - day, - ) - - -def date_add( - date_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], datetime.date], - ], - int64_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ], - date_part: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Adds a specified time interval to a DATE.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _DATE_ADD_OP, - date_expression, - int64_expression, - date_part, - ) - - -def date_diff( - end_date: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], datetime.date], - ], - start_date: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], datetime.date], - ], - granularity: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Gets the number of unit boundaries between two DATE values (end_date - start_date) at a particular time granularity.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _DATE_DIFF_OP, - end_date, - start_date, - granularity, - ) - - -def date_from_unix_date( - int64_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Interprets an INT64 expression as the number of days since 1970-01-01.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _DATE_FROM_UNIX_DATE_OP, - int64_expression, - ) - - -def date_sub( - date_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], datetime.date], - ], - int64_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ], - date_part: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Subtracts a specified time interval from a DATE.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _DATE_SUB_OP, - date_expression, - int64_expression, - date_part, - ) - - -def date_trunc( - date_value: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], datetime.date], - ], - granularity: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Truncates a DATE, DATETIME, or TIMESTAMP value at a particular granularity.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _DATE_TRUNC_OP, - date_value, - granularity, - ) - - -def extract( - date_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[ - Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], - datetime.date, - datetime.datetime, - datetime.time, - ], - ], - part: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ], - time_zone: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, -) -> Union[series.Series, bigframes.core.col.Expression]: - """Returns the value corresponding to the specified date part.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _EXTRACT_OP, - date_expression, - part, - time_zone, - ) - - -def format_date( - format_string: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], - date_expr: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], datetime.date], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Formats a DATE value according to a specified format string.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _FORMAT_DATE_OP, - format_string, - date_expr, - ) - - -def generate_date_array( - start_date: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], datetime.date], - ], - end_date: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], datetime.date], - ], - int64_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], int], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, - date_part: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, -) -> Union[series.Series, bigframes.core.col.Expression]: - """Generates an array of dates in a range.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _GENERATE_DATE_ARRAY_OP, - start_date, - end_date, - int64_expression, - date_part, - ) - - -def last_day( - date_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], datetime.date], - ], - date_part: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Any, Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]], - ] = sentinels.Sentinel.ARGUMENT_DEFAULT, -) -> Union[series.Series, bigframes.core.col.Expression]: - """Returns the last day from a date expression. This is commonly used to return the last day of the month.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _LAST_DAY_OP, - date_expression, - date_part, - ) - - -def parse_date( - format_string: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], - date_string: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], str], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Converts a STRING value to a DATE value.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _PARSE_DATE_OP, - format_string, - date_string, - ) - - -def unix_date( - date_expression: Union[ - series.Series, - bigframes.core.col.Expression, - Union[Literal[sentinels.Sentinel.ARGUMENT_DEFAULT], datetime.date], - ], -) -> Union[series.Series, bigframes.core.col.Expression]: - """Returns the number of days since 1970-01-01.""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - _UNIX_DATE_OP, - date_expression, - ) diff --git a/bigframes/operations/json_ops.py b/bigframes/operations/json_ops.py index c9b5849f9ed..7260a792230 100644 --- a/bigframes/operations/json_ops.py +++ b/bigframes/operations/json_ops.py @@ -105,11 +105,10 @@ def output_type(self, *input_types): @dataclasses.dataclass(frozen=True) class ToJSON(base_ops.UnaryOp): name: typing.ClassVar[str] = "to_json" - safe: bool = True def output_type(self, *input_types): input_type = input_types[0] - if not dtypes.is_json_encoding_type(input_type, strict=True): + if not dtypes.is_json_encoding_type(input_type): raise TypeError( "The value to be assigned must be a type that can be encoded as JSON." + f"Received type: {input_type}" @@ -221,7 +220,6 @@ def output_type(self, *input_types): class JSONDecode(base_ops.UnaryOp): name: typing.ClassVar[str] = "json_decode" to_type: dtypes.Dtype - safe: bool = True def output_type(self, *input_types): input_type = input_types[0] @@ -230,11 +228,4 @@ def output_type(self, *input_types): "Input type must be a valid JSON object or JSON-formatted string type." + f" Received type: {input_type}" ) - if self.to_type not in ( - dtypes.INT_DTYPE, - dtypes.FLOAT_DTYPE, - dtypes.BOOL_DTYPE, - dtypes.STRING_DTYPE, - ): - raise TypeError(f"Cannot cast from {dtypes.JSON_DTYPE} to {self.to_type}") return self.to_type diff --git a/bigframes/operations/lists.py b/bigframes/operations/lists.py index c0ff8d51650..9974e686933 100644 --- a/bigframes/operations/lists.py +++ b/bigframes/operations/lists.py @@ -19,16 +19,16 @@ import bigframes_vendored.pandas.core.arrays.arrow.accessors as vendoracessors -import bigframes.operations as ops -import bigframes.series as series -from bigframes._tools import docs from bigframes.core.logging import log_adapter +import bigframes.operations as ops from bigframes.operations._op_converters import convert_index, convert_slice +import bigframes.series as series @log_adapter.class_logger -@docs.inherit_docs(vendoracessors.ListAccessor) -class ListAccessor: +class ListAccessor(vendoracessors.ListAccessor): + __doc__ = vendoracessors.ListAccessor.__doc__ + def __init__(self, data: series.Series): self._data = data diff --git a/bigframes/operations/numeric_ops.py b/bigframes/operations/numeric_ops.py index af1eef74527..83e2078c881 100644 --- a/bigframes/operations/numeric_ops.py +++ b/bigframes/operations/numeric_ops.py @@ -15,9 +15,9 @@ import dataclasses import typing -import bigframes.operations.type as op_typing from bigframes import dtypes from bigframes.operations import base_ops +import bigframes.operations.type as op_typing SinOp = base_ops.create_unary_op( name="sin", type_signature=op_typing.UNARY_REAL_NUMERIC diff --git a/bigframes/operations/plotting.py b/bigframes/operations/plotting.py index ecaa28e9747..21a23a9ab54 100644 --- a/bigframes/operations/plotting.py +++ b/bigframes/operations/plotting.py @@ -17,14 +17,14 @@ import bigframes_vendored.constants as constants import bigframes_vendored.pandas.plotting._core as vendordt -import bigframes.operations._matplotlib as bfplt -from bigframes._tools import docs from bigframes.core.logging import log_adapter +import bigframes.operations._matplotlib as bfplt @log_adapter.class_logger -@docs.inherit_docs(vendordt.PlotAccessor) -class PlotAccessor: +class PlotAccessor(vendordt.PlotAccessor): + __doc__ = vendordt.PlotAccessor.__doc__ + _common_kinds = ("line", "area", "hist", "bar", "barh", "pie") _dataframe_kinds = ("scatter", "hexbin,") _all_kinds = _common_kinds + _dataframe_kinds diff --git a/bigframes/operations/python_op_maps.py b/bigframes/operations/python_op_maps.py index b4c58e14c7b..39f153ec05d 100644 --- a/bigframes/operations/python_op_maps.py +++ b/bigframes/operations/python_op_maps.py @@ -22,7 +22,6 @@ array_ops, bool_ops, comparison_ops, - generic_ops, numeric_ops, string_ops, ) @@ -48,8 +47,6 @@ operator.and_: bool_ops.and_op, operator.or_: bool_ops.or_op, operator.xor: bool_ops.xor_op, - operator.invert: generic_ops.invert_op, - operator.not_: generic_ops.invert_op, ## math math.log: numeric_ops.ln_op, math.log10: numeric_ops.log10_op, @@ -69,25 +66,16 @@ ## str str.upper: string_ops.upper_op, str.lower: string_ops.lower_op, - str.isalnum: string_ops.isalnum_op, - str.isalpha: string_ops.isalpha_op, - str.isdecimal: string_ops.isdecimal_op, - str.isdigit: string_ops.isdigit_op, - str.isnumeric: string_ops.isnumeric_op, - str.isspace: string_ops.isspace_op, - str.islower: string_ops.islower_op, - str.isupper: string_ops.isupper_op, - str.capitalize: string_ops.capitalize_op, ## builtins len: string_ops.len_op, abs: numeric_ops.abs_op, pow: numeric_ops.pow_op, ### builtins -- iterable - all: array_ops.ArrayReduceOp(aggregations.all_op), # type: ignore - any: array_ops.ArrayReduceOp(aggregations.any_op), # type: ignore - sum: array_ops.ArrayReduceOp(aggregations.sum_op), # type: ignore - min: array_ops.ArrayReduceOp(aggregations.min_op), # type: ignore - max: array_ops.ArrayReduceOp(aggregations.max_op), # type: ignore + all: array_ops.ArrayReduceOp(aggregations.all_op), + any: array_ops.ArrayReduceOp(aggregations.any_op), + sum: array_ops.ArrayReduceOp(aggregations.sum_op), + min: array_ops.ArrayReduceOp(aggregations.min_op), + max: array_ops.ArrayReduceOp(aggregations.max_op), } @@ -95,32 +83,3 @@ def python_callable_to_op(obj) -> Optional[bigframes.operations.RowOp]: if obj in PYTHON_TO_BIGFRAMES: return PYTHON_TO_BIGFRAMES[obj] return None - - -SERIES_METHOD_TO_OP = { - "abs": numeric_ops.abs_op, - "sqrt": numeric_ops.sqrt_op, - "sin": numeric_ops.sin_op, - "cos": numeric_ops.cos_op, - "tan": numeric_ops.tan_op, - "log": numeric_ops.ln_op, - "log10": numeric_ops.log10_op, - "exp": numeric_ops.exp_op, - "floor": numeric_ops.floor_op, - "ceil": numeric_ops.ceil_op, - "isnull": generic_ops.isnull_op, - "isna": generic_ops.isnull_op, - "notnull": generic_ops.notnull_op, - "notna": generic_ops.notnull_op, - "upper": string_ops.upper_op, - "lower": string_ops.lower_op, - "isalnum": string_ops.isalnum_op, - "isalpha": string_ops.isalpha_op, - "isdecimal": string_ops.isdecimal_op, - "isdigit": string_ops.isdigit_op, - "isnumeric": string_ops.isnumeric_op, - "isspace": string_ops.isspace_op, - "islower": string_ops.islower_op, - "isupper": string_ops.isupper_op, - "capitalize": string_ops.capitalize_op, -} diff --git a/bigframes/operations/remote_function_ops.py b/bigframes/operations/remote_function_ops.py index 3ce77d51c61..e610ce61d6e 100644 --- a/bigframes/operations/remote_function_ops.py +++ b/bigframes/operations/remote_function_ops.py @@ -19,22 +19,37 @@ from bigframes.operations import base_ops +# TODO: Enforce input type constraints from function def @dataclasses.dataclass(frozen=True) -class PythonUdfOp(base_ops.NaryOp): - name: typing.ClassVar[str] = "python_udf" - function_def: udf_def.PythonUdf +class RemoteFunctionOp(base_ops.UnaryOp): + name: typing.ClassVar[str] = "remote_function" + function_def: udf_def.BigqueryUdf + apply_on_null: bool @property def expensive(self) -> bool: return True def output_type(self, *input_types): - return self.function_def.signature.output.bf_type + return self.function_def.bigframes_output_type @dataclasses.dataclass(frozen=True) -class RemoteFunctionOp(base_ops.NaryOp): - name: typing.ClassVar[str] = "remote_function" +class BinaryRemoteFunctionOp(base_ops.BinaryOp): + name: typing.ClassVar[str] = "binary_remote_function" + function_def: udf_def.BigqueryUdf + + @property + def expensive(self) -> bool: + return True + + def output_type(self, *input_types): + return self.function_def.bigframes_output_type + + +@dataclasses.dataclass(frozen=True) +class NaryRemoteFunctionOp(base_ops.NaryOp): + name: typing.ClassVar[str] = "nary_remote_function" function_def: udf_def.BigqueryUdf @property @@ -42,4 +57,4 @@ def expensive(self) -> bool: return True def output_type(self, *input_types): - return self.function_def.signature.output.bf_type + return self.function_def.bigframes_output_type diff --git a/bigframes/operations/semantics.py b/bigframes/operations/semantics.py new file mode 100644 index 00000000000..f237959d0d3 --- /dev/null +++ b/bigframes/operations/semantics.py @@ -0,0 +1,1169 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import re +import typing +from typing import List, Optional +import warnings + +import numpy as np + +from bigframes import dtypes, exceptions +from bigframes.core import guid +from bigframes.core.logging import log_adapter + + +@log_adapter.class_logger +class Semantics: + def __init__(self, df) -> None: + import bigframes # Import in the function body to avoid circular imports. + import bigframes.dataframe + + if not bigframes.options.experiments.semantic_operators: + raise NotImplementedError() + + self._df: bigframes.dataframe.DataFrame = df + + def agg( + self, + instruction: str, + model, + cluster_column: typing.Optional[str] = None, + max_agg_rows: int = 10, + ground_with_google_search: bool = False, + ): + """ + Performs an aggregation over all rows of the table. + + This method recursively aggregates the input data to produce partial answers + in parallel, until a single answer remains. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.semantic_operators = True + >>> bpd.options.compute.semantic_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.GeminiTextGenerator(model_name="gemini-2.0-flash-001") # doctest: +SKIP + + >>> df = bpd.DataFrame( + ... { + ... "Movies": [ + ... "Titanic", + ... "The Wolf of Wall Street", + ... "Inception", + ... ], + ... "Year": [1997, 2013, 2010], + ... }) + >>> df.semantics.agg( # doctest: +SKIP + ... "Find the first name shared by all actors in {Movies}. One word answer.", + ... model=model, + ... ) + 0 Leonardo + + Name: Movies, dtype: string + + Args: + instruction (str): + An instruction on how to map the data. This value must contain + column references by name enclosed in braces. + For example, to reference a column named "movies", use "{movies}" in the + instruction, like: "Find actor names shared by all {movies}." + + model (bigframes.ml.llm.GeminiTextGenerator): + A GeminiTextGenerator provided by the Bigframes ML package. + + cluster_column (Optional[str], default None): + If set, aggregates each cluster before performing aggregations across + clusters. Clustering based on semantic similarity can improve accuracy + of the sementic aggregations. + + max_agg_rows (int, default 10): + The maxinum number of rows to be aggregated at a time. + + ground_with_google_search (bool, default False): + Enables Grounding with Google Search for the GeminiTextGenerator model. + When set to True, the model incorporates relevant information from Google + Search results into its responses, enhancing their accuracy and factualness. + Note: Using this feature may impact billing costs. Refer to the pricing + page for details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models + The default is `False`. + + Returns: + bigframes.dataframe.DataFrame: A new DataFrame with the aggregated answers. + + Raises: + NotImplementedError: when the semantic operator experiment is off. + ValueError: when the instruction refers to a non-existing column, or when + more than one columns are referred to. + """ + import bigframes.bigquery as bbq + import bigframes.dataframe + import bigframes.series + + self._validate_model(model) + columns = self._parse_columns(instruction) + + if max_agg_rows <= 1: + raise ValueError( + f"Invalid value for `max_agg_rows`: {max_agg_rows}." + "It must be greater than 1." + ) + + work_estimate = len(self._df) * int(max_agg_rows / (max_agg_rows - 1)) + self._confirm_operation(work_estimate) + + df: bigframes.dataframe.DataFrame = self._df.copy() + for column in columns: + if column not in self._df.columns: + raise ValueError(f"Column {column} not found.") + + if df[column].dtype != dtypes.STRING_DTYPE: + df[column] = df[column].astype(dtypes.STRING_DTYPE) + + if len(columns) > 1: + raise NotImplementedError( + "Semantic aggregations are limited to a single column." + ) + column = columns[0] + + if ground_with_google_search: + msg = exceptions.format_message( + "Enables Grounding with Google Search may impact billing cost. See pricing " + "details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models" + ) + warnings.warn(msg, category=UserWarning) + + user_instruction = self._format_instruction(instruction, columns) + + num_cluster = 1 + if cluster_column is not None: + if cluster_column not in df.columns: + raise ValueError(f"Cluster column `{cluster_column}` not found.") + + if df[cluster_column].dtype != dtypes.INT_DTYPE: + raise TypeError( + "Cluster column must be an integer type, not " + f"{type(df[cluster_column])}" + ) + + num_cluster = df[cluster_column].unique().shape[0] + df = df.sort_values(cluster_column) + else: + cluster_column = guid.generate_guid("pid") + df[cluster_column] = 0 + + aggregation_group_id = guid.generate_guid("agg") + group_row_index = guid.generate_guid("gid") + llm_prompt = guid.generate_guid("prompt") + df = ( + df.reset_index(drop=True) + .reset_index() + .rename(columns={"index": aggregation_group_id}) + ) + + output_instruction = ( + "Answer user instructions using the provided context from various sources. " + "Combine all relevant information into a single, concise, well-structured response. " + f"Instruction: {user_instruction}.\n\n" + ) + + while len(df) > 1: + df[group_row_index] = (df[aggregation_group_id] % max_agg_rows + 1).astype( + dtypes.STRING_DTYPE + ) + df[aggregation_group_id] = (df[aggregation_group_id] / max_agg_rows).astype( + dtypes.INT_DTYPE + ) + df[llm_prompt] = "\t\nSource #" + df[group_row_index] + ": " + df[column] + + if len(df) > num_cluster: + # Aggregate within each partition + agg_df = bbq.array_agg( + df.groupby(by=[cluster_column, aggregation_group_id]) + ) + else: + # Aggregate cross partitions + agg_df = bbq.array_agg(df.groupby(by=[aggregation_group_id])) + agg_df[cluster_column] = agg_df[cluster_column].list[0] + + # Skip if the aggregated group only has a single item + single_row_df: bigframes.series.Series = bbq.array_to_string( + agg_df[agg_df[group_row_index].list.len() <= 1][column], + delimiter="", + ) + prompt_s: bigframes.series.Series = bbq.array_to_string( + agg_df[agg_df[group_row_index].list.len() > 1][llm_prompt], + delimiter="", + ) + prompt_s = output_instruction + prompt_s # type:ignore + + # Run model + predict_df = typing.cast( + bigframes.dataframe.DataFrame, + model.predict( + prompt_s, + temperature=0.0, + ground_with_google_search=ground_with_google_search, + ), + ) + agg_df[column] = predict_df["ml_generate_text_llm_result"].combine_first( + single_row_df + ) + + agg_df = agg_df.reset_index() + df = agg_df[[aggregation_group_id, cluster_column, column]] + + return df[column] + + def cluster_by( + self, + column: str, + output_column: str, + model, + n_clusters: int = 5, + ): + """ + Clusters data based on the semantic similarity of text within a specified column. + + This method leverages a language model to generate text embeddings for each value in + the given column. These embeddings capture the semantic meaning of the text. + The data is then grouped into `n` clusters using the k-means clustering algorithm, + which groups data points based on the similarity of their embeddings. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.semantic_operators = True + >>> bpd.options.compute.semantic_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.TextEmbeddingGenerator(model_name="text-embedding-005") + + >>> df = bpd.DataFrame({ + ... "Product": ["Smartphone", "Laptop", "T-shirt", "Jeans"], + ... }) + >>> df.semantics.cluster_by("Product", "Cluster ID", model, n_clusters=2) # doctest: +SKIP + Product Cluster ID + 0 Smartphone 2 + 1 Laptop 2 + 2 T-shirt 1 + 3 Jeans 1 + + [4 rows x 2 columns] + + Args: + column (str): + An column name to perform the similarity clustering. + + output_column (str): + An output column to store the clustering ID. + + model (bigframes.ml.llm.TextEmbeddingGenerator): + A TextEmbeddingGenerator provided by Bigframes ML package. + + n_clusters (int, default 5): + Default 5. Number of clusters to be detected. + + Returns: + bigframes.dataframe.DataFrame: A new DataFrame with the clustering output column. + + Raises: + NotImplementedError: when the semantic operator experiment is off. + ValueError: when the column refers to a non-existing column. + """ + + import bigframes.dataframe + import bigframes.ml.cluster as cluster + import bigframes.ml.llm as llm + + if not isinstance(model, llm.TextEmbeddingGenerator): + raise TypeError(f"Expect a text embedding model, but got: {type(model)}") + + if column not in self._df.columns: + raise ValueError(f"Column {column} not found.") + + if n_clusters <= 1: + raise ValueError( + f"Invalid value for `n_clusters`: {n_clusters}." + "It must be greater than 1." + ) + + self._confirm_operation(len(self._df)) + + df: bigframes.dataframe.DataFrame = self._df.copy() + embeddings_df = model.predict(df[column]) + + cluster_model = cluster.KMeans(n_clusters=n_clusters) + cluster_model.fit(embeddings_df[["ml_generate_embedding_result"]]) + clustered_result = cluster_model.predict(embeddings_df) + df[output_column] = clustered_result["CENTROID_ID"] + return df + + def filter(self, instruction: str, model, ground_with_google_search: bool = False): + """ + Filters the DataFrame with the semantics of the user instruction. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.semantic_operators = True + >>> bpd.options.compute.semantic_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.GeminiTextGenerator(model_name="gemini-2.0-flash-001") # doctest: +SKIP + + >>> df = bpd.DataFrame({"country": ["USA", "Germany"], "city": ["Seattle", "Berlin"]}) + >>> df.semantics.filter("{city} is the capital of {country}", model) # doctest: +SKIP + country city + 1 Germany Berlin + + [1 rows x 2 columns] + + Args: + instruction (str): + An instruction on how to filter the data. This value must contain + column references by name, which should be wrapped in a pair of braces. + For example, if you have a column "food", you can refer to this column + in the instructions like: + "The {food} is healthy." + + model (bigframes.ml.llm.GeminiTextGenerator): + A GeminiTextGenerator provided by Bigframes ML package. + + ground_with_google_search (bool, default False): + Enables Grounding with Google Search for the GeminiTextGenerator model. + When set to True, the model incorporates relevant information from Google + Search results into its responses, enhancing their accuracy and factualness. + Note: Using this feature may impact billing costs. Refer to the pricing + page for details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models + The default is `False`. + + Returns: + bigframes.pandas.DataFrame: DataFrame filtered by the instruction. + + Raises: + NotImplementedError: when the semantic operator experiment is off. + ValueError: when the instruction refers to a non-existing column, or when no + columns are referred to. + """ + import bigframes.dataframe + import bigframes.series + + self._validate_model(model) + columns = self._parse_columns(instruction) + for column in columns: + if column not in self._df.columns: + raise ValueError(f"Column {column} not found.") + + if ground_with_google_search: + msg = exceptions.format_message( + "Enables Grounding with Google Search may impact billing cost. See pricing " + "details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models" + ) + warnings.warn(msg, category=UserWarning) + + self._confirm_operation(len(self._df)) + + df: bigframes.dataframe.DataFrame = self._df[columns].copy() + has_blob_column = False + for column in columns: + if df[column].dtype == dtypes.OBJ_REF_DTYPE: + # Don't cast blob columns to string + has_blob_column = True + continue + + if df[column].dtype != dtypes.STRING_DTYPE: + df[column] = df[column].astype(dtypes.STRING_DTYPE) + + user_instruction = self._format_instruction(instruction, columns) + output_instruction = "Based on the provided context, reply to the following claim by only True or False:" + + if has_blob_column: + results = typing.cast( + bigframes.dataframe.DataFrame, + model.predict( + df, + prompt=self._make_multimodel_prompt( + df, columns, user_instruction, output_instruction + ), + temperature=0.0, + ground_with_google_search=ground_with_google_search, + ), + ) + else: + results = typing.cast( + bigframes.dataframe.DataFrame, + model.predict( + self._make_text_prompt( + df, columns, user_instruction, output_instruction + ), + temperature=0.0, + ground_with_google_search=ground_with_google_search, + ), + ) + + return self._df[ + results["ml_generate_text_llm_result"].str.lower().str.contains("true") + ] + + def map( + self, + instruction: str, + output_column: str, + model, + ground_with_google_search: bool = False, + ): + """ + Maps the DataFrame with the semantics of the user instruction. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.semantic_operators = True + >>> bpd.options.compute.semantic_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.GeminiTextGenerator(model_name="gemini-2.0-flash-001") # doctest: +SKIP + + >>> df = bpd.DataFrame({"ingredient_1": ["Burger Bun", "Soy Bean"], "ingredient_2": ["Beef Patty", "Bittern"]}) + >>> df.semantics.map("What is the food made from {ingredient_1} and {ingredient_2}? One word only.", output_column="food", model=model) # doctest: +SKIP + ingredient_1 ingredient_2 food + 0 Burger Bun Beef Patty Burger + + 1 Soy Bean Bittern Tofu + + + [2 rows x 3 columns] + + Args: + instruction (str): + An instruction on how to map the data. This value must contain + column references by name, which should be wrapped in a pair of braces. + For example, if you have a column "food", you can refer to this column + in the instructions like: + "Get the ingredients of {food}." + + output_column (str): + The column name of the mapping result. + + model (bigframes.ml.llm.GeminiTextGenerator): + A GeminiTextGenerator provided by Bigframes ML package. + + ground_with_google_search (bool, default False): + Enables Grounding with Google Search for the GeminiTextGenerator model. + When set to True, the model incorporates relevant information from Google + Search results into its responses, enhancing their accuracy and factualness. + Note: Using this feature may impact billing costs. Refer to the pricing + page for details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models + The default is `False`. + + Returns: + bigframes.pandas.DataFrame: DataFrame with attached mapping results. + + Raises: + NotImplementedError: when the semantic operator experiment is off. + ValueError: when the instruction refers to a non-existing column, or when no + columns are referred to. + """ + import bigframes.dataframe + import bigframes.series + + self._validate_model(model) + columns = self._parse_columns(instruction) + for column in columns: + if column not in self._df.columns: + raise ValueError(f"Column {column} not found.") + + if ground_with_google_search: + msg = exceptions.format_message( + "Enables Grounding with Google Search may impact billing cost. See pricing " + "details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models" + ) + warnings.warn(msg, category=UserWarning) + + self._confirm_operation(len(self._df)) + + df: bigframes.dataframe.DataFrame = self._df[columns].copy() + has_blob_column = False + for column in columns: + if df[column].dtype == dtypes.OBJ_REF_DTYPE: + # Don't cast blob columns to string + has_blob_column = True + continue + + if df[column].dtype != dtypes.STRING_DTYPE: + df[column] = df[column].astype(dtypes.STRING_DTYPE) + + user_instruction = self._format_instruction(instruction, columns) + output_instruction = ( + "Based on the provided contenxt, answer the following instruction:" + ) + + if has_blob_column: + results = typing.cast( + bigframes.series.Series, + model.predict( + df, + prompt=self._make_multimodel_prompt( + df, columns, user_instruction, output_instruction + ), + temperature=0.0, + ground_with_google_search=ground_with_google_search, + )["ml_generate_text_llm_result"], + ) + else: + results = typing.cast( + bigframes.series.Series, + model.predict( + self._make_text_prompt( + df, columns, user_instruction, output_instruction + ), + temperature=0.0, + ground_with_google_search=ground_with_google_search, + )["ml_generate_text_llm_result"], + ) + + from bigframes.core.reshape.api import concat + + return concat([self._df, results.rename(output_column)], axis=1) + + def join( + self, + other, + instruction: str, + model, + ground_with_google_search: bool = False, + ): + """ + Joines two dataframes by applying the instruction over each pair of rows from + the left and right table. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.semantic_operators = True + >>> bpd.options.compute.semantic_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.GeminiTextGenerator(model_name="gemini-2.0-flash-001") # doctest: +SKIP + + >>> cities = bpd.DataFrame({'city': ['Seattle', 'Ottawa', 'Berlin', 'Shanghai', 'New Delhi']}) + >>> continents = bpd.DataFrame({'continent': ['North America', 'Africa', 'Asia']}) + + >>> cities.semantics.join(continents, "{city} is in {continent}", model) # doctest: +SKIP + city continent + 0 Seattle North America + 1 Ottawa North America + 2 Shanghai Asia + 3 New Delhi Asia + + [4 rows x 2 columns] + + Args: + other (bigframes.pandas.DataFrame): + The other dataframe. + + instruction (str): + An instruction on how left and right rows can be joined. This value must contain + column references by name. which should be wrapped in a pair of braces. + For example: "The {city} belongs to the {country}". + For column names that are shared between two dataframes, you need to add "left." + and "right." prefix for differentiation. This is especially important when you do + self joins. For example: "The {left.employee_name} reports to {right.employee_name}" + For unique column names, this prefix is optional. + + model (bigframes.ml.llm.GeminiTextGenerator): + A GeminiTextGenerator provided by Bigframes ML package. + + max_rows (int, default 1000): + The maximum number of rows allowed to be sent to the model per call. If the result is too large, the method + call will end early with an error. + + ground_with_google_search (bool, default False): + Enables Grounding with Google Search for the GeminiTextGenerator model. + When set to True, the model incorporates relevant information from Google + Search results into its responses, enhancing their accuracy and factualness. + Note: Using this feature may impact billing costs. Refer to the pricing + page for details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models + The default is `False`. + + Returns: + bigframes.pandas.DataFrame: The joined dataframe. + + Raises: + ValueError if the amount of data that will be sent for LLM processing is larger than max_rows. + """ + self._validate_model(model) + columns = self._parse_columns(instruction) + + if ground_with_google_search: + msg = exceptions.format_message( + "Enables Grounding with Google Search may impact billing cost. See pricing " + "details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models" + ) + warnings.warn(msg, category=UserWarning) + + work_estimate = len(self._df) * len(other) + self._confirm_operation(work_estimate) + + left_columns = [] + right_columns = [] + + for col in columns: + if col in self._df.columns and col in other.columns: + raise ValueError(f"Ambiguous column reference: {col}") + + elif col in self._df.columns: + left_columns.append(col) + + elif col in other.columns: + right_columns.append(col) + + elif col.startswith("left."): + original_col_name = col[len("left.") :] + if ( + original_col_name in self._df.columns + and original_col_name in other.columns + ): + left_columns.append(col) + elif original_col_name in self._df.columns: + left_columns.append(col) + instruction = instruction.replace(col, original_col_name) + else: + raise ValueError(f"Column {col} not found") + + elif col.startswith("right."): + original_col_name = col[len("right.") :] + if ( + original_col_name in self._df.columns + and original_col_name in other.columns + ): + right_columns.append(col) + elif original_col_name in other.columns: + right_columns.append(col) + instruction = instruction.replace(col, original_col_name) + else: + raise ValueError(f"Column {col} not found") + + else: + raise ValueError(f"Column {col} not found") + + if not left_columns: + raise ValueError("No left column references.") + + if not right_columns: + raise ValueError("No right column references.") + + # Update column references to be compatible with internal naming scheme. + # That is, "left.col" -> "col_left" and "right.col" -> "col_right" + instruction = re.sub(r"(?>> import bigframes.pandas as bpd + + >>> import bigframes + >>> bigframes.options.experiments.semantic_operators = True + >>> bpd.options.compute.semantic_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.TextEmbeddingGenerator(model_name="text-embedding-005") # doctest: +SKIP + + >>> df = bpd.DataFrame({"creatures": ["salmon", "sea urchin", "frog", "chimpanzee"]}) + >>> df.semantics.search("creatures", "monkey", top_k=1, model=model, score_column='distance') # doctest: +SKIP + creatures distance + 3 chimpanzee 0.635844 + + [1 rows x 2 columns] + + Args: + search_column: + The name of the column to search from. + query (str): + The search query. + top_k (int): + The number of nearest neighbors to return. + model (TextEmbeddingGenerator): + A TextEmbeddingGenerator provided by Bigframes ML package. + score_column (Optional[str], default None): + The name of the the additional column containning the similarity scores. If None, + this column won't be attached to the result. + + Returns: + DataFrame: the DataFrame with the search result. + + Raises: + ValueError: when the search_column is not found from the the data frame. + TypeError: when the provided model is not TextEmbeddingGenerator. + """ + + if search_column not in self._df.columns: + raise ValueError(f"Column `{search_column}` not found") + + self._confirm_operation(len(self._df)) + + import bigframes.ml.llm as llm + + if not isinstance(model, llm.TextEmbeddingGenerator): + raise TypeError(f"Expect a text embedding model, but got: {type(model)}") + + if top_k < 1: + raise ValueError("top_k must be an integer greater than or equal to 1.") + + embedded_df = model.predict(self._df[search_column]) + embedded_table = embedded_df.reset_index().to_gbq() + + import bigframes.pandas as bpd + + embedding_result_column = "ml_generate_embedding_result" + query_df = model.predict(bpd.DataFrame({"query_id": [query]})).rename( + columns={"content": "query_id", embedding_result_column: "embedding"} + ) + + import bigframes.bigquery as bbq + + search_result = ( + bbq.vector_search( + base_table=embedded_table, + column_to_search=embedding_result_column, + query=query_df, + top_k=top_k, + ) + .rename(columns={"content": search_column}) + .set_index("index") + ) + + search_result.index.name = self._df.index.name + + if score_column is not None: + search_result = search_result.rename(columns={"distance": score_column})[ + [search_column, score_column] + ] + else: + search_result = search_result[[search_column]] + + import bigframes.dataframe + + return typing.cast(bigframes.dataframe.DataFrame, search_result) + + def top_k( + self, + instruction: str, + model, + k: int = 10, + ground_with_google_search: bool = False, + ): + """ + Ranks each tuple and returns the k best according to the instruction. + + This method employs a quick select algorithm to efficiently compare the pivot + with all other items. By leveraging an LLM (Large Language Model), it then + identifies the top 'k' best answers from these comparisons. + + **Examples:** + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.semantic_operators = True + >>> bpd.options.compute.semantic_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.GeminiTextGenerator(model_name="gemini-2.0-flash-001") # doctest: +SKIP + + >>> df = bpd.DataFrame( + ... { + ... "Animals": ["Dog", "Bird", "Cat", "Horse"], + ... "Sounds": ["Woof", "Chirp", "Meow", "Neigh"], + ... }) + >>> df.semantics.top_k("{Animals} are more popular as pets", model=model, k=2) # doctest: +SKIP + Animals Sounds + 0 Dog Woof + 2 Cat Meow + + [2 rows x 2 columns] + + Args: + instruction (str): + An instruction on how to map the data. This value must contain + column references by name enclosed in braces. + For example, to reference a column named "Animals", use "{Animals}" in the + instruction, like: "{Animals} are more popular as pets" + + model (bigframes.ml.llm.GeminiTextGenerator): + A GeminiTextGenerator provided by the Bigframes ML package. + + k (int, default 10): + The number of rows to return. + + ground_with_google_search (bool, default False): + Enables Grounding with Google Search for the GeminiTextGenerator model. + When set to True, the model incorporates relevant information from Google + Search results into its responses, enhancing their accuracy and factualness. + Note: Using this feature may impact billing costs. Refer to the pricing + page for details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models + The default is `False`. + + Returns: + bigframes.dataframe.DataFrame: A new DataFrame with the top k rows. + + Raises: + NotImplementedError: when the semantic operator experiment is off. + ValueError: when the instruction refers to a non-existing column, or when no + columns are referred to. + """ + import bigframes.dataframe + import bigframes.series + + self._validate_model(model) + columns = self._parse_columns(instruction) + for column in columns: + if column not in self._df.columns: + raise ValueError(f"Column {column} not found.") + if len(columns) > 1: + raise NotImplementedError("Semantic top K are limited to a single column.") + + if ground_with_google_search: + msg = exceptions.format_message( + "Enables Grounding with Google Search may impact billing cost. See pricing " + "details: https://cloud.google.com/vertex-ai/generative-ai/pricing#google_models" + ) + warnings.warn(msg, category=UserWarning) + + work_estimate = int(len(self._df) * (len(self._df) - 1) / 2) + self._confirm_operation(work_estimate) + + df: bigframes.dataframe.DataFrame = self._df[columns].copy() + column = columns[0] + if df[column].dtype != dtypes.STRING_DTYPE: + df[column] = df[column].astype(dtypes.STRING_DTYPE) + + # `index` is reserved for the `reset_index` below. + if column == "index": + raise ValueError( + "Column name 'index' is reserved. Please choose a different name." + ) + + if k < 1: + raise ValueError("k must be an integer greater than or equal to 1.") + + user_instruction = self._format_instruction(instruction, columns) + + n = df.shape[0] + if k >= n: + return df + + # Create a unique index and duplicate it as the "index" column. This workaround + # is needed for the select search algorithm due to unimplemented bigFrame methods. + df = df.reset_index().rename(columns={"index": "old_index"}).reset_index() + + # Initialize a status column to track the selection status of each item. + # - None: Unknown/not yet processed + # - 1.0: Selected as part of the top-k items + # - -1.0: Excluded from the top-k items + status_column = guid.generate_guid("status") + df[status_column] = bigframes.series.Series( + None, dtype=dtypes.FLOAT_DTYPE, session=df._session + ) + + num_selected = 0 + while num_selected < k: + df, num_new_selected = self._topk_partition( + df, + column, + status_column, + user_instruction, + model, + k - num_selected, + ground_with_google_search, + ) + num_selected += num_new_selected + + result_df: bigframes.dataframe.DataFrame = self._df.copy() + return result_df[df.set_index("old_index")[status_column] > 0.0] + + @staticmethod + def _topk_partition( + df, + column: str, + status_column: str, + user_instruction: str, + model, + k: int, + ground_with_google_search: bool, + ): + output_instruction = ( + "Given a question and two documents, choose the document that best answers " + "the question. Respond with 'Document 1' or 'Document 2'. You must choose " + "one, even if neither is ideal. " + ) + + # Random pivot selection for improved average quickselect performance. + pending_df = df[df[status_column].isna()] + pivot_iloc = np.random.randint(0, pending_df.shape[0]) + pivot_index = pending_df.iloc[pivot_iloc]["index"] + pivot_df = pending_df[pending_df["index"] == pivot_index] + + # Build a prompt to compare the pivot item's relevance to other pending items. + prompt_s = pending_df[pending_df["index"] != pivot_index][column] + prompt_s = ( + f"{output_instruction}\n\nQuestion: {user_instruction}\n" + + f"\nDocument 1: {column} " + + pivot_df.iloc[0][column] + + f"\nDocument 2: {column} " + + prompt_s # type:ignore + ) + + import bigframes.dataframe + + predict_df = typing.cast( + bigframes.dataframe.DataFrame, + model.predict( + prompt_s, + temperature=0.0, + ground_with_google_search=ground_with_google_search, + ), + ) + + marks = predict_df["ml_generate_text_llm_result"].str.contains("2") + more_relavant: bigframes.dataframe.DataFrame = df[marks] + less_relavent: bigframes.dataframe.DataFrame = df[~marks] + + num_more_relavant = more_relavant.shape[0] + if k < num_more_relavant: + less_relavent[status_column] = -1.0 + pivot_df[status_column] = -1.0 + df = df.combine_first(less_relavent).combine_first(pivot_df) + return df, 0 + else: # k >= num_more_relavant + more_relavant[status_column] = 1.0 + df = df.combine_first(more_relavant) + if k >= num_more_relavant + 1: + pivot_df[status_column] = 1.0 + df = df.combine_first(pivot_df) + return df, num_more_relavant + 1 + else: + return df, num_more_relavant + + def sim_join( + self, + other, + left_on: str, + right_on: str, + model, + top_k: int = 3, + score_column: Optional[str] = None, + max_rows: int = 1000, + ): + """ + Joins two dataframes based on the similarity of the specified columns. + + This method uses BigQuery's VECTOR_SEARCH function to match rows on the left side with the rows that have + nearest embedding vectors on the right. In the worst case scenario, the complexity is around O(M * N * log K). + Therefore, this is a potentially expensive operation. + + ** Examples: ** + + >>> import bigframes.pandas as bpd + >>> bpd.options.experiments.semantic_operators = True + >>> bpd.options.compute.semantic_ops_confirmation_threshold = 25 + + >>> import bigframes.ml.llm as llm + >>> model = llm.TextEmbeddingGenerator(model_name="text-embedding-005") # doctest: +SKIP + + >>> df1 = bpd.DataFrame({'animal': ['monkey', 'spider']}) + >>> df2 = bpd.DataFrame({'animal': ['scorpion', 'baboon']}) + + >>> df1.semantics.sim_join(df2, left_on='animal', right_on='animal', model=model, top_k=1) # doctest: +SKIP + animal animal_1 + 0 monkey baboon + 1 spider scorpion + + [2 rows x 2 columns] + + Args: + other (DataFrame): + The other data frame to join with. + left_on (str): + The name of the column on left side for the join. + right_on (str): + The name of the column on the right side for the join. + top_k (int, default 3): + The number of nearest neighbors to return. + model (TextEmbeddingGenerator): + A TextEmbeddingGenerator provided by Bigframes ML package. + score_column (Optional[str], default None): + The name of the the additional column containning the similarity scores. If None, + this column won't be attached to the result. + max_rows: + The maximum number of rows allowed to be processed per call. If the result is too large, the method + call will end early with an error. + + Returns: + DataFrame: the data frame with the join result. + + Raises: + ValueError: when the amount of data to be processed exceeds the specified max_rows. + """ + + if left_on not in self._df.columns: + raise ValueError(f"Left column {left_on} not found") + if right_on not in self._df.columns: + raise ValueError(f"Right column {right_on} not found") + + import bigframes.ml.llm as llm + + if not isinstance(model, llm.TextEmbeddingGenerator): + raise TypeError(f"Expect a text embedding model, but got: {type(model)}") + + joined_table_rows = len(self._df) * len(other) + if joined_table_rows > max_rows: + raise ValueError( + f"Number of rows that need processing is {joined_table_rows}, which exceeds row limit {max_rows}." + ) + + if top_k < 1: + raise ValueError("top_k must be an integer greater than or equal to 1.") + + work_estimate = len(self._df) * len(other) + self._confirm_operation(work_estimate) + + base_table_embedding_column = guid.generate_guid() + base_table = self._attach_embedding( + other, right_on, base_table_embedding_column, model + ).to_gbq() + query_table = self._attach_embedding(self._df, left_on, "embedding", model) + + import bigframes.bigquery as bbq + + join_result = bbq.vector_search( + base_table=base_table, + column_to_search=base_table_embedding_column, + query=query_table, + top_k=top_k, + ) + + join_result = join_result.drop( + ["embedding", base_table_embedding_column], axis=1 + ) + + if score_column is not None: + join_result = join_result.rename(columns={"distance": score_column}) + else: + del join_result["distance"] + + return join_result + + @staticmethod + def _attach_embedding(dataframe, source_column: str, embedding_column: str, model): + result_df = dataframe.copy() + embeddings = model.predict(dataframe[source_column])[ + "ml_generate_embedding_result" + ] + result_df[embedding_column] = embeddings + return result_df + + @staticmethod + def _make_multimodel_prompt( + prompt_df, columns, user_instruction: str, output_instruction: str + ): + prompt = [f"{output_instruction}\n{user_instruction}\nContext: "] + for col in columns: + prompt.extend([f"{col} is ", prompt_df[col]]) + + return prompt + + @staticmethod + def _make_text_prompt( + prompt_df, columns, user_instruction: str, output_instruction: str + ): + prompt_df["prompt"] = f"{output_instruction}\n{user_instruction}\nContext: " + + # Combine context from multiple columns. + for col in columns: + prompt_df["prompt"] += f"{col} is `" + prompt_df[col] + "`\n" + + return prompt_df["prompt"] + + @staticmethod + def _parse_columns(instruction: str) -> List[str]: + """Extracts column names enclosed in curly braces from the user instruction. + For example, _parse_columns("{city} is in {continent}") == ["city", "continent"] + """ + columns = re.findall(r"(? str: + """Extracts column names enclosed in curly braces from the user instruction. + For example, `_format_instruction(["city", "continent"], "{city} is in {continent}") + == "city is in continent"` + """ + return instruction.format(**{col: col for col in columns}) + + @staticmethod + def _validate_model(model): + from bigframes.ml.llm import GeminiTextGenerator + + if not isinstance(model, GeminiTextGenerator): + raise TypeError("Model is not GeminiText Generator") + + @staticmethod + def _confirm_operation(row_count: int): + """Raises OperationAbortedError when the confirmation fails""" + import bigframes # Import in the function body to avoid circular imports. + + threshold = bigframes.options.compute.semantic_ops_confirmation_threshold + + if threshold is None or row_count <= threshold: + return + + if bigframes.options.compute.semantic_ops_threshold_autofail: + raise exceptions.OperationAbortedError( + f"Operation was cancelled because your work estimate is {row_count} rows, which exceeds the threshold {threshold} rows." + ) + + # Separate the prompt out. In IDE such VS Code, leaving prompt in the + # input function makes it less visible to the end user. + print(f"This operation will process about {row_count} rows.") + print( + "You can raise the confirmation threshold by setting `bigframes.options.compute.semantic_ops_confirmation_threshold` to a higher value. To completely turn off the confirmation check, set the threshold to `None`." + ) + print("Proceed? [Y/n]") + reply = input().casefold() + if reply not in {"y", "yes", ""}: + raise exceptions.OperationAbortedError("Operation was cancelled.") diff --git a/bigframes/operations/string_ops.py b/bigframes/operations/string_ops.py index 21d23416314..a50a1b39f6d 100644 --- a/bigframes/operations/string_ops.py +++ b/bigframes/operations/string_ops.py @@ -18,9 +18,9 @@ import pandas as pd import pyarrow as pa -import bigframes.operations.type as op_typing from bigframes import dtypes from bigframes.operations import base_ops +import bigframes.operations.type as op_typing LenOp = base_ops.create_unary_op( name="len", @@ -126,6 +126,15 @@ def output_type(self, *input_types): return op_typing.STRING_PREDICATE.output_type(input_types[0]) +@dataclasses.dataclass(frozen=True) +class StrGetOp(base_ops.UnaryOp): + name: typing.ClassVar[str] = "str_get" + i: int + + def output_type(self, *input_types): + return op_typing.STRING_TRANSFORM.output_type(input_types[0]) + + @dataclasses.dataclass(frozen=True) class StrPadOp(base_ops.UnaryOp): name: typing.ClassVar[str] = "str_pad" diff --git a/bigframes/operations/strings.py b/bigframes/operations/strings.py index ad9bfd6da40..922d26a23c1 100644 --- a/bigframes/operations/strings.py +++ b/bigframes/operations/strings.py @@ -20,15 +20,13 @@ import bigframes_vendored.constants as constants import bigframes_vendored.pandas.core.strings.accessor as vendorstr -import bigframes.core.col import bigframes.core.indexes.base as indices +from bigframes.core.logging import log_adapter import bigframes.dataframe as df import bigframes.operations as ops +from bigframes.operations._op_converters import convert_index, convert_slice import bigframes.operations.aggregations as agg_ops import bigframes.series as series -from bigframes._tools import docs -from bigframes.core.logging import log_adapter -from bigframes.operations._op_converters import convert_index, convert_slice # Maps from python to re2 REGEXP_FLAGS = { @@ -37,12 +35,13 @@ re.DOTALL: "s", } -T = TypeVar("T", series.Series, indices.Index, bigframes.core.col.Expression) +T = TypeVar("T", series.Series, indices.Index) @log_adapter.class_logger -@docs.inherit_docs(vendorstr.StringMethods) -class StringMethods(Generic[T]): +class StringMethods(vendorstr.StringMethods, Generic[T]): + __doc__ = vendorstr.StringMethods.__doc__ + def __init__(self, data: T): self._data: T = data @@ -172,7 +171,7 @@ def fullmatch(self, pat, case=True, flags=0) -> T: return self.contains(pat=adj_pat, case=case, flags=flags) def get(self, i: int) -> T: - return self._data._apply_unary_op(ops.GetItemOp(key=i)) + return self._data._apply_unary_op(ops.StrGetOp(i=i)) def pad(self, width, side="left", fillchar=" ") -> T: return self._data._apply_unary_op( @@ -305,6 +304,30 @@ def join(self, sep: str) -> T: ops.ArrayReduceOp(aggregation=agg_ops.StringAggOp(sep=sep)) ) + def to_blob(self, connection: Optional[str] = None) -> T: + """Create a BigFrames Blob series from a series of URIs. + + .. note:: + BigFrames Blob is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). + + + Args: + connection (str or None, default None): + Connection to connect with remote service. str of the format ... + If None, use default connection in session context. BigQuery DataFrame will try to create the connection and attach + permission if the connection isn't fully set up. + + Returns: + bigframes.series.Series: Blob Series. + + """ + session = self._data._block.session + connection = session._create_bq_connection(connection=connection) + return self._data._apply_binary_op(connection, ops.obj_make_ref_op) + def _parse_flags(flags: int) -> Optional[str]: re2flags = [] diff --git a/bigframes/operations/structs.py b/bigframes/operations/structs.py index c5446510a6d..ec0b5dae526 100644 --- a/bigframes/operations/structs.py +++ b/bigframes/operations/structs.py @@ -17,17 +17,17 @@ import bigframes_vendored.pandas.core.arrays.arrow.accessors as vendoracessors import pandas as pd +from bigframes.core import backports +from bigframes.core.logging import log_adapter import bigframes.dataframe import bigframes.operations import bigframes.series -from bigframes._tools import docs -from bigframes.core import backports -from bigframes.core.logging import log_adapter @log_adapter.class_logger -@docs.inherit_docs(vendoracessors.StructAccessor) -class StructAccessor: +class StructAccessor(vendoracessors.StructAccessor): + __doc__ = vendoracessors.StructAccessor.__doc__ + def __init__(self, data: bigframes.series.Series): self._data = data @@ -69,8 +69,7 @@ def dtypes(self) -> pd.Series: @log_adapter.class_logger -@docs.inherit_docs(vendoracessors.StructFrameAccessor) -class StructFrameAccessor: +class StructFrameAccessor(vendoracessors.StructFrameAccessor): __doc__ = vendoracessors.StructAccessor.__doc__ def __init__(self, data: bigframes.dataframe.DataFrame) -> None: diff --git a/bigframes/operations/time_ops.py b/bigframes/operations/time_ops.py index 3b6845053a7..bf6fa3e7d1e 100644 --- a/bigframes/operations/time_ops.py +++ b/bigframes/operations/time_ops.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import bigframes.operations.type as op_typing from bigframes import dtypes from bigframes.operations import base_ops +import bigframes.operations.type as op_typing HourOp = base_ops.create_unary_op( name="hour", diff --git a/bigframes/operations/to_op.py b/bigframes/operations/to_op.py deleted file mode 100644 index 4f97a61e3c0..00000000000 --- a/bigframes/operations/to_op.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import annotations - -import dataclasses -import inspect -import typing - -import bigframes.core.expression as ex -from bigframes._config import options -from bigframes.exceptions import TranspilationError -from bigframes.functions import Udf -from bigframes.functions.udf_def import BigqueryUdf, PythonUdf -from bigframes.operations import base_ops, remote_function_ops - -ArgKind = typing.Literal[ - "positional_only", - "positional_or_keyword", - "keyword_only", - "var_positional", - "var_keyword", -] - -_ARGKIND_MAP: dict[inspect._ParameterKind, ArgKind] = { - inspect.Parameter.POSITIONAL_ONLY: "positional_only", - inspect.Parameter.POSITIONAL_OR_KEYWORD: "positional_or_keyword", - inspect.Parameter.VAR_POSITIONAL: "var_positional", - inspect.Parameter.KEYWORD_ONLY: "keyword_only", - inspect.Parameter.VAR_KEYWORD: "var_keyword", -} - - -@dataclasses.dataclass(frozen=True) -class ArgumentSpec: - """ - Information about a single argument to a function - """ - - name: str - default_value: typing.Any - argkind: ArgKind - - @property - def is_positional(self) -> bool: - return self.argkind in ["positional_only", "positional_or_keyword"] - - @property - def is_keyword(self) -> bool: - return self.argkind in ["keyword_only", "positional_or_keyword"] - - @property - def is_var_positional(self) -> bool: - return self.argkind == "var_positional" - - @property - def is_var_keyword(self) -> bool: - return self.argkind == "var_keyword" - - @property - def is_varargs(self) -> bool: - return self.is_var_positional - - -@dataclasses.dataclass(frozen=True) -class CallableExpression: - """ - Encodes a calling convention and an expression to bind arguments to. - """ - - expr: ex.Expression - arg_specs: typing.Sequence[ArgumentSpec] - - @classmethod - def from_callable(cls, func: typing.Callable) -> CallableExpression: - sig = inspect.signature(func) - arg_specs = [] - for name, param in sig.parameters.items(): - arg_specs.append( - ArgumentSpec( - name=name, - default_value=param.default, - argkind=_ARGKIND_MAP[param.kind], - ) - ) - - from bigframes.core.bytecode import py_to_expression - - try: - expr = py_to_expression(func) - except Exception as ex: - raise TranspilationError(f"Failed to transpile function {func}") from ex - return cls(expr=expr, arg_specs=arg_specs) - - def apply(self, *args, **kwargs) -> ex.Expression: - """ - Apply the arguments to the expression. - - All args are expected to be column references, or scalars. - """ - return self.bind_partial(*args, _offset=0, **kwargs).expr - - def bind_partial( - self, - *args, - _offset: int = 0, - **kwargs, - ) -> CallableExpression: - """ - Bind a subset of arguments and return a new CallableExpression with the remaining unbound arguments. - """ - bindings: dict[typing.Hashable, ex.Expression] = {} - pos_idx = 0 - allowed_params = self.arg_specs[_offset:] - allowed_names = {spec.name for spec in allowed_params} - - # Validate unexpected keyword arguments - for key in kwargs: - if key not in allowed_names: - raise TypeError(f"got an unexpected keyword argument '{key}'") - - def to_expr(val): - if isinstance(val, ex.Expression): - return val - return ex.const(val) - - for spec in allowed_params: - if spec.is_varargs: - raise NotImplementedError( - "varargs in compiled python functions is not supported" - ) - - if pos_idx < len(args): - if spec.name in kwargs: - raise TypeError( - f"got multiple values for keyword argument '{spec.name}'" - ) - bindings[spec.name] = to_expr(args[pos_idx]) - pos_idx += 1 - elif spec.name in kwargs: - bindings[spec.name] = to_expr(kwargs[spec.name]) - elif spec.default_value is not inspect.Parameter.empty: - bindings[spec.name] = to_expr(spec.default_value) - else: - raise TypeError(f"missing required argument: '{spec.name}'") - - if pos_idx < len(args): - raise TypeError( - f"too many positional arguments: expected {len(allowed_params)}, got {len(args)}" - ) - - new_expr = self.expr.bind_variables(bindings, allow_partial_bindings=True) - remaining_specs = list(self.arg_specs[:_offset]) - return CallableExpression(expr=new_expr, arg_specs=remaining_specs) - - -def func_to_expr(op) -> CallableExpression: - """ - Convert various bigframes, python functions into bigframes CallableExpression. - """ - if isinstance(op, Udf): - bq_op: base_ops.NaryOp - if isinstance(op.udf_def, BigqueryUdf): - bq_op = remote_function_ops.RemoteFunctionOp(function_def=op.udf_def) - elif isinstance(op.udf_def, PythonUdf): - bq_op = remote_function_ops.PythonUdfOp(function_def=op.udf_def) - else: - raise TypeError(f"Unsupported UDF definition: {op.udf_def}") - - inputs_expr = tuple( - ex.free_var(arg.name) for arg in op.udf_def.signature.inputs - ) - expr = ex.OpExpression(bq_op, inputs_expr) - - arg_specs = [ - ArgumentSpec( - name=arg.name, - default_value=inspect.Parameter.empty, - # Udf specs don't have concept of positional only or keyword only yet, - # so default to positional_or_keyword. - argkind="positional_or_keyword", - ) - for arg in op.udf_def.signature.inputs - ] - return CallableExpression(expr=expr, arg_specs=arg_specs) - - elif options.experiments.enable_python_transpiler and callable(op): - return CallableExpression.from_callable(op) - - else: - raise TypeError(f"Unsupported function type: {op}") diff --git a/bigframes/operations/type.py b/bigframes/operations/type.py index 0ddf3a113fc..65422330814 100644 --- a/bigframes/operations/type.py +++ b/bigframes/operations/type.py @@ -34,13 +34,11 @@ def as_method(self): """Convert the signature into an object method. Convenience function for constructing ops that use the signature.""" ... - def __call__(self, *args, **kwargs): - return self.as_method(*args, **kwargs) - class UnaryTypeSignature(TypeSignature): @abc.abstractmethod - def output_type(self, input_type: ExpressionType) -> ExpressionType: ... + def output_type(self, input_type: ExpressionType) -> ExpressionType: + ... @property def as_method(self): @@ -55,7 +53,8 @@ class BinaryTypeSignature(TypeSignature): @abc.abstractmethod def output_type( self, left_type: ExpressionType, right_type: ExpressionType - ) -> ExpressionType: ... + ) -> ExpressionType: + ... @property def as_method(self): diff --git a/bigframes/pandas/__init__.py b/bigframes/pandas/__init__.py index b88816ab5ab..9da2204a713 100644 --- a/bigframes/pandas/__init__.py +++ b/bigframes/pandas/__init__.py @@ -12,64 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -The primary entry point for the BigQuery DataFrames (BigFrames) pandas-compatible API. - -**BigQuery DataFrames** provides a Pythonic DataFrame and machine learning (ML) API -powered by the BigQuery engine. The ``bigframes.pandas`` module implements a large -subset of the pandas API, allowing you to perform large-scale data analysis -using familiar pandas syntax while the computations are executed in the cloud. - -**Key Features:** - -* **Petabyte-Scale Scalability:** Handle datasets that exceed local memory by - offloading computation to the BigQuery distributed engine. -* **Pandas Compatibility:** Use common pandas methods like - :func:`~bigframes.pandas.DataFrame.groupby`, - :func:`~bigframes.pandas.DataFrame.merge`, - :func:`~bigframes.pandas.DataFrame.pivot_table`, and more on BigQuery-backed - :class:`~bigframes.pandas.DataFrame` objects. -* **Direct BigQuery Integration:** Read from and write to BigQuery tables and - queries with :func:`bigframes.pandas.read_gbq` and - :func:`bigframes.pandas.DataFrame.to_gbq`. -* **User-defined Functions (UDFs):** Effortlessly deploy Python functions - functions using the :func:`bigframes.pandas.remote_function` and - :func:`bigframes.pandas.udf` decorators. -* **Data Ingestion:** Support for various formats including CSV, Parquet, JSON, - and Arrow via :func:`bigframes.pandas.read_csv`, - :func:`bigframes.pandas.read_parquet`, etc., which are automatically uploaded - to BigQuery for processing. Convert any pandas DataFrame into a BigQuery - DataFrame using :func:`bigframes.pandas.read_pandas`. - -**Example usage:** - - >>> import bigframes.pandas as bpd - -Initialize session and set options. - - >>> bpd.options.bigquery.project = "your-project-id" # doctest: +SKIP - -Load data from a BigQuery public dataset. - - >>> df = bpd.read_gbq("bigquery-public-data.usa_names.usa_1910_2013") # doctest: +SKIP - -Perform familiar pandas operations that execute in the cloud. - - >>> top_names = ( - ... df.groupby("name") - ... .agg({"number": "sum"}) - ... .sort_values("number", ascending=False) - ... .head(10) - ... ) # doctest: +SKIP - -Bring the final, aggregated results back to local memory if needed. - - >>> local_df = top_names.to_pandas() # doctest: +SKIP - -BigQuery DataFrames is designed for data scientists and analysts who need the -power of BigQuery with the ease of use of pandas. It eliminates the "data -movement bottleneck" by keeping your data in BigQuery for processing. -""" +"""BigQuery DataFrames provides a DataFrame API backed by the BigQuery engine.""" from __future__ import annotations @@ -86,34 +29,32 @@ import bigframes._config as config import bigframes.core.global_session as global_session import bigframes.core.indexes -import bigframes.dataframe -import bigframes.functions._utils as bff_utils -import bigframes.series -import bigframes.session -import bigframes.session._io.bigquery -import bigframes.version -from bigframes.core.col import col from bigframes.core.logging import log_adapter from bigframes.core.reshape.api import concat, crosstab, cut, get_dummies, merge, qcut +import bigframes.dataframe +import bigframes.functions._utils as bff_utils from bigframes.pandas import api from bigframes.pandas.core.api import to_timedelta from bigframes.pandas.io.api import ( - _from_glob_path, _read_gbq_colab, + from_glob_path, read_arrow, - read_avro, read_csv, read_gbq, read_gbq_function, read_gbq_model, + read_gbq_object_table, read_gbq_query, read_gbq_table, read_json, - read_orc, read_pandas, read_parquet, read_pickle, ) +import bigframes.series +import bigframes.session +import bigframes.session._io.bigquery +import bigframes.version try: import resource @@ -146,8 +87,7 @@ def remote_function( cloud_function_vpc_connector_egress_settings: Optional[ Literal["all", "private-ranges-only", "unspecified"] ] = None, - cloud_function_memory_mib: Optional[int] = None, - cloud_function_cpus: Optional[float] = None, + cloud_function_memory_mib: Optional[int] = 1024, cloud_function_ingress_settings: Literal[ "all", "internal-only", "internal-and-gclb" ] = "internal-only", @@ -171,7 +111,6 @@ def remote_function( cloud_function_vpc_connector=cloud_function_vpc_connector, cloud_function_vpc_connector_egress_settings=cloud_function_vpc_connector_egress_settings, cloud_function_memory_mib=cloud_function_memory_mib, - cloud_function_cpus=cloud_function_cpus, cloud_function_ingress_settings=cloud_function_ingress_settings, cloud_build_service_account=cloud_build_service_account, ) @@ -200,9 +139,9 @@ def udf( *, input_types: Union[None, type, Sequence[type]] = None, output_type: Optional[type] = None, - dataset: Optional[str] = None, + dataset: str, bigquery_connection: Optional[str] = None, - name: Optional[str] = None, + name: str, packages: Optional[Sequence[str]] = None, max_batching_rows: Optional[int] = None, container_cpu: Optional[float] = None, @@ -250,7 +189,8 @@ def to_datetime( utc: bool = False, format: Optional[str] = None, unit: Optional[str] = None, -) -> bigframes.series.Series: ... +) -> bigframes.series.Series: + ... @typing.overload @@ -260,7 +200,8 @@ def to_datetime( utc: bool = False, format: Optional[str] = None, unit: Optional[str] = None, -) -> Union[pandas.Timestamp, datetime.datetime]: ... +) -> Union[pandas.Timestamp, datetime.datetime]: + ... def to_datetime( @@ -437,7 +378,6 @@ def reset_session(): pass _functions = [ - _from_glob_path, clean_up_by_session_id, concat, crosstab, @@ -448,23 +388,23 @@ def reset_session(): get_dummies, merge, qcut, - read_arrow, - read_avro, read_csv, + read_arrow, read_gbq, _read_gbq_colab, read_gbq_function, read_gbq_model, + read_gbq_object_table, read_gbq_query, read_gbq_table, read_json, - read_orc, read_pandas, read_parquet, read_pickle, remote_function, to_datetime, to_timedelta, + from_glob_path, ] # Use __all__ to let type checkers know what is part of the public API. @@ -475,7 +415,6 @@ def reset_session(): "clean_up_by_session_id", "concat", "crosstab", - "col", "cut", "deploy_remote_function", "deploy_udf", @@ -483,23 +422,23 @@ def reset_session(): "get_dummies", "merge", "qcut", - "read_arrow", - "read_avro", "read_csv", + "read_arrow", "read_gbq", "_read_gbq_colab", "read_gbq_function", "read_gbq_model", + "read_gbq_object_table", "read_gbq_query", "read_gbq_table", "read_json", - "read_orc", "read_pandas", "read_parquet", "read_pickle", "remote_function", "to_datetime", "to_timedelta", + "from_glob_path", # Other names "api", # pandas dtype attributes diff --git a/bigframes/pandas/api/typing.py b/bigframes/pandas/api/typing.py index 8d8d65eddec..e21216bb685 100644 --- a/bigframes/pandas/api/typing.py +++ b/bigframes/pandas/api/typing.py @@ -21,14 +21,12 @@ from bigframes.core.groupby.series_group_by import SeriesGroupBy from bigframes.core.window import Window from bigframes.operations.datetimes import DatetimeMethods -from bigframes.operations.plotting import PlotAccessor from bigframes.operations.strings import StringMethods from bigframes.operations.structs import StructAccessor, StructFrameAccessor __all__ = [ "DataFrameGroupBy", "DatetimeMethods", - "PlotAccessor", "SeriesGroupBy", "StringMethods", "StructAccessor", diff --git a/bigframes/pandas/core/methods/describe.py b/bigframes/pandas/core/methods/describe.py index 34c116ba27d..6fd7960daf3 100644 --- a/bigframes/pandas/core/methods/describe.py +++ b/bigframes/pandas/core/methods/describe.py @@ -56,10 +56,9 @@ def describe( "max", ] ).intersection(describe_block.column_labels.get_level_values(-1)) - if not stack_cols.empty: - describe_block = describe_block.stack(override_labels=stack_cols) - return dataframe.DataFrame(describe_block).droplevel(level=0) - return dataframe.DataFrame(describe_block) + describe_block = describe_block.stack(override_labels=stack_cols) + + return dataframe.DataFrame(describe_block).droplevel(level=0) def _describe( @@ -121,7 +120,5 @@ def _get_aggs_for_dtype(dtype) -> list[aggregations.UnaryAggregateOp]: dtypes.TIME_DTYPE, ]: return [aggregations.count_op, aggregations.nunique_op] - elif dtypes.is_json_like(dtype) or dtype == dtypes.OBJ_REF_DTYPE: - return [aggregations.count_op] else: return [] diff --git a/bigframes/pandas/core/tools/timedeltas.py b/bigframes/pandas/core/tools/timedeltas.py index 5d08bec5f7c..eb01f9f8461 100644 --- a/bigframes/pandas/core/tools/timedeltas.py +++ b/bigframes/pandas/core/tools/timedeltas.py @@ -14,11 +14,11 @@ import typing -import pandas as pd -import pandas.api.types as pdtypes from bigframes_vendored.pandas.core.tools import ( timedeltas as vendored_pandas_timedeltas, ) +import pandas as pd +import pandas.api.types as pdtypes from bigframes import operations as ops from bigframes import series, session diff --git a/bigframes/pandas/io/api.py b/bigframes/pandas/io/api.py index fa0f503a08b..483bc5e530d 100644 --- a/bigframes/pandas/io/api.py +++ b/bigframes/pandas/io/api.py @@ -19,49 +19,46 @@ import os import threading import typing -import warnings from typing import ( - IO, Any, Callable, Dict, + IO, Iterable, Literal, MutableSequence, Optional, + overload, Sequence, Tuple, Union, - overload, ) +import warnings import bigframes_vendored.constants as constants import bigframes_vendored.pandas.io.gbq as vendored_pandas_gbq +from google.cloud import bigquery import numpy import pandas -import pyarrow as pa -from google.cloud import bigquery from pandas._typing import ( CompressionOptions, FilePath, ReadPickleBuffer, StorageOptions, ) +import pyarrow as pa import bigframes._config as config -import bigframes._importing import bigframes.core.global_session as global_session import bigframes.core.indexes import bigframes.dataframe import bigframes.enums import bigframes.series import bigframes.session +from bigframes.session import dry_runs import bigframes.session._io.bigquery import bigframes.session.clients -import bigframes.session.iceberg import bigframes.session.metrics -from bigframes.core import bq_data -from bigframes.session import dry_runs # Note: the following methods are duplicated from Session. This duplication # enables the following: @@ -94,21 +91,6 @@ def read_arrow(pa_table: pa.Table) -> bigframes.dataframe.DataFrame: return session.read_arrow(pa_table=pa_table) -def read_avro( - path: str | IO["bytes"], - *, - engine: str = "auto", -) -> bigframes.dataframe.DataFrame: - return global_session.with_default_session( - bigframes.session.Session.read_avro, - path, - engine=engine, - ) - - -read_avro.__doc__ = inspect.getdoc(bigframes.session.Session.read_avro) - - def read_csv( filepath_or_buffer: str | IO["bytes"], *, @@ -206,7 +188,8 @@ def read_gbq( # type: ignore[overload-overlap] col_order: Iterable[str] = ..., dry_run: Literal[False] = ..., allow_large_results: Optional[bool] = ..., -) -> bigframes.dataframe.DataFrame: ... +) -> bigframes.dataframe.DataFrame: + ... @overload @@ -222,7 +205,8 @@ def read_gbq( col_order: Iterable[str] = ..., dry_run: Literal[True] = ..., allow_large_results: Optional[bool] = ..., -) -> pandas.Series: ... +) -> pandas.Series: + ... def read_gbq( @@ -269,7 +253,7 @@ def _run_read_gbq_colab_sessionless_dry_run( pyformat_args=pyformat_args, dry_run=True, ) - bqclient, _ = _get_bqclient_and_project() + bqclient = _get_bqclient() job = _dry_run(query_formatted, bqclient) return dry_runs.get_query_stats_with_inferred_dtypes(job, (), ()) @@ -300,26 +284,25 @@ def _try_read_gbq_colab_sessionless_dry_run( def _read_gbq_colab( # type: ignore[overload-overlap] query_or_table: str, *, - callback: Optional[Callable[[bigframes.core.events.EventEnvelope], None]] = None, - pyformat_args: Optional[Dict[str, Any]] = None, - dry_run: Literal[False] = False, -) -> bigframes.dataframe.DataFrame: ... + pyformat_args: Optional[Dict[str, Any]] = ..., + dry_run: Literal[False] = ..., +) -> bigframes.dataframe.DataFrame: + ... @overload def _read_gbq_colab( query_or_table: str, *, - callback: Optional[Callable[[bigframes.core.events.EventEnvelope], None]] = None, - pyformat_args: Optional[Dict[str, Any]] = None, - dry_run: Literal[True], -) -> pandas.Series: ... + pyformat_args: Optional[Dict[str, Any]] = ..., + dry_run: Literal[True] = ..., +) -> pandas.Series: + ... def _read_gbq_colab( query_or_table: str, *, - callback: Optional[Callable[[bigframes.core.events.EventEnvelope], None]] = None, pyformat_args: Optional[Dict[str, Any]] = None, dry_run: bool = False, ) -> bigframes.dataframe.DataFrame | pandas.Series: @@ -331,8 +314,6 @@ def _read_gbq_colab( Args: query_or_table (str): SQL query or table ID (table ID not yet supported). - callback (Optional[Callable[[bigframes.core.events.EventEnvelope], None]]): - Callback to receive query execution events. pyformat_args (Optional[Dict[str, Any]]): Parameters to format into the query string. dry_run (bool): @@ -372,19 +353,15 @@ def _read_gbq_colab( ) _set_default_session_location_if_possible_deferred_query(create_query) if not config.options.bigquery._session_started: - # Don't warning about Polars in SQL cell. - # Related to b/437090788. - try: - bigframes._importing.import_polars() + with warnings.catch_warnings(): + # Don't warning about Polars in SQL cell. + # Related to b/437090788. warnings.simplefilter("ignore", bigframes.exceptions.PreviewWarning) config.options.bigquery.enable_polars_execution = True - except ImportError: - pass # don't fail if polars isn't available return global_session.with_default_session( bigframes.session.Session._read_gbq_colab, query_or_table, - callback=callback, pyformat_args=pyformat_args, dry_run=dry_run, ) @@ -400,6 +377,21 @@ def read_gbq_model(model_name: str): read_gbq_model.__doc__ = inspect.getdoc(bigframes.session.Session.read_gbq_model) +def read_gbq_object_table( + object_table: str, *, name: Optional[str] = None +) -> bigframes.dataframe.DataFrame: + return global_session.with_default_session( + bigframes.session.Session.read_gbq_object_table, + object_table, + name=name, + ) + + +read_gbq_object_table.__doc__ = inspect.getdoc( + bigframes.session.Session.read_gbq_object_table +) + + @overload def read_gbq_query( # type: ignore[overload-overlap] query: str, @@ -413,7 +405,8 @@ def read_gbq_query( # type: ignore[overload-overlap] filters: vendored_pandas_gbq.FiltersType = ..., dry_run: Literal[False] = ..., allow_large_results: Optional[bool] = ..., -) -> bigframes.dataframe.DataFrame: ... +) -> bigframes.dataframe.DataFrame: + ... @overload @@ -429,7 +422,8 @@ def read_gbq_query( filters: vendored_pandas_gbq.FiltersType = ..., dry_run: Literal[True] = ..., allow_large_results: Optional[bool] = ..., -) -> pandas.Series: ... +) -> pandas.Series: + ... def read_gbq_query( @@ -475,7 +469,8 @@ def read_gbq_table( # type: ignore[overload-overlap] use_cache: bool = ..., col_order: Iterable[str] = ..., dry_run: Literal[False] = ..., -) -> bigframes.dataframe.DataFrame: ... +) -> bigframes.dataframe.DataFrame: + ... @overload @@ -489,7 +484,8 @@ def read_gbq_table( use_cache: bool = ..., col_order: Iterable[str] = ..., dry_run: Literal[True] = ..., -) -> pandas.Series: ... +) -> pandas.Series: + ... def read_gbq_table( @@ -520,29 +516,13 @@ def read_gbq_table( read_gbq_table.__doc__ = inspect.getdoc(bigframes.session.Session.read_gbq_table) -def read_orc( - path: str | IO["bytes"], - *, - engine: str = "auto", - write_engine: constants.WriteEngineType = "default", -) -> bigframes.dataframe.DataFrame: - return global_session.with_default_session( - bigframes.session.Session.read_orc, - path, - engine=engine, - write_engine=write_engine, - ) - - -read_orc.__doc__ = inspect.getdoc(bigframes.session.Session.read_orc) - - @typing.overload def read_pandas( pandas_dataframe: pandas.DataFrame, *, write_engine: constants.WriteEngineType = "default", -) -> bigframes.dataframe.DataFrame: ... +) -> bigframes.dataframe.DataFrame: + ... @typing.overload @@ -550,7 +530,8 @@ def read_pandas( pandas_dataframe: pandas.Series, *, write_engine: constants.WriteEngineType = "default", -) -> bigframes.series.Series: ... +) -> bigframes.series.Series: + ... @typing.overload @@ -558,7 +539,8 @@ def read_pandas( pandas_dataframe: pandas.Index, *, write_engine: constants.WriteEngineType = "default", -) -> bigframes.core.indexes.Index: ... +) -> bigframes.core.indexes.Index: + ... def read_pandas( @@ -626,43 +608,38 @@ def read_gbq_function( read_gbq_function.__doc__ = inspect.getdoc(bigframes.session.Session.read_gbq_function) -def _from_glob_path( +def from_glob_path( path: str, *, connection: Optional[str] = None, name: Optional[str] = None ) -> bigframes.dataframe.DataFrame: return global_session.with_default_session( - bigframes.session.Session._from_glob_path, + bigframes.session.Session.from_glob_path, path=path, connection=connection, name=name, ) -_from_glob_path.__doc__ = inspect.getdoc(bigframes.session.Session._from_glob_path) +from_glob_path.__doc__ = inspect.getdoc(bigframes.session.Session.from_glob_path) _default_location_lock = threading.Lock() -def _get_bqclient_and_project() -> Tuple[bigquery.Client, str]: +def _get_bqclient() -> bigquery.Client: # Address circular imports in doctest due to bigframes/session/__init__.py # containing a lot of logic and samples. - import bigframes._config.auth from bigframes.session import clients - credentials, project = bigframes._config.auth.resolve_credentials_and_project( - config.options.bigquery - ) - clients_provider = clients.ClientsProvider( - project=project, + project=config.options.bigquery.project, location=config.options.bigquery.location, use_regional_endpoints=config.options.bigquery.use_regional_endpoints, - credentials=credentials, + credentials=config.options.bigquery.credentials, application_name=config.options.bigquery.application_name, bq_kms_key_name=config.options.bigquery.kms_key_name, client_endpoints_override=config.options.bigquery.client_endpoints_override, requests_transport_adapters=config.options.bigquery.requests_transport_adapters, ) - return clients_provider.bqclient, project + return clients_provider.bqclient def _dry_run(query, bqclient) -> bigquery.QueryJob: @@ -707,7 +684,7 @@ def _set_default_session_location_if_possible_deferred_query(create_query): return query = create_query() - bqclient, default_project = _get_bqclient_and_project() + bqclient = _get_bqclient() if bigquery.is_query(query): # Intentionally run outside of the session so that we can detect the @@ -715,13 +692,6 @@ def _set_default_session_location_if_possible_deferred_query(create_query): # aren't necessary. job = _dry_run(query, bqclient) config.options.bigquery.location = job.location - elif bq_data.is_irc_table(query): - irc_table = bigframes.session.iceberg.get_table( - default_project, query, bqclient._credentials - ) - config.options.bigquery.location = bq_data.get_default_bq_region( - irc_table.metadata.location - ) else: table = bqclient.get_table(query) config.options.bigquery.location = table.location diff --git a/bigframes/series.py b/bigframes/series.py index 07dcb0de628..814d59befff 100644 --- a/bigframes/series.py +++ b/bigframes/series.py @@ -18,49 +18,51 @@ import datetime import functools +import inspect import itertools import numbers import textwrap import typing -import warnings from typing import ( Any, Callable, + cast, Iterable, List, Literal, Mapping, Optional, + overload, Sequence, Tuple, - TypeVar, Union, - cast, - overload, ) +import warnings import bigframes_vendored.constants as constants import bigframes_vendored.pandas.core.series as vendored_pandas_series -import google.cloud.bigquery.job +import google.cloud.bigquery as bigquery import numpy import pandas +from pandas.api import extensions as pd_ext import pyarrow as pa import typing_extensions -from pandas.api import extensions as pd_ext import bigframes.core +from bigframes.core import agg_expressions, groupby import bigframes.core.block_transforms as block_ops import bigframes.core.blocks as blocks -import bigframes.core.col import bigframes.core.expression as ex import bigframes.core.identifiers as ids import bigframes.core.indexers import bigframes.core.indexes as indexes +from bigframes.core.logging import log_adapter import bigframes.core.ordering as order import bigframes.core.scalar as scalars import bigframes.core.utils as utils import bigframes.core.validations as validations import bigframes.core.window +from bigframes.core.window import rolling import bigframes.core.window_spec as windows import bigframes.dataframe import bigframes.dtypes @@ -69,24 +71,19 @@ import bigframes.functions import bigframes.operations as ops import bigframes.operations.aggregations as agg_ops +import bigframes.operations.blob as blob +import bigframes.operations.datetimes as dt import bigframes.operations.lists as lists import bigframes.operations.plotting as plotting import bigframes.operations.python_op_maps as python_ops import bigframes.operations.structs as structs import bigframes.session -from bigframes._tools import docs -from bigframes.core import agg_expressions, groupby -from bigframes.core.logging import log_adapter -from bigframes.core.window import rolling if typing.TYPE_CHECKING: - import bigframes.extensions.bigframes.series_accessor as series_bigquery_accessor import bigframes.geopandas.geoseries - import bigframes.operations.datetimes as datetimes import bigframes.operations.strings as strings -U = TypeVar("U") LevelType = typing.Union[str, int] LevelsType = typing.Union[LevelType, typing.Sequence[LevelType]] @@ -100,8 +97,7 @@ @log_adapter.class_logger -@docs.inherit_docs(vendored_pandas_series.Series) -class Series: +class Series(vendored_pandas_series.Series): # Must be above 5000 for pandas to delegate to bigframes for binops __pandas_priority__ = 13000 @@ -119,7 +115,7 @@ def __init__( *, session: Optional[bigframes.session.Session] = None, ): - self._query_job: Optional[google.cloud.bigquery.job.QueryJob] = None + self._query_job: Optional[bigquery.QueryJob] = None import bigframes.pandas # Ignore object dtype if provided, as it provides no additional @@ -195,7 +191,6 @@ def __init__( else: if isinstance(dtype, str) and dtype.lower() == "json": dtype = bigframes.dtypes.JSON_DTYPE - pd_series = pandas.Series( data=data, index=index, # type:ignore @@ -210,10 +205,8 @@ def __init__( self._block.session._register_object(self) @property - def dt(self) -> datetimes.DatetimeMethods: - import bigframes.operations.datetimes as datetimes - - return datetimes.DatetimeMethods(self) + def dt(self) -> dt.DatetimeMethods: + return dt.DatetimeMethods(self) @property def dtype(self): @@ -245,6 +238,7 @@ def loc(self) -> bigframes.core.indexers.LocSeriesIndexer: return bigframes.core.indexers.LocSeriesIndexer(self) @property + @validations.requires_ordering() def iloc(self) -> bigframes.core.indexers.IlocSeriesIndexer: return bigframes.core.indexers.IlocSeriesIndexer(self) @@ -302,26 +296,7 @@ def keys(self) -> indexes.Index: return self.index @property - def bigquery( - self, - ) -> series_bigquery_accessor.BigframesBigQuerySeriesAccessor: - """ - Accessor for BigQuery functionality. - - Returns: - bigframes.extensions.core.series_accessor.BigQuerySeriesAccessor: - Accessor that exposes BigQuery functionality on a Series, - with method names closer to SQL. - """ - # Import the accessor here to avoid circular imports. - import bigframes.extensions.bigframes.series_accessor - - return bigframes.extensions.bigframes.series_accessor.BigframesBigQuerySeriesAccessor( - self - ) - - @property - def query_job(self) -> Optional[google.cloud.bigquery.job.QueryJob]: + def query_job(self) -> Optional[bigquery.QueryJob]: """BigQuery job metadata for the most recent query. Returns: @@ -340,6 +315,10 @@ def struct(self) -> structs.StructAccessor: def list(self) -> lists.ListAccessor: return lists.ListAccessor(self) + @property + def blob(self) -> blob.BlobAccessor: + return blob.BlobAccessor(self) + @property @validations.requires_ordering() def T(self) -> Series: @@ -361,32 +340,17 @@ def _struct_fields(self) -> List[str]: struct_type = typing.cast(pa.StructType, self._dtype.pyarrow_dtype) return [struct_type.field(i).name for i in range(struct_type.num_fields)] - @property - def sql(self) -> str: - """Compiles this Series's expression tree to SQL. - - Returns: - A string representing the compiled SQL. - """ - - return self.to_frame().sql - @validations.requires_ordering() def transpose(self) -> Series: return self - def _set_internal_query_job( - self, query_job: Optional[google.cloud.bigquery.job.QueryJob] - ): + def _set_internal_query_job(self, query_job: Optional[bigquery.QueryJob]): self._query_job = query_job def __len__(self): return self.shape[0] - def __bool__(self): - raise ValueError( - "Cannot convert Series into bool. Consider using .empty(), .item(), .any(), or .all() methods." - ) + __len__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__len__) def __iter__(self) -> typing.Iterator: return itertools.chain.from_iterable( @@ -403,7 +367,8 @@ def copy(self) -> Series: def rename( self, index: Union[blocks.Label, Mapping[Any, Any]] = None, - ) -> Series: ... + ) -> Series: + ... @overload def rename( @@ -412,7 +377,8 @@ def rename( *, inplace: Literal[False], **kwargs, - ) -> Series: ... + ) -> Series: + ... @overload def rename( @@ -421,7 +387,8 @@ def rename( *, inplace: Literal[True], **kwargs, - ) -> None: ... + ) -> None: + ... def rename( self, @@ -467,6 +434,7 @@ def rename( # rename the Series name if isinstance(index, typing.Hashable): + # Python 3.9 doesn't allow isinstance of Optional index = typing.cast(Optional[str], index) block = self._block.with_column_labels([index]) @@ -482,7 +450,8 @@ def rename( def rename_axis( self, mapper: typing.Union[blocks.Label, typing.Sequence[blocks.Label]], - ) -> Series: ... + ) -> Series: + ... @overload def rename_axis( @@ -491,7 +460,8 @@ def rename_axis( *, inplace: Literal[False], **kwargs, - ) -> Series: ... + ) -> Series: + ... @overload def rename_axis( @@ -500,7 +470,8 @@ def rename_axis( *, inplace: Literal[True], **kwargs, - ) -> None: ... + ) -> None: + ... @validations.requires_index def rename_axis( @@ -544,7 +515,8 @@ def reset_index( drop: Literal[False] = ..., inplace: Literal[False] = ..., allow_duplicates: Optional[bool] = ..., - ) -> bigframes.dataframe.DataFrame: ... + ) -> bigframes.dataframe.DataFrame: + ... @overload def reset_index( @@ -555,7 +527,8 @@ def reset_index( drop: Literal[True] = ..., inplace: Literal[False] = ..., allow_duplicates: Optional[bool] = ..., - ) -> Series: ... + ) -> Series: + ... @overload def reset_index( @@ -566,7 +539,8 @@ def reset_index( drop: bool = ..., inplace: Literal[True] = ..., allow_duplicates: Optional[bool] = ..., - ) -> None: ... + ) -> None: + ... @validations.requires_ordering() def reset_index( @@ -595,9 +569,6 @@ def reset_index( block = block.assign_label(self._value_column, name) return bigframes.dataframe.DataFrame(block) - def _prepare_display_df(self) -> bigframes.dataframe.DataFrame: - return self.to_frame()._prepare_display_df() - def _repr_mimebundle_(self, include=None, exclude=None): """ Custom display method for IPython/Jupyter environments. @@ -646,17 +617,9 @@ def astype( if errors not in ["raise", "null"]: raise ValueError("Argument 'errors' must be one of 'raise' or 'null'") dtype = bigframes.dtypes.bigframes_type(dtype) - safe = errors == "null" - if dtype == bigframes.dtypes.JSON_DTYPE: - return self._apply_unary_op(bigframes.operations.ToJSON(safe=safe)) - elif self.dtype == bigframes.dtypes.JSON_DTYPE: - return self._apply_unary_op( - bigframes.operations.JSONDecode(to_type=dtype, safe=safe) - ) - else: - return self._apply_unary_op( - bigframes.operations.AsTypeOp(to_type=dtype, safe=safe) - ) + return self._apply_unary_op( + bigframes.operations.AsTypeOp(to_type=dtype, safe=(errors == "null")) + ) def to_pandas( self, @@ -790,7 +753,6 @@ def to_pandas_batches( max_results: Optional[int] = None, *, allow_large_results: Optional[bool] = None, - cell_execution_count: Optional[int] = None, ) -> Iterable[pandas.Series]: """Stream Series results to an iterable of pandas Series. @@ -843,11 +805,10 @@ def to_pandas_batches( page_size=page_size, max_results=max_results, allow_large_results=allow_large_results, - cell_execution_count=cell_execution_count, ) return map(lambda df: cast(pandas.Series, df.squeeze(1)), batches) - def _compute_dry_run(self) -> google.cloud.bigquery.job.QueryJob: + def _compute_dry_run(self) -> bigquery.QueryJob: _, query_job = self._block._compute_dry_run((self._value_column,)) return query_job @@ -949,6 +910,7 @@ def ffill(self, *, limit: typing.Optional[int] = None) -> Series: return self._apply_window_op(agg_ops.LastNonNullOp(), window) pad = ffill + pad.__doc__ = inspect.getdoc(vendored_pandas_series.Series.ffill) @validations.requires_ordering() def bfill(self, *, limit: typing.Optional[int] = None) -> Series: @@ -1196,33 +1158,45 @@ def isna(self) -> "Series": return self._apply_unary_op(ops.isnull_op) isnull = isna + isnull.__doc__ = inspect.getdoc(vendored_pandas_series.Series.isna) def notna(self) -> "Series": return self._apply_unary_op(ops.notnull_op) notnull = notna + notnull.__doc__ = inspect.getdoc(vendored_pandas_series.Series.notna) def __and__(self, other: bool | int | Series) -> Series: return self._apply_binary_op(other, ops.and_op) + __and__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__and__) + __rand__ = __and__ def __or__(self, other: bool | int | Series) -> Series: return self._apply_binary_op(other, ops.or_op) + __or__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__or__) + __ror__ = __or__ def __xor__(self, other: bool | int | Series) -> Series: return self._apply_binary_op(other, ops.xor_op) + __or__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__xor__) + __rxor__ = __xor__ def __add__(self, other: float | int | pandas.Timedelta | Series) -> Series: return self.add(other) + __add__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__add__) + def __radd__(self, other: float | int | pandas.Timedelta | Series) -> Series: return self.radd(other) + __radd__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__radd__) + def add(self, other: float | int | pandas.Timedelta | Series) -> Series: return self._apply_binary_op(other, ops.add_op) @@ -1232,9 +1206,13 @@ def radd(self, other: float | int | pandas.Timedelta | Series) -> Series: def __sub__(self, other: float | int | Series) -> Series: return self.sub(other) + __sub__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__sub__) + def __rsub__(self, other: float | int | Series) -> Series: return self.rsub(other) + __rsub__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__rsub__) + def sub(self, other) -> Series: return self._apply_binary_op(other, ops.sub_op) @@ -1242,13 +1220,18 @@ def rsub(self, other) -> Series: return self._apply_binary_op(other, ops.sub_op, reverse=True) subtract = sub + subtract.__doc__ = inspect.getdoc(vendored_pandas_series.Series.sub) def __mul__(self, other: float | int | Series) -> Series: return self.mul(other) + __mul__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__mul__) + def __rmul__(self, other: float | int | Series) -> Series: return self.rmul(other) + __rmul__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__rmul__) + def mul(self, other: float | int | Series) -> Series: return self._apply_binary_op(other, ops.mul_op) @@ -1256,29 +1239,40 @@ def rmul(self, other: float | int | Series) -> Series: return self._apply_binary_op(other, ops.mul_op, reverse=True) multiply = mul + multiply.__doc__ = inspect.getdoc(vendored_pandas_series.Series.mul) def __truediv__(self, other: float | int | pandas.Timedelta | Series) -> Series: return self.truediv(other) + __truediv__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__truediv__) + def __rtruediv__(self, other: float | int | pandas.Timedelta | Series) -> Series: return self.rtruediv(other) + __rtruediv__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__rtruediv__) + def truediv(self, other: float | int | pandas.Timedelta | Series) -> Series: return self._apply_binary_op(other, ops.div_op) def rtruediv(self, other: float | int | pandas.Timedelta | Series) -> Series: return self._apply_binary_op(other, ops.div_op, reverse=True) + truediv.__doc__ = inspect.getdoc(vendored_pandas_series.Series.truediv) div = divide = truediv rdiv = rtruediv + rdiv.__doc__ = inspect.getdoc(vendored_pandas_series.Series.rtruediv) def __floordiv__(self, other: float | int | pandas.Timedelta | Series) -> Series: return self.floordiv(other) + __floordiv__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__floordiv__) + def __rfloordiv__(self, other: float | int | pandas.Timedelta | Series) -> Series: return self.rfloordiv(other) + __rfloordiv__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__rfloordiv__) + def floordiv(self, other: float | int | pandas.Timedelta | Series) -> Series: return self._apply_binary_op(other, ops.floordiv_op) @@ -1288,9 +1282,13 @@ def rfloordiv(self, other: float | int | pandas.Timedelta | Series) -> Series: def __pow__(self, other: float | int | Series) -> Series: return self.pow(other) + __pow__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__pow__) + def __rpow__(self, other: float | int | Series) -> Series: return self.rpow(other) + __rpow__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__rpow__) + def pow(self, other: float | int | Series) -> Series: return self._apply_binary_op(other, ops.pow_op) @@ -1324,9 +1322,13 @@ def ge(self, other) -> Series: def __mod__(self, other) -> Series: # type: ignore return self.mod(other) + __mod__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__mod__) + def __rmod__(self, other) -> Series: # type: ignore return self.rmod(other) + __rmod__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__rmod__) + def mod(self, other) -> Series: # type: ignore return self._apply_binary_op(other, ops.mod_op) @@ -1349,9 +1351,13 @@ def dot(self, other): def __matmul__(self, other): return self.dot(other) + __matmul__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__matmul__) + def __rmatmul__(self, other): return self.dot(other) + __rmatmul__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__rmatmul__) + def combine_first(self, other: Series) -> Series: result = self._apply_binary_op(other, ops.coalesce_op) result.name = self.name @@ -1366,6 +1372,8 @@ def update(self, other: Union[Series, Sequence, Mapping]) -> None: def __abs__(self) -> Series: return self.abs() + __abs__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.abs) + def abs(self) -> Series: return self._apply_unary_op(ops.abs_op) @@ -1440,6 +1448,7 @@ def agg(self, func: str | typing.Sequence[str]) -> scalars.Scalar | Series: return self._apply_aggregation(agg_ops.lookup_agg_func(func)[0]) aggregate = agg + aggregate.__doc__ = inspect.getdoc(vendored_pandas_series.Series.agg) def describe(self) -> Series: from bigframes.pandas.core.methods import describe @@ -1479,6 +1488,7 @@ def kurt(self): return (numerator / denominator) - adjustment kurtosis = kurt + kurtosis.__doc__ = inspect.getdoc(vendored_pandas_series.Series.kurt) def mode(self) -> Series: block = self._block @@ -1543,6 +1553,7 @@ def prod(self) -> float: return typing.cast(float, self._apply_aggregation(agg_ops.product_op)) product = prod + product.__doc__ = inspect.getdoc(vendored_pandas_series.Series.prod) def __eq__(self, other: object) -> Series: # type: ignore return self.eq(other) @@ -1553,6 +1564,8 @@ def __ne__(self, other: object) -> Series: # type: ignore def __invert__(self) -> Series: return self._apply_unary_op(ops.invert_op) + __invert__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__invert__) + def __pos__(self) -> Series: return self._apply_unary_op(ops.pos_op) @@ -1572,9 +1585,9 @@ def ne(self, other: object) -> Series: def items(self): for batch_df in self._block.to_pandas_batches(): - assert batch_df.shape[1] == 1, ( - f"Expected 1 column in the dataframe, but got {batch_df.shape[1]}." - ) + assert ( + batch_df.shape[1] == 1 + ), f"Expected 1 column in the dataframe, but got {batch_df.shape[1]}." for item in batch_df.squeeze(axis=1).items(): yield item @@ -1582,7 +1595,7 @@ def _apply_callable(self, condition): """ "Executes the possible callable condition as needed.""" if callable(condition): # When it's a bigframes function. - if isinstance(condition, bigframes.functions.Udf): + if hasattr(condition, "bigframes_bigquery_function"): return self.apply(condition) # When it's a plain Python function. else: @@ -1730,6 +1743,8 @@ def __getitem__(self, indexer): return Series(block) return self.loc[indexer] + __getitem__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__getitem__) + def __getattr__(self, key: str): # Protect against recursion errors with uninitialized Series objects. # We use "_block" attribute to check whether the instance is initialized. @@ -1737,7 +1752,7 @@ def __getattr__(self, key: str): # https://github.com/googleapis/python-bigquery-dataframes/issues/728 # and # https://nedbatchelder.com/blog/201010/surprising_getattr_recursion.html - if "_block" not in self.__dict__ or key == "_block": + if key == "_block": raise AttributeError(key) elif hasattr(pandas.Series, key): log_adapter.submit_pandas_labels( @@ -1802,9 +1817,10 @@ def sort_values( axis=..., inplace: Literal[True] = ..., ascending: bool | typing.Sequence[bool] = ..., - kind: str | None = ..., + kind: str = ..., na_position: typing.Literal["first", "last"] = ..., - ) -> None: ... + ) -> None: + ... @typing.overload def sort_values( @@ -1813,9 +1829,10 @@ def sort_values( axis=..., inplace: Literal[False] = ..., ascending: bool | typing.Sequence[bool] = ..., - kind: str | None = ..., + kind: str = ..., na_position: typing.Literal["first", "last"] = ..., - ) -> Series: ... + ) -> Series: + ... def sort_values( self, @@ -1823,21 +1840,19 @@ def sort_values( axis=0, inplace: bool = False, ascending=True, - kind: str | None = None, + kind: str = "quicksort", na_position: typing.Literal["first", "last"] = "last", ) -> Optional[Series]: if axis != 0 and axis != "index": raise ValueError(f"No axis named {axis} for object type Series") if na_position not in ["first", "last"]: raise ValueError("Param na_position must be one of 'first' or 'last'") - is_stable = (kind or constants.DEFAULT_SORT_KIND) in constants.STABLE_SORT_KINDS block = self._block.order_by( [ order.ascending_over(self._value_column, (na_position == "last")) if ascending else order.descending_over(self._value_column, (na_position == "last")) ], - stable=is_stable, ) if inplace: self._set_block(block) @@ -1847,35 +1862,19 @@ def sort_values( @typing.overload # type: ignore[override] def sort_index( - self, - *, - axis=..., - inplace: Literal[False] = ..., - ascending=..., - kind: str | None = ..., - na_position=..., - ) -> Series: ... + self, *, axis=..., inplace: Literal[False] = ..., ascending=..., na_position=... + ) -> Series: + ... @typing.overload def sort_index( - self, - *, - axis=0, - inplace: Literal[True] = ..., - ascending=..., - kind: str | None = ..., - na_position=..., - ) -> None: ... + self, *, axis=0, inplace: Literal[True] = ..., ascending=..., na_position=... + ) -> None: + ... @validations.requires_index def sort_index( - self, - *, - axis=0, - inplace: bool = False, - ascending=True, - kind: str | None = None, - na_position="last", + self, *, axis=0, inplace: bool = False, ascending=True, na_position="last" ) -> Optional[Series]: # TODO(tbergeron): Support level parameter once multi-index introduced. if axis != 0 and axis != "index": @@ -1890,8 +1889,7 @@ def sort_index( else order.descending_over(column, na_last) for column in block.index_columns ] - is_stable = (kind or constants.DEFAULT_SORT_KIND) in constants.STABLE_SORT_KINDS - block = block.order_by(ordering, stable=is_stable) + block = block.order_by(ordering) if inplace: self._set_block(block) return None @@ -1930,22 +1928,6 @@ def expanding(self, min_periods: int = 1) -> bigframes.core.window.Window: self._block, window_spec, self._block.value_columns, is_series=True ) - def pipe( - self, - func: Union[Callable[..., U], tuple[Callable[..., U], str]], - *args, - **kwargs, - ) -> U: - import bigframes_vendored.pandas.core.common as common - - return common.pipe(self, func, *args, **kwargs) - - def get(self, key, default=None): - try: - return self[key] - except (KeyError, ValueError, IndexError): - return default - def groupby( self, by: typing.Union[ @@ -2013,12 +1995,9 @@ def _groupby_values( value_col = self._value_column for key in by: if isinstance(key, Series): - ( - block, - ( - get_column_left, - get_column_right, - ), + block, ( + get_column_left, + get_column_right, ) = block.join(key._block, how="inner" if dropna else "left") value_col = get_column_left[value_col] @@ -2073,48 +2052,33 @@ def apply( " are supported." ) - # Highest priority: try to map directly to an operator, for eg numpy - # ufuncs, or simple arithmetic/logic operators. + if isinstance(func, bigframes.functions.BigqueryCallableRoutine): + # We are working with bigquery function at this point + if args: + result_series = self._apply_nary_op( + ops.NaryRemoteFunctionOp(function_def=func.udf_def), args + ) + # TODO(jialuo): Investigate why `_apply_nary_op` drops the series + # `name`. Manually reassigning it here as a temporary fix. + result_series.name = self.name + else: + result_series = self._apply_unary_op( + ops.RemoteFunctionOp(function_def=func.udf_def, apply_on_null=True) + ) + result_series = func._post_process_series(result_series) + + return result_series + bf_op = python_ops.python_callable_to_op(func) if bf_op and isinstance(bf_op, ops.UnaryOp): return self._apply_unary_op(bf_op) + # It is neither a remote function nor a managed function. + # Then it must be a vectorized function that applies to the Series + # as a whole. if by_row: - from bigframes._config import options - - enable_transpile = options.experiments.enable_python_transpiler - return self._apply_by_row( - func, args=args, transpile_enabled=enable_transpile - ) - try: - return func(self) # type: ignore - except Exception as ex: - # This could happen if any of the operators in func is not - # supported on a Series. Let's guide the customer to use a - # bigquery function instead - if hasattr(ex, "message"): - ex.message += f"\n{_bigquery_function_recommendation_message}" - raise - - def _apply_by_row( - self, - func: typing.Callable, - args: typing.Tuple = (), - transpile_enabled: bool = False, - ) -> Series: - """ - Apply callable or deployed udf row-wise on the series. - """ - if not callable(func): - raise ValueError( - "Expected a callable function. If you meant to use a BigQuery function, please wrap it with bigframes.pandas.udf(...)" - ) - try: - expr = ops.func_to_expr(func) - # We get this message even if transpiler could have in theory translated it. - except Exception: raise ValueError( - "You have passed a functi1on as-is. If your intention is to " + "You have passed a function as-is. If your intention is to " "apply this function in a vectorized way (i.e. to the " "entire Series as a whole, and you are sure that it " "performs only the operations that are implemented for a " @@ -2128,12 +2092,15 @@ def _apply_by_row( "or `bigframes.pandas.remote_function` before passing." ) - result_series = self._apply_callable_expr(expr, args) - # TODO(jialuo): Investigate why `_apply_nary_op` drops the series - # `name`. Manually reassigning it here as a temporary fix. - result_series.name = self.name - - return result_series + try: + return func(self) # type: ignore + except Exception as ex: + # This could happen if any of the operators in func is not + # supported on a Series. Let's guide the customer to use a + # bigquery function instead + if hasattr(ex, "message"): + ex.message += f"\n{_bigquery_function_recommendation_message}" + raise def combine( self, @@ -2147,16 +2114,11 @@ def combine( " are supported." ) - from bigframes._config import options - - if isinstance(func, bigframes.functions.Udf) or ( - options.experiments.enable_python_transpiler and callable(func) - ): - result_series = self._apply_callable_expr(ops.func_to_expr(func), (other,)) - if hasattr(other, "name") and other.name != self._name: # type: ignore - result_series.name = None - else: - result_series.name = self.name + if isinstance(func, bigframes.functions.BigqueryCallableRoutine): + result_series = self._apply_binary_op( + other, ops.BinaryRemoteFunctionOp(function_def=func.udf_def) + ) + result_series = func._post_process_series(result_series) return result_series bf_op = python_ops.python_callable_to_op(func) @@ -2312,14 +2274,11 @@ def mask(self, cond, other=None) -> Series: return self.where(~cond, other) def to_frame(self, name: blocks.Label = None) -> bigframes.dataframe.DataFrame: - provided_name = name if name is not None else self.name + provided_name = name if name else self.name # To be consistent with Pandas, it assigns 0 as the column name if missing. 0 is the first element of RangeIndex. - column_names: List[blocks.Label] - if provided_name is None or pandas.isna([cast(Any, provided_name)])[0]: - column_names = [0] - else: - column_names = [provided_name] - block = self._block.with_column_labels(column_names) + block = self._block.with_column_labels( + [provided_name] if provided_name else [0] + ) return bigframes.dataframe.DataFrame(block) def to_csv( @@ -2351,16 +2310,13 @@ def to_dict( *, allow_large_results: Optional[bool] = None, ) -> typing.Mapping: - return typing.cast( - dict, - self.to_pandas(allow_large_results=allow_large_results).to_dict(into=into), - ) # type: ignore + return typing.cast(dict, self.to_pandas(allow_large_results=allow_large_results).to_dict(into=into)) # type: ignore def to_excel( self, excel_writer, sheet_name="Sheet1", *, allow_large_results=None, **kwargs ) -> None: return self.to_pandas(allow_large_results=allow_large_results).to_excel( - excel_writer, sheet_name=sheet_name, **kwargs + excel_writer, sheet_name, **kwargs ) def to_json( @@ -2384,12 +2340,8 @@ def to_json( ) else: pd_series = self.to_pandas(allow_large_results=allow_large_results) - # Pandas Series.to_json only supports a subset of orients, but bigframes Series.to_json allows all of them. return pd_series.to_json( - path_or_buf=path_or_buf, - orient=orient, # type: ignore[arg-type] - lines=lines, - index=index, # type: ignore + path_or_buf=path_or_buf, orient=orient, lines=lines, index=index # type: ignore ) def to_latex( @@ -2414,6 +2366,7 @@ def tolist( return self.to_pandas(allow_large_results=allow_large_results).to_list() to_list = tolist + to_list.__doc__ = inspect.getdoc(vendored_pandas_series.Series.tolist) def to_markdown( self, @@ -2424,9 +2377,7 @@ def to_markdown( allow_large_results: Optional[bool] = None, **kwargs, ) -> typing.Optional[str]: - return self.to_pandas(allow_large_results=allow_large_results).to_markdown( - buf, mode=mode, index=index, **kwargs - ) # type: ignore + return self.to_pandas(allow_large_results=allow_large_results).to_markdown(buf, mode=mode, index=index, **kwargs) # type: ignore def to_numpy( self, @@ -2446,6 +2397,8 @@ def __array__(self, dtype=None, copy: Optional[bool] = None) -> numpy.ndarray: raise ValueError("Cannot convert to array without copy.") return self.to_numpy(dtype=dtype) + __array__.__doc__ = inspect.getdoc(vendored_pandas_series.Series.__array__) + def to_pickle(self, path, *, allow_large_results=None, **kwargs) -> None: return self.to_pandas(allow_large_results=allow_large_results).to_pickle( path, **kwargs @@ -2522,19 +2475,14 @@ def map( map_df = map_df.set_index("keys") elif callable(arg): # This is for remote function and managed funtion. - from bigframes._config import options - - enable_transpile = options.experiments.enable_python_transpiler - return self._apply_by_row(arg, transpile_enabled=enable_transpile) + return self.apply(arg) else: # Mirroring pandas, call the uncallable object arg() # throws TypeError: object is not callable self_df = self.to_frame(name="series") result_df = self_df.join(map_df, on="series") - result = cast(Series, result_df[self.name]) - result.name = self.name - return result + return result_df[self.name] @validations.requires_ordering() def sample( @@ -2619,6 +2567,8 @@ def hist( ): return self.plot.hist(by=by, bins=bins, **kwargs) + hist.__doc__ = inspect.getdoc(plotting.PlotAccessor.hist) + def line( self, x: typing.Optional[typing.Hashable] = None, @@ -2627,6 +2577,8 @@ def line( ): return self.plot.line(x=x, y=y, **kwargs) + line.__doc__ = inspect.getdoc(plotting.PlotAccessor.line) + def area( self, x: typing.Optional[typing.Hashable] = None, @@ -2636,6 +2588,8 @@ def area( ): return self.plot.area(x=x, y=y, stacked=stacked, **kwargs) + area.__doc__ = inspect.getdoc(plotting.PlotAccessor.area) + def bar( self, x: typing.Optional[typing.Hashable] = None, @@ -2644,6 +2598,8 @@ def bar( ): return self.plot.bar(x=x, y=y, **kwargs) + bar.__doc__ = inspect.getdoc(plotting.PlotAccessor.bar) + def _slice( self, start: typing.Optional[int] = None, @@ -2760,20 +2716,7 @@ def _apply_nary_op( others, ignore_self=ignore_self, cast_scalars=False ) block, result_id = block.project_expr(op.as_expr(*values)) - return Series(block.select_column(result_id).with_column_labels([None])) - - def _apply_callable_expr( - self, - callable_expr: bigframes.operations.to_op.CallableExpression, - others: Sequence[typing.Union[Series, scalars.Scalar]], - ignore_self=False, - ): - """Applies a CallableExpression to the series and others.""" - values, block = self._align_n( - others, ignore_self=ignore_self, cast_scalars=False - ) - block, result_id = block.project_expr(callable_expr.apply(*values)) - return Series(block.select_column(result_id).with_column_labels([None])) + return Series(block.select_column(result_id)) def _apply_binary_aggregation( self, other: Series, stat: agg_ops.BinaryAggregateOp @@ -2783,33 +2726,23 @@ def _apply_binary_aggregation( assert isinstance(right, ex.DerefOp) return block.get_binary_stat(left.id.name, right.id.name, stat) - AlignedExprT = Union[ex.ScalarConstantExpression, ex.DerefOp, ex.OmittedArg] + AlignedExprT = Union[ex.ScalarConstantExpression, ex.DerefOp] @typing.overload def _align( self, other: Series, how="outer" - ) -> tuple[ - ex.DerefOp, - ex.DerefOp, - blocks.Block, - ]: ... + ) -> tuple[ex.DerefOp, ex.DerefOp, blocks.Block,]: + ... @typing.overload def _align( self, other: typing.Union[Series, scalars.Scalar], how="outer" - ) -> tuple[ - ex.DerefOp, - AlignedExprT, - blocks.Block, - ]: ... + ) -> tuple[ex.DerefOp, AlignedExprT, blocks.Block,]: + ... def _align( self, other: typing.Union[Series, scalars.Scalar], how="outer" - ) -> tuple[ - ex.DerefOp, - AlignedExprT, - blocks.Block, - ]: + ) -> tuple[ex.DerefOp, AlignedExprT, blocks.Block,]: """Aligns the series value with another scalar or series object. Returns new left column id, right column id and joined tabled expression.""" values, block = self._align_n( [ @@ -2819,13 +2752,7 @@ def _align( ) return (typing.cast(ex.DerefOp, values[0]), values[1], block) - def _align3( - self, - other1: Series | scalars.Scalar, - other2: Series | scalars.Scalar, - how="left", - cast_scalars: bool = True, - ) -> tuple[ex.DerefOp, AlignedExprT, AlignedExprT, blocks.Block]: # type: ignore + def _align3(self, other1: Series | scalars.Scalar, other2: Series | scalars.Scalar, how="left", cast_scalars: bool = True) -> tuple[ex.DerefOp, AlignedExprT, AlignedExprT, blocks.Block]: # type: ignore """Aligns the series value with 2 other scalars or series objects. Returns new values and joined tabled expression.""" values, index = self._align_n([other1, other2], how, cast_scalars=cast_scalars) return ( @@ -2837,32 +2764,25 @@ def _align3( def _align_n( self, - others: typing.Sequence[ - typing.Union[Series, bigframes.core.col.Expression, scalars.Scalar] - ], + others: typing.Sequence[typing.Union[Series, scalars.Scalar]], how="outer", ignore_self=False, cast_scalars: bool = False, ) -> tuple[ - typing.Sequence[Union[ex.ScalarConstantExpression, ex.DerefOp, ex.OmittedArg]], + typing.Sequence[Union[ex.ScalarConstantExpression, ex.DerefOp]], blocks.Block, ]: if ignore_self: - value_ids: List[ - Union[ex.ScalarConstantExpression, ex.DerefOp, ex.OmittedArg] - ] = [] + value_ids: List[Union[ex.ScalarConstantExpression, ex.DerefOp]] = [] else: value_ids = [ex.deref(self._value_column)] block = self._block for other in others: if isinstance(other, Series): - ( - block, - ( - get_column_left, - get_column_right, - ), + block, ( + get_column_left, + get_column_right, ) = block.join(other._block, how=how) rebindings = { ids.ColumnId(old): ids.ColumnId(new) @@ -2875,17 +2795,6 @@ def _align_n( *remapped_value_ids, # type: ignore ex.deref(get_column_right[other._value_column]), ] - elif isinstance(other, bigframes.core.col.Expression): - if isinstance(other._value, ex.OmittedArg): - value_ids = [*value_ids, other._value] - continue - - label_to_col_ref = { - label: ex.deref(id) for id, label in block.col_id_to_label.items() - } - resolved_expr = other._value.bind_variables(label_to_col_ref) - block = block.project_block_exprs([resolved_expr], labels=[None]) - value_ids = [*value_ids, ex.deref(block.value_columns[-1])] else: # Will throw if can't interpret as scalar. dtype = typing.cast(bigframes.dtypes.Dtype, self._dtype) diff --git a/bigframes/session/__init__.py b/bigframes/session/__init__.py index e20f61901f9..ca8fbf29196 100644 --- a/bigframes/session/__init__.py +++ b/bigframes/session/__init__.py @@ -16,6 +16,7 @@ from __future__ import annotations +from collections import abc import datetime import fnmatch import inspect @@ -24,23 +25,22 @@ import secrets import threading import typing -import warnings -import weakref -from collections import abc from typing import ( - IO, Any, Callable, Dict, + IO, Iterable, Literal, MutableSequence, Optional, + overload, Sequence, Tuple, Union, - overload, ) +import warnings +import weakref import bigframes_vendored.constants as constants import bigframes_vendored.google_cloud_bigquery.retry as third_party_gcb_retry @@ -52,35 +52,34 @@ import google.cloud.bigquery as bigquery import numpy as np import pandas -import pyarrow as pa from pandas._typing import ( CompressionOptions, FilePath, ReadPickleBuffer, StorageOptions, ) +import pyarrow as pa +from bigframes import exceptions as bfe +from bigframes import version import bigframes._config -import bigframes._config.auth import bigframes._config.bigquery_options as bigquery_options import bigframes.clients import bigframes.constants import bigframes.core +from bigframes.core import blocks, utils import bigframes.core.events import bigframes.core.indexes import bigframes.core.indexes.multi +from bigframes.core.logging import log_adapter import bigframes.core.pyformat import bigframes.formatting_helpers +import bigframes.functions._function_session as bff_session import bigframes.functions.function as bff +from bigframes.session import bigquery_session, bq_caching_executor, executor import bigframes.session._io.bigquery as bf_io_bigquery import bigframes.session.clients import bigframes.session.validation -from bigframes import exceptions as bfe -from bigframes import version -from bigframes.core import blocks, utils -from bigframes.core.logging import log_adapter -from bigframes.functions import _function_client, _function_session -from bigframes.session import bigquery_session, executor, proxy_executor # Avoid circular imports. if typing.TYPE_CHECKING: @@ -110,58 +109,6 @@ logger = logging.getLogger(__name__) -class _ExecutionHistory: - def __init__(self, jobs: list[dict]): - self._df = pandas.DataFrame(jobs) - if self._df.empty: - self._df = pandas.DataFrame( - columns=[ - "job_id", - "query_id", - "job_type", - "status", - "query", - "total_bytes_processed", - "job_url", - ] - ) - - def to_dataframe(self) -> pandas.DataFrame: - """Returns the execution history as a pandas DataFrame.""" - return self._df - - def _repr_html_(self) -> str | None: - import bigframes.formatting_helpers as formatter - - if self._df.empty: - return "
    No executions found.
    " - - cols = ["job_type", "job_id", "status", "total_bytes_processed", "job_url"] - - # Filter columns to only those that exist in the dataframe - available_cols = [c for c in cols if c in self._df.columns] - - def format_url(url): - return f'Open Job' if url else "" - - try: - df_display = self._df[available_cols].copy() - if "total_bytes_processed" in df_display.columns: - df_display["total_bytes_processed"] = df_display[ - "total_bytes_processed" - ].apply(formatter.get_formatted_bytes) - if "job_url" in df_display.columns: - df_display["job_url"] = df_display["job_url"].apply(format_url) - - # Rename job_id to query_id to match user expectations - if "job_id" in df_display.columns: - df_display = df_display.rename(columns={"job_id": "query_id"}) - - return df_display.to_html(escape=False, index=False) - except Exception: - return self._df.to_html() - - @log_adapter.class_logger class Session( third_party_pandas_gbq.GBQIOMixin, @@ -195,65 +142,46 @@ def __init__( # Publisher needs to be created before the other objects, especially # the executors, because they access it. self._publisher = bigframes.core.events.Publisher() - self._publisher.subscribe( - bigframes.formatting_helpers.create_progress_callback() - ) + self._publisher.subscribe(bigframes.formatting_helpers.progress_callback) if context is None: context = bigquery_options.BigQueryOptions() + if context.location is None: + self._location = "US" + msg = bfe.format_message( + f"No explicit location is set, so using location {self._location} for the session." + ) + # User's code + # -> get_global_session() + # -> connect() + # -> Session() + # + # Note: We could also have: + # User's code + # -> read_gbq() + # -> with_default_session() + # -> get_global_session() + # -> connect() + # -> Session() + # but we currently have no way to disambiguate these + # situations. + warnings.warn(msg, stacklevel=4, category=bfe.DefaultLocationWarning) + else: + self._location = context.location + self._bq_kms_key_name = context.kms_key_name # Instantiate a clients provider to help with cloud clients that will be # used in the future operations in the session if clients_provider: - # this path is only for unit testing. Not meant to be used by end users. self._clients_provider = clients_provider - self._location = context.location or "US" - project = "test_project" else: - ( - credentials, - project, - ) = bigframes._config.auth.resolve_credentials_and_project(context) - if context.location is None: - with bigquery.Client( - project=project, - credentials=credentials, - ) as temp_client: - row_iter = temp_client.query_and_wait( - "SELECT 1", - job_config=bigquery.QueryJobConfig(dry_run=True), - ) - self._location = row_iter.location or "US" - msg = bfe.format_message( - f"No explicit location is set, so using location {self._location} for the session." - ) - # User's code - # -> get_global_session() - # -> connect() - # -> Session() - # - # Note: We could also have: - # User's code - # -> read_gbq() - # -> with_default_session() - # -> get_global_session() - # -> connect() - # -> Session() - # but we currently have no way to disambiguate these - # situations. - warnings.warn( - msg, stacklevel=4, category=bfe.DefaultLocationWarning - ) - else: - self._location = context.location - self._clients_provider = clients.ClientsProvider( - project=project, - credentials=credentials, + project=context.project, location=self._location, use_regional_endpoints=context.use_regional_endpoints, + credentials=context.credentials, application_name=context.application_name, bq_kms_key_name=self._bq_kms_key_name, client_endpoints_override=context.client_endpoints_override, @@ -305,7 +233,7 @@ def __init__( ) self._metrics = metrics.ExecutionMetrics() - self._publisher.subscribe(self._metrics.on_event) + self._function_session = bff_session.FunctionSession() self._anon_dataset_manager = anonymous_dataset.AnonymousDatasetManager( self._clients_provider.bqclient, location=self._location, @@ -313,24 +241,6 @@ def __init__( kms_key=self._bq_kms_key_name, publisher=self._publisher, ) - self._function_session = _function_session.FunctionSession( - _function_client.FunctionClient( - gcp_project_id=project, - bq_location=self._location, - bq_client=self._clients_provider.bqclient, - bq_connection_manager=bigframes.clients.BqConnectionManager( - self._clients_provider.bqconnectionclient, - self._clients_provider.resourcemanagerclient, - ), - cloud_functions_client=self._clients_provider.cloudfunctionsclient, - publisher=self._publisher, - ), - dataset_manager=self._anon_dataset_manager, - default_connection=self._bq_connection, - location=self._location, - session_id=self._session_id, - manage_connections=not self._skip_bq_connection_check, - ) # Session temp tables don't support specifying kms key, so use anon dataset if kms key specified self._session_resource_manager = ( bigquery_session.SessionResourceManager( @@ -355,21 +265,15 @@ def __init__( metrics=self._metrics, publisher=self._publisher, ) - - labels = {} - if not self._strictly_ordered: - labels["bigframes-mode"] = "unordered" - - self._executor: executor.Executor = proxy_executor.DualCompilerProxyExecutor( + self._executor: executor.Executor = bq_caching_executor.BigQueryCachingExecutor( bqclient=self._clients_provider.bqclient, bqstoragereadclient=self._clients_provider.bqstoragereadclient, loader=self._loader, storage_manager=self._temp_storage_manager, + strictly_ordered=self._strictly_ordered, metrics=self._metrics, enable_polars_execution=context.enable_polars_execution, publisher=self._publisher, - labels=tuple(labels.items()), - function_manager=self._function_session, ) def __del__(self): @@ -462,80 +366,6 @@ def slot_millis_sum(self): """The sum of all slot time used by bigquery jobs in this session.""" return self._metrics.slot_millis - def execution_history( - self, - *, - events: Optional[Iterable[bigframes.core.events.Event]] = None, - job_ids: Optional[Iterable[str]] = None, - all_cells: bool = True, - ) -> _ExecutionHistory: - """Returns the history of executions initiated by BigFrames in the current session. - - Use `.to_dataframe()` on the result to get a pandas DataFrame. - - Args: - events (Iterable[Event], optional): - Filter execution history to only include jobs associated with the given events. - job_ids (Iterable[str], optional): - Filter execution history to only include jobs matching the given job IDs. - all_cells (bool, optional): - If True, do not filter execution history by notebook cell. If False, - and running in Colab/Jupyter, automatically filter history to only include - jobs executed within the current cell. Defaults to True. - """ - jobs = [job.__dict__ for job in self._metrics.jobs] - - if events is not None: - event_job_ids = { - getattr(event, "job_id", None) - for event in events - if getattr(event, "job_id", None) is not None - } - event_query_ids = { - getattr(event, "query_id", None) - for event in events - if getattr(event, "query_id", None) is not None - } - jobs = [ - job - for job in jobs - if ( - job.get("job_id") is not None and job.get("job_id") in event_job_ids - ) - or ( - job.get("query_id") is not None - and job.get("query_id") in event_query_ids - ) - ] - - elif job_ids is not None: - target_job_ids = set(job_ids) - jobs = [ - job - for job in jobs - if ( - job.get("job_id") is not None - and job.get("job_id") in target_job_ids - ) - or ( - job.get("query_id") is not None - and job.get("query_id") in target_job_ids - ) - ] - - elif not all_cells: - from bigframes.core.utils import get_ipython_execution_count - - current_count = get_ipython_execution_count() - if current_count is not None: - jobs = [ - job - for job in jobs - if job.get("cell_execution_count") == current_count - ] - - return _ExecutionHistory(jobs) - @property def _allows_ambiguity(self) -> bool: return self._allow_ambiguity @@ -544,16 +374,6 @@ def _allows_ambiguity(self) -> bool: def _anonymous_dataset(self): return self._anon_dataset_manager.dataset - @property - def bq_connection(self) -> str: - msg = bfe.format_message( - f"""You are using the BigFrames session default connection: {self._bq_connection}, - which can be different from the BigQuery project default connection. - This default connection may change in the future.""" - ) - warnings.warn(msg, category=FutureWarning) - return self._bq_connection - def __hash__(self): # Stable hash needed to use in expression tree return hash(str(self._session_id)) @@ -573,7 +393,9 @@ def close(self): remote_function_session = getattr(self, "_function_session", None) if remote_function_session: - remote_function_session.clean_up() + remote_function_session.clean_up( + self.bqclient, self.cloudfunctionsclient, self.session_id + ) publisher_session = getattr(self, "_publisher", None) if publisher_session: @@ -595,7 +417,8 @@ def read_gbq( # type: ignore[overload-overlap] col_order: Iterable[str] = ..., dry_run: Literal[False] = ..., allow_large_results: Optional[bool] = ..., - ) -> dataframe.DataFrame: ... + ) -> dataframe.DataFrame: + ... @overload def read_gbq( @@ -611,7 +434,8 @@ def read_gbq( col_order: Iterable[str] = ..., dry_run: Literal[True] = ..., allow_large_results: Optional[bool] = ..., - ) -> pandas.Series: ... + ) -> pandas.Series: + ... def read_gbq( self, @@ -681,29 +505,27 @@ def _read_gbq_colab( self, query: str, *, - callback: Optional[Callable[[bigframes.core.events.EventEnvelope], None]] = ..., pyformat_args: Optional[Dict[str, Any]] = None, dry_run: Literal[False] = ..., - ) -> dataframe.DataFrame: ... + ) -> dataframe.DataFrame: + ... @overload def _read_gbq_colab( self, query: str, *, - callback: Optional[Callable[[bigframes.core.events.EventEnvelope], None]] = ..., pyformat_args: Optional[Dict[str, Any]] = None, dry_run: Literal[True] = ..., - ) -> pandas.Series: ... + ) -> pandas.Series: + ... @log_adapter.log_name_override("read_gbq_colab") def _read_gbq_colab( self, query: str, + # TODO: Add a callback parameter that takes some kind of Event object. *, - callback: Optional[ - Callable[[bigframes.core.events.EventEnvelope], None] - ] = None, pyformat_args: Optional[Dict[str, Any]] = None, dry_run: bool = False, ) -> Union[dataframe.DataFrame, pandas.Series]: @@ -716,8 +538,6 @@ def _read_gbq_colab( query (str): A SQL query string to execute. Results (if any) are turned into a DataFrame. - callback (Optional[Callable[[bigframes.core.events.EventEnvelope], None]]): - Callback to receive query execution events. pyformat_args (dict): A dictionary of potential variables to replace in ``query``. Note: strings are _not_ escaped. Use query parameters for these, @@ -737,19 +557,13 @@ def _read_gbq_colab( dry_run=dry_run, ) - def _run_query(): - return self._loader.read_gbq_query( - query=query, - index_col=bigframes.enums.DefaultIndexKind.NULL, - force_total_order=False, - dry_run=typing.cast(Union[Literal[False], Literal[True]], dry_run), - allow_large_results=allow_large_results, - ) - - if callback is not None: - with self._publisher.subscribe(callback): - return _run_query() - return _run_query() + return self._loader.read_gbq_query( + query=query, + index_col=bigframes.enums.DefaultIndexKind.NULL, + force_total_order=False, + dry_run=typing.cast(Union[Literal[False], Literal[True]], dry_run), + allow_large_results=allow_large_results, + ) @overload def read_gbq_query( # type: ignore[overload-overlap] @@ -765,7 +579,8 @@ def read_gbq_query( # type: ignore[overload-overlap] filters: third_party_pandas_gbq.FiltersType = ..., dry_run: Literal[False] = ..., allow_large_results: Optional[bool] = ..., - ) -> dataframe.DataFrame: ... + ) -> dataframe.DataFrame: + ... @overload def read_gbq_query( @@ -781,7 +596,8 @@ def read_gbq_query( filters: third_party_pandas_gbq.FiltersType = ..., dry_run: Literal[True] = ..., allow_large_results: Optional[bool] = ..., - ) -> pandas.Series: ... + ) -> pandas.Series: + ... def read_gbq_query( self, @@ -833,11 +649,9 @@ def read_gbq_query( ... WHERE year = 2016 ... GROUP BY pitcherFirstName, pitcherLastName ... ''', index_col="rowindex") - >>> print("START_OF_OUTPUT"); df.head(2) # doctest: +ELLIPSIS,+NORMALIZE_WHITESPACE - START_OF_OUTPUT - ... + >>> df.head(2) pitcherFirstName pitcherLastName averagePitchSpeed - ... + rowindex 1 Albertin Chapman 96.514113 2 Zachary Britton 94.591039 @@ -928,7 +742,8 @@ def read_gbq_table( # type: ignore[overload-overlap] use_cache: bool = ..., col_order: Iterable[str] = ..., dry_run: Literal[False] = ..., - ) -> dataframe.DataFrame: ... + ) -> dataframe.DataFrame: + ... @overload def read_gbq_table( @@ -942,7 +757,8 @@ def read_gbq_table( use_cache: bool = ..., col_order: Iterable[str] = ..., dry_run: Literal[True] = ..., - ) -> pandas.Series: ... + ) -> pandas.Series: + ... def read_gbq_table( self, @@ -1093,7 +909,8 @@ def read_pandas( pandas_dataframe: pandas.Index, *, write_engine: constants.WriteEngineType = "default", - ) -> bigframes.core.indexes.Index: ... + ) -> bigframes.core.indexes.Index: + ... @typing.overload def read_pandas( @@ -1101,7 +918,8 @@ def read_pandas( pandas_dataframe: pandas.Series, *, write_engine: constants.WriteEngineType = "default", - ) -> bigframes.series.Series: ... + ) -> bigframes.series.Series: + ... @typing.overload def read_pandas( @@ -1109,7 +927,8 @@ def read_pandas( pandas_dataframe: pandas.DataFrame, *, write_engine: constants.WriteEngineType = "default", - ) -> dataframe.DataFrame: ... + ) -> dataframe.DataFrame: + ... def read_pandas( self, @@ -1519,7 +1338,7 @@ def read_parquet( "The provided path contains a wildcard character (*), which is not " "supported by the current engine. To read files from wildcard paths, " "please use the 'bigquery' engine by setting `engine='bigquery'` in " - "the function call." + "your configuration." ) read_parquet_kwargs: Dict[str, Any] = {} @@ -1535,87 +1354,6 @@ def read_parquet( ) return self._read_pandas(pandas_obj, write_engine=write_engine) - def read_orc( - self, - path: str | IO["bytes"], - *, - engine: str = "auto", - write_engine: constants.WriteEngineType = "default", - ) -> dataframe.DataFrame: - """Load an ORC file to a BigQuery DataFrames DataFrame. - - Args: - path (str or IO): - The path or buffer to the ORC file. Can be a local path or Google Cloud Storage URI. - engine (str, default "auto"): - The engine used to read the file. Supported values: `auto`, `bigquery`, `pyarrow`. - write_engine (str, default "default"): - The write engine used to persist the data to BigQuery if needed. - - Returns: - bigframes.pandas.DataFrame: - A new DataFrame representing the data from the ORC file. - """ - bigframes.session.validation.validate_engine_compatibility( - engine=engine, - write_engine=write_engine, - ) - if engine == "bigquery": - job_config = bigquery.LoadJobConfig() - job_config.source_format = bigquery.SourceFormat.ORC - job_config.labels = {"bigframes-api": "read_orc"} - table_id = self._loader.load_file(path, job_config=job_config) - return self._loader.read_gbq_table(table_id) - elif engine in ("auto", "pyarrow"): - if isinstance(path, str) and "*" in path: - raise ValueError( - "The provided path contains a wildcard character (*), which is not " - "supported by the current engine. To read files from wildcard paths, " - "please use the 'bigquery' engine by setting `engine='bigquery'` in " - "your configuration." - ) - - read_orc_kwargs: Dict[str, Any] = {} - if not pandas.__version__.startswith("1."): - read_orc_kwargs["dtype_backend"] = "pyarrow" - - pandas_obj = pandas.read_orc(path, **read_orc_kwargs) - return self._read_pandas(pandas_obj, write_engine=write_engine) - else: - raise ValueError( - f"Unsupported engine: {repr(engine)}. Supported values: 'auto', 'bigquery', 'pyarrow'." - ) - - def read_avro( - self, - path: str | IO["bytes"], - *, - engine: str = "auto", - ) -> dataframe.DataFrame: - """Load an Avro file to a BigQuery DataFrames DataFrame. - - Args: - path (str or IO): - The path or buffer to the Avro file. Can be a local path or Google Cloud Storage URI. - engine (str, default "auto"): - The engine used to read the file. Only `bigquery` is supported for Avro. - - Returns: - bigframes.pandas.DataFrame: - A new DataFrame representing the data from the Avro file. - """ - if engine not in ("auto", "bigquery"): - raise ValueError( - f"Unsupported engine: {repr(engine)}. Supported values: 'auto', 'bigquery'." - ) - - job_config = bigquery.LoadJobConfig() - job_config.use_avro_logical_types = True - job_config.source_format = bigquery.SourceFormat.AVRO - job_config.labels = {"bigframes-api": "read_avro"} - table_id = self._loader.load_file(path, job_config=job_config) - return self._loader.read_gbq_table(table_id) - def read_json( self, path_or_buf: str | IO["bytes"], @@ -1635,6 +1373,7 @@ def read_json( write_engine=write_engine, ) if engine == "bigquery": + if dtype is not None: raise NotImplementedError( "BigQuery engine does not support the dtype arguments." @@ -1754,6 +1493,13 @@ def deploy_remote_function( """ return self._function_session.deploy_remote_function( func, + # Session-provided arguments. + session=self, + bigquery_client=self._clients_provider.bqclient, + bigquery_connection_client=self._clients_provider.bqconnectionclient, + cloud_functions_client=self._clients_provider.cloudfunctionsclient, + resource_manager_client=self._clients_provider.resourcemanagerclient, + # User-provided arguments. **kwargs, ) @@ -1781,8 +1527,7 @@ def remote_function( cloud_function_vpc_connector_egress_settings: Optional[ Literal["all", "private-ranges-only", "unspecified"] ] = None, - cloud_function_memory_mib: Optional[int] = None, - cloud_function_cpus: Optional[float] = None, + cloud_function_memory_mib: Optional[int] = 1024, cloud_function_ingress_settings: Literal[ "all", "internal-only", "internal-and-gclb" ] = "internal-only", @@ -1963,10 +1708,6 @@ def remote_function( default memory of cloud functions be allocated, pass `None`. See for more details https://cloud.google.com/functions/docs/configuring/memory. - cloud_function_cpus (float, Optional): - The number of cpus to allocate for the cloud - function (2nd gen) created. - https://docs.cloud.google.com/run/docs/configuring/services/cpu. cloud_function_ingress_settings (str, Optional): Ingress settings controls dictating what traffic can reach the function. Options are: `all`, `internal-only`, or `internal-and-gclb`. @@ -1994,6 +1735,12 @@ def remote_function( `bigframes_remote_function` - The bigquery remote function capable of calling into `bigframes_cloud_function`. """ return self._function_session.remote_function( + # Session-provided arguments. + session=self, + bigquery_client=self._clients_provider.bqclient, + bigquery_connection_client=self._clients_provider.bqconnectionclient, + cloud_functions_client=self._clients_provider.cloudfunctionsclient, + resource_manager_client=self._clients_provider.resourcemanagerclient, # User-provided arguments. input_types=input_types, output_type=output_type, @@ -2011,7 +1758,6 @@ def remote_function( cloud_function_vpc_connector=cloud_function_vpc_connector, cloud_function_vpc_connector_egress_settings=cloud_function_vpc_connector_egress_settings, cloud_function_memory_mib=cloud_function_memory_mib, - cloud_function_cpus=cloud_function_cpus, cloud_function_ingress_settings=cloud_function_ingress_settings, cloud_build_service_account=cloud_build_service_account, ) @@ -2040,6 +1786,10 @@ def deploy_udf( """ return self._function_session.deploy_udf( func, + # Session-provided arguments. + session=self, + bigquery_client=self._clients_provider.bqclient, + # User-provided arguments. **kwargs, ) @@ -2048,9 +1798,9 @@ def udf( *, input_types: Union[None, type, Sequence[type]] = None, output_type: Optional[type] = None, - dataset: Optional[str] = None, + dataset: str, bigquery_connection: Optional[str] = None, - name: Optional[str] = None, + name: str, packages: Optional[Sequence[str]] = None, max_batching_rows: Optional[int] = None, container_cpu: Optional[float] = None, @@ -2142,7 +1892,7 @@ def udf( be specified. The supported output types are `bool`, `bytes`, `float`, `int`, `str`, `list[bool]`, `list[float]`, `list[int]` and `list[str]`. - dataset (str, Optional): + dataset (str): Dataset in which to create a BigQuery managed function. It should be in `.` or `` format. @@ -2200,6 +1950,10 @@ def udf( deployed for the user defined code. """ return self._function_session.udf( + # Session-provided arguments. + session=self, + bigquery_client=self._clients_provider.bqclient, + # User-provided arguments. input_types=input_types, output_type=output_type, dataset=dataset, @@ -2400,7 +2154,7 @@ def _start_query_ml_ddl( # so we must reset any encryption set in the job config # https://cloud.google.com/bigquery/docs/customer-managed-encryption#encrypt-model job_config.destination_encryption_configuration = None - iterator, query_job = bf_io_bigquery.start_query_with_job( + iterator, query_job = bf_io_bigquery.start_query_with_client( self.bqclient, sql, job_config=job_config, @@ -2408,25 +2162,13 @@ def _start_query_ml_ddl( location=None, project=None, timeout=None, + query_with_job=True, job_retry=third_party_gcb_retry.DEFAULT_ML_JOB_RETRY, publisher=self._publisher, session=self, ) return iterator, query_job - def _from_glob_path( - self, path: str, *, connection: Optional[str] = None, name: Optional[str] = None - ) -> dataframe.DataFrame: - """Create a BigFrames DataFrame that contains a BigFrames ObjectRef column from a global wildcard path.""" - import bigframes.bigquery as bq - - connection = self._create_bq_connection(connection=connection) - table = self._create_object_table(path, connection) - s = bq.obj.make_ref( - self._loader.read_gbq_table(table)["uri"], authorizer=connection - ) - return s.rename(name).to_frame() - def _create_object_table(self, path: str, connection: str) -> str: """Create a random id Object Table from the input path and connection.""" table = str(self._anon_dataset_manager.generate_unique_resource_id()) @@ -2442,7 +2184,7 @@ def _create_object_table(self, path: str, connection: str) -> str: uris = ['{path}']); """ ) - bf_io_bigquery.start_query_with_job( + bf_io_bigquery.start_query_with_client( self.bqclient, sql, job_config=bigquery.QueryJobConfig(), @@ -2450,6 +2192,7 @@ def _create_object_table(self, path: str, connection: str) -> str: location=None, project=None, timeout=None, + query_with_job=True, publisher=self._publisher, session=self, ) @@ -2468,6 +2211,40 @@ def _create_temp_table( schema=schema, cluster_cols=cluster_cols ) + def from_glob_path( + self, path: str, *, connection: Optional[str] = None, name: Optional[str] = None + ) -> dataframe.DataFrame: + r"""Create a BigFrames DataFrame that contains a BigFrames Blob column from a global wildcard path. + This operation creates a temporary BQ Object Table under the hood and requires bigquery.connections.delegate permission or BigQuery Connection Admin role. + If you have an existing BQ Object Table, use read_gbq_object_table(). + + .. note:: + BigFrames Blob is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). + + Args: + path (str): + The wildcard global path, such as "gs:////\*". + connection (str or None, default None): + Connection to connect with remote service. str of the format ... + If None, use default connection in session context. BigQuery DataFrame will try to create the connection and attach + permission if the connection isn't fully set up. + name (str): + The column name of the Blob column. + Returns: + bigframes.pandas.DataFrame: + Result BigFrames DataFrame. + """ + # TODO(garrettwu): switch to pseudocolumn when b/374988109 is done. + connection = self._create_bq_connection(connection=connection) + + table = self._create_object_table(path, connection) + + s = self._loader.read_gbq_table(table)["uri"].str.to_blob(connection) + return s.rename(name).to_frame() + def _create_bq_connection( self, *, @@ -2475,9 +2252,8 @@ def _create_bq_connection( iam_role: Optional[str] = None, ) -> str: """Create the connection with the session settings and try to attach iam role to the connection SA. - If any of project, location or connection isn't specified, use the session defaults. Returns fully-qualified connection name. - """ - connection = self.bq_connection if not connection else connection + If any of project, location or connection isn't specified, use the session defaults. Returns fully-qualified connection name.""" + connection = self._bq_connection if not connection else connection connection = bigframes.clients.get_canonical_bq_connection_id( connection_id=connection, default_project=self._project, @@ -2495,6 +2271,33 @@ def _create_bq_connection( return connection + def read_gbq_object_table( + self, object_table: str, *, name: Optional[str] = None + ) -> dataframe.DataFrame: + """Read an existing object table to create a BigFrames Blob DataFrame. Use the connection of the object table for the connection of the blob. + This function dosen't retrieve the object table data. If you want to read the data, use read_gbq() instead. + + .. note:: + BigFrames Blob is subject to the "Pre-GA Offerings Terms" in the General Service Terms section of the + Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available "as is" + and might have limited support. For more information, see the launch stage descriptions + (https://cloud.google.com/products#product-launch-stages). + + Args: + object_table (str): name of the object table of form ... + name (str or None): the returned blob column name. + + Returns: + bigframes.pandas.DataFrame: + Result BigFrames DataFrame. + """ + # TODO(garrettwu): switch to pseudocolumn when b/374988109 is done. + table = self.bqclient.get_table(object_table) + connection = table._properties["externalDataConfiguration"]["connectionId"] + + s = self._loader.read_gbq_table(object_table)["uri"].str.to_blob(connection) + return s.rename(name).to_frame() + # ========================================================================= # bigframes.pandas attributes # diff --git a/bigframes/session/_io/bigquery/__init__.py b/bigframes/session/_io/bigquery/__init__.py index 3d60bcc8074..98b5f194c74 100644 --- a/bigframes/session/_io/bigquery/__init__.py +++ b/bigframes/session/_io/bigquery/__init__.py @@ -22,14 +22,7 @@ import textwrap import types import typing -from typing import ( - Dict, - Iterable, - Mapping, - Optional, - Tuple, - Union, -) +from typing import Dict, Iterable, Literal, Mapping, Optional, overload, Tuple, Union import bigframes_vendored.google_cloud_bigquery.retry as third_party_gcb_retry import bigframes_vendored.pandas.io.gbq as third_party_pandas_gbq @@ -39,16 +32,13 @@ import google.cloud.bigquery._job_helpers import google.cloud.bigquery.table +import bigframes.core.compile.googlesql as googlesql import bigframes.core.events +from bigframes.core.logging import log_adapter import bigframes.core.sql import bigframes.session.metrics -from bigframes.core.compile.sqlglot import sql as sg_sql -from bigframes.core.logging import log_adapter -CHECK_DRIVE_PERMISSIONS = ( - "\nCheck https://cloud.google.com/bigquery/docs/" - "query-drive-data#Google_Drive_permissions." -) +CHECK_DRIVE_PERMISSIONS = "\nCheck https://cloud.google.com/bigquery/docs/query-drive-data#Google_Drive_permissions." IO_ORDERING_ID = "bqdf_row_nums" @@ -64,11 +54,13 @@ def create_job_configs_labels( ) -> Dict[str, str]: if job_configs_labels is None: job_configs_labels = {} - else: - job_configs_labels = dict(job_configs_labels) + + # If the user has labels they wish to set, make sure we set those first so + # they are preserved. + for key, value in bigframes.options.compute.extra_query_labels.items(): + job_configs_labels[key] = value if api_methods and "bigframes-api" not in job_configs_labels: - api_methods = list(api_methods) job_configs_labels["bigframes-api"] = api_methods[0] del api_methods[0] @@ -93,10 +85,7 @@ def create_job_configs_labels( def create_export_data_statement( - table_id: str, - uri: str, - format: str, - export_options: Dict[str, Union[bool, str]], + table_id: str, uri: str, format: str, export_options: Dict[str, Union[bool, str]] ) -> str: all_options: Dict[str, Union[bool, str]] = { "uri": uri, @@ -150,10 +139,10 @@ def create_temp_table( if cluster_columns: destination.clustering_fields = cluster_columns if kms_key: - enc_config = bigquery.EncryptionConfiguration(kms_key_name=kms_key) - destination.encryption_configuration = enc_config - # Ok if already exists, since this will only happen from retries - # internal to this method + destination.encryption_configuration = bigquery.EncryptionConfiguration( + kms_key_name=kms_key + ) + # Ok if already exists, since this will only happen from retries internal to this method # as the requested table id has a random UUID4 component. bqclient.create_table(destination, exists_ok=True) return f"{table_ref.project}.{table_ref.dataset_id}.{table_ref.table_id}" @@ -176,8 +165,7 @@ def create_temp_view( destination.expires = expiration destination.view_query = sql - # Ok if already exists, since this will only happen from retries - # internal to this method + # Ok if already exists, since this will only happen from retries internal to this method # as the requested table id has a random UUID4 component. bqclient.create_table(destination, exists_ok=True) return f"{table_ref.project}.{table_ref.dataset_id}.{table_ref.table_id}" @@ -211,10 +199,7 @@ def bq_field_to_type_sql(field: bigquery.SchemaField): if field.mode == "REPEATED": nested_type = bq_field_to_type_sql( bigquery.SchemaField( - field.name, - field.field_type, - mode="NULLABLE", - fields=field.fields, + field.name, field.field_type, mode="NULLABLE", fields=field.fields ) ) return f"ARRAY<{nested_type}>" @@ -245,15 +230,10 @@ def format_option(key: str, value: Union[bool, str]) -> str: return f"{key}={repr(value)}" -def add_and_trim_labels( - job_config, - session=None, - extra_query_labels: Optional[Mapping[str, str]] = None, -): +def add_and_trim_labels(job_config, session=None): """ - Add additional labels to the job configuration and trim the total - number of labels to ensure they do not exceed MAX_LABELS_COUNT labels - per job. + Add additional labels to the job configuration and trim the total number of labels + to ensure they do not exceed MAX_LABELS_COUNT labels per job. """ api_methods = log_adapter.get_and_reset_api_methods( dry_run=job_config.dry_run, session=session @@ -264,96 +244,95 @@ def add_and_trim_labels( ) -def create_bq_event_callback(publisher, cell_execution_count=None): - event_map = { - google.cloud.bigquery._job_helpers.QueryFinishedEvent: ( - bigframes.core.events.BigQueryFinishedEvent - ), - google.cloud.bigquery._job_helpers.QueryReceivedEvent: ( - bigframes.core.events.BigQueryReceivedEvent - ), - google.cloud.bigquery._job_helpers.QueryRetryEvent: ( - bigframes.core.events.BigQueryRetryEvent - ), - google.cloud.bigquery._job_helpers.QuerySentEvent: ( - bigframes.core.events.BigQuerySentEvent - ), - } - +def create_bq_event_callback(publisher): def publish_bq_event(event): - bf_event = bigframes.core.events.BigQueryUnknownEvent(event) - for bq_type, bf_type in event_map.items(): - if isinstance(event, bq_type): - bf_event = bf_type.from_bqclient(event) # type: ignore - break - envelope = bigframes.core.events.EventEnvelope( - event=bf_event, - progress_bar=bigframes.core.events._DEFAULT, - cell_execution_count=cell_execution_count, - ) - publisher.publish(envelope) + if isinstance(event, google.cloud.bigquery._job_helpers.QueryFinishedEvent): + bf_event = bigframes.core.events.BigQueryFinishedEvent.from_bqclient(event) + elif isinstance(event, google.cloud.bigquery._job_helpers.QueryReceivedEvent): + bf_event = bigframes.core.events.BigQueryReceivedEvent.from_bqclient(event) + elif isinstance(event, google.cloud.bigquery._job_helpers.QueryRetryEvent): + bf_event = bigframes.core.events.BigQueryRetryEvent.from_bqclient(event) + elif isinstance(event, google.cloud.bigquery._job_helpers.QuerySentEvent): + bf_event = bigframes.core.events.BigQuerySentEvent.from_bqclient(event) + else: + bf_event = bigframes.core.events.BigQueryUnknownEvent(event) + + publisher.publish(bf_event) return publish_bq_event -def start_query_with_job( +@overload +def start_query_with_client( bq_client: bigquery.Client, sql: str, *, job_config: bigquery.QueryJobConfig, - location: Optional[str] = None, - project: Optional[str] = None, - timeout: Optional[float] = None, - metrics: Optional[bigframes.session.metrics.ExecutionMetrics] = None, - # TODO(tswast): We can stop providing our own default once we use a - # google-cloud-bigquery version with - # https://github.com/googleapis/python-bigquery/pull/2256 merged, likely - # version 3.36.0 or later. - job_retry: google.api_core.retry.Retry = (third_party_gcb_retry.DEFAULT_JOB_RETRY), # noqa: E501 + location: Optional[str], + project: Optional[str], + timeout: Optional[float], + metrics: Optional[bigframes.session.metrics.ExecutionMetrics], + query_with_job: Literal[True], publisher: bigframes.core.events.Publisher, session=None, - cell_execution_count: Optional[int] = None, ) -> Tuple[google.cloud.bigquery.table.RowIterator, bigquery.QueryJob]: - """ - Starts query job and waits for results. - """ - if cell_execution_count is None: - from bigframes.core.utils import get_ipython_execution_count + ... - cell_execution_count = get_ipython_execution_count() - # Note: Ensure no additional labels are added to job_config after this - # point, as `add_and_trim_labels` ensures the label count does not - # exceed MAX_LABELS_COUNT. - add_and_trim_labels(job_config, session=session) +@overload +def start_query_with_client( + bq_client: bigquery.Client, + sql: str, + *, + job_config: bigquery.QueryJobConfig, + location: Optional[str], + project: Optional[str], + timeout: Optional[float], + metrics: Optional[bigframes.session.metrics.ExecutionMetrics], + query_with_job: Literal[False], + publisher: bigframes.core.events.Publisher, + session=None, +) -> Tuple[google.cloud.bigquery.table.RowIterator, Optional[bigquery.QueryJob]]: + ... - try: - query_job = bq_client.query( - sql, - job_config=job_config, - location=location, - project=project, - timeout=timeout, - job_retry=job_retry, - ) - except google.api_core.exceptions.Forbidden as ex: - if "Drive credentials" in ex.message: - ex.message += CHECK_DRIVE_PERMISSIONS - raise - results_iterator = query_job.result() - _publish_events( - query_job=query_job, - total_rows=results_iterator.total_rows, - sql=sql, - publisher=publisher, - metrics=metrics, - cell_execution_count=cell_execution_count, - ) - return results_iterator, query_job +@overload +def start_query_with_client( + bq_client: bigquery.Client, + sql: str, + *, + job_config: bigquery.QueryJobConfig, + location: Optional[str], + project: Optional[str], + timeout: Optional[float], + metrics: Optional[bigframes.session.metrics.ExecutionMetrics], + query_with_job: Literal[True], + job_retry: google.api_core.retry.Retry, + publisher: bigframes.core.events.Publisher, + session=None, +) -> Tuple[google.cloud.bigquery.table.RowIterator, bigquery.QueryJob]: + ... -def start_query_job_optional( +@overload +def start_query_with_client( + bq_client: bigquery.Client, + sql: str, + *, + job_config: bigquery.QueryJobConfig, + location: Optional[str], + project: Optional[str], + timeout: Optional[float], + metrics: Optional[bigframes.session.metrics.ExecutionMetrics], + query_with_job: Literal[False], + job_retry: google.api_core.retry.Retry, + publisher: bigframes.core.events.Publisher, + session=None, +) -> Tuple[google.cloud.bigquery.table.RowIterator, Optional[bigquery.QueryJob]]: + ... + + +def start_query_with_client( bq_client: bigquery.Client, sql: str, *, @@ -362,103 +341,85 @@ def start_query_job_optional( project: Optional[str] = None, timeout: Optional[float] = None, metrics: Optional[bigframes.session.metrics.ExecutionMetrics] = None, + query_with_job: bool = True, # TODO(tswast): We can stop providing our own default once we use a # google-cloud-bigquery version with # https://github.com/googleapis/python-bigquery/pull/2256 merged, likely # version 3.36.0 or later. - job_retry: google.api_core.retry.Retry = (third_party_gcb_retry.DEFAULT_JOB_RETRY), # noqa: E501 - publisher: Optional[bigframes.core.events.Publisher] = None, + job_retry: google.api_core.retry.Retry = third_party_gcb_retry.DEFAULT_JOB_RETRY, + publisher: bigframes.core.events.Publisher, session=None, - cell_execution_count: Optional[int] = None, -) -> google.cloud.bigquery.table.RowIterator: +) -> Tuple[google.cloud.bigquery.table.RowIterator, Optional[bigquery.QueryJob]]: """ - Run a bigquery query, with job optional. - - See: - https://docs.cloud.google.com/bigquery/docs/running-queries#optional-job-creation + Starts query job and waits for results. """ - if cell_execution_count is None: - from bigframes.core.utils import get_ipython_execution_count - - cell_execution_count = get_ipython_execution_count() - + # Note: Ensure no additional labels are added to job_config after this + # point, as `add_and_trim_labels` ensures the label count does not + # exceed MAX_LABELS_COUNT. add_and_trim_labels(job_config, session=session) + try: - results_iterator = bq_client._query_and_wait_bigframes( + if not query_with_job: + results_iterator = bq_client._query_and_wait_bigframes( + sql, + job_config=job_config, + location=location, + project=project, + api_timeout=timeout, + job_retry=job_retry, + callback=create_bq_event_callback(publisher), + ) + if metrics is not None: + metrics.count_job_stats(row_iterator=results_iterator) + return results_iterator, None + + query_job = bq_client.query( sql, job_config=job_config, location=location, project=project, - api_timeout=timeout, + timeout=timeout, job_retry=job_retry, - callback=create_bq_event_callback( - publisher, cell_execution_count=cell_execution_count - ) - if publisher - else lambda _: None, ) - if metrics is not None: - metrics.count_job_stats( - row_iterator=results_iterator, cell_execution_count=cell_execution_count - ) - return results_iterator except google.api_core.exceptions.Forbidden as ex: if "Drive credentials" in ex.message: ex.message += CHECK_DRIVE_PERMISSIONS raise - -def _publish_events( - query_job: bigquery.QueryJob, - sql: str, - total_rows: Optional[int], - publisher: bigframes.core.events.Publisher, - metrics: Optional[bigframes.session.metrics.ExecutionMetrics] = None, - cell_execution_count: Optional[int] = None, -): if not query_job.configuration.dry_run: publisher.publish( - bigframes.core.events.EventEnvelope( - event=bigframes.core.events.BigQuerySentEvent( - sql, - billing_project=query_job.project, - location=query_job.location, - job_id=query_job.job_id, - request_id=None, - ), - cell_execution_count=cell_execution_count, + bigframes.core.events.BigQuerySentEvent( + sql, + billing_project=query_job.project, + location=query_job.location, + job_id=query_job.job_id, + request_id=None, ) ) + results_iterator = query_job.result() if not query_job.configuration.dry_run: publisher.publish( - bigframes.core.events.EventEnvelope( - event=bigframes.core.events.BigQueryFinishedEvent( - billing_project=query_job.project, - location=query_job.location, - query_id=query_job.query_id, - job_id=query_job.job_id, - destination=query_job.destination, - total_rows=total_rows, - total_bytes_processed=query_job.total_bytes_processed, - slot_millis=query_job.slot_millis, - created=query_job.created, - started=query_job.started, - ended=query_job.ended, - ), - cell_execution_count=cell_execution_count, + bigframes.core.events.BigQueryFinishedEvent( + billing_project=query_job.project, + location=query_job.location, + job_id=query_job.job_id, + destination=query_job.destination, + total_rows=results_iterator.total_rows, + total_bytes_processed=query_job.total_bytes_processed, + slot_millis=query_job.slot_millis, + created=query_job.created, + started=query_job.started, + ended=query_job.ended, ) ) if metrics is not None: - metrics.count_job_stats( - query_job=query_job, cell_execution_count=cell_execution_count - ) + metrics.count_job_stats(query_job=query_job) + return results_iterator, query_job def delete_tables_matching_session_id( - client: bigquery.Client, - dataset: bigquery.DatasetReference, - session_id: str, + client: bigquery.Client, dataset: bigquery.DatasetReference, session_id: str ) -> None: """Searches within the dataset for tables conforming to the expected session_id form, and instructs bigquery to delete them. @@ -512,12 +473,11 @@ def create_bq_dataset_reference( The project id of the project to create the dataset in. Returns: - bigquery.DatasetReference: The constructed reference to the - anonymous dataset. + bigquery.DatasetReference: The constructed reference to the anonymous dataset. """ job_config = google.cloud.bigquery.QueryJobConfig() - _, query_job = start_query_with_job( + _, query_job = start_query_with_client( bq_client, "SELECT 1", location=location, @@ -525,6 +485,7 @@ def create_bq_dataset_reference( project=project, timeout=None, metrics=None, + query_with_job=True, publisher=publisher, ) @@ -546,8 +507,7 @@ def is_query(query_or_table: str) -> bool: def is_table_with_wildcard_suffix(query_or_table: str) -> bool: - """Determine if `query_or_table` is a table and contains a wildcard - suffix.""" + """Determine if `query_or_table` is a table and contains a wildcard suffix.""" return not is_query(query_or_table) and query_or_table.endswith("*") @@ -559,40 +519,33 @@ def to_query( time_travel_timestamp: Optional[datetime.datetime] = None, ) -> str: """Compile query_or_table with conditions(filters, wildcards) to query.""" - if is_query(query_or_table): - from_item = f"({query_or_table})" - else: - # Table ID can have 1, 2, 3, or 4 parts. Quoting all parts to be safe. - # See: - # https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#identifiers - parts = query_or_table.split(".") - from_item = ".".join(f"`{part}`" for part in parts) + sub_query = ( + f"({query_or_table})" if is_query(query_or_table) else f"`{query_or_table}`" + ) # TODO(b/338111344): Generate an index based on DefaultIndexKind if we # don't have index columns specified. if columns: # We only reduce the selection if columns is set, but we always # want to make sure index_cols is also included. - select_clause = "SELECT " + ", ".join( - f"`_bf_source`.`{column}`" for column in columns - ) + select_clause = "SELECT " + ", ".join(f"`{column}`" for column in columns) else: select_clause = "SELECT *" time_travel_clause = "" if time_travel_timestamp is not None: - time_travel_literal = sg_sql.to_sql(sg_sql.literal(time_travel_timestamp)) # noqa: E501 + time_travel_literal = bigframes.core.sql.simple_literal(time_travel_timestamp) time_travel_clause = f" FOR SYSTEM_TIME AS OF {time_travel_literal}" limit_clause = "" if max_results is not None: - limit_clause = f" LIMIT {sg_sql.to_sql(sg_sql.literal(max_results))}" + limit_clause = f" LIMIT {bigframes.core.sql.simple_literal(max_results)}" where_clause = f" WHERE {sql_predicate}" if sql_predicate else "" return ( f"{select_clause} " - f"FROM {from_item} AS _bf_source" + f"FROM {sub_query}" f"{time_travel_clause}{where_clause}{limit_clause}" ) @@ -614,11 +567,10 @@ def compile_filters(filters: third_party_pandas_gbq.FiltersType) -> str: "!=": "!=", } - # If single layer filter, add another pseudo layer. So the single - # layer represents "and" logic. + # If single layer filter, add another pseudo layer. So the single layer represents "and" logic. filters_list: list = list(filters) if isinstance(filters_list[0], tuple) and ( - len(filters_list[0]) == 0 or not isinstance(list(filters_list[0])[0], tuple) # noqa: E501 + len(filters_list[0]) == 0 or not isinstance(list(filters_list[0])[0], tuple) ): filter_items = [filters_list] else: @@ -632,16 +584,14 @@ def compile_filters(filters: third_party_pandas_gbq.FiltersType) -> str: for filter_item in group: if not isinstance(filter_item, tuple) or (len(filter_item) != 3): raise ValueError( - f"Elements of filters must be tuples of length 3, " - f"but got {repr(filter_item)}.", + f"Elements of filters must be tuples of length 3, but got {repr(filter_item)}.", ) column, operator, value = filter_item if not isinstance(column, str): raise ValueError( - f"Column name should be a string, but received " - f"'{column}' of type {type(column).__name__}." + f"Column name should be a string, but received '{column}' of type {type(column).__name__}." ) if operator not in valid_operators: @@ -649,11 +599,11 @@ def compile_filters(filters: third_party_pandas_gbq.FiltersType) -> str: operator_str = valid_operators[operator] - column_ref = sg_sql.to_sql(sg_sql.identifier(column)) + column_ref = googlesql.identifier(column) if operator_str in ["IN", "NOT IN"]: value_literal = bigframes.core.sql.multi_literal(*value) else: - value_literal = sg_sql.to_sql(sg_sql.literal(value)) + value_literal = bigframes.core.sql.simple_literal(value) expression = bigframes.core.sql.infix_op( operator_str, column_ref, value_literal ) diff --git a/bigframes/session/_io/bigquery/read_gbq_query.py b/bigframes/session/_io/bigquery/read_gbq_query.py index cd6368974b6..b650266a0de 100644 --- a/bigframes/session/_io/bigquery/read_gbq_query.py +++ b/bigframes/session/_io/bigquery/read_gbq_query.py @@ -16,20 +16,20 @@ from __future__ import annotations -from typing import Iterable, Optional, Tuple, cast +from typing import cast, Iterable, Optional, Tuple +from google.cloud import bigquery import google.cloud.bigquery.table import pandas -from google.cloud import bigquery +from bigframes import dataframe +from bigframes.core import local_data, pyarrow_utils import bigframes.core as core import bigframes.core.blocks as blocks import bigframes.core.guid import bigframes.core.schema as schemata import bigframes.enums import bigframes.session -from bigframes import dataframe -from bigframes.core import local_data, pyarrow_utils def should_return_query_results(query_job: bigquery.QueryJob) -> bool: diff --git a/bigframes/session/_io/bigquery/read_gbq_table.py b/bigframes/session/_io/bigquery/read_gbq_table.py index faaf0f01912..e12fe502c0f 100644 --- a/bigframes/session/_io/bigquery/read_gbq_table.py +++ b/bigframes/session/_io/bigquery/read_gbq_table.py @@ -20,18 +20,18 @@ import datetime import typing +from typing import Dict, Iterable, List, Optional, Sequence, Tuple import warnings -from typing import Dict, Iterable, Optional, Sequence, Tuple, Union import bigframes_vendored.constants as constants import google.api_core.exceptions import google.cloud.bigquery as bigquery +import google.cloud.bigquery.table import bigframes.core import bigframes.core.events import bigframes.exceptions as bfe import bigframes.session._io.bigquery -from bigframes.core import bq_data # Avoid circular imports. if typing.TYPE_CHECKING: @@ -98,6 +98,81 @@ def get_information_schema_metadata( return table +def get_table_metadata( + bqclient: bigquery.Client, + *, + table_id: str, + default_project: Optional[str], + bq_time: datetime.datetime, + cache: Dict[str, Tuple[datetime.datetime, bigquery.Table]], + use_cache: bool = True, + publisher: bigframes.core.events.Publisher, +) -> Tuple[datetime.datetime, google.cloud.bigquery.table.Table]: + """Get the table metadata, either from cache or via REST API.""" + + cached_table = cache.get(table_id) + if use_cache and cached_table is not None: + snapshot_timestamp, table = cached_table + + if is_time_travel_eligible( + bqclient=bqclient, + table=table, + columns=None, + snapshot_time=snapshot_timestamp, + filter_str=None, + # Don't warn, because that will already have been taken care of. + should_warn=False, + should_dry_run=False, + publisher=publisher, + ): + # This warning should only happen if the cached snapshot_time will + # have any effect on bigframes (b/437090788). For example, with + # cached query results, such as after re-running a query, time + # travel won't be applied and thus this check is irrelevent. + # + # In other cases, such as an explicit read_gbq_table(), Cache hit + # could be unexpected. See internal issue 329545805. Raise a + # warning with more information about how to avoid the problems + # with the cache. + msg = bfe.format_message( + f"Reading cached table from {snapshot_timestamp} to avoid " + "incompatibilies with previous reads of this table. To read " + "the latest version, set `use_cache=False` or close the " + "current session with Session.close() or " + "bigframes.pandas.close_session()." + ) + # There are many layers before we get to (possibly) the user's code: + # pandas.read_gbq_table + # -> with_default_session + # -> Session.read_gbq_table + # -> _read_gbq_table + # -> _get_snapshot_sql_and_primary_key + # -> get_snapshot_datetime_and_table_metadata + warnings.warn(msg, category=bfe.TimeTravelCacheWarning, stacklevel=7) + + return cached_table + + if is_information_schema(table_id): + table = get_information_schema_metadata( + bqclient=bqclient, table_id=table_id, default_project=default_project + ) + else: + table_ref = google.cloud.bigquery.table.TableReference.from_string( + table_id, default_project=default_project + ) + table = bqclient.get_table(table_ref) + + # local time will lag a little bit do to network latency + # make sure it is at least table creation time. + # This is relevant if the table was created immediately before loading it here. + if (table.created is not None) and (table.created > bq_time): + bq_time = table.created + + cached_table = (bq_time, table) + cache[table_id] = cached_table + return cached_table + + def is_information_schema(table_id: str): table_id_casefold = table_id.casefold() # Include the "."s to ensure we don't have false positives for some user @@ -111,7 +186,7 @@ def is_information_schema(table_id: str): def is_time_travel_eligible( bqclient: bigquery.Client, - table: Union[bq_data.GbqNativeTable, bq_data.BiglakeIcebergTable], + table: google.cloud.bigquery.table.Table, columns: Optional[Sequence[str]], snapshot_time: datetime.datetime, filter_str: Optional[str] = None, @@ -145,55 +220,50 @@ def is_time_travel_eligible( # -> is_time_travel_eligible stacklevel = 7 - if isinstance(table, bq_data.GbqNativeTable): - # Anonymous dataset, does not support snapshot ever - if table.dataset_id.startswith("_"): - return False + # Anonymous dataset, does not support snapshot ever + if table.dataset_id.startswith("_"): + return False - # Only true tables support time travel - if table.table_id.endswith("*"): + # Only true tables support time travel + if table.table_id.endswith("*"): + if should_warn: + msg = bfe.format_message( + "Wildcard tables do not support FOR SYSTEM_TIME AS OF queries. " + "Attempting query without time travel. Be aware that " + "modifications to the underlying data may result in errors or " + "unexpected behavior." + ) + warnings.warn( + msg, category=bfe.TimeTravelDisabledWarning, stacklevel=stacklevel + ) + return False + elif table.table_type != "TABLE": + if table.table_type == "MATERIALIZED_VIEW": if should_warn: msg = bfe.format_message( - "Wildcard tables do not support FOR SYSTEM_TIME AS OF queries. " - "Attempting query without time travel. Be aware that " - "modifications to the underlying data may result in errors or " - "unexpected behavior." + "Materialized views do not support FOR SYSTEM_TIME AS OF queries. " + "Attempting query without time travel. Be aware that as materialized views " + "are updated periodically, modifications to the underlying data in the view may " + "result in errors or unexpected behavior." ) warnings.warn( msg, category=bfe.TimeTravelDisabledWarning, stacklevel=stacklevel ) return False - elif table.metadata.type != "TABLE": - if table.metadata.type == "MATERIALIZED_VIEW": - if should_warn: - msg = bfe.format_message( - "Materialized views do not support FOR SYSTEM_TIME AS OF queries. " - "Attempting query without time travel. Be aware that as materialized views " - "are updated periodically, modifications to the underlying data in the view may " - "result in errors or unexpected behavior." - ) - warnings.warn( - msg, - category=bfe.TimeTravelDisabledWarning, - stacklevel=stacklevel, - ) - return False - elif table.metadata.type == "VIEW": - return False + elif table.table_type == "VIEW": + return False # table might support time travel, lets do a dry-run query with time travel if should_dry_run: snapshot_sql = bigframes.session._io.bigquery.to_query( - query_or_table=table.get_full_id( - quoted=False - ), # to_query will quote for us + query_or_table=f"{table.reference.project}.{table.reference.dataset_id}.{table.reference.table_id}", columns=columns or (), sql_predicate=filter_str, time_travel_timestamp=snapshot_time, ) try: # If this succeeds, we know that time travel will for sure work. - bigframes.session._io.bigquery.start_query_job_optional( + bigframes.session._io.bigquery.start_query_with_client( bq_client=bqclient, sql=snapshot_sql, job_config=bigquery.QueryJobConfig(dry_run=True), @@ -201,6 +271,7 @@ def is_time_travel_eligible( project=None, timeout=None, metrics=None, + query_with_job=False, publisher=publisher, ) return True @@ -228,8 +299,8 @@ def is_time_travel_eligible( def infer_unique_columns( - table: Union[bq_data.GbqNativeTable, bq_data.BiglakeIcebergTable], - index_cols: Sequence[str], + table: google.cloud.bigquery.table.Table, + index_cols: List[str], ) -> Tuple[str, ...]: """Return a set of columns that can provide a unique row key or empty if none can be inferred. @@ -238,7 +309,7 @@ def infer_unique_columns( """ # If index_cols contain the primary_keys, the query engine assumes they are # provide a unique index. - primary_keys = table.primary_key or () + primary_keys = tuple(_get_primary_keys(table)) if (len(primary_keys) > 0) and frozenset(primary_keys) <= frozenset(index_cols): # Essentially, just reordering the primary key to match the index col order return tuple(index_col for index_col in index_cols if index_col in primary_keys) @@ -251,8 +322,8 @@ def infer_unique_columns( def check_if_index_columns_are_unique( bqclient: bigquery.Client, - table: Union[bq_data.GbqNativeTable, bq_data.BiglakeIcebergTable], - index_cols: Sequence[str], + table: google.cloud.bigquery.table.Table, + index_cols: List[str], *, publisher: bigframes.core.events.Publisher, ) -> Tuple[str, ...]: @@ -261,11 +332,9 @@ def check_if_index_columns_are_unique( # TODO(b/337925142): Avoid a "SELECT *" subquery here by ensuring # table_expression only selects just index_cols. - is_unique_sql = bigframes.core.sql.is_distinct_sql( - index_cols, table.get_table_ref() - ) + is_unique_sql = bigframes.core.sql.is_distinct_sql(index_cols, table.reference) job_config = bigquery.QueryJobConfig() - results = bigframes.session._io.bigquery.start_query_job_optional( + results, _ = bigframes.session._io.bigquery.start_query_with_client( bq_client=bqclient, sql=is_unique_sql, job_config=job_config, @@ -273,6 +342,7 @@ def check_if_index_columns_are_unique( location=None, project=None, metrics=None, + query_with_job=False, publisher=publisher, ) row = next(iter(results)) @@ -282,8 +352,49 @@ def check_if_index_columns_are_unique( return () +def _get_primary_keys( + table: google.cloud.bigquery.table.Table, +) -> List[str]: + """Get primary keys from table if they are set.""" + + primary_keys: List[str] = [] + if ( + (table_constraints := getattr(table, "table_constraints", None)) is not None + and (primary_key := table_constraints.primary_key) is not None + # This will be False for either None or empty list. + # We want primary_keys = None if no primary keys are set. + and (columns := primary_key.columns) + ): + primary_keys = columns if columns is not None else [] + + return primary_keys + + +def _is_table_clustered_or_partitioned( + table: google.cloud.bigquery.table.Table, +) -> bool: + """Returns True if the table is clustered or partitioned.""" + + # Could be None or an empty tuple if it's not clustered, both of which are + # falsey. + if table.clustering_fields: + return True + + if ( + time_partitioning := table.time_partitioning + ) is not None and time_partitioning.type_ is not None: + return True + + if ( + range_partitioning := table.range_partitioning + ) is not None and range_partitioning.field is not None: + return True + + return False + + def get_index_cols( - table: Union[bq_data.GbqNativeTable, bq_data.BiglakeIcebergTable], + table: google.cloud.bigquery.table.Table, index_col: Iterable[str] | str | Iterable[int] @@ -292,7 +403,7 @@ def get_index_cols( *, rename_to_schema: Optional[Dict[str, str]] = None, default_index_type: bigframes.enums.DefaultIndexKind = bigframes.enums.DefaultIndexKind.SEQUENTIAL_INT64, -) -> Sequence[str]: +) -> List[str]: """ If we can get a total ordering from the table, such as via primary key column(s), then return those too so that ordering generation can be @@ -300,9 +411,9 @@ def get_index_cols( """ # Transform index_col -> index_cols so we have a variable that is # always a list of column names (possibly empty). - schema_len = len(table.physical_schema) + schema_len = len(table.schema) - index_cols = [] + index_cols: List[str] = [] if isinstance(index_col, bigframes.enums.DefaultIndexKind): if index_col == bigframes.enums.DefaultIndexKind.SEQUENTIAL_INT64: # User has explicity asked for a default, sequential index. @@ -327,7 +438,7 @@ def get_index_cols( f"Integer index {index_col} is out of bounds " f"for table with {schema_len} columns (must be >= 0 and < {schema_len})." ) - index_cols = [table.physical_schema[index_col].name] + index_cols = [table.schema[index_col].name] elif isinstance(index_col, Iterable): for item in index_col: if isinstance(item, str): @@ -340,7 +451,7 @@ def get_index_cols( f"Integer index {item} is out of bounds " f"for table with {schema_len} columns (must be >= 0 and < {schema_len})." ) - index_cols.append(table.physical_schema[item].name) + index_cols.append(table.schema[item].name) else: raise TypeError( "If index_col is an iterable, it must contain either strings " @@ -355,19 +466,19 @@ def get_index_cols( # If the isn't an index selected, use the primary keys of the table as the # index. If there are no primary keys, we'll return an empty list. if len(index_cols) == 0: - primary_keys = table.primary_key or () + primary_keys = _get_primary_keys(table) # If table has clustering/partitioning, fail if we haven't been able to # find index_cols to use. This is to avoid unexpected performance and # resource utilization because of the default sequential index. See # internal issue 335727141. if ( - (table.partition_col is not None or table.cluster_cols) + _is_table_clustered_or_partitioned(table) and not primary_keys and default_index_type == bigframes.enums.DefaultIndexKind.SEQUENTIAL_INT64 ): msg = bfe.format_message( - f"Table '{str(table.get_full_id())}' is clustered and/or " + f"Table '{str(table.reference)}' is clustered and/or " "partitioned, but BigQuery DataFrames was not able to find a " "suitable index. To avoid this warning, set at least one of: " # TODO(b/338037499): Allow max_results to override this too, @@ -379,6 +490,6 @@ def get_index_cols( # If there are primary keys defined, the query engine assumes these # columns are unique, even if the constraint is not enforced. We make # the same assumption and use these columns as the total ordering keys. - index_cols = list(primary_keys) + index_cols = primary_keys return index_cols diff --git a/bigframes/session/_io/pandas.py b/bigframes/session/_io/pandas.py index 8d41474d8c4..9340e060aca 100644 --- a/bigframes/session/_io/pandas.py +++ b/bigframes/session/_io/pandas.py @@ -89,7 +89,7 @@ def arrow_to_pandas( # Preserve NA/NaN distinction. Note: This is currently needed, even if we use # nullable Float64Dtype in the types_mapper. See: # https://github.com/pandas-dev/pandas/issues/55668 - mask = pyarrow.compute.is_null(column) # type: ignore[attr-defined] + mask = pyarrow.compute.is_null(column) nonnull = pyarrow.compute.fill_null(column, float("nan")) # Regarding type: ignore, this class has been public at this # location since pandas 1.2.0. See: @@ -106,7 +106,7 @@ def arrow_to_pandas( elif dtype == pandas.Int64Dtype(): # Avoid out-of-bounds errors in Pandas 1.5.x, which incorrectly # casts to float64 in an intermediate step. - mask = pyarrow.compute.is_null(column) # type: ignore[attr-defined] + mask = pyarrow.compute.is_null(column) nonnull = pyarrow.compute.fill_null(column, 0) pd_array = pandas.arrays.IntegerArray( nonnull.to_numpy() diff --git a/bigframes/session/anonymous_dataset.py b/bigframes/session/anonymous_dataset.py index ed718ff909f..bdc6e7f59c8 100644 --- a/bigframes/session/anonymous_dataset.py +++ b/bigframes/session/anonymous_dataset.py @@ -14,19 +14,18 @@ import datetime import threading +from typing import List, Optional, Sequence import uuid import warnings -from concurrent.futures import ThreadPoolExecutor -from typing import List, Optional, Sequence -import google.cloud.bigquery as bigquery from google.api_core import retry as api_core_retry +import google.cloud.bigquery as bigquery +from bigframes import constants import bigframes.core.events import bigframes.exceptions as bfe -import bigframes.session._io.bigquery as bf_io_bigquery -from bigframes import constants from bigframes.session import temporary_storage +import bigframes.session._io.bigquery as bf_io_bigquery _TEMP_TABLE_ID_FORMAT = "bqdf{date}_{session_id}_{random_id}" # UDFs older than this many days are considered stale and will be deleted @@ -171,19 +170,9 @@ def _cleanup_old_udfs(self): def close(self): """Delete tables that were created with this session's session_id.""" - if self._table_ids: - try: - with ThreadPoolExecutor() as executor: - futures = [ - executor.submit( - self.bqclient.delete_table, table_ref, not_found_ok=True - ) - for table_ref in self._table_ids - ] - for future in futures: - future.result() - finally: - self._table_ids.clear() + for table_ref in self._table_ids: + self.bqclient.delete_table(table_ref, not_found_ok=True) + self._table_ids.clear() try: # Before closing the session, attempt to clean up any uncollected, diff --git a/bigframes/session/bigquery_session.py b/bigframes/session/bigquery_session.py index 18f8cdeaff4..99c13007d85 100644 --- a/bigframes/session/bigquery_session.py +++ b/bigframes/session/bigquery_session.py @@ -17,17 +17,17 @@ import datetime import logging import threading -import uuid from typing import Callable, Optional, Sequence +import uuid # TODO: Non-ibis implementation import bigframes_vendored.ibis.backends.bigquery.datatypes as ibis_bq import google.cloud.bigquery as bigquery +from bigframes.core.compile import googlesql import bigframes.core.events -import bigframes.session._io.bigquery as bfbqio -from bigframes.core.compile.sqlglot import sql as sg_sql from bigframes.session import temporary_storage +import bigframes.session._io.bigquery as bfbqio KEEPALIVE_QUERY_TIMEOUT_SECONDS = 5.0 @@ -80,7 +80,7 @@ def create_temp_table( ibis_schema = ibis_bq.BigQuerySchema.to_ibis(list(schema)) fields = [ - f"{sg_sql.to_sql(sg_sql.identifier(name))} {ibis_bq.BigQueryType.from_ibis(ibis_type)}" + f"{googlesql.identifier(name)} {ibis_bq.BigQueryType.from_ibis(ibis_type)}" for name, ibis_type in ibis_schema.fields.items() ] fields_string = ",".join(fields) @@ -88,14 +88,14 @@ def create_temp_table( cluster_string = "" if cluster_cols: cluster_cols_sql = ", ".join( - f"{sg_sql.to_sql(sg_sql.identifier(cluster_col))}" + f"{googlesql.identifier(cluster_col)}" for cluster_col in cluster_cols ) cluster_string = f"\nCLUSTER BY {cluster_cols_sql}" - ddl = f"CREATE TEMP TABLE `_SESSION`.{sg_sql.to_sql(sg_sql.identifier(table_ref.table_id))} ({fields_string}){cluster_string}" + ddl = f"CREATE TEMP TABLE `_SESSION`.{googlesql.identifier(table_ref.table_id)} ({fields_string}){cluster_string}" - _, job = bfbqio.start_query_with_job( + _, job = bfbqio.start_query_with_client( self.bqclient, ddl, job_config=job_config, @@ -103,6 +103,7 @@ def create_temp_table( project=None, timeout=None, metrics=None, + query_with_job=True, publisher=self._publisher, ) job.result() @@ -116,18 +117,15 @@ def close(self): self._sessiondaemon.stop() if self._session_id is not None and self.bqclient is not None: - bfbqio.start_query_job_optional( + bfbqio.start_query_with_client( self.bqclient, f"CALL BQ.ABORT_SESSION('{self._session_id}')", - # Assume this is being called in the user thread, so we can access - # this thread-local config. - job_config=bigquery.QueryJobConfig( - labels=dict(bigframes.options.compute.extra_query_labels) - ), + job_config=bigquery.QueryJobConfig(), location=self.location, project=None, timeout=None, metrics=None, + query_with_job=False, publisher=self._publisher, ) @@ -139,7 +137,7 @@ def _get_session_id(self) -> str: job_config = bigquery.QueryJobConfig(create_session=True) # Make sure the session is a new one, not one associated with another query. job_config.use_query_cache = False - _, query_job = bfbqio.start_query_with_job( + _, query_job = bfbqio.start_query_with_client( self.bqclient, "SELECT 1", job_config=job_config, @@ -147,6 +145,7 @@ def _get_session_id(self) -> str: project=None, timeout=None, metrics=None, + query_with_job=True, publisher=self._publisher, ) query_job.result() # blocks until finished @@ -170,7 +169,7 @@ def _keep_session_alive(self): ] ) try: - bfbqio.start_query_job_optional( + bfbqio.start_query_with_client( self.bqclient, "SELECT 1", job_config=job_config, @@ -178,6 +177,7 @@ def _keep_session_alive(self): project=None, timeout=KEEPALIVE_QUERY_TIMEOUT_SECONDS, metrics=None, + query_with_job=False, publisher=self._publisher, ) except Exception as e: diff --git a/bigframes/session/bq_caching_executor.py b/bigframes/session/bq_caching_executor.py index dede318d813..ca19d1be86f 100644 --- a/bigframes/session/bq_caching_executor.py +++ b/bigframes/session/bq_caching_executor.py @@ -14,54 +14,42 @@ from __future__ import annotations -import asyncio -import concurrent.futures -import dataclasses import math import threading -from typing import Literal, Optional, Sequence, Tuple +from typing import Literal, Mapping, Optional, Sequence, Tuple +import weakref import google.api_core.exceptions -import google.cloud.bigquery_storage_v1 from google.cloud import bigquery +import google.cloud.bigquery.job as bq_job +import google.cloud.bigquery.table as bq_table +import google.cloud.bigquery_storage_v1 import bigframes +from bigframes import exceptions as bfe import bigframes.constants import bigframes.core +from bigframes.core import bq_data, compile, local_data, rewrite +import bigframes.core.compile.sqlglot.sqlglot_ir as sqlglot_ir import bigframes.core.events import bigframes.core.guid +import bigframes.core.identifiers import bigframes.core.nodes as nodes -import bigframes.core.ordering import bigframes.core.schema as schemata import bigframes.core.tree_properties as tree_properties import bigframes.dtypes -import bigframes.functions._function_session as bff_session -import bigframes.operations as ops -import bigframes.session._io.bigquery as bq_io -import bigframes.session.execution_cache as execution_cache -import bigframes.session.execution_spec as ex_spec -import bigframes.session.metrics -import bigframes.session.planner -import bigframes.session.temporary_storage -from bigframes.core import ( - compile, - expression, - guid, - identifiers, - local_data, - rewrite, -) -from bigframes.core.compile.sqlglot import sql as sg_sql -from bigframes.core.compile.sqlglot import sqlglot_ir -from bigframes.functions import udf_def from bigframes.session import ( - direct_gbq_execution, executor, loader, local_scan_executor, read_api_execution, semi_executor, ) +import bigframes.session._io.bigquery as bq_io +import bigframes.session.execution_spec as ex_spec +import bigframes.session.metrics +import bigframes.session.planner +import bigframes.session.temporary_storage # Max complexity that should be executed as a single query QUERY_COMPLEXITY_LIMIT = 1e7 @@ -70,42 +58,57 @@ _MAX_CLUSTER_COLUMNS = 4 MAX_SMALL_RESULT_BYTES = 10 * 1024 * 1024 * 1024 # 10G - -_bg_loop = None -_bg_thread = None -_bg_lock = threading.Lock() +SourceIdMapping = Mapping[str, str] -def _get_bg_loop(): - global _bg_loop, _bg_thread - with _bg_lock: - if _bg_loop is None: - loop = asyncio.new_event_loop() - _bg_loop = loop +class ExecutionCache: + def __init__(self): + # current assumption is only 1 cache of a given node + # in future, might have multiple caches, with different layout, localities + self._cached_executions: weakref.WeakKeyDictionary[ + nodes.BigFrameNode, nodes.CachedTableNode + ] = weakref.WeakKeyDictionary() + self._uploaded_local_data: weakref.WeakKeyDictionary[ + local_data.ManagedArrowTable, + tuple[bq_data.BigqueryDataSource, SourceIdMapping], + ] = weakref.WeakKeyDictionary() - def run(): - asyncio.set_event_loop(loop) - loop.run_forever() + @property + def mapping(self) -> Mapping[nodes.BigFrameNode, nodes.BigFrameNode]: + return self._cached_executions - _bg_thread = threading.Thread( - target=run, daemon=True, name="bigframes-bg-loop" + def cache_results_table( + self, + original_root: nodes.BigFrameNode, + data: bq_data.BigqueryDataSource, + ): + # Assumption: GBQ cached table uses field name as bq column name + scan_list = nodes.ScanList( + tuple( + nodes.ScanItem(field.id, field.id.sql) for field in original_root.fields ) - _bg_thread.start() - return _bg_loop - - -def _run_sync(coro): - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = None + ) + cached_replacement = nodes.CachedTableNode( + source=data, + scan_list=scan_list, + table_session=original_root.session, + original_node=original_root, + ) + assert original_root.schema == cached_replacement.schema + self._cached_executions[original_root] = cached_replacement - if loop is None: - return asyncio.run(coro) - else: - bg_loop = _get_bg_loop() - future = asyncio.run_coroutine_threadsafe(coro, bg_loop) - return future.result() + def cache_remote_replacement( + self, + local_data: local_data.ManagedArrowTable, + bq_data: bq_data.BigqueryDataSource, + ): + # bq table has one extra column for offsets, those are implicit for local data + assert len(local_data.schema.items) + 1 == len(bq_data.table.physical_schema) + mapping = { + local_data.schema.items[i].column: bq_data.table.physical_schema[i].name + for i in range(len(local_data.schema)) + } + self._uploaded_local_data[local_data] = (bq_data, mapping) class BigQueryCachingExecutor(executor.Executor): @@ -124,24 +127,20 @@ def __init__( bqstoragereadclient: google.cloud.bigquery_storage_v1.BigQueryReadClient, loader: loader.GbqDataLoader, *, + strictly_ordered: bool = True, metrics: Optional[bigframes.session.metrics.ExecutionMetrics] = None, enable_polars_execution: bool = False, publisher: bigframes.core.events.Publisher, - labels: tuple[tuple[str, str], ...] = (), - compiler_name: Literal["ibis", "sqlglot"] = "sqlglot", - cache: Optional[execution_cache.ExecutionCache] = None, - function_manager: bff_session.FunctionSession, ): self.bqclient = bqclient self.storage_manager = storage_manager - self.cache: execution_cache.ExecutionCache = ( - cache or execution_cache.ExecutionCache() - ) + self.strictly_ordered: bool = strictly_ordered + self.cache: ExecutionCache = ExecutionCache() self.metrics = metrics self.loader = loader + self.bqstoragereadclient = bqstoragereadclient self._enable_polars_execution = enable_polars_execution self._publisher = publisher - self._compiler_name = compiler_name # TODO(tswast): Send events from semi-executors, too. self._semi_executors: Sequence[semi_executor.SemiExecutor] = ( @@ -158,15 +157,7 @@ def __init__( *self._semi_executors, polars_executor.PolarsExecutor(), ) - self._gbq_executor = direct_gbq_execution.DirectGbqExecutor( - bqclient, - compiler=compiler_name, - bqstoragereadclient=bqstoragereadclient, - metrics=self.metrics, - publisher=self._publisher, - labels=dict(labels), - ) - self._function_manager = function_manager + self._upload_lock = threading.Lock() def to_sql( self, @@ -178,15 +169,12 @@ def to_sql( if offset_column: array_value, _ = array_value.promote_offsets() node = ( - self._prepare_plan_simplify(array_value.node) + self.prepare_plan(array_value.node, target="simplify") if enable_cache else array_value.node ) - node = _run_sync(self._substitute_large_local_sources(node)) - compiled = compile.compile_sql( - compile.CompileRequest(node, sort_rows=ordered), - compiler_name=self._compiler_name, - ) + node = self._substitute_large_local_sources(node) + compiled = compile.compile_sql(compile.CompileRequest(node, sort_rows=ordered)) return compiled.sql def execute( @@ -194,159 +182,58 @@ def execute( array_value: bigframes.core.ArrayValue, execution_spec: ex_spec.ExecutionSpec, ) -> executor.ExecuteResult: - # Need to grab thread local before starting async execution. - execution_spec = execution_spec.with_compute_options(bigframes.options.compute) - return _run_sync( - self._execute_async( - array_value, - execution_spec, - ) - ) - - async def _execute_async( - self, - array_value: bigframes.core.ArrayValue, - execution_spec: ex_spec.ExecutionSpec, - ) -> executor.ExecuteResult: - await self._publisher.publish_async(bigframes.core.events.ExecutionStarted()) - maybe_result = await self._try_execute_semi_executors( - array_value, execution_spec - ) - if maybe_result is not None: - return maybe_result - result = await self._execute_bigquery( - array_value, - execution_spec, - ) - await self._publisher.publish_async( - bigframes.core.events.EventEnvelope( - event=bigframes.core.events.ExecutionFinished(result=result), - cell_execution_count=execution_spec.cell_execution_count, - ) - ) - return result - - async def _try_execute_semi_executors( - self, - array_value: bigframes.core.ArrayValue, - execution_spec: ex_spec.ExecutionSpec, - ) -> Optional[executor.ExecuteResult]: - plan = self._prepare_plan_simplify(array_value.node) - for exec in self._semi_executors: - maybe_result = await exec.execute(plan, execution_spec) - if maybe_result: - await self._publisher.publish_async( - bigframes.core.events.EventEnvelope( - event=bigframes.core.events.ExecutionFinished( + self._publisher.publish(bigframes.core.events.ExecutionStarted()) + + # TODO: Support export jobs in combination with semi executors + if execution_spec.destination_spec is None: + plan = self.prepare_plan(array_value.node, target="simplify") + for exec in self._semi_executors: + maybe_result = exec.execute( + plan, ordered=execution_spec.ordered, peek=execution_spec.peek + ) + if maybe_result: + self._publisher.publish( + bigframes.core.events.ExecutionFinished( result=maybe_result, - ), - cell_execution_count=execution_spec.cell_execution_count, + ) ) - ) - return maybe_result - return None + return maybe_result - async def _execute_bigquery( - self, - array_value: bigframes.core.ArrayValue, - execution_spec: ex_spec.ExecutionSpec, - ) -> executor.ExecuteResult: - dest_spec = execution_spec.destination_spec - # Recursive handlers for different cases, maybe extract to explicit interface. - if isinstance(dest_spec, ex_spec.GcsOutputSpec): - execution_spec = dataclasses.replace( - execution_spec, destination_spec=ex_spec.EphemeralTableSpec() - ) - results = await self._execute_bigquery( - array_value, - execution_spec, - ) - await self._export_result_gcs(results, dest_spec) - return results - elif isinstance(dest_spec, ex_spec.TableOutputSpec): - return await self._execute_gbq_table_export( - array_value, - execution_spec, - ) - # Force table creation if result might be large (and user explicitly allowed large results) - elif isinstance(dest_spec, ex_spec.EphemeralTableSpec) or (dest_spec is None): - if not execution_spec.promise_under_10gb: - table = await asyncio.to_thread( - self.storage_manager.create_temp_table, - array_value.schema.to_bigquery(), - ) - execution_spec = dataclasses.replace( - execution_spec, - destination_spec=ex_spec.TableOutputSpec( - table=table, if_exists="append" - ), + if isinstance(execution_spec.destination_spec, ex_spec.TableOutputSpec): + if execution_spec.peek or execution_spec.ordered: + raise NotImplementedError( + "Ordering and peeking not supported for gbq export" ) - # We don't use _execute_gbq_table_export, as this result is internal, not exported. - return await self._execute_gbq_query_only( - array_value, - execution_spec, + # separate path for export_gbq, as it has all sorts of annoying logic, such as possibly running as dml + result = self._export_gbq(array_value, execution_spec.destination_spec) + self._publisher.publish( + bigframes.core.events.ExecutionFinished( + result=result, ) - # At this point, dst should be unspecified, a specific bq table, or an ephemeral temp table that fits in <10gb - return await self._execute_gbq_query_only( - array_value, - execution_spec, - ) - - async def _execute_gbq_table_export( - self, - array_value: bigframes.core.ArrayValue, - execution_spec: ex_spec.ExecutionSpec, - ) -> executor.ExecuteResult: - dest_spec = execution_spec.destination_spec - assert isinstance(dest_spec, ex_spec.TableOutputSpec) - existing_table = await self._maybe_find_existing_table(dest_spec) - if (existing_table is not None) and _is_schema_match( - existing_table.schema, array_value.schema - ): - # Special DML path - maybe this should be configurable, dml vs query destination has tradeoffs - execution_spec = dataclasses.replace( - execution_spec, destination_spec=ex_spec.EphemeralTableSpec() - ) - results = await self._execute_bigquery( - array_value, - execution_spec, - ) - assert isinstance(results, executor.BQTableExecuteResult) - await self._export_gbq_with_dml(results, dest_spec) - result: executor.ExecuteResult = results - else: - result = await self._execute_gbq_query_only( - array_value, - execution_spec, ) + return result - has_special_dtype_col = any( - t in (bigframes.dtypes.TIMEDELTA_DTYPE, bigframes.dtypes.OBJ_REF_DTYPE) - for t in array_value.schema.dtypes + result = self._execute_plan_gbq( + array_value.node, + ordered=execution_spec.ordered, + peek=execution_spec.peek, + cache_spec=execution_spec.destination_spec + if isinstance(execution_spec.destination_spec, ex_spec.CacheSpec) + else None, + must_create_table=not execution_spec.promise_under_10gb, ) - if dest_spec.if_exists != "append" and has_special_dtype_col: - table = await asyncio.to_thread(self.bqclient.get_table, dest_spec.table) - table.schema = array_value.schema.to_bigquery() - await asyncio.to_thread(self.bqclient.update_table, table, ["schema"]) + # post steps: export + if isinstance(execution_spec.destination_spec, ex_spec.GcsOutputSpec): + self._export_result_gcs(result, execution_spec.destination_spec) - return result - - async def _execute_gbq_query_only( - self, - array_value: bigframes.core.ArrayValue, - execution_spec: ex_spec.ExecutionSpec, - ) -> executor.ExecuteResult: - gbq_plan = await self._prepare_plan_bq_execution( - array_value.node, execution_spec.bigquery_config - ) - result = await self._gbq_executor.execute(gbq_plan, execution_spec) - if result is None: - raise ValueError( - f"Couldn't execute plan {array_value.node} with {execution_spec}" + self._publisher.publish( + bigframes.core.events.ExecutionFinished( + result=result, ) + ) return result - async def _export_result_gcs( + def _export_result_gcs( self, result: executor.ExecuteResult, gcs_export_spec: ex_spec.GcsOutputSpec ): query_job = result.query_job @@ -359,8 +246,7 @@ async def _export_result_gcs( format=gcs_export_spec.format, export_options=dict(gcs_export_spec.export_options), ) - await asyncio.to_thread( - bq_io.start_query_with_job, + bq_io.start_query_with_client( self.bqclient, export_data_statement, job_config=bigquery.QueryJobConfig(), @@ -368,39 +254,94 @@ async def _export_result_gcs( project=None, location=None, timeout=None, + query_with_job=True, publisher=self._publisher, ) - async def _export_gbq_with_dml( - self, result: executor.BQTableExecuteResult, spec: ex_spec.TableOutputSpec - ): + def _maybe_find_existing_table( + self, spec: ex_spec.TableOutputSpec + ) -> Optional[bigquery.Table]: + # validate destination table + try: + table = self.bqclient.get_table(spec.table) + if spec.if_exists == "fail": + raise ValueError(f"Table already exists: {spec.table.__str__()}") + + if len(spec.cluster_cols) != 0: + if (table.clustering_fields is None) or ( + tuple(table.clustering_fields) != spec.cluster_cols + ): + raise ValueError( + "Table clustering fields cannot be changed after the table has " + f"been created. Requested clustering fields: {spec.cluster_cols}, existing clustering fields: {table.clustering_fields}" + ) + return table + except google.api_core.exceptions.NotFound: + return None + + def _export_gbq( + self, array_value: bigframes.core.ArrayValue, spec: ex_spec.TableOutputSpec + ) -> executor.ExecuteResult: """ - Export the ArrayValue to an existing BigQuery table, using DML. + Export the ArrayValue to an existing BigQuery table. """ - # b/409086472: Uses DML for table appends and replacements to avoid - # BigQuery `RATE_LIMIT_EXCEEDED` errors, as per quota limits: - # https://cloud.google.com/bigquery/quotas#standard_tables - assert result.query_job is not None - assert result.query_job.destination is not None - ir = sqlglot_ir.SQLGlotIR.from_table( - result.query_job.destination.project, - result.query_job.destination.dataset_id, - result.query_job.destination.table_id, + plan = self.prepare_plan(array_value.node, target="bq_execution") + + # validate destination table + existing_table = self._maybe_find_existing_table(spec) + + compiled = compile.compile_sql(compile.CompileRequest(plan, sort_rows=False)) + sql = compiled.sql + + if (existing_table is not None) and _if_schema_match( + existing_table.schema, array_value.schema + ): + # b/409086472: Uses DML for table appends and replacements to avoid + # BigQuery `RATE_LIMIT_EXCEEDED` errors, as per quota limits: + # https://cloud.google.com/bigquery/quotas#standard_tables + job_config = bigquery.QueryJobConfig() + ir = sqlglot_ir.SQLGlotIR.from_query_string(sql) + if spec.if_exists == "append": + sql = ir.insert(spec.table) + else: # for "replace" + assert spec.if_exists == "replace" + sql = ir.replace(spec.table) + else: + dispositions = { + "fail": bigquery.WriteDisposition.WRITE_EMPTY, + "replace": bigquery.WriteDisposition.WRITE_TRUNCATE, + "append": bigquery.WriteDisposition.WRITE_APPEND, + } + job_config = bigquery.QueryJobConfig( + write_disposition=dispositions[spec.if_exists], + destination=spec.table, + clustering_fields=spec.cluster_cols if spec.cluster_cols else None, + ) + + # TODO(swast): plumb through the api_name of the user-facing api that + # caused this query. + iterator, job = self._run_execute_query( + sql=sql, + job_config=job_config, + session=array_value.session, ) - sql = "" - if spec.if_exists == "append": - sql = sg_sql.to_sql(sg_sql.insert(ir.expr.as_select_all(), spec.table)) - else: # for "replace" - assert spec.if_exists == "replace" - sql = sg_sql.to_sql(sg_sql.replace(ir.expr.as_select_all(), spec.table)) - - await asyncio.to_thread( - bq_io.start_query_with_job, - self.bqclient, - sql, - job_config=bigquery.QueryJobConfig(), - metrics=self.metrics, - publisher=self._publisher, + + has_timedelta_col = any( + t == bigframes.dtypes.TIMEDELTA_DTYPE for t in array_value.schema.dtypes + ) + + if spec.if_exists != "append" and has_timedelta_col: + # Only update schema if this is not modifying an existing table, and the + # new table contains timedelta columns. + table = self.bqclient.get_table(spec.table) + table.schema = array_value.schema.to_bigquery() + self.bqclient.update_table(table, ["schema"]) + + return executor.EmptyExecuteResult( + bf_schema=array_value.schema, + execution_metadata=executor.ExecutionMetadata.from_iterator_and_job( + iterator, job + ), ) def dry_run( @@ -413,23 +354,6 @@ def dry_run( def cached( self, array_value: bigframes.core.ArrayValue, *, config: executor.CacheConfig - ) -> None: - # Get compute options before passing to async method, can be thread-local - bq_compute_options = ex_spec.BqComputeOptions.from_compute_options( - bigframes.options.compute - ) - return _run_sync( - self._cached_async( - array_value, config=config, compute_options=bq_compute_options - ) - ) - - async def _cached_async( - self, - array_value: bigframes.core.ArrayValue, - *, - config: executor.CacheConfig, - compute_options: ex_spec.BqComputeOptions, ) -> None: """Write the block to a session table.""" # First, see if we can reuse the existing cache @@ -451,64 +375,75 @@ async def _cached_async( raise ValueError(f"Unexpected 'if_cached' arg: {config.if_cached}") if config.optimize_for == "auto": - await self._cache_with_session_awareness( - array_value, compute_options=compute_options - ) + self._cache_with_session_awareness(array_value) elif config.optimize_for == "head": - await self._cache_with_offsets(array_value, compute_options=compute_options) + self._cache_with_offsets(array_value) else: assert isinstance(config.optimize_for, executor.HierarchicalKey) - await self._cache_with_cluster_cols( - array_value, - cluster_cols=config.optimize_for.columns, - compute_options=compute_options, + self._cache_with_cluster_cols( + array_value, cluster_cols=config.optimize_for.columns ) - async def _execute_to_cached_table( + # Helpers + def _run_execute_query( self, - plan: nodes.BigFrameNode, - cache_spec: ex_spec.CacheSpec, - compute_options: ex_spec.BqComputeOptions, - ) -> executor.ExecuteResult: - # "ephemeral" temp tables created in the course of exeuction, don't need to be allocated - # materialized ordering only really makes sense for internal temp tables used by caching - cluster_cols = cache_spec.cluster_cols - # Rewrite plan to materialize ordering as extra columns - if cache_spec.ordering == "offsets_col": - order_col_id = guid.generate_guid() - plan = nodes.PromoteOffsetsNode(plan, identifiers.ColumnId(order_col_id)) - cluster_cols = (order_col_id,) - ordering: bigframes.core.ordering.RowOrdering = ( - bigframes.core.ordering.TotalOrdering.from_offset_col(order_col_id) + sql: str, + job_config: Optional[bq_job.QueryJobConfig] = None, + query_with_job: bool = True, + session=None, + ) -> Tuple[bq_table.RowIterator, Optional[bigquery.QueryJob]]: + """ + Starts BigQuery query job and waits for results. + """ + job_config = bq_job.QueryJobConfig() if job_config is None else job_config + if bigframes.options.compute.maximum_bytes_billed is not None: + job_config.maximum_bytes_billed = ( + bigframes.options.compute.maximum_bytes_billed ) - elif cache_spec.ordering == "order_key": - plan, ordering = rewrite.pull_out_order(plan) - destination_table = await asyncio.to_thread( - self.storage_manager.create_temp_table, - plan.schema.to_bigquery(), - cluster_cols, - ) - arr_value = bigframes.core.ArrayValue(plan) - execution_spec = ex_spec.ExecutionSpec( - destination_spec=ex_spec.TableOutputSpec( - table=destination_table, - cluster_cols=cluster_cols, - if_exists="replace", - ), - bigquery_config=compute_options, - ) - # We don't use _execute_gbq_table_export, as this result is internal, not exported. - result = await self._execute_gbq_query_only( - arr_value, - execution_spec, - ) - assert isinstance(result, executor.BQTableExecuteResult), ( - "expected result to be BQTableExecuteResult" - ) - result._data = dataclasses.replace(result._data, ordering=ordering) - return result - # Helpers + if not self.strictly_ordered: + job_config.labels["bigframes-mode"] = "unordered" + + try: + # Trick the type checker into thinking we got a literal. + if query_with_job: + return bq_io.start_query_with_client( + self.bqclient, + sql, + job_config=job_config, + metrics=self.metrics, + project=None, + location=None, + timeout=None, + query_with_job=True, + publisher=self._publisher, + session=session, + ) + else: + return bq_io.start_query_with_client( + self.bqclient, + sql, + job_config=job_config, + metrics=self.metrics, + project=None, + location=None, + timeout=None, + query_with_job=False, + publisher=self._publisher, + session=session, + ) + + except google.api_core.exceptions.BadRequest as e: + # Unfortunately, this error type does not have a separate error code or exception type + if "Resources exceeded during query execution" in e.message: + new_message = "Computation is too complex to execute as a single query. Try using DataFrame.cache() on intermediate results, or setting bigframes.options.compute.enable_multi_query_execution." + raise bfe.QueryComplexityError(new_message) from e + else: + raise + + def replace_cached_subtrees(self, node: nodes.BigFrameNode) -> nodes.BigFrameNode: + return nodes.top_down(node, lambda x: self.cache.mapping.get(x, x)) + def _is_trivially_executable(self, array_value: bigframes.core.ArrayValue): """ Can the block be evaluated very cheaply? @@ -516,129 +451,62 @@ def _is_trivially_executable(self, array_value: bigframes.core.ArrayValue): """ # Once rewriting is available, will want to rewrite before # evaluating execution cost. - simplified_plan = self._prepare_plan_simplify(array_value.node) - return tree_properties.is_trivially_executable(simplified_plan) - - def _prepare_plan_simplify(self, plan: nodes.BigFrameNode) -> nodes.BigFrameNode: - """Prepare the plan by simplifying it with caches and removing unused operators.""" - plan = self.cache.subsitute_cached_subplans(plan) - plan = rewrite.column_pruning(plan) - plan = plan.top_down(rewrite.fold_row_counts) - return plan - - async def _deploy_undeployed_udfs( - self, plan: nodes.BigFrameNode - ) -> nodes.BigFrameNode: - referenced_udfs = list(set(self._collect_udf_defs(plan))) - deployed_mapping: dict[udf_def.PythonUdf, udf_def.BigqueryUdf] = {} - tasks = [ - asyncio.to_thread( - self._function_manager._deploy_udf, - udf, - ) - for udf in referenced_udfs - ] - results = await asyncio.gather(*tasks) - deployed_mapping = dict(zip(referenced_udfs, results)) - - return self._subsitute_temporary_functions(plan, deployed_mapping) - - def _collect_udf_defs(self, plan: nodes.BigFrameNode) -> list[udf_def.PythonUdf]: - udf_defs: list[udf_def.PythonUdf] = [] - exprs = [ - expr for node in plan.unique_nodes() for expr in node._node_expressions - ] - expr_nodes = [expr for expr in exprs for expr in expr.walk()] - for expr_node in expr_nodes: - if ( - isinstance(expr_node, expression.OpExpression) - and isinstance(expr_node.op, ops.PythonUdfOp) - and isinstance(expr_node.op.function_def, udf_def.PythonUdf) - ): - udf_defs.append(expr_node.op.function_def) - return udf_defs + return tree_properties.is_trivially_executable( + self.prepare_plan(array_value.node) + ) - def _subsitute_temporary_functions( + def prepare_plan( self, plan: nodes.BigFrameNode, - deployed_mapping: dict[udf_def.PythonUdf, udf_def.BigqueryUdf], + target: Literal["simplify", "bq_execution"] = "simplify", ) -> nodes.BigFrameNode: - def replace_udf_expr(e: expression.Expression) -> expression.Expression: - if isinstance(e, expression.OpExpression) and isinstance( - e.op, ops.PythonUdfOp - ): - func_def = e.op.function_def - # We will have already deployed the function - assert func_def in deployed_mapping - deployed_func = deployed_mapping[func_def] - rf_op = ops.RemoteFunctionOp(function_def=deployed_func) - return dataclasses.replace(e, op=rf_op) - return e + """ + Prepare the plan by simplifying it with caches, removing unused operators. Has modes for different contexts. - def replace_in_expr(expr: expression.Expression) -> expression.Expression: - return expr.bottom_up(replace_udf_expr) + "simplify" removes unused operations and subsitutes subtrees with their previously cached equivalents + "bq_execution" is the most heavy option, preparing the plan for bq execution by also caching subtrees, uploading large local sources + """ + # TODO: We should model plan decomposition and data uploading as work steps rather than as plan preparation. + if ( + target == "bq_execution" + and bigframes.options.compute.enable_multi_query_execution + ): + self._simplify_with_caching(plan) - def replace_in_node(node: nodes.BigFrameNode) -> nodes.BigFrameNode: - if hasattr(node, "transform_exprs"): - return node.transform_exprs(replace_in_expr) - return node + plan = self.replace_cached_subtrees(plan) + plan = rewrite.column_pruning(plan) + plan = plan.top_down(rewrite.fold_row_counts) - return plan.bottom_up(replace_in_node) + if target == "bq_execution": + plan = self._substitute_large_local_sources(plan) - async def _prepare_plan_bq_execution( - self, - plan: nodes.BigFrameNode, - compute_options: Optional[ex_spec.BqComputeOptions] = None, - ) -> nodes.BigFrameNode: - """Prepare the plan for BigQuery execution by caching subtrees and uploading large local sources.""" - plan = await self._deploy_undeployed_udfs(plan) - if compute_options is not None and compute_options.enable_multi_query_execution: - await self._simplify_with_caching(plan, compute_options=compute_options) - plan = self._prepare_plan_simplify(plan) - plan = await self._substitute_large_local_sources(plan) return plan - async def _cache_with_cluster_cols( - self, - array_value: bigframes.core.ArrayValue, - cluster_cols: Sequence[str], - compute_options: ex_spec.BqComputeOptions, + def _cache_with_cluster_cols( + self, array_value: bigframes.core.ArrayValue, cluster_cols: Sequence[str] ): """Executes the query and uses the resulting table to rewrite future executions.""" - cluster_cols = [ - col - for col in cluster_cols - if bigframes.dtypes.is_clusterable(array_value.schema.get_type(col)) - ] - cluster_cols = cluster_cols[:_MAX_CLUSTER_COLUMNS] - result = await self._execute_to_cached_table( - array_value.node, - ex_spec.CacheSpec(cluster_cols=tuple(cluster_cols), ordering="order_key"), - compute_options=compute_options, + execution_spec = ex_spec.ExecutionSpec( + destination_spec=ex_spec.CacheSpec(cluster_cols=tuple(cluster_cols)) + ) + self.execute( + array_value, + execution_spec=execution_spec, ) - assert isinstance(result, executor.BQTableExecuteResult) - assert result._data.ordering is not None - self.cache.cache_results_table(array_value.node, result._data) - async def _cache_with_offsets( - self, - array_value: bigframes.core.ArrayValue, - compute_options: ex_spec.BqComputeOptions, - ): + def _cache_with_offsets(self, array_value: bigframes.core.ArrayValue): """Executes the query and uses the resulting table to rewrite future executions.""" - result = await self._execute_to_cached_table( - array_value.node, - ex_spec.CacheSpec(ordering="offsets_col"), - compute_options=compute_options, + execution_spec = ex_spec.ExecutionSpec( + destination_spec=ex_spec.CacheSpec(cluster_cols=tuple()) + ) + self.execute( + array_value, + execution_spec=execution_spec, ) - assert isinstance(result, executor.BQTableExecuteResult) - assert result._data.ordering is not None - self.cache.cache_results_table(array_value.node, result._data) - async def _cache_with_session_awareness( + def _cache_with_session_awareness( self, array_value: bigframes.core.ArrayValue, - compute_options: ex_spec.BqComputeOptions, ) -> None: session_forest = [obj._block._expr.node for obj in array_value.session.objects] # These node types are cheap to re-compute @@ -647,158 +515,227 @@ async def _cache_with_session_awareness( ) cluster_cols_sql_names = [id.sql for id in cluster_cols] if len(cluster_cols) > 0: - await self._cache_with_cluster_cols( - bigframes.core.ArrayValue(target), - cluster_cols_sql_names, - compute_options=compute_options, - ) - elif not target.order_ambiguous: - await self._cache_with_offsets( - bigframes.core.ArrayValue(target), - compute_options=compute_options, + self._cache_with_cluster_cols( + bigframes.core.ArrayValue(target), cluster_cols_sql_names ) + elif self.strictly_ordered: + self._cache_with_offsets(bigframes.core.ArrayValue(target)) else: - await self._cache_with_cluster_cols( - bigframes.core.ArrayValue(target), - [], - compute_options=compute_options, - ) + self._cache_with_cluster_cols(bigframes.core.ArrayValue(target), []) - async def _simplify_with_caching( - self, plan: nodes.BigFrameNode, compute_options: ex_spec.BqComputeOptions - ): + def _simplify_with_caching(self, plan: nodes.BigFrameNode): """Attempts to handle the complexity by caching duplicated subtrees and breaking the query into pieces.""" # Apply existing caching first for _ in range(MAX_SUBTREE_FACTORINGS): if ( - self._prepare_plan_simplify(plan).planning_complexity + self.prepare_plan(plan, "simplify").planning_complexity < QUERY_COMPLEXITY_LIMIT ): return - did_cache = await self._cache_most_complex_subtree( - plan, compute_options=compute_options - ) + did_cache = self._cache_most_complex_subtree(plan) if not did_cache: return - async def _cache_most_complex_subtree( - self, node: nodes.BigFrameNode, compute_options: ex_spec.BqComputeOptions - ) -> bool: + def _cache_most_complex_subtree(self, node: nodes.BigFrameNode) -> bool: # TODO: If query fails, retry with lower complexity limit selection = tree_properties.select_cache_target( node, min_complexity=(QUERY_COMPLEXITY_LIMIT / 500), max_complexity=QUERY_COMPLEXITY_LIMIT, - cache=self.cache, + cache=dict(self.cache.mapping), # Heuristic: subtree_compleixty * (copies of subtree)^2 - heuristic=lambda complexity, count: ( - math.log(complexity) + 2 * math.log(count) - ), + heuristic=lambda complexity, count: math.log(complexity) + + 2 * math.log(count), ) if selection is None: # No good subtrees to cache, just return original tree return False - await self._cache_with_cluster_cols( - bigframes.core.ArrayValue(selection), - [], - compute_options=compute_options, - ) + self._cache_with_cluster_cols(bigframes.core.ArrayValue(selection), []) return True - async def _substitute_large_local_sources(self, original_root: nodes.BigFrameNode): + def _substitute_large_local_sources(self, original_root: nodes.BigFrameNode): """ Replace large local sources with the uploaded version of those datasources. """ # Step 1: Upload all previously un-uploaded data - needs_upload = [] for leaf in original_root.unique_nodes(): if isinstance(leaf, nodes.ReadLocalNode): if ( leaf.local_data_source.metadata.total_bytes > bigframes.constants.MAX_INLINE_BYTES ): - needs_upload.append(leaf.local_data_source) - - futures: dict[concurrent.futures.Future, local_data.ManagedArrowTable] = dict() - for local_source in needs_upload: - future = self.loader.read_data_async( - local_source, bigframes.core.guid.generate_guid() - ) - futures[future] = local_source - try: - results = await asyncio.gather( - *(asyncio.wrap_future(f) for f in futures.keys()) - ) - for future, result in zip(futures.keys(), results): - self.cache.cache_remote_replacement(futures[future], result) - except Exception as e: - # cancel all futures - for future in futures: - future.cancel() - raise e + self._upload_local_data(leaf.local_data_source) # Step 2: Replace local scans with remote scans def map_local_scans(node: nodes.BigFrameNode): if not isinstance(node, nodes.ReadLocalNode): return node - uploaded_local_data = self.cache.get_uploaded_local_data( - node.local_data_source - ) - if uploaded_local_data is None: + if node.local_data_source not in self.cache._uploaded_local_data: return node - - scan_list = node.scan_list.remap_source_ids( - uploaded_local_data.source_mapping - ) + bq_source, source_mapping = self.cache._uploaded_local_data[ + node.local_data_source + ] + scan_list = node.scan_list.remap_source_ids(source_mapping) # offsets_col isn't part of ReadTableNode, so emulate by adding to end of scan_list if node.offsets_col is not None: # Offsets are always implicitly the final column of uploaded data # See: Loader.load_data scan_list = scan_list.append( - uploaded_local_data.bq_source.table.physical_schema[-1].name, + bq_source.table.physical_schema[-1].name, bigframes.dtypes.INT_DTYPE, node.offsets_col, ) - return nodes.ReadTableNode( - uploaded_local_data.bq_source, scan_list, node.session - ) + return nodes.ReadTableNode(bq_source, scan_list, node.session) return original_root.bottom_up(map_local_scans) - async def _maybe_find_existing_table( - self, spec: ex_spec.TableOutputSpec - ) -> Optional[bigquery.Table]: - # validate destination table - try: - table = await asyncio.to_thread(self.bqclient.get_table, spec.table) - if spec.if_exists == "fail": - raise ValueError(f"Table already exists: {spec.table.__str__()}") + def _upload_local_data(self, local_table: local_data.ManagedArrowTable): + if local_table in self.cache._uploaded_local_data: + return + # Lock prevents concurrent repeated work, but slows things down. + # Might be better as a queue and a worker thread + with self._upload_lock: + if local_table not in self.cache._uploaded_local_data: + uploaded = self.loader.load_data( + local_table, bigframes.core.guid.generate_guid() + ) + self.cache.cache_remote_replacement(local_table, uploaded) - if len(spec.cluster_cols) != 0: - if (table.clustering_fields is None) or ( - tuple(table.clustering_fields) != spec.cluster_cols - ): - raise ValueError( - "Table clustering fields cannot be changed after the table has " - f"been created. Requested clustering fields: {spec.cluster_cols}, existing clustering fields: {table.clustering_fields}" - ) - return table - except google.api_core.exceptions.NotFound: - return None + def _execute_plan_gbq( + self, + plan: nodes.BigFrameNode, + ordered: bool, + peek: Optional[int] = None, + cache_spec: Optional[ex_spec.CacheSpec] = None, + must_create_table: bool = True, + ) -> executor.ExecuteResult: + """Just execute whatever plan as is, without further caching or decomposition.""" + # TODO(swast): plumb through the api_name of the user-facing api that + # caused this query. + + og_plan = plan + og_schema = plan.schema + + plan = self.prepare_plan(plan, target="bq_execution") + create_table = must_create_table + cluster_cols: Sequence[str] = [] + if cache_spec is not None: + if peek is not None: + raise ValueError("peek is not compatible with caching.") + + create_table = True + if not cache_spec.cluster_cols: + + offsets_id = bigframes.core.identifiers.ColumnId( + bigframes.core.guid.generate_guid() + ) + plan = nodes.PromoteOffsetsNode(plan, offsets_id) + cluster_cols = [offsets_id.sql] + else: + cluster_cols = [ + col + for col in cache_spec.cluster_cols + if bigframes.dtypes.is_clusterable(plan.schema.get_type(col)) + ] + cluster_cols = cluster_cols[:_MAX_CLUSTER_COLUMNS] + + compiled = compile.compile_sql( + compile.CompileRequest( + plan, + sort_rows=ordered, + peek_count=peek, + materialize_all_order_keys=(cache_spec is not None), + ) + ) + # might have more columns than og schema, for hidden ordering columns + compiled_schema = compiled.sql_schema + + destination_table: Optional[bigquery.TableReference] = None + + job_config = bigquery.QueryJobConfig() + if create_table: + destination_table = self.storage_manager.create_temp_table( + compiled_schema, cluster_cols + ) + job_config.destination = destination_table + + iterator, query_job = self._run_execute_query( + sql=compiled.sql, + job_config=job_config, + query_with_job=(destination_table is not None), + session=plan.session, + ) + + # we could actually cache even when caching is not explicitly requested, but being conservative for now + result_bq_data = None + if query_job and query_job.destination: + # we might add extra sql columns in compilation, esp if caching w ordering, infer a bigframes type for them + result_bf_schema = _result_schema(og_schema, list(compiled.sql_schema)) + dst = query_job.destination + result_bq_data = bq_data.BigqueryDataSource( + table=bq_data.GbqTable( + dst.project, + dst.dataset_id, + dst.table_id, + tuple(compiled_schema), + is_physically_stored=True, + cluster_cols=tuple(cluster_cols), + ), + schema=result_bf_schema, + ordering=compiled.row_order, + n_rows=iterator.total_rows, + ) + if cache_spec is not None: + assert result_bq_data is not None + assert compiled.row_order is not None + self.cache.cache_results_table(og_plan, result_bq_data) -def _is_schema_match( - table_schema: Tuple[bigquery.SchemaField, ...], - schema: schemata.ArraySchema, + execution_metadata = executor.ExecutionMetadata.from_iterator_and_job( + iterator, query_job + ) + result_mostly_cached = ( + hasattr(iterator, "_is_almost_completely_cached") + and iterator._is_almost_completely_cached() + ) + if result_bq_data is not None and not result_mostly_cached: + return executor.BQTableExecuteResult( + data=result_bq_data, + project_id=self.bqclient.project, + storage_client=self.bqstoragereadclient, + execution_metadata=execution_metadata, + selected_fields=tuple((col, col) for col in og_schema.names), + ) + else: + return executor.LocalExecuteResult( + data=iterator.to_arrow().select(og_schema.names), + bf_schema=plan.schema, + execution_metadata=execution_metadata, + ) + + +def _result_schema( + logical_schema: schemata.ArraySchema, sql_schema: list[bigquery.SchemaField] +) -> schemata.ArraySchema: + inferred_schema = bigframes.dtypes.bf_type_from_type_kind(sql_schema) + inferred_schema.update(logical_schema._mapping) + return schemata.ArraySchema( + tuple(schemata.SchemaItem(col, dtype) for col, dtype in inferred_schema.items()) + ) + + +def _if_schema_match( + table_schema: Tuple[bigquery.SchemaField, ...], schema: schemata.ArraySchema ) -> bool: if len(table_schema) != len(schema.items): return False - for field, schema_item in zip(table_schema, schema.items): - if field.name != schema_item.column: + for field in table_schema: + if field.name not in schema.names: return False - _, field_dtype = bigframes.dtypes.convert_schema_field(field) - if field_dtype != schema_item.dtype: + if bigframes.dtypes.convert_schema_field(field)[1] != schema.get_type( + field.name + ): return False return True diff --git a/bigframes/session/clients.py b/bigframes/session/clients.py index 49822bac16b..2a5c9d64991 100644 --- a/bigframes/session/clients.py +++ b/bigframes/session/clients.py @@ -14,7 +14,9 @@ """Clients manages the connection to Google APIs.""" +import os import threading +import typing from typing import Optional, Sequence, Tuple import google.api_core.client_info @@ -30,11 +32,13 @@ import google.cloud.storage # type: ignore import requests +import bigframes._config.auth import bigframes.constants import bigframes.version from . import environment +_ENV_DEFAULT_PROJECT = "GOOGLE_CLOUD_PROJECT" _APPLICATION_NAME = f"bigframes/{bigframes.version.__version__} ibis/9.2.0" @@ -46,6 +50,10 @@ _BIGQUERYSTORAGE_REGIONAL_ENDPOINT = "bigquerystorage.{location}.rep.googleapis.com" +def _get_default_credentials_with_project(): + return bigframes._config.auth.get_default_credentials_with_project() + + def _get_application_names(): apps = [_APPLICATION_NAME] @@ -66,10 +74,10 @@ class ClientsProvider: def __init__( self, - project: str, - credentials: google.auth.credentials.Credentials, + project: Optional[str] = None, location: Optional[str] = None, use_regional_endpoints: Optional[bool] = None, + credentials: Optional[google.auth.credentials.Credentials] = None, application_name: Optional[str] = None, bq_kms_key_name: Optional[str] = None, client_endpoints_override: dict = {}, @@ -78,6 +86,26 @@ def __init__( Tuple[str, requests.adapters.BaseAdapter] ] = (), ): + credentials_project = None + if credentials is None: + credentials, credentials_project = _get_default_credentials_with_project() + + # Prefer the project in this order: + # 1. Project explicitly specified by the user + # 2. Project set in the environment + # 3. Project associated with the default credentials + project = ( + project + or os.getenv(_ENV_DEFAULT_PROJECT) + or typing.cast(Optional[str], credentials_project) + ) + + if not project: + raise ValueError( + "Project must be set to initialize BigQuery client. " + "Try setting `bigframes.options.bigquery.project` first." + ) + self._application_name = ( f"{_get_application_names()} {application_name}" if application_name diff --git a/bigframes/session/deferred.py b/bigframes/session/deferred.py deleted file mode 100644 index 75906e2a124..00000000000 --- a/bigframes/session/deferred.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from typing import Any, Callable, Optional, Union - -import pandas as pd - -import bigframes.dataframe -import bigframes.series - - -class DeferredBigQueryDataFrame: - """A proxy object that defers the execution of a BigQuery job until requested.""" - - def __init__( - self, - execution_func: Callable[ - [], - Union[ - bigframes.dataframe.DataFrame, - bigframes.series.Series, - pd.Series, - pd.DataFrame, - ], - ], - ): - self._execution_func = execution_func - self._result: Optional[ - Union[ - bigframes.dataframe.DataFrame, - bigframes.series.Series, - pd.Series, - pd.DataFrame, - ] - ] = None - - @property - def executed(self) -> bool: - return self._result is not None - - def execute( - self, - ) -> Union[ - bigframes.dataframe.DataFrame, - bigframes.series.Series, - pd.Series, - pd.DataFrame, - ]: - """Executes the deferred operation and returns the resulting DataFrame.""" - if self._result is None: - self._result = self._execution_func() - return self._result - - def _repr_mimebundle_(self, include=None, exclude=None): - from bigframes.display.anywidget import TableWidget - - return TableWidget(self)._repr_mimebundle_(include=include, exclude=exclude) # type: ignore - - def __getattr__(self, name: str) -> Any: - raise AttributeError( - f"'{type(self).__name__}' object has no attribute '{name}'. " - "This is a deferred object. Display it to run the query interactively." - ) diff --git a/bigframes/session/direct_gbq_execution.py b/bigframes/session/direct_gbq_execution.py index bcfc29ba971..3ec10bf20f2 100644 --- a/bigframes/session/direct_gbq_execution.py +++ b/bigframes/session/direct_gbq_execution.py @@ -13,182 +13,83 @@ # limitations under the License. from __future__ import annotations -import asyncio -from typing import Literal, Mapping, Optional, Tuple +from typing import Literal, Optional, Tuple -import google.api_core.exceptions +from google.cloud import bigquery import google.cloud.bigquery.job as bq_job import google.cloud.bigquery.table as bq_table -import google.cloud.bigquery_storage_v1 -from google.cloud import bigquery -import bigframes -import bigframes.core.compile +from bigframes.core import compile, nodes +from bigframes.core.compile import sqlglot import bigframes.core.events +from bigframes.session import executor, semi_executor import bigframes.session._io.bigquery as bq_io -import bigframes.session.metrics -from bigframes import exceptions as bfe -from bigframes.core import bq_data, compile, nodes -from bigframes.core.compile.configs import CompileRequest -from bigframes.session import execution_spec, executor, semi_executor - -_WRITE_DISPOSITIONS = { - "fail": bigquery.WriteDisposition.WRITE_EMPTY, - "replace": bigquery.WriteDisposition.WRITE_TRUNCATE, - "append": bigquery.WriteDisposition.WRITE_APPEND, -} +# used only in testing right now, BigQueryCachingExecutor is the fully featured engine +# simplified, doesnt not do large >10 gb result queries, error handling, respect global config +# or record metrics. Also avoids caching, and most pre-compile rewrites, to better serve as a +# reference for validating more complex executors. class DirectGbqExecutor(semi_executor.SemiExecutor): def __init__( self, bqclient: bigquery.Client, - bqstoragereadclient: google.cloud.bigquery_storage_v1.BigQueryReadClient, + compiler: Literal["ibis", "sqlglot"] = "ibis", *, publisher: bigframes.core.events.Publisher, - compiler: Literal["ibis", "sqlglot"] = "sqlglot", - metrics: Optional[bigframes.session.metrics.ExecutionMetrics] = None, - labels: Mapping[str, str] = {}, ): self.bqclient = bqclient - self._compiler_name = compiler - self._bqstoragereadclient = bqstoragereadclient + self._compile_fn = ( + compile.compile_sql if compiler == "ibis" else sqlglot.compile_sql + ) self._publisher = publisher - self._metrics = metrics - self._labels = labels - async def execute( + def execute( self, plan: nodes.BigFrameNode, - spec: execution_spec.ExecutionSpec, + ordered: bool, + peek: Optional[int] = None, ) -> executor.ExecuteResult: """Just execute whatever plan as is, without further caching or decomposition.""" - compiled = compile.compile_sql( - CompileRequest( - plan, - sort_rows=spec.ordered, - peek_count=spec.peek, - ), - compiler_name=self._compiler_name, - ) - job_config = bigquery.QueryJobConfig() - dest_spec = spec.destination_spec - cluster_cols = None - can_skip_job = True - if isinstance(dest_spec, execution_spec.TableOutputSpec): - job_config.destination = dest_spec.table - job_config.write_disposition = _WRITE_DISPOSITIONS[dest_spec.if_exists] - cluster_cols = dest_spec.cluster_cols if dest_spec.cluster_cols else None - job_config.clustering_fields = cluster_cols - can_skip_job = False - elif isinstance(dest_spec, execution_spec.EphemeralTableSpec): - # Need destination table, but jobless execution might not create a destination table - can_skip_job = False - elif dest_spec is not None: - raise ValueError( - f"Direct GBQ Executor does not support destination: {dest_spec}" - ) + # TODO(swast): plumb through the api_name of the user-facing api that + # caused this query. - job_config.labels["bigframes-dtypes"] = compiled.encoded_type_refs - if self._labels: - job_config.labels.update(self._labels) - if spec.bigquery_config is not None: - if spec.bigquery_config.extra_query_labels: - job_config.labels.update(spec.bigquery_config.extra_query_labels) - if spec.bigquery_config.maximum_bytes_billed is not None: - job_config.maximum_bytes_billed = ( - spec.bigquery_config.maximum_bytes_billed - ) + compiled = self._compile_fn( + compile.CompileRequest(plan, sort_rows=ordered, peek_count=peek) + ) - iterator, query_job = await asyncio.to_thread( - self._run_execute_query, + iterator, query_job = self._run_execute_query( sql=compiled.sql, - job_config=job_config, - query_with_job=(not can_skip_job), session=plan.session, - cell_execution_count=spec.cell_execution_count, ) - result_bq_data = None - if query_job and query_job.destination: - dst = query_job.destination - result_bq_data = bq_data.BigqueryDataSource( - table=bq_data.GbqNativeTable.from_ref_and_schema( - dst, - tuple(compiled.sql_schema), - cluster_cols=cluster_cols or (), - location=iterator.location or self.bqclient.location, - table_type="TABLE", - ), - schema=plan.schema, - ordering=compiled.row_order, - n_rows=iterator.total_rows, - ) - execution_metadata = executor.ExecutionMetadata.from_iterator_and_job( - iterator, query_job - ) - result_mostly_cached = ( - hasattr(iterator, "_is_almost_completely_cached") - and iterator._is_almost_completely_cached() + # just immediately downlaod everything for simplicity + return executor.LocalExecuteResult( + data=iterator.to_arrow(), + bf_schema=plan.schema, + execution_metadata=executor.ExecutionMetadata.from_iterator_and_job( + iterator, query_job + ), ) - if (isinstance(dest_spec, execution_spec.EphemeralTableSpec)) or ( - (result_bq_data is not None) and not result_mostly_cached - ): - assert result_bq_data is not None, "expected result table but none exists" - return executor.BQTableExecuteResult( - data=result_bq_data, - project_id=self.bqclient.project, - storage_client=self._bqstoragereadclient, - execution_metadata=execution_metadata, - selected_fields=tuple((col, col) for col in plan.schema.names), - ) - else: - return executor.LocalExecuteResult( - data=iterator.to_arrow().select(plan.schema.names), - bf_schema=plan.schema, - execution_metadata=execution_metadata, - ) - def _run_execute_query( self, sql: str, - job_config: bq_job.QueryJobConfig, - query_with_job: bool, - session, - cell_execution_count: Optional[int] = None, + job_config: Optional[bq_job.QueryJobConfig] = None, + session=None, ) -> Tuple[bq_table.RowIterator, Optional[bigquery.QueryJob]]: """ Starts BigQuery query job and waits for results. """ - try: - if query_with_job: - return bq_io.start_query_with_job( - self.bqclient, - sql, - job_config=job_config, - metrics=self._metrics, - publisher=self._publisher, - session=session, - cell_execution_count=cell_execution_count, - ) - else: - return ( - bq_io.start_query_job_optional( - self.bqclient, - sql, - job_config=job_config, - metrics=self._metrics, - publisher=self._publisher, - session=session, - cell_execution_count=cell_execution_count, - ), - None, - ) - except google.api_core.exceptions.BadRequest as e: - # Unfortunately, this error type does not have a separate error code or exception type - if "Resources exceeded during query execution" in e.message: - new_message = "Computation is too complex to execute as a single query. Try using DataFrame.cache() on intermediate results, or setting bigframes.options.compute.enable_multi_query_execution." - raise bfe.QueryComplexityError(new_message) from e - else: - raise + return bq_io.start_query_with_client( + self.bqclient, + sql, + job_config=job_config or bq_job.QueryJobConfig(), + project=None, + location=None, + timeout=None, + metrics=None, + query_with_job=False, + publisher=self._publisher, + session=session, + ) diff --git a/bigframes/session/dry_runs.py b/bigframes/session/dry_runs.py index 03688b38cd3..bd54bb65d7b 100644 --- a/bigframes/session/dry_runs.py +++ b/bigframes/session/dry_runs.py @@ -14,18 +14,16 @@ from __future__ import annotations import copy -from typing import Any, Dict, List, Sequence, Union +from typing import Any, Dict, List, Sequence -import pandas from google.cloud import bigquery +import pandas from bigframes import dtypes -from bigframes.core import bigframe_node, bq_data, nodes +from bigframes.core import bigframe_node, nodes -def get_table_stats( - table: Union[bq_data.GbqNativeTable, bq_data.BiglakeIcebergTable], -) -> pandas.Series: +def get_table_stats(table: bigquery.Table) -> pandas.Series: values: List[Any] = [] index: List[Any] = [] @@ -34,7 +32,7 @@ def get_table_stats( values.append(False) # Populate column and index types - col_dtypes = dtypes.bf_type_from_type_kind(table.physical_schema) + col_dtypes = dtypes.bf_type_from_type_kind(table.schema) index.append("columnCount") values.append(len(col_dtypes)) index.append("columnDtypes") @@ -42,22 +40,17 @@ def get_table_stats( # Add raw BQ schema index.append("bigquerySchema") - values.append(table.physical_schema) + values.append(table.schema) - index.append("numBytes") - values.append(table.metadata.numBytes) - index.append("numRows") - values.append(table.metadata.numRows) - index.append("location") - values.append(table.metadata.location) - index.append("type") - values.append(table.metadata.type) + for key in ("numBytes", "numRows", "location", "type"): + index.append(key) + values.append(table._properties[key]) index.append("creationTime") - values.append(table.metadata.created_time) + values.append(table.created) index.append("lastModifiedTime") - values.append(table.metadata.modified_time) + values.append(table.modified) return pandas.Series(values, index=index) diff --git a/bigframes/session/execution_cache.py b/bigframes/session/execution_cache.py deleted file mode 100644 index ef4f324afce..00000000000 --- a/bigframes/session/execution_cache.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import dataclasses -import weakref -from typing import Mapping, Optional - -from bigframes.core import bq_data, local_data, nodes - -SourceIdMapping = Mapping[str, str] - - -@dataclasses.dataclass(frozen=True) -class UploadedLocalData: - bq_source: bq_data.BigqueryDataSource - source_mapping: SourceIdMapping - - -class ExecutionCache: - def __init__(self): - # effectively two separate caches that don't interact - self._cached_executions: weakref.WeakKeyDictionary[ - nodes.BigFrameNode, bq_data.BigqueryDataSource - ] = weakref.WeakKeyDictionary() - # This upload cache is entirely independent of the plan cache. - self._uploaded_local_data: weakref.WeakKeyDictionary[ - local_data.ManagedArrowTable, - UploadedLocalData, - ] = weakref.WeakKeyDictionary() - - def subsitute_cached_subplans(self, root: nodes.BigFrameNode) -> nodes.BigFrameNode: - def replace_if_cached(node: nodes.BigFrameNode) -> nodes.BigFrameNode: - if node not in self._cached_executions: - return node - # Assumption: GBQ cached table uses field name as bq column name - scan_list = nodes.ScanList( - tuple(nodes.ScanItem(field.id, field.id.sql) for field in node.fields) - ) - bq_data = self._cached_executions[node] - cached_replacement = nodes.CachedTableNode( - source=bq_data, - scan_list=scan_list, - table_session=node.session, - original_node=node, - ) - assert node.schema == cached_replacement.schema - return cached_replacement - - return nodes.top_down(root, replace_if_cached) - - def cache_results_table( - self, - original_root: nodes.BigFrameNode, - data: bq_data.BigqueryDataSource, - ): - self._cached_executions[original_root] = data - - ## Local data upload caching - def cache_remote_replacement( - self, - local_data: local_data.ManagedArrowTable, - bq_data: bq_data.BigqueryDataSource, - ): - # bq table has one extra column for offsets, those are implicit for local data - assert len(local_data.schema.items) + 1 == len(bq_data.table.physical_schema) - mapping = { - local_data.schema.items[i].column: bq_data.table.physical_schema[i].name - for i in range(len(local_data.schema)) - } - self._uploaded_local_data[local_data] = UploadedLocalData(bq_data, mapping) - - def get_uploaded_local_data( - self, local_data: local_data.ManagedArrowTable - ) -> Optional[UploadedLocalData]: - return self._uploaded_local_data.get(local_data) diff --git a/bigframes/session/execution_spec.py b/bigframes/session/execution_spec.py index 89de6eec902..c9431dbd116 100644 --- a/bigframes/session/execution_spec.py +++ b/bigframes/session/execution_spec.py @@ -15,126 +15,33 @@ from __future__ import annotations import dataclasses -from typing import Literal, Mapping, Optional, Union +from typing import Literal, Optional, Union from google.cloud import bigquery -from bigframes._config import ComputeOptions - - -@dataclasses.dataclass(frozen=True) -class BqComputeOptions: - enable_multi_query_execution: bool = True - maximum_bytes_billed: Optional[int] = None - extra_query_labels: tuple[tuple[str, str], ...] = () - - @classmethod - def from_compute_options(cls, compute_options: ComputeOptions) -> BqComputeOptions: - return cls( - enable_multi_query_execution=compute_options.enable_multi_query_execution, - maximum_bytes_billed=compute_options.maximum_bytes_billed, - extra_query_labels=tuple(compute_options.extra_query_labels.items()), - ) - - def push_labels(self, labels: Mapping[str, str]) -> BqComputeOptions: - return dataclasses.replace( - self, - extra_query_labels=tuple(labels.items()) + self.extra_query_labels, - ) - @dataclasses.dataclass(frozen=True) class ExecutionSpec: - # destination for the result of the operation. Executor may also incidentally create other temporary tables for its own purposes. - destination_spec: Union[ - TableOutputSpec, GcsOutputSpec, EphemeralTableSpec, None - ] = None - # If set, the result will be truncated to the given number of rows. Which N rows is - # implementation dependent and not stable. + destination_spec: Union[TableOutputSpec, GcsOutputSpec, CacheSpec, None] = None peek: Optional[int] = None - # Controls whether output iterator is ordered. Cannot be true if destination is not - # guaranteed to be ordered. - ordered: bool = False + ordered: bool = ( + False # ordered and promise_under_10gb must both be together for bq execution + ) # This is an optimization flag for gbq execution, it doesn't change semantics, but if promise is falsely made, errors may occur promise_under_10gb: bool = False - # BigQuery specific options - bigquery_config: Optional[BqComputeOptions] = None - cell_execution_count: Optional[int] = None - - def with_bq_labels(self, labels: Mapping[str, str]) -> ExecutionSpec: - bq_config = self.bigquery_config or BqComputeOptions() - return dataclasses.replace(self, bigquery_config=bq_config.push_labels(labels)) - - def with_compute_options(self, compute_options: ComputeOptions) -> ExecutionSpec: - """ - Grabs the current global or thread-local config and binds it to the execution spec. - - Returns a new ExecutionSpec with the current configuration applied. - """ - new_bq_config = BqComputeOptions.from_compute_options(compute_options) - if self.bigquery_config: - # merge labels, new ComputeOptions takes priority for everything else - new_bq_config = new_bq_config.push_labels( - dict(self.bigquery_config.extra_query_labels) - ) - - cell_execution_count = self.cell_execution_count - if cell_execution_count is None: - from bigframes.core.utils import get_ipython_execution_count - - cell_execution_count = get_ipython_execution_count() - - return dataclasses.replace( - self, - bigquery_config=new_bq_config, - cell_execution_count=cell_execution_count, - ) - - -# Used internally by execution -@dataclasses.dataclass(frozen=True) -class EphemeralTableSpec: - """ - Specifies that the result of an operation should be a temporary table of some sort. - - No guarantees on lifetime, may be a session temp table, or a bq-created temp table with <24hr life. - - Used internally when results need temporary staging, because they are large (>10GB), or needed in subsequent operations. - """ - - pass - +# This one is temporary, in future, caching will not be done through immediate execution, but will label nodes +# that will be cached only when a super-tree is executed @dataclasses.dataclass(frozen=True) class CacheSpec: - """ - Specifies that the result of an operation should be a session temp table. - The table will be automatically deleted after the session ends. - """ - - cluster_cols: tuple[ - str, ... - ] = () # if empty, will cluster using order key if ordering_key is set - # Controls ordering and whether extra columns are materialized to preserve ordering - # Any extra columns will be appended to the end of the schema. - # None: ordering may be discarded entirely (ordering metadata will still be provided if ordering is derivable from materialized columns) - # order_rows: the result iterator itself will be ordered. For gbq execution, result cannot exceed 10GB. - # order_key: the result set ordered by a key, may materialize extra columns. - # offsets_col: order the result set by an offsets column, materializes one extra column. - ordering: Literal["order_rows", "offsets_col", "order_key"] | None = None + cluster_cols: tuple[str, ...] @dataclasses.dataclass(frozen=True) class TableOutputSpec: - """ - Specifies that the result of an operation should be exported to a specific named table. - - The executor is not responsible for managing lifecycle of the table. - """ - table: bigquery.TableReference - cluster_cols: tuple[str, ...] = () + cluster_cols: tuple[str, ...] if_exists: Literal["fail", "replace", "append"] = "fail" diff --git a/bigframes/session/executor.py b/bigframes/session/executor.py index ba5ac60d74f..2cbf6d8705c 100644 --- a/bigframes/session/executor.py +++ b/bigframes/session/executor.py @@ -20,19 +20,19 @@ import itertools from typing import Iterator, Literal, Optional, Sequence, Union +from google.cloud import bigquery, bigquery_storage_v1 import google.cloud.bigquery.table as bq_table import pandas as pd import pyarrow import pyarrow as pa -from google.cloud import bigquery, bigquery_storage_v1 import bigframes import bigframes.core +from bigframes.core import bq_data, local_data, pyarrow_utils import bigframes.core.schema import bigframes.dtypes import bigframes.session._io.pandas as io_pandas import bigframes.session.execution_spec as ex_spec -from bigframes.core import bq_data, local_data, pyarrow_utils _ROW_LIMIT_EXCEEDED_TEMPLATE = ( "Execution has downloaded {result_rows} rows so far, which exceeds the " @@ -118,9 +118,9 @@ def to_pandas_batches( ) -> Iterator[pd.DataFrame]: assert (page_size is None) or (page_size > 0) assert (max_results is None) or (max_results > 0) - batch_iter: Iterator[Union[pyarrow.Table, pyarrow.RecordBatch]] = ( - self.arrow_batches - ) + batch_iter: Iterator[ + Union[pyarrow.Table, pyarrow.RecordBatch] + ] = self.arrow_batches if max_results is not None: batch_iter = pyarrow_utils.truncate_pyarrow_iterable( batch_iter, max_results @@ -152,14 +152,17 @@ def to_py_scalar(self): class ExecuteResult(abc.ABC): @property @abc.abstractmethod - def execution_metadata(self) -> ExecutionMetadata: ... + def execution_metadata(self) -> ExecutionMetadata: + ... @property @abc.abstractmethod - def schema(self) -> bigframes.core.schema.ArraySchema: ... + def schema(self) -> bigframes.core.schema.ArraySchema: + ... @abc.abstractmethod - def batches(self, sample_rate: Optional[float] = None) -> ResultsIterator: ... + def batches(self, sample_rate: Optional[float] = None) -> ResultsIterator: + ... @property def query_job(self) -> Optional[bigquery.QueryJob]: @@ -300,7 +303,6 @@ class HierarchicalKey: class CacheConfig(abc.ABC): optimize_for: Union[Literal["auto", "head"], HierarchicalKey] = "auto" if_cached: Literal["reuse-strict", "reuse-any", "replace"] = "reuse-any" - enable_multi_query_execution: Optional[bool] = None class Executor(abc.ABC): diff --git a/bigframes/session/iceberg.py b/bigframes/session/iceberg.py deleted file mode 100644 index 0d2539f5554..00000000000 --- a/bigframes/session/iceberg.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import datetime -import json -import urllib.parse -from typing import List - -import google.auth.transport.requests -import google.cloud.bigquery as bq -import pyiceberg -import pyiceberg.schema -import pyiceberg.types -import requests -from pyiceberg.catalog import load_catalog - -from bigframes.core import bq_data - - -def get_table( - user_project_id: str, full_table_id: str, credentials -) -> bq_data.BiglakeIcebergTable: - table_parts = full_table_id.split(".") - if len(table_parts) != 4: - raise ValueError("Iceberg catalog table must contain exactly 4 parts") - - catalog_project_id, catalog_id, namespace, table = table_parts - - credentials.refresh(google.auth.transport.requests.Request()) - token = credentials.token - - base_uri = "https://biglake.googleapis.com/iceberg/v1/restcatalog" - - # Maybe can drop the pyiceberg dependency at some point, but parsing through raw schema json seems a bit painful - catalog = load_catalog( - f"{catalog_project_id}.{catalog_id}", - **{ - "uri": base_uri, - "header.x-goog-user-project": user_project_id, - "oauth2-server-uri": "https://oauth2.googleapis.com/token", - "token": token, - "warehouse": f"gs://{catalog_id}", - }, - ) - - response = requests.get( - f"{base_uri}/extensions/projects/{urllib.parse.quote(catalog_project_id, safe='')}/catalogs/{urllib.parse.quote(catalog_id, safe='')}", - headers={ - "Authorization": f"Bearer {credentials.token}", - "Content-Type": "application/json", - "header.x-goog-user-project": user_project_id, - }, - ) - response.raise_for_status() - location = _extract_location_from_catalog_extension_data(response) - - iceberg_table = catalog.load_table(f"{namespace}.{table}") - bq_schema = pyiceberg.schema.visit(iceberg_table.schema(), SchemaVisitor()) - # TODO: Handle physical layout to help optimize - # TODO: Use snapshot metadata to get row, byte counts - return bq_data.BiglakeIcebergTable( - catalog_project_id, - catalog_id, - namespace, - table, - physical_schema=bq_schema, # type: ignore - cluster_cols=(), - metadata=bq_data.TableMetadata( - location=location, - type="TABLE", - modified_time=datetime.datetime.fromtimestamp( - iceberg_table.metadata.last_updated_ms / 1000.0 - ), - ), - ) - - -def _extract_location_from_catalog_extension_data(data): - catalog_extension_metadata = json.loads(data.text) - storage_region = catalog_extension_metadata["storage-regions"][ - 0 - ] # assumption: exactly 1 region - replicas = tuple(item["region"] for item in catalog_extension_metadata["replicas"]) - return bq_data.GcsRegion(storage_region, replicas) - - -class SchemaVisitor(pyiceberg.schema.SchemaVisitorPerPrimitiveType[bq.SchemaField]): - # Override returns a tuple of fields instead of a single field, violating supertype signature but intentional for this visitor. - def schema( # type: ignore[override] - self, schema: pyiceberg.schema.Schema, struct_result: bq.SchemaField - ) -> tuple[bq.SchemaField, ...]: - return tuple(f for f in struct_result.fields) - - def struct( - self, struct: pyiceberg.types.StructType, field_results: List[bq.SchemaField] - ) -> bq.SchemaField: - return bq.SchemaField("", "RECORD", fields=field_results) - - def field( - self, field: pyiceberg.types.NestedField, field_result: bq.SchemaField - ) -> bq.SchemaField: - return bq.SchemaField( - field.name, - field_result.field_type, - mode=field_result.mode or "NULLABLE", - fields=field_result.fields, - ) - - def map( - self, - map_type: pyiceberg.types.MapType, - key_result: bq.SchemaField, - value_result: bq.SchemaField, - ) -> bq.SchemaField: - return bq.SchemaField("", "UNKNOWN") - - def list( - self, list_type: pyiceberg.types.ListType, element_result: bq.SchemaField - ) -> bq.SchemaField: - return bq.SchemaField( - "", element_result.field_type, mode="REPEATED", fields=element_result.fields - ) - - def visit_fixed(self, fixed_type: pyiceberg.types.FixedType) -> bq.SchemaField: - return bq.SchemaField("", "UNKNOWN") - - def visit_decimal( - self, decimal_type: pyiceberg.types.DecimalType - ) -> bq.SchemaField: - # BIGNUMERIC not supported in iceberg tables yet, so just assume numeric - return bq.SchemaField("", "NUMERIC") - - def visit_boolean( - self, boolean_type: pyiceberg.types.BooleanType - ) -> bq.SchemaField: - return bq.SchemaField("", "NUMERIC") - - def visit_integer( - self, integer_type: pyiceberg.types.IntegerType - ) -> bq.SchemaField: - return bq.SchemaField("", "INTEGER") - - def visit_long(self, long_type: pyiceberg.types.LongType) -> bq.SchemaField: - return bq.SchemaField("", "INTEGER") - - def visit_float(self, float_type: pyiceberg.types.FloatType) -> bq.SchemaField: - # 32-bit IEEE 754 floating point - return bq.SchemaField("", "FLOAT") - - def visit_double(self, double_type: pyiceberg.types.DoubleType) -> bq.SchemaField: - # 64-bit IEEE 754 floating point - return bq.SchemaField("", "FLOAT") - - def visit_date(self, date_type: pyiceberg.types.DateType) -> bq.SchemaField: - # Date encoded as an int - return bq.SchemaField("", "DATE") - - def visit_time(self, time_type: pyiceberg.types.TimeType) -> bq.SchemaField: - return bq.SchemaField("", "TIME") - - def visit_timestamp( - self, timestamp_type: pyiceberg.types.TimestampType - ) -> bq.SchemaField: - return bq.SchemaField("", "DATETIME") - - def visit_timestamp_ns( - self, timestamp_type: pyiceberg.types.TimestampNanoType - ) -> bq.SchemaField: - return bq.SchemaField("", "UNKNOWN") - - def visit_timestamptz( - self, timestamptz_type: pyiceberg.types.TimestamptzType - ) -> bq.SchemaField: - return bq.SchemaField("", "TIMESTAMP") - - def visit_timestamptz_ns( - self, timestamptz_ns_type: pyiceberg.types.TimestamptzNanoType - ) -> bq.SchemaField: - return bq.SchemaField("", "UNKNOWN") - - def visit_string(self, string_type: pyiceberg.types.StringType) -> bq.SchemaField: - return bq.SchemaField("", "STRING") - - def visit_uuid(self, uuid_type: pyiceberg.types.UUIDType) -> bq.SchemaField: - return bq.SchemaField("", "UNKNOWN") - - def visit_unknown( - self, unknown_type: pyiceberg.types.UnknownType - ) -> bq.SchemaField: - """Type `UnknownType` can be promoted to any primitive type in V3+ tables per the Iceberg spec.""" - return bq.SchemaField("", "UNKNOWN") - - def visit_binary(self, binary_type: pyiceberg.types.BinaryType) -> bq.SchemaField: - return bq.SchemaField("", "BINARY") diff --git a/bigframes/session/loader.py b/bigframes/session/loader.py index 43f45a500f0..9c18d727c80 100644 --- a/bigframes/session/loader.py +++ b/bigframes/session/loader.py @@ -25,68 +25,57 @@ import os import threading import typing -import warnings from typing import ( - IO, + cast, Dict, Hashable, + IO, Iterable, Iterator, List, Literal, Optional, + overload, Sequence, Tuple, TypeVar, - Union, - cast, - overload, ) import bigframes_vendored.constants as constants import bigframes_vendored.pandas.io.gbq as third_party_pandas_gbq import google.api_core.exceptions +from google.cloud import bigquery_storage_v1 import google.cloud.bigquery import google.cloud.bigquery as bigquery import google.cloud.bigquery.table +from google.cloud.bigquery_storage_v1 import types as bq_storage_types import pandas import pyarrow as pa -from google.cloud import bigquery_storage_v1 -from google.cloud.bigquery.job.load import LoadJob -from google.cloud.bigquery.job.query import QueryJob -from google.cloud.bigquery_storage_v1 import ( - types as bq_storage_types, -) -from google.cloud.bigquery_storage_v1 import ( - writer as bq_storage_writer, -) import bigframes._tools import bigframes._tools.strings +from bigframes.core import ( + bq_data, + guid, + identifiers, + local_data, + nodes, + ordering, + utils, +) import bigframes.core as core import bigframes.core.blocks as blocks import bigframes.core.events import bigframes.core.schema as schemata import bigframes.dtypes -import bigframes.exceptions as bfe import bigframes.formatting_helpers as formatting_helpers +from bigframes.session import dry_runs import bigframes.session._io.bigquery as bf_io_bigquery import bigframes.session._io.bigquery.read_gbq_query as bf_read_gbq_query import bigframes.session._io.bigquery.read_gbq_table as bf_read_gbq_table -import bigframes.session.iceberg import bigframes.session.metrics import bigframes.session.temporary_storage import bigframes.session.time as session_time -from bigframes.core import ( - bq_data, - guid, - identifiers, - local_data, - nodes, - ordering, - utils, -) -from bigframes.session import dry_runs # Avoid circular imports. if typing.TYPE_CHECKING: @@ -109,8 +98,6 @@ bigframes.dtypes.TIMEDELTA_DTYPE: "INTEGER", } -TABLE_TYPE = Union[bq_data.GbqNativeTable, bq_data.BiglakeIcebergTable] - def _to_index_cols( index_col: Iterable[str] | str | bigframes.enums.DefaultIndexKind = (), @@ -300,24 +287,13 @@ def __init__( self._default_index_type = default_index_type self._scan_index_uniqueness = scan_index_uniqueness self._force_total_order = force_total_order - self._df_snapshot: Dict[str, Tuple[datetime.datetime, TABLE_TYPE]] = {} + self._df_snapshot: Dict[str, Tuple[datetime.datetime, bigquery.Table]] = {} self._metrics = metrics self._publisher = publisher # Unfortunate circular reference, but need to pass reference when constructing objects self._session = session self._clock = session_time.BigQuerySyncedClock(bqclient) self._clock.sync() - self._threadpool = concurrent.futures.ThreadPoolExecutor( - max_workers=1, thread_name_prefix="bigframes-loader" - ) - - def read_data_async( - self, local_data: local_data.ManagedArrowTable, offsets_col: str - ) -> concurrent.futures.Future[bq_data.BigqueryDataSource]: - future = self._threadpool.submit( - self._load_data_or_write_data, local_data, offsets_col - ) - return future def read_pandas( self, @@ -368,38 +344,16 @@ def read_managed_data( session=self._session, ) - def _load_data_or_write_data( - self, - data: local_data.ManagedArrowTable, - offsets_col: str, - ) -> bq_data.BigqueryDataSource: - """Write local data into BigQuery using the local API if possible, - otherwise use the write API.""" - can_load = all( - _is_dtype_can_load(item.column, item.dtype) for item in data.schema.items - ) - if can_load: - return self.load_data(data, offsets_col=offsets_col) - else: - return self.write_data(data, offsets_col=offsets_col) - def load_data( self, data: local_data.ManagedArrowTable, offsets_col: str, ) -> bq_data.BigqueryDataSource: """Load managed data into bigquery""" - cannot_load_columns = { - item.column: item.dtype - for item in data.schema.items - if not _is_dtype_can_load(item.column, item.dtype) - } - - if cannot_load_columns: - raise NotImplementedError( - f"Nested JSON types are currently unsupported for BigQuery Load API. " - f"Unsupported columns: {cannot_load_columns}. {constants.FEEDBACK_LINK}" - ) + + # JSON support incomplete + for item in data.schema.items: + _validate_dtype_can_load(item.column, item.dtype) schema_w_offsets = data.schema.append( schemata.SchemaItem(offsets_col, bigframes.dtypes.INT_DTYPE) @@ -437,7 +391,7 @@ def load_data( # must get table metadata after load job for accurate metadata destination_table = self._bqclient.get_table(load_table_destination) return bq_data.BigqueryDataSource( - bq_data.GbqNativeTable.from_table(destination_table), + bq_data.GbqTable.from_table(destination_table), schema=schema_w_offsets, ordering=ordering.TotalOrdering.from_offset_col(offsets_col), n_rows=data.metadata.row_count, @@ -491,7 +445,7 @@ def stream_data( ) destination_table = self._bqclient.get_table(load_table_destination) return bq_data.BigqueryDataSource( - bq_data.GbqNativeTable.from_table(destination_table), + bq_data.GbqTable.from_table(destination_table), schema=schema_w_offsets, ordering=ordering.TotalOrdering.from_offset_col(offsets_col), n_rows=data.metadata.row_count, @@ -527,51 +481,38 @@ def write_data( ) serialized_schema = schema.serialize().to_pybytes() - def stream_worker( - work: Iterator[pa.RecordBatch], max_outstanding: int = 5 - ) -> str: + def stream_worker(work: Iterator[pa.RecordBatch]) -> str: requested_stream = bq_storage_types.WriteStream( type_=bq_storage_types.WriteStream.Type.PENDING ) stream = self._write_client.create_write_stream( parent=parent, write_stream=requested_stream ) - base_request = bq_storage_types.AppendRowsRequest( - write_stream=stream.name, - ) - base_request.arrow_rows.writer_schema.serialized_schema = serialized_schema - - stream_manager = bq_storage_writer.AppendRowsStream( - client=self._write_client, initial_request_template=base_request - ) stream_name = stream.name - current_offset = 0 - futures: list[bq_storage_writer.AppendRowsFuture] = [] - - for batch in work: - if len(futures) >= max_outstanding: - row_errors = futures.pop(0).result().row_errors - if row_errors: - raise ValueError( - f"Problem loading rows: {row_errors}. {constants.FEEDBACK_LINK}" - ) - request = bq_storage_types.AppendRowsRequest(offset=current_offset) - request.arrow_rows.rows.serialized_record_batch = ( - batch.serialize().to_pybytes() - ) + def request_generator(): + current_offset = 0 + for batch in work: + request = bq_storage_types.AppendRowsRequest( + write_stream=stream.name, offset=current_offset + ) + + request.arrow_rows.writer_schema.serialized_schema = ( + serialized_schema + ) + request.arrow_rows.rows.serialized_record_batch = ( + batch.serialize().to_pybytes() + ) - futures.append(stream_manager.send(request)) - current_offset += batch.num_rows + yield request + current_offset += batch.num_rows - for future in futures: - row_errors = future.result().row_errors - if row_errors: + responses = self._write_client.append_rows(requests=request_generator()) + for resp in responses: + if resp.row_errors: raise ValueError( - f"Problem loading rows: {row_errors}. {constants.FEEDBACK_LINK}" + f"Errors in stream {stream_name}: {resp.row_errors}" ) - - stream_manager.close() self._write_client.finalize_write_stream(name=stream_name) return stream_name @@ -603,12 +544,8 @@ def stream_worker( for error in response.stream_errors: raise ValueError(f"Errors commiting stream {error}") - result_table = bq_data.GbqNativeTable.from_ref_and_schema( - bq_table_ref, - schema=bq_schema, - cluster_cols=[offsets_col], - location=self._storage_manager.location, - table_type="TABLE", + result_table = bq_data.GbqTable.from_ref_and_schema( + bq_table_ref, schema=bq_schema, cluster_cols=[offsets_col] ) return bq_data.BigqueryDataSource( result_table, @@ -625,9 +562,6 @@ def _start_generic_job(self, job: formatting_helpers.GenericJob): else: job.result() - if self._metrics is not None and isinstance(job, (QueryJob, LoadJob)): - self._metrics.count_job_stats(query_job=job) - @overload def read_gbq_table( # type: ignore[overload-overlap] self, @@ -649,7 +583,8 @@ def read_gbq_table( # type: ignore[overload-overlap] n_rows: Optional[int] = None, index_col_in_columns: bool = False, publish_execution: bool = True, - ) -> dataframe.DataFrame: ... + ) -> dataframe.DataFrame: + ... @overload def read_gbq_table( @@ -672,7 +607,8 @@ def read_gbq_table( n_rows: Optional[int] = None, index_col_in_columns: bool = False, publish_execution: bool = True, - ) -> pandas.Series: ... + ) -> pandas.Series: + ... def read_gbq_table( self, @@ -780,33 +716,33 @@ def read_gbq_table( # Fetch table metadata and validate # --------------------------------- - time_travel_timestamp, table = self._get_table_metadata( + time_travel_timestamp, table = bf_read_gbq_table.get_table_metadata( + self._bqclient, table_id=table_id, default_project=self._bqclient.project, bq_time=self._clock.get_time(), + cache=self._df_snapshot, use_cache=use_cache, + publisher=self._publisher, ) - if not bq_data.is_compatible( - table.metadata.location, self._storage_manager.location - ): + if table.location.casefold() != self._storage_manager.location.casefold(): raise ValueError( - f"Current session is in {self._storage_manager.location} but table '{table.get_full_id()}' is located in {table.metadata.location}" + f"Current session is in {self._storage_manager.location} but dataset '{table.project}.{table.dataset_id}' is located in {table.location}" ) - table_column_names = [field.name for field in table.physical_schema] + table_column_names = [field.name for field in table.schema] rename_to_schema: Optional[Dict[str, str]] = None if names is not None: _check_names_param(names, index_col, columns, table_column_names) # Additional unnamed columns is going to set as index columns len_names = len(list(names)) - len_schema = len(table.physical_schema) + len_schema = len(table.schema) if len(columns) == 0 and len_names < len_schema: index_col = range(len_schema - len_names) names = [ - field.name - for field in table.physical_schema[: len_schema - len_names] + field.name for field in table.schema[: len_schema - len_names] ] + list(names) assert len_schema >= len_names @@ -863,7 +799,7 @@ def read_gbq_table( itertools.chain(index_cols, columns) if columns else () ) query = bf_io_bigquery.to_query( - table.get_full_id(quoted=False), + f"{table.project}.{table.dataset_id}.{table.table_id}", columns=all_columns, sql_predicate=bf_io_bigquery.compile_filters(filters) if filters @@ -948,7 +884,7 @@ def read_gbq_table( bigframes.core.events.ExecutionFinished(), ) - selected_cols = None if include_all_columns else (*index_cols, *columns) + selected_cols = None if include_all_columns else index_cols + columns array_value = core.ArrayValue.from_table( table, columns=selected_cols, @@ -1023,90 +959,6 @@ def read_gbq_table( df.sort_index() return df - def _get_table_metadata( - self, - *, - table_id: str, - default_project: Optional[str], - bq_time: datetime.datetime, - use_cache: bool = True, - ) -> Tuple[ - datetime.datetime, Union[bq_data.GbqNativeTable, bq_data.BiglakeIcebergTable] - ]: - """Get the table metadata, either from cache or via REST API.""" - - cached_table = self._df_snapshot.get(table_id) - if use_cache and cached_table is not None: - snapshot_timestamp, table = cached_table - - if bf_read_gbq_table.is_time_travel_eligible( - bqclient=self._bqclient, - table=table, - columns=None, - snapshot_time=snapshot_timestamp, - filter_str=None, - # Don't warn, because that will already have been taken care of. - should_warn=False, - should_dry_run=False, - publisher=self._publisher, - ): - # This warning should only happen if the cached snapshot_time will - # have any effect on bigframes (b/437090788). For example, with - # cached query results, such as after re-running a query, time - # travel won't be applied and thus this check is irrelevent. - # - # In other cases, such as an explicit read_gbq_table(), Cache hit - # could be unexpected. See internal issue 329545805. Raise a - # warning with more information about how to avoid the problems - # with the cache. - msg = bfe.format_message( - f"Reading cached table from {snapshot_timestamp} to avoid " - "incompatibilies with previous reads of this table. To read " - "the latest version, set `use_cache=False` or close the " - "current session with Session.close() or " - "bigframes.pandas.close_session()." - ) - # There are many layers before we get to (possibly) the user's code: - # pandas.read_gbq_table - # -> with_default_session - # -> Session.read_gbq_table - # -> _read_gbq_table - # -> _get_snapshot_sql_and_primary_key - # -> get_snapshot_datetime_and_table_metadata - warnings.warn(msg, category=bfe.TimeTravelCacheWarning, stacklevel=7) - - return cached_table - - if bf_read_gbq_table.is_information_schema(table_id): - client_table = bf_read_gbq_table.get_information_schema_metadata( - bqclient=self._bqclient, - table_id=table_id, - default_project=default_project, - ) - table = bq_data.GbqNativeTable.from_table(client_table) - elif bq_data.is_irc_table(table_id): - table = bigframes.session.iceberg.get_table( - self._bqclient.project, table_id, self._bqclient._credentials - ) - else: - table_ref = google.cloud.bigquery.table.TableReference.from_string( - table_id, default_project=default_project - ) - client_table = self._bqclient.get_table(table_ref) - table = bq_data.GbqNativeTable.from_table(client_table) - - # local time will lag a little bit do to network latency - # make sure it is at least table creation time. - # This is relevant if the table was created immediately before loading it here. - if (table.metadata.created_time is not None) and ( - table.metadata.created_time > bq_time - ): - bq_time = table.metadata.created_time - - cached_table = (bq_time, table) - self._df_snapshot[table_id] = cached_table - return cached_table - def load_file( self, filepath_or_buffer: str | IO["bytes"], @@ -1156,7 +1008,8 @@ def read_gbq_query( # type: ignore[overload-overlap] dry_run: Literal[False] = ..., force_total_order: Optional[bool] = ..., allow_large_results: bool, - ) -> dataframe.DataFrame: ... + ) -> dataframe.DataFrame: + ... @overload def read_gbq_query( @@ -1172,7 +1025,8 @@ def read_gbq_query( dry_run: Literal[True] = ..., force_total_order: Optional[bool] = ..., allow_large_results: bool, - ) -> pandas.Series: ... + ) -> pandas.Series: + ... def read_gbq_query( self, @@ -1442,7 +1296,7 @@ def _prepare_job_config( job_config = bigquery.QueryJobConfig() if job_config is None else job_config if bigframes.options.compute.maximum_bytes_billed is not None: - # Maybe this should be pushed down into start_query_with_job + # Maybe this should be pushed down into start_query_with_client job_config.maximum_bytes_billed = ( bigframes.options.compute.maximum_bytes_billed ) @@ -1462,7 +1316,7 @@ def _start_query_with_job_optional( Do not execute dataframe through this API, instead use the executor. """ job_config = self._prepare_job_config(job_config) - rows = bf_io_bigquery.start_query_job_optional( + rows, _ = bf_io_bigquery.start_query_with_client( self._bqclient, sql, job_config=job_config, @@ -1470,6 +1324,7 @@ def _start_query_with_job_optional( location=None, project=None, metrics=None, + query_with_job=False, publisher=self._publisher, session=self._session, ) @@ -1488,7 +1343,7 @@ def _start_query_with_job( Do not execute dataframe through this API, instead use the executor. """ job_config = self._prepare_job_config(job_config) - _, query_job = bf_io_bigquery.start_query_with_job( + _, query_job = bf_io_bigquery.start_query_with_client( self._bqclient, sql, job_config=job_config, @@ -1496,6 +1351,7 @@ def _start_query_with_job( location=None, project=None, metrics=None, + query_with_job=True, publisher=self._publisher, session=self._session, ) @@ -1524,7 +1380,7 @@ def _transform_read_gbq_configuration(configuration: Optional[dict]) -> dict: return configuration -def _is_dtype_can_load(name: str, column_type: bigframes.dtypes.Dtype) -> bool: +def _validate_dtype_can_load(name: str, column_type: bigframes.dtypes.Dtype): """ Determines whether a datatype is supported by bq load jobs. @@ -1532,19 +1388,23 @@ def _is_dtype_can_load(name: str, column_type: bigframes.dtypes.Dtype) -> bool: we're using a workaround: storing JSON as strings and then parsing them into JSON objects. TODO(b/395912450): Remove workaround solution once b/374784249 got resolved. + + Raises: + NotImplementedError: Type is not yet supported by load jobs. """ # we can handle top-level json, but not nested yet through string conversion if column_type == bigframes.dtypes.JSON_DTYPE: - return True + return if isinstance( column_type, pandas.ArrowDtype ) and bigframes.dtypes.contains_db_dtypes_json_arrow_type( column_type.pyarrow_dtype ): - return False - - return True + raise NotImplementedError( + f"Nested JSON types, found in column `{name}`: `{column_type}`', " + f"are currently unsupported for upload. {constants.FEEDBACK_LINK}" + ) # itertools.batched not available in python <3.12, so we use this instead diff --git a/bigframes/session/local_scan_executor.py b/bigframes/session/local_scan_executor.py index 22007ec5eb7..fee0f557ea7 100644 --- a/bigframes/session/local_scan_executor.py +++ b/bigframes/session/local_scan_executor.py @@ -16,7 +16,7 @@ from typing import Optional from bigframes.core import bigframe_node, rewrite -from bigframes.session import execution_spec, executor, semi_executor +from bigframes.session import executor, semi_executor class LocalScanExecutor(semi_executor.SemiExecutor): @@ -24,20 +24,18 @@ class LocalScanExecutor(semi_executor.SemiExecutor): Executes plans reducible to a arrow table scan. """ - async def execute( + def execute( self, plan: bigframe_node.BigFrameNode, - execution_spec: execution_spec.ExecutionSpec, + ordered: bool, + peek: Optional[int] = None, ) -> Optional[executor.ExecuteResult]: - if execution_spec.destination_spec is not None: - return None - reduced_result = rewrite.try_reduce_to_local_scan(plan) if not reduced_result: return None node, limit = reduced_result - peek = execution_spec.peek + if limit is not None: if peek is None or limit < peek: peek = limit diff --git a/bigframes/session/metrics.py b/bigframes/session/metrics.py index a9a444ecb38..8d43a83d730 100644 --- a/bigframes/session/metrics.py +++ b/bigframes/session/metrics.py @@ -15,161 +15,16 @@ from __future__ import annotations import dataclasses -import datetime import os -from typing import Any, Mapping, Optional, Tuple, Union +from typing import Optional, Tuple import google.cloud.bigquery as bigquery +import google.cloud.bigquery.job as bq_job import google.cloud.bigquery.table as bq_table -from google.cloud.bigquery.job.load import LoadJob -from google.cloud.bigquery.job.query import QueryJob LOGGING_NAME_ENV_VAR = "BIGFRAMES_PERFORMANCE_LOG_NAME" -@dataclasses.dataclass -class JobMetadata: - job_id: Optional[str] = None - query_id: Optional[str] = None - location: Optional[str] = None - project: Optional[str] = None - creation_time: Optional[datetime.datetime] = None - start_time: Optional[datetime.datetime] = None - end_time: Optional[datetime.datetime] = None - duration_seconds: Optional[float] = None - status: Optional[str] = None - total_bytes_processed: Optional[int] = None - total_slot_ms: Optional[int] = None - job_type: Optional[str] = None - error_result: Optional[Mapping[str, Any]] = None - cached: Optional[bool] = None - job_url: Optional[str] = None - query: Optional[str] = None - destination_table: Optional[str] = None - source_uris: Optional[list[str]] = None - input_files: Optional[int] = None - input_bytes: Optional[int] = None - output_rows: Optional[int] = None - source_format: Optional[str] = None - cell_execution_count: Optional[int] = None - - @classmethod - def from_job( - cls, - query_job: Union[QueryJob, LoadJob], - exec_seconds: Optional[float] = None, - cell_execution_count: Optional[int] = None, - ) -> "JobMetadata": - query_text = getattr(query_job, "query", None) - if query_text and len(query_text) > 1024: - query_text = query_text[:1021] + "..." - - job_id = getattr(query_job, "job_id", None) - job_url = None - if job_id: - job_url = ( - f"https://console.cloud.google.com/bigquery?" - f"project={query_job.project}&j=bq:{query_job.location}:" - f"{job_id}&page=queryresults" - ) - - if cell_execution_count is None: - from bigframes.core.utils import get_ipython_execution_count - - cell_execution_count = get_ipython_execution_count() - - metadata = cls( - job_id=query_job.job_id, - location=query_job.location, - project=query_job.project, - creation_time=query_job.created, - start_time=query_job.started, - end_time=query_job.ended, - duration_seconds=exec_seconds, - status=query_job.state, - job_type=query_job.job_type, - error_result=query_job.error_result, - query=query_text, - job_url=job_url, - cell_execution_count=cell_execution_count, - ) - if isinstance(query_job, QueryJob): - metadata.cached = getattr(query_job, "cache_hit", None) - metadata.destination_table = ( - str(query_job.destination) if query_job.destination else None - ) - metadata.total_bytes_processed = getattr( - query_job, "total_bytes_processed", None - ) - metadata.total_slot_ms = getattr(query_job, "slot_millis", None) - elif isinstance(query_job, LoadJob): - metadata.output_rows = getattr(query_job, "output_rows", None) - metadata.input_files = getattr(query_job, "input_files", None) - metadata.input_bytes = getattr(query_job, "input_bytes", None) - metadata.destination_table = ( - str(query_job.destination) - if getattr(query_job, "destination", None) - else None - ) - if getattr(query_job, "source_uris", None): - metadata.source_uris = list(query_job.source_uris) - if query_job.configuration and hasattr( - query_job.configuration, "source_format" - ): - metadata.source_format = query_job.configuration.source_format - - return metadata - - @classmethod - def from_row_iterator( - cls, - row_iterator: bq_table.RowIterator, - exec_seconds: Optional[float] = None, - cell_execution_count: Optional[int] = None, - ) -> "JobMetadata": - query_text = getattr(row_iterator, "query", None) - if query_text and len(query_text) > 1024: - query_text = query_text[:1021] + "..." - - job_id = getattr(row_iterator, "job_id", None) - job_url = None - if job_id: - project = getattr(row_iterator, "project", "") - location = getattr(row_iterator, "location", "") - job_url = ( - f"https://console.cloud.google.com/bigquery?" - f"project={project}&j=bq:{location}:{job_id}&page=queryresults" - ) - - if cell_execution_count is None: - from bigframes.core.utils import get_ipython_execution_count - - cell_execution_count = get_ipython_execution_count() - - # fmt: off - return cls( - job_id=job_id, - query_id=getattr(row_iterator, "query_id", None), - location=getattr(row_iterator, "location", None), - project=getattr(row_iterator, "project", None), - creation_time=getattr(row_iterator, "created", None), - start_time=getattr(row_iterator, "started", None), - end_time=getattr(row_iterator, "ended", None), - duration_seconds=exec_seconds, - status="DONE", - total_bytes_processed=getattr( - row_iterator, "total_bytes_processed", None - ), - total_slot_ms=getattr(row_iterator, "slot_millis", None), - job_type="query", - cached=getattr(row_iterator, "cache_hit", None), - query=query_text, - job_url=job_url, - cell_execution_count=cell_execution_count, - ) - # fmt: on - - @dataclasses.dataclass class ExecutionMetrics: execution_count: int = 0 @@ -177,24 +32,17 @@ class ExecutionMetrics: bytes_processed: int = 0 execution_secs: float = 0 query_char_count: int = 0 - jobs: list[JobMetadata] = dataclasses.field(default_factory=list) - # fmt: off def count_job_stats( self, - query_job: Optional[Union[QueryJob, LoadJob]] = None, + query_job: Optional[bq_job.QueryJob] = None, row_iterator: Optional[bq_table.RowIterator] = None, - *, - cell_execution_count: Optional[int] = None, ): if query_job is None: assert row_iterator is not None - # TODO(tswast): Pass None after making benchmark publishing robust - # to missing data. - bytes_processed = ( - getattr(row_iterator, "total_bytes_processed", 0) or 0 - ) + # TODO(tswast): Pass None after making benchmark publishing robust to missing data. + bytes_processed = getattr(row_iterator, "total_bytes_processed", 0) or 0 query_char_count = len(getattr(row_iterator, "query", "") or "") slot_millis = getattr(row_iterator, "slot_millis", 0) or 0 created = getattr(row_iterator, "created", None) @@ -209,126 +57,41 @@ def count_job_stats( self.slot_millis += slot_millis self.execution_secs += exec_seconds - self.jobs.append( - JobMetadata.from_row_iterator( - row_iterator, - exec_seconds=exec_seconds, - cell_execution_count=cell_execution_count, - ) - ) - - elif ( - isinstance(query_job, QueryJob) - and query_job.configuration.dry_run - ): - query_char_count = len(getattr(query_job, "query", "")) + elif query_job.configuration.dry_run: + query_char_count = len(query_job.query) - # TODO(tswast): Pass None after making benchmark publishing robust - # to missing data. + # TODO(tswast): Pass None after making benchmark publishing robust to missing data. bytes_processed = 0 slot_millis = 0 exec_seconds = 0.0 - elif isinstance(query_job, bigquery.QueryJob): - if (stats := get_performance_stats(query_job)) is not None: - ( - query_char_count, - bytes_processed, - slot_millis, - exec_seconds, - ) = stats - self.execution_count += 1 - self.query_char_count += query_char_count or 0 - self.bytes_processed += bytes_processed or 0 - self.slot_millis += slot_millis or 0 - self.execution_secs += exec_seconds or 0 - - metadata = JobMetadata.from_job( - query_job, - exec_seconds=exec_seconds, - cell_execution_count=cell_execution_count, - ) - self.jobs.append(metadata) - - else: + elif (stats := get_performance_stats(query_job)) is not None: + query_char_count, bytes_processed, slot_millis, exec_seconds = stats self.execution_count += 1 - duration = ( - (query_job.ended - query_job.created).total_seconds() - if query_job.ended and query_job.created - else None - ) - self.jobs.append( - JobMetadata.from_job( - query_job, - exec_seconds=duration, - cell_execution_count=cell_execution_count, - ) - ) - - # For pytest runs only, log information about the query job - # to a file in order to create a performance report. - if ( - isinstance(query_job, bigquery.QueryJob) - and not query_job.configuration.dry_run - ): - stats = get_performance_stats(query_job) - if stats: - write_stats_to_disk( - query_char_count=stats[0], - bytes_processed=stats[1], - slot_millis=stats[2], - exec_seconds=stats[3], - ) - elif row_iterator is not None: - bytes_processed = ( - getattr(row_iterator, "total_bytes_processed", 0) or 0 - ) - query_char_count = len(getattr(row_iterator, "query", "") or "") - slot_millis = getattr(row_iterator, "slot_millis", 0) or 0 - created = getattr(row_iterator, "created", None) - ended = getattr(row_iterator, "ended", None) - exec_seconds = ( - (ended - created).total_seconds() if created and ended else 0.0 - ) + self.query_char_count += query_char_count or 0 + self.bytes_processed += bytes_processed or 0 + self.slot_millis += slot_millis or 0 + self.execution_secs += exec_seconds or 0 write_stats_to_disk( query_char_count=query_char_count, bytes_processed=bytes_processed, slot_millis=slot_millis, exec_seconds=exec_seconds, ) - # fmt: on - - def on_event(self, envelope: Any): - try: - import bigframes.core.events - from bigframes.session.executor import LocalExecuteResult - except ImportError: - return - - # Publisher.publish automatically wraps raw Event objects in an - # EventEnvelope, ensuring subscribers receive a consistent contract. - assert isinstance(envelope, bigframes.core.events.EventEnvelope) - event = envelope.event - cell_execution_count = envelope.cell_execution_count - - if isinstance(event, bigframes.core.events.ExecutionFinished): - if event.result and isinstance(event.result, LocalExecuteResult): - self.execution_count += 1 - bytes_processed = event.result.total_bytes_processed or 0 - self.bytes_processed += bytes_processed - if cell_execution_count is None: - from bigframes.core.utils import get_ipython_execution_count - - cell_execution_count = get_ipython_execution_count() + else: + # TODO(tswast): Pass None after making benchmark publishing robust to missing data. + bytes_processed = 0 + query_char_count = 0 + slot_millis = 0 + exec_seconds = 0 - metadata = JobMetadata( - job_type="polars", - status="DONE", - total_bytes_processed=bytes_processed, - cell_execution_count=cell_execution_count, - ) - self.jobs.append(metadata) + write_stats_to_disk( + query_char_count=query_char_count, + bytes_processed=bytes_processed, + slot_millis=slot_millis, + exec_seconds=exec_seconds, + ) def get_performance_stats( diff --git a/bigframes/session/polars_executor.py b/bigframes/session/polars_executor.py index f757de130ce..575beff8fc0 100644 --- a/bigframes/session/polars_executor.py +++ b/bigframes/session/polars_executor.py @@ -14,9 +14,8 @@ from __future__ import annotations import itertools -from typing import TYPE_CHECKING, Optional +from typing import Optional, TYPE_CHECKING -import bigframes.operations from bigframes.core import ( agg_expressions, array_value, @@ -24,6 +23,7 @@ expression, nodes, ) +import bigframes.operations from bigframes.operations import aggregations as agg_ops from bigframes.operations import ( bool_ops, @@ -34,7 +34,7 @@ numeric_ops, string_ops, ) -from bigframes.session import execution_spec, executor, semi_executor +from bigframes.session import executor, semi_executor if TYPE_CHECKING: import polars as pl @@ -79,8 +79,6 @@ numeric_ops.SubOp, numeric_ops.MulOp, numeric_ops.DivOp, - numeric_ops.CeilOp, - numeric_ops.FloorOp, numeric_ops.FloorDivOp, numeric_ops.ModOp, generic_ops.AsTypeOp, @@ -122,7 +120,7 @@ def _is_node_polars_executable(node: nodes.BigFrameNode): return False for expr in node._node_expressions: if isinstance(expr, agg_expressions.Aggregation): - if type(expr.op) not in _COMPATIBLE_AGG_OPS: + if not type(expr.op) in _COMPATIBLE_AGG_OPS: return False if isinstance(expr, expression.Expression): if not set(map(type, _get_expr_ops(expr))).issubset(_COMPATIBLE_SCALAR_OPS): @@ -137,25 +135,24 @@ def __init__(self): self._compiler = PolarsCompiler() - async def execute( + def execute( self, plan: bigframe_node.BigFrameNode, - execution_spec: execution_spec.ExecutionSpec, + ordered: bool, + peek: Optional[int] = None, ) -> Optional[executor.ExecuteResult]: if not self._can_execute(plan): return None - if execution_spec.destination_spec is not None: - return None + # Note: Ignoring ordered flag, as just executing totally ordered is fine. try: lazy_frame: pl.LazyFrame = self._compiler.compile( array_value.ArrayValue(plan).node ) except Exception: return None - if execution_spec.peek is not None: - lazy_frame = lazy_frame.limit(execution_spec.peek) - pl_df = await lazy_frame.collect_async() - pa_table = pl_df.to_arrow() + if peek is not None: + lazy_frame = lazy_frame.limit(peek) + pa_table = lazy_frame.collect().to_arrow() return executor.LocalExecuteResult( data=pa_table, bf_schema=plan.schema, diff --git a/bigframes/session/proxy_executor.py b/bigframes/session/proxy_executor.py deleted file mode 100644 index f6c914790cb..00000000000 --- a/bigframes/session/proxy_executor.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import uuid -import warnings -from typing import Optional - -import google.cloud.bigquery as bigquery -import google.cloud.exceptions - -import bigframes.core -import bigframes.functions._function_session as bff_session -from bigframes import exceptions as bfe -from bigframes.session import ( - bq_caching_executor, - execution_cache, - execution_spec, - executor, - loader, - temporary_storage, -) - -_COMPILER_LABEL_KEY = "bigframes-compiler" - - -class DualCompilerProxyExecutor(executor.Executor): - """ - Used to rollout new compiler implementation. - """ - - def __init__( - self, - bqclient: bigquery.Client, - storage_manager: temporary_storage.TemporaryStorageManager, - bqstoragereadclient: google.cloud.bigquery_storage_v1.BigQueryReadClient, - loader: loader.GbqDataLoader, - *, - metrics: Optional[bigframes.session.metrics.ExecutionMetrics] = None, - enable_polars_execution: bool = False, - publisher: bigframes.core.events.Publisher, - function_manager: bff_session.FunctionSession, - labels: tuple[tuple[str, str], ...] = (), - ): - self._enable_polars_execution = enable_polars_execution - shared_cache = execution_cache.ExecutionCache() - self._ibis_executor = bq_caching_executor.BigQueryCachingExecutor( - bqclient, - storage_manager, - bqstoragereadclient, - loader, - metrics=metrics, - enable_polars_execution=self._enable_polars_execution, - publisher=publisher, - labels=labels, - cache=shared_cache, - compiler_name="ibis", - function_manager=function_manager, - ) - self._sqlglot_executor = bq_caching_executor.BigQueryCachingExecutor( - bqclient, - storage_manager, - bqstoragereadclient, - loader, - metrics=metrics, - enable_polars_execution=self._enable_polars_execution, - publisher=publisher, - labels=labels, - cache=shared_cache, - compiler_name="sqlglot", - function_manager=function_manager, - ) - - def to_sql( - self, - array_value: bigframes.core.ArrayValue, - offset_column: Optional[str] = None, - ordered: bool = False, - enable_cache: bool = True, - ) -> str: - """ - Convert an ArrayValue to a sql query that will yield its value. - """ - compiler_option = bigframes.options.experiments.sql_compiler - # Use ibis unless sqlglot explicitly selected, since we can't handle errors resulting - # from use of the sql produced by this method. - if compiler_option == "experimental": - return self._sqlglot_executor.to_sql( - array_value, - offset_column=offset_column, - ordered=ordered, - enable_cache=enable_cache, - ) - # stable or legacy use ibis - # TODO(b/510408650): Use sqlglot by default. - return self._ibis_executor.to_sql( - array_value, - offset_column=offset_column, - ordered=ordered, - enable_cache=enable_cache, - ) - - def execute( - self, - array_value: bigframes.core.ArrayValue, - execution_spec: execution_spec.ExecutionSpec, - ) -> executor.ExecuteResult: - compiler_option = bigframes.options.experiments.sql_compiler - if compiler_option == "legacy": - return self._ibis_executor.execute( - array_value, - execution_spec.with_bq_labels({_COMPILER_LABEL_KEY: "ibis"}), - ) - elif compiler_option == "experimental": - return self._sqlglot_executor.execute( - array_value, - execution_spec.with_bq_labels({_COMPILER_LABEL_KEY: "sqlglot"}), - ) - else: # stable - correlation_id = f"{uuid.uuid1().hex[:12]}" - try: - return self._sqlglot_executor.execute( - array_value, - execution_spec.with_bq_labels( - {_COMPILER_LABEL_KEY: f"sqlglot-{correlation_id}"} - ), - ) - except Exception as e: - msg = bfe.format_message( - f"Compiler ID {correlation_id}: Exception on sqlglot. " - f"Falling back to ibis. Details: {e}" - ) - warnings.warn(msg, category=UserWarning) - return self._ibis_executor.execute( - array_value, - execution_spec.with_bq_labels( - {_COMPILER_LABEL_KEY: f"ibis-{correlation_id}"} - ), - ) - - def dry_run( - self, array_value: bigframes.core.ArrayValue, ordered: bool = True - ) -> bigquery.QueryJob: - """ - Dry run executing the ArrayValue. - - Does not actually execute the data but will get stats and indicate any invalid query errors. - """ - # TODO(b/510408650): Use sqlglot for dry runs when sqlglot has been validated. - return self._ibis_executor.dry_run(array_value, ordered=ordered) - - def cached( - self, - array_value: bigframes.core.ArrayValue, - *, - config: executor.CacheConfig, - ) -> None: - compiler_option = bigframes.options.experiments.sql_compiler - if compiler_option == "legacy": - return self._ibis_executor.cached(array_value, config=config) - elif compiler_option == "experimental": - return self._sqlglot_executor.cached(array_value, config=config) - else: # stable - correlation_id = f"{uuid.uuid1().hex[:12]}" - try: - return self._sqlglot_executor.cached(array_value, config=config) - except Exception as e: - msg = bfe.format_message( - f"Compiler ID {correlation_id}: Exception on sqlglot. " - f"Falling back to ibis. Details: {e}" - ) - warnings.warn(msg, category=UserWarning) - return self._ibis_executor.cached( - array_value, - config=config, - ) diff --git a/bigframes/session/read_api_execution.py b/bigframes/session/read_api_execution.py index fff8022e40a..c7138f7b307 100644 --- a/bigframes/session/read_api_execution.py +++ b/bigframes/session/read_api_execution.py @@ -17,8 +17,8 @@ from google.cloud import bigquery_storage_v1 -from bigframes.core import bigframe_node, bq_data, nodes, rewrite -from bigframes.session import execution_spec, executor, semi_executor +from bigframes.core import bigframe_node, nodes, rewrite +from bigframes.session import executor, semi_executor class ReadApiSemiExecutor(semi_executor.SemiExecutor): @@ -34,28 +34,22 @@ def __init__( self.bqstoragereadclient = bqstoragereadclient self.project = project - async def execute( + def execute( self, plan: bigframe_node.BigFrameNode, - execution_spec: execution_spec.ExecutionSpec, + ordered: bool, + peek: Optional[int] = None, ) -> Optional[executor.ExecuteResult]: - if execution_spec.destination_spec is not None: - return None - - adapt_result = self._try_adapt_plan(plan, execution_spec.ordered) + adapt_result = self._try_adapt_plan(plan, ordered) if not adapt_result: return None node, limit = adapt_result - if node.explicitly_ordered and execution_spec.ordered: - return None - - if not isinstance(node.source.table, bq_data.GbqNativeTable): + if node.explicitly_ordered and ordered: return None if not node.source.table.is_physically_stored: return None - peek = execution_spec.peek if limit is not None: if peek is None or limit < peek: peek = limit diff --git a/bigframes/session/semi_executor.py b/bigframes/session/semi_executor.py index 1f827ce9d93..c41d7c96d3e 100644 --- a/bigframes/session/semi_executor.py +++ b/bigframes/session/semi_executor.py @@ -15,7 +15,7 @@ from typing import Optional from bigframes.core import bigframe_node -from bigframes.session import execution_spec, executor +from bigframes.session import executor # Unstable interface, in development @@ -24,9 +24,10 @@ class SemiExecutor(abc.ABC): A semi executor executes a subset of possible plans, returns None for unsupported plans. """ - async def execute( + def execute( self, plan: bigframe_node.BigFrameNode, - execution_spec: execution_spec.ExecutionSpec, + ordered: bool, + peek: Optional[int] = None, ) -> Optional[executor.ExecuteResult]: raise NotImplementedError("execute not implemented for this executor") diff --git a/bigframes/session/temporary_storage.py b/bigframes/session/temporary_storage.py index 42617c8f6c1..0c2a36f3fed 100644 --- a/bigframes/session/temporary_storage.py +++ b/bigframes/session/temporary_storage.py @@ -19,11 +19,14 @@ class TemporaryStorageManager(Protocol): @property - def location(self) -> str: ... + def location(self) -> str: + ... def create_temp_table( self, schema: Sequence[bigquery.SchemaField], cluster_cols: Sequence[str] = [] - ) -> bigquery.TableReference: ... + ) -> bigquery.TableReference: + ... # implementations should be robust to repeatedly closing - def close(self) -> None: ... + def close(self) -> None: + ... diff --git a/bigframes/session/time.py b/bigframes/session/time.py index 1452b2952dc..bef4bbc17f5 100644 --- a/bigframes/session/time.py +++ b/bigframes/session/time.py @@ -15,7 +15,7 @@ import datetime import threading import time -from typing import Optional, cast +from typing import cast, Optional import google.cloud.bigquery as bigquery diff --git a/bigframes/streaming/__init__.py b/bigframes/streaming/__init__.py index 49687090fe6..0d91e5f91a2 100644 --- a/bigframes/streaming/__init__.py +++ b/bigframes/streaming/__init__.py @@ -18,10 +18,10 @@ import sys import bigframes.core.global_session as global_session -import bigframes.session -import bigframes.streaming.dataframe as streaming_dataframe from bigframes.core.logging import log_adapter from bigframes.pandas.io.api import _set_default_session_location_if_possible +import bigframes.session +import bigframes.streaming.dataframe as streaming_dataframe def read_gbq_table(table: str) -> streaming_dataframe.StreamingDataFrame: diff --git a/bigframes/streaming/dataframe.py b/bigframes/streaming/dataframe.py index 98d6da45399..b7b67178cea 100644 --- a/bigframes/streaming/dataframe.py +++ b/bigframes/streaming/dataframe.py @@ -13,25 +13,24 @@ # limitations under the License. """Module for bigquery continuous queries""" - from __future__ import annotations +from abc import abstractmethod +from datetime import date, datetime import functools import inspect import json -import warnings -from abc import abstractmethod -from datetime import date, datetime from typing import Optional, Union +import warnings -import pandas as pd from google.cloud import bigquery +import pandas as pd -import bigframes.exceptions as bfe -import bigframes.session from bigframes import dataframe from bigframes.core import nodes from bigframes.core.logging import log_adapter +import bigframes.exceptions as bfe +import bigframes.session def _return_type_wrapper(method, cls): @@ -252,7 +251,7 @@ def _from_table_df(cls, df: dataframe.DataFrame) -> StreamingDataFrame: def _original_table(self): def traverse(node: nodes.BigFrameNode): if isinstance(node, nodes.ReadTableNode): - return node.source.table.get_full_id(quoted=False) + return f"{node.source.table.project_id}.{node.source.table.dataset_id}.{node.source.table.table_id}" for child in node.child_nodes: original_table = traverse(child) if original_table: diff --git a/bigframes/testing/__init__.py b/bigframes/testing/__init__.py index 098a67bddf3..529c08241d7 100644 --- a/bigframes/testing/__init__.py +++ b/bigframes/testing/__init__.py @@ -17,5 +17,3 @@ These modules are provided for testing the BigQuery DataFrames package. The interface is not considered stable. """ - -# Do not import modules contains pytest. (b/490160312) diff --git a/bigframes/testing/engine_utils.py b/bigframes/testing/engine_utils.py index 385ca7e45cc..edb68c3a9b0 100644 --- a/bigframes/testing/engine_utils.py +++ b/bigframes/testing/engine_utils.py @@ -12,16 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio - import pandas.testing from bigframes.core import nodes -from bigframes.session import execution_spec, semi_executor - -SPEC = execution_spec.ExecutionSpec( - ordered=True, -) +from bigframes.session import semi_executor def assert_equivalence_execution( @@ -29,8 +23,8 @@ def assert_equivalence_execution( engine1: semi_executor.SemiExecutor, engine2: semi_executor.SemiExecutor, ): - e1_result = asyncio.run(engine1.execute(node, SPEC)) - e2_result = asyncio.run(engine2.execute(node, SPEC)) + e1_result = engine1.execute(node, ordered=True) + e2_result = engine2.execute(node, ordered=True) assert e1_result is not None assert e2_result is not None # Convert to pandas, as pandas has better comparison utils than arrow diff --git a/bigframes/testing/mocks.py b/bigframes/testing/mocks.py index f8ad43dd664..ff210419fde 100644 --- a/bigframes/testing/mocks.py +++ b/bigframes/testing/mocks.py @@ -14,15 +14,15 @@ import copy import datetime -import unittest.mock as mock from typing import Any, Dict, Literal, Optional, Sequence +import unittest.mock as mock +from bigframes_vendored.google_cloud_bigquery import _pandas_helpers import google.auth.credentials import google.cloud.bigquery import google.cloud.bigquery.table import pyarrow import pytest -from bigframes_vendored.google_cloud_bigquery import _pandas_helpers import bigframes import bigframes.clients @@ -148,7 +148,6 @@ def query_and_wait_mock(query, *args, job_config=None, **kwargs): clients_provider = mock.create_autospec(bigframes.session.clients.ClientsProvider) type(clients_provider).bqclient = mock.PropertyMock(return_value=bqclient) clients_provider._credentials = credentials - clients_provider.project = anonymous_dataset.project bqoptions = bigframes.BigQueryOptions( credentials=credentials, diff --git a/bigframes/testing/polars_session.py b/bigframes/testing/polars_session.py index 2806dab53f9..ca1fa329a2e 100644 --- a/bigframes/testing/polars_session.py +++ b/bigframes/testing/polars_session.py @@ -13,8 +13,8 @@ # limitations under the License. import dataclasses -import weakref from typing import Union +import weakref import pandas import polars @@ -26,7 +26,6 @@ import bigframes.session.execution_spec import bigframes.session.executor import bigframes.session.metrics -from bigframes.functions import _utils, function, udf_def # Does not support to_sql, dry_run, peek, cached @@ -112,29 +111,6 @@ def read_pandas(self, pandas_dataframe, write_engine="default"): return bf_df - def udf( - self, - *, - input_types=None, - output_type=None, - **kwargs, - ): - def wrapper(func): - udf_sig = _utils.get_func_signature( - func, - input_types, - output_type, - ) - - code_def = udf_def.CodeDef.from_func(func) - udf_definition = udf_def.PythonUdf( - signature=udf_sig, - code=code_def, - ) - return function.UdfRoutine(func=func, _udf_def=udf_definition) - - return wrapper - @property def bqclient(self): # prevents logger from trying to call bq upon any errors diff --git a/bigframes/testing/utils.py b/bigframes/testing/utils.py index 79e99968f58..6679f53b2ce 100644 --- a/bigframes/testing/utils.py +++ b/bigframes/testing/utils.py @@ -15,22 +15,23 @@ import base64 import decimal import re -from typing import Iterable, Optional, Sequence, Set, TypeVar, Union +from typing import Iterable, Optional, Sequence, Set, Union import geopandas as gpd # type: ignore import google.api_core.operation +from google.cloud import bigquery, functions_v2 +from google.cloud.functions_v2.types import functions import numpy as np import pandas as pd import pandas.api.types as pd_types import pyarrow as pa # type: ignore import pytest -from google.cloud import bigquery, functions_v2 -from google.cloud.functions_v2.types import functions -import bigframes.functions._utils as bff_utils -import bigframes.pandas as bpd from bigframes import operations as ops from bigframes.core import expression as ex +import bigframes.dtypes +import bigframes.functions._utils as bff_utils +import bigframes.pandas as bpd ML_REGRESSION_METRICS = [ "mean_absolute_error", @@ -68,8 +69,6 @@ "content", ] -SeriesOrIndexT = TypeVar("SeriesOrIndexT", pd.Series, pd.Index) - def pandas_major_version() -> int: match = re.search(r"^v?(\d+)", pd.__version__.strip()) @@ -91,38 +90,19 @@ def assert_series_equivalent(pd_series: pd.Series, bf_series: bpd.Series, **kwar def _normalize_all_nulls(col: pd.Series) -> pd.Series: - if pd_types.is_float_dtype(col.dtype): - col = col.astype("float64").astype("Float64") - elif col.dtype == "object": - if any(isinstance(x, decimal.Decimal) for x in col): - pass - else: - try: - col = col.astype("Float64") - except (TypeError, ValueError, SystemError): - pass + if col.dtype in (bigframes.dtypes.FLOAT_DTYPE, bigframes.dtypes.INT_DTYPE): + col = col.astype("float64") + if pd_types.is_object_dtype(col): + col = col.fillna(float("nan")) return col -def _normalize_index_nulls(idx: pd.Index) -> pd.Index: - if isinstance(idx, pd.MultiIndex): - new_levels = [ - _normalize_index_nulls(idx.get_level_values(i)) for i in range(idx.nlevels) - ] - return pd.MultiIndex.from_arrays(new_levels, names=idx.names) - if idx.hasnans: - if pd_types.is_float_dtype(idx.dtype): - idx = idx.astype("float64").astype("Float64") - return idx - - def assert_frame_equal( left: pd.DataFrame, right: pd.DataFrame, *, ignore_order: bool = False, nulls_are_nan: bool = True, - downcast_object: bool = True, **kwargs, ): if ignore_order: @@ -138,17 +118,9 @@ def assert_frame_equal( left = left.sort_index() right = right.sort_index() - # Pandas sometimes likes to produce object dtype columns - # However, nan/None/Null inconsistency makes comparison futile, convert to typed column - if downcast_object: - left = left.apply(lambda x: x.infer_objects()) - right = right.apply(lambda x: x.infer_objects()) - if nulls_are_nan: left = left.apply(_normalize_all_nulls) right = right.apply(_normalize_all_nulls) - left.index = _normalize_index_nulls(left.index) - right.index = _normalize_index_nulls(right.index) pd.testing.assert_frame_equal(left, right, **kwargs) @@ -179,20 +151,12 @@ def assert_series_equal( right.index = right.index.astype("Int64") if nulls_are_nan: - left = _normalize_all_nulls(left.infer_objects()) - right = _normalize_all_nulls(right.infer_objects()) - left.index = _normalize_index_nulls(left.index) - right.index = _normalize_index_nulls(right.index) - left.name = pd.NA if pd.isna(left.name) else left.name # type: ignore - right.name = pd.NA if pd.isna(right.name) else right.name # type: ignore + left = _normalize_all_nulls(left) + right = _normalize_all_nulls(right) pd.testing.assert_series_equal(left, right, **kwargs) -def assert_index_equal(left, right, **kwargs): - pd.testing.assert_index_equal(left, right, **kwargs) - - def _standardize_index(idx): return pd.Index(list(idx), name=idx.name) @@ -436,11 +400,11 @@ def get_cloud_functions( ) -> Iterable[functions.ListFunctionsResponse]: """Get the cloud functions in the given project and location.""" - assert not name or not name_prefix, ( - "Either 'name' or 'name_prefix' can be passed but not both." - ) + assert ( + not name or not name_prefix + ), "Either 'name' or 'name_prefix' can be passed but not both." - location = bff_utils.gcf_location_from_bq_location(location) + _, location = bff_utils.get_remote_function_locations(location) parent = f"projects/{project}/locations/{location}" request = functions_v2.ListFunctionsRequest(parent=parent) page_result = functions_client.list_functions(request=request) @@ -516,6 +480,20 @@ def cleanup_function_assets( pass +def get_function_name(func, package_requirements=None, is_row_processor=False): + """Get a bigframes function name for testing given a udf.""" + # Augment user package requirements with any internal package + # requirements. + package_requirements = bff_utils.get_updated_package_requirements( + package_requirements, is_row_processor + ) + + # Compute a unique hash representing the user code. + function_hash = bff_utils.get_hash(func, package_requirements) + + return f"bigframes_{function_hash}" + + def _apply_ops_to_sql( obj: bpd.DataFrame, ops_list: Sequence[ex.Expression], diff --git a/bigframes/version.py b/bigframes/version.py index 3eecebee5a1..f36c6789c1a 100644 --- a/bigframes/version.py +++ b/bigframes/version.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.48.0" +__version__ = "2.32.0" # {x-release-please-start-date} -__release_date__ = "2026-06-12" +__release_date__ = "2026-01-05" # {x-release-please-end} diff --git a/conftest.py b/conftest.py index 5d3f116b521..e0f059fa432 100644 --- a/conftest.py +++ b/conftest.py @@ -29,7 +29,7 @@ warnings.simplefilter("ignore", pd.errors.SettingWithCopyWarning) -@pytest.fixture() +@pytest.fixture(scope="session") def polars_session_or_bpd(): # Since the doctest imports fixture is autouse=True, don't skip if polars # isn't available. diff --git a/docs/README.rst b/docs/README.rst deleted file mode 100644 index a3aef5380bb..00000000000 --- a/docs/README.rst +++ /dev/null @@ -1,94 +0,0 @@ -BigQuery DataFrames (BigFrames) -=============================== - - -|GA| |pypi| |versions| - -BigQuery DataFrames (also known as BigFrames) provides a Pythonic DataFrame -and machine learning (ML) API powered by the BigQuery engine. It provides modules -for many use cases, including: - -* `bigframes.pandas `_ - is a pandas API for analytics. Many workloads can be - migrated from pandas to bigframes by just changing a few imports. -* `bigframes.ml `_ - is a scikit-learn-like API for ML. -* `bigframes.bigquery.ai `_ - are a collection of powerful AI methods, powered by Gemini. - -BigQuery DataFrames is an `open-source package `_. - -.. |GA| image:: https://img.shields.io/badge/support-GA-gold.svg - :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#general-availability -.. |pypi| image:: https://img.shields.io/pypi/v/bigframes.svg - :target: https://pypi.org/project/bigframes/ -.. |versions| image:: https://img.shields.io/pypi/pyversions/bigframes.svg - :target: https://pypi.org/project/bigframes/ - -Getting started with BigQuery DataFrames ----------------------------------------- - -The easiest way to get started is to try the -`BigFrames quickstart `_ -in a `notebook in BigQuery Studio `_. - -To use BigFrames in your local development environment, - -1. Run ``pip install --upgrade bigframes`` to install the latest version. - -2. Setup `Application default credentials `_ - for your local development environment enviroment. - -3. Create a `GCP project with the BigQuery API enabled `_. - -4. Use the ``bigframes`` package to query data. - -.. code-block:: python - - import bigframes.pandas as bpd - - bpd.options.bigquery.project = your_gcp_project_id # Optional in BQ Studio. - bpd.options.bigquery.ordering_mode = "partial" # Recommended for performance. - df = bpd.read_gbq("bigquery-public-data.usa_names.usa_1910_2013") - print( - df.groupby("name") - .agg({"number": "sum"}) - .sort_values("number", ascending=False) - .head(10) - .to_pandas() - ) - -Documentation -------------- - -To learn more about BigQuery DataFrames, visit these pages - -* `Introduction to BigQuery DataFrames (BigFrames) `_ -* `Sample notebooks `_ -* `API reference `_ -* `Source code (GitHub) `_ - -License -------- - -BigQuery DataFrames is distributed with the `Apache-2.0 license -`_. - -It also contains code derived from the following third-party packages: - -* `Ibis `_ -* `pandas `_ -* `Python `_ -* `scikit-learn `_ -* `XGBoost `_ -* `SQLGlot `_ - -For details, see the `third_party -`_ -directory. - - -Contact Us ----------- - -For further help and provide feedback, you can email us at `bigframes-feedback@google.com `_. diff --git a/docs/README.rst b/docs/README.rst new file mode 120000 index 00000000000..89a0106941f --- /dev/null +++ b/docs/README.rst @@ -0,0 +1 @@ +../README.rst \ No newline at end of file diff --git a/docs/CHANGELOG.md b/docs/changelog.md similarity index 100% rename from docs/CHANGELOG.md rename to docs/changelog.md diff --git a/docs/conf.py b/docs/conf.py index 2cc3ffa130d..9883467edfa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,12 +59,9 @@ "sphinx.ext.todo", "sphinx.ext.viewcode", "sphinx_sitemap", - "myst_nb", + "myst_parser", ] -# myst-nb configuration -nb_execution_mode = "off" - # autodoc/autosummary flags autoclass_content = "both" autodoc_default_options = {"members": True} @@ -121,7 +118,6 @@ "samples/AUTHORING_GUIDE.md", "samples/CONTRIBUTING.md", "samples/snippets/README.rst", - "README.rst", # used for include in overview.rst only ] # The reST default role (used for this markup: `text`) to use for all @@ -163,10 +159,20 @@ # documentation. # https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/layout.html#references html_theme_options = { - "github_url": "https://github.com/googleapis/google-cloud-python", + "github_url": "https://github.com/googleapis/python-bigquery-dataframes", "logo": { "text": "BigQuery DataFrames (BigFrames)", }, + "external_links": [ + { + "name": "Getting started", + "url": "https://docs.cloud.google.com/bigquery/docs/dataframes-quickstart", + }, + { + "name": "User guide", + "url": "https://docs.cloud.google.com/bigquery/docs/bigquery-dataframes-introduction", + }, + ], "analytics": { "google_analytics_id": "G-XVSRMCJ37X", }, @@ -272,16 +278,12 @@ suppress_warnings = [ - # Allow unknown mimetype so we can use widgets in tutorial notebooks. - "mystnb.unknown_mime_type", # Temporarily suppress this to avoid "more than one target found for # cross-reference" warning, which are intractable for us to avoid while in # a mono-repo. # See https://github.com/sphinx-doc/sphinx/blob # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843 - "ref.python", - # Allow external websites to be down occasionally. - "intersphinx.external", + "ref.python" ] # -- Options for LaTeX output --------------------------------------------- @@ -390,8 +392,7 @@ "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), - # TODO(tswast): re-enable if we can get temporary failures to be ignored. - # "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), "pydata-google-auth": ( "https://pydata-google-auth.readthedocs.io/en/latest/", None, diff --git a/docs/index.rst b/docs/index.rst index 51d05e7d368..b17ac7cbd9c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,85 +1,20 @@ -.. BigQuery DataFrames documentation main file +.. include:: README.rst -Scalable Python Data Analysis with BigQuery DataFrames (BigFrames) -================================================================== - -.. meta:: - :description: BigQuery DataFrames (BigFrames) provides a scalable, pandas-compatible Python API for data analysis and machine learning on petabyte-scale datasets using the BigQuery engine. - -**BigQuery DataFrames** (``bigframes``) is an open-source Python library that brings the power of **distributed computing** to your data science workflow. By providing a familiar **pandas** and **scikit-learn** compatible API, BigFrames allows you to analyze and model massive datasets where they live—directly in **BigQuery**. - -Why Choose BigQuery DataFrames? -------------------------------- - -BigFrames eliminates the "data movement bottleneck." Instead of downloading large datasets to a local environment, BigFrames translates your Python code into optimized SQL, executing complex transformations across the BigQuery fleet. - -* **Petabyte-Scale Scalability:** Effortlessly process datasets that far exceed local memory limits. -* **Familiar Python Ecosystem:** Use the same ``read_gbq``, ``groupby``, ``merge``, and ``pivot_table`` functions you already know from pandas. -* **Integrated Machine Learning:** Access BigQuery ML's powerful algorithms via a scikit-learn-like interface (``bigframes.ml``), including seamless **Gemini AI** integration. -* **Enterprise-Grade Security:** Maintain data governance and security by keeping your data within the BigQuery perimeter. -* **Hybrid Flexibility:** Easily move between distributed BigQuery processing and local pandas analysis with ``to_pandas()``. - -Core Components of BigFrames ----------------------------- - -BigQuery DataFrames is organized into specialized modules designed for the modern data stack: - -1. :mod:`bigframes.pandas`: A high-performance, pandas-compatible API for scalable data exploration, cleaning, and transformation. -2. :mod:`bigframes.bigquery`: Specialized utilities for direct BigQuery resource management, including integrations with Gemini and other AI models in the :mod:`bigframes.bigquery.ai` submodule. - - -Quickstart: Scalable Data Analysis in Seconds ---------------------------------------------- - -Install BigQuery DataFrames via pip: - -.. code-block:: bash - - pip install --upgrade bigframes - -The following example demonstrates how to perform a distributed aggregation on a public dataset with millions of rows using just a few lines of Python: - -.. code-block:: python - - import bigframes.pandas as bpd - - # If running in your local environment or Colab, uncomment these lines and add your GCP project ID - # PROJECT_ID = "bigframes-dev" - # bpd.options.bigquery.project = PROJECT_ID - - # Initialize BigFrames and load a public dataset - df = bpd.read_gbq("bigquery-public-data.usa_names.usa_1910_2013") - - # Perform familiar pandas operations that execute in the cloud - top_names = ( - df.groupby("name") - .agg({"number": "sum"}) - .sort_values("number", ascending=False) - .head(10) - ) - - # Bring the final, aggregated results back to local memory if needed - print(top_names.to_pandas()) - - -Explore the Documentation -------------------------- +API reference +------------- .. toctree:: - :maxdepth: 2 - :caption: User Documentation - - user_guide/index - -.. toctree:: - :maxdepth: 2 - :caption: API Reference + :maxdepth: 3 reference/index supported_pandas_apis +Changelog +--------- + +For a list of all BigQuery DataFrames releases: + .. toctree:: - :maxdepth: 1 - :caption: Community & Updates + :maxdepth: 2 - CHANGELOG + changelog diff --git a/docs/notebooks b/docs/notebooks deleted file mode 120000 index 8f9a5b2e6d2..00000000000 --- a/docs/notebooks +++ /dev/null @@ -1 +0,0 @@ -../notebooks \ No newline at end of file diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 99228010b24..bdf38e977da 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -9,7 +9,6 @@ packages. bigframes._config bigframes.bigquery - bigframes.bigquery.aead bigframes.bigquery.ai bigframes.bigquery.ml bigframes.bigquery.obj @@ -20,19 +19,6 @@ packages. bigframes.pandas.api.typing bigframes.streaming -Pandas Extensions -~~~~~~~~~~~~~~~~~ - -BigQuery DataFrames provides extensions to pandas DataFrame and Series objects. - -.. autosummary:: - :toctree: api - - bigframes.extensions.core.dataframe_accessor.BigQueryDataFrameAccessor - bigframes.extensions.core.dataframe_accessor.AIAccessor - bigframes.extensions.core.series_accessor.BigQuerySeriesAccessor - bigframes.extensions.core.series_accessor.AeadSeriesAccessor - ML APIs ~~~~~~~ diff --git a/docs/templates/toc.yml b/docs/templates/toc.yml index 394f2a7d3cc..5d043fd85f2 100644 --- a/docs/templates/toc.yml +++ b/docs/templates/toc.yml @@ -42,7 +42,7 @@ - name: DataFrame uid: bigframes.dataframe.DataFrame - name: PlotAccessor - uid: bigframes.pandas.api.typing.PlotAccessor + uid: bigframes.operations.plotting.PlotAccessor - name: StructAccessor uid: bigframes.operations.structs.StructFrameAccessor name: DataFrame @@ -86,7 +86,10 @@ - name: ListAccessor uid: bigframes.operations.lists.ListAccessor - name: PlotAccessor - uid: bigframes.pandas.api.typing.PlotAccessor + uid: bigframes.operations.plotting.PlotAccessor + - name: BlobAccessor + uid: bigframes.operations.blob.BlobAccessor + status: beta name: Series - name: Window uid: bigframes.core.window.Window diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst deleted file mode 100644 index 0c0935ac40a..00000000000 --- a/docs/user_guide/index.rst +++ /dev/null @@ -1,127 +0,0 @@ -User Guide -********** - -.. include:: ../README.rst - -.. toctree:: - :caption: Guides - :maxdepth: 1 - - Getting Started - Cloud Docs User Guides - -.. toctree:: - :caption: Getting Started - :maxdepth: 1 - - Quickstart Template <../notebooks/getting_started/bq_dataframes_template.ipynb> - Getting Started <../notebooks/getting_started/getting_started_bq_dataframes.ipynb> - Magics <../notebooks/getting_started/magics.ipynb> - ML Fundamentals <../notebooks/getting_started/ml_fundamentals_bq_dataframes.ipynb> - Pandas Extensions <../notebooks/getting_started/pandas_extensions.ipynb> - -.. toctree:: - :caption: DataFrames - :maxdepth: 1 - - Anywidget Mode <../notebooks/dataframes/anywidget_mode.ipynb> - Dataframe <../notebooks/dataframes/dataframe.ipynb> - Index Col Null <../notebooks/dataframes/index_col_null.ipynb> - Integrations <../notebooks/dataframes/integrations.ipynb> - Magics for Python and SQL Interoperability <../notebooks/dataframes/magics_with_local_data.ipynb> - Pypi <../notebooks/dataframes/pypi.ipynb> - -.. toctree:: - :caption: Data Types - :maxdepth: 1 - - Array <../notebooks/data_types/array.ipynb> - Json <../notebooks/data_types/json.ipynb> - Struct <../notebooks/data_types/struct.ipynb> - Timedelta <../notebooks/data_types/timedelta.ipynb> - -.. toctree:: - :caption: Generative AI - :maxdepth: 1 - - AI Functions <../notebooks/generative_ai/ai_functions.ipynb> - AI Functions for Poster Analysis <../notebooks/generative_ai/ai_movie_poster.ipynb> - AI Forecast <../notebooks/generative_ai/bq_dataframes_ai_forecast.ipynb> - LLM Code Generation <../notebooks/generative_ai/bq_dataframes_llm_code_generation.ipynb> - LLM KMeans <../notebooks/generative_ai/bq_dataframes_llm_kmeans.ipynb> - LLM Output Schema <../notebooks/generative_ai/bq_dataframes_llm_output_schema.ipynb> - LLM Vector Search <../notebooks/generative_ai/bq_dataframes_llm_vector_search.ipynb> - Drug Name Generation <../notebooks/generative_ai/bq_dataframes_ml_drug_name_generation.ipynb> - Large Language Models <../notebooks/generative_ai/large_language_models.ipynb> - -.. toctree:: - :caption: Machine Learning - :maxdepth: 1 - - ML Cross Validation <../notebooks/ml/bq_dataframes_ml_cross_validation.ipynb> - Linear Regression <../notebooks/ml/bq_dataframes_ml_linear_regression.ipynb> - Linear Regression BBQ <../notebooks/ml/bq_dataframes_ml_linear_regression_bbq.ipynb> - Linear Regression Big <../notebooks/ml/bq_dataframes_ml_linear_regression_big.ipynb> - Easy Linear Regression <../notebooks/ml/easy_linear_regression.ipynb> - Sklearn Linear Regression <../notebooks/ml/sklearn_linear_regression.ipynb> - Timeseries Analysis <../notebooks/ml/timeseries_analysis.ipynb> - -.. toctree:: - :caption: Visualization - :maxdepth: 1 - - COVID Line Graphs <../notebooks/visualization/bq_dataframes_covid_line_graphs.ipynb> - Tutorial <../notebooks/visualization/tutorial.ipynb> - -.. toctree:: - :caption: Geospatial Data - :maxdepth: 1 - - Geoseries <../notebooks/geo/geoseries.ipynb> - -.. toctree:: - :caption: Regionalized BigQuery - :maxdepth: 1 - - Regionalized <../notebooks/location/regionalized.ipynb> - -.. toctree:: - :caption: Multimodal - :maxdepth: 1 - - Multimodal Dataframe <../notebooks/multimodal/multimodal_dataframe.ipynb> - -.. toctree:: - :caption: Remote Functions - :maxdepth: 1 - - Remote Function <../notebooks/remote_functions/remote_function.ipynb> - Remote Function Usecases <../notebooks/remote_functions/remote_function_usecases.ipynb> - Remote Function Vertex Claude Model <../notebooks/remote_functions/remote_function_vertex_claude_model.ipynb> - -.. toctree:: - :caption: Streaming - :maxdepth: 1 - - Streaming Dataframe <../notebooks/streaming/streaming_dataframe.ipynb> - -.. toctree:: - :caption: Experimental - :maxdepth: 1 - - AI Operators <../notebooks/experimental/ai_operators.ipynb> - Semantic Operators <../notebooks/experimental/semantic_operators.ipynb> - -.. toctree:: - :caption: Apps - :maxdepth: 1 - - Synthetic Data Generation <../notebooks/apps/synthetic_data_generation.ipynb> - -.. toctree:: - :caption: Kaggle - :maxdepth: 1 - - AI Forecast <../notebooks/kaggle/bq_dataframes_ai_forecast.ipynb> - Describe Product Images <../notebooks/kaggle/describe-product-images-with-bigframes-multimodal.ipynb> - Vector Search Over National Jukebox <../notebooks/kaggle/vector-search-with-bigframes-over-national-jukebox.ipynb> diff --git a/mypy.ini b/mypy.ini index e3f44c262ac..7709eb200a3 100644 --- a/mypy.ini +++ b/mypy.ini @@ -44,6 +44,3 @@ ignore_missing_imports = True [mypy-anywidget] ignore_missing_imports = True - -[mypy-bigframes_vendored.*] -ignore_errors = True diff --git a/notebooks/apps/synthetic_data_generation.ipynb b/notebooks/apps/synthetic_data_generation.ipynb index 00d30fc8a8a..b59777a5da3 100644 --- a/notebooks/apps/synthetic_data_generation.ipynb +++ b/notebooks/apps/synthetic_data_generation.ipynb @@ -98,7 +98,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -111,7 +111,7 @@ "source": [ "from bigframes.ml.llm import GeminiTextGenerator\n", "\n", - "model = GeminiTextGenerator(model_name=\"gemini-2.5-flash\")" + "model = GeminiTextGenerator(model_name=\"gemini-2.0-flash-001\")" ] }, { diff --git a/notebooks/dataframes/anywidget_mode.ipynb b/notebooks/dataframes/anywidget_mode.ipynb index 9cae55b26dc..5dd8af1c5f1 100644 --- a/notebooks/dataframes/anywidget_mode.ipynb +++ b/notebooks/dataframes/anywidget_mode.ipynb @@ -63,7 +63,7 @@ "outputs": [], "source": [ "bpd.options.bigquery.ordering_mode = \"partial\"\n", - "bpd.options.display.render_mode = \"anywidget\"" + "bpd.options.display.repr_mode = \"anywidget\"" ] }, { @@ -92,8 +92,20 @@ { "data": { "text/html": [ - "\n", - " Query processed 171.4 MB in 19 seconds of slot time. [Job bigframes-dev:US.50efe672-74c6-4292-98d9-520cba9ca516 details]\n", + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 0 Bytes in a moment of slot time.\n", " " ], "text/plain": [ @@ -131,14 +143,57 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "220340b0", "metadata": {}, "outputs": [ + { + "data": { + "text/html": [ + "\n", + " Query started with request ID bigframes-dev:US.161c75bd-f9f8-4b21-8a45-1d7dfc659034.
    SQL
    SELECT\n",
+       "`state` AS `state`,\n",
+       "`gender` AS `gender`,\n",
+       "`year` AS `year`,\n",
+       "`name` AS `name`,\n",
+       "`number` AS `number`\n",
+       "FROM\n",
+       "(SELECT\n",
+       "  `t0`.`state`,\n",
+       "  `t0`.`gender`,\n",
+       "  `t0`.`year`,\n",
+       "  `t0`.`name`,\n",
+       "  `t0`.`number`,\n",
+       "  `t0`.`bfuid_col_2` AS `bfuid_col_15`\n",
+       "FROM `bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._c58be946_1477_4c00_b699_0ae022f13563_bqdf_8e323719-899f-4da2-89cd-2dbb53ab1dfc` AS `t0`)\n",
+       "ORDER BY `bfuid_col_15` ASC NULLS LAST
    \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 215.9 MB in 7 seconds of slot time. [Job bigframes-dev:US.job_IuiJsjhfPtOrKuTIOqPIjnVLX820 details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b1080dddbe4140d2b88ef85566e52955", + "model_id": "e68fbb9eb4d24bab837c77730d31c8a1", "version_major": 2, "version_minor": 1 }, @@ -174,80 +229,80 @@ " AL\n", " F\n", " 1910\n", - " Lillian\n", - " 99\n", + " Hazel\n", + " 51\n", " \n", " \n", " 1\n", " AL\n", " F\n", " 1910\n", - " Ruby\n", - " 204\n", + " Lucy\n", + " 76\n", " \n", " \n", " 2\n", - " AL\n", + " AR\n", " F\n", " 1910\n", - " Helen\n", - " 76\n", + " Nellie\n", + " 39\n", " \n", " \n", " 3\n", - " AL\n", + " AR\n", " F\n", " 1910\n", - " Eunice\n", - " 41\n", + " Lena\n", + " 40\n", " \n", " \n", " 4\n", - " AR\n", + " CO\n", " F\n", " 1910\n", - " Dora\n", - " 42\n", + " Thelma\n", + " 36\n", " \n", " \n", " 5\n", - " CA\n", + " CO\n", " F\n", " 1910\n", - " Edna\n", - " 62\n", + " Ruth\n", + " 68\n", " \n", " \n", " 6\n", - " CA\n", + " CT\n", " F\n", " 1910\n", - " Helen\n", - " 239\n", + " Elizabeth\n", + " 86\n", " \n", " \n", " 7\n", - " CO\n", + " DC\n", " F\n", " 1910\n", - " Alice\n", - " 46\n", + " Mary\n", + " 80\n", " \n", " \n", " 8\n", " FL\n", " F\n", " 1910\n", - " Willie\n", - " 71\n", + " Annie\n", + " 101\n", " \n", " \n", " 9\n", " FL\n", " F\n", " 1910\n", - " Thelma\n", - " 65\n", + " Alma\n", + " 39\n", " \n", " \n", "\n", @@ -255,25 +310,67 @@ "[5552452 rows x 5 columns in total]" ], "text/plain": [ - "state gender year name number\n", - " AL F 1910 Lillian 99\n", - " AL F 1910 Ruby 204\n", - " AL F 1910 Helen 76\n", - " AL F 1910 Eunice 41\n", - " AR F 1910 Dora 42\n", - " CA F 1910 Edna 62\n", - " CA F 1910 Helen 239\n", - " CO F 1910 Alice 46\n", - " FL F 1910 Willie 71\n", - " FL F 1910 Thelma 65\n", + "state gender year name number\n", + " AL F 1910 Hazel 51\n", + " AL F 1910 Lucy 76\n", + " AR F 1910 Nellie 39\n", + " AR F 1910 Lena 40\n", + " CO F 1910 Thelma 36\n", + " CO F 1910 Ruth 68\n", + " CT F 1910 Elizabeth 86\n", + " DC F 1910 Mary 80\n", + " FL F 1910 Annie 101\n", + " FL F 1910 Alma 39\n", "...\n", "\n", "[5552452 rows x 5 columns]" ] }, - "execution_count": 5, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 215.9 MB in 9 seconds of slot time. [Job bigframes-dev:US.job_IEjIRaqt2w-_pAttPw1VAVuRPxA7 details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 215.9 MB in 5 seconds of slot time. [Job bigframes-dev:US.job_Mi-3m2AkEC1iPgWi7hmcWa1M1oIA details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 215.9 MB in 6 seconds of slot time. [Job bigframes-dev:US.job_j8pvY385WwIY7tGvhI7Yxc62aBwd details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ @@ -298,8 +395,22 @@ { "data": { "text/html": [ - "\n", - " Query processed 44.4 MB in a moment of slot time.\n", + "✅ Completed. \n", + " Query processed 171.4 MB in 30 seconds of slot time. [Job bigframes-dev:US.ff90d507-bec8-4d24-abc3-0209ac28e21f details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 88.8 MB in a moment of slot time.\n", " " ], "text/plain": [ @@ -309,20 +420,32 @@ "metadata": {}, "output_type": "display_data" }, + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "name": "stdout", "output_type": "stream", "text": [ - "1967\n", - "1981\n", - "2009\n", - "1956\n", - "1960\n", - "2001\n", - "2009\n", - "2003\n", - "1985\n", - "1993\n", + "1910\n", + "1910\n", + "1910\n", + "1910\n", + "1910\n", + "1910\n", + "1910\n", + "1910\n", + "1910\n", + "1910\n", "Name: year, dtype: Int64\n", "...\n", "\n", @@ -350,36 +473,64 @@ "id": "da23e0f3", "metadata": {}, "outputs": [ + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 88.8 MB in 3 seconds of slot time. [Job bigframes-dev:US.job_517TdI--FMoURkV7QQNMltY_-dZ7 details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 88.8 MB in 2 seconds of slot time. [Job bigframes-dev:US.job_rCeYkeBPqmTKNFWFgwXjz5Ed8uWI details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "46e836f10d9e47afb4d82b5c7da69660", + "model_id": "3e630b1a56c740e781772ca5f5c7267a", "version_major": 2, "version_minor": 1 }, "text/html": [ "
    0    1910\n",
-       "1    1912\n",
-       "2    1912\n",
-       "3    1911\n",
-       "4    1912\n",
+       "1    1910\n",
+       "2    1910\n",
+       "3    1910\n",
+       "4    1910\n",
        "5    1910\n",
-       "6    1913\n",
-       "7    1912\n",
-       "8    1913\n",
-       "9    1913

    [5552452 rows]

    " + "6 1910\n", + "7 1910\n", + "8 1910\n", + "9 1910

    [5552452 rows]

    " ], "text/plain": [ "1910\n", - "1912\n", - "1912\n", - "1911\n", - "1912\n", "1910\n", - "1913\n", - "1912\n", - "1913\n", - "1913\n", + "1910\n", + "1910\n", + "1910\n", + "1910\n", + "1910\n", + "1910\n", + "1910\n", + "1910\n", "Name: year, dtype: Int64\n", "...\n", "\n", @@ -451,6 +602,34 @@ "id": "6920d49b", "metadata": {}, "outputs": [ + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 215.9 MB in 11 seconds of slot time. [Job bigframes-dev:US.job_XwXTDb6gWVkuyIFMeWA0waE33bSg details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 215.9 MB in 7 seconds of slot time. [Job bigframes-dev:US.job_bCW0LYK5_PzyyGPf9OAg4YfNMG1C details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "name": "stdout", "output_type": "stream", @@ -461,12 +640,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6e5f603b56fb408bb1ea41519ea8702e", + "model_id": "a6a2b19314b04283a5a66ca9d66eb771", "version_major": 2, "version_minor": 1 }, "text/plain": [ - "" + "" ] }, "execution_count": 8, @@ -475,10 +654,9 @@ } ], "source": [ - "import math\n", - "\n", "from bigframes.display.anywidget import TableWidget\n", - "\n", + "import math\n", + " \n", "# Create widget programmatically \n", "widget = TableWidget(df)\n", "print(f\"Total pages: {math.ceil(widget.row_count / widget.page_size)}\")\n", @@ -539,6 +717,34 @@ "id": "a9d5d13a", "metadata": {}, "outputs": [ + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 215.9 MB in a moment of slot time.\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 215.9 MB in a moment of slot time.\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "name": "stdout", "output_type": "stream", @@ -549,12 +755,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "20c94621c4ae4eb5a94fd3596ae8c236", + "model_id": "beb362548a6b4fd4a163569edd6f1a90", "version_major": 2, "version_minor": 1 }, "text/plain": [ - "" + "" ] }, "execution_count": 10, @@ -591,14 +797,14 @@ { "cell_type": "code", "execution_count": 11, - "id": "75000341", + "id": "added-cell-1", "metadata": {}, "outputs": [ { "data": { "text/html": [ - "\n", - " Query processed 0 Bytes in a moment of slot time. [Job bigframes-dev:US.job_cpfa9oehjApkQgrbTrKRxTpEtuQX details]\n", + "✅ Completed. \n", + " Query processed 85.9 kB in 19 seconds of slot time.\n", " " ], "text/plain": [ @@ -608,10 +814,58 @@ "metadata": {}, "output_type": "display_data" }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/python-bigquery-dataframes/bigframes/dtypes.py:987: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + }, + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/python-bigquery-dataframes/bigframes/dtypes.py:987: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/python-bigquery-dataframes/bigframes/dtypes.py:987: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d5bf0a9438954c6890b5d8cd16bff7cd", + "model_id": "02a46cf499b442d4bfe03934195e67df", "version_major": 2, "version_minor": 1 }, @@ -654,43 +908,43 @@ " \n", " \n", " 0\n", - " {\"application_number\":\"18157874.1\",\"class_inte...\n", + " {'application_number': None, 'class_internatio...\n", " gs://gcs-public-data--labeled-patents/espacene...\n", " EU\n", " DE\n", - " 29.08.018\n", - " E04H 6/12\n", + " 03.10.2018\n", + " H01L 21/20\n", " <NA>\n", - " 18157874.1\n", - " 21.02.2018\n", - " 22.02.2017\n", - " Liedtke & Partner Patentanw√§lte\n", - " SHB Hebezeugbau GmbH\n", - " VOLGER, Alexander\n", - " STEUERUNGSSYSTEM F√úR AUTOMATISCHE PARKH√ÑUSER\n", - " EP 3 366 869 A1\n", + " 18166536.5\n", + " 16.02.2016\n", + " <NA>\n", + " Scheider, Sascha et al\n", + " EV Group E. Thallner GmbH\n", + " Kurz, Florian\n", + " VORRICHTUNG ZUM BONDEN VON SUBSTRATEN\n", + " EP 3 382 744 A1\n", " \n", " \n", " 1\n", - " {\"application_number\":\"18165514.3\",\"class_inte...\n", + " {'application_number': None, 'class_internatio...\n", " gs://gcs-public-data--labeled-patents/espacene...\n", " EU\n", " DE\n", " 03.10.2018\n", - " H05B 6/12\n", - " <NA>\n", - " 18165514.3\n", - " 03.04.2018\n", - " 30.03.2017\n", + " A01K 31/00\n", " <NA>\n", - " BSH Hausger√§te GmbH\n", - " Acero Acero, Jesus\n", - " VORRICHTUNG ZUR INDUKTIVEN ENERGIE√úBERTRAGUNG\n", - " EP 3 383 141 A2\n", + " 18171005.4\n", + " 05.02.2015\n", + " 05.02.2014\n", + " Stork Bamberger Patentanw√§lte\n", + " Linco Food Systems A/S\n", + " Thrane, Uffe\n", + " MASTH√ÑHNCHENCONTAINER ALS BESTANDTEIL EINER E...\n", + " EP 3 381 276 A1\n", " \n", " \n", " 2\n", - " {\"application_number\":\"18157347.8\",\"class_inte...\n", + " {'application_number': None, 'class_internatio...\n", " gs://gcs-public-data--labeled-patents/espacene...\n", " EU\n", " DE\n", @@ -708,39 +962,39 @@ " \n", " \n", " 3\n", - " {\"application_number\":\"18166536.5\",\"class_inte...\n", + " {'application_number': None, 'class_internatio...\n", " gs://gcs-public-data--labeled-patents/espacene...\n", " EU\n", " DE\n", " 03.10.2018\n", - " H01L 21/20\n", + " H05B 6/12\n", " <NA>\n", - " 18166536.5\n", - " 16.02.2016\n", + " 18165514.3\n", + " 03.04.2018\n", + " 30.03.2017\n", " <NA>\n", - " Scheider, Sascha et al\n", - " EV Group E. Thallner GmbH\n", - " Kurz, Florian\n", - " VORRICHTUNG ZUM BONDEN VON SUBSTRATEN\n", - " EP 3 382 744 A1\n", + " BSH Hausger√§te GmbH\n", + " Acero Acero, Jesus\n", + " VORRICHTUNG ZUR INDUKTIVEN ENERGIE√úBERTRAGUNG\n", + " EP 3 383 141 A2\n", " \n", " \n", " 4\n", - " {\"application_number\":\"18171005.4\",\"class_inte...\n", + " {'application_number': None, 'class_internatio...\n", " gs://gcs-public-data--labeled-patents/espacene...\n", " EU\n", " DE\n", - " 03.10.2018\n", - " A01K 31/00\n", + " 29.08.018\n", + " E04H 6/12\n", " <NA>\n", - " 18171005.4\n", - " 05.02.2015\n", - " 05.02.2014\n", - " Stork Bamberger Patentanw√§lte\n", - " Linco Food Systems A/S\n", - " Thrane, Uffe\n", - " MASTH√ÑHNCHENCONTAINER ALS BESTANDTEIL EINER E...\n", - " EP 3 381 276 A1\n", + " 18157874.1\n", + " 21.02.2018\n", + " 22.02.2017\n", + " Liedtke & Partner Patentanw√§lte\n", + " SHB Hebezeugbau GmbH\n", + " VOLGER, Alexander\n", + " STEUERUNGSSYSTEM F√úR AUTOMATISCHE PARKH√ÑUSER\n", + " EP 3 366 869 A1\n", " \n", " \n", "\n", @@ -748,47 +1002,47 @@ "[5 rows x 15 columns in total]" ], "text/plain": [ - " result \\\n", - "{\"application_number\":\"18157874.1\",\"class_inter... \n", - "{\"application_number\":\"18165514.3\",\"class_inter... \n", - "{\"application_number\":\"18157347.8\",\"class_inter... \n", - "{\"application_number\":\"18166536.5\",\"class_inter... \n", - "{\"application_number\":\"18171005.4\",\"class_inter... \n", + " result \\\n", + "0 {'application_number': None, 'class_internatio... \n", + "1 {'application_number': None, 'class_internatio... \n", + "2 {'application_number': None, 'class_internatio... \n", + "3 {'application_number': None, 'class_internatio... \n", + "4 {'application_number': None, 'class_internatio... \n", "\n", - " gcs_path issuer language \\\n", - "gs://gcs-public-data--labeled-patents/espacenet... EU DE \n", - "gs://gcs-public-data--labeled-patents/espacenet... EU DE \n", - "gs://gcs-public-data--labeled-patents/espacenet... EU DE \n", - "gs://gcs-public-data--labeled-patents/espacenet... EU DE \n", - "gs://gcs-public-data--labeled-patents/espacenet... EU DE \n", + " gcs_path issuer language \\\n", + "0 gs://gcs-public-data--labeled-patents/espacene... EU DE \n", + "1 gs://gcs-public-data--labeled-patents/espacene... EU DE \n", + "2 gs://gcs-public-data--labeled-patents/espacene... EU DE \n", + "3 gs://gcs-public-data--labeled-patents/espacene... EU DE \n", + "4 gs://gcs-public-data--labeled-patents/espacene... EU DE \n", "\n", - "publication_date class_international class_us application_number filing_date \\\n", - " 29.08.018 E04H 6/12 18157874.1 21.02.2018 \n", - " 03.10.2018 H05B 6/12 18165514.3 03.04.2018 \n", - " 03.10.2018 G06F 11/30 18157347.8 19.02.2018 \n", - " 03.10.2018 H01L 21/20 18166536.5 16.02.2016 \n", - " 03.10.2018 A01K 31/00 18171005.4 05.02.2015 \n", + " publication_date class_international class_us application_number \\\n", + "0 03.10.2018 H01L 21/20 18166536.5 \n", + "1 03.10.2018 A01K 31/00 18171005.4 \n", + "2 03.10.2018 G06F 11/30 18157347.8 \n", + "3 03.10.2018 H05B 6/12 18165514.3 \n", + "4 29.08.018 E04H 6/12 18157874.1 \n", "\n", - "priority_date_eu representative_line_1_eu applicant_line_1 \\\n", - " 22.02.2017 Liedtke & Partner Patentanw√§lte SHB Hebezeugbau GmbH \n", - " 30.03.2017 BSH Hausger√§te GmbH \n", - " 31.03.2017 Hoffmann Eitle FUJITSU LIMITED \n", - " Scheider, Sascha et al EV Group E. Thallner GmbH \n", - " 05.02.2014 Stork Bamberger Patentanw√§lte Linco Food Systems A/S \n", + " filing_date priority_date_eu representative_line_1_eu \\\n", + "0 16.02.2016 Scheider, Sascha et al \n", + "1 05.02.2015 05.02.2014 Stork Bamberger Patentanw√§lte \n", + "2 19.02.2018 31.03.2017 Hoffmann Eitle \n", + "3 03.04.2018 30.03.2017 \n", + "4 21.02.2018 22.02.2017 Liedtke & Partner Patentanw√§lte \n", "\n", - " inventor_line_1 title_line_1 \\\n", - " VOLGER, Alexander STEUERUNGSSYSTEM F√úR AUTOMATISCHE PARKH√ÑUSER \n", - "Acero Acero, Jesus VORRICHTUNG ZUR INDUKTIVEN ENERGIE√úBERTRAGUNG \n", - " Kukihara, Kensuke METHOD EXECUTED BY A COMPUTER, INFORMATION PROC... \n", - " Kurz, Florian VORRICHTUNG ZUM BONDEN VON SUBSTRATEN \n", - " Thrane, Uffe MASTH√ÑHNCHENCONTAINER ALS BESTANDTEIL EINER EI... \n", + " applicant_line_1 inventor_line_1 \\\n", + "0 EV Group E. Thallner GmbH Kurz, Florian \n", + "1 Linco Food Systems A/S Thrane, Uffe \n", + "2 FUJITSU LIMITED Kukihara, Kensuke \n", + "3 BSH Hausger√§te GmbH Acero Acero, Jesus \n", + "4 SHB Hebezeugbau GmbH VOLGER, Alexander \n", "\n", - " number \n", - "EP 3 366 869 A1 \n", - "EP 3 383 141 A2 \n", - "EP 3 382 553 A1 \n", - "EP 3 382 744 A1 \n", - "EP 3 381 276 A1 \n", + " title_line_1 number \n", + "0 VORRICHTUNG ZUM BONDEN VON SUBSTRATEN EP 3 382 744 A1 \n", + "1 MASTH√ÑHNCHENCONTAINER ALS BESTANDTEIL EINER E... EP 3 381 276 A1 \n", + "2 METHOD EXECUTED BY A COMPUTER, INFORMATION PRO... EP 3 382 553 A1 \n", + "3 VORRICHTUNG ZUR INDUKTIVEN ENERGIE√úBERTRAGUNG EP 3 383 141 A2 \n", + "4 STEUERUNGSSYSTEM F√úR AUTOMATISCHE PARKH√ÑUSER EP 3 366 869 A1 \n", "\n", "[5 rows x 15 columns]" ] @@ -799,12 +1053,12 @@ } ], "source": [ - "bpd.read_gbq(\"\"\"\n", + "bpd._read_gbq_colab(\"\"\"\n", " SELECT\n", " AI.GENERATE(\n", - " prompt=>(\"Extract the values.\", OBJ.GET_ACCESS_URL(OBJ.FETCH_METADATA(OBJ.MAKE_REF(gcs_path, \"us.bigframes-default-connection\")), \"r\")),\n", - " connection_id=>\"us.bigframes-default-connection\",\n", - " output_schema=>\"publication_date string, class_international string, application_number string, filing_date string\") AS result,\n", + " prompt=>(\\\"Extract the values.\\\", OBJ.GET_ACCESS_URL(OBJ.FETCH_METADATA(OBJ.MAKE_REF(gcs_path, \\\"us.conn\\\")), \\\"r\\\")),\n", + " connection_id=>\\\"bigframes-dev.us.bigframes-default-connection\\\",\n", + " output_schema=>\\\"publication_date string, class_international string, application_number string, filing_date string\\\") AS result,\n", " *\n", " FROM `bigquery-public-data.labeled_patents.extracted_data`\n", " LIMIT 5;\n", @@ -814,7 +1068,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "venv", "language": "python", "name": "python3" }, diff --git a/notebooks/dataframes/dataframe.ipynb b/notebooks/dataframes/dataframe.ipynb index f26b4ff1cf1..de9bb1d04f4 100644 --- a/notebooks/dataframes/dataframe.ipynb +++ b/notebooks/dataframes/dataframe.ipynb @@ -49,7 +49,7 @@ "id": "13861abc-120c-4db6-ad0c-e414b85d3443", "metadata": {}, "source": [ - "## Select a subset of the DF" + "### Select a subset of the DF" ] }, { diff --git a/notebooks/dataframes/index_col_null.ipynb b/notebooks/dataframes/index_col_null.ipynb index f77051e553b..655745dd2be 100644 --- a/notebooks/dataframes/index_col_null.ipynb +++ b/notebooks/dataframes/index_col_null.ipynb @@ -358,7 +358,7 @@ "id": "13861abc-120c-4db6-ad0c-e414b85d3443", "metadata": {}, "source": [ - "## Select a subset of the DataFrame", + "### Select a subset of the DataFrame\n", "\n", "Filter columns by selecting a list of columns from the DataFrame.\n", "\n", diff --git a/notebooks/dataframes/magics_with_local_data.ipynb b/notebooks/dataframes/magics_with_local_data.ipynb deleted file mode 100644 index 675ac83988b..00000000000 --- a/notebooks/dataframes/magics_with_local_data.ipynb +++ /dev/null @@ -1,2488 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "c5f9e86e", - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2026 Google LLC\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "id": "71383fa0", - "metadata": {}, - "source": [ - "# Unlock SQL and Python interoperability for BigQuery with %%bqsql magic\n", - "\n", - "In this tutorial, you will learn how to seamlessly chain data processing across\n", - "SQL and Python code cells using `%%bqsql` IPython magic and BigQuery DataFrames\n", - "(BigFrames). This interoperability is now available to all Jupyter users,\n", - "whether you're in Colab, JupyterLab, or VS Code. \n", - "\n", - "While we begin by loading a local Excel dataset into a local Pandas DataFrame,\n", - "the main focus is on how you can transition between Pandas' Python-centric API\n", - "and BigQuery's SQL-centric engine. This hybrid workflow combines the best of\n", - "both worlds: the expressive power of SQL for complex transformations and the\n", - "versatile Python ecosystem for visualization and further analysis.\n", - "\n", - "Thanks to open-source packages like Jupyter, Pandas, BigFrames, and the\n", - "[BigQuery sandbox](https://docs.cloud.google.com/bigquery/docs/sandbox), you can\n", - "follow all steps in this guide for free\\* and without a credit card.\n", - "\n", - "_\\*See the [BigQuery sandbox](https://docs.cloud.google.com/bigquery/docs/sandbox) documentation for limitations._\n", - "\n", - "## The %%bqsql Magic\n", - "\n", - "Last year, Google introduced [SQL cells in Colab Enterprise\n", - "notebooks](https://docs.cloud.google.com/colab/docs/sql-cells). Now, with the\n", - "[%%bqsql cell\n", - "magics](https://dataframes.bigquery.dev/notebooks/getting_started/magics.html)\n", - "in BigQuery DataFrames, this same powerful interoperability is available to all\n", - "Jupyter users, whether you're in Colab, JupyterLab, or VS Code. These magics\n", - "allow you to write SQL queries that run directly on local pandas DataFrames,\n", - "BigFrames DataFrames, or BigQuery tables.\n", - "\n", - "\n", - "## Getting Started\n", - "\n", - "To get started,\n", - "\n", - "1. Enable the [BigQuery\n", - " sandbox](https://docs.cloud.google.com/bigquery/docs/sandbox). Make note of your\n", - " Google Cloud project ID.\n", - "\n", - "2. Set up a local Python development environment (see: [Setting up a Python\n", - " development environment](https://docs.cloud.google.com/python/docs/setup)) for\n", - " Google Cloud.\n", - "\n", - "3. Create and activate a venv to isolate Python dependencies.\n", - " On Linux or macOS, use these commands (update to your preferred Python\n", - " version):\n", - "\n", - " ```\n", - " python3.12 -m venv ~/venv\n", - " . ~/venv/bin/activate\n", - " ```\n", - "\n", - "4. Install the Jupyter, bigframes, and python-calamine packages:\n", - "\n", - " ```\n", - " pip install --upgrade jupyterlab bigframes python-calamine\n", - " ```\n", - "\n", - "5. Start Jupyter Lab.\n", - "\n", - " ```\n", - " jupyter lab\n", - " ```\n", - "\n", - "6. Open a web browser to the URL listed in the output. It will be something like\n", - " `http://localhost:8888/lab?token=somesupersecretvaluehere`.\n", - "\n", - "7. Create a new notebook using the Jupyter Lab UI.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d00aeb28", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install python-calamine pandas bigframes" - ] - }, - { - "cell_type": "markdown", - "id": "5ba39d0d", - "metadata": {}, - "source": [ - "## Accessing the Dataset\n", - "\n", - "In this tutorial, you'll analyze the [USDA wheat\n", - "data](https://www.ers.usda.gov/data-products/wheat-data). Use the standard\n", - "`requests` package to download the data to a temporary file, mimicking a typical\n", - "local data analysis workflow.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "fb1dfdc2", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\n", - "import tempfile\n", - "\n", - "import requests\n", - "\n", - "url = \"https://www.ers.usda.gov/media/5706/wheat-data-all-years.xlsx?v=52690\"\n", - "\n", - "tmp = tempfile.NamedTemporaryFile(delete=True)\n", - "\n", - "with requests.get(url, stream=True) as r:\n", - " r.raise_for_status()\n", - " for chunk in r.iter_content(chunk_size=8192):\n", - " tmp.write(chunk)\n", - "\n", - "tmp.flush()\n", - "tmp.seek(0)" - ] - }, - { - "cell_type": "markdown", - "id": "50f896bb", - "metadata": {}, - "source": [ - "Use the `pyarrow` `dtype_backend` when preparing local Pandas data for SQL\n", - "processing. This ensures more consistent handling of NULL values and seamless\n", - "schema mapping when you hand off the data to the BigQuery SQL engine. For this\n", - "example, read the 'Table05' sheet, which contains annual wheat supply and\n", - "disappearance data:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "8a8a137b", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    Marketing year 1/Time periodBeginning stocksProductionImports 2/Total supply 3/Food useSeed useFeed and residual useTotal domestic use 3/Exports 2/Total disappearance 3/Ending stocks
    01950/51MY Jun-May496.01019.011.01526.0580.0--109.0689.0345.01034.0492.0
    11951/52MY Jun-May492.0988.030.01510.0585.0--110.0695.0485.01180.0330.0
    21952/53MY Jun-May330.01306.024.01660.0578.0--78.0656.0332.0988.0672.0
    31953/54MY Jun-May672.01173.06.01851.0556.0--87.0643.0214.0857.0994.0
    41954/55MY Jun-May994.0984.03.01981.0552.0--53.0605.0267.0872.01109.0
    ..........................................
    2811/ June–May. Latest data may be preliminary or...<NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA>
    2822/ Includes flour and selected other products ...<NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA>
    2833/ Totals may not add due to rounding.<NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA>
    284Source: USDA, Economic Research Service, based...<NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA>
    285Updated: May 12, 2026<NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA>
    \n", - "

    286 rows × 13 columns

    \n", - "
    " - ], - "text/plain": [ - " Marketing year 1/ Time period \\\n", - "0 1950/51 MY Jun-May \n", - "1 1951/52 MY Jun-May \n", - "2 1952/53 MY Jun-May \n", - "3 1953/54 MY Jun-May \n", - "4 1954/55 MY Jun-May \n", - ".. ... ... \n", - "281 1/ June–May. Latest data may be preliminary or... \n", - "282 2/ Includes flour and selected other products ... \n", - "283 3/ Totals may not add due to rounding. \n", - "284 Source: USDA, Economic Research Service, based... \n", - "285 Updated: May 12, 2026 \n", - "\n", - " Beginning stocks Production Imports 2/ Total supply 3/ Food use \\\n", - "0 496.0 1019.0 11.0 1526.0 580.0 \n", - "1 492.0 988.0 30.0 1510.0 585.0 \n", - "2 330.0 1306.0 24.0 1660.0 578.0 \n", - "3 672.0 1173.0 6.0 1851.0 556.0 \n", - "4 994.0 984.0 3.0 1981.0 552.0 \n", - ".. ... ... ... ... ... \n", - "281 \n", - "282 \n", - "283 \n", - "284 \n", - "285 \n", - "\n", - " Seed use Feed and residual use Total domestic use 3/ Exports 2/ \\\n", - "0 -- 109.0 689.0 345.0 \n", - "1 -- 110.0 695.0 485.0 \n", - "2 -- 78.0 656.0 332.0 \n", - "3 -- 87.0 643.0 214.0 \n", - "4 -- 53.0 605.0 267.0 \n", - ".. ... ... ... ... \n", - "281 \n", - "282 \n", - "283 \n", - "284 \n", - "285 \n", - "\n", - " Total disappearance 3/ Ending stocks \n", - "0 1034.0 492.0 \n", - "1 1180.0 330.0 \n", - "2 988.0 672.0 \n", - "3 857.0 994.0 \n", - "4 872.0 1109.0 \n", - ".. ... ... \n", - "281 \n", - "282 \n", - "283 \n", - "284 \n", - "285 \n", - "\n", - "[286 rows x 13 columns]" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\n", - "import pandas as pd\n", - "\n", - "df = pd.read_excel(\n", - " tmp,\n", - " sheet_name=\"Table05\",\n", - " dtype_backend=\"pyarrow\",\n", - " engine=\"calamine\",\n", - " header=1, # Skip the first row.\n", - ")\n", - "tmp.close()\n", - "df" - ] - }, - { - "cell_type": "markdown", - "id": "1a7ec573", - "metadata": {}, - "source": [ - "## Preparing the data\n", - "\n", - "Before querying the local DataFrame with SQL, ensure that the column names are\n", - "SQL-friendly. BigQuery supports [flexible column\n", - "names](https://docs.cloud.google.com/bigquery/docs/schemas#flexible-column-names),\n", - "allowing most unicode characters, but special characters like \"/\" and \"\\\" must\n", - "be removed or replaced.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "d5674020", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    Marketing year 1Time periodBeginning stocksProductionImports 2Total supply 3Food useSeed useFeed and residual useTotal domestic use 3Exports 2Total disappearance 3Ending stocks
    01950/51MY Jun-May496.01019.011.01526.0580.0--109.0689.0345.01034.0492.0
    11951/52MY Jun-May492.0988.030.01510.0585.0--110.0695.0485.01180.0330.0
    21952/53MY Jun-May330.01306.024.01660.0578.0--78.0656.0332.0988.0672.0
    31953/54MY Jun-May672.01173.06.01851.0556.0--87.0643.0214.0857.0994.0
    41954/55MY Jun-May994.0984.03.01981.0552.0--53.0605.0267.0872.01109.0
    ..........................................
    2811/ June–May. Latest data may be preliminary or...<NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA>
    2822/ Includes flour and selected other products ...<NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA>
    2833/ Totals may not add due to rounding.<NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA>
    284Source: USDA, Economic Research Service, based...<NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA>
    285Updated: May 12, 2026<NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA><NA>
    \n", - "

    286 rows × 13 columns

    \n", - "
    " - ], - "text/plain": [ - " Marketing year 1 Time period \\\n", - "0 1950/51 MY Jun-May \n", - "1 1951/52 MY Jun-May \n", - "2 1952/53 MY Jun-May \n", - "3 1953/54 MY Jun-May \n", - "4 1954/55 MY Jun-May \n", - ".. ... ... \n", - "281 1/ June–May. Latest data may be preliminary or... \n", - "282 2/ Includes flour and selected other products ... \n", - "283 3/ Totals may not add due to rounding. \n", - "284 Source: USDA, Economic Research Service, based... \n", - "285 Updated: May 12, 2026 \n", - "\n", - " Beginning stocks Production Imports 2 Total supply 3 Food use \\\n", - "0 496.0 1019.0 11.0 1526.0 580.0 \n", - "1 492.0 988.0 30.0 1510.0 585.0 \n", - "2 330.0 1306.0 24.0 1660.0 578.0 \n", - "3 672.0 1173.0 6.0 1851.0 556.0 \n", - "4 994.0 984.0 3.0 1981.0 552.0 \n", - ".. ... ... ... ... ... \n", - "281 \n", - "282 \n", - "283 \n", - "284 \n", - "285 \n", - "\n", - " Seed use Feed and residual use Total domestic use 3 Exports 2 \\\n", - "0 -- 109.0 689.0 345.0 \n", - "1 -- 110.0 695.0 485.0 \n", - "2 -- 78.0 656.0 332.0 \n", - "3 -- 87.0 643.0 214.0 \n", - "4 -- 53.0 605.0 267.0 \n", - ".. ... ... ... ... \n", - "281 \n", - "282 \n", - "283 \n", - "284 \n", - "285 \n", - "\n", - " Total disappearance 3 Ending stocks \n", - "0 1034.0 492.0 \n", - "1 1180.0 330.0 \n", - "2 988.0 672.0 \n", - "3 857.0 994.0 \n", - "4 872.0 1109.0 \n", - ".. ... ... \n", - "281 \n", - "282 \n", - "283 \n", - "284 \n", - "285 \n", - "\n", - "[286 rows x 13 columns]" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.columns = [name.replace(\"/\", \"\") for name in df.columns]\n", - "df" - ] - }, - { - "cell_type": "markdown", - "id": "b50c5798", - "metadata": {}, - "source": [ - "## Filtering with Pandas\n", - "\n", - "Perform a basic filter using standard Python/Pandas syntax to remove rows with missing data. This represents the initial Python-only stage of a processing chain.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "1dbad481", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    Marketing year 1Time periodBeginning stocksProductionImports 2Total supply 3Food useSeed useFeed and residual useTotal domestic use 3Exports 2Total disappearance 3Ending stocks
    01950/51MY Jun-May496.01019.011.01526.0580.0--109.0689.0345.01034.0492.0
    11951/52MY Jun-May492.0988.030.01510.0585.0--110.0695.0485.01180.0330.0
    21952/53MY Jun-May330.01306.024.01660.0578.0--78.0656.0332.0988.0672.0
    31953/54MY Jun-May672.01173.06.01851.0556.0--87.0643.0214.0857.0994.0
    41954/55MY Jun-May994.0984.03.01981.0552.0--53.0605.0267.0872.01109.0
    ..........................................
    2752025/26MY Jun-May854.7341984.537125.02964.271960.059.7100.01119.7910.02029.7934.571
    2762025/26Q1 Jun-Aug854.7341984.53730.5932869.864241.0912.653239.522483.266252.58735.8462134.018
    2772025/26Q2 Sep-Nov2134.0180.030.0782164.096245.5839.658-54.047231.191255.802486.9931677.103
    2782025/26Q3 Dec-Feb1677.1030.032.3631709.466230.9751.75-24.747207.978201.291409.2691300.197
    2792026/27MY Jun-May934.5711561.322140.02635.893960.05980.01099.0775.01874.0761.893
    \n", - "

    280 rows × 13 columns

    \n", - "
    " - ], - "text/plain": [ - " Marketing year 1 Time period Beginning stocks Production Imports 2 \\\n", - "0 1950/51 MY Jun-May 496.0 1019.0 11.0 \n", - "1 1951/52 MY Jun-May 492.0 988.0 30.0 \n", - "2 1952/53 MY Jun-May 330.0 1306.0 24.0 \n", - "3 1953/54 MY Jun-May 672.0 1173.0 6.0 \n", - "4 1954/55 MY Jun-May 994.0 984.0 3.0 \n", - ".. ... ... ... ... ... \n", - "275 2025/26 MY Jun-May 854.734 1984.537 125.0 \n", - "276 2025/26 Q1 Jun-Aug 854.734 1984.537 30.593 \n", - "277 2025/26 Q2 Sep-Nov 2134.018 0.0 30.078 \n", - "278 2025/26 Q3 Dec-Feb 1677.103 0.0 32.363 \n", - "279 2026/27 MY Jun-May 934.571 1561.322 140.0 \n", - "\n", - " Total supply 3 Food use Seed use Feed and residual use \\\n", - "0 1526.0 580.0 -- 109.0 \n", - "1 1510.0 585.0 -- 110.0 \n", - "2 1660.0 578.0 -- 78.0 \n", - "3 1851.0 556.0 -- 87.0 \n", - "4 1981.0 552.0 -- 53.0 \n", - ".. ... ... ... ... \n", - "275 2964.271 960.0 59.7 100.0 \n", - "276 2869.864 241.091 2.653 239.522 \n", - "277 2164.096 245.58 39.658 -54.047 \n", - "278 1709.466 230.975 1.75 -24.747 \n", - "279 2635.893 960.0 59 80.0 \n", - "\n", - " Total domestic use 3 Exports 2 Total disappearance 3 Ending stocks \n", - "0 689.0 345.0 1034.0 492.0 \n", - "1 695.0 485.0 1180.0 330.0 \n", - "2 656.0 332.0 988.0 672.0 \n", - "3 643.0 214.0 857.0 994.0 \n", - "4 605.0 267.0 872.0 1109.0 \n", - ".. ... ... ... ... \n", - "275 1119.7 910.0 2029.7 934.571 \n", - "276 483.266 252.58 735.846 2134.018 \n", - "277 231.191 255.802 486.993 1677.103 \n", - "278 207.978 201.291 409.269 1300.197 \n", - "279 1099.0 775.0 1874.0 761.893 \n", - "\n", - "[280 rows x 13 columns]" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "full_rows = df[~df['Beginning stocks'].isna()]\n", - "full_rows" - ] - }, - { - "cell_type": "markdown", - "id": "e914ce69", - "metadata": {}, - "source": [ - "## Interoperate with SQL using the BigQuery SQL magics (%%bqsql)\n", - "\n", - "The BigQuery DataFrames library provides the `%%bqsql` magic, which acts as the bridge between your Python and SQL environments. It allows the BigQuery query engine to directly reference and query your local Pandas DataFrames (by implicitly uploading them as temporary tables) as well as actual BigQuery tables and external tables in GCS (Parquet, Iceberg, CSV).\n", - "\n", - "To enable this integration in your notebook, load the `bigframes` extension. This is already completed in BigQuery Studio, Colab Enterprise, and Colab notebooks. For other environments, such as VS Code and Jupyter Lab, run the following cell:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "3d837a5e", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext bigframes\n" - ] - }, - { - "cell_type": "markdown", - "id": "315a53b5", - "metadata": {}, - "source": [ - "To ensure the correct Google Cloud project is billed for query usage, including free tier usage, configure the project ID used by the magics. Even in the free sandbox tier, a project ID is required to allocate query resources. If you don't set it explicitly, BigFrames will try to discover it from your environment (e.g., your Application Default Credentials).\n" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "ffe5757c", - "metadata": {}, - "outputs": [], - "source": [ - "import bigframes.pandas as bpd\n", - "\n", - "PROJECT_ID = \"\" # @param {type:\"string\"}\n", - "bpd.options.bigquery.project = PROJECT_ID\n" - ] - }, - { - "cell_type": "markdown", - "id": "fe174ed2", - "metadata": {}, - "source": [ - "### Querying Local Pandas DataFrames with SQL\n", - "\n", - "With the project configured, you can now run SQL queries directly against your local Pandas DataFrame (`full_rows`) as if it were a table in BigQuery. Simply reference the variable name inside braces `{full_rows}` in your SQL query.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "fbbf52d6", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes. [Job bigframes-dev:US.c3c67902-6a45-492a-9491-a91daddaada1 details]\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Load job c22ec1ce-09da-4ea1-b0a0-f28eee65aa20 is DONE. Open Job" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " Query processed 30.0 kB in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    Marketing year 1Time periodBeginning stocksProductionImports 2Total supply 3Food useSeed useFeed and residual useTotal domestic use 3Exports 2Total disappearance 3Ending stocks
    01980/81Q2 Sep-Nov2714.00.00.62714.6162.1764.865242.965379.335622.32092.3
    11987/88Q2 Sep-Nov2976.4620.04.5252980.987193.04858-79.082171.966308.453480.4192500.568
    22014/15Q2 Sep-Nov1907.220.034.5511941.771248.18748.802-92.585204.404207.737412.1411529.63
    31976/77Q2 Sep-Nov2385.20.00.52385.7153.064-2.795214.205277.295491.51894.2
    41994/95Q2 Sep-Nov2069.4940.021.4232090.917229.29760.954-28.64261.611338.202599.8131491.104
    52002/03Q2 Sep-Nov1748.9870.023.0871772.074237.75454.599-74.678217.675234.53452.2051319.869
    62007/08Q2 Sep-Nov1716.9270.021.4861738.413245.02659.915-119.882185.059421.416606.4751131.938
    72025/26Q2 Sep-Nov2134.0180.030.0782164.096245.5839.658-54.047231.191255.802486.9931677.103
    81995/96Q2 Sep-Nov1881.0990.016.2521897.351232.15164.356-98.182198.325360.759559.0841338.267
    92001/02Q2 Sep-Nov2155.8140.029.042184.854245.08851.601-23.073273.616287.783561.3991623.455
    \n", - "

    10 rows × 13 columns

    \n", - "
    [280 rows x 13 columns in total]" - ], - "text/plain": [ - " Marketing year 1 Time period Beginning stocks Production Imports 2 \\\n", - "0 1980/81 Q2 Sep-Nov 2714.0 0.0 0.6 \n", - "1 1987/88 Q2 Sep-Nov 2976.462 0.0 4.525 \n", - "2 2014/15 Q2 Sep-Nov 1907.22 0.0 34.551 \n", - "3 1976/77 Q2 Sep-Nov 2385.2 0.0 0.5 \n", - "4 1994/95 Q2 Sep-Nov 2069.494 0.0 21.423 \n", - "5 2002/03 Q2 Sep-Nov 1748.987 0.0 23.087 \n", - "6 2007/08 Q2 Sep-Nov 1716.927 0.0 21.486 \n", - "7 2025/26 Q2 Sep-Nov 2134.018 0.0 30.078 \n", - "8 1995/96 Q2 Sep-Nov 1881.099 0.0 16.252 \n", - "9 2001/02 Q2 Sep-Nov 2155.814 0.0 29.04 \n", - "\n", - " Total supply 3 Food use Seed use Feed and residual use \\\n", - "0 2714.6 162.1 76 4.865 \n", - "1 2980.987 193.048 58 -79.082 \n", - "2 1941.771 248.187 48.802 -92.585 \n", - "3 2385.7 153.0 64 -2.795 \n", - "4 2090.917 229.297 60.954 -28.64 \n", - "5 1772.074 237.754 54.599 -74.678 \n", - "6 1738.413 245.026 59.915 -119.882 \n", - "7 2164.096 245.58 39.658 -54.047 \n", - "8 1897.351 232.151 64.356 -98.182 \n", - "9 2184.854 245.088 51.601 -23.073 \n", - "\n", - " Total domestic use 3 Exports 2 Total disappearance 3 Ending stocks \n", - "0 242.965 379.335 622.3 2092.3 \n", - "1 171.966 308.453 480.419 2500.568 \n", - "2 204.404 207.737 412.141 1529.63 \n", - "3 214.205 277.295 491.5 1894.2 \n", - "4 261.611 338.202 599.813 1491.104 \n", - "5 217.675 234.53 452.205 1319.869 \n", - "6 185.059 421.416 606.475 1131.938 \n", - "7 231.191 255.802 486.993 1677.103 \n", - "8 198.325 360.759 559.084 1338.267 \n", - "9 273.616 287.783 561.399 1623.455 \n", - "...\n", - "\n", - "[280 rows x 13 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%%bqsql\n", - "SELECT * FROM {full_rows}\n" - ] - }, - { - "cell_type": "markdown", - "id": "2fcd5284", - "metadata": {}, - "source": [ - "You should see the results from full_rows.\n", - "\n", - "\n", - "## Chaining SQL and Python: Saving SQL Results\n", - "\n", - "The true power of the `%%bqsql` magic lies in chaining. By providing a destination variable name as an argument to `%%bqsql` (e.g., `%%bqsql destination_var`), the query result is saved as a BigQuery DataFrame (a.k.a. BigFrames DataFrame) to that variable. \n", - "\n", - "This DataFrame lives on the BigQuery engine but behaves like a Pandas DataFrame in Python. You can immediately use it in subsequent Python cells, or reference it again in another SQL cell. This allows you to build a multi-step, hybrid processing pipeline.\n", - "\n", - "Filter the data to only yearly entries using SQL, and save the result into a new BigFrames DataFrame named `yearly`:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "75fe0e10", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes in a moment of slot time. [Job bigframes-dev:US.71850fc1-147f-44f7-b4c0-b94592f55639 details]\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Load job aaa74c26-b3ff-422f-a670-93222188fe9f is DONE. Open Job" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " Query processed 30.0 kB in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    Marketing year 1Time periodBeginning stocksProductionImports 2Total supply 3Food useSeed useFeed and residual useTotal domestic use 3Exports 2Total disappearance 3Ending stocks
    01955/56MY Jun-May1109.0937.010.02056.0553.0--51.0604.0322.0926.01130.0
    11957/58MY Jun-May1004.0956.010.01970.0547.0--43.0590.0418.01008.0962.0
    21954/55MY Jun-May994.0984.03.01981.0552.0--53.0605.0267.0872.01109.0
    31951/52MY Jun-May492.0988.030.01510.0585.0--110.0695.0485.01180.0330.0
    41956/57MY Jun-May1130.01005.08.02143.0541.0--57.0598.0541.01139.01004.0
    51950/51MY Jun-May496.01019.011.01526.0580.0--109.0689.0345.01034.0492.0
    61962/63MY Jun-May1420.61092.05.32517.9502.761.434.7598.8649.41248.21269.7
    71959/60MY Jun-May1368.01118.07.02493.0558.0--49.0607.0502.01109.01384.0
    81963/64MY Jun-May1269.71146.84.02420.5487.964.928.6581.4845.61427.0993.5
    91953/54MY Jun-May672.01173.06.01851.0556.0--87.0643.0214.0857.0994.0
    \n", - "

    10 rows × 13 columns

    \n", - "
    [77 rows x 13 columns in total]" - ], - "text/plain": [ - " Marketing year 1 Time period Beginning stocks Production Imports 2 \\\n", - "0 1955/56 MY Jun-May 1109.0 937.0 10.0 \n", - "1 1957/58 MY Jun-May 1004.0 956.0 10.0 \n", - "2 1954/55 MY Jun-May 994.0 984.0 3.0 \n", - "3 1951/52 MY Jun-May 492.0 988.0 30.0 \n", - "4 1956/57 MY Jun-May 1130.0 1005.0 8.0 \n", - "5 1950/51 MY Jun-May 496.0 1019.0 11.0 \n", - "6 1962/63 MY Jun-May 1420.6 1092.0 5.3 \n", - "7 1959/60 MY Jun-May 1368.0 1118.0 7.0 \n", - "8 1963/64 MY Jun-May 1269.7 1146.8 4.0 \n", - "9 1953/54 MY Jun-May 672.0 1173.0 6.0 \n", - "\n", - " Total supply 3 Food use Seed use Feed and residual use \\\n", - "0 2056.0 553.0 -- 51.0 \n", - "1 1970.0 547.0 -- 43.0 \n", - "2 1981.0 552.0 -- 53.0 \n", - "3 1510.0 585.0 -- 110.0 \n", - "4 2143.0 541.0 -- 57.0 \n", - "5 1526.0 580.0 -- 109.0 \n", - "6 2517.9 502.7 61.4 34.7 \n", - "7 2493.0 558.0 -- 49.0 \n", - "8 2420.5 487.9 64.9 28.6 \n", - "9 1851.0 556.0 -- 87.0 \n", - "\n", - " Total domestic use 3 Exports 2 Total disappearance 3 Ending stocks \n", - "0 604.0 322.0 926.0 1130.0 \n", - "1 590.0 418.0 1008.0 962.0 \n", - "2 605.0 267.0 872.0 1109.0 \n", - "3 695.0 485.0 1180.0 330.0 \n", - "4 598.0 541.0 1139.0 1004.0 \n", - "5 689.0 345.0 1034.0 492.0 \n", - "6 598.8 649.4 1248.2 1269.7 \n", - "7 607.0 502.0 1109.0 1384.0 \n", - "8 581.4 845.6 1427.0 993.5 \n", - "9 643.0 214.0 857.0 994.0 \n", - "...\n", - "\n", - "[77 rows x 13 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%%bqsql yearly\n", - "SELECT *\n", - "FROM {full_rows}\n", - "WHERE STARTS_WITH(`Time period`, 'MY')\n" - ] - }, - { - "cell_type": "markdown", - "id": "19a70e9e", - "metadata": {}, - "source": [ - "### Chaining Step 2: Complex SQL Transformation on the BigFrames DataFrame\n", - "\n", - "Now, you can chain another SQL operation. Reference the `yearly` BigFrames DataFrame that you just created, extract the year using SQL regular expressions, cast it to a timestamp, and save the results into a new BigFrames DataFrame named `timeseries`.\n", - "\n", - "Notice how you are building a chain: Local Pandas -> [SQL filter] -> BigFrames `yearly` -> [SQL transform] -> BigFrames `timeseries`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "8fbb5224", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes in a moment of slot time. [Job bigframes-dev:US.fdcdabc9-e1a3-47e6-ab27-9e19d9aaa106 details]\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Load job ec6be16a-722d-4151-b7a6-5e410557577d is DONE. Open Job" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " Query processed 8.3 kB in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    Time periodBeginning stocksProductionImports 2Total supply 3Food useSeed useFeed and residual useTotal domestic use 3Exports 2Total disappearance 3Ending stocksyear
    0MY Jun-May1004.0956.010.01970.0547.0--43.0590.0418.01008.0962.01957-01-01 00:00:00+00:00
    1MY Jun-May672.01173.06.01851.0556.0--87.0643.0214.0857.0994.01953-01-01 00:00:00+00:00
    2MY Jun-May496.01019.011.01526.0580.0--109.0689.0345.01034.0492.01950-01-01 00:00:00+00:00
    3MY Jun-May330.01306.024.01660.0578.0--78.0656.0332.0988.0672.01952-01-01 00:00:00+00:00
    4MY Jun-May962.01457.08.02427.0561.0--48.0609.0450.01059.01368.01958-01-01 00:00:00+00:00
    5MY Jun-May994.0984.03.01981.0552.0--53.0605.0267.0872.01109.01954-01-01 00:00:00+00:00
    6MY Jun-May492.0988.030.01510.0585.0--110.0695.0485.01180.0330.01951-01-01 00:00:00+00:00
    7MY Jun-May1130.01005.08.02143.0541.0--57.0598.0541.01139.01004.01956-01-01 00:00:00+00:00
    8MY Jun-May1109.0937.010.02056.0553.0--51.0604.0322.0926.01130.01955-01-01 00:00:00+00:00
    9MY Jun-May1368.01118.07.02493.0558.0--49.0607.0502.01109.01384.01959-01-01 00:00:00+00:00
    \n", - "

    10 rows × 13 columns

    \n", - "
    [77 rows x 13 columns in total]" - ], - "text/plain": [ - " Time period Beginning stocks Production Imports 2 Total supply 3 \\\n", - "0 MY Jun-May 1004.0 956.0 10.0 1970.0 \n", - "1 MY Jun-May 672.0 1173.0 6.0 1851.0 \n", - "2 MY Jun-May 496.0 1019.0 11.0 1526.0 \n", - "3 MY Jun-May 330.0 1306.0 24.0 1660.0 \n", - "4 MY Jun-May 962.0 1457.0 8.0 2427.0 \n", - "5 MY Jun-May 994.0 984.0 3.0 1981.0 \n", - "6 MY Jun-May 492.0 988.0 30.0 1510.0 \n", - "7 MY Jun-May 1130.0 1005.0 8.0 2143.0 \n", - "8 MY Jun-May 1109.0 937.0 10.0 2056.0 \n", - "9 MY Jun-May 1368.0 1118.0 7.0 2493.0 \n", - "\n", - " Food use Seed use Feed and residual use Total domestic use 3 Exports 2 \\\n", - "0 547.0 -- 43.0 590.0 418.0 \n", - "1 556.0 -- 87.0 643.0 214.0 \n", - "2 580.0 -- 109.0 689.0 345.0 \n", - "3 578.0 -- 78.0 656.0 332.0 \n", - "4 561.0 -- 48.0 609.0 450.0 \n", - "5 552.0 -- 53.0 605.0 267.0 \n", - "6 585.0 -- 110.0 695.0 485.0 \n", - "7 541.0 -- 57.0 598.0 541.0 \n", - "8 553.0 -- 51.0 604.0 322.0 \n", - "9 558.0 -- 49.0 607.0 502.0 \n", - "\n", - " Total disappearance 3 Ending stocks year \n", - "0 1008.0 962.0 1957-01-01 00:00:00+00:00 \n", - "1 857.0 994.0 1953-01-01 00:00:00+00:00 \n", - "2 1034.0 492.0 1950-01-01 00:00:00+00:00 \n", - "3 988.0 672.0 1952-01-01 00:00:00+00:00 \n", - "4 1059.0 1368.0 1958-01-01 00:00:00+00:00 \n", - "5 872.0 1109.0 1954-01-01 00:00:00+00:00 \n", - "6 1180.0 330.0 1951-01-01 00:00:00+00:00 \n", - "7 1139.0 1004.0 1956-01-01 00:00:00+00:00 \n", - "8 926.0 1130.0 1955-01-01 00:00:00+00:00 \n", - "9 1109.0 1384.0 1959-01-01 00:00:00+00:00 \n", - "...\n", - "\n", - "[77 rows x 13 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%%bqsql timeseries\n", - "SELECT\n", - " * EXCEPT (`Marketing year 1`),\n", - " TIMESTAMP(CONCAT(\n", - " REGEXP_EXTRACT(`Marketing year 1`, r'([0-9]+)\\/'),\n", - " '-01-01')) AS `year`\n", - "FROM {yearly}\n" - ] - }, - { - "cell_type": "markdown", - "id": "76ba8a7d", - "metadata": {}, - "source": [ - "## Chaining Back to Python: Visualizing BigFrames Data\n", - "\n", - "Now that you've completed some SQL transformations, you can chain back to Python for visualization. Because BigFrames DataFrames implement the Pandas API, you can call standard visualization methods (like `.plot.line()`) directly on the `timeseries` DataFrame without downloading the full dataset first. The computations happen in BigQuery, and only the summarized chart data is sent back to the notebook.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "d3ff4eec", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " Query processed 8.8 kB in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Load job c6b4d65a-4555-4efc-9f8b-7156f4c62835 is DONE. Open Job" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGwCAYAAAC3qV8qAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjksIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvJkbTWQAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsnXd4FGXXh+/Zmk3Z9JCEBAiE3psSUKlSBARBxPJRFPRFUcQCigVFVCxgVywooIgKFlREigioFEGQIqGGQAIklPS2fb4/Zneym56QkATmvq65ksw8M/PMbnbnzDm/c44giqKIgoKCgoKCgkI9QlXbE1BQUFBQUFBQqCyKAaOgoKCgoKBQ71AMGAUFBQUFBYV6h2LAKCgoKCgoKNQ7FANGQUFBQUFBod6hGDAKCgoKCgoK9Q7FgFFQUFBQUFCod2hqewI1hcPh4OzZs/j5+SEIQm1PR0FBQUFBQaECiKJITk4OkZGRqFSl+1muWAPm7NmzREdH1/Y0FBQUFBQUFKpAcnIyUVFRpW6/Yg0YPz8/QHoBjEZjLc9GQUFBQUFBoSJkZ2cTHR0t38dL44o1YFxhI6PRqBgwCgoKCgoK9Yzy5B+KiFdBQUFBQUGh3qEYMAoKCgoKCgr1DsWAUVBQUFBQUKh3XLEamIpit9uxWq21PQ0FhTqHVqtFrVbX9jQUFBQUSuSqNWBEUSQ1NZXMzMzanoqCQp0lICCA8PBwpZaSgoJCneOqNWBcxktYWBje3t7KF7SCghuiKJKfn8/58+cBiIiIqOUZKSgoKHhyVRowdrtdNl6Cg4NrezoKCnUSg8EAwPnz5wkLC1PCSQoKCnWKq1LE69K8eHt71/JMFBTqNq7PiKITU1BQqGtclQaMCyVspKBQNspnREFBoa5yVRswCgoKCgoKCvUTxYBRUFBQUFBQqHcoBoxCifTp04fp06dX6zGXLFlCQEBAtR6zrlATr5eCgoKCQulckgHzyiuvIAiCxxe3yWRi6tSpBAcH4+vry+jRozl37pzHfklJSQwdOhRvb2/CwsKYMWMGNpvNY8zmzZvp0qULer2e2NhYlixZcilTvSKYOHEigiDIS3BwMIMHD2b//v3Vfq7vv/+euXPnVusxx44dy9GjR6v1mJXlSjai6jOixYLDYqntaSgoKNQjqmzA7Nq1i48++ogOHTp4rH/kkUf4+eefWblyJVu2bOHs2bOMGjVK3m632xk6dCgWi4Vt27axdOlSlixZwuzZs+UxiYmJDB06lL59+7J3716mT5/O5MmTWbduXVWne8UwePBgUlJSSElJYePGjWg0GoYNG1bt5wkKCiq3lXllMRgMhIWFVesxFeo/jrw8EobcxInBQ7AkJdX2dBQUFOoLYhXIyckRmzdvLm7YsEHs3bu3+PDDD4uiKIqZmZmiVqsVV65cKY89dOiQCIjbt28XRVEU16xZI6pUKjE1NVUes3DhQtFoNIpms1kURVGcOXOm2LZtW49zjh07Vhw0aFCpczKZTGJWVpa8JCcni4CYlZVVbGxBQYEYHx8vFhQUiKIoig6HQ8wzW2tlcTgcFX7dJ0yYII4YMcJj3Z9//ikC4vnz5+V1SUlJ4pgxY0R/f38xMDBQvPnmm8XExER5u9VqFR966CHR399fDAoKEmfOnCmOHz/e49ju76soimLjxo3Fl156Sbz77rtFX19fMTo6Wvzoo4/k7YmJiSIgfvfdd2KfPn1Eg8EgdujQQdy2bZs8ZvHixaK/v7/893PPPSd27NhR/Pzzz8XGjRuLRqNRHDt2rJidnS2Pyc7OFu+8807R29tbDA8PF994441icyvK3r17xT59+oi+vr6in5+f2KVLF3HXrl3ipk2bRMBjee6550RRFMX09HRx3LhxYkBAgGgwGMTBgweLR48e9TjuX3/9Jfbu3Vs0GAxiQECAOHDgQDE9Pb3E12v16tWi0WgUly1bJoqiKG7atEns3r276O3tLfr7+4s9e/YUT548Weo11BWKflYqgi07W0x+eLp4cvwE0ZaTU+74i4sXi/EtW4nxLVuJR/v0Fc3JyZcyZQUFhXpOVlZWqfdvd6pUyG7q1KkMHTqUAQMG8OKLL8rrd+/ejdVqZcCAAfK6Vq1a0ahRI7Zv306PHj3Yvn077du3p0GDBvKYQYMGcf/993Pw4EE6d+7M9u3bPY7hGlOWxmDevHnMmTOnKpdDgdVOm9m1492Jf2EQ3rqq1RPMzc1l2bJlxMbGygX5rFYrgwYNIi4ujj///BONRsOLL74oh5p0Oh2vvvoqX375JYsXL6Z169a8/fbbrFq1ir59+5Z5vgULFjB37lyeeuopvv32W+6//3569+5Ny5Yt5TFPP/008+fPp3nz5jz99NPccccdHD9+HI2m5GtMSEhg1apVrF69moyMDG677TZeeeUVXnrpJQAeffRRtm7dyk8//USDBg2YPXs2e/bsoVOnTqXO86677qJz584sXLgQtVrN3r170Wq19OzZk7feeovZs2dz5MgRAHx9fQEpPHfs2DF++uknjEYjTzzxBDfddBPx8fFotVr27t1L//79ueeee3j77bfRaDRs2rQJu91e7PzLly9nypQpLF++nGHDhmGz2Rg5ciT33nsvX331FRaLhZ07d16RKcrW8+dJvu9/mA8fBiB96VJCp04tdbxosZC+eAkAKh8fbCkpJE2YSOPPl6Jt2PByTFlBQaGeUuk759dff82ePXvYtWtXsW2pqanodLpiGoMGDRqQmpoqj3E3XlzbXdvKGpOdnU1BQYFcIdSdWbNm8eijj8p/Z2dnEx0dXdnLq/OsXr1avunm5eURERHB6tWrUamkaOA333yDw+Fg0aJF8g1y8eLFBAQEsHnzZgYOHMi7777LrFmzuOWWWwB47733WLNmTbnnvummm3jggQcAeOKJJ3jzzTfZtGmThwHz+OOPM3ToUADmzJlD27ZtOX78OK1atSrxmA6HgyVLlsjhqnHjxrFx40ZeeuklcnJyWLp0KcuXL6d///7ytURGRpY5z6SkJGbMmCGfs3nz5vI2f39/BEEgPDxcXucyXLZu3UrPnj0B+PLLL4mOjmbVqlWMGTOG1157jW7duvHBBx/I+7Vt27bYud9//32efvppfv75Z3r37g1I/4tZWVkMGzaMZs2aAdC6desyr6E+Yjl5kqRJk7GeOYNgMCAWFJC+eAlBd92FuhTdUdbPq7GdO4cmNJTGX31F8qRJWE6d4tTEu2n8xedo3d4nBQUFBXcqZcAkJyfz8MMPs2HDBry8vGpqTlVCr9ej1+urtK9Bqyb+hUHVPKOKn7sy9O3bl4ULFwKQkZHBBx98wJAhQ9i5cyeNGzdm3759HD9+vJh+xWQykZCQQFZWFufOneOaa66Rt6nVarp27YrD4Sjz3O56J5cR4OqVU9IYV/+c8+fPl2rANGnSxGOuERER8jFPnDiB1Wr1mKu/v7+HwVQSjz76KJMnT+aLL75gwIABjBkzRjYcSuLQoUNoNBquvfZaeV1wcDAtW7bk0KFDAOzdu5cxY8aUed5vv/2W8+fPs3XrVrp37y6vDwoKYuLEiQwaNIgbb7yRAQMGcNttt11R/YUKDvxH8v/+hz09HW2jRjT65GNOPzwd8+HDpH36GWGPPVpsH9HhIG3RIgCCJk5EF9WQRkuXcGrceKzJyZyaMIHGn3+BtoGim1JQUChOpUS8u3fv5vz583Tp0gWNRoNGo2HLli288847aDQaGjRogMViKdbh+dy5c/ITb3h4eLGsJNff5Y0xGo0lel8uFUEQ8NZpamWpbBjBx8eH2NhYYmNj6d69O4sWLSIvL49PPvkEkMJKXbt2Ze/evR7L0aNHufPOOy/pddJqtcVet6JGj/sY17WVZRhV5JiV5fnnn+fgwYMMHTqU33//nTZt2vDDDz9c0jEr8n/XuXNnQkND+eyzzxBF0WPb4sWL2b59Oz179uSbb76hRYsW7Nix45LmVFfI3bqVUxMmYE9Px6tNG5os/xJd48aETpsGQPqyZdguXiy2X87GjVgSE1EZjQSMvQ0AbXg4jZcuQduwIdZTSSRNmIC1iJGsoKCgAJU0YPr378+BAwc8bozdunXjrrvukn/XarVs3LhR3ufIkSMkJSURFxcHQFxcHAcOHPB4ct+wYQNGo5E2bdrIY9yP4RrjOoZCIYIgoFKpKCgoAKBLly4cO3aMsLAw2dBxLf7+/vj7+9OgQQOPEKDdbmfPnj21dQml0rRpU7Rarcdcs7KyKpSK3aJFCx555BHWr1/PqFGjWLx4MQA6na6YbqV169bYbDb+/vtveV1aWhpHjhyR/yc7dOhQ7H+yKM2aNWPTpk38+OOPPPTQQ8W2d+7cmVmzZrFt2zbatWvH8uXLy72OuoQ9Nw9zYiL5u3aRvWYN6Z9/zrl580iecj9ifj7ecT1o9PlSNCEhAPj27YNXhw6IBQVc/Phjj2OJokjaJ5L3JfDOO1A7w6IA2shIGi1diiYyAsvJk5x5pLj3RkFBQaFSISQ/Pz/atWvnsc7Hx4fg4GB5/aRJk3j00UcJCgrCaDTy0EMPERcXR48ePQAYOHAgbdq0Ydy4cbz22mukpqbyzDPPMHXqVDkENGXKFN577z1mzpzJPffcw++//86KFSv45ZdfquOa6zVms1nWCmVkZPDee++Rm5vL8OHDAUnA+vrrrzNixAheeOEFoqKiOHXqFN9//z0zZ84kKiqKhx56iHnz5hEbG0urVq149913ycjIqHOiUj8/PyZMmMCMGTMICgoiLCyM5557DpVKVepcCwoKmDFjBrfeeisxMTGcPn2aXbt2MXr0aEAKWeXm5rJx40Y6duyIt7c3zZs3Z8SIEdx777189NFH+Pn58eSTT9KwYUNGjBgBSBqr9u3b88ADDzBlyhR0Oh2bNm1izJgxhDhv2CAZTps2baJPnz5oNBreeustEhMT+fjjj7n55puJjIzkyJEjHDt2jPHjx9f8i1hNXFj4IbkffVTqdr8hg4l89VVUOp28ThAEwqY/TNI9k8j86muC774brTNslv/3Tkz79yPo9QSNG1fseLqohkR/+CGJN4+gYM8eRLsdQemGraCg4EbV0l/K4M0330SlUjF69GjMZjODBg3yED6q1WpWr17N/fffT1xcHD4+PkyYMIEXXnhBHhMTE8Mvv/zCI488wttvv01UVBSLFi1i0KDa0anUJdauXStrJ/z8/GjVqhUrV66kT58+gNQ9+I8//uCJJ55g1KhR5OTk0LBhQ/r374/RaAQkAW5qairjx49HrVZz3333MWjQINR18AbxxhtvMGXKFIYNG4bRaGTmzJkkJyeXqsFSq9WkpaUxfvx4zp07R0hICKNGjZIz1Hr27MmUKVMYO3YsaWlpPPfcczz//PMsXryYhx9+mGHDhmGxWLjhhhtYs2aNHOJq0aIF69ev56mnnuKaa67BYDBw7bXXcscddxSbQ8uWLfn999/p06cParWamTNncvjwYZYuXUpaWhoRERFMnTqV//3vfzX3wlUjos1G9urVqACVtzfq0BA0oaHSEhKKV6tW+N8yEkFV3KHrHReH9zXXkL9zJxcXfkjEC9L7kOb0yASMHo3GmUFXFH1MjHMCIvbMzFLHKSgoXJ0IYtFg/RVCdnY2/v7+ZGVlyTduFyaTicTERGJiYuqcGLk2cDgctG7dmttuu63aq+9WN3l5eTRs2JAFCxYwadKk2p7OFY/JZOL4v/8iPPU0fs2a0WjRJ5U+Rv6ePZy68y7QaGi25hfs2TmcvPVWUKtptm4tuqioUvc9em0P7FlZNP35J/Ru2WQKCgpXLmXdv92pdg+MQt3n1KlTrF+/nt69e2M2m3nvvfdITEy8ZJFvTfDvv/9y+PBhrrnmGrKysmRPnSu0o1CziDYbjoIC1EDw5KoZjN5duuBzw/Xk/fEnF99/H4dZahlgvOmmMo0XAHVQEPasLGzpGVQtx1BBQeFKRTFgrkJUKhVLlizh8ccfRxRF2rVrx2+//VZna5PMnz+fI0eOoNPp6Nq1K3/++aeH7kSh5rBlZYEooo+NxdstzbyyhE57mLw//iTrp5/ldcGTJ5e7nzooCBITsaenVfncCgoKVyaKAXMVEh0dzdatW2t7GhWic+fO7N69u7ancVUiOhw4nCUR/EePuiSRt6FdW/xuvJGcDRsA8O3dG6+WLcrdTxMUBIAtPb3K51ZQULgyuaRu1AoKClcu9owMRLsd1Gp8r7vuko8XOu0hcBpBwffdW6F91E4Dxp6eccnnV1BQuLJQPDAKCgrFEEURW5oUtlH7+CCU0suqMuibN6fhGwuw5+bi3bVrhfbRBLs8MEoISUFBwRPFgFFQUCiGIzsb0WJBUKsRvL2r7bjGIUMqNV4dqHhgFBQUSkYJISkoKHggiqJc+l/l719ifZfLhTooEAB7muKBUVBQ8EQxYBQUFDxw5OfjKCgAQUBTShfpy4WreJ0tQ/HAKCgoeKIYMAolMnHiREaOHFnj52nSpAlvvfVWjZ9HoeLYnd4XdUBgtWhfLoXCEJKShaSgoOCJYsDUMyZOnIggCAiCgE6nIzY2lhdeeAGbzVbbUyuTJUuWEFDC0/yuXbu47777Lv+EFErEYTJhz8kBQBNS+6X7Na4QUmamlBGloKCg4EQxYOohgwcPJiUlhWPHjvHYY4/x/PPP8/rrrxcbZ7FYamF2lSM0NBTvahSJKlwatjTJ06E2GlHpa7/2rTpQMmBc/ZAUFBQUXCgGTD1Er9cTHh5O48aNuf/++xkwYAA//fSTHPZ56aWXiIyMpGXLlgAcOHCAfv36YTAYCA4O5r777iM3N1c+nt1u59FHHyUgIIDg4GBmzpxJ0RZZJYV6OnXqxPPPPy//nZmZyf/+9z8aNGiAl5cX7dq1Y/Xq1WzevJm7776brKws2Xvk2q/ocZOSkhgxYgS+vr4YjUZuu+02zp07J29//vnn6dSpE1988QVNmjTB39+f22+/nRyn10Dh0hAL8gFQ17L2xYWg0aD29weUMJKCgoInigEDIIpgyaudpRp6aRoMBtnbsnHjRo4cOcKGDRtYvXo1eXl5DBo0iMDAQHbt2sXKlSv57bffePDBB+X9FyxYwJIlS/jss8/466+/SE9P54cffqjUHBwOB0OGDGHr1q0sW7aM+Ph4XnnlFdRqNT179uStt97CaDSSkpJCSkoKjz/+eInHGDFiBOnp6WzZsoUNGzZw4sQJxo4d6zEuISGBVatWsXr1alavXs2WLVt45ZVXqvDKKbgjiiIO5/+RUAe8Ly7ULiFvmmLAKCgoFKLUgQGw5sPLkbVz7qfOgs6nSruKosjGjRtZt24dDz30EBcuXMDHx4dFixah0+kA+OSTTzCZTHz++ef4+Ejnee+99xg+fDivvvoqDRo04K233mLWrFmMGjUKgA8//JB169ZVai6//fYbO3fu5NChQ7RoIZWIb9q0qbzd398fQRAIDw8v9RgbN27kwIEDJCYmEh0dDcDnn39O27Zt2bVrF927dwckQ2fJkiX4+fkBMG7cODZu3MhLL71UqTkreCLabOBwAAKCVlvb05FRBwXCCbBnKAaMgoJCIYoHph6yevVqfH198fLyYsiQIYwdO1YOybRv3142XgAOHTpEx44dZeMFoFevXjgcDo4cOUJWVhYpKSlc69aoT6PR0K1bt0rNae/evURFRcnGS1U4dOgQ0dHRsvEC0KZNGwICAjh06JC8rkmTJrLxAhAREcH58+erfF4FCdHlfdFpa7X2S1E0gUo/JAUFheIoHhgArbfkCamtc1eSvn37snDhQnQ6HZGRkWjcUl3dDZXqRKVSFdPFWK1W+XeDwVAj5y0JbRHvgCAIOByOy3b+KxXRbAZAcDOA6wJqZzsBuxJCUlBQcKPuPGbVJoIghXFqY6lCh18fHx9iY2Np1KiRh/FSEq1bt2bfvn3k5eXJ67Zu3YpKpaJly5b4+/sTERHB33//LW+32WzFOkCHhoaSkpIi/52dnU1iYqL8d4cOHTh9+jRHjx4tcR46nQ57OWmwrVu3Jjk5meTkZHldfHw8mZmZtGnTpsx9FS4dlwdGpas7+hdw60ithJAUFBTcUAyYK5y77roLLy8vJkyYwH///cemTZt46KGHGDduHA0aNADg4Ycf5pVXXmHVqlUcPnyYBx54gMwiKav9+vXjiy++4M8//+TAgQNMmDABtVotb+/duzc33HADo0ePZsOGDSQmJvLrr7+ydu1aQAr75ObmsnHjRi5evEh+fn6xuQ4YMID27dtz1113sWfPHnbu3Mn48ePp3bt3pUNaCpVHDiHp65gHJlDxwCgoKBRHMWCucLy9vVm3bh3p6el0796dW2+9lf79+/Pee+/JYx577DHGjRvHhAkTiIuLw8/Pj1tuucXjOLNmzaJ3794MGzaMoUOHMnLkSJo1a+Yx5rvvvqN79+7ccccdtGnThpkzZ8pel549ezJlyhTGjh1LaGgor732WrG5CoLAjz/+SGBgIDfccAMDBgygadOmfPPNNzXwyigUpa6GkFwdqZU0agUFBXcEsaiw4QohOzsbf39/srKyMBqNHttMJhOJiYnExMTg5eVVSzNUUKg7iKKIKT4eRBF98+ZyEbu68FnJ27GDpIl3o2vWjGa/rK6VOSgoKFw+yrp/u6N4YBQUFBCtVqkmkSDUOQ9MYQhJ6UitoKBQiGLAKCgoFOpftFqEKgjLaxI5hJSVpfRDUlBQkFEMGAUFhcIMpDpUgdeF3NZA6YekoKDghmLAKCgoIJpdRezqVvgInP2QnEaMTQkjKSgoOFEMGAUFBURL3cxAcqEOcmUiZdTyTBQUFOoKigGjoKBQ2MSxjhWxc6EOCgSUfkgKCgqFKAaMgsJVjiiKdbaInQtNkNKRWkFBwRPFgFFQuMrxSKGuQ12o3ZE9MEoxOwUFBSeKAaOgcJVT2IVaV+dSqF0o/ZAUFBSKohgwCgpXOa4WAqo6KuAFUDtDSEo/JAUFBReKAVOPmDhxIiNHjqztaZTKyZMnEQSBvXv3XtIxJk2aRExMDAaDgWbNmvHcc89hcXoJFKofdw9MXUWjhJAUFBSKUCkDZuHChXTo0AGj0YjRaCQuLo5ff/1V3t6nTx8EQfBYpkyZ4nGMpKQkhg4dire3N2FhYcyYMQObzeYxZvPmzXTp0gW9Xk9sbCxLliyp+hUqXBaqy8A4fPgwDoeDjz76iIMHD/Lmm2/y4Ycf8tRTT1XL8RWKUyjgrZsZSFDogbEpBoyCgoKTShkwUVFRvPLKK+zevZt//vmHfv36MWLECA4ePCiPuffee0lJSZEX967DdrudoUOHYrFY2LZtG0uXLmXJkiXMnj1bHpOYmMjQoUPp27cve/fuZfr06UyePJl169ZVw+VeWfTp04eHHnqI6dOnExgYSIMGDfjkk0/Iy8vj7rvvxs/Pj9jYWA8jc/PmzQiCwC+//EKHDh3w8vKiR48e/Pfffx7H/u6772jbti16vZ4mTZqwYMECj+1NmjRh7ty5jB8/HqPRyH333UdMTAwAnTt3RhAE+vTpI5/zmmuuwcfHh4CAAHr16sWpU6dKvKbBgwezePFiBg4cSNOmTbn55pt5/PHH+f7776vxlVNwx1FHu1C7o4h4FRQUiiFeIoGBgeKiRYtEURTF3r17iw8//HCpY9esWSOqVCoxNTVVXrdw4ULRaDSKZrNZFEVRnDlzpti2bVuP/caOHSsOGjSozHmYTCYxKytLXpKTk0VAzMrKKja2oKBAjI+PFwsKCkRRFEWHwyHmWfJqZXE4HBV6nUVRFCdMmCCOGDFC/rt3796in5+fOHfuXPHo0aPi3LlzRbVaLQ4ZMkT8+OOPxaNHj4r333+/GBwcLObl5YmiKIqbNm0SAbF169bi+vXrxf3794vDhg0TmzRpIlosFlEURfGff/4RVSqV+MILL4hHjhwRFy9eLBoMBnHx4sXyuRs3biwajUZx/vz54vHjx8Xjx4+LO3fuFAHxt99+E1NSUsS0tDTRarWK/v7+4uOPPy4eP35cjI+PF5csWSKeOnWqwtf99NNPi127dq3weIWK43A4xPz//hPzDxwQ7c7PoDtFPyu1hfXCBTG+ZSsxvlVr0WG11upcFBQUapasrKxS79/uaKpq+NjtdlauXEleXh5xcXHy+i+//JJly5YRHh7O8OHDefbZZ/H29gZg+/bttG/fngYNGsjjBw0axP3338/Bgwfp3Lkz27dvZ8CAAR7nGjRoENOnTy9zPvPmzWPOnDlVupYCWwHXLr+2SvteKn/f+TfeWu8q79+xY0eeeeYZAGbNmsUrr7xCSEgI9957LwCzZ89m4cKF7N+/nx49esj7Pffcc9x4440ALF26lKioKH744Qduu+023njjDfr378+zzz4LQIsWLYiPj+f1119n4sSJ8jH69evHY489Jv+tVqsBCA4OJjw8HID09HSysrIYNmwYzZo1A6B169YVvr7jx4/z7rvvMn/+/Mq+NAoVQLRY6nwKNTj7IQmC3A9JExJS21NSUFCoZSot4j1w4AC+vr7o9XqmTJnCDz/8QJs2bQC48847WbZsGZs2bWLWrFl88cUX/N///Z+8b2pqqofxAsh/p6amljkmOzubgoKCUuc1a9YssrKy5CU5Obmyl1Yv6dChg/y7Wq0mODiY9u3by+tcr+X58+c99nM3OoOCgmjZsiWHDh0C4NChQ/Tq1ctjfK9evTh27Bh2t27A3bp1K3d+QUFBTJw4kUGDBjF8+HDefvttUlJSKnRtZ86cYfDgwYwZM0Y2yBSqF7mJYx1OoQZnPyR/f0DRwSgoKEhU2gPTsmVL9u7dS1ZWFt9++y0TJkxgy5YttGnThvvuu08e1759eyIiIujfvz8JCQny03dNodfr0VdRhGjQGPj7zr+reUYVP/eloC3y1CwIgsc6103J4XBc0nlKwsfHp0LjFi9ezLRp01i7di3ffPMNzzzzDBs2bPDwCBXl7Nmz9O3bl549e/Lxxx9X15QVilAfBLwu1EFB2DMzlX5ICgoKQBUMGJ1OR2xsLABdu3Zl165dvP3223z00UfFxl57rRSWOX78OM2aNSM8PJydO3d6jDl37hyAHHIIDw+X17mPMRqNGAyXdrMvDUEQLimMUx/ZsWMHjRo1AiAjI4OjR4/KoZ3WrVuzdetWj/Fbt26lRYsWcpioJHROEai7l8ZF586d6dy5M7NmzSIuLo7ly5eXasCcOXOGvn370rVrVxYvXoxKpWT71xR1uQt1UTRBQVhOnMCernSkVlBQqIY6MA6HA7Mzi6EornogERERgBS2OHDggEc4Y8OGDRiNRjkMFRcXx8aNGz2Os2HDBo+Qh8Kl88ILL7Bx40b+++8/Jk6cSEhIiFxj5rHHHmPjxo3MnTuXo0ePsnTpUt577z0ef/zxMo8ZFhaGwWBg7dq1nDt3jqysLBITE5k1axbbt2/n1KlTrF+/nmPHjpWqgzlz5gx9+vShUaNGzJ8/nwsXLpCamiqHGBWqF0cd70LtjqsjtU3xwCgoKFBJD8ysWbMYMmQIjRo1Iicnh+XLl7N582bWrVtHQkICy5cv56abbiI4OJj9+/fzyCOPcMMNN8g6jYEDB9KmTRvGjRvHa6+9RmpqKs888wxTp06Vwz9TpkzhvffeY+bMmdxzzz38/vvvrFixgl9++aX6r/4q5pVXXuHhhx/m2LFjdOrUiZ9//ln2oHTp0oUVK1Ywe/Zs5s6dS0REBC+88IKHgLckNBoN77zzDi+88AKzZ8/m+uuv55tvvuHw4cMsXbqUtLQ0IiIimDp1Kv/73/9KPMaGDRs4fvw4x48fJyoqymObKIrVcu0KhYh1vAu1O+pgyYBRUqkVFBSAyqVR33PPPWLjxo1FnU4nhoaGiv379xfXr18viqIoJiUliTfccIMYFBQk6vV6MTY2VpwxY0axNKiTJ0+KQ4YMEQ0GgxgSEiI+9thjorVIWuSmTZvETp06iTqdTmzatKlH+m5FKSsNq66khtYGrjTqjIyM2p6KQi3jsNvF/APOFGpnCn1R6tJn5fzb74jxLVuJZ597rranoqCgUIPUSBr1p59+Wuq26OhotmzZUu4xGjduzJo1a8oc06dPH/7999/KTE1BQaGSiFYrIIJKhaCpckWFy4YrhKSIeBXqKumff45gMBA4ZkxtT+WqoO5/aykoKNQI9SWF2oUm2KWBUUS8CnUPa2oq516eB4KA8cYbpdpFCjWKkt5xldGnTx9EUSRA+XBd9Yj1oIWAO+rAq9MD48jPx1qkjpNC3cNy4oT0iyhSsH9/7U7mKkExYBQUrlLqQxdqd67WfkinH5rG8Rt6c/qhhzAdPVrb01EoBfPJk/LvBXv31d5EriIUA0ZB4SrFUY+K2AFogqWO1PbMTMQiHeyvVBz5+eRt3w5AzobfSBwxkjOPz8BSSjNUhdrD4mHA7K21eVxNKAaMgsJVSr0LIbn6ISEZMVcDpsNHwOFAHRSE3+DBIIpkr15Nwk1DSXn2WawVbMuhUPN4GDD79yPWQPVzBU8UA0ZB4SpEdDicWUiSiLc+IKjVV10/JNN//wFg6NCBqLfeJOb77/Dt3RvsdjJXfkvC0GHk7dhRy7NUALAknpR/d+TmFmpiFGoMxYBRULgKkfUvKhXUgxRqF2pXGOlqMWAOHgTAq1076WebNkR/9CGNly/H0KkTYn4+yf+bQu4ff9TmNK96HBYL1jNnANDFxABKGOlyoBgwCgpXIe4C3vqQQu1CE1h5Ia8oiuTt2IEto/5lLxUclDwwXm3beKz37tKZRp8vxbdfP0SzmeSpD5K9YUNtTFEBsCYng8OByscHvwH9ASjYpwh5axrFgFEoE0EQWLVqVW1Po0xOnjyJIAhy7y2F8qlPXajdcXlgKtMPKX/HDpIm3k3KM8/W1LRqBEd+PpYTiQB4tW1bbLtKpyPq7bckbYzVypnpj5CltFypFVz6F12TJhg6dQKUTKTLgWLA1BMEQShzef7550vdV7nBl8/3339Pt27dCAgIwMfHh06dOvHFF1/U9rRqDEc96kLtTmEqdcWL2ZkOHQbq3xOx6fBhcDjQhIWhDQsrcYyg1dJw/uv4j7gZ7HbOzphJ5vc/XOaZKngYMM7ef+bjx7Hn5NTirK586k/w+yonxS3b4JtvvmH27NkcOXJEXufr61sb07piCAoK4umnn6ZVq1bodDpWr17N3XffTVhYGIMGDart6VU7jrw8AFReXrU8k8qhCXRV4614CMl6+jQA9osXsaWno3G2JKjruAS8JXlf3BE0GiLmzUPQe5G5YgUpTz2FaLEQePvYyzFNBcCcKHnKdE2aoAkNRduwIdYzZzAdOIBPz561PLsrF8UDU08IDw+XF39/fwRBkP8OCwvjjTfeICoqCr1eT6dOnVi7dq28b4xTVNa5c2cEQaBPnz4A7Nq1ixtvvJGQkBD8/f3p3bs3e/bsqdS8vv32W9q3b4/BYCA4OJgBAwaQ57w59unTh+nTp3uMHzlypEdX6yZNmjB37lzuuOMOfHx8aNiwIe+//77HPoIgsHDhQoYMGYLBYKBp06Z8++23Jc5HFEViY2OZP3++x/q9e/ciCALHjx8vcb8+ffpwyy230Lp1a5o1a8bDDz9Mhw4d+Ouvvyr1etQHHGYzosUMgoCqnhm+ckfqtIobMJbTyfLv5mMlv/91kUIBb9kGDEhi7PA5zxM4bhwAqS++WC81P/UVdw8MUBhGqmdev/qGYsAg3fQc+fm1soiieMnzf/vtt1mwYAHz589n//79DBo0iJtvvpljx44BsHPnTgB+++03UlJS+P777wHIyclhwoQJ/PXXX+zYsYPmzZtz0003kVNBt2dKSgp33HEH99xzD4cOHWLz5s2MGjWq0tf0+uuv07FjR/7991+efPJJHn74YTYUESQ+++yzjB49mn379nHXXXdx++23c+jQoWLHEgSBe+65h8WLF3usX7x4MTfccAOxsbHlzkcURTZu3MiRI0e44YYbKnUt9QF7djYAKh8fBLW6lmdTOVzeE1tGZTwwZ+Tfzc7PRH2g4D+nAVOOB8aFIAg0eGqWdBO12TAp5ewvG5aTUmFB2YDp2BGAfCVsX6MoISRALCjgSJeutXLulnt2I3h7X9Ix5s+fzxNPPMHtt98OwKuvvsqmTZt46623eP/99wkNDQUgODiY8PBweb9+/fp5HOfjjz8mICCALVu2MGzYsHLPm5KSgs1mY9SoUTRu3BiA9u3bV3r+vXr14sknnwSgRYsWbN26lTfffJMbb7xRHjNmzBgmT54MwNy5c9mwYQPvvvsuH3zwQbHjTZw4kdmzZ7Nz506uueYarFYry5cvL+aVKUpWVhYNGzbEbDajVqv54IMPPOZwpeBwGjBqo7GWZ1J5KtsPSXQ45BAS1B8DxpGXJ9cRMVTQgAHJiDF07Ijl5EkK9h+QasZcweTv2UPGV1/TYNaTtRYatOfkYL94EQBdTBMADJ07AWDauw9RFOtVpl99QvHA1HOys7M5e/YsvXr18ljfq1evEj0U7pw7d457772X5s2b4+/vj9FoJDc3l6SkpAqdu2PHjvTv35/27dszZswYPvnkEzKq4LaOi4sr9nfRuVdkjIvIyEiGDh3KZ599BsDPP/+M2WxmTDkt7v38/Ni7dy+7du3ipZde4tFHH2Xz5s2VvJq6jcNqxVFQAIDaz6+WZ1N5NHIIqWIiXtuFi3LGFdQfA8Z0+DCIIpoGDdA4H0AqilcH6SGi4MCV74E5//p8sn/+mYxly2ptDi7vizo0BLUzJOvVsiWCToc9Kwur0vahxlA8MIBgMNByz+5aO3dtMWHCBNLS0nj77bdp3Lgxer2euLg4LG5f+GWhVqvZsGED27ZtY/369bz77rs8/fTT/P3338TExKBSqYqFk6zO6q81zeTJkxk3bhxvvvkmixcvZuzYsXiX4+lSqVRyiKlTp04cOnSIefPmyZqhKwGX90Xl7Y2g1dbybCqP2vmUbc/KQrTZEMopwmd16V/UarDbMR87Vi+eiCsq4C0JVxaMaf+BenGtVcWelSVrTPK2bSd02rRamYflpCTg1TduIq8TdDq82ral4N9/yd+7Vw4tKVQvigcGye2q8vauleVSv1yMRiORkZFs3brVY/3WrVtp00YqfqVzpsra7fZiY6ZNm8ZNN91E27Zt0ev1XHS6QiuKIAj06tWLOXPm8O+//6LT6fjhBymNMzQ01CN7ym6385/zi9mdHUVKoe/YsYPWrVtXeow7N910Ez4+PixcuJC1a9dyzz33VOq6ABwOB2Znv6ArBVdaZ330vkDl+yG5wkeGTp1Ao8GRk4Pt3Lmam2A1UVAJAW9R9C1bImi12DMzPcJnVxp527aBs99QwYEDtZay7Goh4AofuVCEvDWP4oG5ApgxYwbPPfcczZo1o1OnTixevJi9e/fy5ZdfAhAWFobBYGDt2rVERUXh5eWFv78/zZs354svvqBbt25kZ2czY8YMDJXwCP39999s3LiRgQMHEhYWxt9//82FCxdkw6Jfv348+uij/PLLLzRr1ow33niDzBJuOlu3buW1115j5MiRbNiwgZUrV/JLkYJcK1eupFu3blx33XV8+eWX7Ny5k08//bTUuanVaiZOnMisWbNo3rx5sRBUUebNm0e3bt1o1qwZZrOZNWvW8MUXX7Bw4cIKvx51HdFmK0yfrof6F3D2QwoIwJ6RgS0tHU1ISJnjLcnSDVwX0wR7ZiaWhATMx46hddOC1UVMTgFvZfQvLlQ6HfrWrTHt30/B/v3ooqOre3p1gtw/3TIE7Xbyd+3Cr4iu73JQNAPJhUvIqxgwNYfigbkCmDZtGo8++iiPPfYY7du3Z+3atfz00080b94cAI1GwzvvvMNHH31EZGQkI0aMAODTTz8lIyODLl26MG7cOKZNm0ZYKQWzSsJoNPLHH39w00030aJFC5555hkWLFjAkCFDALjnnnuYMGEC48ePp3fv3jRt2pS+ffsWO85jjz3GP//8Q+fOnXnxxRd54403itVemTNnDl9//TUdOnTg888/56uvvpI9TKUxadIkLBYLd999d7nXkpeXxwMPPEDbtm3p1asX3333HcuWLZOFw1cC9txcEEVUej2qelaB1x05jFSBTCSXB0IXFY3e+Xmo66nU9tw8LImlV+CtCAanmP5KzUQSRZG8P/8EQNe0KSCFkWqDUg2YTpIBYz5yFEd+/mWe1dWB4oGph0ycONGjlopKpeK5557jueeeK3WfyZMnF7sZd+7cmV27dnmsu/XWWz3+LislunXr1h71Zoqi1Wr54IMPSswUcsdoNLJixYoyx0RGRrJ+/foStzVp0qTEeZ45cwatVsv48ePLPDbAiy++yIsvvljuuNpGdDiwZ2aiNhrL1X8URda/1FPviwtNUBCWhIQK9UNy1YDRRkUh2m3krK37Ql7zoXhJwBseXq6HqTQMHdqT8SUU7D9QzbOrG5iPHsV24QKCwUDI/fdzdsYM8rZffgNGFMVCA8ZZb8uFNjwcTXg4ttRUCv77D59rrrns87vSUTwwClccZrOZ06dP8/zzzzNmzBgaNGhQ21OqNuzp6VjPnsXqpi2qCKLDIXlgqJ/p0+64PDC2ChSzc9WA0UVHuXlg6rYBcyn6Fxde7Z1C3vh4xMsknL+cuLwvPtdcg+8N14MgYElIwHqZ9U22Cxck74pKhS4qqth2JYxUsygGjMIVx1dffUXjxo3JzMzktddeq+3pVCsuV7Q9OxuxiCi7zP1yc8HhQNBqEepZ+4CiyP2QygkhOSwWWbCrjXIzYI4fR3SKP+sipoPxQNX0Ly50TRqj8vNDNJvrvMFWFVz6F5/rr0ft749Xu3bA5Q8juQS82qioEvuKyQaM0tixRlAMGIVa5eTJk8XaDRRFFEVGjhxZ4WNOnDgRu93O7t27adiw4aVNsI7hquGCKFYq68JVfVftZ6z3abWaIGdH6nI8MNYzZ0AUEby9UQcFoWvUCEGnQzSZ6nR2zqWkULsQVCoM7aWb+pUWRrLn5pHvbHnie/11APg4Rfp527dd1rkU6l8al7jdPROpOqquK3iiGDAKCvUE0Wr1CAfYs7Iqtp8o4nAaOypj/UyfdqewI3U5BoxLwNuwodS1Xa1GF9sMqLthJHturnxTvBQDBgrDSFdaQbv8v3eA1Yq2USN0zgrgPj1dBsz2Mg2F1Bde4NgNvaVCgdVAaQJeF15tWoNWi/3iRaxnzsrrRZuNjBUrONavH4c7duJojziO9+tPwrBhJI65jVMT7yb711+rZY5XMooBo6BQT3B5X1z9ixy5uYg2W/n75eUj2u0IajUqH58anePlQBPs9MCUE0JyGTBaN22CVx3XwZjinQLeiAj5OquKoYMrE+nK8sDkOvUvvtdfL68zdO6M4OWF/cJFLKU0bDUnJJCx/Cts589z5uHpsibsUijPgFF5eeHVqhUABXv3IooiuX/8QeItt5A6+zlsZ1MQzWapZs/Zs1iOJ2A6cID8HTtIee55JXupHJQsJAWFeoLLgFH5+SGaTDhMJuzZ2eX2gHHkZMv71ffwEbj1Q7pYdjsBVw0YbXShAaNzVlo2H62jBoxL/3IJAl4XXs5UavPx49hz81D71n/jVUqfdulfrpPXq/R6vLt2JW/rVvK2b5f1Tu6kfbJI/t1y6hSps2cTuWDBJX0mXAaMvkgGkjuGTp0wHThA9i+/kPX9d7JOR+3vT8jUB/Dt2xdHQQFiQQGOggIc+fmce3ke1tOnyfp5NYFjb6vy/K50FA+MgkI9QTZgDAZU/v5A+WEkURQL9S/1PPvIhc5pkFhOn8ZhMpU6rrAGTKEB4y7krYuYDlauA3VZaMPC0ISHgyhiij94ycerC1gST2I9cwZBqy2WliyHkUoQ8lrPniVr9WoAGjw1CzQastf8SsZXX1V5LqLViiVZStMvq1WAS8ibu2kTedu2I2i1BN1zD83WryNo/Hh00dF4tWiBoWNHfHr0wK9fPwLvuguAjC+/VLQzZaAYMAoK9QBRFBHdDBi104Bx5OXhKCNNVjSZJN2MoELlbDRX39FERKAOCQGbDVMZDUsLa8AUVqKVQ0iJiXUyvbg6BLzuyAXtDlwZYaS8v6TwkXf3bqiK9DZzCXnzd+4s9t6mfbYYbDa8e/QgaPx4wh57DIDz816h4EDx9iYVwXrmDNhsCF5eaMoo1eDdrSuCs3Ck8aabaPrrGhrMnCF/hksiYNQtCAYD5qNHKfjnnyrN72pAMWAUFOoBotUqpU0LAoKXFyqdDpVB+gJ3ZGWXup+rX5DazxdBdWV83AVBqNCN2VUDRhtVmImmiYyUbnxWK5Y61iW4OgW8LuTO1FeIDkZOn77u+mLb9K1aoQ4IwJGfT4Hb/4UtPZ3Mb78FIOS+ewEImjgB3/79Ea1WzjzyiOylrAxml/6lceMyP1va8HCafPM1MT/9SMM3FpRYL6Yoan9//IcPByB92ZeVntvVwpXxjaZQY2zevBlBEErsYaRw+ZC9L3q9/GWpLieM5MgvkFON1QEBNT/Jy4hLoFqwr+QMG3tWllx52P2GIQjCJRW0q0ztncri0r9oIiPK1TVVFMMVlInkMJnI37kTKEyfdkdQqfCO6wFA3tbCdOr0L75ANJnwatcOb6eXRhAEIl9+CW1UFNbTpzn71FOVDtWUJ+B1x6tVK7xatKjU8QPvuhOAnN9+w5qaWql9rxYqZcAsXLiQDh06YDQaMRqNxMXF8atbqpfJZGLq1KkEBwfj6+vL6NGjOVekMmJSUhJDhw7F29ubsLAwZsyYga1IJsXmzZvp0qULer2e2NhYlixZUvUrvIKYOHGilA5aZDleR+P5CtWHnIFkKHSbq/yNzm35OCwWj/Giw4HlzGlARO3vf8XoX1x4dXDdmEv2LFic+hd1cHCxUIO+ReUMGOu586R//gUn77iTwx06kvrSy4hFXu/qwKV/MbRtV23H9GrXFgQB29kUbBcuVNtxa4P8Xf8gms1owsNlMXZRCuvBSDoYe24uGV8uByD4vns9BLtqf38avvkmglZL7m8bSV+6tFLzqYwBUxW8WrbEu1s3sNvJ+OabGjlHfadSBkxUVBSvvPIKu3fv5p9//qFfv36MGDGCg84P3iOPPMLPP//MypUr2bJlC2fPnmXUqFHy/na7naFDh2KxWNi2bRtLly5lyZIlzJ49Wx6TmJjI0KFD6du3L3v37mX69OlMnjyZdevWVdMl128GDx5MSkqKxxJThgJe4cqgUMBbWEVXpdXKadFFvTC28+cRzWYEjQZtRMTlm+hlwuCsvGpNSsKWkVFsu9xCoAR3fUU8MLa0NNKXL+fUuPEc79OHcy+/TMG//0o3ky++4NS48ZVu51Ae1SngdaH29UXvrH1TVa1HXSH3zz8AyftSWuaQT89egFQ4zp6bS+Y33+DIzkYXE4PfgAHFxhvatyNs1pMAnJ+/QDZ8K4KrCq8upkklrqJyBP7f/wGQuWJlsYcUhUoaMMOHD+emm26iefPmtGjRgpdeeglfX1927NhBVlYWn376KW+88Qb9+vWja9euLF68mG3btrFjxw4A1q9fT3x8PMuWLaNTp04MGTKEuXPn8v7772NxvjkffvghMTExLFiwgNatW/Pggw9y66238uabb1b/1ddD9Ho94eHhHovaWRdky5YtXHPNNej1eiIiInjyySc9vFtms1nuOO3l5cV1111XrJnjmjVraNGiBQaDgb59+3LS+ZRRGidPnkQQBPbu3Suvy8zMRBAENm/eDEBGRgZ33XUXoaGhGAwGmjdvzuLFi+XxycnJ3HbbbQQEBBAUFMSIESPKPe/VhIeAt4g3QRbzuhkwjvx8bBcvAqCNjKx008f6gNrfX37ydQlf3bG6NXEsimzAlJJKnbNpE8f79OXcC3PJ37ULRBFDp040eGoWka+/jspopGDfPhJHjSZ369ZquR5RFCUDCeSy+NXFlVLQLs+tfUBp6KIaom3UCOx28v7aSprTex88eXKpOpXAO+7A0LUr2Gzkbtpc4fnIKdQ15IEB8OvfD02DBtjT0sgpo3Hu1UqVNTB2u52vv/6avLw84uLi2L17N1arlQFuVm6rVq1o1KgR253uvO3bt9O+fXuP5nqDBg0iOztb9uJs377d4xiuMdvL6TRqNpvJzs72WCqKKIpYzfZaWaorRe7MmTPcdNNNdO/enX379rFw4UI+/fRTjw7LM2fO5LvvvmPp0qXs2bOH2NhYBg0aRLqzomlycjKjRo1i+PDh7N27l8mTJ/Pkk09e8tyeffZZ4uPj+fXXXzl06BALFy4kxNll12q1MmjQIPz8/Pjzzz/ZunUrvr6+DB48WDZqr3ZEs1nq3SOo5GwGF2qjEQQBh8mEwzlODp8EBFxxoSN3DB2dN+YSdDCu18C9BowLvTP8YElKKpaG7SgoIHXOC4hWK/pWrQibMYPY3zfS5OuvCBo/Hv/hw4j57lu82rTBnpFB8uR7ufD++5fcW8l6+jTWs2dBo8G7S+dLOlZR5IJ2peiF6gOW06exJCaCWi2HiUrDtf3cvHnYL1xEEx6O//BhpY4XBAG/fn0ByPvrrwrNx5GXJ/fZqqkQEoCg1RJ4+1gA0r9UxLxFqfSj2YEDB4iLi8NkMuHr68sPP/xAmzZt2Lt3LzqdjoAiYsEGDRqQ6hQgpaamFusM7Pq7vDHZ2dkUFBRgMBhKnNe8efOYM2dOZS8HAJvFwccPb6nSvpfKfW/3RqtXV3j86tWr8XVLhx0yZAgrV67kgw8+IDo6mvfeew9BEGjVqhVnz57liSeeYPbs2RQUFLBw4UKWLFnCkCFDAPjkk0/YsGEDn376KTNmzGDhwoU0a9aMBQsWANCyZUsOHDjAq6++eknXmJSUROfOnenWrRsATdw+8N988w0Oh4NFixbJbuHFixcTEBDA5s2bGThw4CWd+0rAPXxU1HUuaDSofH1x5ORgz8wChx3RYpFCR+HhtTHdy4ZX+w5k/fhTiZ4Fa3LxGjAu1CEhqAMCsGdmYk5I8GiamPbpZ9hSU9FERtDk669QldD4UhcdTeOvlnPuxZfIXLmSi+++R8G/e2n41puoq5iq7tJsGDp2LOZlu1RcBe0K/vsP0eGod9looiiS9f33ABg6d0LtV3Y7DJ+4ODK/+UY2MILvubvERose+/TqBa/PJ2/nThwWC6pyxluSkgDnQ0INC+QDbruNix8sxLRvPwUHDsgZeApV8MC0bNmSvXv38vfff3P//fczYcIE4uPja2JulWLWrFlkZWXJS7KzwNCVhksb5FreeecdAA4dOkRcXJzHDa5Xr17k5uZy+vRpEhISsFqt9OrVS96u1Wq55pprOOSspXHo0CGuvfZaj/PFlfO0UxHuv/9+vv76azp16sTMmTPZtq0wQ2Dfvn0cP34cPz8/fH198fX1JSgoCJPJREJCwiWf+0rAvf5LScjZSOlp2NKk6rTahg2vyNCRO+6l8ot6MgvbCEQX2889E8m97Lw1JYW0RVK11gYzZ5ZovLhQ6fVEzH2BiHnzELy8yPvrL9I+/bTK15K/428AfHr0qPIxSsOrRQsEnQ5HdnadSx0vj/zduzl5++1c/GAhQIk6lqJ4X3sNOL8H1QEBBNx6a7n76Fu2RB0SglhQQMGef8sdX9MCXnc0wcH4DRkMQEYtpVTX1WJ6lf6G0+l0xDpdsF27dmXXrl28/fbbjB07FovFQmZmpocX5ty5c4Q7nwTDw8PZ6UyDc9/u2ub6WTRz6dy5cxiNxlK9LyBpQ/RF3OsVRaNTcd/bvau076Wi0VXOhvTx8ZFf/7qAyvk05/4Pbi1SRGrIkCGcOnWKNWvWsGHDBvr378/UqVOZP38+ubm5dO3alS9LcI+GhobW7OTrCYUZSKUYMH5+WAVBTvFVBwaW+5R6JaBv1QpBq8WekYH19Gl00ZKxIjocUpExStbAgKSDyd+1y0PIe37+AkSTCUO3rvgNGlShOQTcMhJBJXD2iSfJWb+BsIcfrvR1iKJI3t8uA+backZXHkGrxatNGwr27sV04ECZZe/rCubERC688QY5G34DQPD2JnjSPQQ5Ra1loQkMxKtdO0wHDhA4flyFPFqCIODbqydZP/5E3tat5b4P5sRE4PIYMABBd91F9k8/k71mDWFPzKy2NPuKkLZoERc+WEjjpUvqnPfnkn2JDocDs9lM165d0Wq1bNy4Ud525MgRkpKS5Kf4uLg4Dhw4wPnz5+UxGzZswGg00qZNG3mM+zFcY6rDE1AagiCg1atrZamu3jStW7dme5FOrFu3bsXPz4+oqCiaNWuGTqdjq5vo0Gq1smvXLvm1b926dTED0yXALg2XkZHilpHhLuh1HzdhwgSWLVvGW2+9xccffwxAly5dOHbsGGFhYcTGxnos/mVUqrxaEB0OWadRmgdGUKtlg0XQaq/40JELlU6HvnVrAAr2F4aRbOfPS5VYNRq04SVXSHWlUpucBkz+nj1k//ILCALhTz1Vqc+lb79+oNViSUjAfCKx0tdhPnYMe1oagpcXXs6y89VNfSlo5zCZSH3xJU4Mv1kyXlQqAm67jdh1awmdOrXCXsWIOc8TOv1hgidNqvC5fZze6dyt5etgLqcHBqSyAV7t2iFarWSuWHlZzglgPX+eC+++h5ifT/aautcdu1IGzKxZs/jjjz84efIkBw4cYNasWWzevJm77roLf39/Jk2axKOPPsqmTZvYvXs3d999N3FxcfRwukUHDhxImzZtGDduHPv27WPdunU888wzTJ06VfaeTJkyhRMnTjBz5kwOHz7MBx98wIoVK3jkkUeq/+qvIB544AGSk5N56KGHOHz4MD/++CPPPfccjz76KCqVCh8fH+6//35mzJjB2rVriY+P59577yU/P59Jzg/5lClTOHbsGDNmzODIkSMsX7683Bo8BoOBHj168Morr3Do0CG2bNnCM8884zFm9uzZ/Pjjjxw/fpyDBw+yevVqWjtvPHfddRchISGMGDGCP//8k8TERDZv3sy0adM4XYmUxisV0WwGUURQqcuM42vCwlD7+aGLjpa7VV8NyBV53W7MVmf4WBsRUeoNzz2VWnQ4OPfSywAE3HorXk6DvqKo/fzwcYZec377rXIXQGH4yLtr13K1F1WlvhS0S/tkERnLloHNhk/vG2j64yoiXpiDppLeWK82bQiZMgVVJbzyPj17AmCOPySHYkvDclIKxekukzdLEAQC77gdgJwiD/g1SdqiRdJ3EFJqel2jUgbM+fPnGT9+PC1btqR///7s2rWLdevWceONNwLw5ptvMmzYMEaPHs0NN9xAeHg43zvFVwBqtZrVq1ejVquJi4vj//7v/xg/fjwvvPCCPCYmJoZffvmFDRs20LFjRxYsWMCiRYsYVEGX7tVKw4YNWbNmDTt37qRjx45MmTKFSZMmeRgTr7zyCqNHj2bcuHF06dKF48ePs27dOgIDAwFo1KgR3333HatWraJjx458+OGHvPzyy+We+7PPPsNms9G1a1emT5/ukfkEUthx1qxZdOjQgRtuuAG1Ws3XX38NgLe3N3/88QeNGjVi1KhRtG7dmkmTJmEymTBewRk0FaUwfFRcwOuOyssLXePG1S4ArevIFXndCtpZXDVgSshAcuHKRLKdTSFj2TJMBw+i8vUldHrlQ0BQqM2oigHjCh9510D4yIXrdTLHH6qTPaBc5Dk9vmEzZ9Loo49K7CpdU2hCQmSPXkkNIV2IDoeUEcXl88AAUlE7wHzkyGV5D63nzpH5dWEBPdPBg3Xuf0cQ66o65xLJzs7G39+frKysYjdCk8lEYmIiMTExeJUh1FNQqG0sZ85gz8hAExJSK6Ghuv5ZMScmcmLITQh6PS3/2YWg1XLhnXe5+MEHBIwZQ8TcF0rd91jvPlKmilYLVithM2cSfM/dVZqH7cIFjt3QG0SR2C2b0ZbR3M8d0WbjaFxPHDk5NFm5osY0BqLDwZEuXRFNJpr+uqZO6mBEq5Uj3bojms00XfML+qZNL/sczi9YQNoni/AfMYLIV18pcUzuli0k/28KKl9fmm/9q1JenktBFEWOXnMtjpwcYlb9gFerVjV6vtQX5pKxfDmGbl0xHz2GIzubJt9965G1V1OUdf92p37l0ykoXGWUl4F0taNr3BiV0YhoNsuCXOsZVw2Y4hlI7ri8MFit6Bo3Juj/7qryPDShoRg6dQIq54UxHTqEIycHlZ9fpUNXlUFQqdA1bgwU6jfKQhRFMr76iry/d5Y7trowHTmKaDajMhovq2fDHVkHs21rqZk3ruJ4AWPGXDbjBaQwkpfTQ+Sq2lxTWFNSyFwpaW1CH5rmFqqtWyFIxYBRUKijSAJeKf5cWgbS1Y6gUsltBVwCVYtcA6ZhqfsBHuGJsFlPllsrpDyqEkbK2y6FTLyvuabGtUsuvYarBH5ZmP77j9Q5L5A8ZQrWIlmhNUXBfkljYejQodZq1Ri6dEEwGLBfuIj56NFi202HD5O/fQeo1Zdk8FYVV5sJV+PPmuLiRx8hWq14X3stPtdeU1g0cm/d0sEoBoyCQh3FUWACRASNBkGrre3p1Fm8XF+uzqdDuQZMOR4Y7+6SpsC3d298e196GQW/GyUDJn/nLuwV7N6e79R8+Fxbc/oXF66ePRXxwJgOHwYkD+CFN9+quUm5n9MpEjXUUCZWRVDpdHhf0x2AvL+Kt4lIXyI1fDQOGoi2YdkGck3g8tKZarD2muX0GTK/k7SroQ89KJ23g+dnrK6gGDAKCnUUsSAfkMJH1ZVufyXiyrAxHdiPw2TC5izTUFoNGBe+/frR+KvlNHzn7Wp5fXWNGqFv0QLsdnKcfcDKwmGxkL9nDwA+cdVfwK4orp49LgFqWVgSTsi/Z61aRcF/NRuygMKne0On2jNgAHx7XQdAXpF0auv582T98gsAQRMnXu5pAW4emMOHEd363FUnaR99CFYrPj3jZOGwy6i0JCYWaxxbmygGjIJCHUXywCjho/KQM2yOJ8g6GJWPT7kl3gVBwLtz52rVMVQmjFSwdy+iyYQ6OBjdZShOKYeQKuCBMZ+QqmCrnLWYzr0yr0arsdoyMuQqwbVdLM3nOkkHk//PbjkLECBj+XKwWjF07ozB6ZG43OiaSJmGoslUIUO0sliSk8n8/gcAQh58SF6vCQyUmmRSt7qaKwaMgkIdxeHmgVEoHU1ICNrISBBFsp0de7VRUbXitXKFkfL+2upx8ysJuX3Atddelrm6hLG2Cxew5+aWOdZyXDJgwp95BsHLi4J/dpOzbn2Nzc0lDtU1aVLjvYXKQxcTgyYiAtFiIf+ffwCpnEHmV1Lph9ryvoCk+ZKLN9aAkPfiwg/Bbsfn+uuLNRV1GW0F+/ZW+3mrimLAKCjUQUS71JQRFAOmIrhi9Nm/StVCS+pCfTnQt2qFtmFDRJOJ3HI6G8v1Xy5D+AikzuXq4GCgsBBbSTjy86XO2EjeiOB77gHg/Pz5OJxFzaqbgjqgf3EhCAK+Ti+MSweT9eOP2LOy0EZF4Tegf21OD6+2NaODsZw8SdaPPwKF2hd3XO9NXdLBKAaMgkIdRC5gp9Ve8U0ZqwNX2MF2VmppoWtYOwaMIAhyGCm3jDCSIy9PvmnXRAPH0tBVQAfjaoegDgxEExhI8ORJaMLCsJ4+TcYXX9TIvAr2STfF2ta/uHClU+dt24rocJC+9HMAgsaPq/VK17KQt5ozkdK/WAZ2O769e5cYInNlIpn27a8zzR0VA0ZBoQ7iyHOGj66yyrpVxaWDcVGegLcmcYWRcjZtLrVyaf6ePWCzoY2MvKxzrUgmksWpf9E3awZI/4OhzlYuFxd+WG6Z/coiOhzyU31d8MCA06hUqTAfO07mipVYEhNR+friP2p0bU9NLiRnOnQI0eGotuPm79oFgP/oUSVul5unZmbK7TpqG8WAUSiT559/nk7OAl2XmyZNmvDWW29d1nNW5HonTpzIyJEjq/W8S5Ys8eji7sjNASQx6mWnjjxdVQavtm3BrXZIbYWQAAydO6MOCsKRnS3fFIriKpnvHdfjsmp1KpKJZHZmIOmcBgyA/4ib8WrbFkdeHhfeebda52RJTMSRk4Pg5SVlcdUB1AEBeLWX6gude0WqyBtw222ofWvh81gEXUwMgpcXYn5+hQTZFcGekyML4L07dy5xjEqnQ9/Gqb+pI32RFAOmHjFx4kQEQSi2HD9+vLandsXw+OOPF+uGfrkRbTY5hKRydpm+bGSnQMpeuHBE+t2SXy8MGpW3t0dhOl0temAEtRrffn2B0rOR8p0F7C5n+AgqlolkTpC+T/TNCkv5CyoVDWY9CUDmypWYjhQv8lZVXOnTXu3a1qlwqSudWjSZaq1wXUkIGo3cRqC6wkgF+/aDKKKNji6zcaasg9lXN3QwigFTzxg8eDApKSkeS0wd7GtSG1icotdLwdfXl2Cn0LG2cOTlAaDS61Fd7gJ2pgzppzUfclMh8yRkn4X1z0L8T3XamHEPI9VGkTF3CtOpNxZz89szMzEdOgSA9zU1X8DOHVkDc/JkqToGSwkeGJCaCfoNHAgOB2cefRTTkSPVMqe6JOB1x5VODc7CdZGRtTgbTwp1MNWTiVTw778AGDp3KnOcoUPdEvIqBgxS3w+ryVQrS2XFUHq9nvDwcI9F7RSV/fjjj3Tp0gUvLy+aNm3KnDlzsLkVO8rMzGTy5MmEhoZiNBrp168f+4q4Al955RUaNGiAn5+f3BW6LOx2O5MmTSImJgaDwUDLli15++23Pca4Qi7z588nIiKC4OBgpk6ditVNH3D+/HmGDx+OwWAgJiaGL7/8stzXwnXcl156icjISFq2bAlAcnIyt912GwEBAQQFBTFixAhOuj1xbt68mWuuuQYfHx8CAgLo1asXp5w1KIqGkOx2O48++igBAQEEBwczc+bMYu9ZSaGuTp068fzzz8t/v/HGG7Rv3x4fHx+io6N54IEHyC0lldWe4wwf+RZ6XzZv3owgCGS6VXjdu3cvgiDI13bq1CmGDx9OYGAgPj4+tG3bljVr1sjj//vvP4YMGYKvry8NGjRg3LhxXLx4sfDEDgfYnFkmxobgFQCoQbTD0V9hxTg4WXZmTW3iykRSh4bUeuaWT1wcKm9vbOfPkzJ7Nhlff03+rl3Y0tPJ27kTRBFds2ZoG4Rd1nnpoqNBpcKRn4/t/IVi20WLBUtSElCogXEnbOYM1MHBWBISOHnrGNIWLUK02y9pTnXVgDF06IA6KAio3dTpkpAL2lVTJpLLgCktfOTCJeQ1HzqEoxoeGC+VuuOvq0VsZjPvTLi1Vs49bem3aKuhy++ff/7J+PHjeeedd7j++utJSEjgvvvuA+C5554DYMyYMRgMBn799Vf8/f356KOP6N+/P0ePHiUoKIgVK1bw/PPP8/7773Pdddfx+ZIlvPvuu8RER2M+eRJBrUHQqCU3r1aL2tcXhygSFRXFypUrCQ4OZtu2bdx3331ERERw2223yfPbtGkTERERbNq0iePHjzN27Fg6derEvffeC0jGyNmzZ9m0aRNarZZp06Zx3llRtSw2btyI0Whkw4YNAFitVgYNGkRcXBx//vknGo2GF198kcGDB7N//35UKhUjR47k3nvv5auvvsJisbBz585SdQgLFixgyZIlfPbZZ7Ru3ZoFCxbwww8/0K9fv0q9PyqVinfeeYeYmBhOnDjBAw88wMyZM/nggw88xomiiMNp2Kj8fCt1jqlTp2KxWPjjjz/w8fEhPj4eX1/pGJmZmfTr14/Jkyfz5ptvUlBQwBNPPMFtt93G77//Lh3A5jRWBTX4hIJvGBjyIUuEBh0gNxmSdkDM9ZWa1+XCt3dvNJERGG+8sbangkqvx3dAf7J/+pmsb78j69vvCjc6vWqXo31AUQSdDm1UFNakJCyJicUMKMupU2C3o/LxQVNCR21dVBRNf1xFyrOzyd20ifPzF5CzaTORr8yTjKNK4sjLk7UXho6dqnRNNYWg0dDos0+xZ2TUWuG60nBPpRYdjkvqHSXa7YVGZDkGjDYqCnVgIPaMDMzx8XID09pCMWDqGatXr5ZvSgBDhgxh5cqVzJkzhyeffJIJEyYA0LRpU+bOncvMmTN57rnn+Ouvv9i5cyfnz59H76w8On/+fFatWsW3337Lfffdx1tvvcWkSZOYNGkSAM8/+igb1qzBZDbLN1V3HEYjukaNmDNnjrwuJiaG7du3s2LFCg8DJjAwkPfeew+1Wk2rVq0YOnQoGzdu5N577+Xo0aP8+uuv7Ny5k+7dpT4kn376Ka2dBZvKwsfHh0WLFqFzNuJbtmwZDoeDRYsWyUbJ4sWLCQgIYPPmzXTr1o2srCyGDRtGM+cTZlnneeutt5g1axajRknK/A8//JB169aVO6+iTJ8+Xf69SZMmvPjii0yZMqW4AWM2SyXCVapKZyAlJSUxevRo2jtTips2LdQwvPfee3Tu3JmXX35ZXvfZZ58RHR3N0aNHadGiBdichde0BnAZdIIKNHqI7Q8Jv8DZPZWa0+VEGxZGc5cx5o7DAYd+gqju4H/5Qkvhzz6Lz7U9MCckYE44jiXhhNSnyel59O1fOSO4utDFNJEMmJMn8enhaUS5C3hLM+o1ISFEffA+Wd9/z7mXXqZg924SR4wkbNaTBNx6a6VEyQX/HQSHA01ExGX3RlUEl9akrqFv1gxBp8ORm4s1OVnuNF4VzMeP48jLK6YjKwlBEDB07Eju5s0U7N+vGDB1AY1ez7Sl39bauStD3759Wbhwofy3jzNLZd++fWzdupWXXnpJ3ma32zGZTOTn57Nv3z5yc3OL6TsKCgpISJDSJg8dOsSUKVPkbY7cXK7t0IE/9uyRNAU2G6LNjmizYs/Kwp6Tg2i388GHH/LZZ5+RlJREQUEBFoulWCZP27Zt5VAXQEREBAcOHJDPq9Fo6Nq1q7y9VatWHlk5pdG+fXvZeHG9DsePH8eviPjVZDKRkJDAwIEDmThxIoMGDeLGG29kwIAB3HbbbURERBQ7dlZWFikpKVzr9qSs0Wjo1q1bpUN/v/32G/PmzePw4cNkZ2djs9nk98bbzVBxOMNHah+fSj9VTZs2jfvvv5/169czYMAARo8eTQfnk+O+ffvYtGmTh/HrIiEhQTJgrE4PjKYEj2CY9MTH2X8rNac6wYnfYeUEaDEY7vzmsp1W7edHQJGUVEdBAZbERES7vdZK5uubxJC35Y8SM5FcLQT0bsZvSQiCQMDo0Xhfey0pT84i/59/SH12NtakZMIee7TCc6mr4aO6jqDVom/ZEtOBA5ji4y/JgJH1L506VqjGjaFjB8mAqQNCXsWAQfowVkcY53Lg4+NDbAl9U3Jzc5kzZ47sKXDHy8uL3NxcIiIi2FxCk7mSDAXRbseRL9UiETQaNIGBhdtEEUdBAaLFwldLl/L444+zYMEC4uLi8PPz4/XXX+dvZ5VRF9oiYlRBEHBUQw0DnyJpxrm5uXTt2rVEDU2oU12/ePFipk2bxtq1a/nmm2945pln2LBhAz2qmBGiUqmKGTTu+p6TJ08ybNgw7r//fl566SWCgoL466+/mDRpEhaLxcOAcZV4VxUxNFROY8b9PNYiNUYmT57MoEGD+OWXX1i/fj3z5s1jwYIFPPTQQ+Tm5jJ8+HBeffXVYvOXjTd3D0xRQltK3picFCk7yVjc4KuzXHRm6aUl1O48kKoquwSYtUVZtWBcLQT0scX1LyUeKyqKRp8vJe3jj7nw1ttkfP01oQ89iOD2UFEWsgFTx0I09QGvNm0kA+bgQYxDhlT5OIUGTNnhI/m8dagztSLivULo0qULR44cITY2ttiiUqno0qULqampaDSaYttDQkIAKZTiMjwceXkgiuz877/CcIITQRBQG40A/PXnn/Ts2ZMHHniAzp07ExsbK3t0KkqrVq2w2Wzs3r1bXnfkyBEPwWplXodjx44RFhZW7Dr9nY3pADp37sysWbPYtm0b7dq1Y/ny5cWO5e/vT0REhIcxVnSeIBlGKSkp8t/Z2dkkuj3d7t69G4fDwYIFC+jRowctWrTgrLNUe1FcRmNRA8ZlfLmfZ+/evcX2j46OZsqUKXz//fc89thjfPLJJ/LrcvDgQZo0aVLsdZGNwLI8MDpvCHW60+ubFybH+Zrllq+puhrQNZGyFs0nS/LAOENITStmwICUYh18332oQ0Jw5OSQt7Pk2jdFEUWx0ICpIxV46xPV1VIg/9+9QPn6Fxcuz6E1ORlbevolnftSUQyYK4TZs2fz+eefM2fOHA4ePMihQ4f4+uuveeaZZwAYMGAAcXFxjBw5kvXr13Py5Em2bdvG008/zT/OhmUPP/wwn332GYsXL+bwvn3Mff99DpVSY0blNGCaRUTwzz//sG7dOo4ePcqzzz7LrlKKd5VGy5YtGTx4MP/73//4+++/2b17N5MnT8ZQhUySu+66i5CQEEaMGMGff/5JYmIimzdvZtq0aZw+fZrExERmzZrF9u3bOXXqFOvXr+fYsWOl6mAefvhhXnnlFVatWsXhw4d54IEHihlW/fr144svvuDPP//kwIEDTJgwwSNcFhsbi9Vq5d133+XEiRN88cUXfPjhhyVfgCgi6HTFnmBjY2OJjo7m+eef59ixY/zyyy8sWLDAY8z06dNZt24diYmJ7Nmzh02bNsnXNXXqVNLT07njjjvYtWsXCQkJrFu3jrvvvhu73Q4OGzicHp2SPDAAkV2kn5dRB5OSuoq0tD8v7SA5qdJPcxZYy26weDXg8sBYT5+R+22B5HW1OA0Y9xowFUFQqfBzCttznIL68rCeOYv94kXQaGrdK1Uf8WojZSIVHIyvcml/28WLWJOSQBDkDKPyUBuNcop9bRe0UwyYK4RBgwaxevVq1q9fT/fu3enRowdvvvkmjZ2xUUEQWLNmDTfccAN33303LVq04Pbbb+fUqVM0cGYbjB07lmeffZaZM2fSY/Bgks+e5X9OQW9RVAYDgkbDpFtv5Zbhwxk7dizXXnstaWlpPPDAA5We/+LFi4mMjKR3796MGjWK++67j7Cwyov6vL29+eOPP2jUqBGjRo2idevWcjq40WjE29ubw4cPM3r0aFq0aMF9993H1KlT+d///lfi8R577DHGjRvHhAkT5BDZLbfc4jFm1qxZ9O7dm2HDhjF06FBGjhwpC4QBOnbsyBtvvMGrr75Ku3bt+PLLL5k3b17xkzm/hNS+vsWEkFqtlq+++orDhw/ToUMHXn31VV588UWPMXa7nalTp9K6dWsGDx5MixYtZJFwZGQkW7duxW63M3DgQNq3b8/06dMJCAiQwlMu74taB6pS4uCRnaSfl8kDk59/ivj4x9h/4H7s9ktoIphT6LVSvDCgCQtD8PYGux3L6dPyeusZyaBxZSpVFj9n9lfO78Vr35SEab+zgF2rVqjqSQi/LqFv0Ry0WhxZWVjPlOzRLY8CpxdXHxsre9UrgqGOhJEEsa50ZapmsrOz8ff3JysrC2ORN8ZkMpGYmEhMTAxeygenGA6zWUptFAS8WrUqVdhlOXsWe3o66sBAdLVcOKy+I4oi5qPHEK0WdI0aVerLpFrIuwBZp0FvhOBC48vjs5J2ED7pB4YgmHmiWGixurlwcSP790ulALp0+ZrAgO5VO9B718BFZ9G1Sb9BdBWPcwVxYtQozPGHiPrg/ULPyaZNnL7/AfQtW9L0x1WVPqZosXC013U4cnJovHw53l3KDkmcmzeP9KWfE3jXXYQ/+0xVLuOqx/U+Nnz7bYyDBlZ6/3Ovv076p58RcNttRLwwp/wdnGR8/TWpz8/Bp2dPGn32aaXPWx5l3b/dUTwwCsWQ65B4e5epSnfdZB05OXWmO2l9RbRYEK0WEITa6X9kLUPA66JBO1BpoSAdMpNqfEoF+Sfl3zMz/i59YHl4eGBSq36cKwi9UwfjnolkSfBs4lhZBJ0O3969gdJbKLjjaiGg6F+qjlyRt4o6mIJK6l9cyC0F9u+v1oaSlUUxYBSK4SglE6YoLgNHtNlk8alC1aio0VhjlCXgdaHRQwMp7n45dDD5BSfl3zMzK6erkjHngjm78O/cc5c2qSsE95YCLgprwFRO/+JOYQuF38p8qHFYLPJNV0mhrjqGS6jI67BYMP33HwDe5bQQKIq+eXMEgwFHbm6ZjUFrGsWAUfBAdDiwO3vxqMsxYASVSm426MjOLnOsQtm4DJjyXvMaQRQLq/BqDWV70yKdT2qXQQeT7+aBycreg8NhLX1waRQ1WBQNDFDY1NHsdvORa8BU0QMD4Hv9dQg6HdakJMxHS2/4aD50CNFqRR0YiLYKFXwVJNx7IlXWC26Oj0e0WKT3oJJ1ZASNRs6Cqs16MIoBo+CBIz8fHA4EjQahAvogVxjJnp2thJGqiLvRWJ7Xq0awW6R+RwiY7dnk5sZjt5eSrdPQmYl0puY9MO4hJLs9n5yc/yp/EPfwESgeGCeFHhipB5goioU1YC7BgFH5+OBzndTFOWdD6WGk3D+kzDJDhw6Vqtyr4Im+ZUtQq7Gnp2M7V7n/bff06aq8BwGjbyV0+sMeTVQvN4oBo+CBe/ioIv/UKl9fEFSIVqvUdl6h0lTWaKx2XN4XjR6rLRNRdGCxpJU81uWBSdknlegv9ZgWsFQ9rGi3mzCZpcwKf3/JaMrM3Fn5A+UU0bwoHhigMJXafvEi9pwcbOfPS7Wf1OpLquoKnmGkkrCmpJD22WcAGIcNvaRzXe2ovLxkg7Oynakr2oG6NAJuGUnIlCnoSyiserlQDBgFDyqqf3EhqFSonU0H7UoYqUpU1misdpwCXofWC4dDqgtis2UhiiUYKKGtQWOQdCXppRQsdDhg8RB4u0OVDYaCAskzoNH4ERYqVRnNqJIB4/TA6JytJYoaNFcpal9f1KFSAUvLyZOYnfWedNHRFa6iWxq+ffuAWo358GGPNG0X5159DbGgAEOXLhiHDbukcym4daY+WHEdjCiK5P8reVHL60Bdl1EMGAUZh9WKw+lFqYwWw1XUTtHBVA25/1GR/k2XDaeA164pFA+LogObLaf4WLUGIpwFr0oJI1mPr+GEJp7jYXmI/yyp0pRcAl6DoQkBgdcAkJn5D6Jor9yBsp0GjGvOigdGxj0TyeIS8FawhUBZaAID8e7WDSgeRsrbto2ctWtBpSJ89rNK+Kga8GonGTA5mzZVOIxvPXMW+wVnEcF27WpyejWKYsAoyMieAC+pSF1FUfv6giDgMJtxmC+h4NhViMNqlV+zWkmfBrkHkl1VtJ9TVsnjSxHy2u0FnDz1EduSHiWxsTenor3Ji/8M7LZKT8mlf/H2boKfb2vUal/s9lxycg9V7kAuD0yEM9Ml95xcMPBqxyXktZw86dbE8dINGHAraudWlVe0WEidKxVfDLzzzjrb6bm+YRwyBMHbG/OhQ+Ru2lyhfVzhI682bep1EUHFgFGQkQ0Yv8oJSQWNRr75KmGkyuFwiXcNlTMaqw3RATbJgLIjZfnodFLHcps9u2SPR5GWAg6HldNnlrNtez8SEl7Dpircx2S9CEfWVHpargwkb0MTBEFNQID0RJ+ZUckwkitk5DJgHFYoyKj0fGoVe83M2SXkNScmVrqJY3n4DegPSDdK28WLAKR//jmWxETUwcGETnuoWs6jAJqgIILuuhOAi++9VyEvjMuAqWz6dF1DMWAUAGeH6RL0L4IgsGrVqnL3V7uFkfr06cP06dNrYpo1zuWeu+j0vtSKeBecxouIKKixO6S5aLVBqFR6EEWs1hLCSLKQdz/pF/9gx98DOXLkWSyW83jhR5vDOQTnSwXxTF5q2LWo0tPKd2pgDN5NAAgMcIWRKmvAOD0wAY3BK0D6vT5lImUmw8KesKB1tRcPLOxKfapKTRzLQhsejlf79iCK5Gz8HWtqKhc+WAhA2GOPXf5K01c4QXffjeDtjSk+vkJemPy9LgFv/dW/QCUNmHnz5tG9e3f8/PwICwtj5MiRHDlyxGNMnz59EATBY5kyZYrHmKSkJIYOHYq3tzdhYWHMmDEDm83Tzbx582a6dOmCXq8nNjaWJUuWVO0KrxCKvqZFl+eff77UfU+ePIkgCCV2L3YhFhQg2u1SbZcqNFGU68EUFNQLF/3mzZsRBKFYY8bvv/+euXPnlru/6HBgOX0G6/kLlzQPlwGj0uurfIybb76ZRo0a4eXlRUREBOPGjSu123UxXAJenQ5EB4KgQqXSo9UGAGC35xbfJzgWdH6YVSb2/3c/BQVJ6HQhtIh9lrj9NiLOm/EKlDQnZr0aErfAhdJrgpREfr5Un8TbWwpzBARcC0BG5q6SxcUlIYqFHhi/cGmB+mPAXDwGnw2Gi0elMN/hX6r18LIH5vhx7M6uwvqmMdV2fPdspPOvvYaYn4+hc2f8R46otnMoSFTGC+PIy8N8WLpvGzp1uhzTqzEqZcBs2bKFqVOnsmPHDjZs2IDVamXgwIHkOd3gLu69915SUlLk5bXXXpO32e12hg4disViYdu2bSxdupQlS5Ywe/ZseUxiYiJDhw6lb9++7N27l+nTpzN58mTWrVt3iZdbf3F/Pd966y2MRqPHuscff/ySjm93z4RRVd4xp9JqUXl7AyDaKq95qCsEBQXhV46YVhRFrGfPYs/MwHb+HA5rFQqsOXHpX4RLMGD69u3LihUrOHLkCN999x0JCQnceuutFdvZ5hLwSuErtdobQRDQav2l9fb84mEklQoiO5HQxAe7w4TRrwNxPX4n2hSJKjMZvALwiugJgCnU2RTwn4r3S7HZ8rBYJLGtt6EJAH5+bVGrvbHZMsnLO1axA5kyZX0PfuHg62wOWh+EvGf3wmeDIPu01GAT4Hj55fkrgy4qCjQacP7/aiMj5c9wdeB3o2TA5G3dSvaaXwuFu2V9v7iMznrwEFTXqKgXpuDAAXA40ERGoA0Pv3wTrAEqdadau3YtEydOpG3btnTs2JElS5aQlJTE7t27PcZ5e3sTHh4uL+7NmNavX098fDzLli2jU6dODBkyhLlz5/L+++9jcbZ2//DDD4mJiWHBggW0bt2aBx98kFtvvZU333yzGi65OKIo4rDYa2WpqGrc/fX09/dHEAT577CwMN544w2ioqLQ6/V06tSJtWvXyvvGOMV6nZ0Fi/r06QPArl27uPHGGwkJCSGkWTMGTpzI3oRSUmNLIS8vj/Hjx+Pr60uTuDjeXrpUqgnjViMkIyOD8ePHExgYiLe3N0OGDOHYscKb0JIlSwgICGD16tW0bNkSb29vbr31VvLz81m6dClNmjQhMDCQadOmYbcX3kzNZjOPP/44DRs2xMfHh2uvvZbNmzfL20+dOsXw4cMJDAzEx8eHtm3bsmbNGk6ePEnfvn0BCAwMRBAEJk6cCBQPIZnNZp544gmio6Nlb+Anb7+N3c1z4559VVLILSAgQPYgWiwWHnzwQSIiIvDy8qJF7968vmiRbMBkZmYyefJkQkNDMRqN9OvXj33ltKx/5JFH6NGjB40bN6Znz548+eST7NixA2tFDCurp4BXrZZuYCqVXv69pKJ22ZGNSGkgzblFi9loND7wj1Tbg0534uXdCABTgKSnYe9yqax/BXClUGu1gbIhpVJp8TdK2psKp1O7vC9eAVKPJ1+p63qd98Cc/AuWDIP8NIjoBON+KFxvLaXAYBUQtFrJiHGiu4QCdiWhb9pUOqbzuyDw9tvxat267J3+XQYLWsK/X1TrXK4GKuKFEW02Mr75BgDvTvU7fARwSarBrCwpSyEoKMhj/ZdffsmyZcsIDw9n+PDhPPvss3g7Lfvt27fTvn17GjRoII8fNGgQ999/PwcPHqRz585s376dAU73o/uYsrQJZrMZs1sGTHYlxKSi1cHZ2dsqPL46iXyhJ4Lu0nrfvP322yxYsICPPvqIzp0789lnn3HzzTdz8OBBmjdvzs6dO7nmmmv47bffaNu2LTpnnYecnBwmTJjAO2++ienECd5eupSb77iDY8eOleuFcDFjxgy2bNnCjz/+SGhICLMeeYS98fF0bNcOURRl4+DYsWP89NNPGI1GnnjiCW666Sbi4+PRarUA5Ofn88477/D111+Tk5PDqFGjuOWWWwgICGDNmjWcOHGC0aNH06tXL8aOHQvAgw8+SHx8PF9//TWRkZH88MMPDB48mAMHDtC8eXOmTp2KxWLhjz/+wMfHh/j4eHx9fYmOjua7775j9OjRHDlyBKPRiKGUsNn48ePZvn0777zzDh07duR4fDznDh8GpCJSDpMJe04OmuDgCr1e77zzDj/99BMrVqwgKiyME3//zelz5xCcr8OYMWMwGAz8+uuv+Pv789FHH9G/f3+OHj1a7HNWEunp6Xz55Zf07NlTfm3LxOWBQfKauYwWAK3Wn4KCvGIGjCiKHDUcAatAeLY3/v6dpU7Wx5we0q5346WVPn9moQCCmkL6CTiwArrdU+6U3FOo3QkI6E56xl9kZu4kOmpc+dfm0r8YI6Wf9cGAOfIrrJwovS+Nr4M7vgK9HxijJG/Mya3QfEC5h6koupgYuR+SvmnVeyCVht+AAaQlJKAOCiL04Wnl75CwUfp58AfoMr7a53OlE3T33aR/uVz2wvj16ytvc5jNnH38cSm1XaXCf9SoWpxp9VBlA8bhcDB9+nR69epFO7c88jvvvJPGjRsTGRnJ/v37eeKJJzhy5Ajff/89AKmpqR7GCyD/nZqaWuaY7OxsCgoKSrzZzJs3jzlzKt4O/Epi/vz5PPHEE9x+++0AvPrqq2zatIm33nqL999/n9DQUACCg4MJd3MZ9uvXD9Fmw3IqCUfTpix8+WUadO/Oli1bGFaBAlO5ubl8+umnLFu2jP79payDpcuW0ahZM0SrFfvFNBIzM/jpp5/YunUrPXtKYYUvv/yS6OhoVq1axZgxYwCwWq0sXLiQZs6nwFtvvZUvvviCc+fO4evrS5s2bejbty+bNm1i7NixJCUlsXjxYpKSkoiMlG5Qjz/+OGvXrmXx4sW8/PLLJCUlMXr0aNq3l0pdN3X7gnYZA2FhYQQEBJR4fUePHmXFihVs2LCBAQMG4LBYiDSbEZs0QR0QgCY0FPOxYzhy8xBttgplESUlJdG8eXOuu+46HNnZhHfpImUgCQJ//fUXO3fu5Pz58+idHpn58+ezatUqvv32W+67775Sj/vEE0/w3nvvkZ+fT48ePVi9enW5c8FhA7sFhwAOUTJgVKrCz5ZG4w+cRRQtFBScwctLem/OnV9NlvUkKrtIs8OpkhB4z+dSRlOT6yG0BfqCMwCYzKmI3R5EWP807PoUut4N5dT+KEyh9qwIGxB4LSRKQl6XcVwm7voXKAwh5ZRtwBQUnOZU0ifo9WH4+bXFz68del1IqeMdDgt2uwmt9hKFqQdXwbf3SG0dWt4Et35W2B08tj/sWSqFkarTgHHqYKB6asAUJWjCeKynTxMw5lbU/v7l73DR6ZlN3iml36trITOvHuPywqR9soiL772Hb19Jk2rPzeP0gw+Sv2MHglZL5BsL8L2uV21P95Kp8n/H1KlT+e+///jrr7881rt/ybZv356IiAj69+9PQkKCfHOqCWbNmsWjjz4q/52dnU10BZuECVoVkS/0rKmplXvuSyE7O5uzZ8/Sq5fnP2OvXr3KDT2knj7NU48+xh9/7+BCejp2USQ/P5+kpIplOyQkJGCxWLj22mvldSGRkbRo3hwA6/lzHIyPR6PReIwJDg6mZcuWHDpUWNPD29vb4/+jQYMGNGnSBF+3jKgGDRpw/rykXzhw4AB2u50WLVp4zMlsNhPs9IZMmzaN+++/n/Xr1zNgwABGjx5Nhw4dKnRtAHv37kWtVtO7d29EhwNrUhKi3Y7Ky4A2MlISPLt7YQIDyz3mxIkTufHGG2nZsiUDb7iBQddcw6CbbgJg37595ObmyvN3UVBQQEI5ob0ZM2YwadIkTp06xZw5cxg/fjyrV68u+yYvF7DTAiIqlR6VqvArQaXSyh6ZtLRNBAY2w24v4PjxVwFokiLiZbJIbQV2L5V26joRAL0+DFAhilYsbQeh//1FOPcfJO2AxnFlXkthCrWnoNTf2AGVSofFcpH8/ER8fMrxGLg8MH4R0k/f8kW8eXkJ/PvvOMwWzzE6nWTM+PjEYrPlYDafcy6pWK2SALZly7lENbyz7DmVhsMBa2dJxkuH22HE+54379gBhQZMNeLKRIJL64FUGpqgIBoumF+xwQ4HpDn/zy25cP5gYfq7QoUp6oUxdO5E8n3/w3TgACpvb6I+eB+fHj1qe5rVQpUMmAcffJDVq1fzxx9/EOUWQy0J143r+PHjNGvWjPDwcHbu9Ixhn3M2oXJ5B8LDw+V17mPKcvXr9Xr5qbWyCIJwyWGc+obDYmH8HXeQlpHB/Keeoln37hiMRuLi4mQtUlUR1GqpHLkoyjUgyqNouEMSkhZf53DG03Nzc1Gr1ezevRu12vO9cxk9kydPZtCgQfzyyy+sX7+eefPmsWDBAh56qGI1KFz/a6IoYj1zFofJhKDWoG0ULQsRVUYjDpNJ0sE49TRFY8/uWpQuXbqQmJjIr7/+yvpVqxj3+OP0+/lnvv/5Z3Jzc4mIiPDQ8bgozUvkIiQkhJCQEFq0aEHr1q2Jjo5mx44dxMWVYSy4Cthp1YDNI3zkQq2WQokXLv5Os2aTOJW0CLM5BS99JI2EYGATbJ4HuangHQKth0uvi0qLXh+G2ZyKSchH3/5WSdewa1H5BkyBKwOpicd6lUqP0diZzMy/ycz8uwIGTCkemFJEvDk58fy7dwJWazreeTb8rAZywhuSbzqFxXKetLTzpKVtKvV0R4++gNGvHUZjxY1kmaRtkHMWvPzh5neKex6a9gZBDWnHIOMkBDap/DlKwN0DUxMhpEqRfbpQdA2SsasYMJXG3Qtz4a23EO12LAkJqAMCiP7kYwzta6/5YnVTqcd/URR58MEH+eGHH/j9999lcWhZuFJ3IyKkp6C4uDgOHDggP0kDbNiwAaPRSBtna/C4uDg2btzocZwNGzaU/WV8lWI0GomMjGTr1q0e67du3Sq/ni7Ni0sA6zCZsJw4wfY9e5g6fjw3T5xIe2fK+sUKGhwAzZo1Q6vV8vfff8vrMjIyOHr0KCofHwStlpaNGmGz2TzGpKWlceTIEXl+VaFz587Y7XbOnz9PbGysx+IeJouOjmbKlCl8//33PPbYY3zyySclviYl0b59exwOB5t+/hl7ViYgoI2ORuXWK0buxp2bi2i3ExoaSkpKYQfkY8eOkZ/v2dTQaDQyduxYPpg7l89ff50fVq8mPT2dLl26kJqaikajKXZNISGlhzCK4jLyzOVVRXZ5YJzfAiUZMFqtLyBgMiVz8eJGTp36EIDY2CdQN5SKy5Hwu/Sz8/+BpvAhwksvfebNphToPllaGf9juVlALg+MoYgBA+71YHaVfW0A2c5UctkDU7oGJitrD3v+vQurNR0/u5Gu+7Jotz+FuH8y6NN5Hd26rqRFi+eJippATMzDtGr5Eh07LOKa7qu5/rqdhIYOQhStHPjvwdIrGJfFgW+ln62He7yGMl7+EO30Yh7fWHx7FfFq3RqV0Yi+dWvU5RjJNc7FIqn2SdtrZx5XAK6MJPPRo1gSEtCEh9N4+ZdXlPEClfTATJ06leXLl/Pjjz/i5+cna1b8/f0xGAwkJCSwfPlybrrpJoKDg9m/fz+PPPIIN9xwg+y6HzhwIG3atGHcuHG89tprpKam8swzzzB16lTZgzJlyhTee+89Zs6cyT333MPvv//OihUr+OWX6q2DcKUwY8YMnnvuOZo1a0anTp1YvHgxe/fu5csvvwQknYfBYGDt2rVEBgWhunABo7c3sU2a8NWGDcQNH052djYzZswo1cNVEr6+vkyaNIkZM2YQHBxMWFgYTz/9NCqVCkGlQhsVRazVxrC+fbl30iQ++uQT/Pz8ePLJJ2nYsCEjRlS9HkSLFi246667GD9+PAsWLKBz585cuHCBjRs30qFDB4YOHcr06dMZMmQILVq0ICMjg02bNtHamQXRuHFjBEFg9erV3HTTTRgMBo9wFUCTJk0Yf9ddTH7wQeY/+SRdrrueM//s4vz589x2222AlP4s6HSIFgv2nBz69evHe++9R1xcHHa7nSeeeMLDk/TGG28QERFBp06dsJ5I5Pv16wkPDycgIIABAwYQFxfHyJEjee2112jRogVnz57ll19+4ZZbbqGbs7+MO3///Te7du3iuuuuIzAwkISEBJ599lmaNWtWvsFvK0AEHEhGXEkGjCCoUamkInv/HXwYh8OEv383wsKGQkaR8FTXCR5/6r0iIPtfTKaz0GgwRHWH07ukcFPvGSVPyZYjh2S8i4h4QRLyAmRk/l2+DsbpgbH5BHD00JNoRS3BAVoCstJR2SygkQzR9PRt7D/wP+z2fPz9u9Fpfzoa2wlQaSA9AfUXt+I/YTX+UV1KPVXrVq+Qm3OIAlMS8Ydm0qH9hxXv82OzQPwq6fd2xdPfXXVvhNj+kqfm+EboPqlixy4HtZ8fsRvW114hRXcuSg0l8QmFvAuSB0YUy9VMXSoFBWdIPr0Eh8NMi+azPcKo9RVNUBBB//d/pH38MbqYGBp9ugitUyt4JVEpD8zChQvJysqiT58+REREyMs3zrQsnU7Hb7/9xsCBA2nVqhWPPfYYo0eP5ueff5aPoVarWb16NWq1mri4OP7v//6P8ePH88ILL8hjYmJi+OWXX9iwYQMdO3ZkwYIFLFq0iEGDBlXTZV9ZTJs2jUcffZTHHnuM9u3bs3btWn766SeaO7UoGo2Gd955h48++oio2FjGTJ2KymDg0yVLyMzMpEuXLowbN45p06YRFhZWqXO//vrrXH/99QwfPpwBAwZw3XXX0bVrVwDUPj5owkL5aO5cOrVowbBhw4iLi0MURdasWVOxLJkyWLx4MePHj+exxx6jZcuWjBw5kl27dtGokZTCa7fbmTp1Kq1bt2bw4MG0aNGCDz74AICGDRsyZ84cnnzySRo0aMCDDz5Y7Piiw8HbTzzJLTfeyCMvv0y7nnHce++9HnWPBEHwqEK8YMECoqOjuf7667nzzjt5/PHH5Qw8AD8/P1577TW6d+/O9bePJclpoKhUKgRBYM2aNdxwww3cfffdtGjRgttvv51Tp04VE7W78Pb25vvvv6d///60bNmSSZMm0aFDB7Zs2VJ2SFUUwWrCoQIRUS5gVxJqtWTUOhwmQKBF82ekm3Ok2w29WT8p28gNLy/pC9Nkdnqkut8r/dy9GBwle75cBex0uhA0muItLfz9uyAIWik0ZUou/fpANmDO2P8jJWUlSanL+beDP3/EBbF//32cPbuC1NQf2bd/EnZ7PkGB19G502I0F09K+9/6GQQ0kjKolgyVMq1KQas10q79uwiCjosXfyMpueJ1bzixSWoV4BMGMTd4bBJFkf0HprDlj46c8E+TvGWJWySjp5pQ+/tfUiHFaiPNKeBtdyuotJKGKfNUjZ0uL+848fEz2L6jH8nJn3HmzJdkZO6osfNdbkKnP0z0xx/RZMU3V6TxAiCIFS1EUs/Izs7G39+frKwsjzo0ACaTicTERGJiYvCqC08elwl7Tg6WU6cQtFr0sbEI6prX/YiiiCXxJI78PNR+fugaNy5/pzqC9dw5bBcuIKjV6Js3LzXLyJFfIDXDU6nwatWqQoUAXe+FSq9H7zQ0Lys2C5w/iEWrwqRXodH4ylVv3TGZTJw4cYKLaY9htR4lImIMbVq/UjjgzXaQlQxjl8n6FxfJyUs4emwuoaGD6dD+fSlb6fVYMGfDlL8gvLg7OzX1Jw7GP4K/fze6df2mxKn/s3sMWVl7aN36VSIjSinY53DAi6HgsLFr8HVk5x/G39iZgrR/sZRgN4eG3Ei7dm+jspjhFaf4/8kkMGVLxkvmKakdwcTVklFTCqfPLOfIkWcRBDVdOi+XeziVyXf3Sinm106BIa96bLpwYT37D9wv/623QGxCDg2GfIvQ9IaiR6rfLB0OiX/AyIVSTaHTu+CWj6Dj7dV6muzs/Zw89SEXLqwHCusf2e35NI99mkaNyk/1V6hZyrp/u6P0QrqKkMvWGwyXxXgBpxi3oWT923NypFYD9QBHQYEsQNZGRpaZIi0YvKQ6Lg6H3E+qPGq/B5IrA0n6P1CpSw8dCoJAk8ZTCAkZQGyzIqGf0Z/CsLegVfG0e5cHxuzywGj0hX2UzuwuNh4Ka8CUZEy5CHDpYMpq7JifBg4bBV5qsvMPAyrad/iQ605F0X1PJjF+QzEaOwECEeGjaNfuXckDle7MgvEJlXQnAdFw9xoIjJGMmCVDIaN0r0DDyDto0GA4omjnv4MPY7Gklz5HAEt+YYuAIuEjh8PK8QSpinloyI14eTXErIODrf3YnTiD7Oz9xQ5nt5sxmVOx2+thV3hXCCm4OTRyZslUsw4mIWE+u/65hQsX1gEioSE30q3b9zSKlkJyFa7yrFAnqP/BPoUKUx1l66uCSq9H7R+APSsT2/kL6BqX/gRbFxAdDqxnzoAoojYaUZXTeM4VRrKlpWHPzq5Qozr5vdDVkuveVYFXLQAialXZJeTDwgbRqFEJmqVG10pLCeidIl6Tya0vU1Q3KQRy+h855dqdArcu1KURGHANp059SHrGVkTRjiCUYIw7U6jPRQYBIoGBPaRaLr7hGM/uxajqQtOu7+BwWFCpCkXZchpvkFtKsX8UTPxF8hCkJ8DnI+CetYXZTW4IgkCrli+Sk/Mf+fmJxMc/RseOnyIIpTwrHl0L1jzJuxPl6a05e3YF+fmJaLVBtGnzOoKgJWnXI5zKWUeW+iK7/rkFf/8uOOxmrNYMLNYMHA7pfdXrw4nr8Zsc/qvzmHOkLCyAkFhoFAfb3pV0MNV1CstFTiVJIv7wBiNp3Ph/+PpKZRhMBVJ4MFcxYOoVigfmKkI0S3Hzy23AAGjCpGJ69pzsOu+FsV286EyZVqONiKiQGFMl62ByPNoolIZocnrDvGrJgLEV4AAcSHMtScB7qbg8MBbLBRzOTtc0lPRRnNlT4j6FHpgmpR43IKAHWm0gZnMqFy/+XvIgpwFzPkR6RmsQJtXaKZpK7WG8gKR3AQguUhPFv6EzfNQYMhLhi1sgv2TvikbjS/t276NS6UlL/4OEhNdLvRb++0762W60h1jVZsvlROLbAMTETEOj8UOt9iKm/VzidmUQkSp50LKy9pCTexCT+axsvACYzamkZ9ROdfEqkeYm4DUEFmZcXThc6usMUtHBpOTFFfI4pZz9FlG0YTR2pm3bBbLxAuDjI4Vx8/KOVbi9i0LtoxgwVxGixXnTrIWnfpcXBsB2iR2caxJHQQG2C9L8NBERcon/8lB5eyNoNIgOO44izU2LIoqi/F7UhjGJKII5x+l9kW7iNZF5odUGycJgs9mZuuwyYC4cKrE3Ulkp1C7Uaj2REVIF59Onl5U8KCeFfC8VOV5WBEFNaOhAab2rmJ2rRkxRXDfSogYMSC0Jxv8oHeN8PHx5q+Q5KAFf35a0avUyAKeSPiY5eUnxQQWZcGy99Ht7z/DRqaRPsFrTMBia0DDSTQPiE4I+pBNtjuZyjd8DtGmzgI4dFtGt2/fE9fidG67/l4YN/w+AixerL926xnEPHwH4hECI08Aowwtz7Pg8jh17kROJZffJE0UHZ85+BUDDhsU1Nd7eTRAEDXZ7bmHIU6HOoxgwVwmi3S53iRb0unJG1wx1xQtjz87GmpKCLSMDR0GB7DEpGjqqUOlzJ4IgyF4Yezl9uESbDdFZf0bQ1cJ7Yc0Hhw27Wvr414T3BaTXpDCM5KqKGy719REdkLLXc1rWDGw2qYaKt6FssXfDhncCAukZf8mZSx7kpHI+VDKeAgPi0Omc1Y1lD0wp1XhLCiG5ExQD41dJXoIzu+HrO+V6OkWJCB9Js6ZSl/ijx17k3LkiZSAO/Qx2C4S2hgZt5dVm8zmSkqQspthmM1GpihjRzW8EwO/kf0SEjyQkpC/+xo54ezdGqzUSGiK19bh4cVOVvQl2u5l9/97DieMLqrR/pXHVgAmJLVzXyFkGoAwdTHb2XgDOnPkSqzWj1HHp6X9iMp1GozHSIGxose0qlU7WXeXmHS22XaFuohgwVwmyaFSjuWwC3qLUBS+MaLNhSU7GlpaG9cwZzAkJmA4dwpyQgCUpqdKhI3fU7mGkMm4c8nuh01UoY6nacXoNHFrJ61JTBgyAl1cJOpiGzvTrIkJel/dFrw8vV7thMEQTEiw1qivRC5OTwjmnAdOggdsNSy5mV0oxPZeItyQPjIuw1vB/34HOV8qa+fZusJfc+btx4ylENRwHiByMf5z0DLeb8X/O4nVFvC8nEt/G4SjA39i50HPkTqyzF1LC7yWmowcEXIta7Y3Fcp6cnP9Kv44yyEhZy8WMLSQmfUBG+mUoKOdKoQ4pDOsUGjAle2AslouYzZInzW7PJ6kkL5eT02eWAxARPqrU/y33MJJC/UAxYK4SakvAW5Ta9sLY0tNBFBG0OqlasEoFooijoEDOIKpM6Mgdlbc3glqNaLfhKFJ91x05G6y23gtTFiJgF2pO/+JCrsbr7pZ3iVVP/+Mx1uVJKUvA605UlBQqSUn9Drvd8/XOyz9Jrq8GARWhoTcWbiirI3V+ulSPBYrVtClGw65wx9eg8YIja2DVA1LqdhEEQaBFi2cJDR2MKFrYv38KObmHpYaSiX9Ig9qNlsfn5h7l7NmVAMQ2n1WyER3ZBbwCwJRZopZIrdYTFHQdQOkaoXLIPfen/PuRQ0/hcJRsoFUbRUNIUJiJdPZfWXTuTk5OPACCIBnip08vxWYrHtIzmVLkFhANG95R6hR8fCTjKS9X8cDUFxQD5iqh1m+aTmrTCyM6HNjTJUGgpkEY+pgY9K1bo2/eHF10NJqQELTh4ZUKHbkjqFSo/KTeQY6s0sNIYm0ak3YbWPOLFLCruVRuvauYnYcHpmQhr0vAW5b+xZ2goOsxGBpjs+WQmvqjx7bzgpTqHKRvhVbr1mTTXcRb1EvmCh/5RYLOp/wJxFwPt30uVew9sAJ2fFDiMEFQ07bNGwQEXIPdnsvevXdT8N9SKYzWsJsUlnIiCX4dhIYOIsC/a8nnVWugmeR9Kq25Y0iwM4yUVjUDJqeg0AuRZ07i9JlStEZVxGrN4OixFzl7dgU2S1ah9ijEzYAJbCLpjRzWEg01lwETGjoQb2+pyWby6c+LjTubshJRtBMQcC0+PrHFtrvwdXpgLikTyZQFlrI1cArVh2LAXCWIltrLQCpKbXlh7FlZiDYbglYrh3sEQXAaVf5ow8PRhIRUOnTkjtwbKTur1DBSrXrDzJJhZddK2huVynBJ11seLg+Myd0DE9EJBJXUvM9NTCt3oa6gASMIKqIa3gXA6TPLPF7vcz7Sk3hYUD/PnVweGFuB/FrIVCR8VJQWgwqLz/25oFRRr1qtp0P7D/HxaY7Fcp5/sz/lcKwPCS0bcOrUx5w5+w1JSZ9xMe13BEFdvN5OUVxhpFIMmOCQPoBATs5/hQLqSpBrOQNA6AXpf/XEibcwm6vvgePUqY9JTl7MocOz+HNrD/6L1XAx2IDDv2HhIEEosx5MTu5BAIx+7Yhp8gAAycmLsdkKDQiHw8bZs1JBRA8xdAnIHpi843L7hkphzoEP4uCTfiV64xSqH8WAuUqo9bojbtSGF0YURexpaQCog4JqTHui8vWVwkg2W6lF7WrVGyYbMDWvfwG3dgLuHhi9ryRcBQ8dTIErhbqCISSAiIhbUam8yM09TFaWdKzc7EPkGUBwiIRGeFYHRucNemednqI6GFnAW8muzF0mQnAsFKTD3x+WOkyr9adTx8XotaEUaG2ciTRw0vY3xxNe5fDhpzh2/CUAIiPvKLOQHwDNJA8LZ3ZD0t/FNut1IRiNUifnixdL76BdEnZ7AfmiFEprkZCHMduK3Z7L8YRXytmz4qSnS81ntdogHKKFc2F69rX1YeuO3hw99lKhILcMHYxL3+Pn146wsKEYDI2xWjPkbCOAtLRNmM2paLVBhIWV3YrGYGiEIOhwOAowmc5U/qJObIbsM1Lqd17dzbS8klAMmHrExIkTEQSh2DJ48OAy9xMdDjcPTM1lvUycOJGRI0dWaKy7F8aeU/jU+sknn3D99dcTGBhIYGAgAwYMYOfOMiquVhBHXh4OkwlUKjSBgeXvUEUElUru6mvPKJ4VIdpsbtlgl9mAcaVPC2BF+n/QaMovuncp6L1K0MBAMSGvKIrk50thn3Jv3m5otf6EN7gZQA4fnD8j1VYJyrShNZZgjJSWiSR7YEoPM5SIWgN9Zkm/b3tXSo8uBS+vCLqLQ2lxPJeYrBCiosYTHj6SkOB++Pt3JTi4N01jHi7/nMYIaDEYEOGLkdLNswgukXNlw0i5eUdBAK3FgV4TSIuEPBAhNXUVmZn/lH+AcrBY0snJlcI/116zhu5e44g6U4DWocFiuUhy8mecPLlQGuzywCTv9BAsW63ZFBQkAeDn1waVSkOTxlLLhaSkRdjtUmbYGad4NzLi1lJ7fblQqTT4+DQtfA0qy7ENhb9nJlV+f4VKoxgw9YzBgweTkpLisXz11Vdl7iNardLNS6Wqkji1POx2O45KukxVer18o7ecOiVlAFksbN68mTvuuINNmzaxfft2oqOjGThwIGfOVOGJyH2OLu9LQECZbQGqA9mAycmRjRUXDlcxQa328meDOdOnLXrpvBqNHxpNDXtgnCEkmy3HU2Dp0sE4hbwWy0Xs9lxAhcEQXalzREWNA+DChXWYzec5lybVVmmQ4w0ledpKE/KWVQOmPNqOkrxKpizY/n7p47JOo9/+KdFnTTRtPJWWLZ6jbZsFdOz4Cd26rqBTx8/Q6YIqds5bF0tNNK358OVtcORXj80hznTq9PSt8g29IuTmHALAL8+GcO39+OfYiHQWzjty9DkcDltZu5eL1DBRxMenBXp9KMaMHFom5HEdY2nWbKY0Z1cRvgbtpGwvcxacP1Q4R6cB5OXVUNY4hYePwEsficVygbMpKygoSCYtXRIjR5YTPnIhZyLlVlIHI4pFDJiaa0KpUIhiwOBsOGix1MpS2ToNer2e8PBwjyXQ6VHYvHkzOp2OP/8szCB47bXXCI+K4tzFi6h0Ovr27cuDDz7Igw8+iL+/PyEhITz77LMe88jIyGD8+PEEBgbi7e3NkCFDOHas8AO9ZMkSAgIC+Omnn2jTpg16vZ577rmHpUuX8uOPP8qeoc2bN2OxWHjwwQeJiIjAy8uLxo0bM2/ePAC0ERFogqQva3t2NuZjx1jyxhvc/7//0alTJ1q1asWiRYtwOBxs3Fj1olwOs1n28miCg6t8nIqiMhhQeXmBKGLPzPTYJpqlG0Gt6F9M2ZL3RSNpXvT6ynUerwoaja/s5ZFrwUBhJtLZf8HhkAW8Xl6R5T4pF8XPrw3+/l0QRRtHj80l33JGCh85IkreweWByXEzYEQR0pxVeEurAVMWKhX0dXphdiwsuXqs3Qrf3iNlOkV0gg5jK38ed3TeUiZUq2FgN8M3/1dY2Rfw9W2FXh+Bw2EiI6O4hqQ0cnMPS/vn2qBxT4juQbPEPDToyc09zJmzyy9p2hnpknESFNhTWuGsAaMKbklExGjnHA5JYSS1BqK6S+PcdDA5OZL+xc+vsH6OSqWjceMpAJw69RGnT38BiAQFXoe3t7Ou0IWjcODbUjUqVU6lPh9f2AoBFA/MZULphQRYrVZefvnlWjn3U089ha6aipn16dOH6dOnM27cOPbt28eJEyd49tln+XrRIhqEhMg3zaVLlzJp0iR27tzJP//8w3333UejRo249957ASkUdOzYMX766SeMRiNPPPEEN910E/Hx8WidHpz8/HxeffVVFi1aRHBwMBERERQUFJCdnc3ixYsBCAoK4p133uGnn35ixYoVNGrUiOTkZJKTkwGkeiuRkaiDgrCmpODIy8N24QL2jAw04eFoAgLIz8/HarUSFFTBp9ISsKdJNxOVn99l052oAwNxpKRgz8hEHRwsC2VrW/9i1kvPLBqNscb1Ly68vCLJzc3GZD5bWL49tDVoDJImJ+0YBdaTQOX0L+5ENRxHVtYezp9fA0BwugWNT2TJg13VeN09MHkXwJIDCB5ZQZWi1XCpw3bqAdj6Ntw4x3P77y9C8t+SBmfMEqm55aWi0UvHWnU/HFgJ302WUo47/z975x0eR3l18d/M9tVq1btVLBe5dxvbYKrBNiUQOoFQA6ElIRCSjxQCAUKHUB0IwXQI1RAgYJoNGNwtd8uyrGart1XZvjPfH+9Wdclygz3Ps8+uprwzO9rdOe+95557CZIkkZx8Ivv2vUpDwxckJ5/QryHb2kWkw9LhE9GqKT9DX7mKEVUaijJhz55HSEs9Fb0+eVCn3NQs9C+JiUeLBQ2hCiSDPpmYmFF0dBTT3LxG6FZy5sCer4QOZpb4jQpUIMVaxkWMnZFxLqVlT+Jy1VBR+TwQMD304+0roHarIJH+scJhCQp5B0hgAo7KAUQJzEFBNAJzhOHDDz/EYrFEPMLJ1913301CQgLXXHMNl1xyCZdddhmnnSgqMQIEJjs7m0cffZSCggIuvvhifvWrX/Hoo8KKO0BcnnvuOebNm8fkyZN59dVX2bdvH0uXLg0ex+Px8PTTTzN37lwKCgqwWq2YTKaICJFer6eiooJRo0ZxzDHHkJubyzHHHMNFF0V6MchGI/q8PPQ5OUg6ParXi2fvXnw2G3/4wx/IzMxk/vz5g7peqs+Ht0VoUQ5G9CUATVwcSBKKy4nqDIXvD1kFks+Dz+fAqxVfeYMh7aAdOuDG6wqPwGi0kDlFvN63fsAl1J2RmroAnS70/02rd0NsHxGYcBFvQMAbnz14YiHLcMKfxOs1z0aOv2sZrPyHeH3mk4MnSd1Bo4OfPgPTLhOl2e/fAGtE08LkZPHdb2jsnyuvqiq0+wlMbLsXYtNg/E9BayJrdwWxhjy83jaKdv1tUC6/DsdeHI4KJElDfPzMyCaOfu1RQrzQvTS3+CMuwUqkkJA3UIEUGzsh8lJoDOTmBIiJil6fGrwG2PYK8gLwxV3dmhkGIzD23ahqV5PAHlHsrwbLEMLpKIE5OIhGYACdTscf//jHQ3bsgeCEE05g8eLFEcvCoxN6vZ5XX32VSZMmkZuby6OPPopaK2aagVn/7NmzI0pn58yZw8MPP4zP52PHjh1otVqOOirUYTgpKYmCggJ27NgRcZxJkyb1eb6XX345J598MgUFBSxcuJDTTz+dU07p6i4a6OgsWyx4qqrwtbRw37338sYbb7B8+XKMxsF5lfiam0FRkA0G5Jh+eHsMESStFo3Vis9mw9fcjGwS7p+HzAPG1YZLL/7nWl0cGs2B837pjG4rkUDoYCq+FwQmR5C8/pZQd4YsG8jKvICy8qeRVZnkJjdM7NotGuheAxPQvwwmfRSO0QvF+9q3Hr59FBbeC7Z98N4vxfpZ18C4brp67y9kDZzxmNCLrHoK/vcHGPsTEuLnIMsmXK4a2tu3R6RcuoPTuRefrwNJUTF7DWI8SYJxZyJtfoOClizWmSqoq/uIPebhjMj/7YBOs9mvbbFaJ6PVxkLdRrHCnAxm8TuWkDCbvfteprnZT1iGzQBJI8ruWyrxxSbR0SEIZ3fvJyvrQsrKF+PxNJGZeX6oFUNJWDWWywaf3Q4/jawaM5mykWUDiuLC4ajon6DcaYOK71GBrePi0MRaGNtYzoEzJ4gigGgEBnHz1Ov1h+QxUA+OmJgYRo4cGfHonF757jvxI9HU1ERTU1OEdf1QwWTqn3/ItGnTKC0t5a677sLhcHD++edz7rnn9ri9JMtoEhL4xwsv8ODTT/Ppp5/2iyh1B1VV8QbEu2FpnIMFjV+b5LPZRCWYzycE1Rz8FJLP1RyKvugPvPYlHN16wUCEkHcwJdSdMSz7MqzWyeS2paD1qaL5Ynforp3AYDxguoMkhaIwa/8tZuJvXylKrDMmwyl379/4fR17wT1C+Kr6oOwbvyuvSNXU98OVNzx9JMekhTpkTxV+O3Gbv2LMyL8AUFb2JPv29V5A0BkBcW5CUP/S1cAuPn4WINI4bneDMBUMRDYKX/WLjBX0+pRudVwajZlx4x4kI/1scrKvDK0o8b//UacAEmx6HcojO3ZLkoYY88jg8fuFPctB9dE6LI86z3aq043Y3fu6GiVGMeSIEpgfGEpKSvjtb3/Lv/71L4466iguu/RSfP6bZmDWv3p1pG/EqlWrGDVqFBqNhrFjx+L1eiO2aWxspKioiHHjIvPNnaHX6/H5uoZdrVYrF1xwAf/617/4z3/+wzvvvENTUzciRz8eevJJ7nvmGd5fvJjpEyf2+713htLaiurxIGk0wcqggwk5JgZJp0P1+fC1toZK2TXaA14JFQFVxYWw2tdpLAc1+gK9RGD8Ql61duuATey6g0GfzMwZ75Jf5b9xxPYUgQmkkMI6UjcOsoS6O4w4Ueg2fC749ylQuWpodS+9QZJg+LHidZkQ8wdSKI39IDCBCiRLhzdE9AByj4H4HHC1ktViJC/vRgB2Ft3e73YFqqrSFBTw+vUvwR5IIQKj1ydisYwBoLnZ/zs0xt/Pavm9tH5xE9B99CWA5KTjGTfuQXQ6v6u24hM6GoB5t8C0S8Xrj27p0scqxjJAR16//qUpJy+4qCGOnvttRTFkiBKYIwwul4uampqIR0NDAyDKmS+55BIWLFjAFVdcwZIlS9i8eTOPvfiiKNv1l5RWVFRw8803U1RUxOuvv84TTzzBb34jvCdGjRrFmWeeydVXX823337Lpk2buOSSS8jKyuLMM3sPfefl5bF582aKiopoaGjA4/HwyCOP8Prrr7Nz50527drFW2+9RXp6OvE9EIr777+f22+/nWceeICcrCyqSkqoqamhvQdTuN7gPQjGdb1BkqRQSXVLS5j+5eB2oPa6mvD6K7b1xh6iEgcQhu76IQHEZUNMCu0mFUVxIssGjMZh+3/AtkDn6x40MAFi09EgWisANO1HBVJnhEdhAufyk8cHbpA3WOTNE8+lfgLj94NpbduMy9X7TTUYgQnoXwKQZZjsF8MWvkr+8JvIyDgXUNiy9dfYWjf1eVodHbvweBqRZRNxcVPEwkAX6vAeSITrYPxppKNvgvl3gtZIu1P8r2Jtzm6bWXaL6k1CuGuwisjf/DvAlCiqh1Y/E7HpgHoiqWpQ/9IcEyJCDYn6qA7mICBKYI4wfPLJJ2RkZEQ8jjlGNG675557KC8v55lnxBcyIyODxY88wp1PPMGWkpLgGJdeeikOh4NZs2Zxww038Jvf/IZrrrkmuH7JkiVMnz6d008/nTlz5qCqKh9//HGfep2rr76agoICZsyYQUpKCitXriQ2NpYHHniAGTNmMHPmTMrKyvj444+ReyAUixcvxu12c9H115N/wglkT5hARkYGDz300ICuk+JwiIaKkoRmPyqY9heBNJLS3o7SJkiYZDi4ERCXW7iC6lQtGs3Br34KRWBqIi3aJQmyposfe0RaIahXGCw8zlBDxp4iMOYk0coAFewNoqS2cYhSSAEMnwf5x4vXM38hhLAHC7lzxftrKoHWKgyGVGJjRSSzsXF5r7sGSqhjO3yhaq0ApvjF93tWINn2MqbgbpISj0VRHGza9ItgFK0nBNx3E+JnIst+Eh9MIY2O2DYhwU9gAjoYjRaOuQmu+462RBFVid20DJ5fAHU7ez0uEEofDT9WiJ7NiYLEACy/F1pD0UHLQEqpa7ZAew0+gxmbuzS42BanxdPUj/OKYr8QJTBHEF544QVUVe3y2LlTfFFuv/12qqqqSAqrtjlrwQJaNmxgytSpwWU6nY7Fixdjs9loamrinnvuidCHJCQk8NJLL9HS0oLdbueTTz5h1KjQDOnyyy+npZO/CUBKSgrLli2jra0NVVU5/vjjufrqq9m4cSPt7e3YbDY+//xzpoadS2eUlZUJ23+7HfuWLTi2bUPx+bjjjjsGdK2C0RerFfkAmPf1F7JeHxQP+2wtYtlB1L94ve34EDNDg/bgVWGFQ1Q8SaiqG7enU+owawYNSeJmluI3XtsvBCIeWqPo2NwdZA3ECCdo2mvFPl6HaMoYn7P/5xDAuUtEs8eF9w/dmP2BKR7S/bqxQBTGf23rG3r2U/J4WnE69wKBFFInfUlCnj+6o0Lha8iyjgkTniQ2djweTxOFm64QmpUeENS/JPr1L4rSfRNHAjoYCbt9T0QvJyVhGO1GQYJj3QbYuxaemddtO4UIBAS8gSaYAFN/Ljxm3O3w6Z+Ci0OVSKV9m/btFuZ1tlFTUFQ3BkM6Mb4YVEmiqeW73veNYr8RJTBHOBSXK5ia6A4HQsB7MCAZjaKnkKKgOPrvIgrCrt9nswFCvHuooenUukAy9kJgVFV0s22tEuH1QDRhEFBVNVi6rPcoyMYD10KhN8iyLigcdnXSwbjSR9IaK/RAwXLX/UGgOWRsekiA2h3CS6kDAt74XDE7HyqYE0XFkeYQFHsO96eRyr4GIMV/bZuaVkY0OwxHIPpi9OrQedXuI1hTLxHPK+6HlY+hlU1MnvRvjMZhOBwVbNn6624bISqKh5YW0RIkqH9p3esnjjpx7cOg08URGys0d0EdDEKXoqoetNo4jFetEoTK5xaNNHuCq03474DQJwUgy3DawyJate3dYDsGozELjcaMqrpxOPpw1PW77zali9+ZhIQ5JGlEqrDBub33faPYb0QJzBEMX0cHrt27cZeUdLGsD+CQdj7eD0iSFIxcKB0D0794m5tBVZGNpmD58qGExmpFkkNtA7r8LxRFlGK2VAifioZdIjLg7oDmCvAOjMAF4PXa8ClOJFVFrxiG9uY8QAR6IkW48QKNRhtIErFtHgy+ITi/oP6lD61PQKDaVrN/LQQOV+QFhLzfAmCxjMNkykVRHNTV/6/bXQL+Lxan/7YQLuINYMK5MPF8UeX02e3w+oUYfBqmTF6CLJtoaVnNvqo3uuzW2roJn68DnS4k0KXBn6JJzO+W5AV1MGEuwuEOvFL8MDj9H2JF8TJoLuv+WpStBMUjIkiddUgZk0WKD+Cj34HXhSTJwUqkXnsiOZpFjyagWSsii4kJc0m2CHF6o1w9MC+ZKAaMKIE5QqG43XgqK0FVURWli2U9BJo4RlYgLV++nH/84x8H8UwHD9liAUBp737G2B1UVcXnr3DSJCUe9NLp7iDJMnJ8XPB1RAWS1ylIS9MesDeC4hWeF8Z40JkBBZrLB1ySqaoKLpeIRujdKnKgA/MhQlAH44qMwDTYhMYhudEN+zbs/4HCIzC9IdyNN9iF+gdEYHJmi89Rcxm0VCJJEpkZwr6guurtbncJthBo80d0uyMwGi2c/awgDhoDFH8K/5xHTFMDI0f8DoDdu+/rUnEWKp+ejST5bzsNXSuQwpGQIDpRB4W8hBMYf0Vk8kjIPwFQYd3z3Y4T1L+M6CHCd8KfICZVVET5Izn9ailQ8hWoPjypo2i1C6KTkDiXuOSj0XoUPBofrf0QN0cxeEQJzBEIVVHwVFSKqItfDOttau7ijCnKdtWuN80jBMEIjMOO2s9mkRGl03FxB/L0BgRtYiJIErLFEkmqnK1iNitrhZlX4ghInyCcWhOGi5uQx9618WAfcLsbURQPkirSR+gPnolfdzB248br87lobBIRguQmN+zr1Om4vV40KGwsod/oqwIpgIgUkr8C6YcUgTFaQ07H/nLq9PSfAjIttrXY7aVddgl0iI5t9jfd7I7AgEjNzbgCfvG5+Ly27oUlixhW0UqcdRo+Xwc7d/4psr9a5/5HECqh7qF0PT5+BpKkweGoCBKiUAuBsBLqQEuADS8LEXdn9EVgTPGwyK9T+uZhqNkaLKXulcDs9lcfjRwPqJjN+RgN6cgJw0lqFhPH/paYRzE4RAnMEQZVVfHs24fidCBpNBjy85FkGdXtQumIjFQEfUcMhsMiEjFQSHq96J6tqigd9n7t4/X3PTpUpdM9QTYaMRYUoBvWqUzY7X9fMSnCxt5o9VfIAFo9xPm3b6sJbdsHFMWL2y3KZQ0un3AE1R3aVFoohRSamTe3fI+iODBIFmLbfVD6NWx+E/77G3hyJjw0El6/EF44rf/lskEC01cEJsyNd6grkA4XdCqnNhozSEoSy6qq34nYVFG8dPjTJZZ2j/gMxvTR6yhjElyzXHTiVrxIn/2FsS3DkGU9jU1fU1PzLgBebwe2VuG4G+x/BGERmNF0B602NtgqoLl5FYriDbU5CG8hMGoBWIcJs8Bt70YO0lIhiJKkCV2P7jD+p6IppuKF928gxihSTT0SGEUJ6l+aE0JVdABYs0hq8hOYus+63b2/cLlqqav7lOLd97F+/YWs+HoaZWVP79eYPyQcPr/wUfQLvoYGIVCVJHQ5OchGI3LAa6STOdyRqn8JQOhg/GmkfuhgFKcTxd4BHNrS6Z4gabVdSZXHT0p0PTRWNCWAMQ5QoaW8xy664XC5a1FVBY2sF2JMWSuEkocQRkMghRSKwDT4K2KSrbMFySpfCe9eDetfCPmDIAlSUtdPQWQwhdTPCExbNTT7oxE/pBQShAl5vw0uysg4D4Ca6ncjKmzsjlIUxY1GNmJyKoJQh+m2eoTRCuc+DyffBUBM4ccMz/s1ALuK78blqqPFthZV9WI0DsNkCqvy6iOFBOE6mFXY7XtQFCcaTUyk4aFGKyJCEOwBFUSg+mjYDBFp6QmSJAS9xjioLsRSJCJGdru4Ll1Qsxk66kBvoclbBkBioLpKoyPJmwKqSrtjd1cDxz7gdFaxffutrFw5j29XzmXL1uupqPgXLba1eL02ysqfwedzDGjMHyqiBOYIgq+tDY+/r5EuPR2NP8Wi9d+sfa1tKJ6QmdKRWoEUDtniTyP1QwcTKp2OPaSl0/2G4hVurdAzgZEkYfgma4Vepq26++388PmceNyCyBqw+KMv5t4rcg4CjJ0iMKqqBsPrydnnCnGlpIHMaTDnRrjwdfh9aajstfz7bsftgoFGYKo3iyoWjSEU7fqhIHu2+NzYKoIC15TkE9HpEnC5a2lq+ia4adCBV5spPjM9pY+6gySJHk8aPbTXkGM5idjYCXi9rRTt+mv36SNXe5cmjt0h5AfzPW1tohGjxTI2pKMJYNpl4vhVG0QfqgD6Sh+FIzYdFojGuIYVT6KRzaiqt9t0WyD64hwxB7tjDyAHyRaAPjaXuFZBEBv68N4JQFVVqqvfYdXqRVTXvOvXi8lYLGPJyryIsWPvx2gchs/XTn39/kV2fiiIEpgjBIrLJUS7iLLc8AiDbDQim82AKpoX+hEgMAe7785QIqiDcTp6rLQCf+l0y+FTOt0vBFJCGn3vpbYaXcifpKNO/Pj3gIBwV6u1ovX4r1dP5OggwuAX8brd9SiKm/b27bhcNciyiYSkY+HGdXDbXrjmK9HPZ8ypogw513/Tq+inp0YgAtNTH6QAAgTH65/JJg7vX8ThSILBIgghBNNIsmwgPU04aleHpZGCFUj4f1f6IoCdoTMGe1vJlWsYO/Z+JElLff2yYFVS0P8FQpVfYU0cu0Nc3HQkSYvTVUVd/afi1AIC3nBYUmDcWeL12n+LZ8UXLI3uF4EBmHIxjDgRyeskxi6inV3SSB4nFH0EQHN2tv+cxofaFgDE5whdF9DY8BV9weVuYPOWa9m+4/f4fO1YrVOYOuUljjt2I0fN+pAxY+4mM+NcMtKFIWJ1zbt9jPjjQJTAHAFQvV7c5eWoioJsNqPLyOiiaQkQGp9fzKuqKqorpIE5UiHrdMHz76zxCYevuQVUJYzMHQHoK30UDmOccJAFfyqpqybE623H6xUCTIMhHTz+m/Mh1r8A6HWJSJIeUHG5aoONBRMTjxbuwLIG9N1chxz/Ta/8u74rsVxtwpQM+o4gdDZp+6GljwIIppFC0ZYMfzVSfcPnuP3RumALAa//f9D5+vQHOaJqiPLviLWMIS/3OgB8PvG9TfRXFQH9Sh8BaLUxWK3ClC+QcuyxB1KgHHrrO2BvgqpCcLaAIS5E5PqCJIkKK10MluYWIKwnktclUlSPT4WqjSDJNJld/vc2N3Kc+ByS/ASmqfk7fL6erRDq6j5h9epFNDR8jiTpGJH/O6ZP+w+JiUej1Voitk33E5impm+7WBL8GBElMIc5VK8Xd1kZqtuNpNOhz87uVpyqsVqF8ZvXg9LWBl4vqv8mtz8pJEmSWLp06YD2Of7447npppuCf+fl5e1X6bbGr4Px9UBgVFXF2xTW9+hIESwHCEx3N+7uYM0S0RqfO7IRIX7TOr++RK9PQiPpQtGFwyACI0kyRqOY1Tud1cGbUZ/uu1nT/amJ2lC1UE8IRF8MVhF96A16S+R1STpIfYoONsKFvH4CGBs7ltjYCaiqh5ra94GwFgJ2/3encxuB/iDXL9D1d3jOy7s+2FfIYhmDXh8mCq4uFM8pBX0OG0rNBM5/QvcbZs+C9Iki1brx5bD2AfMGZiaYkAvz7yDGLiKYHc0bYe1zgrh8/DuR+rJmoZ79L5o7RJl0RHQJID4HS4cPg1eHojgjSsED8Ho72LbtFrZsvQGPpwmLZQwzZ7xHXt51yHL352s25/pdilVqapb2/z39QDEgAnPvvfcyc+ZMYmNjSU1N5ayzzqKoqChiG6fTyQ033EBSUhIWi4VzzjmH2trIEtCKigpOO+00zGYzqamp3HrrrXg7pQeWL1/OtGnTMBgMjBw5khdeeGFw7/AIhurz4S4vR3E6MU+ciGnMGGS9HkmSujzu/Nvfgo6v3qYmlEAFkl5PeUUFkiRRWFh4SN7H2rVrI3otDRQhHUz3qROlre2Qdp0eNAIpJF0/S5xlTUic6mwNLg54vvh8TiRJRq9PDZEXWRs0sFu8eDGTJk3CarVitVqZM2cO//tf96ZmBwKBpo6trYW0tW0BICnphN52iUhNBG6MPSJQTdSXgBfETDs8yjAUXagPR2QfJQTcbVURBDDTL+atrnoLl7sBt7sekLC0+SMFA9HABI81S1QvNZdCazWyrGf8uIeJjR1Pbk6n7/8ukQ4K9ovqBQEdDIAs64kx9xAtkySY6S+pXvvvYJlzv9NH4Zj5i1BTx+qvRdfq1n3CIPHUh+DXG7HnT8PlqkGS9MTHTY/cPz4HCUhuFYSwoVMayeHYx/oN51NTuxSQyc29jpkz3iU2dmyfp5aRfg4A1TXvdLHOGGqoqkrl3pfYsPHnffa6OhQYEIFZsWIFN9xwA6tWreKzzz7D4/Fwyimn0BE2M/7tb3/Lf//7X9566y1WrFhBVVUVZ599dnC9z+fjtNNOw+1289133/Hiiy/ywgsvcPvttwe3KS0t5bTTTuOEE06gsLCQm266iV/84hd8+umnQ/CWjwyoioK7ogLFIcql95WWUl1dTXV1Nf/4xz+wWq3Bv6urq/nd734XTCMp7e0orSKVcDikj1JSUjDvR1onoINR3e4IkTL4tUE1YuatSUg4pKXTPp8PpZ9+Nfg8wh0UBpbiCRjSeZ3g8+DzOeiwlwR70BgMaWL2FkxPmYIC3mHDhnHfffexfv161q1bx4knnsiZZ57Jtm3b+n/8/UDAzG5f1esAWK2TMRhS+t4xkJqo6EPIu8tPxvKO7n27AMJv0j/UFJLeLPr9QEQaKS3tDGRZT3tHEdVVbwFgNuehaff3MoodBIExWkUEBIKapdjYccya+QHp6WGd7Jv2iNJmWdsvciF0MCKKbIkZ03vDz4nniXRrSzlU+qMegyEwsozlhAcBsJtkfHHpsOhB+PVG4TujNdDsN+eLj5uGRtPpO+zXrCXXtADQ2PhVkGy0tKxj7bqzaG/fiU6XxLRprzFyxO+Q5f79VqemLkSWTdjtpbT6y9MPBBTFw86iP7Fr1500N39HyZ5e2jUcIgzo1/6TTz7h8ssvZ/z48UyePJkXXniBiooK1q8Xqm+bzca///1vHnnkEU488USmT5/OkiVL+O6771i1SnyYli1bxvbt23nllVeYMmUKixYt4q677uKpp57C7Y8a/POf/2T48OE8/PDDjB07lhtvvJFzzz2XRx99dIjfvoCqqvh89kPy6I5Bq4qCp7ISpaMDSZbR5+aSmZdHeno66enpxMXFIUlS8O/U1FQeeeQRcvLziZ8+naPOPZePPxChYVmvZ/jw4QBMnToVSZI4/vjjAREZOfnkk0lOTiYuLo7jjjuODRsG5oba0dHBpZdeisViISMjg4cf7vohD08hqarKHXfcQU5ODgaDgczMTH79618Ht3355ZeZMWMGsbGxpKen87Of/Yz6xsZgS4CvPvkESZL46KOPmDR+PGarlWPPO4/te/YExbsvvPAC8fHxLF26lFGjRmE0GlmwYAGVfhF0AO+//z7Tpk3DaDSSn5/PnXfeGREJfOSRR5g4cSIxMTFkZ2dz/fXX0x4WBQoc54MPPmDcuHEYDAYqKir6dV0lrZ7nXnuPn/7iVsyWWEaNGsUHH3wQsc22bds4/fTTsVqtxMbGMm/ePErKykFnQgUWL36EcePGk5w0gRkzzuTFF5eFwvSerumjM844g1NPPZVRo0YxevRo7rnnHiwWS/C7eaARMLNzOCoASE7q542lU2qiWyg+2PmxeD3m9P6NG05gfmgeMOEYHukHA6LXUErKAgDKyhcDoronaJg4mAgMRGqWeoK/goecOX6LgN6h0RiJi5sC9KJ/CUBvhimXhP5OGC4E2oOAPv0oDBphQFl47FS80y8SEUE/ujSnDIc1CyQNCU12ZEmP07mPjo5dVFW9xYaNl/hTRuOYNXMpCfEzB3ReWq2F1NSFQFc/n6GCx2OjcNOVVFX9hwBNqKv7JPjdPVywX/asNn/DvET/zH/9+vV4PB7mz58f3GbMmDHk5OTw/fffM3v2bL7//nsmTpxIWlroC7JgwQKuu+46tm3bxtSpU/n+++8jxghsE66r6AyXy4UrrKlha2trj9t2hqI4WL5iYr+3H0ocf9wWNJrQTSZgVOdra/N7veT2KUp97LHHePjhh3nmmWeYNGoU/37qKc678UbWL13K2MxM1qxZw6xZs/j8888ZP348er8mpq2tjcsuu4wnnngCVVV5+OGHOfXUUykuLiY2NrZf53/rrbeyYsUK3n//fVJTU/njH//Ihg0bmDJlSrfbv/POOzz66KO88cYbjB8/npqaGjZtCtltezwe7rrrLgoKCqirq+Pmm2/m8ssv5/3nl6A4HChOEeL+3W9/y4O/+x1pycnc8dRTnPub37Br0aIgI7fb7dxzzz289NJL6PV6rr/+ei688EJWrlwJwDfffMOll17K448/LohBSUkwzfXXv/4VAFmWefzxxxk+fDh79uzh+uuv5/e//z1PPx0ykrLb7dx///0899xzJCUlkZqayp49e/p1Xe985Fke+NsfefDxf/LEE09w8cUXU15eTmJiIvv27ePYY4/l+OOP58svv8RqtbJy5Uq8Xi8+g5kX33uPv93zKA8+eBvTps1k+/YqfvnL67Bak7nsssvC0lPdf3Z8Ph9vvfUWHR0dzJkzp9tthhqBCEwAySnze9iyEzqlJrB2kyLau1ZUaBniejcsC0fgJq0z9y/tdKQi7xjRfLHMr4PxR+QyM86jtva/QZFtbMwYaBPRmEETmNy5sHpx72XvgfTRqFP6PWz2sMuw28tIz/hp3xvPvApWPSVeDyb64ockSYyb9DibN19Li20t6zdcyJTJz2MwpKGqPpqbBfHvIuAFobmxZqGxVZBgGkejvZCt234TrGhKTVnEuHEPRPz2DwQZ6WdTU/MedXUfMXrUX9BojH3v1E/Y7eVs2nw1dnsJGo2ZCWMeorLqVZqaV1JR+TwFo+8YsmPtLwZNYBRF4aabbuLoo49mwgQhqqqpqUGv1xPfSYeQlpZGjT/MX1NTE0FeAusD63rbprW1FYfDgambBn333nsvd95552DfzmEDb3V10KhOn5ODxtK3PuKhhx7iD3/4AxdeeCGqqnLPH/7AirVrefLll3n6+ONJSRFh+qSkJNLTQ+K8E0+M/HI/++yzxMfHs2LFCk4/ve9ZbHt7O//+97955ZVXOOkkIcZ88cUXGdbZbTYMFRUVpKenM3/+fHQ6HTk5OcyaNSu4/sorrwy+zs/P5/HHH2fmzJnYUdEDqkNEFv54zTWcNHcu2uQUXjruOLKzs3nvvfc4//zzAUGEnnzySY466qjgeY0dOzZI5u68807+7//+T9zs/ce66667+P3vfx8kMJ2FyHfffTfXXnttBIHxeDw8/fTTTJ48ecDX9fLzz+CiCy8CSwp///vfefzxx1mzZg0LFy7kqaeeIi4ujjfeeAOd39Nm9OjR+Hx2Ojr2cPd9i7nnnls4//zL0eniGTdOYufOXTzzzDNcdunPQw0gO6WntmzZwpw5c3A6nVgsFt577z3GjeumLPUAIODGC8LYzhLTt4BTbGyFtAnCPKziO5hwTtdtdvxXPI9eIByM+4PATTpxxCH3yTmgGDZL+Ny014rqnxSh7UhImIPRkBnsT2Ux5IW0U4OOwPjJcN02UQnUuUTa3REy1hsAgUlNXRiMOvSJpBEiCrfzQxh/Vr+P0R0SE+YwfdprFG66kvb2naxbfx5Tp7yI19uG12tDo7EQG9vD5Dc+B2wVJMt5NFIYJC/Dh/+G4Xk3dvWyGQASEmYH/3f1DZ+RnnbGoMcKorGEltV3slm/Go/sxeCRmLzFRuxX5yMnmmiaEENV1dvkD/8NOt2h6WzfGYMmMDfccANbt27l22+/7Xvjg4DbbruNm2++Ofh3a2sr2f4a/b4gyyaOP27LgTq1Po8dgGK34/W76eqHDUPTjyhIa2srVVVVHH20CLNLkoQmIZE5U6awZdeuXj1gamtr+fOf/8zy5cupq6vD5/Nht9upqOhfmLCkpAS32x0kCSCicQUFPd+YzjvvPP7xj3+Qn5/PwoULOfXUUznjjDPQ+ns1rV+/njvuuINNmzbR3Nwc1JTsbWggX5JQfaKy6qipUwXBs1pJAgoKCtixY0fwOFqtlpkzQ6HZMWPGEB8fz44dO5g1axabNm1i5cqV3HPPPcFtfD4fTqcTu92O2Wzm888/595772Xnzp20trbi9Xoj1gPo9XomTRJlnqqq4vXaqKws4u67n+Cbb1Z3f139acNJY0cFK5BiYmKwWq3U1YkWAIWFhcybNy9IXgJwu5vp6OigtLSSG2+4g1//+q7gOq/XS1xcnL8fjCqM4TSRN/OCggIKCwux2Wy8/fbbXHbZZaxYseKgkJiAGy9AcvJJA6sWyz1aEJjybgiMqoqbFcDYfqaPIHgjJ3Ny79sd6dAZRRSr7Bvx8L9vSZLJyDiX0rLHAbDgvykZrP2vjOsMS4poDdCwCypXQ8GiyPV7VgjzxvicflUgDRpn/wtslUNyjNjY8cyY/hYbCy/H4Shn3frzSUw8BoCEhKN6rBgiPgfKIdlpZZekR5I0jBv3IGmpi7rffgCQJJn0jLMpK3uS6up3hoTANH7xSzYl7kGVRXf4ydtaMbjFb1VCkwOLmk270sTefa8xPO+G/T7eUGBQBObGG2/kww8/5Ouvv46Ybaenp+N2u2lpaYmIwtTW1gZn/unp6axZsyZivECVUvg2nSuXamtrsVqt3UZfAAwGA4ZBClYlSRp0KG+ooKpq0GVXEx+/X40ItQnxosmjLIOmZ3Ouyy67jMbGRh577DFyc3MxGAzMmTMnqEU6EMjOzqaoqIjPP/+czz77jOuvv54HH3yQFStW4Ha7WbBgAQsWLODVV18lJSWFiooKFixYgMfrDbZMANDn5qKxDr7Dcnt7O3feeWeEwDwAo9FIWVkZp59+Otdddx333HMPiYmJfPvtt1x11VW43e4ggQl8Hj2eVlzuWhSfk2uuuZmmJhv/+Mcj5OXld72uPiHe1el0oA19niVJChK2nj7nPl87HR1ilvyvB//MUcfOD/nDABqNJtJfphNJ0Ov1jBwpKm6mT5/O2rVreeyxx3jmmWcGdP0GA2NYBCY5eYCh/dw5PacmarcJp1mtEUb2My0FYpb+86WQOXVg53IkIm9eiMDMvCq4OCPjXCoqn8NgSMfg9Ou/Bht9CSBnjiAw5Su7EpjiZeJ51IIDG/XSm4eUIJlMOcyY/iaFm66irW0rtbVCr5aQ0Ev61S/kNdqamHXCUjQaCyZT1pCdU0a6IDBNTStxumowGgZR+u6H2ribXeZiVFlLimYk40dejWZKpjAa3PEB0pd3kduayLa4JvbufYmc7F8I/6ZDjAHFsFRV5cYbb+S9997jyy+/DIpDA5g+fTo6nY4vvvgiuKyoqIiKiopgnn3OnDls2bIlONME+Oyzz7BarcFZ4Jw5cyLGCGxzsHL1hwJKe7swapMktKn9N5GyWq1kZmYGtR0Akk7H6u3bGT9tGpIkBTUvPl+k+dnKlSv59a9/zamnnsr48eMxGAw0NDT0+9gjRoxAp9OxevXq4LLm5mZ27drVy17i5nzGGWfw+OOPs3z5cr7//nu2bNnCzp07aWxs5L777mPevHmMGTMm4nOiy8xE60+HrQkTxQaOOXZsqATR6/Wybl2ou3FRUREtLS3BbaZNm0ZRUREjR47s8pBlmfXr16MoCg8//DCzZ89m9OjRVFVF9jQJCLDt9hIcjnIUfxnz6tWFXHvtz1iw4Pjur2uAYMi6YDfxzpg0aRLffPMNnrCqK0XxoChuUlOTyMxIZ0/5PkYOS4k49+HDh4f5y/Rd3aQoSoR27EBCq40lLfV0EuJnk5BwVN87hCMgDq3bLlIT4Qikj0acOLCu27JGtCrorUfODwXhfZHCCgdMpiyOmvUx06a9jtTh/67tL4EJiq47kU1VDRGY0Qv27xiHAHp9MtOmvkZiYkhj1a3+JYCAe3ZLBRZLwZCSF/B7wsTNBBRqqpfu11gNG/6O3axFq8iMO/odNOPPFXqmlNGijxSQWlmHwZCB293gL/8+9BhQBOaGG27gtdde4/333yc2NjaoWYmLi8NkMhEXF8dVV13FzTffTGJiIlarlV/96lfMmTOH2bNFLf8pp5zCuHHj+PnPf84DDzxATU0Nf/7zn7nhhhuCEZRrr72WJ598kt///vdceeWVfPnll7z55pt89NFHQ/z2Dw+oqorXfy21SUnIAzSeu/XWW/nrX//KiBEjmDJlCkuWLKFw0yZefe01AFJTUzGZTHzyyScMGzYMo9FIXFwco0aNClb9tLa2cuutt/Y48+8OFouFq666iltvvTUoYP3Tn/6E3Esp8wsvvIDP5+Ooo47CbDbzyiuvYDKZyM3NRVEU9Ho9TzzxBNdeey1bt27lrrtCKRJZr0djEQZlf/vb30hKSiItLY0//elPJCcnc9ZZZwW31el0/OpXv+Lxxx9Hq9Vy4403Mnv27KDe5vbbb+f0008nJyeHc889F1mW2bRpE1u3buXuu+9m5MiReDwennjiCc444wxWrlzJP//5z+D4qqr6S5cVfD4HkiSj0yWh1yczYkQ+b7zxIbNmHYPLZeh6XYMRkp5nMDfeeCNPPPEEF154IbfddhtxcXF8++0XTJqUyZgx47jz9j/z69/+jjhrLAvPvxKX2826detobm7m5p+f5h8/Mqp42223sWjRInJycmhra+O1115j+fLlB9WeYMKExwa3oyUFkkaJ8tvOqYlA+qi/1Uc/RgQMATvqhRg6MWTcZzL5U+2BCqTBlFCHI9c/0awuFJqXAKms3Sa8VLQmISw+AqHVxjB50rOU7HkECSnoFdMtwgjMgNBSAd89CSNP6pPoZWScTYttLdU175Cb+8vBmXh6XZTbV0AsZMWe0MX9lxQx6ZOby8jJuo/iPQ9QUfEcmRnn7ZeOZ0igDgAIK8QujyVLlgS3cTgc6vXXX68mJCSoZrNZ/elPf6pWV1dHjFNWVqYuWrRINZlManJysnrLLbeoHo8nYpuvvvpKnTJliqrX69X8/PyIY/QHNptNBVSbzdZlncPhULdv3646HI4BjXmg4GlqUu1btqiO7dtVpdN16A5LlixR4+Lign/7fD71jjvuULOyslSdTqdOnjxZ/d///hexz7/+9S81OztblWVZPe6441RVVdUNGzaoM2bMUI1Gozpq1Cj1rbfeUnNzc9VHH300uB+gvvfeez2eS1tbm3rJJZeoZrNZTUtLUx944AH1uOOOU3/zm98Etwkf87333lOPOuoo1Wq1qjExMers2bPVzz//PLjta6+9publ5akGg0GdM2eO+sEHH6iAunHjRlVVxecCUP/73/+q48ePV/V6vTpr1ix106ZNXa7PO++8o+bn56sGg0GdP3++Wl5eHnHun3zyiTp37lzVZDKpVqtVnTVrlvrss88G1z/yyCNqRkaGajKZ1AULFqgvvfSSCqjNzc2q12tXn376LjUuLlZ1OKpUn88d3G/16uXq1KnjVaPR0P11rS8W1/X1FyPOJy4uLuJzvmnTJvWUU05RzWazGhsbqx599Cy1sPBj1eGoUlVFUV998u/qlPEFql6vVxMSEtRjjz1Wffedt1V130ZV3bdBVT3OiPGvvPJKNTc3V9Xr9WpKSop60kknqcuWLevxf6uqh9l35f0bVfWvVlX99M+hZY17xLI7ElS1o/HQnduRgGeOF9dq81vdr1/2F7H+f/+3/8d6ZLwYq+Sr0LKvHxLLXjlv/8c/EtBUJt7v31JU1efre3ufT1XX/EtV78kU+92VJj7fvcDjaVW//Gqc+vkX+WpLy8ZBnWbz+gfVz7/IV7/4LF91duztuoGiqOp9uar6V6vqqVylLl8xWf38i3y1rv7zrtsOEXq7f4dDUtUDbOV3iNDa2kpcXBw2mw1rJ62E0+mktLSU4cOHYzQOXfnZYKAqCq5dxaheD7r0dLTJyX3v9CPG8uXLOeGEE2hubu5S7RbACy+8wE033URLS8sBOw+PpwWHoxKNxkxMTKSHiGhWKByqY2PHIUlhOiRVhZotoPoguWBAYsn29iIUxY3JnIdOGyucZ12tonFhIOzvtkNDkRDwpk/cb53BUH9XtrbZ6fApHBUfOcvbWNGMLElMzo7veedNb8B7v4SsGXC1P8X83ROw7M9C43H5h/t9fj9ofHSLsMSfc6NomNkZ7/4SNr8B8++EY27av2O9czVseROO+wOc8Eex7N8LhLncaQ+H+hb9kOHzwt2p4rt+S1HvDTKb9sAHvw6ZDWpNoiJsxIlwybu9fo+3bbuFmtqlxMSMZtTI20hMnDegSMzmDydTb24ng9GMO7EHZ+7nF4kKwLP/xW5zGeUVzxAfN5Pp09/o93EGgt7u3+GI9kI6xPA2NqJ6PUg6XUSH6SgObyiKv9N3N+6ZsqxHlkUa0Ovt1L/J5xI/aEigM6IoHjo6SnC7G/s4nhtFESJgbUBwbvBXqbnaQhuGN3A8zEqDfarKuYUlnLlxNx/WtQSXtzk9XPSvVZy9+Du+Le5Fg5XTKTUBsCNQfTQEZaQ/dAQaGu7rwaxyf03swhHoIh4wtLM3wV5/8caoI0//Mij4vWCAntNIig9WLYbFRwvyojPDwvvh2m9E6XvJl7Dl7V4Pk5N7NRqNhY6OXRRuuoING39GS8u6XvcJoGPvF9SbxO9H7rg/97xh6hjxXLeD7OzLkCQdLba12GyF/TrOgUKUwBxCqF4vvvp6ALRpaYfUBj+KgSFAJgJEpTM0GpH3D5iEBRE0mDOBJOP2NOLz2XG5alHVntsQeL3t/nHNoYhOkMB0QGDfgXS4PsjY63TT4hVC8l/tKKewVZzrnvoOnB4Fn6Jy3avr2V3Xfc8r4nPEDUHxwt510F4n9DAAY047GG/hyEagp1T1pm67mQ+ZBgZCBGbvWvC6xY1YVSB1HMT3z97iB4HedDBuO7z4E/jk/8T3Nm8eXLcSZl8runQfd6vY7pP/6ypcD0OsZQxz53xBdvaVyLKelpY1rN9wAYWbrqS1bWuvp1ex4z6QJJJd8cSk99KCI8VPYOp3YjCkkZ72E7F/xXO9jn+gEb1jHkJ46+tRFQXZaNyvsukfE44//nhUVe0xfQRw+eWXH9D0EfQegQHQaHsgMGEdqFVVxesRbtaq6sPrbaMnBMYJECNAlA3LWkAJRSTCeyAdZiixh6qdHIrKZVv2UOV0U9oQukZtTi9XvbiW5o5uSvklKXJmv/MjQBVl0HE9mydG4UfyKNGF29MB9UVd1wcjMIMvxw0da7Qo7/c6RcQs6L578v6PfSQhSGDKu65b9zyUfyv+J6c9Apd+ECGuZu5vhIDW3gCf/aXXw+j1yYwe9SfmzP6SzMwLkSQNjY0rWLv2TIqL/97t5MjVUUmNJBqg5mb3kdJLCUVgAHJyRCl+Xf2n2O3dvLeDhCiBOURQ3O6gaZ02PX1w6vEoDglUVe2TwGg1QuPh8zlQ1bDZbliERFGcwUgOCF1NT8fz+glMRIWAJIE+EIVpF/oaT8CB9/CLwOxxiGt2bIKFMTFGat1eLttSSlGDiLjMH5tGdqKJ8kY7v3xlPW5vNxGpYGPH76LVRwOFrIGMKeL1vvWR67xusPvTmEORQpKk0P+q9OtQZ+gfS/oogJ4iMB4HrPRX5C28V3jzdI7Aa/Vwhn+bja9E9LLqCUZjBmPH3MPso5aRniYaaFZU/pstW2/E53NEbLt3019RZAlrh0zc2Kt7HzjVb1HRXAYeBxZLAUlJxwEKdXWHrjo4SmAOEby1taCqyBZLsDQ4iiMDquoNzmh6SiHJsq6rDkZVI5osBgiL7O9j4vW2oSjezkOhKG5UxQN0Y7ho8H92XG3+9gGK6BukPfQmU50RiMBMijXz0sThJOm0bGl38KbXjgrMyEvg35fNJNagZU1pE396b0vXZqeBCEzlWuHqClH9y0CQ5Tftq+qkg+kQqWxkHZiGyCY+4Aez9jlwNInGjdkD9P850tETgdnwkujdFZcDky7sef+co2CGv73KhzeFJih9wGzOY/z4Rxg/7lEkSU99/ads2HgxLn/Xeq+3g70dghDlWk9G0vThqBKTAqZEQBUmhUB+/s1MnfIyubnX9eucDgSiBOYQQPV68fmbTerShmC2E8VBRUj/ouvVB6GLDsbrFDoASUbVGvF6RfrIoE/1kxg1uCwcofSRqevxAjoYT4eIwkC3DryHA/b4CUy+2UCOycALE4djkCX2GSW8o6wMT45hdFosT148DVmCt9bv5Zmv90QOklwgfki9DlA8whvmQFrS/9DQk5C3XfhQYUnt0VxxwAj4wbRVi+cRJwlh648J3REYjxO+fVS8Puamvnt3nfRXERVr3A3fPDygw6en/4SpU19Cq42ntXUT69adS0dHCVVFj+HVKJgcPlKm9iLeDUCSQlGYup0AWGMnkJg495BmD6IE5hDA12IT0RejCXkAxnFRHB7oK30UQCDd4/X5iUVY+sjnc6AoHiRJRquNRaeNF5t0k0by+ffvYjAFItIS6HcUmEUfhvoXgBKHmD2OMInrNjMuhkcKhKDTlx/LDq1ItR03OoW/njEegPs/2cknW2tCg8hyKDUBA+t9FAVk+QlM7Tbwhjkwtw1hBVIAaRNDKU4YUPPGHwyCBKYS/G1CKHxFkLrYTJh6Sd9jmOJh0QPi9bePBglEf5EQP5OZM97GZMrB6axk3fpzKa95BYAc32gka2YfI/gRFPLu6H27g4gogTkE8LU0A6BJiD+0JxLFoNBfAhOIwCg+p0gNuUMEJhBp0Wpj/S6+8QD4fHZ8vtCNRVXVYAoqQsAbjkAUJrDfYah/cfoU9jlFW4R8c+i6zYsxo9ktopFP1Dbi9v/IXzY3j0vn5KKq8Jf3t0amkgJpJIAx0fTRgBCfKyJYigdqwipUhrKEOgCNVjSRBED68Ql4QVTNSRrx3eyoE1qjb/8h1h1zU/9TvePOhNELxf9tWT8iJp1gNg9nxvS3ibNOxettxS250LkVMsbd2v9BOkVgDgdECcxBhuJ0ojidIEnRyqMjFH2VUAcgdDDiB8rn6whGYFSdCU+QwMQHtw1GbLwtYcdyoape8XnpqeGoITby78OQwJQ5XaiAVSuTrAulEcoa7GhL2pB8Kh1hJAfg/xaJGV99mwubI7ScEScAEiTk/TgaMQ4lJCkUhQnXwQxlCXU4AmQzazrE/AhNOjt7wWx6XXTJtqTBtEv7P44kwYK/i9e7PxPGdwOEXp/E1KmvkKoVRGR4YwyaEQNofhpI1UYjMD9e+JpbANDExiJpD4988PLly5EkKVh6/MILL/Rapnwk4/jjj+emm27arzH6G4GBsHJqb0dQwOvTyKjB9FEoLaT1R2E8npZgxCGkfzH3rLfRh1cmHZ4C3qD+xWSMyJmXNrQjAWafeL+VzlBVllmvJdkiSOLe5rAKirTxcMXHopN01Dtp4Aj4wezrhsAMZQQGRHXN1EtEpc2PFYE0UmNJSMNy9G8GnupNGiF0RADrlgzqVDSSnombajj2u0ay868bmFYuJVCJVB6KJh9iRL/9BxGqquKztQCgGQRBuPzyy5Ekqctj4cKFQ3qeF1xwQZ8dpQ80DlcSJUqoAxGYvomC1p/28XpbARUkDV5VfPm1WmsEKdH5/1YUNz6f3b+fv3xa00ulmkYnrMdBPO+HqE5V1S5dy4cCgQqkEebIa7bH7wGT5DfnCycwAMMSRDRpb3OnH8zcuZA4fMjP80eBzG4iMAdCAwOiounMp8JSST9CBAjMyseEH4w5GaZfMbixZgr/FTa+0u+KpAjs+h80FqPTWAcWAQLRUNWcRHgl0qFGlMAcRCjt7aheL5JGgzzI0umFCxdSXV0d8Xj99deH9DxNJhOpqalDOuYPBarqRhARCUnS9bl9UAejelAA1ZyAxxNIH0WmECVJg1Yr+n54vC1+MtEeMU6PMPr7hRj62K4P2Gw2amtrcblcfW88AAQ8YPJNkQSmtF4QmEyDuJYVXQiMIGaVTZEeFlHsBwIppPqiUBuKYAppCEzsoohEgMAEUi9zfzWgHmgRGLVApKQcTbD9/YHvH/CemXll19RzfxCIwtQfHjqYKIFBzDo7fL4D/mhrbMauqriscdj9xxxoL02DwUB6enrEIyEh5NsgSRLPPfccP/3pTzGbzYwaNYoPPvggYoyPP/6Y0aNHYzKZOOGEEygrK4tY3zn6cccddzBlyhRefvll8vLyiIuL48ILL6StLeQc29bWxsUXX0xMTAwZGRk8+uijfaZrNm3axAknnEBsbCxWq5Xp06ezbt06li9fzhVXXIHNZgtGme644w4AmpubufTSS0lISMBsNrNo0SKKi4sjxl25ciXHH388ZrOZhIQEFixYQHNzc7fn8NFHHxEXF8err74KiHTarFmziImJIT4+nqOPPpry8pDTZDD6Iun7VT4ou+3Iij8dZLbgM8ejql4/WelKYnU68b/0emwoit8ET5LRaPoIN1vSxQ/lfrioqqqKwyGIgtM5iNldL9jTQwQm4MI7wiK8cPodgYli8LCkgnUYoEJVoVh2oFJIUYQIDIiIVCCKMhhotDD9cvF63fMD27dilWi9odHDUdcO7vipkY68hxqHhwjjEMOuKIz4esvBO2BDHeyqA6Dk2InEaDR97DAw3HnnnTzwwAM8+OCDPPHEE1x88cWUl5eTmJhIZWUlZ599NjfccAPXXHMN69at45ZbbulzzJKSEpYuXcqHH35Ic3Mz559/Pvfddx/33CO62t58882sXLmSDz74gLS0NG6//XY2bNjAlClTehzz4osvZurUqSxevBiNRkNhYSE6nY65c+fyj3/8g9tvv52iImF5bvFHrC6//HKKi4v54IMPsFqt/OEPf+DUU09l+/bt6HQ6CgsLOemkk7jyyit57LHH0Gq1fPXVV92mRV577TWuvfZaXnvtNU4//XS8Xi9nnXUWV199Na+//jput5s1a9ZEEJWB6F9w2qCpFK1Bwi1L+AxmVK+ouOmcPgpAo4lBkrSoqhenU5QPa3vTvwQgy/7w7uDh8XiChNrpdBI3hCLzkjAPmAB8ikp5kyAmExNioLWVCkdk5Cc7URC3CA1MFPuPrKnQulekkfKOiRKYA4lwAjPnhsFFPsIx7VJYcb/o7F27TWjC+oOVj4vnSRcMPtIW1hPpcECUwBxh+PDDD4M38wD++Mc/8sc//jH49+WXX85FF10EwN///ncef/xx1qxZw8KFC1m8eDEjRozg4YeFmKygoIAtW7Zw//3393pcRVF44YUXiI0VX76f//znfPHFF9xzzz20tbXx4osv8tprr3HSSUJktmTJEjIze/cXqKio4NZbb2XMGPGlGDVqVHBdXFwckiSRnh76ogWIy8qVK5k7V1Q3vPrqq2RnZ7N06VLOO+88HnjgAWbMmMHTTz8d3G/8+K5f8Keeeoo//elP/Pe//+W4444DRAt3m83G6aefzogRIwAYO3Zsp+vQT/2Ln7yAikaOAZx4fR2iogiCZdOdIUkSOl08bndDmID34Dg1h0ddfD4fXm9XV+DBwObx0uARY4WnkKpaHLi9CnqNzKQkC5T3HIGpjEZghhaZ02DHf4WQ19EMPv91t0RTx0OOlDGis7Q+BmZds//jxaaL5qXb34e1/4bTH+l7n/pdUOS3/J/768EfO1hKHY3AHDYwyzIlx048oMdw7ylFcTrQpaWhSQrNlM0DrKI44YQTWLx4ccSyxMTEiL8nTZoUfB0TE4PVaqWuTkR8duzYwVFHRdp5z5kzh76Ql5cXJC8AGRkZwTH37NmDx+Nh1qyQUC8uLo6Cgt4dUm+++WZ+8Ytf8PLLLzN//nzOO++8IHHoDjt27ECr1Uacf1JSEgUFBezYIb5QhYWFnHfeeb0e9+2336auro6VK1cyc+bM4PLExEQuv/xyFixYwMknn8z8+fM5//zzycjICG4TisD0UkIdRl4wxqOJGwbtO4P7SpKmV02LTpeA22/5DaFKpgONzroXl8uFZgiig3sc4uaYptdi0YbGC6SPcpLM5PqJTa3bi9OnYNSI70V2QigCo6pqtGfYUCG8lDoQfTElHJYVbEc8LCnwi8/AYBXtFIYCM64UBGbzf+DkO/uO6nz/hHguOA1SRg/+uIEITEu5aCCrPzi/TT0hqoFBzHpjNJoD9jB5vRhdTsySTGxCQsS6gf4gx8TEMHLkyIhHZwKj00WKSyVJQgm4QA4SB2LMO+64g23btnHaaafx5ZdfMm7cON577739GtPUD2fjqVOnkpKSwvPPP99Fg7RkyRK+//575s6dy3/+8x9Gjx7NqlWrguv7TCF5XRHkhYRc4Qfj73cEoNXF9fp/12iMwe0lSUYjH3hnXUVR8HiE10rgGg6VkHePXUR28nvQvwxPjiFRpyHGT1r2ukJRmMx4cS52t4+m7jpURzE4BJo6tlSINARE00cHEhmTh7ZqbvhxkDQS3O2w+c3et22rgU1viNdH70f0BYSXj9nv59NdR/ODjCiBOQgIeb9YkHR9V64cSIwdO5Y1a9ZELAu/QQ8G+fn56HQ61q5dG1xms9n6VYo9evRofvvb37Js2TLOPvtsliwR/gZ6vb6LbmXs2LF4vV5Wr14dXNbY2EhRURHjxo0DRPTpiy++6PWYI0aM4KuvvuL999/nV7/6VZf1U6dO5bbbbuO7775jwoQJvPbaawCoqoKiiJt8jwSmox5QhTdLQq7wZSFUTg2g0/Y9CwuIeTUay0GJOgTIilarxWwWaRu32z1gkXl3KPHrWkaYjBHLAwQmPzkGSZLINoqoVqUjRFSMOg2pseJaR3UwQwhTvOgjBbDrU/EcJTBHDiQp1ORx3fOiUWxPWP1PkSLMPgpyZu//sQNppMNABxMlMAcY++v90hkul4uampqIR0NDQ987+nHttddSXFzMrbfeSlFREa+99hovvPDCfp1TbGwsl112GbfeeitfffUV27Zt46qrrkKW5R5vvg6HgxtvvJHly5dTXl7OypUrWbt2bVBzkpeXR3t7O1988QUNDQ3Y7XZGjRrFmWeeydVXX823337Lpk2buOSSS8jKyuLMM0Xr+Ntuu421a9dy/fXXs3nzZnbu3MnixYu7XKPRo0fz1Vdf8c477wQrpUpLS7ntttv4/vvvKS8vZ9myZRQXFwfPKaB/kSQNktRNakXxgb1JvLakBckLhHQskqTtuyQa0OuSMBqHYTT2s0/JfiJAYAwGA3q9PhhhC0Rl9gd7uhHwQmQEBiDHT2A6l1JnJwYqkaIEZkgRSCMVLxPPUQJzZGHyRaA1Qu1WqFzT/TauNljrr1Y6+jdDc9zDSMgbJTAHGEpHB6rHI7xfYvdTfQ588sknZGRkRDyOOeaYfu+fk5PDO++8w9KlS5k8eTL//Oc/+fvf/77f5/XII48wZ84cTj/9dObPn8/RRx/N2LFjMRqN3W6v0WhobGzk0ksvZfTo0Zx//vksWrSIO++8E4C5c+dy7bXXcsEFF5CSksIDD4hmZkuWLGH69OmcfvrpzJkzB1VV+fjjj4MprtGjR7Ns2TI2bdrErFmzmDNnDu+//z7ablyPCwoK+PLLL3n99de55ZZbMJvN7Ny5k3POOYfRo0dzzTXXcMMNN/DLX/4SiNS/dEvM7E2g+oRgr1NOWquNxWBIx2TK6VdERZIk9PoEZPnAR+xUVQ0KeA0Gg//YgkwMCYEJRGD6IDDBCExPXjBRIe/QImBo52wRz0PdRiCKAwtzIkw4R7zuqaR6/Yvgsolo2+hFQ3PcYCn1oScwkjoUMeLDEK2trcTFxWGz2bBarRHrnE4npaWlDB8+vMcb7FDBXVmJz2ZDk5iIvo+qnB8SOjo6yMrK4uGHH+aqq/bD9+AwgstVh8tVi04Xj8mUHblSVYUy3+eCuGEQk3JoTnIQ8Hg81NeLTtbp6enIskx7ezutra3BFhOD/a6oqsqob7bQ7lP4etYYRseIMVxeH2P/8gmKCmv+dBKpsUb+WVHHHSVVnJkazzPj84JjPPRpEU9+tZtLZudw91kHVmz/o0LlGvh3WIPFU+6BuTceuvOJYuDYux6eO1FMmm7ZKUhNAB4HPDEdWvfBGY/D9MuG5phlK+GFU0V5+E0Hxn6kt/t3OKJVSAcQituNr9Xv+3EY2uIPJTZu3MjOnTuZNWsWNpuNv/3tbwDB1M4PAb0KeF1tgrxIGtHt9whCIH2k1+uR/VVxBoN4j+HeMINBvdtLu09BBnJNocqtyiY7igoxeg0pFnGsHP/6Ckf3EZhoCmmIkT4RZC0o/nL5aArpyEPWNCEQrt4E/5wn0tbuNlEhFCyNTxPeL0OFgAampQJc7WA4ODYP3SGaQjqA8DU2gqoix8Qgmw+/DsFDjYceeojJkyczf/58Ojo6+Oabb0hO/uF0oO21C3WHKCnHnAjy0BoTHmgECEx4hEWr1QbJzP74wQQEvNlGPYYwy4A9/hYCw1Nigim1nlNIfi+YpmgKaUihM4VuRhBNIR2JkCSYfb143boXbBWRvj6yFk74I+iGMNNgTgxFmBsObSVSNAJzgKB6vXj99vXa5CMnnTBYTJ06lfXr1x/q0zgw8PeL6TEC43GGesocQakjEOXT4QLeACRJwmg04na794vA9CTgLWsM6F9Cs7cAgWnweOnw+YIO1eFuvFEvmCFG5jSo8acBohGYIxOTLoD4XBEB1scKbxaDRTzrLaLZ61AjZYyouKzbGepufggQJTAHCN7GRlAUZKMR2XJozX6i2A/4PNBYgoKKahFfly4RmA6hH8EQd8QZgbndgaiS3EXoHCA0+xWB6aMHUkDACxCv02LVyrR6FfY6PRTECAKTEWdCksDlVahvd5Eae2B1az8qZE2DDS+K11ECc2RCkiC3bzPSIUXqWCj7JtSg8hDhR51COlD6ZdXnw9ckymm1KSnRGeORDFcboKLI4n/YpYRa8YrOsCAcN48whEdfOn9O9Xq9vyO2j46OjkGNv8fhN7Hr1IU6kELKT44k992lkfRamQyrIC1RHcwQI2uGeNbFDJ1LbBQ/fARLqaMppIOOQMmt3W7vl3PrQOFrakb1+ZD0euReFNRRHAFwtwOg+NMZstcrcswmfwdwexOoivBj0B86Mdtg0V36KACNRoOiKPh8PiorK0lKGnizyFAEpnsTu+GdCEyO0cC2dicVDheKovD2229jtVoZlhBHlc3J3mYH03ISiGKIkDYeTviTqJyLTrSi6C+CPZEObSn1j5LAaDQa4uPjg718zGbzkEVJVEXBXV+Hqiho4+JgiOzYf0xwu1vweBoxGDLQag+h+FlVod0GiorLaMbj7kDrUXC2lYLVI7xeWmpB8bcNOML+1z6fL+j/Eu4FE/jbbrdjs9koLy/HbDb32lm82/FVlTJ/RVG4BqbD5aWuTVyrvF4iMLW1tWzfvh2AnOEnsIaokHfIIUlw3O8P9VlEcaQhEIGxHdpKpB8lgQGCXY4DJGaooHR04LPZQNag02rBn0qKov9wuWpRVS+SVI9enzp4cqkq4iEP8mPu80BbNSDhNttRFCdaVYfW5YR9DeJL62oTpYtWA0gtgzvOIYLb7cZut6PRaHpMEel0OoqLi7FYLAMW0O51uvGoKgZZIssQEhIGoi9JMXriTJECw3ACY/OGCGGCoxJIiKaQoojicIA5UWim2mtFGmnYoRHy/mgJjCRJZGRkkJqaOiRuoyC0LxW//CXsqyLpmquJnzMEfSd+ZOjoKGXzlr8E/87NvY7MjLMHNoiqiv4uK+4XZk5nPTU4pfyWt2HlfZA5nU0jwW7fw5jRfyNh3Wew65PQdlMvgynzBj7+Icann35KcXEx06dPj+hgHoBOp0NRFLRaLe3t7dTV1ZGW1n+hZyB9NNxkQA4jPj2ljyDMC8bpxuazBZf76svRYmVv1I03iigOD6QU+AnMjiiBOVTQaDRoNEPj29H68cco69aji48n9cwzkQ+wy+8PEVVVy1CUKrTaOLxeG5WVDzIsayF6fT/N4exN8NHNsC2sq/XHv4HrVg68QmjPp9BeiZp5KXb7iyiKk7j44RhPuw+c9bB9qTCum34hHGH/a0VRKCoqwm639+qyq9FoyM3NpaSkhD179gyIwPS3hUA4AhGYvU43LR0tofP1eRihaWRvc1RTFkUUhwVmXQMTzoW8/reyGWoMuArp66+/5owzziAzMxNJkli6dGnE+ssvvxxJkiIeCxcujNimqamJiy++GKvVSnx8PFdddRXt7e0R22zevJl58+ZhNBrJzs4O9sI51Gh8fgkVV15F/RNP0v7tSnxtwv9DVVUanv0XAAk/v+RHYVx3IFBXLyIbo0b+EYtlLF5vK3tK/9G/nYs/h6fnCPIiaWDe7yAmFRqLYeVjAzsRxSfKBAFXzkQUxYkkaTEah4FGC+c8B8f9AX7yBMRn9zHY4Yeamhrsdjt6vZ7s7N7Pf8SIEQCUlJQM6BiBCEznCqQggUnpmcA0eXzU+l2sExMFeS3Q1LGv2Y6i/CC7n0QRxZGFsWeI9gQJeYfsFAYcgeno6GDy5MlceeWVnH1296H9hQsXsmTJkuDfnSscLr74Yqqrq/nss8/weDxcccUVXHPNNbz22muA6INwyimnMH/+fP75z3+yZcsWrrzySuLj47nmmmsGespDio7vvgs+AJAkDKNGoc/NxbVzJ7LZTOLFFx/SczxSYbeX0tGxC0nSkpJyMiZTNhs2/ox9+14nK+tnxFrGdL+juwOW/QXW/Vv8nTwafvqM8LhIHQvvXAVfPwTjz4bkkT0ev67uE1zuerIyL0Su2SqqjQxW7FZxozWZcpADehqNTjhcHqHYvXs3AMOHD+8zApmfn09x6jA+NyRyisNJqikUrXl1dTlrS5u4+6cTsRgif05K++hC3bmEGiBWqyFBq6HZ66O8Q4iK582bx0cffUSi10G8p5W6NhfpcUdWxCuKKKIYegyYwCxatIhFi3rvamkwGIIi2c7YsWMHn3zyCWvXrmXGDOFB8MQTT3Dqqafy0EMPkZmZyauvvorb7eb5559Hr9czfvx4CgsLeeSRR3okMC6XK1gSCoIEHQik3nor9hNPwLGxEEdhIZ7KSly7duHatQuA+AsuQPMD6XvU6Pbyl937WJBs5czUUOlqbauTv324nbOnZnHS2KEzv6qrE9GXhIQ56HRxJCQcRWrKIurq/0fxrruYOvWV7kWk7/0SdvxXvD7qWph/h7BJB9GttfBVKPkSPvotXPpBt+WiHR0lbNl6I6BSW/M+ExyTMALkzcPurADAbM4fsvd6KOHz+di4cSMAo0aN6nP7tLQ01uaPp9Vg4qZ123ltnuhirKoq9/1vJ21OL0kWA385fVzEfoE2AiPCIjCqqrKnXkRbO1cgBZBt1NPc7mCf20u6//gTJkygsLCQAm09e5vtUQITRRRRHBgju+XLl5OamkpBQQHXXXcdjY2NwXXff/898fHxQfICMH/+fGRZZvXq1cFtjj32WPT6kOPpggULKCoqotlvz98Z9957L3FxccFHX2HxwcJYMJrEn/2MrAcfYORnyxj1zddkPfE4iVddSfx555J87S8PyHEPBe4s2ce7tc1ct62czxtDhPAvS7fy0eZqrntlA6v3NPYywsAQSB+lpiwILhs58v+QZT3NLauor1/Wdae2Wtj5kXh98duw6P4QeQFBVk57WPi0lH4Nm9/s9thl5YsBkZqwtW5kjfMVGhN0kH88dnspAGZz3n6/x8MBhYWFNDc3ExMT0614tzOavT5aDeKafumVeXubMK8qb7TT5hQuvS98V8b2qtBnxOlT2OsMlFCHyEaz3UOrf5+8pB4IjF/IW+83DIyPj2fmzJliH7mJsprufwOiiCKKHxeGnMAsXLiQl156iS+++IL777+fFStWsGjRInw+HyBy76mpqRH7aLVaEhMTqampCW7TWSwY+DuwTWfcdttt2Gy24KOysnKo31q30KakYD35ZNJuvZWMu+5CE/fDcLNca+vgTf+NQgGu2VbGljY7XxXVsWx7LQBun8I1L6+npL69l5H6B4djL21tWwGZlJSTg8tNpmHk5FwNQPHue/H5OnmtbHtPlEoPmwmjTqZbJOaHvC4+vU0IfSOOXUFt7QcATBj/GLExY/FofBROsFJirqDDLrQfZtPw/X6fhxper5evv/4agGOOOSZiktATtrRFli7/tayOvTU1bNkXViWkqPzl/a1BfUqZ04UKxGk1JOlCKarSBvFZyYo3YdR1n7oK6GDajGZ0Oh0mk4msrCx8xng0ksqeoq39f8NRRBHFDxZDTmAuvPBCfvKTnzBx4kTOOussPvzwQ9auXcvy5cuH+lARMBgMWK3WiEcUg4NXUblt114AzktP4NgEC3afwiWb9/Cn/wljsZ/PzmVqTjw2h4crlqyloX3/TNzq6z8FID5+Jnp9ZAfrvNxrMRjScTorqax8PnLHLW+J5wnn9n6AOb8S5kv2Rvjs9ohVZeX/RFV9JCUeS1ra6UxP+g1ZVQ6QJMrqX6epSYh5zeYjn8Bs2LABm81GbGxsRBS0N2xuE6XLx8bHEOPz0GiO5Xeff8uWUjGZmD82FbNew/ryZt5eLz43e8JKqMPTfsEu1D2kjyBEYFqNZuLj44P7xw4rAKB93y4URen3e44iiih+mDjgvZDy8/NJTk4OigbT09O7mMd5vV6ampqCupn09HRqa2sjtgn83ZO2Jop+wOOAF8/ocgPvjBerGtja7iBOq+GvI7J4bsJwCmKM1Lq9lObHkJJg5A+LxvCvS2eQk2imosnOL15ch9PjG/Sp1fkJTHj6KACNxsyIEbcCUFb+NHZ7uVjRVAr71gkjufE/7f0AWj2c/g/xeuPLUC5E2E5nFdXV7wKQl3eDOF7ZSsbs7mC8fSIaTaia7EjXwHg8Hr75RpCxefPmBVtq9IVNAQKTaOWvo4YB8G16HtXF36PFx8nj0vjt/NEA3Pu/HTR3uHts4hjqQt0zgckJRGAMZuLCIprDR43BrWqQ3R3s2bOnX+ceRRRR/HBxwAnM3r17aWxsJCMjA4A5c+bQ0tLC+vXrg9t8+eWXKIrCUUcdFdzm66+/jjCY++yzzygoKCAhIdoHZdCoXCN0IN8/Dd7uIyb1bg/3l1YD8H/5GSTrtVi1Gh7KyUBy+VBjdVjmpmPQaUi2GFhyxUziTDoKK1v47X8KB1Xi6nLVYbNtACAltSuBAUhP+wlx1qn4fHbWb7iAtrZtsPVtsXL4sRDbDzFx7hyYdpl4/d+bwOOkvOJZVNVDQvxs4uP9EYk9y8Uxsy9i5oylxMVNIzl5fpfI0JGGdevW0dbWhtVqZdq0af3eb7M/hTQ51swlOelMizHg1WhZn5vHsbo9jEuP5fKj8yhIi6XZ7uGBT3cGPWB6LKHuLQJjCqWQwglMbmocu31JwfcSRRRR/LgxYALT3t5OYWEhhYWFAJSWllJYWEhFRQXt7e3ceuutrFq1irKyMr744gvOPPNMRo4cyYIF4sY0duxYFi5cyNVXX82aNWtYuXIlN954IxdeeCGZmZkA/OxnP0Ov13PVVVexbds2/vOf//DYY49x8803D907/zGiyT9rVTxQ072O4J6Salq9ChMtJi7NDDXve3ZZMbr1jciKyg6vh9/vqkRVVUakWHj259PRa2T+t7WG+z7p2tzL57NTXbMUj6f7yjAhzlWxWqdiNHQfYZMkmYkTn8JiGYPbXc/6DRfRuEeU3TPxvP5fg/l3QEwKNBTh+uTXVFX9BwhFX7A3QVWheJ1/HDExI5gx/S0mT3rmiO4q7na7+fbbbwE47rjj0Gr7V4DY7PFS4RfjTow1IUsSD4/LQwb2pGRBip7d679mz+5ibj46iTjJwXtr9rCpSWhdOkdg+pVCMggC49bp0cXFB5cPSzBR5BP6uaKiImw2W3e7RxFFFD8SDJjArFu3jqlTpzJ16lQAbr75ZqZOncrtt9+ORqNh8+bN/OQnP2H06NFcddVVTJ8+nW+++SbCC+bVV19lzJgxnHTSSZx66qkcc8wxPPvss8H1cXFxLFu2jNLSUqZPn84tt9zC7bfffsg9YI54NIWF3fet77J6na2DN2qEwPXe0cPQ+G/YX+2s47Ptteg7vPw9NwMZeL26iWcq6wE4Kj+JB88T1SzPfr2Ht9ZFCqhL9jzK9u23sGHDhbjdDV2OG6w+6iH6EoDBkMb0aW+QED8bn6+DTcNsVKfHwJjT+/f+QfTwOPtZQKKi6b8oips461QSEuaI9WXfACqkjIXYH066cu3atXR0dBAfHz+gpowBAW+uUU+8TpCesRYTJxlFVdK3IyezfvMmXn/9dVZ8+h5jctswzraw3SVIT45eg83u4ZVV5Zz11Ep21gjjx94ITIxWQ4xPRF8dlpCWLc1qxC6bqVFiUVWVDRs29P8CRBFFFD84DNgH5vjjj0dVe04TfPrpp32OkZiYGDSt6wmTJk0K5uujGCKEE5iqyB9/nxoS7l6YnsiMOHGDcXp83PHfbQBcecxwLh+Zjteo4c/F+/hHeS2XZCZh0Wo4c0oWpQ0d/OPzYh75bBdnTslCr5Xx+VxUV78DQHtHERs2XsLUKS9jMKQA4HY30dKyBuhe/9IZWm0sU6Y8z/avTqVWLmP7aBOuuv+Qm3NN/yMkI07EfcLN7HW/AECedVFoX3/6iPzj+jfWEQCXyxURfRlI64yA/mVSbKSz9MgWH5/ho9UUw6Zpx+D2KWy3JOCRxdiSqjK8vool7zbwaakHt1eIbjWyxFlTsshN6t2p2upy0GHW0WoMER2NLJEZb2Jncwrp+jY2bNjAscceO2StQKKIIoojCwdcAxPFYYSm0tDrThGYl6oa2eIX7v5pREZw+bNf76G80U6a1cCvTxKmZ1dkJTPCZKDF6+PlqpAPzHXHjyA11kC1zcnSwn0A1Nd/gtdrw6BPw2BIp6OjmA0bf4bLJUTZDQ2fo6o+LJZxmEw5/XobsqRn/OZacirFzbWk5AF2Fd+Jqva/MqUyy4yikYht85D08aOh0uoggTm+32Md7li9ejUOh4OkpKR++b6EI6B/mRRrili+a18rup0tAKyNSWSTNRmPrGGU2cCpWiOnfPctp+xYy7aSfbi9CmPSY/nzaWNZddtJPHz+5F7JpqIomDtEpKZFG1nmPSzBRIWSgEZvpK2tLVgcEEUU/YHi9OKpjzYE/aEgSmB+LFCUiAiM2lBMRUsD79Y2c9uuvfy9pAqA3+el02pz8ea6Sv7w9mae+krcIP502rigVbxGkrghR2gRnqmsx+UvaTVoNfxinig1/ueKEhRFZV/VGwBkZl3EtKmvYTRkYrfvYf2Gi3A6q7o1r+sTlWuQbJWM2iczKv8PgMTevS9TuffFfu3u8dio3PcyAMObYpFaKuHda6C5TFwjSQO5R/f/fA5jOBwOvvO3vRho9AVCJdSTwyIwqqqydV8rcq2TE2NjSNRpuDQziY+njeLrWWP419EFJPtTTMfmmvjwV8fwv9/M4xfz8kmJ7buhZltbG7EOoZWpI5LoZCeYUZDRJucBBB2Ff8hw7mqmdXklqi9aOr6/aHxlB7WPrMdZHDVD/CHgR9+N+keD9hrwOngj/VQ+SzuBtaYR1G3cG7FJvEvhiSUbucvuiVh+fEEKZ0zKiFh2TnoCD5TWUOP28E5NMz/zC34vmpXDk1/uZk99B8s2r0HXsgaQycw4F6Mxg2nTXmfDxotxOMpZv+FnuFzCSyQ1NbLhZ68IeL+MOZ2cvGuQtEZ27bqTkpKHSE46EbM5t9fdK/e+hM/XTkzMaJIX3AvPL4Ddn8F/fi42GDYDjD8MH6E1a9bgdDpJSUlhwoQJA9q3xeOlPEzAG0BlkwObw4NBI/PClBHotZHzII0EZx1VwBdf7GVSipYJWQMzd7TZbMQ6BXGqdEZ+FocliPNoMmURw06KiooE4YmN7XVMb5MTTawOqQfzvMMVPpuLxpe3o3oUvA0OEs4ZdUSLyQ8lvM1OXLtbALB9XIrhV/FIcvRaHsmIRmB+LGjaww7zcG4q+AMfxc+izpCEDoWpsWbOiLWgK2zE8XUNLXYPBq3MrLxErjt+BO+e4uJf5w7v8qNpkGWuzRY6lqcq6vD5dVGxRh2XzskDYHOxiHIkJx2P0SgIkMk0jOnTXsdkysHprERVPZjNI4mJ6bnJYgR8XuG+CzBRmNcNy7qEhPjZKIqTHTtv6zWV1Na2jYoKIRjPy7seKXNKyB+mZrN4/oGkjxRFCdoVzJs3D1ke2Nc9IODNMepJ0IXmOgEH3oL02C7kJYBAB+mmpqZu1/eGlpaWMALjjlg3LEFEgiodWoYNG4aqqmzatKnX8dyVbdQ8uJamd4r7dXxVVamtrT0szPJsn5ahesR52NfV0vbVwXEY/yHCsSVUQOCp7sBeWNfL1lEcCYgSmB8LmvawOVaYjY2V2nl/443sqnuc/80YzVEOGU2tk+nD4njv+rlsuWMBb147hz8ML2Xa11ege+UsQRw64ZLMJOK1GkocLj6uD5W0Xn50HjE6H6NjhWV9ZtaFEfsZjZlMm/Z60Nk2Le20/r+P0uVgbwBzUpBoSJLM2LH3IssmWlpWs29f9wJxp7OaTZuuxuezk5hwDGmpp4oVUy6CGVeGNvyBEJiSkhJaW1sxmUyMHTt2wPuHBLyR+pcAgektshLwaxoMgQmPwFQ43RFFA9mJ4lwqmxxBL5uNGzf2Wljg3NUMqriBKa6un+Nw+Hw+3n33XRYvXsyHH3444HMfSrgr27BvEDfZmNliAtC6rBz7xuiNdzBwbBUERpsmSHDrsvIgOYziyESUwPxY0LSHnTGCMMyN1XNU6xZM+9YCUOQvbZ03KoWpOQmhWfX298Vz7VZYv6TLkBathiuyhLnbExW1wZtIssXAtbOriNV30OFNICmxa0WP0ZDO9GlvMm7sg+T6ex31C1tERRPjzgJNyEnWZMphpN+td3fJ/Tgckekxr7eDTZuvweWuJSZmFBMnPokkhaUTFt4HBadB/gmir9IPAIEy40mTJvXbdTccm9tDBnbh2OonMBN7ITCBCIzdbsfpdA7ouDabjViXIDAdPoVmb8jhORCBqWl1UjBmLDqdjsbGRioqKnocz1Pl79XlU3EVt/S4naIoLF26lC1btgDi+lVVVQ3o3IcKqqrS8qHQrJmnpZJw1kgsx2YB0PT2LpwlLT3u621y4m3Zv9YePzR4W1y4K9pAguRLx6GJ0+NrcdH+/aH5/0YxNIgSmB8LGkvYGSOs8MemZAn7/da90FZDUa0gMGPSw3QEig+Kw7o/f3VPsFLH53NRVfU2DsdefjEsBZMssbnNwTfNoaaOM1JE9OXL8llsq+ro9pT0+kQyMs5Go4mc4aOq4lidQ/geB+z4r3jdjXndsGE/Jz5uJj6fnZ07/xgkVKrqY9u2m2hv345Ol8TkSc+h1XbSTGgNcNFrcOnSCGJ0pKK9vZ2iItE1eiCuu+HY3E0JtaqqwQhMbwTGaDRiNov9euog3xNaWlrQKgoJkvj/haeRUiwGUZ6vqDQ51aCupzdPGHd16PPn2N599/Rw8iLLctBU89NPP+01unOg4Nhcj7u8FUknE7cgD4C4hcMxTUwGn0rjyzvw1IWqaVRFxbG9kfp/b6HmgbXUProeny1KYgIIpI/0uVa0SSasJ+cB0PplJUonzV8URw6iBOZAw9EMtdsO9VlAUyk7AgQmLh6SRWM839717PITmIJwArN3rWh8aIiD1HHifSy/F0Vxs2Xr9ezY+QfWrvspBk8pF/sFvE9UiNJou70ce/tqVFXim32z+eeKkv6do9cNm96Afx4DDwyHv2fC4mPgzcvgy7vFw90G1mGQfVSX3UOpJANNzSuDLru7iu+hofFLZNnA5EnPYjING8wVPKJQWFiIoihkZWV16ezeH9g8XsocXQW8e5uFgFenkRidbul1jMHqYAIOu5l+wW2FI0RgZFliWLw/jdRsD5Kzbdu2dRvpUZxefE2h5c6iJtRO7S4C5GXz5s1IksS5557LBRdcgFarpby8nB07dgzo/PcXituH7eMyAGKPz0YTJyq3JFki8fwC9LlWVKeXhue34qnpoHV5JTUPrKXxpe3BCJPq8tEa1csE4dgiTDfNE0XE2DwtFV26GdXppXV59DodqYgSmAONd6+BxXOh6JNDdw6qSnNrHTV+87iCGCNkTQegrWQNTo+CQSuTmxTmjrrLf76j5ov0CqCu/TfbNlxNY+NyADyeJjZuvITLkhxoJfimuZ2NrXaqqt8EwGiZQ6MziY+3VrOnPhSd6QKnDVY+Bo9Nhvd+KVJWAF4H1G6B7Uvh6wfh+yfF8onnQA+CVLN5OCPybwGgePe9lJQ8zF5/efW4cQ8TFzdlABfuyES4S+306dMHNcYWf/oo26gnsQcBr0Hbe0XPYHQwqqrS0tICQI6/j1JnIW+WvxJpb7ODYcOGkZKSgtfrZevWru0xAukjjVWPZNSidHhxV4RaWiiKwvvvvx8kL+eddx7jxo0jLi6OuXPnAqIPm9fbu3ZmIOjYWEfdM5txbG3oNrrT/s0+fDYXmngDsf60UQCSTibp0nFok4z4WlzU/mMDrZ+U4WtxIZu1WI4bRuIFYnLSsbYGb9PA0ndHIvqKkIWnj0x+AiPJEtaFIqXe/l0V3uYf/nX6ISJKYA4kvG7Ys0K8/vyvIi1zKNBex069mIVnG3RYtBrIEq0gvJWiKd6oNAua8JLCAOEavQjyj0Mdezo7R5qoa/0WSdIxftyjWCxjcbsbqNl+KWcmibTLE+XVVFeLJoujh1/C/LGpqKowxOsCZyt8+id4ZLzokN1WBZY0OOmvcGsJ/GoDXPQfOOVumH455B4jIi+zem8pkZ19ub/xYztl5U8DMCL/VtJSFw32Ch5RKC8vp6mpCb1ez/jx4wc1Rk8Gdv1JHwUQiMAMJIXkcDiCTVzzYwWhruhEYLITRWpqb5MdSZKCbU26SyO5/elL3bBYjGMEoXLsEIRKURQ++OADNm3aFIy8jBs3Lrjv0UcfjcViobm5mTVr1vR4zs7iZprf342vte+UjeLy0fL+btylNhpf2UHji9sjSIbP5qLNHxGIWzS827JvTYyO5CsmIMdo/e/NQsJ5o8m4bRbxi4ZjnpqKYVQ8+FRav+hZG/RDwHfffcfdd9/dqwYqIN7V51rRWEM+RMaCBAz5ceBVaV1WfsDPNYqhR5TAHEjUbAaf/0etfqdIjxwKNO0JpY8s/huSPwJjadwMqBSkhfmeNJdB/Q5h6DbyJFRVZfe4YVRlGEFVmRB3CenpP2HqlJewxBTgdtdxTMtfAPhfQytlbgN6fTLJySdy3fEjAHhnw15qbJ1mOf/9tYiquNtE76Ezn4abtsC8myEmGZJGQMFCmPsrOOMxuOIjuGoZxPWeApIkDWPH3o8sCxfXzIzzyc395f5dwyMIgRv5hAkTInqQDQTdGdhBSMDbH2+XwaSQAumjmJgY8mKMAFQ6OpdShyIwAJMnT0aWZaqqqqipqYnYNhCB0WfGYBorUp1Ovw5m8+bNFBYWIkkS55xzTheyZzAYOOmkkwBYsWIFHR2RWi7F6aXp7V00/HsrHd9X0/JB36lS+/paVKcP2awFjYRzZxO1j66n9asKVK+C7RNRNq3PtWKa1HP3c22yibSbppN20zTSbpxKzPS0CLJjPVl4Idk31B5RzrOK3UPbN3v7rUvZtm0bPp+vVw1UQP9imhB5PSVJIu5UEYWxF9bhruolShzFYYkogTmQqPTP2jT+m8hXfwfPIQhVNoUEvGP8NwVSx4NGj9HbSq5US0G4nmGXv59VzmwwJ1Ja9iQV9SKqMnZXO6lfvwkeJ3p9IlOnvkRMzChSPZuYKW1CRWIZi8jIOBdZ1jE9N5FZeYl4fCqvrQ6b5VRv8vu5SHDBK3D99zD1YiGmHQLExIxg8uTnGTnyNgoK7vzRmH85HA62b98ODF68C91HYPor4A1gMAQmkD6Ki4sjx9h9CilQiRQgMDExMYwZMwboGoXxBCIwmRaMBQkgS3jrHXgaHMFqo2OPPbZHk7/JkyeTnp6Oy+Vi+fLlweWOnU3UPrIe+7ra0LJtjXhquxesgxDatq8ULTasJ+eS9ptpGPLjUD0KrZ+WU/Po+mCJdPzp+X1+ZjWxenTp3TfFNORYMY5NBBVaPz/8ojAdHR189NFHNDRENne1LSvH9lEpzUv7bhGhKAr19ULbUlxc3K1vj8/mwl0uUoaB9FE49MNiMU1OARVs/yvtsj6KwxtRAnMgsddPYI7+DVizRNXP2ucO/nmElVAHIzBaPaSLvjiTpT0UpIdFYIr+J55HL6Sicgmlpf8Qf+b/gUx7ErSUB/Uoen0yU6e+gtk8gmOVjwHYwmQyM84PDnf+zGwAvi4O+7H68h7xPPFcGHsGHACCkZgwh9ycXwQjMT8GbNmyBa/XS2pqKllZWX3v0A1avT72OETkMLwCaW+zgxa7EPBGCL57QEAD09raGkwL9YVABCY+Pp5so/i/VTpdkV4wCSERbwABsrZ58+bgsVSvEqzU0WXGIBu1ImUANG/ex549Iq3ZW38oWZZZsEC0uVi3bh21FVU0vVlE4wvb8LW60SYZSfnlJEzjk0CFtuV7exzLubMJb6MTyaTFPD0NXaqZ5KsnknBBAbJFh69RTG7M01LRZ/d9fftCIArj2FyPp6ZnYnUosGrVKtauXcuyZaFKR1VRg+kex5aGXskgiM+K2y3IbUdHR7cl7/aw9JE2rvvJUdwpuaCRcBW34NjW0O02URyeiBKYA4lK4bPC8Hlw/G3i9TcPCdHqQYQaRmCCERjAmyG0A5PlklAJtasNykTn4qasDIqL7wYgf/hvyc67Bk6+U2z3zSPQKn4wDPpkpk19hammDiTVR42USYscaj0wZ4QI3W/ZZ6PN6RGRqeJPRYoqcF2i2G+Ei3enTZs26KjTFn/6aJhRFyHgDaSPRqf1LeAFERnR6wUJCURW+kJ4BCbLqEMCHIpKgyckog33ggl0uc7Pz8dqteJ0Otm5cycAnlo7KCqyWRus5DGOFVGh7Zu2oaoqaWlpJCUl9XpOw4cPZ8yYMaiqyodL3hPmchJYjski9TfTMAyPI/ZE0YjUXliHt9GBoiisWLGCZ555JhglaP9WRF8ss9KR9f6u3ZJEzNRU0m+ZgeXoTAyjE4hbNLxf16ov6DMtIg2lisjG4YR9+8S1KC0tDQqk3RWtKO1+oqvSZxVVXV2koV9xcVen5WD6qJvoSwDaJBOxxwiy37x0d7Ss+ghClMAcKLRWiYiLJEPmNJh8ESSPFuXIKx8/qKeyr7WZVm0sWlRGmEOzkBqLECxO0+4hNdBkr+RLUDyoicPZXf8qAJmZF5CXd4NYP/E8IaT1dMAXdwXHMhhSmTftecYaRB55ZUson5wVbyIvyYxPUVlb1gRf/E2smHqx0LlEMSSorq6mpqYGjUYz4K7T4QikjzrrXwaSPgJxcx5oGikQgYmLi8Mgy6QbhDg8XAeTbNFj1MmoKlS1iHOVZTko5g20TwjoX3SZliCZC+hgipvFDb2/Iuf5J56EjESlWs/OuFqSfzmJ+NPzg0REn+VPUalQ81kxL730El999RXV1dWsW7cOd1U7rj02kCVi5mZ2GV82aYk/YwQpV05AEzt0EUPr/FyQhO7HvbdtyMbdH6iqGoyWeDweysvF/8KxVWiTdMNEOtuxqb5X/U6AwARI8q5duyLW+1p7Tx+Fwzo/F22KCaXNEzQQjOLwR5TAHCgE9C9p48FgAY0WTrpdLFv1NLTV9rzvUEJV2ekWP94jDTL6sPLj7ZLoPzSOMqRAhZS/+qi+YBJtbVvQaGIYkX9LaDYvSbDgXvF68xvQEMpVGwypnJg+CoBvmyMFcYEoTNWGT6DsG9Do4djfD+17PQzQ0dHBK6+8wldffXXQjx2IvowdOzZoIjcYBA3sLN0TmIE0ZxwsgYmPjwcIppHCK5EkSeqigwGYOnUqkiRRVlZGbW1tUJSpywjpRLSJRrwpWqokURkVXnXUG/Sb7Yz3ilTot66tvPDJ68EUVACxJ+awV27klR0fUlZWFly+a9cu2r4RqSXTxOQeUxkHArpUM+aponP84RKFaW5ujvDsKS4uRlVD6SPrCdlB/U7blz1HYQIEJpA+rK6uprU1VCLv2NIAKuhzYvu85pJOJuHc0SCBfUMdjqKBt8CI4uAjSmAOFAIEZtis0LIxpwubeo8dvn7g4JxHRwM79OkAjLVG5tXXtyfRqpow4BJVR373XRXYYxaCtuzsK9DrO4XYh00X5dWqAivuj1h1TIKYPX3b3BahW5gzIhlQmVHylFgw/QqIzx6693kYwOPx8Prrr7N7926+/vpr2toO3ozX7XYHRan7I96FngW8/Wkh0Bn99YLxdXjwdXgiUkggGklCVyFvdzqY+Pj4YM+n1atXBwW8+sxIw719KW2okkqyIZ7k5N5n5gCeOjutX1Uy0zuC48bPwWAwUFNTw0svvcRrr71GfX09Pp+Pb3ev5RN9IU7JTZIxnquvvhqNRkNzczPVmwV5CKQqDiasJ+WALOHa1Yyr7OCmr7tDIPoSmBTt3r0bT1UHvhYXkk7GMCpBnDMiJedpcHQ7ToDADB8+PKj3Ck8j2fuRPgqHIdeKxR8da3m3GMU5dN4/URwYRAnMgUJAwJsdRmAkCebfIV6vfwEa++lQuz9o2hOqQLJEViwU1XWwWRHr2LdePOwN1GTG0+GpQquNIzfnF92Pe/z/ieetb0N9KHQ7My4GnSSxz+WJmDXPyU/iJHkDY3xFqFoTzLtl6N7jYQBFUXj33XfZu1fMtFVVZdu2g+fAvG7dOlwuFwkJCeTl5Q16nDavj5JuBLz7Whw02z1o5f4JeAPojxeMr8ND7SPr2PfwGux2QUg6R2B67ErdFBB1yC8AANhPSURBVJlimD17NiDEvG3V4pi6zMjP/W6X0F/kOZNRvb0381MVleZ3i8GnYi5I4vhzT+HXv/41s2bNQpZldu3axdNPP83ixYv55ptvABjjzeSM9qmkW1OC/4sK6tHnWodEnDtQaJNMxMwQPlC2T8pQfX23RvA2OKh9bAN1izfRvqp6SHUhAQIzYcIEJEmioaGBmnViwmQsSEDWa9APi8U4JhCF6VpF5fP5ghVMqampjB4tGtUGCIyv1d3v9FE4rAvy0CQZ8dnc2D6OViUd7ogSmCFCQ8NXbNn6K9zuJvC6RJkwdG0MmHcMjDwZFK/oL3SgEVGBZIxYVVTTxmbVr0HZtwGK/ociQelwMWPNzf1l155BAWROEc0PO0VhYjQaplnFzSU8jZQSo+OPRtGIsXTEJRA7cHv7wxmff/45O3bsQKPRMHHiREDcRPsDu92Oz9c/k8Nal4cHS6spc4RM09avXx+s5hg7aRpyDy7F/cEWf/Qly6AjSd+9gNfYjblaT+hPCql95T6UDi+tDhGx0uv1GI3is5pt6iECkxjpBRNcnp1NRkYGXq+XHb5K0Mpok0NEzG63U1YlbojD3Sm4SnuPSHSsq8Fd1oqkl4k/a6QQ3cbEcOqpp3L99ddTUFCAqqo0NDSg1+s555xzODFzFlqvTNs3exk1QqRpK+UGLMd01b4cLMSemANaGXdZK81v7+rSTiEcXpuL+ue24KnuwF3eSsvS3VT9fTWNr+7AsbOpXwSoNwQEvPn5+eTkiEjLrh2ib1e4V0t4FMbbGPl/bmpqwufzodPpiIuLCxKYkpISvF6vqCZSQZ8dizY+8nevOzQ0NFBcXIys15BwtkiDd6ypwbm7e+LtbXbiKm/FXdWOp8GBz+bCa/dw644KbiuqxBt2fb0+hZvfLOTvH+9A6eW6RzFwaPveJIq+4PPZ2b7j93g8TZhNeYzQzwOfG8xJkJjfdYf5f4Xdn8HWd+DYWyF17AE7N09TKcXm+UBkBZLN7qHa5mSTHEZgVIWqdCMOjRO9PoXsYT/vffDj/w+KPgp7H8KL4+gEC6ttHXzb3Bbsk8T2pYxQymhVTfxHfzY/pNqjNWvW8N133wFw5plnkp+fz9atW6mqqqKhoaHXNEV5eTkvvvgikydP5swzz+z1OCV2Jxdu2kOl083ODif/njCcdevW8eGHHwKw3ZuKs9HKKfvxXnoysBuogDeAAIFpaWnB5/Oh0USSH8XhpX2lmJG3S0IXEauLCaYXAimkii5mdv4ITHNkBEaSJGbPns17773Hdu1epqWMR9KEqrF27tyJqqokG+OJc5px7mjCOCqh23P3tYZm4dZT8tAmRN4Ik5OTueiiiygrK6OoqIgZM2aQlJSEQ9dE4wvb6FhVTeaceABqNDak/N57Rx1IaOMNJF1UQOOrO7BvrEPSysT/dCRSuPs24Gt30/DcFnwtLrRJRmJmpWPfWIenxo5jSwOOLQ3IFh3GMYkYhsdhGB6HJsHQ74o3RVGorq4GIDMzk/b2dsrLyynvqGGMJl1EXfzQZ8diGJ2Aa1czrV9Vknju6OC6QPooNTUVWZZJT08nNjaWtrY2SotLiF0pyHB/oi9er5cXX3yRtrY2rrnmGjJHZBIzO4OOVdU0v1NM2k3TkfQy3lo7jq0Nwu+numuJ98Z4DS8fJT6XPkXl/jHZSJLE18X1vLtBkDaNLPGHhWP6da2i6BvRCMwQYF/Vf/B4xAyzpmYpasVqsWLYrO79TdInCj0MwMZXDui57bE14Zb1xOBlmDFU3bCrTnzBayx+8lS7FV/DNkpzxBcwL+8GNJo+hKAZk4SHCyqsuC+4+Jh4EbVZ2dIudDA+rzDxA57znsaX5T+cMsVdu3bxv/8J35wTTzyRSZMmYbFYGDFCEMOALqUnrFixAkVR2LJlS9DTojtsbLVzxobiYCRiRVMbq9aGk5c01nhzWL6r+/46/UVhsAN1ZAuBzXv9At5hfRMYxS4qOZrfLSbWbEGj0aAoSlCgG47276pQXT60qWaUyeJzY27TBD1AAimkvS43SoQXTFcRbwDjx4/HrDNhl1yUmyN9PQJpvbEjxU3Esb2xx+vV8t8SVKcP3TBLUBvRHfLy8liwYEGwHNtYkIAuMwbVrSCvaCJeMaOiUlJ6EFLGvcA0PpnEC8aAJPoktXxQEvHeFX+DSG+9A59V5v2Y9XzZsoGUX08l9VdTsRydiRyjQ2n3YF9XS/Nbu6h5YC01966h8fWdtH9fhbcHvUoAjY2NuN1udDodycnJjBoloh1VchPaEbHIxsg5tXW+PwqzoS6i5UI4gQFBXANjbfl0Hd4GB5o4fTB11hu2bdsW1KsFxNdxi/LQxBvwNbtoWLKV2ofWib5Tn1cI8iKDJtGIHKtHMmpAlliWETr3l2qaeLpSlM9/UBjyp1m8vITXVh9+xoJHKqIEZj/h87moKP9X8G+nq4rmus/EH9kze9gLmHqJeN78H/AN/oZut5dit/dcXbDDIVITY3Q+5DAytbNGfGET0/PAkg6o7M004TbIGI3DyMq8oH8ncJxfC7NtKdQKB9jpcWaMskSd20txuwO+uhsai1FMiSxRFlJc105d25HfPK2qqoq33noLVVWZOnUq8+bNC64LlDFv3ry5xxtkdXV1sJLF6/Wye3f37qMrmto4p3A3TR4fkywmEnUa2n0KS779HoCt3jTUrEkYtBpqWp0U1w3OEt3m8fJpg9ANHJ0QSh1W2xx8VyJKXOfkJ3a7L/idZtdUU/PQOtq/3UfHmhravqgICnk762AUly/kTHtiNg7/ZNmiGGh6owjVq5Bp0CMDLkWlzh0SVQZSSPVtLpyeyPSbVqtlglmkTTfZioPX3263U1oqIiqTjp4GWhlfiwtvbddSXcf2RlHFIkPC2aO6RCp6gyRJxJ6QE/w7RxJNVDuX+R4KmCenkHCeqLbpWFWN7aNSVFVFcftoeGEbnqoO5BgdthPMVNfVsHHjRlauXIk+y0L8GSPI+OMskq4Yj+XYYehzYkGW8LW6cWyqp+X9EmoeWkfNw+to+XgPrj22LummgP4lPT0djUZDWloaMbIRn6TQkNaVwBtyrKKvk6LSFuYL05nAAME0UmlTJaqsknjRGGSzrtfroaoqq1evDv5dWSmOIRu0JJwjCJG7rBVvoxO0EsaxiSScO5qMP80m4/czyfzTUWTdMZf0u4/myxHiM3lirfg9v6ukireqGvlsu6g4XTRBFFP85f2tLC+K9LA51FB9Ch1ra2j6TxHt31fha+t5MnU4IUpg9hPVNe/gctdiMKSTkX42ADXeHWJleAVSZ4ycDzEp0FEPu7/ocTNVVdnR7uCJ8lpeq24MLlcUlU3lFXy76id8t/p0XK5uyrJVlSKfCHuPjYmcURfViBtVQUYcZE3Dq5EozxbbDB/+q/6716ZPgHFnEh6FMcgyM+OEcPLbzx+Hbx8FQD7hj+RkiC/x9yWN3Q53pCBQceTxeMjPz+f000+PCKOPGTMGnU5Hc3NzUNjbGYG0U0CzsmPHji7bvFfbzCWb92D3KcxLsPDu1JFMUMSPS0VCKlu96SSOmsHLv5jN7HwRAVhRVD+o9/SfmiYcisLYGCMzrKHo2+urK/ApKkcNT2RkaveaKPfeNuoWb6Ll3d0odi+aBFG22rZiL/Em4fLcWQfTsaoaxe5Fm2zCNCklGKGJ1ZnxVHfQ+nkFOlkiI+AFE6aDiTPpsBjEjLe7KExBexoaVabGVh+8/jt37kRRFNLS0kjJSMU4Kl6c11u7aHqziJYPSrAtK6Pt6720vC+iJZZ5w7pUMfUHpvFJaFPF92l0gegO3ZPd/cFGzLQ0En4qbs7t3+7D9kkZja/sEFofo4bkqyZQZQv9nnz55ZdB4idpZEwFicSfOpzU66eQeccckq+eiHV+jnA59rdqaP96H/XPbqbq7lU0vrEzGFELEJjMTBHR8rW4GOYWpLhS6f6mHtDCdKyvDUZ4uiMww0ypaFSZNtmJ75gEDHl9RwsrKysjHHwrKiqChNc4KoG4M/IxT0sl8WdjyPzLbJIvG0/MjDQ0MZHE6NuWNho9PhJ1Gp5ISObCcvFZvbmokjazhmEJJp762TTOnpaFT1G54dUNbKs69BVhqkehfVUVNQ+uo/mdYuwb62h5v4Tqv6+m/tnNtK86vMlMlMDsBxTFQ3n5PwHIzbmGTH/Uoi7eh0+jgaxeylk1Opjkj3IUvhqxyuFT+Lyxlf/btZcZ32/nhLVF3LOnmpt3VvLQhnL+753NzL73Cx58/1k02JFUO2u3PNj1GI5mdhjFD8WYhMhS6F01YpZekG6BrGlUZBnx6GTMhmzS084a2IU47v8ACba/DzVbAThGIyI8Kz1G0BrhzKdg1tXM9fvBrNpzZBOYuro62traMJlMnH/++V20HXq9PljS210aqaWlha1bxbUKWNXv2rUr6EoK8Nzeeq7bXo5HVTkzNZ5XJuWzd3cxmm2FYvuELEZMnsPTl0zHqNNw7Ggx0/+6eOAERlFVXtgn/idXZCUHyZjbq/DaGjEr/fmc3K772T00v1tM3VOFeCrbkAwa4k7PJ/13M4T/iAqmGnFDCCcwqscX9EaJPX4YkiwFCUzaTBE9aVtRiavMRo4poIMJCZeFF0zXUmoQ2hVDh8wIRaQPVq1aBYTSRwHvl4A+wrOvHfuGOtq/q6Lty0psH5fis7nQJBqDN8+BQpIlEi8cg+WYLArOmIbRaMThcPRIZg82YmalE3+mSHO2r9iLa1czkk4m+YoJ6DMtwe7OCQkJqKrK22+/HeGxEoCs12AcEY91fi4p10wi8/bZJP5sDOapqchmLarDi6Ownvpnt+BpcATJQqDs2bGtkWGK+E3YXda9gZwhLw7DyHhQVOqeLqR1Q3XwsxQgMIrLS/tbe8hQ4gHYZ+16rt0hEH2ZMGECsizT0dERESmMPTqLxPMLME9KQTb0LBl9r7YFgDNS4kmcn8cfHHqOq/XgAdxTkzhmSjqyLHHf2ZOYk59Eh9vHlS+spdoWIt+qquJtcgoS8d8SbP8rpX11Nc7iZryNDlRfiPwqTi+emg4cO5toX1WF7dMybJ+W0fp5Oa3LK2n7dh/tq6roWFuDY3sj7so2vC2uYNWd4vbR9u0+qh9cS8vSEnwtLmSLDssxWaJSTgXXHhstS0Nkpu2bvXjq7fuVoh5qREW8+4GamvdxOveh0yWRmXkBsmzApEnCQSN1+cPJ0HffaC2IyReJnkJF/wN7E5gTuX9PNf+srMMRplY3yhIGr4pNhof31qFfV4ekws9Hbwpu47Atpb39l1gso0Ljh3WhHhMX6nWkqio7AxGYNCtt5iwqnOJmkD/y98jyAD8WaeNg/E9h27uw/F4YdTJHf/M8TH6c7xJmoFy5DDlzMiAM7f71TWkwJXGkIvAjl5KSEqyY6YyJEyeyefNmtm7dyoIFCyJIzvfff4+qquTn5zNz5ky+/vprOjo6KCsrY+TIkXzd1Mafi0V65aqsZO4alYUsSXz0+QqybeLHu8Nq4Xdzx6LViHnIcaOTuQtYXdqEw+3DpO9/tdDXzW3scbiI1cickxYStX66rYaGdhcpsQYWjE/vsl/Di9uD5armqanELRqOxioIR/wZ+ThLWrB06EAXSWA61tSgtHvQxBuCRmsBD5iUCcMwtzuwb6ij6T9FZJ+Wyvd0X0q9s6atSwQmYGA3yTqKXR3VbN++nZqammAUIeC+a56SimzS4rO5UZxeVKcPxSWeVa9C7HHDgk67g4E+0xKM3owcOZKtW7dSXFwcrLw51LDMyRQdsD8qBY1E0qXjMORacbvdwa7eP/vZz3jrrbeoq6vj7bff5rLLLutC1sMhG7WYJ6VgnpSCqqi4K1pp+e8ePPvaqXtuE9XekIAXwLG1gSwlEVmSaGxspKmpKSj8DkfCT0fS+MoOPNUdlL61EdWgYjKasFgsqKpK87u78TY4yLWks9fbxK7iXRx9zNG9vn+bzRZsfHr00UfT0tLC3r17qays7PYceoJLUfi4vgWAn6YlIGkkUi4awz2Pb+Aao8z2OA3LdAq/c3tJ1mv558+nc+7i7yiua+fPz67l/pl5SFUduMpaUVp7iXZIoLHqUZw+VFf/qha7HcakBVVFdYoxNHF6Yo/LJmZmqKO5t8mJY2sD9i0NeCrbcO2x4dpjw/ZRKZokI6aCxKCQW9IdujhINAIzSKiqj7LypwHIzfkFGo0RSZJId4kf+ZrU3nOvgEi/ZEwGxQNb3ubLxlYeLa/FoahkGnRcmpnEyxOH86A1CedXVeBWUGN1HHVsDi9eXsD4ZJFTL2/NQZZUviuMLMvuaCyl3CRmOmPCUkg1rU5anV60sorB9QZrq/6GTytjNY0mNXXh4C7IcX8AJNj5Ifz3N0y2bSVGcdGstbA9NkSqZuYlopElyhvt7GvpXfB3OCNwMw7oO7pDfn4+MTEx2O12SkpCAk673R50zZ07dy6yLAe7KQfSSIFWDD9JjeduP3mpq6ujraEao9tFokfMpL4OK1UfkWIhK96E26uwqnRgBHHJPiF2PT89kZiwPkcvfy/0VRfNykGnify58NSKMls0EinXTCTxgoIgeQGQzToSzxmFVRXpqKZacQzVq9C2IhB9yUbSyPh8vuAMPz4+nvifjECTIESUKWUi/dBjKXUnLxhPtbgmGdmZ5OXloaoqb775ZjB9FKgKk2QJ09gkLLMzsB6fTdzCPBLOHEniBQUkXTwW/bCh82wJCEwPBx1MOGLnDSP56omk/WpqsBpr3759KIpCbGwsycnJnH/++ej1eioqKvjii57T3Z0hyRKGvDiSrxiPNslIg60Jr9eLwWAgMTERX5vwatGjJTtLmFp2188IhJdN6g1TiD0phyaN+P/GO404tzeJKMOmepBh0llzAJEKcjh6/31Zu3YtqqqSm5tLRkYG2dniHAI6mP7iy8ZW2nwKmQYds/ypc228kfKpSTy6wUGmXaHK4+XkdUXcWlTJigYbj07K5iXZwp2NEs5PynFsbhDkRZaCovGYORkYxySiTTWDVgYVfDZ3kLzIZi26zBiM45KImZ0h9pmVjnlaKqbJKZjGJwlB+TALmjg9+KvxVIcg6ZpEI/FnjyT91plY5mYGyQsIx+rYY4eRdsMU0n8/k7jT80UUTCPha3TS/l0VDc9vpepv39OxtmZA12soEY3ADBK1tR/hcJSj1caTlXVxcHnGvhZKc6FJ04DTWYXR2If3w5SLoXoTjk1vcptyDABXD0vmbyOzkCQJm8PDSR+tRfKqzNca+BwP22MlMmI2Ual6sVjG4FJ+i6Jei8b9DXUNa0hNFtqboqYGYBQpip3kME+Popo2EgzN3DjtdcpKReO75OT5jB1zL5I0SE6bOgYmnCOM7SQZ3Yl/ZrY1mS+a2ljZ0s4Ef1lurFHHpGFxbKxo4fuSRs6dPmxwxzvECERgeiMwGo2GCRMmsHr1ajZv3hwUGa5btw6Px0NaWlqwWmns2LGsX7+enTt3ctppp7HbLkTOM62hkuJ169YBUKnEMy3GzOduJ181tXFOupgtSpLEsaOTeX1NJSuK6jmhILXzKXWLSqebz/zi3cuzQmWnO2taWVPWhEaW+NmsrlED+yaRqjKOSsCQH9/t2MaCRNImZkNRIc0tzficHhybGvC1upGtoSqRtjbh3CzLMhaLBVmWSTy/gPpnN5OypxUmmiKMEYFu2wkAIQfeDAuz02ZTVlYWJJz9bR0w1Bg5UnjI1NbW0tLSEjTqGyyqqqr44osvaG1tpaCggAkTJpCWljaoBp7GEZHnEriBZ2eLMuDk5GTOOuss3nzzTb777juys7OD6dH+QGPRk3zlBLY/9REokKxakXxg394Iquh9NGrMaMr3VrB7926OOuqobseRtDJxJ+fibtsKmyHea6bx5e3BaXjcguHEThhG8vJkGhoaKCkpYcKECd2O5fF4gj2zAsfLzs7m+++/HzCBWVrXAojJRnihxIv1Nqa6nTy2QeLaWWaq8fByVSMvA7Ks8v/snXd4HOXV9n8z27t6L5YsWZJ77xUDpgaDaaYTSkIKJLwhedNICCQkJG8gEJpD6BB6Cd3GuGIb9ypZVrV610rb28z3x+yuJEtylY3h831duiTtzs7Ozs48z3nOuc99F00xM6U9SEajlxaXnz2EKJUlRqpEZhtNXDYhg6x45RqXZRnJGSDY6UU0KOakx5odlGUZ2RMk5PAj+yU0aSYE1ZHHe3WcHsvsdCyz05F8QXzldrz7O/GUdiB1+1HFHVln52ThTAbmOCDLUjT7kpV5M2p1uFQU8GKoLSbGHgBkmpr+e+Sdjb4cRA2Pasdw0OsnVafhFzmp0YHob5+V0ub0MTzRxFNzChhu0NEWCPJ4XaQGfAE3zp3PthYlXbp59/3RGuV+pzKwF6n6dvzUN3zAfTP/zDDLfkTRQGHhnxg75im02qNPmw6I8/4M038IN34Ac+6OdrL080UKk003VHxzreuPJoABoqJ2+/fvx+fzEQgEonX3WbNmRb/nYcOGodPpcLlc1NXVccClfGd5YfNNv9/Prl1KyXB/KImF8QpBcXWHo0978bzj4MG8WN+GBMyJNZPfSysokn1ZNCqZFFvfQUqWZWXVCxjHJx52/5mXjEZAIIhE/bv7erIvczMQ1MoQFOG/WK3WKKlZl2PDNC2VVI/y+WoP0YIZyE4AekpImjQTI0aM6PMdHa1541DDZDKRkaEE64NlGY4GDoeD999/n2XLllFRUUFrayvr16/nqaee4vHHH2f16tVRhdrjRYT/0rvUNXLkyKjK8XvvvXfU3lYRqOMNOAuVCTfObaDjtf1R7yPDqJ526qqqKgKBw3dltnnsAKTkZYAASKAvjMM8R8k2RxYKh8t27dmzB4/Hg81mi2Y/IxmYlpaWPl5Nh4MrGGJ5m3LtLk7quc7anT6+LG/jCXwMM+t4d62Th7e7ufqgnxxnCEkQ2Bej4vnhOv4828aeuUl0xOtwyzJbD3byyOdlnPePtby5tRZZlhEEAZVFiy7LiibReFylTUEQEI0aNMkmtJmWowpeDoWoU2MYlUDsknxSfzmVpB9PQDfMeuQXniScCWCOA62tK3C5ylCpzGRk3NDzROMukAKk2pXyUWPTO0cmPJniKRt5Df/MvAaAB/LTMYdT+Dtr7bz8lTKJ3L94NGatmnvzlIzOO95xtJJIctIFGLVq8vN+ii+kwUAxlXUfA7A/qOynyKBkX4JBB/uKf0aK/CdMGg8euYBpUz8gPe2q41q59YM5Ec77k6I2TI8v0ka7s48y5czhyip/Y8XgGhynOyID+JFq5enp6cTFxREMBtm/fz+7du3C5XJhs9n6TKZqtTo68O4pKaE6PFlHAoq9e/fi8/nolnQ0SlYuyozDpBJpCwTZ6+zJQMzMS0AlClS2uvrJ7A8EnyTxSmMPeTcChzfAuzsUDs510/uTdwP1ToLtXgSNiL4ovt/zvaEx6rCalWC2eXcNoQ4volmDaWoPp+ZQE8cIbOdmkxHOCtZ7/YR6XS8D2QlI3iChdmXy0aQpmZzICrt3+ejrwNFMrIMhGAyyfv16HnvsMXbs2AEowfFll11GUVERKpWKtrY2Vq9ezT//+U9eeumlIwYCA0GSpCjRODKhR3DOOeeQmZmJz+fj5ZdfPuYgpsmuBLyJ2PDsa8dXZgfAMDqepKQkrFYrwWCwjxHmQIh0IA1bMJLEO8ZhXZRN3NUF0Vb3yHkuLy8fsOtLluUosTtiCQFgsViIiYlBluWoWvCRsLy9G48kk2PQMq6XdtLHexoJSTJFGTZSrx+FWa9mbjf8zhbLmgkj2Da9iEcKM5kTayYIbDTIZC/K5oO75/LgZWOYnB2L2x/inrd2c9drO3F4Tz/tLEEQ0KabjysQGiqcCWCOEbIsU12tGBJmZt6IRtMr+gz7HyUZJyGKOtzuChyOwwuZybLML1KvJSBqONu+jQvCg3IwJPHrd/cgy3DZhPTopH9uvJXJRjcBtLyjvgNjWO/i0knj2NmhdLPs2/8QkhSkRIgBoMBmo6trJ5s3f4empneRZIH/VixCk7ws+vqTgVFmAzFqRbNkt7NnkpmUHYtWJdLY5eVg+5En2dMNwWAwytc4UgZGEISoJsyuXbvYuFHRbpk+fXo/MmRkJfhVldJ5ZBBF0sItxJHyUWkokUSLnkSTjjnhAHF1R49ppFWvYWJWDHB0WZgPWux0BEKk6zScG9/TdvrO9nrc/hB5SeZoxqw33LvD5aPCOETdkVeD8UnK9esQlWDLMie9zyryUBPHCESjhvyzhqGSZAJAfWtPNi8jzIHpdAdw+pTurYhCqsqmjba6TpkyhfPOO48lS5Yc8ThPJqI6JVVVA4oWyrKM3W6noaGBiooK9uzZw+bNm1m1ahWPP/44n3/+OX6/n/T0dG655RaWLFnC2LFjueqqq7jnnntYvHhxtFRVUVHBihUrjvkY29ra8Hq9aDQaUlL6krZVKhWXX345NpuNjo4O/v3vf/dpQT4cgsFglBicd9F4JXMCqJOMaBKNCIJAXp5iuzCYHhKA1+uNBruJiYnosqxYF2T1EcDLzMxEr9fjdrtZvXp1v3NdXV1NS0sLGo2mn/HpsfJg3m1WMrGLk2KRJImXXnqJ//znP3wQFq/7zrg01AkGUn4xlbTfTif20ny0GRbSDTquTo3njXHDeSA/Ha0gsLy9m+vLasjMj+X1783gnkUFqESB/+5q4IJH17GjZnA/sf9fcSaAOUa0t6/G4dyHSmUkM+Omvk/WKqUBdfp0EhMVQffGxncOu7+3mzvZ4NdhCPn4Y+lDCBVfAPDSpoPsa+jGqlfzqwt76s2CIHCT+n0EWWJtaBzbupQBWxQFFky6G4ffhElVx659yygxKDdjjLSNbduvwuOtQadL5/+2/4T3Ky6kMPXwk++JQiUIzIhRJtkve5WRDFoV48OT7DexGyky2Wo0GkymI3Sa0VNGqqyspL29Hb1eP6BjdF5eHmq1moNhgm6+UYcoCNTX19PQ0IAgipSHEhieqLzn/DgleP6ivW/L6Nx8paRzNHowEfLu9WnxqMMrWFmWeWmTkvm7fnp2v+ycLMl4dimvM447fPkogkimymlVdF9M01P7PD9YBgbAMimZ1KByDCWrekQbrXoNNoMSpNSFy0g95aMe7RaVSsX06dP7aIZ8HUhKSsJmsxEMBqMdUaCc77KyMp588kkeeeQRli1bxksvvcTbb7/Nxx9/zJo1a+js7MRsNrN48WJuueWWftkRvV7P+PHjue6667jmGiWbu3nz5mPO9kTKR+np6QN2G9lsNm699VZSUlJwuVw899xzR1USa2lpQZIkDAYDqdNyiVmcB2oR88ye6yBSRjrc/lpblWvaYrFgNA6sFK5SqZgwYQIAa9eu5Z///Cc7duyIZmMi2Zdx48ZhMPTVxzqWAMYeCLIqvHhYnBwbDTxLS0tprq1AEODCscrnE7WqaLm0NwRB4NaMRD6ZPIJ8o44mf4ArdlbwUHUTt88bzhvfm0F6jIHaDg9XPLWRJ1aXn/FT6oVjDmDWrl3LxRdfTFpaGoIg8N577/V5XpZl7r33XlJTUzEYDJx99tn9LsiOjg6uvfZarFYrMTEx3HLLLTidfXkSu3fvZs6cOej1ejIzM3nooYeO/dOdBFSX/Q2A9NSr+3JGZBlqtyh/Z07tEbVr/gBJGrg1zh4I8rtyJVL/qVxKtrcJdr5CU5eX/1uuDDz/e34RCWZd9DWBgJ247veYyyoAfldeHy3DzMjLptx9OQClja/Spo1DkCXkpr8hy0GSky4iKfc19rfnYNCoonLsJxOzwlmCQ3kwET2YbyIPpjf/5WhKb/Hx8VHdC1AyAjqdrt92Op2O4cOHYzcq5Za8cPkokn0R4zLxoSEvSTmnC+KU7bZ2u3AEe9oq5xUoQcWGinYCof4p9Ah2O9xs63ajEYQezypgY2U75S1OjFoVl01M7/c6f003oS4fgk6FvuDoeFORAMaXqyXlZ5P7aWoMloEBZZDPjlEmmqpmB57SntJFTyeSktmJEHg1xyE+d7LRW+4+MiY2NDTw4osv8sorr9DS0hIlMSclJZGdnU1RURGTJk3i3HPP5cc//jHjx48/ollnfn5+tGz2/vvv9xtbD4fIxH24Vm+LxcJNN91Ebm4ugUCAV199NVrWGgy9BewEQcA8LZX0P8zEPL2nySE3NxdRFOno6KC9feCFzUACdgPhnHPO4bLLLsNms9Hd3c3777/P008/zY4dOygtVYwjByILRwKYurq6I4oOftzaRUCWKTLpKTDpo6raAOPUDUzJjiXVZjjMHnowymzg08kjuDY1Dhn4x8Fmfrq/hknZsXx81xwuGptKUJJ56NNS/vBh8VHt8/8HHHMA43K5GDduHI8//viAzz/00EM8+uijPPXUU3z11VeYTCYWLVrUhxR17bXXsm/fPlasWMGHH37I2rVruf3226PPd3d3c+6555Kdnc22bdv461//yu9//3uWLVt2HB9xaJHfbCKxzUfWJy/C2r+BK3yjddWCswkEFaRNIC5uFlptEsGgnbb2VQPu60+VjbQHguQbdXx/tLJioPQT/v7+Rpy+IBOyYrh6St+VVmvr58hykBsNmzGIIlu73fy3uedmXzL7x7R64mlRKxNSEs2YVCpGFj3EqFGPUN4WrhMnmxGPQR79eDErnIHZ3OXE12tAiJQlNlV+83gwR8t/6Y1IGUmlUjF16uAKzYWFhXSGA5h8ow6PxxMVvGvWKsFEXmLYLdygI9egIyjD+s6eMtLoNBtxJi1OX5DtBwdPO0eyLxcl2kjU9rT9R8i7l05Ix6LvLwcQ6T4yjIo/ag2ISKltMN5EJAMzUAADkB2eCBoMouLhE85SZcT0NXUMhDMw2tQjZ8a+DkTKSKWlpbz99tssW7aMqqoqVCoVM2bM4Gc/+xk/+9nP+MEPfsDNN9/MVVddxcUXX8zMmTMHDHoHw9lnn01SUhIul4v333//qO+xSAbm0AzPodDr9VxzzTWMHTsWWZZ5//33WbNmzaDvE+GURPRfgH72DDqdLho4RYKMQ3G0AYwoiowdO5Yf/ehHnHPOOeh0Opqbm3n//fcBGD58OImJ/bOHSUlJaLVafD5fNNszGN5rUe6tS8O6Sb2zanGih7NSgwO+bjCYVCr+rzCLp0YqnLN3mjtp9PmxGTQ8tnQC9y9WOqr+s7mGLs/px4n5OnDMAcz555/PAw88wKWXXtrvOVmWeeSRR/jNb37DJZdcwtixY3nxxRdpaGiIZmpKSkr49NNPeeaZZ5g2bRqzZ8/mscce47XXXotG6a+88gp+v59nn32WUaNGcfXVV3PnnXfy97//fdDj8vl8dHd39/k5GbDFT2dsjRFdZxN8cT88PBL+eyfsek3ZIGU0aE0IgoqUFMVduKnx3X772drl4sUGJfD4y4hMtKljIGUsSAH0pe+iEgX+uHhMvyCjpVUh6BalzOaHccpg8Zvde/hox0pkWSYnKZZO8WZqUW6CbKmBqVP+S2rqEgRBiHogFaQMncbF4VBo0hOvUeORZHZ09/BdxmfFoNeItDn9x+3d83XhaDuQemP8+PEUFRVx3nnnYbEMfu4LCgqiGZgUKcju3bsJBAIkJiZS3K0EE8OTerILkSzMql48GFEUmJOvcE4G48F0BoLR+n1v8m5Tl5flYe+WG2YM6/c6OSQrHkGA4SjLR9AT7HV0dPSb5GRZPmwJCXpMHRutakLtXhxrwq2+kQxMpwc5KBFoUa6x0zEDA5CTk4NarcbhcEQVmseMGcOPfvQjFi1aNGhZ5Fih0WhYsmQJKpWKsrIytmzZcsTXOJ3O6LUd6Zg6HNRqNZdeeimzZyuk/VWrVrF8+fIBtz3UQmAwRNrcN23aNCAJ+WgDmAg0Gg2zZs3irrvuYvr06dHs1YwZMwbcXqVSRT/74cpILb5ANKt8SVIMgUAgun11SBkXQvX7jmtxtjg5luk2ExLweqMS8AuCwLVTMxmdqMYfDPLfXUfHPfq2Y0g5MFVVVTQ1NXH22WdHH7PZbEybNi1KXty4cSMxMTFMnjw5us3ZZ5+NKIrR9tKNGzcyd+5ctNoeUaxFixZRWlrazxAuggcffBCbzRb9OdIK4rgx5274yR649Gkl4Ah6YfsLsCosItfL/yg1RQny2tpX4/f3XXk+UKFcgFemxDIzXGZhvKInc4VqDTfPHMbItL7taYFAFx0din9OkpzNHZ9cTr6rmlZNLLfY47l6xUeUdbaxdO53OehU/FfEmhB3vtlOTZgsG/VASjk1rW+CIES7kV6ob4u2/OrUKqYMUya1FzdWn9IsTGcg2E8UDZTWx6au/u2TBz0+6nptH7kGJa2J5u6ja7fU6XRcddVVTJlyGINPwGAw0GUOfzf1NdHy0cRJk6kJl0nyegUw83sFML3PYZQHc2DgAOblhna8kswosz7qWwXw0qZqQpLM1Jy4AYNcX6UdyRlANKrR58Uc4VP3IBLs+Xy+fgJj27dvJxAIoFKpsFoHvi6zwgFMa5oSsHSvVmwGxvsEbkTLtF2dND28DUIygl4d9WI63aDRaKLaJDk5Odx+++0sWbLkmILho0VycjLnnqtw8ZYvXx6d/AdDJPuSlJTUjxsyGARB4Oyzz+aCCy4AlLE7ItIYQSAQiL73kQKYCRMmYLVa6e7ujl77vXGsAUwERqOR8847jzvvvJNbb701ShgeCJG5I3I+BsIHrXYkYKLVSLZBR21tLaFQCEFrYGMgG0lQ0drSfNwt80tTlQz1fxo7aA13lz3++ONMdmxkhvogb2w5Nq2abyuGNICJsMyTk/tamCcnJ0efa2pq6nfxqdVq4uLi+mwz0D56v8eh+OUvf0lXV1f051jFiI4Jah2Muxq+txZu/gSKLoaIAFzewuhmZnMBFstoZDlATe2/o4+7QxJbupVa/f8M62H6t+ZcjF9WMUas5heOB8HZd/JpbVuBLAcwazMxvfYDTO4WPmv5Fz8NlaCT/KzRZLBgRzWP7N9Fm1bpLhKcEp+XNHP2w2v4+/JSihsjFgKnJgMDcG1qPCLwboudXx6oi060S8PiaC9vquGvn5WekiDmoMfH/M37mfNVCZXuHm+dsmYHC/++hnP+vqaPU3aLL8DCLaWcv+1AtAQWCWD+/EUtVz29cUiPu9kfxCeqEGSZqk1f0traikajwZY+nKAkY9KqSLH2aLLMjDWjFQRqvX4qe3kFzRmhZFX21nfT5vT1eY/tXS7+Vq3cR7dkJEZ5PBvK23hqjVLHv2nmsAGPL1o+GpNwTO2TWq02mnnqXUaqq6vj44+VrOK8efPQaAZWsI5kYOpUsuJOHJRpfWo34za3cRt6RjqkaPu0cULi0MgCnCRceOGF3HXXXdxwww1HnNBPFFOnTiUvL49gMMjbb7992Nbq3gJ2x/M+8+fPB+Cjjz7qM/42NTUhyzJms3nQADUCjUbDvHnzAFi3bh0+n3Ltlru93LOviuawl0+k/LO3vov7PthH+yHX+GCIiYk5YnbpaIi874W9jxYnxQBE+S9NkhUfGpJylUzS4cpqh8M8vYgBmYNeP796+TVWr14d5QUNV7VTWt/OvoYuQkGJjkYXVbvbKN/WQvm2Fsq2Nis/W5TfTZVdeJ2HLznJsozPHaCr1U3Qf/w2Baca3xolXp1Od0w14iGBIED2TOWn86DCgwlroESQM+zH7N7zPWprnyMj/Tr0+lR2OdyEZEjRaqIrS4DtbSo2BK/jt5qX0ZT+F2rWw4V/g1GXgSDQ0vIJAEnlFeB3wLA5GJe+xi90Zq4u38S9e7fzmW0iT3UDOmVg/Om8cSzbIrK+vI1Hv+hpTzxVJSSAuXEWHi3K4sclNbzQ0I5GFLg/L50LxqRy33dG8bv/7uOJ1RWoVSJ3nzPipB1Hmz/I0l2VNPuV2vSyulb+PCKDBruHG57djN2t3OQvbzzI3ecqGazn6ttwhiScIYlyt4+RJn00gOkIanG0u2ns8pIWc3Qr1iMhosBr9bjwu5Qgd8yYMdR0Kcc2PMncZ3I2qVRMizGxrtPJqg4Hw41KcJNk0TMy1UpxYzfrylq5dIIyaDf5Aty8twqfJHNegpWrwyq+Ne1ufvDqdkKSzKUT0jl/dH/fIzko4dmrDKKGsUdfPoogNjYWh8NBR0cHGRkZOJ1O3njjDUKhEIWFhdFSxEDIDBs6NvoDmC/Ox//EbmR/CClWx/L2burU8L83TkSTakJlPkon9a8JGo3mpGRcBoIgCCxevJgnn3yS5uZmVq5cyXnnDWwXMpCA3bFg7ty5NDc3U1JSwmuvvcbtt9+OzWbrR+A9EsaPH8/69evp7Ozkq6++Yu7cufxfVRPvtnSRWjiJG2v2RbPzf/l0P+vK2qjr9PCvGyYfYc9HRigkkRinLJY7Ozup3t+IXmtEVAkIooAoCrRKIbZ0uxBQzBuhh/9ywG1EqxZZcsFZ/PvJEurr61nx9iaEbiVwsyYasCUasCUYsCYaMIatN3yuIK4uHwcra9i5bxu1TZUMyx9LSVoO+1OymSDoyEwaTlntbrrddi4NdPP533ayxishH2VXks6oJibZiC3JgCVWj8cZwNnpxdHhw9npJeDtCVxMMTqsCfrocZpjdYNyJlPzYrAmDM34d6wY0gAmohvQ3NxMampPe1xzczPjx4+PbnNoKjMYDNLR0RF9fUpKCs3NzX22ifx/qDbBaYPYbOXnECQkLMRmm0xX11Yqq/7ByKI/R1ufJ9mMfW7o7TWdvBBaRFzRPO5yPAzNe+Ct78Ledwgs+j0d7esASGp2Qd45cNVLoFEunOy86byQOYqVnz/Jbymi0piJMeRm5ohRzBlr5dO9Tdz/YTENXV6SLDoSLac22Ls8JY6ALPPT/bU8U9eGWhD43fA0bpw5jEBI4oGPSnh0ZRkaUeDHC/OPvMNjhCsY4rrdlVR6fMSoVdiDIV5v7OCOlHhufXYzjV1erHo13d4gL39Vww8W5CGLAi809HRJ7XN6yJKDBAIBJBlcsjL4lDR2D1kAE1HgTZF6VkyTJ0/m3VLlmokQeHtjQZxVCWDaHdya0RNYzCtIpLixm7UH2rh0QgbekMR391bR7A9SYNLzz6JsREHA6Qty24tbsbsDjMuw8eBlYwacaLwHOpG9QUSLFl3OwGTbwyEuLo6amho6OzsJhUK8+eabdHd3R6XqD9ddk6zVoBUE/LJMi0VN1q+ngiDgkSQeuPczCMJPM8zoDUfhQfYNg0+SKHZ6cQZDOEMhHCEJRzBEpy+I3i8xTqXF5Q/h9AXp7vDgrnGRHmNg0Tk5GK1azGYzl1xyCa+++iqbNm1i5MiR/YKUQCBAY6NitHi85XdRFFm8eDHt7e20tLTw+uuvc/PNN/fjv3hdAbpaPLi6fCRkmrHG9713VCoVCxYs4J133uHLL79kypQp0Yx1Y0wCVYKyyJEkmZ01dgBWFDfz6d4mzjsk8JYlGafdR3ebJ/zjxWn34XcH8XmC+D09v/3eIFJQCQZU8UZCGjfvPLkOna+v+OGWPB1MMpHeFuD9ezZgjFNRLygk5XR3LLNtZtYvK0fbnUzQWM/m7RuJ6RiHQP97Sq0VCUkSXlU7HlMdAW1X9LnRtW2UpOVQFZ+B50szNSUyIVMsWOxYdW2oO9KRAY1ORUyyEbVW7HPfCoJyjhztXpydPnzuIM1V3TRXDc4PValFQkEJl92Hy+6jsbxr0G0jOPeWUd+OACYnJ4eUlBRWrlwZDVi6u7v56quvuOOOOwCFPGW329m2bRuTJk0C4IsvvkCSpGhb24wZM/j1r39NIBCIppRXrFhBQUHBKVu5DBUEQSA/7xds3XYFjY1vk5X5XbaFyZiTrH07JXaEb8bUgqkw4QtY/zCsfQj2f0ir80vkXBUmVxBT9vlw+bNKKas3dBYWXvhzZpd+xhubniFVr0NtmAnA+WNSmV+QxOtbak4Z/+VQLE2NJyjL3FNax1O1rWgEgV/lpnLrnFyCksyfP9nP/604gFolcsf84UP2vn5J4tZ91ex0uInTqPjvxHxu31tNscvLVZ/uoanFSYpVz5vfn8HVyzZRb/fw7o56pAwTHYGeVck+p4fpfoW450aLFK7AljR2s7AoecD3PlaUh8taheFrIy0tjbS0NMrX7QT6EngjWBBn4Q8VsMHuwBuS0IdLO3PzE3lydQVrD7QSCkn8/EAt27vdxKhVvDAmB7NahSTJ3P36TkqbHSRadDx9/WT0vUzdeiMiXmccm9Cvg+Ro0JvIu2LFCg4ePIhWq+Wqq64a1NE7AlEQyNBrqfT4qPX6yQ5zXIyIJJi1tDn91Ha4saUfe2B1uuPqXRVstLsGfd5W5mDyPje5QRUpIREBaACe+7ye9PwYhk9MInd8NhMmTGDHjh188MEHfO9730Ot7hn+GxoakCQJs9l8QmOsTqdj6dKlLFu2jIaGBt549e1oYFS53kX5f9fic/ftzolPN5E9JoFhYxJIzrEiigKjR49m/fr1tLS08OGXG6lT9XT8fWBN5jf+AB0dXhy+nn394f19jFBrcNS7aarsoq3OSXe7JxqUHAu0QRsejRssLizmDGRJRpJkZEnmQJZy7RXWBQgFJNrsrRALqqCBaV4zeAO0EEAvZuA2NBLUdpM330BSbDpdbR66Wjx0t3pwdLpwqBrxWOsIqSO8MIE4bQbZiUVYTXFsCvio1UDL3HjO6hLxBkzsaK4ioO3ifXM33790ApfOzDpiZivgC9HV6sHe7Kar1Y2z04fBrMEcp8cSq8ccp8Mcq0etFfG6AnS3eulu89DV5sHe5qbK7SflkCqdPyTR4fJHs0hfB445gHE6nX2UEquqqti5cydxcXFkZWXxk5/8hAceeID8/HxycnL47W9/S1paGosXLwaIdmLcdtttPPXUUwQCAX70ox9x9dVXRyP0a665hvvuu49bbrmFX/ziF+zdu5d//OMfPPzww0PzqU8xbLaJJCaeS2vrcsor/sY2550ATLL2dBwEQxK76+wATMiKAbUW5v8CCi+A935AS0w1oCJJzIMrXgDV4F+drmAR1+efo4TgvWDQqrhp1slT3j0aXJ+WQECS+VVZPY/VtKARBX6ek8r35w0nJMn89bNS/vLpfjQqgVvn5J7w+0myzN37a1nV4cAgirw8Jpc8o57b0hP46YE6qi0qEg1qXvjuVDLjjNw8axgPfFTCM+urCM5SgpIik54Sl5dip4c2WQlguiUdhSkW9jc5KGl0HO4Qjgll4RLSnOHDyGRB1G6gPNypNXyADEyhSU+KVkOTP8DaTgfnJiiT+KTsWExaFe0uP7/fW8MbHXZE4OlRwxhmUAbhR1aWsby4Ga1K5OnrJ/XzPIqeR38Ib3G4fHQM3Ue9EQlgIr5QAJdeeumA7awDISscwBxq6pgea6TN6aeu083ob1kAc9DjY6NdKVcUmPRYVCr0Mvg6fdQ3OanP0NOVbyHkUZFSolw7LrOIwxMkJSRSf8BO/QE7a18/QGJOElq1ntbWVj5693Nmz5yNKUaHzqju0z59IvwhrytAa5mPXPNU9rnXUhY2iwVwNagQJSXgMMXoMFg0tNc5aa930V7vYvunB9GbNWSNiiM22cSI9Am0tHzGR2VVUBhHst9DyOejzRLDz3dUM/9ggPE+FSNMBkzOEDF2mU8e2dXvmERRwByvx5agx5pgwByrR29SozUoPzqDGq1RjVavRqNTodGp2LM3jvfeew9bjsQNt8yM7qsjEOS3X+4FGf5w6wTivbDi88/YWwptISt12iBXTM8iOz+W5Bwr676S2LJlC03eA5y3SNlPd3c3W7ZsYevW7VFCu06nY/LkyUydOrWPlMBttS3cW97AjmE6/jJFKWt3Pl9CdXU1Ol0bbxU3ctms/pn/Q6HRqUjIMJOQceTuPINZi8GsJTnHil+SuH53FWs6/Tw9KptLwn5Pbn+Qpf/6il3Ndmju5LaCryexcMwBzNatW1mwYEH0/7vvvhuAG2+8keeff56f//znuFwubr/9dux2O7Nnz+bTTz/ts8J65ZVX+NGPfsTChQsRRZElS5bw6KOPRp+32WwsX76cH/7wh0yaNImEhATuvffePlox3zQMz72HtraV7G/fTYsQRC3AWEtPALO/yYE3IGHRq/tMUqHEfA7MnUl7k5KGTZ77xGGDlyiOIHb1deK7GYkEZZl7yxv4e3UzBSY9lyTF8sMFeQRCEo98XsYDH5UwPMl81I7Kg+H+igbeau5ELcAzo4cx0WZClmV2fFUPxhDoVVx/2cgoJ+iqKZk88nkZB0IBAm4vRpXIfXnpXLmrgn1OD5vrlIE+qDZyz6ICbnlhKyWNQ9eyX+ZSJvYii4lJYTKjLMtUhCX08wbIwAiCwPw4C681dXDDnirGWQycn2BjUYKN6cMTWN5s51/tnSAInCPoyAwpE9Qnexp5dKXSJfHHS0czMWvwQci7vwPZL6GK06PNPD7+VO9OJIA5c+Yck6txpkELnQObOu6qtfdzpf4mwe8NcnBvOx2NLjQ6lTKpGtS8LSufabxayy+rROpK2uls6pEj2Fgg8fl4I6vHGimakMQ947MwWrX89bP9PP15JSMCKs6xWgi2eGmt9KDTD8Mfs58de7+ienUQdUgpPXTF7gMB3PUa1r5+AKNVi8mmRatXo9KIqDUiaq0KlUZEpRbxuQI4w2WGyO/uNi+tB7uJcFbNxlyc1goADFoTF94+CVuiwqnQhG0kvK4ANfvaqd7TTs2+drzOAAe+UugCMjLqOAuNJiVjHN/aTGFTNe9OnMcnbhexxQ7O8WjBE8mSCrgFmfThNkaMTCB5mBVbUpi/cYx+PZEyWkNDA8FgMJqtWtHWTUiGkSY9uRYDWKClUxmbdwpmUkdbOPuawuh+Zs+ezbZt26iurmbLli3U1NSwb9++qEheTEwM06ZNY+LEiQNyOJckx3F/RSO7nR72OtyMthgZO3Ys1dXVDFe18V5FGzXt7qhr9VBCCpf814Q1pp6ubeWSpFiCIYkfv7qDXbV2YowaFhR+fQrXxxzAzJ8//7CsakEQ+MMf/sAf/vCHQbeJi4vj1VdfPez7jB07lnXr1h3r4Z22MJlySUu9kg0NygQ4ymzA0OumivhcjM+MiZKl3O4q9uz9MU5nCSAwfPg9mCwFp/zYTwZuz0yizhtgWV0r7zfbo5H9XQvzqe/08Oa2Oj7c1XhCAcwzda08WauUPf5emMXCeGUg/OcX5byxuQ5NroVAvpXPvW7uCTu+WvQarpqSyZNuJShZmhLHFJsJEegIhNhW10Y6kJ+VwrjMGACq2l24/UGM2hOryDqCIZr8Cvcl4kIN0Njlxe0PoRYFsgcZqH6QlUSVx8fmLhe7HB52OTz8uaqJtOF6pPR4EATEeher99az5tNK8pLM1Icn/O/OyuGKyYPzHiRfEOd6pcZvHHv8HT69hf/y8vL6LISOBhHC+6Et8AOZOn4T4PcEqd7bRsW2Vg7uaycU6K/8+s4CCyRpSNrcyZ6ycA5fgE4dlMoBLsrPYHy2hb8dbObJkJNsRzc3WRP42bkF+AISz6yvYlugk79eO5IRQQ3tDWnsqO6kO9CMO7YcS9sYAv4QHrkTBOiqEtlzoO6EPldcmolhY+LJGjWBrcVr2bFzB3kFueQO4FquN2kYMTWFEVNTCIUkmiq6qD9gx9mhcFX0HYU0hbvukh3tJDm6mFru46t8PR9ONnLO5x3MHJHIpAnJPLe/gdf3N1Go1vDBoiw0J2AyGBcXh9FoxO1209jYGA1oPmmzA3B+opIlcTqdUU5nk2ThplF9OTg2m40JEyawbds2Pvroo+jj2dnZTJ8+nYKCgsNyv+K1as5LsPFBq51XGzv4k8VIUVERH330ETF4iRM8vLG1lp8tGvp54U+Vjbzd3IkoyyDLbO92s6vbxWufV7Byfws6tci/b5w84KLqVOFb04X0TUBOzp2UNz4HwChNG9Bz0UX4LxPCq+CWlk8pLvkFoZATjSaO0aMeIS5u1qk+5JOKS5NjWVbXyrpOB0FJRi0KCILAJePTeXNbHevKWqNW8scKSZb5vyqlVfjXualcGe628QVDPL5aKYH+ZkwmD/q72e30sKnLFfVtmjcxlcdL/SDLLNQbMahEhht1lLl9tKkF0oF5Y3JIMCtk6FaHj9ImR/S7O15EykdJWjU2Tc+tGSkfZccbBx2UR5j0vD8xn1Z/gOVt3XzS1sW6TgcNgSCoBEYb9VwzPIZVPjUbK9qj+5ydl8CvLigccJ8AwS4f7c/tI9DkQtCIGCcfP9fHYDAwatQouru7WbJkyREl8Q9F5iABTG8xu9MRfm9QyVR0+HDaFUJly0EHtcUdhII9QYs10UBGQSyhgITPE6QtEKQ2Ubn2p/pVjJqbTmZRLIE4Lec9+SVqUeCfZ+VgNajxyzKP1rTwvwfq0IkCS1Pj+fWFRfiCEi9tOsjPPy7msaUTuXBBIeM7k3niiSfwYeecW2KIsyTx4mvrUIkq5l0yHq8zhLvbj7vLT8AXJOiXCAUlgv4QoYBEMCChM6oxxegwx+gw9fpJzrH2IeWmDL+IEQUjjqqzSaUSSR8RS/qInvvIL43l96t3ApDc3UFCQjwv3zyZOZv30wS8N0bPH64qIt6s4+dj4lj+93b2Nzn49/oqvj/v+Hl0giCQmZlJaWkptbW1ZGZm4gqFosapF4a7jyLO2R2SAR8aFo3q32Qye/Zs9uzZQzAYZPTo0UyfPv2Y2uevSY3jg1Y77zR3cu/wNAwGAwUFBRQXF5OrauOtbXX89JwRqIZQWf2Zulb+WaMEZvNKt1Mbm0R5ciY/31JJ6eZaRAEeXTqBSdlHr0Z+MnAmgDmF0OmSqNHOAD8kOd5FkqYiigqhd3s0A2PiQNkD1NYqgY7NNpnRo/+BXneadl+dAMZaDNGOoB0Od1RQbfKwWHRqkRaHjwPNzuNq+d7n9NAZDGFWidyR2ZPF2X7QjjcgkWjRcfv0YZQfqOOlhnaW1bZGA5gPHcrkLrZ4+bSjlrOy4hhpNlDm9uG1GMAOw9KUfRalWml1tFLSeOIBTITAm2/sy0OJlI8G4r8cikSthmvT4rk2LR5XMMSqDgfFLg83pyeQqNXw3RnD6PIEWF3aQnWbm5tmDUM9SFDkr3fS9sI+pG4/ollDwo2j0Jxgt8EVV1xx3K89Ygam8/TIwMiyTHN1N3tW13FwT3s/0mpvxCQbGT4xkbxJScSn922Rf7mhHbm0lrEWAz/85fjo40+uVsoyM4bHYzMq48cvc1PxSjLL6lq5e38telHk0uRY7vvOKHzBEG9sreOu13agVYucMzKZBQsWsHz5cj5fuYKZMxVuRkZmBhPOHjak50KlUh1TmfBQ7HV4CAoier8Pm8dFcu4wLBo115mt/K2jg1COmWYk4oF4s45fXVDEPW/t5pHPD3DhmFQy446/tNI7gAFY1e7AK8lk67UUhX3KIu3TjZKVsRm2AbsRY2Nj+fGPf4woikdl/noo5sZZSNdpqPcF+KSti0uTYxk7dizFxcXkqTvY1u1h7YHWISvlfNhi57dlSsZ1alUxBc21xAX9lCdnskf2o1EL3HfhyAGDtVONMwHMKYQ3JFEWUMoY2f4vaWh4g5SU71Dfsp2R1ve4MKMGVWsDtQGldTcr6zaG5/5PNMj5tkElCMyJtfBBq53VHd3RAEavUTE1J451ZW2sK2s9rgBmXVjme3qMOeqyDLAxbB45c3g8giBwW0YiLzW082lbF1VuHzaNijebFKE19UEn7zns3LOokIww78hhVoKICJ+jKNXC2gOtQ8KDKQu3UPcuH0FPBuZYU7UmtYqLkmK4iJg+j9sMGi4Z39+ksTc8Je10/Gc/sl9CnWQk4aZRqOMO3yl0shHVgvEF8EkSunAGJzO2l53AABk7vyTxUFUTixJsfVSHXb4gD684wGUTM/qpXh8Kl93H1k+qqdzRSkyykYzCWDIK40gaZkEVDgCDgRDlW1vYs7qOloN9id1avQpznF7JWMTqsCUaGDYmgbg006AZxo9b7QBckNCXmPzZPiWzeG6vCUQQBO7LS8MrSbzY0M6PSw4yzKBjgtXIg5eNxReUeH9nAz96dTsf/Hg206ZNY/fu3TQ1NbFqleLVdtLUy08AW8Pt07mSH4GeVmyxxYvY7kFKNvCz0lo+mJiPShC4fFIG72yvZ2NlO79+by8v3DzluEuevQXtZFnmkzalpfj8RFt0n70DmKWHmdAPZx9yJKgEgatS4/h7dTOvNrZzaXIseXl5ilqyx0OK2M1rW2qGJIDZZHfyw5KDyMCohiom1Bxg3rx51DklvnB102myMmNOBtcPYDPydeBMAHMKscfpISBDnCpAYrCFA2X3U3rgd4DMkrD0STAAanUMI4v+QmLi2Yfd37cB8+OUAGZNh4N7chTtIL/fz7QkmXVlMuvK2o6rGylibjgntu+k/2WF0kkza7ii7TDCpOesOAtfdDh4pq6VJK0GjyQz2mzAZDayq9POKxuqqZX8oId2kxW9Xh+VWh+Zqkx8QxHARDMwpoEzMHlJZnwHu+l86wDqRCOGUfHoC+NQmYY2wHVuaMD+QQXIoMuLIf7aIkTD1z9UJGjUGEQBjyTT4A2QEw70Iqtetz9Eh8tPvLlvAPh2cyf/rGlhfaeTTyf3CCW++lUNz6yv4kCLkxe/O7DBpsfpZ/unB9mzpj7KUXF3+2kos7P5gyo0OhXpI2KwxBso29KM16VwmES1QP7kZEbNTiM+3Yz2GM+fIxiKeu2cHy5XgOJVtbPWDsC5I/uW8wRB4M8jMmgPBPmotYsfFFfz+eQCTGoV/3fFODpcftaVtXHnf3bw3g9ncfHFF/PMM89EOY3HK2B3MrE17J92YX4OC1MXR7vydtR0oqmxQ7KB7d1uXmpo56b0BARB4I+Xjua8f6xj7YFWPtrTyEVjj0/tOC0tDZVKhdPpZE9JCSvalUxaJKC02+10dHQgydAsWVg0amikFAbC1SlxPFzdzLpOJwc9PrINOkaOHMm2bdvIFTtYWdJCq8N3QvpeB1xebtyjiFwO72hmVtkuRo8aRUL+BH68bAN56U1sGTGKcm3wuEv7Q43Tt1XlW4itYQG7KTFxmIw5yHIAkPHJKWxunEiJ8yYmTXyd2bPWf6uCl+bmZl555RXq6vqTA+eGvXx2ONx0BZQB4v3336dt26dMU9fwVVUbvuCxSVsHJJlN4XM9O7Zn5eP0BdkVHvxnDI+PPv79cInpP00d/LteIf1+LzOR2+bkkIbAnNVNXLFJKfHZjRbMvcioRb0CGOkoFTEHQ4QDc2gJqbwlLGJnM9Dxn/0EWz14i9vpfPMAjQ9souXp3TjW1RNsPzEOiCzJ2D+qxP5fJXgxTk4m4eZRp0XwAsoEnREuI/VupdZrVCRblYF7IB7M9vAkuN/lIdSrAWFHrfKdDhR8+twBvvpvJS/9eiM7P68lFJBIHW7jgh+MZd41BQyfmITOpCbgC1G9p509q+vwugKYY3VMX5zLTQ/O4uybRpKaF3PMwQvAyvZu/LJMnlHHiF4ZuRXFSvZlYlYMydb+GTFREPi/gkzSdRqqPH5+HS4FqFUi/3flOOJNWvY3OXjo01LS09Oj2ltwdAaOpxoR0c9p8TbGjx+PRqOJCtgJPolbE5V78S+VjXSEx4/cRDM/COtI/fGjEtz+Y3OFjkCj0USd4x9ftZ7uoESSVs3kcBYvkn1pk01kJVrJSzp56uZZBh1zw2PZa2GDx4jDfa6mE1kK8s72EyNf31tWT1cwRIariwX7viIzPZ1LLrmE376/D6dfJtlvQB0KUofImpbBXe5PJU6Pken/E2wLp0Mn28xMLPgPTlcpFnMhN79Yzpfl7fzx0tHExBy5p/+bhlWrVlFWVkZHRwd33HFHHwGtTL2WPKOOcrePL+1ORnu62bdvHwBF6hacAS3bqjuZmZcw2O77YafDjTskEadRRWvVAFuqOghKMllxxj618Tmx5qjWizukDFKXJMUg6/2kiGaSJYEkH5j9QZxaNb6EnpVWboIJrVrE5Q9R2+kmO/7Ya9ygBF3VnggHpmfC6nIHol5GKdva8Nt9qOL0GCck4S1uJ9Dowl/Vhb+qi66PKjFNSyHmouEImmNbm8ghic63ynDvUIh71kXDsMzPOC1WWb2RqddS5vYNyINp7vZR2+mOdodFsD1833klmUq3L5rhiqi4tjp8tDl9JJh1yJLMvvUNbHqvIspdScyyMO2SXLJGxkXPx+i56ciSTFudk9r9HXQ1u8NibPHH3LI7ED6OlCsSbH2+g8/2KS3Gh+MfxGjUPFaUzZKd5bzW1MGCeAuXJMWSZNHz0OVjueWFrTz7ZRVzRySwYMEC2tvbo103pxMafX7qfQFEYEIvyYnyVicOXxCjVsU9hel84XZT4vLyl8pG/lKglH2+P284b22ro67TwxOrKo67S+fss8+mqamJtWplobLQZkQcoHx0KvggS1PjWNPp4NXGdu4elkJmZiYxMTHY7XYyRTv/2VzDbXNyB5X8PxxKnB5WdzoQZJm5e78i3mLh6quvZneDk521drRqkT/ecD71n69mR1wqj+wrY37ytCPv+CTjTAbmFGJbeCU4yWpCp0skPm42KnU8u2qVwWpC5jdLZfho4Ha7OXDgAADt7e1s2bKl3zbzwiuL1R0Oli9fDkB8vJIhmaKpY9Wmbcf0nuvC5aOZMeboYAOwoRf/pTcEQeD2zJ4Wz5vTE1D7JewvFJMiCdQj8QF+0pzK99fUqSbQovytVomMSFbKVCdSRqry+AjKYFKJpOp6SkLl4fLRBUYD/u0tIEDc5fnYzskm+a6JpPx8CraLc9Hl2kAA11dNtDy585iyMZI/RPuLxUrwIkLsFSOwLjgxQbOThaywAF+/TqTYgTuRXKEQ+1095pzFLuX5lm4vDb2cx/c3OuhscvHu37ez5tVSfO4gsakmzvveaK745WSyR8X3Ox+CKJCYZWHiudksuL6I3PGJQxK8eEMSK9uVaynSrgtgd/vZVKmUQI80Yc6MNXNXthJo31NaGz1fC4uSuWGGskj62Zu7cQTg2muv5fzzzz/h4x5qbO1S7rGRZgMmdY869PaDyup/bIYNvVrFH/OVzNFLDe3sdSiv0WtU/OZCxVBx2dpKDrYPrmR8OKhUKi67/HIOJiqcMcPeHYRCIWRZprLy1AYwFyTaSNSqafYH+bStC1EUGTNmDAAFmk6q292sL287wl4GxlNhuYmc1gYSpCBLly7FYrHwzDrlM146Pp1km4E78pQAcbOsobrTfuIf6gRxJoA5RWjw+mn0BVAJMM7aw1Qva3HgDK8mTqXB4qlCRLQpknVZvXo1LlffwWReuIy0ormduro6NBoNN954Iwm5owHwlW+KtiseDSLcgTmxfc/nl+XK4D9QNuey5FhyDFriNWquT4ql7YV9BBpdCCYNfzQHeCVGIBFl8mtWG2h5bAfOrxqRZZmisDVD8Qko8kZMHPOMuj4TZUWLEwvwI58S1JhnpaPLjYk+r47TY5mVTuLtY0m4eTSiSU2gwUXzYzvw7D3yYBZyBWh7Zg/e0k4EjUj8DaMwTTp5tfwTRaSVusbTV9d8MC2YPQ4PoV6VvRKncp4jPBIAUYa9K2p47YHNNJZ3odapmH1lPlf/dirDJySd8kBubacDV0giVadhfK/Mw8qSFoKSTEGyhWEJR870/c+wFCZajXQHJX5cfDBaPvvVBUWMSDbT5vTxi7d2nxIX+ONBhMDbW7Ecejo2I8KLM2PNXJIUgwT8uqw++nkWjUpmTn4C/pDE/R+WHPdxlARkXBot2mAAVelePv30U9ra2nA6HYRkAZU5gbEZJ18BWiuKXJeqLL6eq1fu7UgZKVWwoyPAS5sOHvN+m30B3m5WylLj6sq59NJLSUlJ4WC7i8/CJctb5ygK7hePHUWGx4Ekivx1654T/kwnijMBzClChIw20mTApOpZTUT0X8ZlxAxpH//pgt27dwOwYMECUlJS8Pl8fPHFF322mRljRi1AYwi69EZmzJiB1WrlisUXUh2KRUTmP/95rZ8J6EDwhKQo12h2LwJvp8tPcThDMiM3vt/rdKLIiskFfDmlEN6qwF/VjaBTkXjLaF7+2Tw+/ckc4tx2ABriLcgBCfu75bS/WMz4GGUyOZEMTESBtx//pdXJT9BjDYE60YBt0eAlRv2IWJLunIg224rsDdH+cgn2DyuRQ/0F0gCCdh+tT+/CX+NAMKhJuHUMhsKvV9fhSBislTqiBVN7SAYmwn+JDHS7O5w4OrzsOtCOWYLMoMgNDh2h3XakoEzWqHiW3juVcWdlHlcqfigQ6XY5L8HWJ4MY6T46WrKoRhR4YmQ2JpXIpi4Xjx5Uyk96jYp/XD0BrVpk5f6W45r0TgUi/JfJtr7B2vbwmNlbOfre4WkYRJGvuly826I8LwgCv7t4JGpR4POSZlaXHnn8GAiRct5csw6VLLNlyxY++OADAJolM+eMPjqX7aHAdWnxiMAGu5NSl5fExMSwcbJMkbqFlSVN1NuPjQv3bF0rQRmSu9qZm5YUbXt/dn0VsgzzCxLJT1YWg4IgcGO6sgD8LKiivfPr5cKcCWBOESL8l4mHrCYiCrwTsmJO9SGddHR0dFBbW4sgCIwZMyaapt6+fTtNTU3R7cxqFQWCMsm2pmQya5Yi2JdsNdAcN5ZmyYzP5+WVV16hu/vwQcLWLhd+WSZVpyHX0MMliaTeRySbB2Xqm1Qi0geViuePWiDhxpFo08xY9RpMWhWGFsWU7qBFi/WCYaAS8JZ0MGdtM5eiYX/DCQQwgxB4teVdLEKLDMRdWYAwiNFiBGqbjsTbx2Ceo6S8nevraXlqN/YPKxWC7sdVdH1SRden1bQ+uZNgiweVVUvS98eiy/56TD6PBZkDkHihJwNTd4gWzNYOJRuX26Bsv62pmxd/tQHjZ83c0W3gaqeOREnEp4JzbhnJRT8a288d+VQiKMl8Fp4wL+xVPvL4Q6wtU9L8i0YffblimEHHgyOUEsvfqpuiwX1RqpX/PU8RMPzjRyWUNg2dn9dQwCdJ7HYoE3Fv09sudyAqK9B7zEzXa7krWyHj/6G8AVeY+J+XZOGmmcOUxz8oxh8cOJgfDLIs80mr8n1cPTyLs846CyDqHdUkWTn3JHYfHYp0vZbzwl1Qz4ezMOPGjQNgvLqBizT7eOnj9VGrgiPBHZJ4rlYJ7CY0VLJo0SJAKVe+sVUhBd92SBforaNGYAwFceqNPLmhPyXgVOJMAHOKcCyriW8L9uxRUow5OTlYrVays7MZNWoUsizz6aefRlO9Xq8X20FFnMuVW9DHE2ROQQpf+PORtGa6urp46aWXWLNmDVu3bqWkpISamhra29sJBJT21Qj/ZXZsX1GwL6P8F2X1IAclgu0efJV2XDta6F5VS/vLJbi3NoMA8UuL+pRquru7sTm7ESUJhyTTNTWZ5DsnoM2yIAYk/gcDv7QLdNYc2X5+IEQCmDxTz2cPOf1c0KhMvN3j44/ag0hQicRcmEv89UUIehWBWgfO9fU419XjXFuHY00djtW1hLr8qBMNJP5gHJrk4yMfn2pkhbVgWvxBPL0yS5nRAMaDJMmEQhK7VtbyZb0dgAmVSoary6TCrxcIIRNCRlAJ7NUEed7qI3fiqS8XHYrNXS46AiFi1Sqm23oyiGsOtOINSGTEGqKt+0eLK5JjuTQphpAMPy45iBS+726eNYx5IxLxBSXueau/CeLXib0OD35ZJk6jIsfQ43Yc6RzLjjf2a5f/fmYS2XotTf4AD4ezTQB3nZ1PgllHZZuL576sOqbjKHZ5Oej1oxcFFsRbmDNnTrSVG8CpjWPqsFObtbwpnAF5s6kDZzDE1KlTmTdvHqJaQ5zowXvgS5588kn27t17xEDmP3UtdMtg9bi4rnB41O7jla9q8ARCFKVa+3EGDWoVl8Yp1+Z/PRJtbcfHuxkKnAlgTgF8ksQe5wCrCU/PamL8tywDI8syu3Ypg2KkTgtwzjnnoFarqa6upqREqUuvX7+elFal3XMvaoK92pFn5yXgQ816ijCZTLS2trJq1So+/PBDXn/9dZ599lkee+wx/vrXv9LW1sZ6e1giP6bvZL+hop00BC52CzT/Yzv1v/mSpr9upXXZHjpfL6X7s2q8+5QsTexl+RhG9b1pOzs7UckS8T5lhV/i9KBJNpH4/XHEfGc4HmTGosb51B66VhxEPoaVnizL/VR4ZVmm/e0yrLJAOSESFg076v1FYBiVQPKdE7Gek41lfgbmeRmY56Zjnp2OeVYa1rOzSPz+ONQxX69A3bEgVq3CFCbK1vXKwqTG6BEF8Acl9m5r4vX7N/PpB+V0GUUEWebOK0eSHiZHZ901ir/HeHkyMcBtj85jbaxEtyxR1XZ8RM+hxMdhr51zEqx9BBiXR8tHKcccZAmCwF8KMjGqRKo8fkrCpGZBEPjrFWMRBdhd10XDMZYeTiZ6+C99hf52HGbBp1eJ3J+vZB6frm2lIrwosOg1/OI8pQvp0ZVlNHd7+712METEBOfHWTCpVIrVySWXgDWF+pCVSUW5gypZnyzMiTWTZ9ThDEm81dyJKIosWLCAn/zkJ5SJmfhlFa2trbz11ls8+eST0W6pQyHJMo9VhhV32+uZO3s2oNxDL2yoBuC2OTkDXm8/LFQ4MTVxyawpKT0Jn/LocCaAOQXY5/Dgk/qvJiKaJFlxRhLMA5c1vqmor6+no6MDjUbTR0o8JiYmKl2+fPly2tvb2bRpEwkOOxYBHCGJnY6eMsDUnDi0apHKbjh78dXMmTOHCRMmMGLECDIyMoiNjUWtVuP3+9m0ew87w5yHWWH+S7DLR8OKav63Fd7AQvJOpfUYALWAOl6PLteGcWISlgWZJNw6GtOU/in6znCtN1NWWmv3hQNSQRQwz0zjyTw9XxJAkGQcK2tofXo3kv/o9GsafQFcIQmVAFkqFa4tTbQ8sQt/SQcBZP6u9ZM0gET50UAdp8e6MAvbeTnEnJ9DzAW5xFyUS8zFw7GenT3kIngnG4IgDOiJJAckpqsMXO7Usu7fJXQ2uWlNVwKzArOB/IJ4iszKOVzVqGTJxmTY0KjEKHm+eAhdxY8HvcsVFyTERB8PhCQ+Lzly+/ThYFWrmBbO/m4MB/kASRY9Y9KVksRXVe3Hte+TgUgH0mTroRnrCIE3ZsDXnRNv5aw4CwFZ5t6yhujjSyZmMD4zBpc/xF8+2X9UxyDLMh+1RtrZe95PrdbwmS+fFYECFo0+PpG8E4EgCNyYpmRhnq9vi2ayrWYTU2fO5S3fWFrMw9Hr9bS2tvLiiy+ybt26ftmYdw820IQKbcDPT8aPRKtV7qv/7mqgxeEj2aobVAQw16hjhkkHgkBN5vF7Tp0ozgQwpwBbo/yXvquJI92M32REyLuFhYX9bOJnz56NxWLBbrfz7LPPEgwGycnOZn64thsxTIOwrUA4RbujOcjChQu55JJLuOaaa7j11lu56667+M53vgPAitpGJCDHoCVDr8X+3wqaHtyMtLKWkaiQUJRlYy/LJ/VX00i/fxYp90wh8fax2Jbk8W7jap5b8Ro+X98OF1D4PADD1cr3FwlgIkjPsvELPHw63IBgUOOvdShS/EchbhfJvmQFBdr/vIXOt8sI1DqQBXgUL6qUweXm/39EhMhb0eVh/6ZGPnpiN8/+bD2z2iAnqAIRxp+die07irJshHc2KhzA7AkHuRPCejERMcL9XzMPZLfTQ70vgEEUo515oPC3ur1B4k1aJmUff6k54vW1odPZ5/FpYVL7poqO4973UCPCGZxk6+EMSpIc7R4bzHdMEATuz09HIwis7Ohm/ub9/KO6mVqfn/u+MwpBgHd21HPRY+v419pKGrsGzzpt6XKx3+VFLwqcm9BTtvtifwuNXV6sejWz849en2oocWVKLAZRZL/LGxXtBLh6aiaSqOHjtjjOv/oWxo0bhyzLrFy5ktdffx2Pp+fz/r1UIW/PdNuZNFopi8myzDPrKgG4aWYOWvXgIcKvCrJ4aUwOd2Z/fV2LZwKYU4CI/svkfgReOzD4zfhNRSgUYu/evUDf8lEEWq2Wc845ByDaUn3uuecyP04ZJFY1dtL+2n58lcrqZ054kBhM4yA/Px9RFCkWlYltTqwFT0k7zg3KCqzOrOL/8PDOjHgSbx2DaWoKKqu2T1CwZ88eqquraW5uZsOGDf3eI5KBKQpzVIqdfdPQkUnwvz4vCTeNArVC8LV/UHHYNtVAs4vtyxX+T1abH9kbQhWnx3reMD6anci7BMg7ChPHbxNkWcbrCtBa66ByZyv71tWz/bODbHi7nC9eKkEqVwKNTz6rYuXzJVTvbiMUlPAbRDbpAsiLUph1eT67w4P1hPB9FxE1rAspWbSI4F1hJID5mjMwkezLWfEWDL3KEj3eR8kn1Kk4MxzAbOpyRnkwANNzlQXC6ZKBafD6aRhMwM4bxKBRUXgYyYnhRj2/y0tDKwjsd3l5sKqRaZtK+FVLC7PPzkHUq9hb380fPy5h5p+/4MqnN/LSpoN0uPoSw5fVKaTpJcmxxPZyh382zKNZOi0L/RFI9ScLNo2aJcnKvBFpqQZItuqjWbrXdzSxePFiLr74YlQqFaWlpSxbtozGxkY+LC6lQmtElCR+NWVMdCxcX97G/iYHRq2Ka6Ye3lpiis3EOQk2VF/j4uqMEu8pQIT535v/0nc1EfM1HNXxo7y8nOXLlzNy5EjmzZvXLztQXl6O2+3GZDKRmzuwj9GYMWPYvHkzdXV1jB49mvT0dOaGSwI7vV5a9jrx7GzFMD6RuRMTeRBlJeoPSv1WBQaDgdzcXF63KWJ0My1G7C8qQYF5bjp376qigQAvjUxkIIRCIVavXh39f8OGDUyZMgWzuVcbdjiAGR9jhVY/VR4frlAo2hIfCWBKmxyoMszEXVlAx6v7cW1sRB1nwDKnv3mia0cL9nfKKM/TAFryjToSbslDNzwGQRTY958dAAw/RhPHbxp87gC7V9XRXNWNo8OLo91LwDd4+U0YoYNYE3ajQFyaieETEhk+MYmXiutZt7Kc9GAQSZaj5cSJ4ftuZDgD49IK6IDxkQxMeDL8ujMwy3up70YgyzJflChdIueOPDGxtHEWIwZRpCMQotTljZbUJg+LQxSgut1NU5eXFNvXy4mKSk4cRsDuSLyTWzMSuTw5lo9bu3i3pZMvO53KQlIE04I0fqAxsXV3C5urO9hcpfz83/JS3v3BLHISTNR6/XwcDihvzegZN0oau9lQ0Y5KFLjhazY0vDkjgZcb2/m41U6zL0BymON13fRsPtrTyHs76vnf8wuZNGkSqampvPHGG3R2dvLvf/+blUWTIS6FabKPsemp0X3+Kyxcd+XkzKjT+emMMxmYIYDkDxHs8BLs7E8Oa/IFqPcFEOhZCQJUtrno8gTQqcXo5PdNwPbt23nllVdoaWlh9erVfPTRR/1qq5Hy0ejRo1GpBl6hCILAFVdcwfz587ngggsASBdUDPPKhASB7Rl6EMCzsxXry6XcojXg84eiZbdDkVZQRLtZGfjH7rQTsvtQxejoHJ9AQ5cXrUpkcvbA3QI7duzAbrdjMplITU0lEAiwZs2aPttEApjhifEkadXIwP5eWZjsOCNGrQpfUKK63YVxbCK2CxSiW9fHlX1E5eSAROc7ZXS+XoockKhJVDJHY6ZkoM+PRQivsqMu1N/SDEwwEGLH8hpe+u1GNn9QxcG97XQ0uKLBi8GiISnbwrAx8RROT2Hc2ZlM+04usyYqE7lqhJWl905j6sW5xKebyYhTApXaTjflbh+OkIRBFCkIE6NzDTo0AqAWiUswkBqeqEeEA5jGLi92t5+vA23+IMVhcm3v8tHBdjcNXV40KoHpA+gXHQs0osDUMA9mQy8ejFWvYVTa6cOD2dZ1BAG7oyyjxWjUXJMWz5vj89g5cxQP5KczwqjHJUlsNsi8/r3pbPjfs/jVBYUMizdidwf48ydKY8Gzda1IwNxYczTQA6JdTOeNTiH9OHlpQ4VRZgNTbSaCMrzc0PO9Tc+NIz/JjNsf4t3tCkk3LS2N22+/nfz8fDpVGvbHKi3nv57Qw08sbXKw9kArogDfnZVzaj/MceJMBuYY4Vhbh6+yi5DTj+QMILkCyIGeCVxfFIft/Bw0ScrNF6nlFpr0mNW9Bex6VhOaU8xiPx7IssyqVatYu3YtoDjX1tTUsHXrVnw+H4sXL0alUuH1eiktVVjpEX2CwWCz2Zg/f350/51vH2CaP0B1tpZds5O44qI47O9X4K91cDMa5iNyYHP9gAN5W3I6OJuId3bh39iKFj0xi/N4p7ZHZ8eg7R9MBQKB6GeaM2cOycnJvPDCC2zbto3p06cTHx+Px+OJ1o5jYmIYZXbS0uFgn9PDpPCEIIoCBSkWdtTYKW50kJdkwTwnnWCHF9emRtpfKyXxdi0qk4b2V0oINLhAAMtZWVQbHeAPkt+rhVqSZCp7uVB/myCFJPZvamLLh1U4OxX+T2yKkTHzM7AlGbDE6bHE6VEP8H0B2Fxefr95P/t9flr9ARK1ykoxcp521NjZEJbiH2cxRLt51KJAoizSgERGdo/HkFWvISPWQF2nh/1NjhMOFI4HEWJtoUkf/TwAG8P6RRMyYwe8fo8VM2JMrOl0sMHu5JZemYVpOXHsqe9iU2U7l4zvny08ldjaPbDkxOE6kI6EJJ2GWzMSWZRgY/ZXJay3O/m8vZtzEmzcPnc4CwqSWPTIWj7b18yaijZeaVTO+229zlGb08d7O5Wy9Okywd+UnsDmLhcvNbRzZ3YyGlFAEASun5HNve/v46VNB7lhRjaCIGA0Glm6dCkr1m5FlkUmaEUmx8dE9/XE6nJAIYpnxZ9evliD4fSfOU8z+BucePd3EKhzErL7eoIXtQgCeEs6aH5kG53vlhFy+NkWYdMfejMegYx2OiEYDPLuu+/2mehvvvlmlixZgiiK7Nmzh9dff51AIEBJSQnBYJCEhISwQuTRwbm+Ac/uNqZ1KKvvtU4X2gwLiXeMI3ZJPgGtSA4qztrZhf2Tqn7k2K0eRQcmzd5KtdCCYUwChsI4NkTsA4YPTLbbvn073d3dWCwWJk2aRE5ODvn5+UiSxMqVK4Ge7IvJZEKn00VLEYcSeXs7U4OSZYq5eDj6wjgISrQ/v4/mx3YQaHAhmtQk3Dwa5qfTEnbLzeslYldv9+ALSmhVIhmxX+9Kb6jg9wYp/aqJ1+7fzKqX9uPs9GGO1XHWDYVcfe80xszPIGtkPLEppkGDF4B8k55JViMBWebF+p6V57gMW3Tl+U6VkvGacMgqXuMOZ3cS+p7TwpSvlwfzZTiAmRXTN1jdWKF8vunDhyaoivBgNtqdfbhZkaDtq8qvl8jrkyT2hAXsJh8iOVE2gIDdsSJTr40GJX+oaIhKNuQnW7g6zPn4+VcVdAclcg06Fsb3ZMdf/aoGf1BiXGbMadN4cWGijQSNmiZ/ICqACHDphHRMWhXlLc5oEAxQ5fXzOUrG91dFPUHYrlo77+9sQBDghwvyTt0HOEGcCWCOEaZJycQuySf+hpEk/mAcKfdMJu2+maTfP5Pkn05CPzIeJMVUr+mvW9hcowwI/RV47UBPJ8TpCq9XUcDdvXs3giBw8cUXs3Dhwqi67tVXX41arebAgQO8/PLLbN++HVDIu0fbOeOrtNP1icJ8P2tqBmoBqj1+qj0+BFHANCUFww/H8S5Ket+5po625/chuQPRfUT8j9LtbVSr24i5OBdJkqM376y8/hOA3++PBmVz585Fo1FWvgsXLgSguLiYurq6aAATG6sEm5FulsGIvL0tBQSVQNzSQjTpZiR3ENkbQpttJfnOiehHxEY7kFK0Gqy9MnSR8lFOgumU60wMJXweJWj5+MndPHvPej5/rpjOJjc6k5qZS/K49g/TKZqZdsyy/RFewgsNbfjDJUxBELhp1jAAdoVb8Scc0obrbFUe9xn6ntORqUrZpuQE/KxOBF+GBRhn9bK/kOWe63cg+4vjwXirEYMoKDwYd8/1OyUnDkFQStstx6CTMtTY00vAblgvyYkIXzA7/sQlJ+7MTiZOo6LM7YtmWgB+evYIjFoVNVblPrwlIyFq5eAPSlHLhe/OGnbadAXqRJFr05Rr46/VTfjC94JFr+HSiUom7W+flRIKB2r3VzQQlOHseCtzwqVKWZb540dK6ezSCemMTj/5vk5DhW/uyPg1QZ8fq0yoI+PRZVlRxxsQdYrAkSbJSMINI0m8fSyaTAuBgMTekDLJ5qyox/5xFe4dLThqu6kIEwZPZwE7t9vNs88+S1VVFRqNhmuuuYZJkyb12WbEiBFcd911aLVaDh48SG1tLUDUJfVICHX5aH91P0hgnJBE8qyM6Mor0pUBkJJs5sNkDb/DTUgl4DvQSfPjOwk0u6j3+qn0+BBlmVR7G01CJx4xwP4mBx0uP0atirEZMf3ee8uWLbhcLmJiYpgwYULPe6WkRMtfn3/+ebSFOqJSOdKsZEqKXZ4+3Rw9k2DfVbyoU5Fw4yj0I+OxLMwi8fYxqGzKILy1V4mxNyL1/hHfEINPWZJxdfloruqmfFsLO5bX8NHju3j2nnV8/lwxVbvaCAUkrIkGplw4jOsfmMmEc7JQH2cXx4WJNpK1alr8QT5s7bvytBjV+AzKfnsvHFodPrrDLuKNh/C2op1ITac+A9PsC1Dm9iHQ0+oMUNHqpNXhQ6sWh4zorxXFaDZ4o72n/dZm0EQVfjdVfX1ZmEjDw+RDJSfCBN6hWPBZ1Sp+NkzhUT1U1YQjbDuQaNFxzvxsZJMaMSixuBeZ+qM9DbSGtVEuGHP0meVTgdszEknQqCl1eflbVY9Fyx3z8zDr1GyvsfPMuko2dDr5tK0blaB4R0Xw2b5mNld3oNeI3LOo4Ov4CMeNMxyYkwBdro2kH4xjx45GvF0tGIMyqcVdOOkZaD/FTIVKJqbRjWzVnzYRfW9s2LCBlpYWzGYz11xzDWlpA4saDRs2jJtuuomXX34Zt9tNVlZWNFtxOMhBifZXSpCcATSpJmIuzUMQBJakxLKpy8Wyula+m5GATlTi7EsmpPHQp6XI1hB/lA2E2r20PL6TVd9RvF5GdklkCGZasVNaWsoOt3IMU4bF9etc8vl8rF+/HoB58+ZF3bIjWLBgAXv37qW6ujrqvxT5THkGPTpRwBWSqPH6GRb2XCoIlyGau310uPzEmXpWkCqrloQbRvY7B5+1Kfs+K75voLKiWBEuW1AwcOfU1w1Jkqk/0EnppiaaKrpwdvoIDaI+HJNsJG9SEsMnJhKfbh6Sa10rityYnsBDVU08U9fKZeGWUqNWzdwp6bwtBtEG5aj6LihpctGhlOtqfH5cwVC0yyXSllva7CAkyafUWDVCqB1tNvRp142UjyZnxw5pu+6MGDPrOp1s6HRyc3pPaXVaTjz7GrrZVNnOd8adeoE2ICpieWjGenedHRi6kvv1aQn8u66NCo+Pxw4286vwhF4Xo4EuH0Kti7e+quV784YjyzL/Xq+Qd2+YMey04yzGa9U8VJDBd/dW83hNC+cl2JhkM5EeY+C3FxXxi7f38LcVB0jVKtne61LjGRFeMPmDUpS4fNucXFJt36xy9en1TXyLIAgC+1OViW2s2UD8pXmYpqeizbYSVAtoESgKibQ/X0zrsj34ar5eDYpD4fV62bJFMeq68MILBw1eIkhLS+O73/0u48ePjxqCHQ4hV4C2F4sVJ2S9mvjrihDDvIcrU+JI0Wpo9AV4s6mn6+i66dlY9Gq+6HSyZ2Eaulwbsl9i1X6lzXRKR5BRE5TMT0lJCRsqBi8fbdq0CY/HQ3x8/IBaNTExMUybNg3oEbGLBDBqUYh2tvTmwZh1arLD5LejcabuDAT5qkuZvBb1Wu3VdrjZ3+RAJQqcVZh0xP2cSnQ2udj4XgUv/XoD/31kJ6Wbmuhq9RAKSggCmGN1pOTayJ+cxNSLc7j63qlc8/tpTPtOLgkZliEN1K9Pi0crCGzvdrO9l5hXVq7yPQU7vFS09jy+s9aOEJAwhDu097t6dZHFm9BrRLwBiYPtp9ZS4Mtw+XNm7CH8lyEuH0UwOA8mrAdT+fV1IkVKf+N7BTCyLLOnvkc9eSigEQV+Gw5altW1Uuf1U+L08GWXEwFQ1bj456pyOlx+th7sZG99Nzq1eERtlK8LFyTGsCQ5Fgm4a39N1CfsysmZLChIxJuo44DXj1kl8rOcnnb8lzcdpLrdTYJZx/fmfX2KuseLMxmYk4iIDsWERAvmvJ60452vbmfr7ib+mJ1Mfr0Hf1UXrU/swjAqHuuiYdEOpq8Tke6ihIQECgqOLq2YkJDA4sWLj7id72A3Ha+WEOryg1okfmkB6l4OwDpR5I6sRH5X3sA/a5q5OiUOtShg1Wu4aeYwHvuinEc3VfHfO2ax59MKVuiVCWBBaiyjpw5n9ZZ1VFRW8qXHAqiZO6JvFsPj8UTF6ubPnz9oq/fs2bPZvn07Xq8y0fXOKo00G9jt9LDP6eHCxJjo40UpVg62uylp7GZW3uFVOle2dxOSlfJRdi/n7Ej2ZXJ2LDFG7WAvPyFIIQmPM4DerEF1mBWlq8tH60EHrbUOqve001LdE5jpjGolszIhCVuSAVOs7rD7GmokajUsTo7hjaZO/l3fxsRwaaQyqJRtRbufFzdW84dLRgM9PIosjZpSKUixq6eLTCUKFCRb2FXXxf4mB7mnsHX9S3uY/9KrfCRJMpvChNoZQ0TgjWCC1YheFGgLBClz+6Kr8alhHkxFq4sWh5cky6nVg+kKBKnyKDy3sb0E7Bq7vLQ5/ahE4ZiNLA+HRQlWZsSY2Gh38efKRnThrNv5CTaaYh0UN3b38U66bGIGsaaTcz8OBf6Yn876Tgflbh9/rmrkvrx0BEHgd5eO5rONSpZlSlAV7XLrcgd49IsyAP7n3BGYdd+8cOCbd8TfIOwcYDUBsLu+iwZkxIVZpCRZ6F5Rg3t7M5597XiK29EXxCHoVIoeiCiEf4OgVSGaNKjMGkSz0pIrmjSoLFoEzeATR7Ddg7fMjq+8E19lF5I3hKAWENQiqEQEjYigFlDHGdBmWxDTjWzauAmAWbNmIYZLOJIvSKDRpfy0uBFUIqJJg2hSozJqEM3K8ajjDVEtk96QZRnnunq6Pq0GSUadYCDu2iK0qf2dkK9Li+cfB5up9vj5b6s9WiK4eVYOz6yrYm99N2vKW/lXpoi/Q2C2pGbhwlxUOjWJiYm0traSLnQxauyYaIdJBBs2bMDn85GUlNTHWfZQGI1GZs+ezeeffw70cGCgN5G3byfSyDQrn+5rorjhyBmYT8NdA+cl9F1VRnxvzhk5NBLdsiRjb3HTctBBy8HuaEAS9CtZE6NNhzlWhzlWjzlOh1oj0lbnpPWgA3d3X10UQRTIGhVH4fRUho2NP24Oy1DhloxE3mjq5L8tdu4dnkayTsP28MJB6ArwVn0dP1tUgFmrZle4DDHOZqK0s6sfCbswxaoEMI3dp4znUO/1U+XxIwLTewUwB1oU/pZBMzB/60SgE0UmWU18aXey0e6MBjAxRi2FKVZKGrvZXNUxqA/OyULE8DZLryWuVyltd51yn4xItgxpKU0QBH43PJ3zth3greZOtOHs4PcyEwleaOHaZ77i5U097t3fDRPET1fEaNT8X2EW1+2uZFltK+cn2JgeY+Y9ezeSXgWeIFs2NlBcmMnINCuPfVGG3R2gINnClZMzv+7DPy6cCWBOEnySREl4gBzfazXR5Q5Q3a4MsGPTbahNWuKuGIFlbjpdn1bjLenAu//YSXSiSY3KplN+YpTfoQ4v3nI7oY7+XQWyX0b29+UsBFs8ePd3sF9Vj1PjxCToyaw00F5cQqDBSbD96LoTBJ0K3TAr2hwbulwb2nQzsi9Ex5sH8JYon80wLpHYy/IQB4n6TSoVt2ck8ueqJv5xsJnFSTGIgkCcScu107J4Zn0Vf9haTUm6Dq0g8NCMfFThfcm2dGhtJVdj5zcX9gg1ybLMl19+ybp16wAl+xIJzgbDtGnT2L9/Pzqdro8yb4TIu2+QTqQjGQP6JIlVYc+nKQEVNfvaQQC3P0TDATuZkshEg4HGii5ElYBKLSCKIqJKCWh97gAeZwCvw4/bEcDr9ONxBvB7QgR8wT6/Pa4AwUGUbWUZXHYfLrtCwD0UggCxqSYSMy0k51gZPjEJo/X0WYWOsxiZajOxucvFiw1t3JSeQK3XjwCM0Gmo6PDx5tY65o1IxOENoteIzEqy8kZnFyWHBJ+FERL2KVTkjbRPj7UY+3ShRfkvw2IP60dzvJgRY+ZLu5MNdic39uHBxFHSqPBgTnUAE8lYj7P0XfDtqbcDyng51BhvNXJ5cixvNXfil2XGWhRxOCHGzIKCRFaVKnYCc/ITyE8+/Qn1Z8dbWZoax38aO/jJ/hpeHzecx2qUEvtEt0BxQOZ/3tzFY0sn8MLGagB+dWHRKeV8DSXOBDDHiP2bGmmu7MbrDuBzBfC6gvjcyu9QQMJg0WCK0dGUpCEwTMYqC3h2deAZFY/BomV3+GbMijP2SUdqkk0k3DgKf60Df003sgTIsqJ3Ev6RfCEkV4BQWEBPcgYIufwQlJFcQSRXUBFIOxQqAW2WBX1eLLr8GNQ2HXJQQg7Jyu+ghOyXCDS58Fbb2V2+EYDR/ky8W1r77sqqRZNmRpNiRJZQjsMdPh5XgJDDj+wL4S3txFuq8FcErYigUSG5AqBWtFFMU1OOyIe4OT2Bx2taKHV5+ayti/PDpZrb5ubywlc1lMQpA/4PspLINSolmE6XnzcrZBYAmaouYvXK4B8IBPjggw+iKsFTpkzp45I9GDQaDbfeemu/x8dYjKgExRG5zusnI2wwOC5coy9tdtDp8g+acv6y04krJGELQMnDe+ntj3tFWKdh4zMlRzy+o4VaI5KYZSEx20JStpWkbAu2RAMeZwBnhw9npxdnpw9HpyLjH59mJjHLQkKGGY3u682yHAm3ZChiXi/Ut1NkUjJjeUYdt0zP4Vfv7uGFDdVY9MpQNybdxthwRrTY5UGW5eh1OFAb/MlGhP8y+1D+SziAGeryUQSH8mAi52B6bjzPb6j+WvRgdoX1X8Za+hJJIxmYoeK/HIr/zU3lw1Y7XknmtozE6Ln41QVFrDnQiiSfPsJ1R4P78tJZ2+Gg2uPn/G0HcIUkxluMPDcpi/OK2ylp7OaKpzYQCMnMHZHIvBGnZ6PA0eBMAHOMqNnbTtnWlkGfd3b6cHb62KrSwTATiU0+Vq4tQRAgZbiNRpNATEiIGskdCm2mBW3m0Uf6siwje4IE7T5C3X5Cdh+hLuVHNGrQ5cWgy7EhHsUkpM+LoSbWTne5G71Oz4zvzIdmPyqTBk2qCU2qCZX58KtvWZIJNLrwVXXhq+zCX92l6J/4JVTxeuKvKUKbfnT8AptGzXczEvnHwWYeOdjMeQmKemqyVU/uzFR26WX0AbmPG+pfl5dy0KPBa9Cjl7xUVFSQkZHBa6+9Rn19PYIgcP755zN16tSjOobBYFWrmGgxsaXbxeoOB9eFtRiSrHoKki2UNjv4sqJtwFVsMBDixW11oIHhB72o1AJxqSZkGeo73Tg8ivNwnFGLFJKQQjJSSCYU/lsOyeiMavQWLQazBoNFi8GiwWDWoDWo0erVaPQqtDo1GoMKnUGNLdGAOAA/xWTTYbLpSM755thZHIoLEmJI1TXQ6Avw56pGQPE/Wjw8jT9/UkJNh5unViveWOMzYxhu1KERBLqDEnW+AJnh4DPSiVTX6aHbG8Cq7+sFs73bxUetXSxNjesjOripsp3Vpa1cNz2LjNhj468Nxn/5KtzKPNQE3ggmWo3oRIEWf5AKjy/6eabmKGXSshYnbU7fUWmuuEMSbzV18H6LnctTYlmaqhyz0xfkjS21fLqviZtnDuP8I5TlogReS18C794wgXfsSQpgMvRanh41jL0OD5cm9fDc8pMtPHzVeJq6vMw/TbsBB4JVreLhwiyu3FVBR0DJvN6Xl0ayVc8Di8fww1e30+kOIArw6wuOvIg7nXEmgDlG5IxPxJZsRG/UoDep0Zk06E0adEY1KrWI2+HH3eVnU2cb4GOcyUBCJrTVOmksV27E29Aj7HSyVjygvNagTDi68OSj0ohIkpJ9if4OyegMamJTTX1S+IIgIBg1aI0aOGSulCWZrlYPdXvaoryH1loHoYCEWqtCo1Wh1qnQaEU0ehV5k5L4skRpLZ46bSoxE49dUlwQBbTpZrTpZiyz05ElmWCLm2CHF91w26Alo8FwW0Yiy2pb2OXwsKbTwfw4K+VuL/sMMsgQ2tNBWVE34zJj2FVr5z+bawCBoqIiqop3sHHjRux2O93d3ej1eq688spBDSaPFfPjLGzpdrGqozsawICSbi5tVnxFDg1gaos7WPVaKV9O04FGZJZKx9LfjiUm2YgvGGLS/Z/j1AR59weTvhEqzacDNKLATWkJPFjVGBUGnGA1YtSquXpqFsvWVlLZpmQmx2XGoBVF8o06il1eSpyeaAATY9SSatPT2OXlQJODycOUyfyAy8tfqhr5KKw383JDO8+NzsHmCfHQp6WsOaBkKd/aVsezN00+as7KQY+POm8AtUDUowiU8mOXJ4BZp2bMSRIV06tEJlqNbLS72Gh3RgOYOJOWwhQL+5scbK7qOCwXqMHr57n6Nl5uaKczrKWypctFFiq+2NbAG1tqcfiUtvXihm4mZseSbB2YGNwZCFITNnMd0ysDU9fpodMdQKNSrDpOFhYl2Pp0AkbwddsqHC/mxlm4KT2B5+vbuDDRxrRwgHzh2FQ+3ZfGB7sauGpK1kk9p6cCZwKYY0T+5MMTK61hifKDX7WDGxbPyuLcS2x0t3uo3t3GG+8dIMkHqq4Ae1bVHdcxGCwa4lJNxKWa0MVKaPRqxKAOjyOAx6FwIdzdfuxNLvzegbkPQb+El0Cfx6qqquiKa0StVkdbiE8UgiigSTGhSelP1D0aJGjVXJcWz7/q2nikupl5sRZ+daCOoAxpPmhv9fL4qnKevG4Sv31/L7IMl01IZ8H0OKqKd1BTU6PsJyGBpUuXEh8/dCva+XEW/lrdxPpOJ15/EEeTh/YGJ/lNQc5xa/BvbGOluwQkGVkGt8NPbXEHDbEqHEYDBgTuvmEMhjD3YVNlB05fkCSLjnFDTNz8tuO6tHj+frAJX1hxNKIjcv30bJ5ZV0nEeSLiQD3SbKDY5aXY6eHcXhNXYYqFxi4vJU0OUlPM/K26idcbO5BQNCfS9VpqvX4u31GGak8nqkYPalHJCtbbPVz19Cb+ec0EFhYdmYAdKR9NsJj6uC5vCrcxTxkWe1JVmGfEmNlod7Gh08n1aYfwYDpd/LmhhZclN4laDYlaNQkaNWJAYvnuRio0Mk1GETlcBU4QRYyiSE0wyBVf7keztR0ByE00IQoC5S1O/vBhMY9fM3HAY9kdLh/lGLTE9CLwRtqnC1Os6NSndynzdMMDeemcFWfpo+4M8LcrxnLx2FTmfYOySoPhTABzEuAMhigLy3RHvFis8QaSJyXy8me7MejhjUsm4Gh04/cG8XuC+L2h8O8gwYCEKApRwqYY7kTyOPx0t3vxOALUO+zUlrXRkbgZkIltm4xK6p/uValF4jPMJGVZFA5ElgWdUU3AFyLgDxH0hQj6JVoOdrNig8IPidNkoVGdmFz3UOKOzCSer29nU5eLe8vrWdvpRCcKPDw6k5vW1LO8uJn7Pyxmd10XFp2a/72gkASTFovFgsPhIC8vj8svvxy9/sTaQv3eYLhEqPBF/B0eTAboIsR9f9hARlswuu141OCH/Rsa++xDEMA+Kx4IclaiNRq8AHwebp9eWJR8zNL6/78jXqvmsuRY/tPYgV4UolyYzDgjZxcls7y4mQSzLuogXGQ2QHNn1AE6gsJUK1+Ut/FCp53ffGWPBkTnJ9j4YWoC73xZzQseD6FkA9LYOApzZJbNyCfBouMHr2xnXVkbt724lfsuGc3107MPe8wRAu+p5r9EMDPGzN9pZqPd1YcHE5NpwadL4oBW5kDHAITmGAFQthU7fKiqnThavXQbVDArGSleT/74JH4zIZu5+YmUNHVz8WPr+Wh3I1dObh2QcxEpH421HCpgd3L5L99mqEWhT3AegU6t4txRKQO84puHIQ9gfv/733Pffff1eaygoID9+xWaotfr5X/+53947bXX8Pl8LFq0iCeeeILk5J4VS01NDXfccQerVq3CbDZz44038uCDD/ZTSz1dsdvhQQbSdZo+zrK7IjoUKRbGzDw+hn/AF6KzyUVHg4u9+/bQXqdMmqphTYwdNguDOcyHsGixJhiITTUelTaHJt5HYJsdZAgeTOC9v+/gwh+OxWQ7uYGMx+nH5w4qAZU3hN+r/C1LMrEpJmJTjaTptVyVEsfLje38q04x6ftxVjLzMuK4cEQSu/a2sW1VLVNkNedkJFDy4UF87iDp4iRcBjvWjmGser4MlUZErRGjv9VaFRqdKvxb+V8Kybi7/EopsNun/N3tx2X34XMH+x1/1gwTJVk6KpI1DHdDfLqZ+DQTH5e1UN3hZn5BIlNz45VSnwiZRXFcUV8HrmCf9mlZlnu1T59e4nXfFHw/M4n/ttg5J96KplcAeMf84aw+0Mr5o3uI4yPDrcOHdiLFJBrwT09kr14GCabbTPxmeBoGV5DvP7uF2g4PaiBjegoHbSr2WASe6OjkwfgMnr1pCr9+dw9vbK3jt+/tpa7TzS8WFQ4YjMqyHM3A9F4hB0MSm6P8l8PrCJ0oJllNaAWBJn+AKo+fHIOWJ2tb+VtXJ+hUCI4Avx+ThUeQWVHeyo4WB2hV6E0aRpv1jAuqUGk0tKVqaLP4sLsDCH6RHQaZlkwDE3LjEEWBUWk2bpqZw7NfVnHv+3v57Cdz+7VDRwKYwTqQTlYp7Qy+2TgpEcGoUaOi2hlAn8Djpz/9KR999BFvvvkmNpuNH/3oR1x22WV8+eWXAIRCIS688EJSUlLYsGEDjY2N3HDDDWg0Gv70pz+djMMdcuwYTP8lvJoYl3n8N6NGpwp3kVjZUb06+nir+yA5M88hIyPjuPYbOf/5wwvxuS201jh46y9buehH44hP60sw9HQrk7osy4gqMZotUtp9RYxW7YA6MBF4nH4ObG6mdFMTrTWHb1kVRIGYZCPjso28mgWSAMkhgZyPmni+tpyiLj9F9ARZgb1d7O1l2QBWahm6jgqtXoU5Tq/opsTpOTdJpAQ33VPjuOVHI6IT5IF1Gl77qASdLsj3F/WsxA96fBQf8KIS6ON0u7e+m8YuLwaNalDn7DM4PApMenbPHIX+kIB9QlYs2397DsZek2ZEx6fC7cMTkjCoRD5qtfMXpx3ZrEH0hXhhUh5nJ1h5Z3s9v3p3D76gRGacgb9cNpaZeQk8U9fKb8vqeamhnQZvgGWjs/nLkrFkxBr5+4oDPL2mkvpOD3+/cny/VuhKj48mfwCtIDCpl+HkvoZuHL4gVr2akWknl1htCPNgNnW5WN7WxZYwSRkgpsOPZ1sbluRk1u9tZF9ZG1rghhnZ/OackYO2dvskiYVbSsNiak08OEIZj+4+dwQf72nkYLubJ1aVc/e5fcUxewKYHv6LLMs9GZgzAcwZDICTEsCo1WpSUvqnqLq6uvj3v//Nq6++yllnnQXAc889R1FREZs2bWL69OksX76c4uJiPv/8c5KTkxk/fjz3338/v/jFL/j973+PVjtwF4zP58Pn80X/j/jXDDXe3F7Lh61djFdridFrsOg1mHVqrHo1KTY9uYnmqJ7B+ENWExEhraEQpvL5fFRUKJ0VmZmZ1NbW8sknn3DLLbccUdvkUOzatYvi4mIAzl60AP35Fj78527szW7eeWgbGUVxUa0QV5dfae0+DHRGNYlZFpKHWUkapgRbBquGg3vaKd3URPWeNqRQzz40OhUavZIN0erVaHQqZFmmo9GFzxWks9EFjS4m+o1sz9WxcF03Tc092RC/XqRBCjB5RAKpiSZ0BjU6oxqdUYNaIxIMSISCEqGARDAQIhiQCPolguESWsAfIuBT/hdEAaNVi9GmxWjVYrJqMVi1mGxKwKIz9L1lRnj9/GNjMbvdHrqCoWj9fk5+IlDCV1XteAOh6Ipzedj7aKrN1Eesa0U4+zJ3RMKQinX9/wbTIDyJQ1VGE7Vq4jVq2gNBSpwePm7r4p9hvQxVpw/1zg5yJhZw7/v7oi7ECwoSeeSqCdiMSlb11oxE0nUaflB8kJUd3TxY2cgD+RncuTCf9BgDv3h7Nx/ubiQvycxPzh7R5/0j2ZdJNiOGXgFXxD5gak78KdHmmBFjZlOXi99XNACgEQTuz0+nYmsjL0kyv3p3DwB6jcifLxvL4gmHJ7XqRJE/j8jg8p0VPF/fxpUpcUywGjHr1Pzu4pHc8cp2nlxTwSUT0hkeVjtu8wep8yp8vN4lpIPtbhzeIFq1yIhvgAbLGZx6nJQApqysjLS0NPR6PTNmzODBBx8kKyuLbdu2EQgEOPvss6PbFhYWkpWVxcaNG5k+fTobN25kzJgxfUpKixYt4o477mDfvn19XIN748EHH+xXujoZ+F1rGx1agZV7W1HXu6OPxwsuslSd/HDpRT0KvIe0A0YzMEMQwJSVlREMBomNjeXKK6/kscceo76+nj179kSdlI8En8/Hxx9/zK5duwD6nPclP5/Ex0/uprG8i8odfbVgBAEMVi2iKERbfKWQhCTJhAISPneQuv2d1O3v8TFSqcU+Zn+JWRYKZ6SSPyUJwyCt2bKslHPa6py01zsZXufAvi9AamEy8QvNJGSYiUszodKq8AZCmL4GKex0vZZ8o44yt491nU4uTooBYESymRSrnqZuL1uqO8IBDXw2iPpuxD7gnJHfjtr06YaDu3eSmD0Moy0GULr3Rpr1rOt0cuPeKlr9SkD8vcxEtuyppNgvsfRfm2h1KIuiuxbmc9fC/H7loPMTY3hmtMi1uyt5ob6d2zISyTboWDIpA5Uo8JPXd7JsbSXXTssm0dKTKVwf5r/Miuk7MZ8q/ksEM2PMPHxQufbSdRr+NXoYE60mPmoP8NJGhQCfk2Diyesm9lO0HgyzYy1RcbhflNbyyeQRqASB80anRMXhfvveXl65dRqCILA7PF4ON+iw9ApAd4cJvEWp1pMi5ncG33wM+Yg/bdo0nn/+eQoKCmhsbOS+++5jzpw57N27l6amJrRaLTExMX1ek5ycTFOTYgPe1NTUJ3iJPB95bjD88pe/5O67747+393dTWbm0Msjz7eZecfjwlIYy4J4Gy5vEKfHR17rbgz4eG3FGmrHKa7DvQWZDra76fIE0KrEIWldKylRRM5GjhyJxWJhzpw5rFy5ks8//5zCwkJ0usNzVxobG3nrrbdob29HEATmzZvH3Llzo8/rTRouuWsCpZubCPolzDE6TOEfo1UzoKYIQCgk0VHvouVgN83V3bRUO+hocBIKShisWgqmJlM4I5X4o9CCEQQh+p7Zow8/oH8dwUsEC+KslLlbWd3RHQ1gBEFgTn4Cb26rY11ZG3PyE7EHgmwcxLyxpLEbUeC0M2/8NqB65zbefvB3ZI+dwOW/vj/6+EiTgXWdTlr9QYwqkb8XZLI4OZb/SWmjuKGbVocPq17NI1eP56zCwbuKFsZbmR9rYXWngwcrG3lq1DAALhmfxnNfVrGrrotHV5Zx/2LFk0mWZTYMIGAXCElsqT65+i+HYmqMidkxZmwaFQ+NyCReq9xH8woSmZAVw7B4E/ddMqqfJs6R8Lu8NFa0d7Pb6eG5+jZuDQvE3fed0Wx4eA0bKtp5f2cDiyekRwOYcYeU3KP6L2fKR2cwCIZ81D///POjf48dO5Zp06aRnZ3NG2+8gcFw8qy6dTrdESftocCfJuXwyYZiOtUS15yfz+xYC1u3buXDD5WVmldQOB3DDTpsvUoEkfJRUdqJryYCgQBlZYoJV0RNdvr06Wzfvp3Ozk7Wr1/PwoULB3ytLMts3ryZ5cuXEwqFsFgsLFmyhGHDhvXbVqURGTnr2MjGKpUY7XYaNUdJNwd8IbrbPcQmGwcNfL7JmB9nYVldK6s7HH26OeaMSOTNbXWsPdDKry4o4osOByFZ4WoM62XeuLIkYt4YR9xpbBb3TUXNPqW7rmbPLjxOBwazsoCYHmPi6bpWcg06nh0zjMJw59K0nDje3l5HUaqVp66bSHb8kSUAfjM8lTVbHbzXYuf7mW7GW40IgsD/nl/E0n9t4j+ba/ju7BxyEkyUur20BYIYRCHapQiwu86O2x8i1qiJiuqdbOhEkbcm5PV73KxT8+4PZh33fhO1Gn6Vm8ovDtTx58pGLkqMIUWnISveyJ0L8/nrZ6U88FExCwqSogq84/op8NqBMx1IZzA4TvpsEhMTw4gRIygvLyclJQW/34/dbu+zTXNzc5Qzk5KSQnNzc7/nI8993YjRqLkqVRG4WlbbSjAYjHrryAjYrUqadTAC7/ghuBkrKirw+/1YrVbS05UgQaPRcO655wKKWWFnZ2e/1zU2NvLqq6/yySefEAqFGDFiBN///vcHDF6GEhqdivg087cyeAHFhE8nCtT7ApS5e3hYs/MSEATY3+Sgpds7qHnjiiE2bzyDvmiqUIJ9WZao3rkt+vh5CTY+mzyCz6cURIMXgMsnZfD+D2fx3g9nHlXwAjDaYmRJ2HD0gYoG5LAB4Izh8SwoSCQoyfz1M6UTM8J/mWIzoevFV4uUj6bnxn8r2uivT4tnotWIMyTxx8qG6OO3zcklL8lMm9PPP1aWDdiBJEkye+sVvtjJUuA9g28+TvqM4nQ6qaioIDU1lUmTJqHRaFi5cmX0+dLSUmpqapgxYwYAM2bMYM+ePbS09Mj1r1ixAqvVysiRI0/24R4Vbs1QukRWtHfz0bYddHV1YTabGTttLq0WZRDLPYRMuHsICbyR8lFRUVEfP6HCwkJycnIIhUIsX74cUDIuZWVlvPDCCzz99NOUlZWhUqk477zzWLp0KSbT8QnMnUEPjCqRaWEl1dUdPeTxOJM22j2x8kArX7Qrzy1K6OESdHkCUd+Zs88EMEMOWZJoriyP/l+xbXP0b0EQGGcxYjwksBZFxerjWIXTfp6TglYQWG93Ro06AX5xfiGCAB/vaWJ9dTuvNyrfd2/+S6fLz3s7lUn+VPFfTjZEQeCP+UoX0jvNndR4lOBeqxb57UXKWP7KjjoafAEEYIy5J4isanfh9Cnmm3mJR2c9cgb//2HIA5if/exnrFmzhurqajZs2MCll16KSqVi6dKl2Gw2brnlFu6++25WrVrFtm3buPnmm5kxYwbTp08H4Nxzz2XkyJFcf/317Nq1i88++4zf/OY3/PCHPzwlJaKjQZ5Rz8I4KzLwdLXCy5kzZw4XL5xFSziAqdl5ILp9MCRFFSVPpIUaIBgMUlpaCtDPjFAQBM477zwEQaCkpISVK1fyxBNP8Morr1BVVYUgCIwePZrbb7+d6dOnH9FM8QyOHvPjlKBk9SHCX3PD5N0XGtpwhiSStOo+5O5/ra0kKMnkJZnJSTgTTA41Opsa8Ht6yPbVu7YRCvbX8xkKZBl03Bxe3DxQ0UAonIUpTLGyZGIGskbg5pKD7HZ6sKhEFifHANDi8HL1sk2UtziJM2k571siMgaKkOe8WAshGZ6s7WkGmJufwJh0G26jMgXlGXV9Osj2hDPWo9JsJ1WN+Ay+2RjyK6Ouro6lS5dSUFDAlVdeSXx8PJs2bSIxURnIH374YS666CKWLFnC3LlzSUlJ4Z133om+XqVS8eGHH6JSqZgxYwbXXXcdN9xwA3/4wx+G+lBPCLdnKp9nd3wqmphYJk6cSKsEHq0OQZaQ92/HE+5sKGtx4g1ImHVqchNObDVRXV2N1+vFZDKRlZXV7/nk5GQmT54MwLp162htbUWr1TJjxgzuuusuLr/88n4k6TM4cSyIU1bTG+1OvKGebqs5+QlINg07w7HJT4elIIYDx3VlrTy+WskO3LUw/9Qe8EmGLEkEA4Ejb3iSESkfpeYVYLBY8blcNBwYOpfvQ3FXdjJWtUixy8vbzT1l3Bvn5xKYkohDL2IWBN6ekEe2QUe93cOVT22ktNlBkkXH67dPJ2kQv6BvKn6crRDT/9PYTqtfuSYEQeCO+cORbQrna5TpUP7LGf2XMzgyhpzE+9prrx32eb1ez+OPP87jjz8+6DbZ2dl8/PHHQ31oQ4oZZh3xHiftBjPuKbPRaDTsDHN7Yl0O4iQn//l8M9+9YGa0fDQ63XrCte2IXkthYeGgei8LFiygvLycUCjEtGnTmDRp0gnL6J/B4VFo0pOi1dDkD7C5y8XccEAzPN1KcFw8iAJzzEZuCps+tnR7+clrO5FlWDo1i4vHHZ8y8+mGoN/P7pWfsfm9N0AQuPZPf8cSd2Rhvj1fLEdUqRg1b2Dy+fGiqULJhKbkjyA2LZ3itV9QuX0LmSPHDOn7RBCnUfPjrGT+WNnIXyob+U5iDB2BIN+vqEOyaMAXIq3Kzai5BqraXFz3zFfU2z2kxxh49bZpR825+SZhVoyZCRYjOxxunqlr45e5ikHkolEp6MvrcAL+tr6KyGcUeM/gaHAmN3ec2L59O6NqlNXdZ4KeoCRHBezSwkJve7dvRpZldkUVeGNO6D0lSYpaMhyOD2Q0Grnzzju5++67mTVr1png5RRAEATmhYOWVWEejCzL/PxAHZJBheAOMt0pIAgCIUnmrtd20u7yU5hi4XcXnx7crhNBKBhg14qP+fdPbmfV80/jsnfi6uxg3asvHPG1VTu3sfzpR/n0iYc5uHvnkB5XJAOTkptP7sSpQF8ezMnArRmJpOk01PsCPFDZwCU7yqnw+EjTaojfZafmYDd//ayUK5/eSL3dQ26iibfumDFo8LJ39ed8/u8nCfi8Az5/ukMQBO4MZ2Geq2/FEXauVokCYqySgdm2pxlf+PHQGQLvGRwlzgQwxwG/38+6desY0VKLVZCp8wX4rL0rKmD3nYJhSLKANWjns83F0QzMiQrY1dTU4Ha70ev1R+wcOsNvOfWYHw5gIjyYf9e38XFbFypAs6uDzQcUH6dHV5axsbIdo1bF49dO/EYr74aCQfZ8sZxnf/I9Pn/mCZztbZjjE5i+5GoQBErWrTpsySbo9/PFs09F/1/xzD+HbKIOBYO0VlUCkDw8n2HjJiKqVHQ21NHZWD8k7zEQDCqRe3IUHsszdW3UehWfof9OyucnM3IAeGpNBa0OH4UpFt743gxSbQNLTHS3trBi2T/ZtfwjNr71n5N2zCcbixJs5Bt1dAclnq9X7oMmX4BuWQZZxt7k5t3tyndS0erEEwhh1KrIPUPgPYPD4EwAcxzYunUrLpeLBKuVGzOUlcXTta3RdsCz0pMgTuGnrFy9lv2NyoR2oquJ3uUjleqbO+l9WzE31oIAlLi8LG/r4r5ypavkrrRExO4A22s6WVHczKNfKFmBP106Jiqn/k3FBw//meVPP0p3awum2DjOuvl73PLIMmZdeR2j5yuK26ueX4YsSQO+fst/38be3Ig5Ng5zfAJdzU1sePPVITm29roaggE/WoORuNR0dEYjGUWKmFzl9q1D8h6D4cqUOArDhpEFJj3vTcgnQ6/l+hnZUUfscZkxvHb7dBLMgzcnfPXuG0ghhUu39cN3aamuPKnHfbIgCgI/ylK4d8vqWvGGpKiAXbKgQgjJPL22kpAkRwm8o9NsQ26nUL+/mK/ee/OkEbnP4NTiTABzjPD7/axfvx6AuXPn8t3MRNQCbO5y0R2U0IsCBSYD31m0AACjuxmj7CHepI0OXMcDSZL6tE+fwemHeK06qr58y95qArLMBQk27hmRRlackUBI5gevbEOW4arJmUf0lTnd0VZTTcXWTQiiyLzrb+GWR//FhPMuRh32K5t99Q1oDQaaKsooXreq3+vtzU1sfu9NAObdcCtn33IHANs+eq9P6/PxIlI+Ss7NQwjzxSJlpMrtJ7eMpBIEXhyTw73D03h3Qh7JOkXJVq9R8cJ3p/KbC4t45dZpxBgHFy7samlm7+oVACQNG44sSSx/+jEkKXRSj/1k4bLkWNJ1Glr9QV5v6ohmrGcnWokxaqhqc/HJ3sZox+ZQC9jJksSHjz7E+v+8wM7PPhzSfZ/B14MzAcwxYsuWLbjdbmJjYxk3bhypOi3fSYqNPj/abEAjCkwuHIbLkIQgwChVE2MzbCdU1qmvr8fhcKDVasnNzR2Kj3IGJwELwu3UAVkmU6/l4cJMBEFg7v9r7z7Do6rWBgw/M5NJ752QRugkgdAJHYEggggooqAgFqzH3lBE8BwOlk+s2PWgKArYKAJKr6GF0AKEJCQESCe9TV3fjyEDQxKSQCqsm2uuGfZeu7zZk8w7a6/SwdSQVWcQdPRxYu640KY8zXpxeON6ANr27EuvsRNQW1vWJDi4utFv4j0A7Fi62KI7sxCCLYu/RK/TEhgeQcfIQbTt2ZcOkYMufVAbru+DOrOi/UvbSz28Qnr2BuDciWNoSkuua/8VTu7axk+vPVcp6Qq0s+GJQG+LSTsB2nk78vCgkEoTTF5pz+/LMBoMBIZHMOHVN7GxdyDzdAKx61bXy3k3NrVSweOBphrrRalZHCwwvR96uDowPTIYgM+3Jl026W39JjDn449TfMF0+2rvH8st3o9SyyQTmDoQQpjHYBkyZIj5Nk7FwHZgOQLvwAGmobjbqXII976+IeIral86dOiAWl23eUmkxlPRnVqtUPBlaJB5OomhHUx/uO3UKhZN7Y6ddcu+BajTlHPiYq1KtxG3Vluu++hxuPq2oiQ/j71/LDcvTzqwl9MH96NUWTH8wcfMyf0tD8zExsGBrJQkYtauvK5zzKgigXHz9cPdzx+jwUDK4djr2j+Ykpe1n7xPRlICe35fdt37q5CfkU7cto0A9J80FUc3dwbfNwOAncuWUJCVebXNm60prTxwV6tILdeyNc90az3CyZ4H+gdjb60iLq2Q2NR8oP57IJ3cvcP8uqyokJi/ru/9JTU9mcDUgUKh4IEHHmDSpEmEh1/qhtnD2YF+F0dijXS91KZh3ICuFKpcUCkEjheufeyJ3Nxcjh41TWsvbx81b31cHHi7gz9Lurahh/OlXiW3dPJm9pjO/PBQH9p5N848Nw0pfvcONKUluHj7ENS16hniAazUaoZOexgw3RrKz0hHV17O5sVfAtDr9gm4+/mbyzu4ujHk/ocA2L38J/Izq5/A9Wr0Wi05Z1MA8G3bwWJdSM/6uY10au8u1n76PkIYL+5vP2VFhTVsVTt7fl+GMBoJ7taD1h1Nv/Phw6Lw7xyGXqNh47efmacraEnsVUoe8fcy/1+lgC6Odrg5WHNvn0vjWjnZWBFcj13KjQYDp/aYbv2HDRsJwIE1v1NaWFBvx5Aan0xg6kipVBIaGlqpEe03YW1YHNaG2y6b50ahUDD97jsASDsdT0pKSp2PFx8fz1dffUVRURHOzs60b39jDXh2o1EoFDzQ2tM8Mm8FpVLBw4NC6B3s3kRnVr+ObDLdPgq/ZZS5fUl1Qnr0Iahrdwx6Pdt+/JY9fyyjKCcbZy9v+k2cXKl82NCRBIR2Ra/VsOHrT6/pgzor5TRGgwE7J2ecPL0s1oX0MN1GSo49cM3tSZJi9vLXR+8ijEZChwzHO7gtRoOek7u21bjt6dj9rPnoXXLOnqlyfV5GGsd3bAZMtS8VFEolI2c+hcrKipRDMZzcvb3K7cuLi+vt9lhDmNHaE4eLo+t2crDF7uLrhwe1Qa0y1cSF1sOYWZc7G3eUssICbJ2cGfHwE3gHt0VbVsb+Vb/V2zGkxicTmHriaW3FrV6V27l06xhCz549AVi7di2GWt7XNxqNbN68mZ9//pny8nJat27NQw89hLW1nK1YalrZZ5JJT4hHqVKZv81ejUKhYNj0R1AolSTu32P+0Bg2fSZqm8pjFCkUCkY+8iRWamtSjx7i+PbNdT7HzNOXbh9d+Tvp16EzNg4OlBUVkp5wqqrNryr5UAyrFy7AaDDQacAQoh57mtAhtwAQt+3q56rTlLNu0QfE797O0tkvkrB3d6Uye377BWE00iaiJ63ad7RY5+7nT9+LSd+WxV9RVlSIEIKc1BT2/rGcn994iUUP38t3zz5KSX7lCV2bA1e1FQ+0Nt127+Nyqca6lYsdd/Yw1cb1qedEvyLZ69C3PyorNQPvuR+AQ+vXUJSbU6/HkhqPTGAawfDhw7GzsyMrK4t9+2quti4pKeGnn35i+3bTL13v3r2ZMWMGLi5yUCep6VU03m3Xqx8Orm41lDbx8A8kImoMYOoNEtKjN2179a22vFur1vS7614Ati35ts4NLs09kK64fQSgsrKiTYRpuo263kY6c/QQK//vPxj0ejr0HcDoJ59HqVTRaeBQlCoVmacTuHAutdrtj23dSPnF20y68jJWLfwvu5YtMXczz007x4kdWwHL2pfL9bnjLjz8AykrLOC3/87hm389zPcvPcXOX34wjbkjBKUF+Wz/8bs6xdaYXm3Tii+6BPFKG8t5n+aOC+WjeyJ4bGjbejuWQa8jYd8uADr1HwxAcERPWncKRa/Tsue3q48eLzVfMoFpBPb29owYcXFMjC1bKCoqqrbsuXPn+Oqrr0hKSsLKyooJEyYwZswYrKzqfdYHSaozXfmlxrtdR4yu07aRk6bg4OqG2taOYQ88WmOvvF5jJ+DWyo+yokJi19et22tVDXgvV3Eb6fTB/bXe57mTcfz57r8x6HS07dWX255+CeXFW8n2zi606W5KiuK2bapye6PBQMyaPwAYNv0Retxmur285/dl/Pnev9GUlphqX4QpwfNtVzn5AlBZqYl69F+gUJB5OpHC7Eys1NaE9OjNiIefYPzLb4BCwfEdWzh34lit42tMaqWC8T5uuF7RQ0uh09Am5yhWBm29HSvlcCyakhIcXN1o3dnU+0+hUJhrYY5t2UBeRlq9HU9qPDKBaSTdu3endevWaLVa/vnnn0rrhRDs2bOH7777joKCAtzd3XnkkUfo1q1bE5ytVKEoN4efXnuOpW+8SGFOds0b3OBO7t6OtqwUV59WBIZ1rdO2do5OTHv3E2Z88DmuPjXPuKyysiLyTlMtzIHVv6MprV0tjLaslNy0c0D1CUxwRE8UCiU5qSkUZmfVuM/sM8n8+c5b6LUa2kT0ZOyzr6K64ktF6GDTPE4ndmypsm1Nwr7dFGRlYuvkTPjwUQyb/gijn3welVrN6YP7+fHVZzm5y1TrWl3tSwW/Dp0Z9ejTRIway/iX3+CJb5cy4ZU36TbyNtr27EvXW0YBsOm7L667O3pjiv71JzZ8/Sl/f/Fxve0zPtrU+6hD5ECUykttF/07h9EmoidGg4Hdy3+qt+NJjUcmMI1EqVQyZoypCv3o0aMkJyeb15WVlbFs2TLWr1+P0WikU6dOzJw5U84a3cQKc7JZPncWGUkJpJ86yc9vvEh2akpTn1aTOrJxHQDhw2tuvFsVexfXWk3uWKHjgMG4+/lTXlLMwXW16/aaeToRhMDJw6vaW1x2jk74XezdkxSz96r7K8jK4LcFb6IpLaF1py7c/sJrWFUxlEGbHr2xdXSiOC+X1CvmdBJCmNv+dB811tz2p8vgW7hn3rs4eniSn5mOEEba9uqHT0i7GuMMGzaS4Q8+RtuefSu1JRp47zRsHZ3ISU1pUYO2VdSIndqzk/Mnj1/3/nRaDYn79wCXbh9dbsA90wBTYp59JrnSeql5kwlMI/Lz86N3b1PVdUWD3nPnzvHll19y8uRJlEolo0ePZvLkyXICxiZWkJXJ8nmvkp+Zjou3Dx7+gRTnXuCXOS+TeuxIU59ek8hMTiIjKQGlyso8TUBDUypVRF5sCxOz5k/KS4pr3Kam20cVKtrgbP/xfxzesLbK3k6lF9uZlOTl4hkQxPiX5lQasK+ClVpNpwFDAIi7ouHx2bgjZJ5OxMrahohRYyzW+bZtz/0LPiSoa3fsnF0YOPm+GmOsiZ2TM4PunQ7AruU/NdsGvZfLz0gnL/3SrZytS76pdgqK2kqOPYCuvAwnTy9ate9Uab1Pm7Z0iBwEQrBz2ZLrOpbU+GQC08huueUW7O3tyc7O5ueff+a7774jPz8fNzc3HnroIfr27SsnYmxi+ZkZLJv3KgVZmbj6tuLuN9/mnnnv0rpTKNqyUn5fMKdW3WVvNBW1L+37RGLv4tpox+0QORAP/0A0pSUcrMXgdhkXR8StqRYjYuRtBHXtjl6nZeM3n7Hy/+ZbjAtiutZzyUtPw9nLmztfewtbx6vPXRU6xHQbKXFftEVX5oral7BhI7F3rtwY397Flbte/zePffkDnoHBNcZYG2G3jMQnpD3astJm3aC3QsrhgwB4BgajtrUjI/HUdf+exV+8JdcxclC1f1cH3H0fCqWS0zH7OBt3c345aalkAtPI7OzsGDnS1PU0MTERo9FIly5dePTRR2ndumXPjXMjyEs/z7J5r1KUk41bq9bc/eYCnD29sHV05K7X/02HfgMx6PX89fF7HFj9e4scTOxaaMtKObHT9GFS18a710upVNF/0hQAYv5aSVlx9Y3gATKTTF2jrxzA7kpqW1vunDWPodMeRmVlRdKBPfzw8r84c/QQBr2Ole//l8zTCdg5OXPna//G0d2jxnP1CWmHh38gep2W+GjTwGlZKadJOXwQhUJJr7Hja4y1viiVKtP8Us28QW+F5MMxgOlWT9/xkwDY/vP31zw7ubaslNOxB8z7rI67X2u6DjeNJr3puy/kRI8tiExgmkC3bt1o164dVlZW3HbbbUyaNEneMmpCBr2O0sIC0k6dZPm8WRRfyMG9dQCT575t0V7Dytqasc+8bO49su3H79i25JtGT2LKi4uvu2q9rk7u2o6uvAy3Vq0JCA2veYN61r5Pf7wCg9GWlRKz5s9qy5UWFpiH2fdpW3M7EoVSSc8x45kyfyHufv6U5OXy6/w3+GnWc6QePYTaxpaJr87F3a92Xy4UCgVdBpvGhDm+3dQb6cDq3wFTTZKLd82Nl+uTb7sOhN8SBdStQW9+ZgbfPTuTxS88wY6li0lPiG/Q95xBr+PsxVuzwRE96THmDpw8vSi+kHPNQ/4nxexDr9Xg6tsK7zZX75Y98J5p2Dm7cOFcaq1q+aTmQfbNbQJKpZIpU6ZgNBpl9+hGlpVymoNrV5GRdApNSTHlpSXoNRqLMp4BQUx6Y36Vt0kUSiXDpj+Ck4cn25Z8S8xfK1GqrBg05YFGufV3fMcW1i1aiFdAEP3uvIf2ffpfU2Paukg7dcLcPqDr8FFNcotToVQSefdUVv3ffA6uW0WP28ZVeSumYkJFt1Z+2Dpc/XbP5byDQ7jv7Q/ZtuRbDm9YR3ZqCkqVFeNefL3a7szV6TJoGDt//oHzJ4+TeuyweRC13rdPrNN+6svAe6aRsHe3qUHvP3/RY/S4q5bXlpXy57tvmdujXDiXyr6Vv+Lg5k7bHn1o27svgWERVTZkvlbnT55ApynH3sUV76A2KJRKBk15gLUfv8e+P1cQNmwkjm51G9yu4ufeqf/gGt+zto6ODJ46g78//5Ddvy6lY//BOF8xgrPU/MgamCaiVCpl8tJIhBAkxx5gxX9ms+SVp4nbtpEL51Ipzsu1SF6s7ewJ6tqdSXP+W2Mbj15jJzBy5lOAqX3Dnt8bfjCssqJCtiz+CoQgOzWF1R+8zfcvPcXJXduueUj8mpzYtY3lb71GWWEB3sFtCR9e/cSNDa1dr354B7dFV17GgYvjqVwp4+LtI5+Quk+5obaxZcTDTzLuxdcJ6BLO2OdeIfgq8zxVx9Hdg6CuEQCsXrgAYTQSGNatVj2LGoK9swsDL/a22fHz95y5oofU5YTRyNpPF3LhXCoOrm5EPfY0HSIHYW1nR0leLkc2reePt+fx+SNTWbdoIacP7seg1133OaZcvH0U3K2HOSHv1H8wrdp3RKcpZ9eyH+u0v/LiYlIOmdrUdLzK7aMKcRfiSPArwrN9O/QaDVt/+LqOEUhNQX6CSjcsvU7HiR1biPnrT/PoqAqlkg59BxA6ZDj2rm7YOjhgY++Itb1dndsfdB1+K7py0x+73ct/Qm1jS6+xExoiFAB2/vwD5cVFeAYE0b5vfw6uXcWFc6n89fF77P71Z/pNuJtOA4fUSzsKIQTRv/5M9K9LAWjbqx+3/esFrG3trnvf10qhUND/7qn8+e5bHFq/hl5jxldKNC/1QKpbrcnl2veOpH3vyOs5VboMGU7K4YPmXlO9x915Xfu7XuHDo0g6sIfkQzH88c5cxjzzMu379K9UbvevS0k6sAeVlRXjXngdvw6dCB8WhV6n49zxoyQe2EvSgT0U517g+PbNHN++GRsHB9r1jqRj5CCCwiPMg/vVRcqhiwlMRE/zMoVCwdBpD/PzGy9xbOsGut86Fu/gkFrtL3F/NEaDHs+AIDwDgqotV64v5+PYj/nx+I8IBK4+au5I9CNh7242bV3O0MF3oqrh90kIQXHuBS6cP0te2jk8/AMJDLMcvyujxDQpqa9D495CvNHJBEZqdoxGA3lp53FvHXDNtyuE0chv898wN1y0trMj/JYoeoy+A2cv73o7155j7rj4DXEJ25Z8i9rGhm4jb6u3/VfISDzFkc1/AzD8ocfx7xxGj9vuIHb9ag7+tZK8tHOsW7SQpIP7GfOvF6/pQ6SCXqvl7y8+MvcA6XX7RAZNmV6vDUyvVUiP3vi2bU9GUgLrP/uA9v0G4NE6AHe/AGwdHc23kGrqQt3Q2vXuh7WdPdqyUryC2lx1xu7GoFSqGPfibNZ+8h4Je3ezeuHbjHr8GXOvKYD46J3mYfVHzvwXfh0udTu2UqsJ7taD4G49GD7jUdJOnSQ+egen9uykJD+PuK0bidu6Ea/AYEY9/mydapuKcy+YxldSKAgKj7BY59ehMx0jBxEfvYNtS77hrtnza/ybUFqQz76VvwKm3kfVOZJ9hNd3vk5KYYqprFtHEhWJxAUXEpbszPbvv+Wt859wS8gIZobPpJVjK8DUXuf4ji2cizvKhfPnyE07h668zLxfhVLJ/W9/hFdQGzJLMvk49mNWJ61GIAj1CCUqOIqRQSMJcAqo9c+oIRVqC0kuSCa5IJnc8lz6+PYh1CO0RfSGVYgbtBtFYWEhLi4uFBQU4OzsXPMGUrMgjEZWvj+fpAN7CQzrxshHnsLVt1Wd93Nk099s+OoT1LZ2RN51L12Hj8LG3qEBztj0DWznz9+b/mgqFIx+4jlzQ876YDQaWPr6i2SeTqDLoGGMfuoFi/Wa0lIO/b2G3SuWYjTo6dh/MLc99cI1JTGlhQWsfO8/pJ06gVKlYvhDT9B1+Kj6CqVeJB+K4fcFb1Zabu/iSmlBPgqFkn8tXo66iRvGb//pf+xf/Tt3vDibdleZ96kxGQ0GNnz9Kce2bABg2AMz6TF6HFkpp/l5zkvoNRp6jhnP0GkP125/RgNpJ09wMnoH8bu2UV5SjEKppM8dk+h35z21aidzbMsG/v7iI3zbdWDq/IWV1hdkZfK/5x/DoNNxy4OP0X3U2Gr3VVZcxIp5s8hOTcHRw5P7/vtBpcEMtQYtnx/+nO+OfYdRGPG282Zu/7kM8h9EgaaAbac3c/K971GV6Iltn8/h9gVYK62Z0uEeBhV04PCqlZVGb1Yolbj6+oEwkpeehk+79hTc1Y7FcYspN5h6USkVSoziUkPozu6diQqO4rY2t+Hn6Ffjz+l6CSFILUolNiuWYznHOF1wmuSCZHLKKk9m2dqxNaOCRzEqeBSd3TubkxmdUUdiXiJHc45yLOcYR3OOMn/gfLp4dKnXc63t57dMYKRmJfq3ny2G9baytqH/pCn0HDO+1h/I5cXFfPfsTMqKChk67WF6jhnfQGd7iRCCLYu/Inb9ahQKJWOffZkO/QbWy76PbFzPhq8/xdrOngc//LLa0WUTD+y9OEuynk4DhjD6qefrVGtSWljA8nmzuHAuFRsHB8Y9/1qlqvDmIj56J+dPxnHh/Flyz5+lOPeCeV2r9h2Z8p/3m/DsTIxGA+XFxVU2Nm5KQgi2LfnG3Lun97g7iY/eQWF2FkFduzPx1bnXlvwW5LPpuy84tcfUfdzDP5BRjz9DiYcKa5U1rRxaoVZVTmhWf/gOp6J34DeyPwW93UkpTMHNxo1WDq1o5diKVg6tyPgnmsMrTecbMWoMQ6c9UmkqB01pCSv+PZvM0wk4uLoxee7buLUy9R4r15eTUphCUn4S3x77loQ8063GsSFjebXPq7jYWF6j+OgdrPnwHRRWKhLucCU9IZ5uiS44l5rO397Vla7DR+Md1Ab31gG4+vqislJTeCGbb56bidDoiA69QHxQMd29u/NSr5do5diKzamb+efMP+zP2G9OZpQKJcMChjG181R6+fSqt5oPg9HA0ZyjHMw6yKGsQxzOPkxueW6VZb3tvWnj0gY7Kzv2pu+lTH+pRinQKZBevr1ILkjmxIUT5oSswuy+s5ncaXK9nHMFmcA0UAJTlJuDpqQEYTRiNBgwGg3m12pbO7yDQ1pE1VtzlHwoht/fngtCMPCeaaQeO2Qe9da7TVuiHn0anxq6Q4Kpu+ihv9fg4R/I/e98XOkPXUMRRiP/fPUJx7ZswEptzT3/fq9W53s1pYUF/O+5xygvLjJ/W76ahP3RrPngbYwGA10GDWPUE8/WKonRlJay4t+vk3k6AUd3D+6a/R88WjePKu7a0JaVknv+HPlZGfh16NwgPUiEEORr8nG2dq6xXURzJIQw/20SQrDn918sviy4tfJjyn8W1jhYX01O7d3Fpm8/p7QgH6GAY20KONQ+H2GlxMfeh9aOrfF38sfTzpPkvNP4/C8Ra52CvyIzyHbTVL1TAT1PexAebzo3n44dmfjiHHNyqCsv59f/ziEt/jjWjg60fngsiVbp5lsjacVpCC591LnbujOn3xyGBw2v+nBC8Ov8N0g9egilygqjwTQ2TJm1gaMhBeR3diCq3a2U6ErI1+RToCkgX5NPTlkO3id19Dvujk4tCH/5YW4LH1/pMyG3PJfNqZtZl7yOfRmXZkTv6NaRqZ2nclvIbdioqh7xuSbxufGsOb2GtafXklVmWVNkrbQm1DOUCK8I2ru1p41LG4Kdg3FQO5CZlMD5+OPYuruSZJPF1oJotp/fgcZgeU2c1E6EeYYR5hlGuGc4Ed4RuNnWblb62pIJTAMlMGs+fMc8OVhVvAKD6Tl2Ap0GDEZlVX/dDG90BVkZ/Pjqs5SXFNN1xK2MfOQphBDEbd3ItiXfmqume42dwIDJ91X7s80+k8ySV55BCCN3zf5PpXvqDc1oNPDnu/8mOfYAzl4+3LfgA+ycrv39989Xn3B00994BQZz39sf1eqbccLe3az+8G2E0UjokOFEPfb0VZMYnVbD7wve5NzxY9g6OXPP3Hfw8G85yUtjiMuJY2HMQvZl7MNJ7USEdwQ9fHrQ06cnoR6hWKus6+U4GoOGzJJMMkoyyCjNMD2XZKAxaPB18KWVQyv8HPzwdTS9trOqvlG13qjnZO5J9mfsZ3/Gfg5mHcRB7cCkDpO4q8NdeNp5cnDdarYs/hJrO3um/Of9ernuOWU5LNr9AVlrdhOSZrpte95bw4aeGXDFdzuvPGvGRLdCY2Vkz50quniF0s61HYWaQtJL0k2P4nSyy7IRCAIy7Rh02BNrvRKNgwKPe4cQ3qEfBz7/Dl1yFjq1YF2fdHJdKveMcrFxIcQlhFCPUB7p+gjutlfvlp2bdp4fXnoSg16PrZMzvcZOIK2Dgs+Of2lulFsVJysnJh9oiz4tjw6Rg7j92Veuepyk/CR+OvETq5NWm2s33KzdCPcOx0HtgIPaAUe1I/ZqexzVjqaHtaN5uaO1KanbkrqFNafXkJifeOlcrJ3o49uHCK8IIrwj6OLRxeK9WlpYwIkdWzm25R9yzp6xOC9rOzvc/QPRuFuR72IgKDyCHh0HEOQchFLRsB2YZQLTQAnMhq8+JWHfbpQqFQqlEqVKhVJpel2Um2Puluvo5k730ePoOuJWi/EojEYDxRcuUJCdiVJlhV+HTjd9jY1Oq+GXOS+TlZyEb9v2TJ73rsW985L8PDYv/opTFxPHwPAIxj3/Gjb29hb7EUKwbO6rnD8ZR4e+A7j9+VmNGkeF8uJifnztWQoyMwju1oMJr755TQ1g0xPiWfrGiyAEk+e9g3+n0Fpve2rPTtZ89K4piRk6gqiZ/6oy+THo9ax6fz6nD+7H2s6Ou+csaLLuvs3RuaJzfBz7MeuS11VbxlppTVevrgwLGEZUcFS1PU2EEBzLOca6lHUcyT5Cmb6Mcn055YZyNAYN5frySt92a+Jk7YSztbPpYeNsfp1VmsXBrIOU6Eqq3E6tVDMqeBRTOk3Bt9QRGwfH66610hq0/HjiR7468pX5uOONA/DYlIlBp6PHpEm4De7G+eLznCs6R3ZZNu4xhZRtP05In35MeGF2tfvWGXQkFSSx7ew2oo9uInBTAS6lavRKI3nOOrzybdCpjPzTJ5M8dyOdPTrT1asrbV3b0sa5DSGuITUmLFU5c/QQeelpdBk0FGs7098bjUHDb6d+I6UwBVcbV4uHi60Lwc7BFJ/L4KfXnkMYjUx49U1CuveutG9hNJJ4YA85Z89QkpdLXk4m5zOSKcnLxVoDWrWRAgcdRfZ6Chx0FDroKbLXIRSg1itRGxRY6ZWo9QqsDAqK7Q3kOmnR2SsZGjiUMW3GMMh/kEXCIoSgrKiQjMRTxG3dSOKBvebaJSu1NQFhXSnOyyX3XGqVIxK7tfIjuFtPgiN6ENAlvNIkovVFJjBN0AamvLiYwxvXEbt+NSV5pnuNals72nTvRXmRaYTQogs5FqNhegWH0Hf83bTvG9ksenk0NiEEf3/xEXFbN2Ln5Mx9b3+Is2fVvYQS9u1m3acL0WnK8Q5uy8RZcy3ag5zYtY21H7+HlbUNMz74vNr9NIbsM8ksnf0ieq2GfhMnM2Dy/XXa3tRw9wUyTycSOmQ4tz7xXJ3PIT56B399/B7CaMTBzZ0ug28hbOgI3P38zcdY9+lCTu7ahpXamjtfewv/LmF1Ps6NKL88ny+PfMkv8b+gN+pRoGBsyFgej3icQm0hBzMPmh5ZByu1K+ju3d3cANLD1oOE/ATWJ69nXfI6zhWfq/HYtipbfB188XHwwdfeF18HX2ytbMkoySCtOI30knTSitMo1ZfWuC8nayd6+fSil08vevr2JKUghaUnl3Ik+9KcP+Ge4YwJGUOYZxgd3Tpia1Xzh1J+eT4phSkkFySTUpjCmcIzHM05Slap6ZZFmEcYr/R5hQjvCHODeoVSyd1z/ot/50vvsaWvv0B6YjxRjz1N+LCoGo9b4Wx2Mqs+XEB5ommwPaNKgdOUAfTqNYIwz7BaxdDQti75lpg1f+Ds5c0D//eZRaPy8yePs+X7r8y95uqTtb093kEheAYG4+7XmtLCAvNEmfmZaWhKLJNan5D2hA0bSacBg81ftg16PXnp58k+k0x2agrpp06SduqExWeXSq3Gv3MYvW6feE3jJV2NTGCasBGvXqcjfvd2Dqz+vVK1HIBSZYWzlxcl+fnm7ndufv70ueMuOg8c2mhtNpqDI5vWs+GrT1EolNz5+ls13vLJSErg97fnUlZYgIuPL3fOmodbq9Zoy0r533OPUZyXy4DJ99NvYv02KrsWJ3ZsYe2npsakd7z0Rq16ogghSDl8kF3LlpB5OhEbewdmfPBFtQ13a3Jqz042fvMZZUWF5mV+HToTOnQEmacTOLJxPUqVijteml3lt8SbTWphKsvjl/N7wu8U6UxzLkW2iuS5ns/R2aNzpfJCCFIKU9idtpu/U/4mNivWvE6BAl8HX9JL0s3L7KzsGBowlKH+Q3G1dcVWZYutlS22KltsrGxwVDvibO1cY62sEIJCbSEXyi5QqC2kUFtIgabA/NpJ7UQv3160d21fZXuduJw4lp5cyrrkdeiMl263qBQq2rm2I9QzlC7uXVAqlWSVZpFVmkVmSSaZpaZHkbbq+ai87bx5tuezjAkZY77NIIRg3aKFnNixBUc3d+5/52PsXVwpKyrks0emghDM/HyxxbQdtWE0Gti9fCkJ+3ZzywOPmgcPbC605WUsfuEJinKy6T3uTgZPnUFhTjY7li42D1FgbWdPh34DcHT3xNHNHUd3dxxc3bF3caW8uIi89PPkpaddfD5PQVYmCoUCta0tals7rG3tsLa1RWmlpiAzndy0c7WaLsLJ04v2vSMJGzYSr6A2tYpHU1pKatxhUmJjSD4cQ1FONgC3P/dqvXVYqCATmGbQC0kIwZkjsWQmJ+Hk4YmzlzcuXj44urmjUCopKy4idt0qYtetNg945ezlTZfBw7Gxt8dKbY1KrTY9rNQYL87ZU1pYQGlBAaWF+ZQVFqC2tcOnTVu827TFJ6Qdrt6+VQ4vL4RAr9Wg1+kQBsPFRshGhNH07OTuiZV1/dzLr4muvJyT0dvZ9M1nGPR6Bt473TyBW03yMtL47b9zKMjMwM7ZhYmvvMmpvbvYv+o3XHx8eeD/Pmu0OGqy+X9fErt+NdZ29ty34ANzr4iqnDtxjJ2/LOH8yTjAVHs36rFn6Bh5fX8cDHodp2P2c2zrBpJjYxCXdeVEoWDM0y9ddbK7G53BaGD7ue0si1/GrrRd5uUd3TryfM/n6d+68oBv1ckoyeCflH/4O+VvjuSYajmsldYM8h/ErW1uZXDrwdir7WvYS+O5UHaBlUkricmM4VjOsWp7qVTF18GXYOdg08MlmDbObeju073Kdjna8jJ+mvUcuWnnTD2dZs3lVPRO/vr4PTwDg5n+3qf1GVazkRSzjz/ffQuFUkn3UWM5sulv9FoNKBSE3xLFwMn31+vM7nqdjtzzZ801J/kZaTi4uOHq2wrXVn64+bTCxbcVautrayBcQQhB7vlzpByOIXToiDpN21EbMoFpBglMbWnLSjn0z1pi/vqT0oL8696ftZ093m1CsLF3pLy4yPQoKaa8uAiDrvphv63U1vh16kJQeASBYd3wbhNivq0ljEbyM9PJSjlNZnISF86ewcbBERdvX1x9fHHx8cXV2xcHV7dq5+YRRiPnThwjbttmTu3dZa59atc7knEvvFantkAl+Xn8/vZcspKTUNvYYtDrMRr0jH95Dm179qnDT6thGfQ6lr/1Omnxx/HwD2TK/PdRqqww6LTotVr0Wg1FF3LY9+cKki+ORqpSq4mIGkOf8ZPqvQtucV4ux7dv5tjWjeRnpDHi4Seb3TgvDaFQW0heeR7F2mIKtYUU64op0haRVpzGqqRV5loSBQoGth7I5I6TGeQ/6LoaK54vPs/p/NNEeEfgZO1UX6E0GCEEmaWZxF2IIy4njhO5J1AqTD2HvO298bH3Mb/2c/SrcyKWc/YMP732PHqthv53T6UgM4O4bZvodftEhtz3YANF1fRWLfwvCXt3m//fulMowx6Yed09FG9kN0QCs2jRIt577z0yMjLo1q0bn3zyCX361O7DqTETmIySDPak7yG5IBkXGxfcbd0tHh52HrXqEqfTaojbuomMpFMYdDoMeh0GnQ69zvSsslJh5+yKvYsL9s6u2Du7YOfsTFlREVnJSWQmJ5J9JvmqScrlFEolSqUShUoFAtM3g8vYOjjSunMYmpJislKS0JaVVbOnS1RqNQ6ubtg7u2Dv4ordxWeEID56J4XZmeayrj6tCB06gp5j7rimxmDaslJWLVzAmSOmavs23Xsx8dW5dd5PQyvOy+XHV5+hJD/vquWUKhVhw0bSb+I9OHnUrTq9roQQ6DWaJh/srSHklOVw4sIJTuSe4PiF45y4cIK0krSrbuNq48qE9hOY1GFSsxkh9UYUt20T6z/7AIVCiZWNDbryMia9Mb/ZjjdUH4pyc1g6+0UUCgWDp86gY+Sgm77jRk1afAKzbNkypk2bxhdffEHfvn358MMPWbFiBfHx8Xh719w4syETmGJtMQcyDxCdFk10ejTJBclXLW+lsCLMM4zevr3p7dubCO+Iq3Z/vB4GvZ7c82fJTE7CoNNh6+iEraMjto5O2Dk6YePgiNrWBoVCafFLZKoSPMuZo4dJPXaIs3FH0ZZZNhJUqdV4BQbj3aYtnoHBaMvKKMjKoCAzg4KsDApzshFG45WnZMHazp6OkQMJHTICv46dr/sX2aDXsWXx16QnxHP787Nw9Wmec42cOxnHb/PnWCaJCgVWamvUNja0iehJ5F1TrmnU4easoq1Gdmk22WXZ5JTlkFeeR4G2wNRmQ1Nofq1AgYuNi/nhauOKi40LOoOOnLIcLpRfIKcsh5yyHHLLc9EatICp1qSie64QgmJdcZXnYm9lj5O1k8XD2dqZSL9IRgWPuuZxN6S6Wf/5h8Rt3QiYJtB84tuf63Vm6+bIoNebeq7KxKVWWnwC07dvX3r37s2nn5rujRqNRgICAvjXv/7Fq6++WuP2DZXAvLbjNdYmr8UgLjWUUiqUhHqEEuoRSqm+lAvlF8gtyyVPk0duWS5ao9ZiH2qlmnDPcMI9w3GwdsBOZYedlR12atOzwWgguyzb/Ec/uyybnNIcyg3lqJVqrFXWWCutTc8qawQCnUGH1qBFa9Sang1aU2MvpRoblQ3WKmvztmqlGpVChUqpsngWCAxGAwZhQKfXYpVZhk1GOdirMXo7oPRwxMba1nxslVKFEiVKhemhEECRBmWpHmWZHkWpHkq1iFItCq0Bx3YBeIZ3wtbOARuVDTYqG9Sqi+dy8aFUKE3PSiVGoxGDMGAUl54FAgUKc1nzsS//w3DZO9ogDOiMOrRGLTqDDp1Rh86gw4gRtVKNldIKK6WV+bVKoTJ/ICpQmPZ98dNRIKj4dREX/1Wco96oR2fUoTfqMQiD+edoFEbzQ1tWCnqjOWlRWV065uX7FEKYB91SKpTmn7FKeenno7hyQI1qzg8wL6v4+VWMAFrxf/PxhGk7vVFfuWGoppByQ7l5XIqKMSgc1A4oFAryy/PJ0+RZPFckHJc3Em0MChQEuwTT2b0zXTy60MWjCx3dO+Js3bxvJd8sdJpylr7+AjlnzxDSozcTXqk8LYR0c6vt53ez7O6i1WqJiYlh1qxL43golUpGjBhBdHR0ldtoNBo0mkvfbgsLC6ssd70crR0xCAOBToFE+kXSr1U/evv2rjQUdQUhBOeLz5sHk9qXsY/M0kwOZpm6YDZ7FXcyii8+6srm4gOgCNh9lbLSDcvFxgUvOy887Dxwt3XH1cYVZ2tnc21LRXJRMappxcim+Zp81Eq1eVtPO0887DzwsPXA1srWItmrSMJ8HHxwUDfMvFfS9VPb2DL+5TfY+8dyut96e1OfjtSCNcsEJicnB4PBgI+Pj8VyHx8fTp48WeU2CxYsYN68eQ1+btNDpzM9dDqtHavvTXI5hUKBv5M//k7+TGg/ASEEZ4vOsi9jH8kFyZTpyyo9lAolXnZeeNl74WXnhaedJ172Xthb2VeqZdEYNCgUCosamYrXAmEud/l2FbUERmFEb9SbazgUKFApVVgprCxqZ/RGPRqDxny8itcVNQsV3+orHpfXSBiMl15X1IJcvg+NQWPexiBMvaH0Qm8e9tyiVuZiTYtRGBFCYBCGS8+XVbtU1E4oFAqUKFGr1KiVaosaKKVCaa4xufz5ylqJimfzPi+7XVFRE2SltDL/3CpqdK4874oHgF7oMRgv/vwvvq7Yn0Jx8XFZrU9VNVHVvt8ui/3y/1f87CqOoeRSrZV5+cV1KoXKYmC0igTDWmVNmb6MEl2JxcNgNOBq64qbrRtuNm642lx8betmfv/W10i10o3BxduXqEefburTkFq4ZpnAXItZs2bx/PPPm/9fWFhIQED9N8arbeJSHYVCQaBzIIHOgfV0RpIkSZJ082mWCYynpycqlYrMzEyL5ZmZmfj6Vt1I08bGBhsb2QhPkiRJkm4GDTsj0zWytramZ8+ebNq0ybzMaDSyadMmIiMjm/DMJEmSJElqDpplDQzA888/z/Tp0+nVqxd9+vThww8/pKSkhBkzZjT1qUmSJEmS1MSabQIzefJksrOzmTNnDhkZGURERLB+/fpKDXslSZIkSbr5NNtxYK5XS5pKQJIkSZIkk9p+fjfLNjCSJEmSJElXIxMYSZIkSZJaHJnASJIkSZLU4sgERpIkSZKkFkcmMJIkSZIktTgygZEkSZIkqcWRCYwkSZIkSS2OTGAkSZIkSWpxZAIjSZIkSVKL02ynErheFQMMFxYWNvGZSJIkSZJUWxWf2zVNFHDDJjBFRUUABAQENPGZSJIkSZJUV0VFRbi4uFS7/oadC8loNJKWloaTkxMKhaLe9ltYWEhAQABnz569KeZYupnilbHeuG6meGWsN66bJV4hBEVFRfj5+aFUVt/S5YatgVEqlfj7+zfY/p2dnW/oN9CVbqZ4Zaw3rpspXhnrjetmiPdqNS8VZCNeSZIkSZJaHJnASJIkSZLU4sgEpo5sbGx48803sbGxaepTaRQ3U7wy1hvXzRSvjPXGdbPFW5MbthGvJEmSJEk3LlkDI0mSJElSiyMTGEmSJEmSWhyZwEiSJEmS1OLIBEaSJEmSpBbnpkxgtm/fzu23346fnx8KhYI///zTYn1mZiYPPPAAfn5+2Nvbc+utt5KQkGBRZujQoSgUCovHY489ZlEmNTWVMWPGYG9vj7e3Ny+99BJ6vb6hw6ukPuIFiI6O5pZbbsHBwQFnZ2cGDx5MWVmZeX1ubi5Tp07F2dkZV1dXHnroIYqLixs6PAvXG2tKSkql61rxWLFihblcc7i29XFdMzIyuP/++/H19cXBwYEePXrw22+/WZRpDtcV6ifepKQkJkyYgJeXF87Oztx9991kZmZalGkO8S5YsIDevXvj5OSEt7c348ePJz4+3qJMeXk5Tz75JB4eHjg6OnLnnXdWiqU279OtW7fSo0cPbGxsaNeuHYsXL27o8CzUV6xPP/00PXv2xMbGhoiIiCqPdeTIEQYNGoStrS0BAQG8++67DRVWleoj1sOHD3PvvfcSEBCAnZ0dnTt35qOPPqp0rKa+ro3hpkxgSkpK6NatG4sWLaq0TgjB+PHjOX36NCtXriQ2NpagoCBGjBhBSUmJRdlHHnmE9PR08+PyXwaDwcCYMWPQarXs3r2b77//nsWLFzNnzpwGj+9K9RFvdHQ0t956K1FRUezbt4/9+/fz1FNPWQzzPHXqVOLi4tiwYQNr1qxh+/btzJw5s1FirHC9sQYEBFhc0/T0dObNm4ejoyOjR48Gms+1rY/rOm3aNOLj41m1ahVHjx5l4sSJ3H333cTGxprLNIfrCtcfb0lJCVFRUSgUCjZv3syuXbvQarXcfvvtGI1G876aQ7zbtm3jySefZM+ePWzYsAGdTkdUVJTFtXvuuedYvXo1K1asYNu2baSlpTFx4kTz+tq8T5OTkxkzZgzDhg3j0KFDPPvsszz88MP8/fffLSrWCg8++CCTJ0+u8jiFhYVERUURFBRETEwM7733HnPnzuWrr75qsNiuVB+xxsTE4O3tzY8//khcXByvv/46s2bN4tNPPzWXaQ7XtVGImxwg/vjjD/P/4+PjBSCOHTtmXmYwGISXl5f4+uuvzcuGDBkinnnmmWr3u3btWqFUKkVGRoZ52eeffy6cnZ2FRqOp1xjq4lrj7du3r5g9e3a1+z1+/LgAxP79+83L1q1bJxQKhTh//nz9BlFL1xrrlSIiIsSDDz5o/n9zvLbXGquDg4P44YcfLPbl7u5uLtMcr6sQ1xbv33//LZRKpSgoKDCXyc/PFwqFQmzYsEEI0XzjzcrKEoDYtm2bEMJ03mq1WqxYscJc5sSJEwIQ0dHRQojavU9ffvllERoaanGsyZMni1GjRjV0SNW6llgv9+abb4pu3bpVWv7ZZ58JNzc3i9/RV155RXTs2LH+g6il6421whNPPCGGDRtm/n9zvK4N4aasgbkajUYDgK2trXmZUqnExsaGnTt3WpT96aef8PT0JCwsjFmzZlFaWmpeFx0dTXh4OD4+PuZlo0aNorCwkLi4uAaOovZqE29WVhZ79+7F29ub/v374+Pjw5AhQyx+HtHR0bi6utKrVy/zshEjRqBUKtm7d28jRXN1dbm2FWJiYjh06BAPPfSQeVlLuLa1jbV///4sW7aM3NxcjEYjv/zyC+Xl5QwdOhRoGdcVahevRqNBoVBYDAJma2uLUqk0l2mu8RYUFADg7u4OmN6XOp2OESNGmMt06tSJwMBAoqOjgdq9T6Ojoy32UVGmYh9N4VpirY3o6GgGDx6MtbW1edmoUaOIj48nLy+vns6+buor1oKCAvM+oHle14YgE5grVLxZZs2aRV5eHlqtlnfeeYdz586Rnp5uLjdlyhR+/PFHtmzZwqxZs1iyZAn33XefeX1GRobFHw7A/P+MjIzGCaYWahPv6dOnAZg7dy6PPPII69evp0ePHgwfPtzcxiAjIwNvb2+LfVtZWeHu7t5s4q3ttb3ct99+S+fOnenfv795WUu4trWNdfny5eh0Ojw8PLCxseHRRx/ljz/+oF27dkDLuK5Qu3j79euHg4MDr7zyCqWlpZSUlPDiiy9iMBjMZZpjvEajkWeffZYBAwYQFhYGmM7T2toaV1dXi7I+Pj7m86zN+7S6MoWFhRbt2xrLtcZaG83t97a+Yt29ezfLli2zuM3Z3K5rQ5EJzBXUajW///47p06dwt3dHXt7e7Zs2cLo0aMt2nvMnDmTUaNGER4eztSpU/nhhx/4448/SEpKasKzr7vaxFvRPuDRRx9lxowZdO/enQ8++ICOHTvy3XffNeXp10ltr22FsrIyli5dalH70lLUNtY33niD/Px8Nm7cyIEDB3j++ee5++67OXr0aBOefd3VJl4vLy9WrFjB6tWrcXR0xMXFhfz8fHr06FHl9W8unnzySY4dO8Yvv/zS1KfS4GSsdXPs2DHuuOMO3nzzTaKiourx7FoGq6Y+geaoZ8+eHDp0iIKCArRaLV5eXvTt29eiWvlKffv2BSAxMZG2bdvi6+vLvn37LMpUtCT39fVtuJO/BjXF26pVKwC6dOlisV3nzp1JTU0FTDFlZWVZrNfr9eTm5jareOtybX/99VdKS0uZNm2axfKWcm1rijUpKYlPP/2UY8eOERoaCkC3bt3YsWMHixYt4osvvmgx1xVqd22joqJISkoiJycHKysrXF1d8fX1JSQkBGh+7+OnnnrK3JDY39/fvNzX1xetVkt+fr7Ft/XMzEzzedbmferr61upN09mZibOzs7Y2dk1REjVup5Ya6O6WCvWNab6iPX48eMMHz6cmTNnMnv2bIt1zem6NqTm+7WjGXBxccHLy4uEhAQOHDjAHXfcUW3ZQ4cOAZc+7CMjIzl69KjFH8MNGzbg7OxcKRFoLqqLNzg4GD8/v0rd/U6dOkVQUBBgijc/P5+YmBjz+s2bN2M0Gs3JXXNSm2v77bffMm7cOLy8vCyWt7RrW12sFW22rqx9UKlU5lq3lnZdoXbX1tPTE1dXVzZv3kxWVhbjxo0Dmk+8Qgieeuop/vjjDzZv3kybNm0s1vfs2RO1Ws2mTZvMy+Lj40lNTSUyMhKo3fs0MjLSYh8VZSr20RjqI9baiIyMZPv27eh0OvOyDRs20LFjR9zc3K4/kFqor1jj4uIYNmwY06dPZ/78+ZWO0xyua6No4kbETaKoqEjExsaK2NhYAYiFCxeK2NhYcebMGSGEEMuXLxdbtmwRSUlJ4s8//xRBQUFi4sSJ5u0TExPFW2+9JQ4cOCCSk5PFypUrRUhIiBg8eLC5jF6vF2FhYSIqKkocOnRIrF+/Xnh5eYlZs2a1uHiFEOKDDz4Qzs7OYsWKFSIhIUHMnj1b2NraisTERHOZW2+9VXTv3l3s3btX7Ny5U7Rv317ce++9LS5WIYRISEgQCoVCrFu3rtK65nJtrzdWrVYr2rVrJwYNGiT27t0rEhMTxf/93/8JhUIh/vrrL3O55nBd6yNeIYT47rvvRHR0tEhMTBRLliwR7u7u4vnnn7co0xziffzxx4WLi4vYunWrSE9PNz9KS0vNZR577DERGBgoNm/eLA4cOCAiIyNFZGSkeX1t3qenT58W9vb24qWXXhInTpwQixYtEiqVSqxfv75FxSqE6Xc2NjZWPProo6JDhw7m90pFr6P8/Hzh4+Mj7r//fnHs2DHxyy+/CHt7e/Hll1+2qFiPHj0qvLy8xH333Wexj6ysLHOZ5nBdG8NNmcBs2bJFAJUe06dPF0II8dFHHwl/f3+hVqtFYGCgmD17tkXXu9TUVDF48GDh7u4ubGxsRLt27cRLL71k0T1TCCFSUlLE6NGjhZ2dnfD09BQvvPCC0Ol0jRmqEOL6462wYMEC4e/vL+zt7UVkZKTYsWOHxfoLFy6Ie++9Vzg6OgpnZ2cxY8YMUVRU1BghmtVXrLNmzRIBAQHCYDBUeZzmcG3rI9ZTp06JiRMnCm9vb2Fvby+6du1aqVt1c7iuQtRPvK+88orw8fERarVatG/fXrz//vvCaDRalGkO8VYVJyD+97//mcuUlZWJJ554Qri5uQl7e3sxYcIEkZ6ebrGf2rxPt2zZIiIiIoS1tbUICQmxOEZjqK9YhwwZUuV+kpOTzWUOHz4sBg4cKGxsbETr1q3F22+/3UhRmtRHrG+++WaV+wgKCrI4VlNf18agEEKI+qrNkSRJkiRJagyyDYwkSZIkSS2OTGAkSZIkSWpxZAIjSZIkSVKLIxMYSZIkSZJaHJnASJIkSZLU4sgERpIkSZKkFkcmMJIkSZIktTgygZEkSZIkqcWRCYwkSZIkSS2OTGAkSZIkSWpxZAIjSdJNxWAwmGfbliSp5ZIJjCRJTeaHH37Aw8MDjUZjsXz8+PHcf//9AKxcuZIePXpga2tLSEgI8+bNQ6/Xm8suXLiQ8PBwHBwcCAgI4IknnqC4uNi8fvHixbi6urJq1Sq6dOmCjY0NqampjROgJEkNRiYwkiQ1mUmTJmEwGFi1apV5WVZWFn/99RcPPvggO3bsYNq0aTzzzDMcP36cL7/8ksWLFzN//nxzeaVSyccff0xcXBzff/89mzdv5uWXX7Y4TmlpKe+88w7ffPMNcXFxeHt7N1qMkiQ1DDkbtSRJTeqJJ54gJSWFtWvXAqYalUWLFpGYmMjIkSMZPnw4s2bNMpf/8ccfefnll0lLS6tyf7/++iuPPfYYOTk5gKkGZsaMGRw6dIhu3bo1fECSJDUKmcBIktSkYmNj6d27N2fOnKF169Z07dqVSZMm8cYbb+Dl5UVxcTEqlcpc3mAwUF5eTklJCfb29mzcuJEFCxZw8uRJCgsL0ev1FusXL17Mo48+Snl5OQqFogkjlSSpPlk19QlIknRz6969O926deOHH34gKiqKuLg4/vrrLwCKi4uZN28eEydOrLSdra0tKSkpjB07lscff5z58+fj7u7Ozp07eeihh9Bqtdjb2wNgZ2cnkxdJusHIBEaSpCb38MMP8+GHH3L+/HlGjBhBQEAAAD169CA+Pp527dpVuV1MTAxGo5H3338fpdLUpG/58uWNdt6SJDUdmcBIktTkpkyZwosvvsjXX3/NDz/8YF4+Z84cxo4dS2BgIHfddRdKpZLDhw9z7Ngx/vOf/9CuXTt0Oh2ffPIJt99+O7t27eKLL75owkgkSWossheSJElNzsXFhTvvvBNHR0fGjx9vXj5q1CjWrFnDP//8Q+/evenXrx8ffPABQUFBAHTr1o2FCxfyzjvvEBYWxk8//cSCBQuaKApJkhqTbMQrSVKzMHz4cEJDQ/n444+b+lQkSWoBZAIjSVKTysvLY+vWrdx1110cP36cjh07NvUpSZLUAsg2MJIkNanu3buTl5fHO++8I5MXSZJqTdbASJIkSZLU4shGvJIkSZIktTgygZEkSZIkqcWRCYwkSZIkSS2OTGAkSZIkSWpxZAIjSZIkSVKLIxMYSZIkSZJaHJnASJIkSZLU4sgERpIkSZKkFuf/AaFoRR+6+Zl0AAAAAElFTkSuQmCC", - "text/plain": [ - "
    " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "timeseries.set_index('year').sort_index().plot.line()\n" - ] - }, - { - "cell_type": "markdown", - "id": "4b15e937", - "metadata": {}, - "source": [ - "### Downloading to Local Pandas (Optional Handoff)\n", - "\n", - "If you need to use local Python libraries that are not supported by BigFrames (such as custom plotting libraries or local ML frameworks), you can explicitly download the final transformed remote DataFrame into a standard local Pandas DataFrame using `.to_pandas()`:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "97757974", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    Time periodBeginning stocksProductionImports 2Total supply 3Food useSeed useFeed and residual useTotal domestic use 3Exports 2Total disappearance 3Ending stocks
    year
    1950-01-01 00:00:00+00:00MY Jun-May496.01019.011.01526.0580.0--109.0689.0345.01034.0492.0
    1951-01-01 00:00:00+00:00MY Jun-May492.0988.030.01510.0585.0--110.0695.0485.01180.0330.0
    1952-01-01 00:00:00+00:00MY Jun-May330.01306.024.01660.0578.0--78.0656.0332.0988.0672.0
    1953-01-01 00:00:00+00:00MY Jun-May672.01173.06.01851.0556.0--87.0643.0214.0857.0994.0
    1954-01-01 00:00:00+00:00MY Jun-May994.0984.03.01981.0552.0--53.0605.0267.0872.01109.0
    .......................................
    2022-01-01 00:00:00+00:00MY Jun-May674.4311649.713121.5852445.729971.67768.36975.5031115.549760.6121876.161569.568
    2023-01-01 00:00:00+00:00MY Jun-May569.5681803.942137.7982511.308961.30362.04685.6171108.966705.9081814.874696.434
    2024-01-01 00:00:00+00:00MY Jun-May696.4341978.697148.9542824.085969.49361.1112.8631143.456825.8951969.351854.734
    2025-01-01 00:00:00+00:00MY Jun-May854.7341984.537125.02964.271960.059.7100.01119.7910.02029.7934.571
    2026-01-01 00:00:00+00:00MY Jun-May934.5711561.322140.02635.893960.05980.01099.0775.01874.0761.893
    \n", - "

    77 rows × 12 columns

    \n", - "
    " - ], - "text/plain": [ - " Time period Beginning stocks Production \\\n", - "year \n", - "1950-01-01 00:00:00+00:00 MY Jun-May 496.0 1019.0 \n", - "1951-01-01 00:00:00+00:00 MY Jun-May 492.0 988.0 \n", - "1952-01-01 00:00:00+00:00 MY Jun-May 330.0 1306.0 \n", - "1953-01-01 00:00:00+00:00 MY Jun-May 672.0 1173.0 \n", - "1954-01-01 00:00:00+00:00 MY Jun-May 994.0 984.0 \n", - "... ... ... ... \n", - "2022-01-01 00:00:00+00:00 MY Jun-May 674.431 1649.713 \n", - "2023-01-01 00:00:00+00:00 MY Jun-May 569.568 1803.942 \n", - "2024-01-01 00:00:00+00:00 MY Jun-May 696.434 1978.697 \n", - "2025-01-01 00:00:00+00:00 MY Jun-May 854.734 1984.537 \n", - "2026-01-01 00:00:00+00:00 MY Jun-May 934.571 1561.322 \n", - "\n", - " Imports 2 Total supply 3 Food use Seed use \\\n", - "year \n", - "1950-01-01 00:00:00+00:00 11.0 1526.0 580.0 -- \n", - "1951-01-01 00:00:00+00:00 30.0 1510.0 585.0 -- \n", - "1952-01-01 00:00:00+00:00 24.0 1660.0 578.0 -- \n", - "1953-01-01 00:00:00+00:00 6.0 1851.0 556.0 -- \n", - "1954-01-01 00:00:00+00:00 3.0 1981.0 552.0 -- \n", - "... ... ... ... ... \n", - "2022-01-01 00:00:00+00:00 121.585 2445.729 971.677 68.369 \n", - "2023-01-01 00:00:00+00:00 137.798 2511.308 961.303 62.046 \n", - "2024-01-01 00:00:00+00:00 148.954 2824.085 969.493 61.1 \n", - "2025-01-01 00:00:00+00:00 125.0 2964.271 960.0 59.7 \n", - "2026-01-01 00:00:00+00:00 140.0 2635.893 960.0 59 \n", - "\n", - " Feed and residual use Total domestic use 3 \\\n", - "year \n", - "1950-01-01 00:00:00+00:00 109.0 689.0 \n", - "1951-01-01 00:00:00+00:00 110.0 695.0 \n", - "1952-01-01 00:00:00+00:00 78.0 656.0 \n", - "1953-01-01 00:00:00+00:00 87.0 643.0 \n", - "1954-01-01 00:00:00+00:00 53.0 605.0 \n", - "... ... ... \n", - "2022-01-01 00:00:00+00:00 75.503 1115.549 \n", - "2023-01-01 00:00:00+00:00 85.617 1108.966 \n", - "2024-01-01 00:00:00+00:00 112.863 1143.456 \n", - "2025-01-01 00:00:00+00:00 100.0 1119.7 \n", - "2026-01-01 00:00:00+00:00 80.0 1099.0 \n", - "\n", - " Exports 2 Total disappearance 3 Ending stocks \n", - "year \n", - "1950-01-01 00:00:00+00:00 345.0 1034.0 492.0 \n", - "1951-01-01 00:00:00+00:00 485.0 1180.0 330.0 \n", - "1952-01-01 00:00:00+00:00 332.0 988.0 672.0 \n", - "1953-01-01 00:00:00+00:00 214.0 857.0 994.0 \n", - "1954-01-01 00:00:00+00:00 267.0 872.0 1109.0 \n", - "... ... ... ... \n", - "2022-01-01 00:00:00+00:00 760.612 1876.161 569.568 \n", - "2023-01-01 00:00:00+00:00 705.908 1814.874 696.434 \n", - "2024-01-01 00:00:00+00:00 825.895 1969.351 854.734 \n", - "2025-01-01 00:00:00+00:00 910.0 2029.7 934.571 \n", - "2026-01-01 00:00:00+00:00 775.0 1874.0 761.893 \n", - "\n", - "[77 rows x 12 columns]" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pddf = timeseries.set_index('year').sort_index().to_pandas()\n", - "pddf\n" - ] - }, - { - "cell_type": "markdown", - "id": "9c1242ab", - "metadata": {}, - "source": [ - "## Conclusion: The Power of Hybrid Chaining\n", - "\n", - "By leveraging BigQuery DataFrames and the `%%bqsql` magic, you have built a powerful, interoperable pipeline that seamlessly transitions between SQL and Python.\n", - "\n", - "This hybrid approach offers several key benefits:\n", - "- **Optimal Tool Selection**: Use SQL for what it does best (complex queries, window functions, regex extractions on large sets) and Python for what it does best (visualization, statistical analysis, ML, orchestrating workflow).\n", - "- **Improved Readability**: Instead of massive, unreadable SQL queries with dozens of CTEs, or long, complex Pandas method chains, you can split your pipeline into logical steps, alternating between SQL and Python.\n", - "- **Seamless Scaling**: The exact same `%%bqsql` code can scale from a tiny local Pandas DataFrame to billions of rows in a production BigQuery table. You only need to swap the initial local Pandas DataFrame with a BigQuery DataFrame reference.\n", - "\n", - "\n", - "## Next Steps\n", - "\n", - "In addition to the `%%bqsql` cell magic, BigFrames also registers a **BigQuery Accessor** on standard Pandas DataFrames, allowing you to run SQL scalar functions directly on local pandas data. \n", - "\n", - "For example, you can call powerful Google Cloud community UDFs from [BigQuery Utils](https://github.com/GoogleCloudPlatform/bigquery-utils/tree/master/udfs#bigquery-udfs), [BigFunctions](https://unytics.io/bigfunctions/bigfunctions/#function-categories), or [CARTO Analytics Toolbox for BigQuery](https://docs.carto.com/data-and-analysis/analytics-toolbox-for-bigquery) using `df.bigquery.sql_scalar(...)`:\n" - ] - }, - { - "cell_type": "markdown", - "id": "6a7928bd", - "metadata": {}, - "source": [ - "### Scaling Up: Advanced BigQuery Features\n", - "\n", - "While the BigQuery sandbox offers a powerful environment to test these hybrid Python-SQL workflows for free, some advanced features like BigQuery Machine Learning (BQML) are restricted. By connecting a billing account to your Google Cloud project, you can unlock advanced capabilities such as `ML.FORECAST` (or the `AI.FORECAST` function) to predict time-series data using Google's state-of-the-art foundational models directly from your SQL/Python chain.\n", - "\n", - "### Feedback & Community\n", - "\n", - "The BigFrames team would love to hear your feedback on the hybrid Python-SQL experience:\n", - "* **Email**: [bigframes-feedback@google.com](mailto:bigframes-feedback@google.com)\n", - "* **Issues**: File bug reports or feature requests on the [open-source BigFrames repository](https://github.com/googleapis/google-cloud-python/issues).\n", - "* **Updates**: To receive news and updates, subscribe to the [BigFrames email list](https://docs.google.com/forms/d/10EnDyYdYUW9HvelHYuBRC8L3GdGVl3rX0aroinbRZyc/edit?resourcekey=0-QUsnpzF91gm9hsp04rSA6Q).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bc1a6dbe-170e-4380-83da-779f37e1c00a", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.14.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/experimental/ai_operators.ipynb b/notebooks/experimental/ai_operators.ipynb index e054484a0bf..e24ec34d86d 100644 --- a/notebooks/experimental/ai_operators.ipynb +++ b/notebooks/experimental/ai_operators.ipynb @@ -1,13 +1,5 @@ { "cells": [ - { - "cell_type": "markdown", - "id": "title-cell", - "metadata": {}, - "source": [ - "# AI Operators (Experimental)" - ] - }, { "cell_type": "code", "execution_count": 1, diff --git a/notebooks/experimental/semantic_operators.ipynb b/notebooks/experimental/semantic_operators.ipynb index 22927e6ef94..c32ac9042b8 100644 --- a/notebooks/experimental/semantic_operators.ipynb +++ b/notebooks/experimental/semantic_operators.ipynb @@ -1,13 +1,5 @@ { "cells": [ - { - "cell_type": "markdown", - "id": "title-cell", - "metadata": {}, - "source": [ - "# Semantic Operators (Experimental)" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/generative_ai/ai_functions.ipynb b/notebooks/generative_ai/ai_functions.ipynb index 0831ea0412b..3783ad83651 100644 --- a/notebooks/generative_ai/ai_functions.ipynb +++ b/notebooks/generative_ai/ai_functions.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "acd53f9d", "metadata": {}, "outputs": [], @@ -76,9 +76,9 @@ "metadata": {}, "outputs": [], "source": [ - "import bigframes.pandas as bpd\n", + "import bigframes.pandas as bpd \n", "\n", - "PROJECT_ID = \"\" # @param {type:\"string\"}\n", + "PROJECT_ID = \"\" # Your project ID here\n", "\n", "bpd.options.bigquery.project = PROJECT_ID\n", "bpd.options.bigquery.ordering_mode = \"partial\"\n", @@ -97,7 +97,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "471a47fe", "metadata": {}, "outputs": [ @@ -105,24 +105,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "/usr/local/google/home/swast/src/github.com/googleapis/google-cloud-python/packages/bigframes/bigframes/core/global_session.py:113: DefaultLocationWarning: No explicit location is set, so using location US for the session.\n", - " _global_session = bigframes.session.connect(\n", - "/usr/local/google/home/swast/src/github.com/googleapis/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + "/usr/local/google/home/sycai/src/python-bigquery-dataframes/bigframes/core/global_session.py:103: DefaultLocationWarning: No explicit location is set, so using location US for the session.\n", + " _global_session = bigframes.session.connect(\n" ] }, { "data": { - "text/html": [ - "
    0    {\"result\":\"Salad\",\"full_response\":{\"candidates...\n",
-       "1    {\"result\":\"Hotdog\",\"full_response\":{\"candidate...
    " - ], "text/plain": [ - "0 {\"result\":\"Salad\",\"full_response\":{\"candidates...\n", - "1 {\"result\":\"Hotdog\",\"full_response\":{\"candidate...\n", - "Name: 0, dtype: string" + "0 {'result': 'Salad\\n', 'full_response': '{\"cand...\n", + "1 {'result': 'Sausageroll\\n', 'full_response': '...\n", + "dtype: struct>, status: string>[pyarrow]" ] }, "execution_count": 3, @@ -158,19 +150,15 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "4a3229a8", "metadata": {}, "outputs": [ { "data": { - "text/html": [ - "
    0    \n",
-       "1
    " - ], "text/plain": [ - "0 \n", - "1 \n", + "0 Lettuce\n", + "1 The food\n", "Name: result, dtype: string" ] }, @@ -213,7 +201,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "718c6622", "metadata": {}, "outputs": [ @@ -301,7 +289,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "6875fe36", "metadata": {}, "outputs": [ @@ -339,7 +327,7 @@ " \n", " 0\n", " tiger\n", - " 7.0\n", + " 8.0\n", " \n", " \n", " 2\n", @@ -354,7 +342,7 @@ "text/plain": [ " animals relative_weight\n", "1 spider 1.0\n", - "0 tiger 7.0\n", + "0 tiger 8.0\n", "2 blue whale 10.0\n", "\n", "[3 rows x 2 columns]" @@ -390,7 +378,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "8cfb844b", "metadata": {}, "outputs": [ @@ -477,7 +465,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "2e66110a", "metadata": {}, "outputs": [ @@ -530,7 +518,7 @@ "[2 rows x 2 columns]" ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -545,7 +533,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "venv (3.10.17)", "language": "python", "name": "python3" }, @@ -559,7 +547,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.14.3" + "version": "3.10.17" } }, "nbformat": 4, diff --git a/notebooks/generative_ai/ai_movie_poster.ipynb b/notebooks/generative_ai/ai_movie_poster.ipynb deleted file mode 100644 index 8f309fa7c49..00000000000 --- a/notebooks/generative_ai/ai_movie_poster.ipynb +++ /dev/null @@ -1,762 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "7add2e44", - "metadata": { - "id": "XZpKUoHjXw3_" - }, - "outputs": [], - "source": [ - "# Copyright 2026 Google LLC\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "id": "ee509844", - "metadata": { - "id": "SEKzWP6jW9Oj" - }, - "source": [ - "# Analyzing movie posters with BigQuery Dataframe AI functions" - ] - }, - { - "cell_type": "markdown", - "id": "81b8de8d", - "metadata": {}, - "source": [ - "\n", - "\n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \"Colab Run in Colab\n", - " \n", - " \n", - " \n", - " \"GitHub\n", - " View on GitHub\n", - " \n", - " \n", - " \n", - " \"BQ\n", - " Open in BQ Studio\n", - " \n", - "
    " - ] - }, - { - "cell_type": "markdown", - "id": "256b6c02", - "metadata": { - "id": "c9CCKXG5XTb-" - }, - "source": [ - "BigQuery Dataframe provides a Pythonic way to use AI functions directly with your dataframes. In this notebook, you will use these functions to analyze old\n", - "movie posters. These posters are images stored in a public Google Cloud Storage bucket: `gs://cloud-samples-data/vertex-ai/dataset-management/datasets/classic-movie-posters`" - ] - }, - { - "cell_type": "markdown", - "id": "3f71d3cb", - "metadata": { - "id": "CUJDa_7MPbL9" - }, - "source": [ - "## Set up" - ] - }, - { - "cell_type": "markdown", - "id": "547145f5", - "metadata": { - "id": "D3iYtBSkYpCK" - }, - "source": [ - "Before you begin, you need to\n", - "\n", - "* Set up your permissions for generative AI functions with [these instructions](https://docs.cloud.google.com/bigquery/docs/permissions-for-ai-functions)\n", - "* Set up your Cloud Resource connection by following [these instructions](https://docs.cloud.google.com/bigquery/docs/create-cloud-resource-connection)\n", - "\n", - "Once you have the permissions set up, import the `bigframes.pandas` package, and\n", - "set your cloud project ID." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "d9cd6da8", - "metadata": { - "id": "6nqoRHYbPAx3" - }, - "outputs": [], - "source": [ - "import bigframes.pandas as bpd\n", - "\n", - "MY_PROJECT_ID = \"bigframes-dev\" # @param {type:\"string\"}\n", - "LOCATION = \"us\" # @param {type:\"string\"}\n", - "\n", - "bpd.options.bigquery.project = MY_PROJECT_ID\n", - "bpd.options.bigquery.location = LOCATION" - ] - }, - { - "cell_type": "markdown", - "id": "015a63c1", - "metadata": { - "id": "2XHcNHtvPhNW" - }, - "source": [ - "## Load data" - ] - }, - { - "cell_type": "markdown", - "id": "254561e0", - "metadata": { - "id": "eS-9A7DijfoQ" - }, - "source": [ - "First, you load the data from the GCS bucket to a BigQuery Dataframe:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "47acbbfe", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "id": "ZNPzFjCyPap0", - "outputId": "346d20b2-d615-4094-d24e-2d40e5c90ee2" - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes in 18 seconds of slot time.\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes in 8 seconds of slot time.\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    poster
    0
    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Replace with your own connection name.\n", - "MY_CONNECTION = 'bigframes-default-connection' # @param {type:\"string\"}\n", - "FULL_CONNECTION_ID = f\"{MY_PROJECT_ID}.{LOCATION}.{MY_CONNECTION}\"\n", - "\n", - "import gcsfs\n", - "import bigframes\n", - "import bigframes.pandas as bpd\n", - "import bigframes.bigquery as bbq\n", - "import json\n", - "from IPython.display import HTML, display\n", - "\n", - "session = bpd.get_global_session()\n", - "\n", - "# Configure global display parameters \n", - "bigframes.options.display.blob_display_width = 200\n", - "\n", - "def get_runtime_json_str(series, mode=\"R\", with_metadata=False):\n", - " s = bbq.obj.fetch_metadata(series) if with_metadata else series\n", - " runtime = bbq.obj.get_access_url(s, mode=mode)\n", - " return bbq.to_json_string(runtime)\n", - "\n", - "def get_read_url(series):\n", - " runtime = bbq.obj.get_access_url(series, mode=\"R\")\n", - " return bbq.json_value(runtime, \"$.access_urls.read_url\")\n", - "\n", - "def render_images(df):\n", - " \"\"\"Helper to display BigFrames DataFrame with rendered image previews.\"\"\"\n", - " from bigframes import dtypes\n", - " if isinstance(df, bpd.Series):\n", - " df = df.to_frame()\n", - " \n", - " object_cols = [col for col, dtype in zip(df.columns, df.dtypes) if dtype == dtypes.OBJ_REF_DTYPE]\n", - " if not object_cols:\n", - " display(df)\n", - " return\n", - "\n", - " limit = bigframes.options.display.max_rows or 10\n", - " view_df = df.head(limit)\n", - " runtime_cols = {\n", - " col: get_runtime_json_str(view_df[col], mode=\"R\", with_metadata=False) \n", - " for col in object_cols\n", - " }\n", - " \n", - " pandas_json_df = bpd.DataFrame(runtime_cols).to_pandas()\n", - " final_pd = view_df.to_pandas()\n", - " width = bigframes.options.display.blob_display_width or 200\n", - " \n", - " def format_cell_html(raw_json):\n", - " if not raw_json: return \"\"\n", - " try:\n", - " obj_rt = json.loads(raw_json)\n", - " if \"access_urls\" not in obj_rt: return \"Error fetching URL\"\n", - " uri = obj_rt.get(\"objectref\", {}).get(\"uri\", \"\")\n", - " url = obj_rt[\"access_urls\"][\"read_url\"]\n", - " if str(uri).lower().endswith((\".png\", \".jpg\", \".jpeg\", \".webp\")):\n", - " return f''\n", - " return f'{uri}'\n", - " except: return \"Format Error\"\n", - "\n", - " for col in object_cols:\n", - " final_pd[col] = pandas_json_df[col].map(format_cell_html)\n", - " display(HTML(final_pd.to_html(escape=False)))\n", - "\n", - "# List files using gcsfs\n", - "fs = gcsfs.GCSFileSystem(anon=True)\n", - "uris = fs.glob(\"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/classic-movie-posters/*\")\n", - "\n", - "# Ensure URIs have gs:// prefix\n", - "uris = [u if u.startswith(\"gs://\") else f\"gs://{u}\" for u in uris]\n", - "\n", - "# Read the URIs into a BigQuery DataFrame\n", - "movies = bpd.read_gbq(f\"SELECT uri FROM UNNEST({uris[:5]}) as uri\")\n", - "\n", - "# Create the object reference column using the fully qualified connection ID\n", - "movies['poster'] = bbq.obj.make_ref(movies['uri'], authorizer=FULL_CONNECTION_ID)\n", - "movies = movies[['poster']]\n", - "render_images(movies.head(1))" - ] - }, - { - "cell_type": "markdown", - "id": "f1096d2f", - "metadata": { - "id": "EfkdDH08QnYw" - }, - "source": [ - "## Extract titles from posters" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "bb30d47c", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "id": "6CoZZ5tSQm1r", - "outputId": "1b3915ce-eb83-4be9-b1c1-d9a326dc9408" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes in 23 seconds of slot time. [Job bigframes-dev:US.job_ZKfuxLQE1U49whg7fgakYFYfiz34 details]\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes in 40 seconds of slot time. [Job bigframes-dev:US.job_VwLv_BxDFdE4adNx1bpnvvM5vfZd details]\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    postertitle
    0The movie title for this poster image is **Au Secours!** (Help!).
    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import bigframes.bigquery as bbq\n", - "\n", - "movies['title'] = bbq.ai.generate(\n", - " (\"What is the movie title for this poster image?\", get_read_url(movies['poster']))\n", - ").struct.field(\"result\")\n", - "render_images(movies.head(1))" - ] - }, - { - "cell_type": "markdown", - "id": "eb9eb261", - "metadata": { - "id": "cFQHQ9S2lr6t" - }, - "source": [ - "Notice that `ai.generate()` has a `struct` return type, which holds not only the LLM response, but also the status. If you do not provide a field name for your answer, `\"result\"` will be the default name. You can access LLM response content with the struct accessor (e.g. `my_response.struct.filed(\"result\")`);." - ] - }, - { - "cell_type": "markdown", - "id": "ea29eb21", - "metadata": { - "id": "R8kkUhgoS5Xz" - }, - "source": [ - "## Get movie release year\n", - "\n", - "In the example below, you will use `ai.generate_int()` to find the release year for each movie poster:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "bf426247", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 976 - }, - "id": "cKZdHq0XS1iW", - "outputId": "72cbad57-4518-4e1e-97bb-333d424dba73" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/core/logging/log_adapter.py:229: ApiDeprecationWarning: The blob accessor is deprecated and will be removed in a future release. Use bigframes.bigquery.obj functions instead.\n", - " return prop(*args, **kwargs)\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes in 51 seconds of slot time. [Job bigframes-dev:US.3cf4ab5b-c360-4b7c-9def-4cd03135a547 details]\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " Query processed 1.2 kB in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    postertitleyear
    0The movie title is **Au Secours!**1924
    \n", - "

    1 rows × 3 columns

    \n", - "
    [1 rows x 3 columns in total]" - ], - "text/plain": [ - " poster \\\n", - "0 {\"access_urls\":{\"expiry_time\":\"2026-05-09T03:1... \n", - "\n", - " title year \n", - "0 The movie title is **Au Secours!** 1924 \n", - "\n", - "[1 rows x 3 columns]" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "movies['year'] = bbq.ai.generate_int(\n", - " (\"What is the release year for this movie?\", movies['title']),\n", - " endpoint='gemini-2.5-pro'\n", - ").struct.field(\"result\")\n", - "\n", - "movies.head(1)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "8bf12352", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 250 - }, - "id": "yqRiNRY8_8fs", - "outputId": "efa60107-6883-4f5c-8e40-43c7287ea7fb" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] - }, - { - "data": { - "text/plain": [ - "poster structSQL
    WITH `bfcte_0` AS (\n",
-       "  SELECT\n",
-       "    *\n",
-       "  FROM UNNEST(ARRAY<STRUCT<`bfcol_0` STRING, `bfcol_1` INT64, `bfcol_2` INT64>>[STRUCT(\n",
-       "    'gs://cloud-samples-data/vertex-ai/dataset-management/datasets/classic-movie-posters/au_secours.jpeg',\n",
-       "    0,\n",
-       "    0\n",
-       "  ), STRUCT(\n",
-       "    'gs://cloud-samples-data/vertex-ai/dataset-management/datasets/classic-movie-posters/barque_sortant_du_port.jpeg',\n",
-       "    1,\n",
-       "    1\n",
-       "  ), STRUCT(\n",
-       "    'gs://cloud-samples-data/vertex-ai/dataset-management/datasets/classic-movie-posters/battling_butler.jpg',\n",
-       "    2,\n",
-       "    2\n",
-       "  ), STRUCT(\n",
-       "    'gs://cloud-samples-data/vertex-ai/dataset-management/datasets/classic-movie-posters/brown_of_harvard.jpeg',\n",
-       "    3,\n",
-       "    3\n",
-       "  ), STRUCT(\n",
-       "    'gs://cloud-samples-data/vertex-ai/dataset-management/datasets/classic-movie-posters/der_student_von_prag.jpg',\n",
-       "    4,\n",
-       "    4\n",
-       "  )])\n",
-       ")\n",
-       "SELECT\n",
-       "  `bfcol_1` AS `bfuid_col_60`,\n",
-       "  TO_JSON_STRING(\n",
-       "    OBJ.GET_ACCESS_URL(OBJ.MAKE_REF(`bfcol_0`, 'bigframes-dev.us.bigframes-default-connection'), 'R')\n",
-       "  ) AS `bfuid_col_66`\n",
-       "FROM `bfcte_0`\n",
-       "WHERE\n",
-       "  AI.IF(\n",
-       "    prompt => (\n",
-       "      'The movie ',\n",
-       "      AI.GENERATE(\n",
-       "        prompt => (\n",
-       "          'What is the movie title for this poster image?',\n",
-       "          JSON_VALUE(\n",
-       "            OBJ.GET_ACCESS_URL(OBJ.MAKE_REF(`bfcol_0`, 'bigframes-dev.us.bigframes-default-connection'), 'R'),\n",
-       "            '$.access_urls.read_url'\n",
-       "          )\n",
-       "        ),\n",
-       "        request_type => 'UNSPECIFIED'\n",
-       "      ).`result`,\n",
-       "      ' was made in US'\n",
-       "    ),\n",
-       "    optimization_mode => 'MINIMIZE_COST'\n",
-       "  )\n",
-       "ORDER BY\n",
-       "  `bfcol_2` ASC NULLS LAST\n",
-       "LIMIT 1
    \n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes in 3 minutes of slot time. [Job bigframes-dev:US.job_NBILG5qU14Aitas81nPCCtYM9KdM details]\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    postertitleyear
    2The movie title for the poster image is **Battling Butler**.1926
    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "us_movies = movies[bbq.ai.if_(\n", - " (\"The movie \", movies['title'], \" was made in US\")\n", - ")]\n", - "render_images(us_movies.head(1))" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.0" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/generative_ai/bq_dataframes_ai_forecast.ipynb b/notebooks/generative_ai/bq_dataframes_ai_forecast.ipynb index 6f8c95d3a48..b9599282b38 100644 --- a/notebooks/generative_ai/bq_dataframes_ai_forecast.ipynb +++ b/notebooks/generative_ai/bq_dataframes_ai_forecast.ipynb @@ -60,7 +60,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Setup" + "### Setup" ] }, { diff --git a/notebooks/generative_ai/bq_dataframes_llm_claude3_museum_art.ipynb b/notebooks/generative_ai/bq_dataframes_llm_claude3_museum_art.ipynb new file mode 100644 index 00000000000..a1bb1e9d89d --- /dev/null +++ b/notebooks/generative_ai/bq_dataframes_llm_claude3_museum_art.ipynb @@ -0,0 +1,1019 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "9A9NkTRTfo2I" + }, + "outputs": [], + "source": [ + "# Copyright 2024 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8fK_rdvvx1iZ" + }, + "source": [ + "## Overview\n", + "\n", + "## Objective\n", + "\n", + "This notebook shows how to conecct BigQuery dataset to Claude models on Vertex AI using BigQuery DataFrames.\n", + "\n", + "### Claude on Vertex AI\n", + "\n", + "Anthropic Claude models on Vertex AI offer fully managed and serverless models. To use a Claude model on Vertex AI, send a request directly to the Vertex AI API endpoint.\n", + "\n", + "For more information, see the [Use Claude](https://cloud.devsite.corp.google.com/vertex-ai/generative-ai/docs/third-party-models/use-claude) documentation.\n", + "\n", + "### BigQuery DataFrames\n", + "BigQuery DataFrames provides a Pythonic DataFrame and machine learning (ML) API powered by the BigQuery engine. BigQuery DataFrames is an open-source package.\n", + "\n", + "For more information, see this documentation\n", + "https://cloud.google.com/bigquery/docs/reference/bigquery-dataframes\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nwYvaaW25jYS" + }, + "source": [ + "### Getting Started\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hVi8v2mxBkeG" + }, + "source": [ + "#### Authenticate your notebook environment (Colab only)\n", + "If you are running this notebook on Google Colab, uncomment and run the following cell to authenticate your environment. This step is not required if you are using [Vertex AI Workbench](https://cloud.google.com/vertex-ai-workbench)." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "OHfMDNI76_Pz" + }, + "outputs": [], + "source": [ + "# from google.colab import auth\n", + "# auth.authenticate_user()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gI3KlxQQ_F_T" + }, + "source": [ + "## Using Anthropic's Vertex SDK + BQ for *Python*" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "E0x3GO6M_O3_" + }, + "source": [ + "### Getting Started\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_CJrqUvqAfR7" + }, + "source": [ + "#### Install the latest bigframes package if bigframes version < 1.15.0\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "executionInfo": { + "elapsed": 11539, + "status": "ok", + "timestamp": 1724257409246, + "user": { + "displayName": "Annie Xu", + "userId": "11935526703047498014" + }, + "user_tz": 420 + }, + "id": "fi_HLdat_Pce", + "outputId": "020149f0-9fe8-45de-f160-abe488c0bed2" + }, + "outputs": [], + "source": [ + "# !pip install bigframes --upgrade" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hUiAYUFbBCpR" + }, + "source": [ + "#### Restart current runtime\n", + "\n", + "To use the newly installed packages in this Jupyter runtime, you must restart the runtime. You can do this by running the cell below, which will restart the current kernel." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "jcqgcj_DBFgt" + }, + "outputs": [], + "source": [ + "# # Restart kernel after installs so that your environment can access the new packages\n", + "# import sys\n", + "\n", + "# if \"google.colab\" in sys.modules:\n", + "# import IPython\n", + "\n", + "# app = IPython.Application.instance()\n", + "# app.kernel.do_shutdown(True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "shZgRl6qbZYP" + }, + "source": [ + "#### Define Google Cloud project and region information" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "JZLqMJ6va9fc" + }, + "outputs": [], + "source": [ + "# Input your project id\n", + "PROJECT_ID = \"bigframes-dev\" # @param {type:\"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "czcmJpKPBMVC" + }, + "source": [ + "#### Select Claude Model and Region Availability:\n", + "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude#anthropic_claude_quotas_and_supported_context_length" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "wIBkGcFkK0Ci" + }, + "outputs": [], + "source": [ + "REGION = \"us-east5\" # @param {type:\"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F3UmCLerH0t0" + }, + "source": [ + "### Load raw sample data to a bigquery dataset\n", + "\n", + "Create a BigQuery Dataset and table. You can use the sample museum data in CSV from [here](https://github.com/googleapis/python-bigquery-dataframes/tree/main/notebooks/generative_ai/museum_art.csv).\n", + "\n", + "The dataset should be in the **same region** as your chosen claude model. Let's say you selected us-east5 for claude 'haiku', then load the sample data to a dataset in us-east5." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gijJ2vr5B5nV" + }, + "source": [ + "### Text generation for BQ Tables using Python BigFrames\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "collapsed": true, + "executionInfo": { + "elapsed": 756, + "status": "ok", + "timestamp": 1724260427446, + "user": { + "displayName": "Annie Xu", + "userId": "11935526703047498014" + }, + "user_tz": 420 + }, + "id": "cU3Gq7TqHFdi", + "outputId": "aa5ec159-a91b-4349-e56a-400e90935edc" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "object_number string[pyarrow]\n", + "is_highlight boolean\n", + "is_public_domain boolean\n", + "object_id Int64\n", + "department string[pyarrow]\n", + "object_name string[pyarrow]\n", + "title string[pyarrow]\n", + "culture string[pyarrow]\n", + "period string[pyarrow]\n", + "dynasty string[pyarrow]\n", + "reign string[pyarrow]\n", + "portfolio string[pyarrow]\n", + "artist_role string[pyarrow]\n", + "artist_prefix string[pyarrow]\n", + "artist_display_name string[pyarrow]\n", + "artist_display_bio string[pyarrow]\n", + "artist_suffix string[pyarrow]\n", + "artist_alpha_sort string[pyarrow]\n", + "artist_nationality string[pyarrow]\n", + "artist_begin_date string[pyarrow]\n", + "artist_end_date string[pyarrow]\n", + "object_date string[pyarrow]\n", + "object_begin_date Int64\n", + "object_end_date Int64\n", + "medium string[pyarrow]\n", + "dimensions string[pyarrow]\n", + "credit_line string[pyarrow]\n", + "geography_type string[pyarrow]\n", + "city string[pyarrow]\n", + "state string[pyarrow]\n", + "county string[pyarrow]\n", + "country string[pyarrow]\n", + "region string[pyarrow]\n", + "subregion string[pyarrow]\n", + "locale string[pyarrow]\n", + "locus string[pyarrow]\n", + "excavation string[pyarrow]\n", + "river string[pyarrow]\n", + "classification string[pyarrow]\n", + "rights_and_reproduction string[pyarrow]\n", + "link_resource string[pyarrow]\n", + "metadata_date timestamp[us, tz=UTC][pyarrow]\n", + "repository string[pyarrow]\n", + "dtype: object" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import bigframes\n", + "import bigframes.pandas as bpd\n", + "bigframes.options._bigquery_options.project = PROJECT_ID # replace to user project\n", + "bigframes.options._bigquery_options.location = REGION #choice a region which the claude model you choice allows\n", + "df = bpd.read_gbq(\"bigframes-dev.garrettwu_us_east5.museum_art\") # replace with your table\n", + "df.dtypes" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 461 + }, + "executionInfo": { + "elapsed": 4568, + "status": "ok", + "timestamp": 1724271168583, + "user": { + "displayName": "Annie Xu", + "userId": "11935526703047498014" + }, + "user_tz": 420 + }, + "id": "exWNXEzLHHaU", + "outputId": "1b33b64c-c8bd-42e6-ecc3-0ea0b5e492be" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 1998408a-4e29-4381-9229-cf8585a47dbe is DONE. 7.7 MB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 817a5321-9852-45da-8b14-004affc20c38 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 36aa1b30-acb5-4188-8377-b9f544443db8 is DONE. 955 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    object_idtitle
    0285844Addie Card, 12 years. Spinner in North Pownal ...
    1437141Portrait of a Man
    2670650[Snow Crystal]
    3268450Newhaven Fisherman
    4646996전(傳) 오원 장승업 (1843–1897) 청동기와 화초가 있는 정물화 조선|傳 吾...
    5287958Bridge of Augustus at Nani
    6435869Antoine Dominique Sauveur Aubert (born 1817), ...
    755834<NA>
    845087<NA>
    956883<NA>
    \n", + "

    10 rows × 2 columns

    \n", + "
    [10 rows x 2 columns in total]" + ], + "text/plain": [ + " object_id title\n", + "0 285844 Addie Card, 12 years. Spinner in North Pownal ...\n", + "1 437141 Portrait of a Man\n", + "2 670650 [Snow Crystal]\n", + "3 268450 Newhaven Fisherman\n", + "4 646996 전(傳) 오원 장승업 (1843–1897) 청동기와 화초가 있는 정물화 조선|傳 吾...\n", + "5 287958 Bridge of Augustus at Nani\n", + "6 435869 Antoine Dominique Sauveur Aubert (born 1817), ...\n", + "7 55834 \n", + "8 45087 \n", + "9 56883 \n", + "\n", + "[10 rows x 2 columns]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# @title query: select top 10 records from table and put into dataframe\n", + "\n", + "df = df[[\"object_id\", \"title\"]].head(10)\n", + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_UZNsP_WDlyr" + }, + "source": [ + "### Enable Claude model on Vertex AI and Create a BQ External Model Connection\n", + "\n", + "\n", + "* Step 1: Visit the Vertex AI Model Garden console and select the model tile for Claude model of your choice. Following this doc [link](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude). Click on the **“Enable”** button and follow the instructions.\n", + "\n", + "* Step 2: Create a BQ External Connection\n", + "Follow the same process like this one: [link](https://cloud.google.com/bigquery/docs/generate-text#create_a_connection). Pay attention to the **supported region** of Claude models and make your conenction follow the same region for example us-east5 for Claude 3.5.\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8Q4aff5m9QOn" + }, + "source": [ + "### Use BigQuery DataFrames ML package with Claude LLM \n", + "\n", + "In this example, we are using the Claude3TextGenerator class from BigQuery DataFrames to translate title of art piece to english.\n", + "\n", + "Documentation for the Claude3TextGenerator Class: https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.ml.llm.Claude3TextGenerator" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 513 + }, + "executionInfo": { + "elapsed": 25662, + "status": "ok", + "timestamp": 1724271197922, + "user": { + "displayName": "Annie Xu", + "userId": "11935526703047498014" + }, + "user_tz": 420 + }, + "id": "1pdyI5KBTyTD", + "outputId": "8f1e976b-1fd0-49ba-e068-f480eafb1765" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 514f5afe-15e0-4474-9e09-fbf94f0fe8ca is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 5d4df544-e8a4-42f3-8a94-5f7e79b23562 is DONE. 635 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 25288d94-b10c-4b39-a272-3969ccb19af3 is DONE. 14 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job d5693878-1037-4798-8aa0-f568ec0be9e3 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 54080328-ba8b-4715-bf2b-3e5b7affa90b is DONE. 4.0 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    ml_generate_text_llm_resultml_generate_text_statusprompt
    0This text is already in English. It appears to...translate this into English: Addie Card, 12 ye...
    1The phrase \"Portrait of a Man\" is already in E...translate this into English: Portrait of a Man
    2The phrase \"[Snow Crystal]\" is already in Engl...translate this into English: [Snow Crystal]
    3The phrase \"Newhaven Fisherman\" is already in ...translate this into English: Newhaven Fisherman
    4Here's the English translation:\n", + "\n", + "\"Attributed t...translate this into English: 전(傳) 오원 장승업 (1843...
    5I apologize, but I'm not sure which language \"...translate this into English: Bridge of Augustu...
    6This title is already in English. It describes...translate this into English: Antoine Dominique...
    7<NA><NA><NA>
    8<NA><NA><NA>
    9<NA><NA><NA>
    \n", + "

    10 rows × 3 columns

    \n", + "
    [10 rows x 3 columns in total]" + ], + "text/plain": [ + " ml_generate_text_llm_result ml_generate_text_status \\\n", + "0 This text is already in English. It appears to... \n", + "1 The phrase \"Portrait of a Man\" is already in E... \n", + "2 The phrase \"[Snow Crystal]\" is already in Engl... \n", + "3 The phrase \"Newhaven Fisherman\" is already in ... \n", + "4 Here's the English translation:\n", + "\n", + "\"Attributed t... \n", + "5 I apologize, but I'm not sure which language \"... \n", + "6 This title is already in English. It describes... \n", + "7 \n", + "8 \n", + "9 \n", + "\n", + " prompt \n", + "0 translate this into English: Addie Card, 12 ye... \n", + "1 translate this into English: Portrait of a Man \n", + "2 translate this into English: [Snow Crystal] \n", + "3 translate this into English: Newhaven Fisherman \n", + "4 translate this into English: 전(傳) 오원 장승업 (1843... \n", + "5 translate this into English: Bridge of Augustu... \n", + "6 translate this into English: Antoine Dominique... \n", + "7 \n", + "8 \n", + "9 \n", + "\n", + "[10 rows x 3 columns]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from bigframes.ml import llm\n", + "model = llm.Claude3TextGenerator(model_name=\"claude-3-5-sonnet\",\n", + " connection_name=\"bigframes-dev.us-east5.bigframes-rf-conn\" ) # replace with your connection\n", + "df[\"input_prompt\"] = \"translate this into English: \" + df[\"title\"]\n", + "result = model.predict(df[\"input_prompt\"])\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 461 + }, + "executionInfo": { + "elapsed": 5249, + "status": "ok", + "timestamp": 1724274172557, + "user": { + "displayName": "Annie Xu", + "userId": "11935526703047498014" + }, + "user_tz": 420 + }, + "id": "Ux1VI5qujHOB", + "outputId": "7b859943-5e7c-4cc0-d9c2-bb3d44682010" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 6b6eceaa-e713-493e-beac-481a3d777a5c is DONE. 4.9 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 5c660da9-318c-424e-9412-43f09e44a8b3 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 82b61007-8370-4514-addb-258d7c48d66c is DONE. 4.9 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    object_idtitleml_generate_text_llm_resultprompt
    0285844Addie Card, 12 years. Spinner in North Pownal ...This text is already in English. It appears to...translate this into English: Addie Card, 12 ye...
    1437141Portrait of a ManThe phrase \"Portrait of a Man\" is already in E...translate this into English: Portrait of a Man
    2670650[Snow Crystal]The phrase \"[Snow Crystal]\" is already in Engl...translate this into English: [Snow Crystal]
    3268450Newhaven FishermanThe phrase \"Newhaven Fisherman\" is already in ...translate this into English: Newhaven Fisherman
    4646996전(傳) 오원 장승업 (1843–1897) 청동기와 화초가 있는 정물화 조선|傳 吾...Here's the English translation:\n", + "\n", + "\"Attributed t...translate this into English: 전(傳) 오원 장승업 (1843...
    5287958Bridge of Augustus at NaniI apologize, but I'm not sure which language \"...translate this into English: Bridge of Augustu...
    6435869Antoine Dominique Sauveur Aubert (born 1817), ...This title is already in English. It describes...translate this into English: Antoine Dominique...
    755834<NA><NA><NA>
    845087<NA><NA><NA>
    956883<NA><NA><NA>
    \n", + "

    10 rows × 4 columns

    \n", + "
    [10 rows x 4 columns in total]" + ], + "text/plain": [ + " object_id title \\\n", + "0 285844 Addie Card, 12 years. Spinner in North Pownal ... \n", + "1 437141 Portrait of a Man \n", + "2 670650 [Snow Crystal] \n", + "3 268450 Newhaven Fisherman \n", + "4 646996 전(傳) 오원 장승업 (1843–1897) 청동기와 화초가 있는 정물화 조선|傳 吾... \n", + "5 287958 Bridge of Augustus at Nani \n", + "6 435869 Antoine Dominique Sauveur Aubert (born 1817), ... \n", + "7 55834 \n", + "8 45087 \n", + "9 56883 \n", + "\n", + " ml_generate_text_llm_result \\\n", + "0 This text is already in English. It appears to... \n", + "1 The phrase \"Portrait of a Man\" is already in E... \n", + "2 The phrase \"[Snow Crystal]\" is already in Engl... \n", + "3 The phrase \"Newhaven Fisherman\" is already in ... \n", + "4 Here's the English translation:\n", + "\n", + "\"Attributed t... \n", + "5 I apologize, but I'm not sure which language \"... \n", + "6 This title is already in English. It describes... \n", + "7 \n", + "8 \n", + "9 \n", + "\n", + " prompt \n", + "0 translate this into English: Addie Card, 12 ye... \n", + "1 translate this into English: Portrait of a Man \n", + "2 translate this into English: [Snow Crystal] \n", + "3 translate this into English: Newhaven Fisherman \n", + "4 translate this into English: 전(傳) 오원 장승업 (1843... \n", + "5 translate this into English: Bridge of Augustu... \n", + "6 translate this into English: Antoine Dominique... \n", + "7 \n", + "8 \n", + "9 \n", + "\n", + "[10 rows x 4 columns]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output_df=df.drop(columns=[\"input_prompt\"]).join(result.drop(columns=\"ml_generate_text_status\"))\n", + "output_df" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "id": "ej70vFMvelsg" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 8c3f1d21-9033-4224-b6f3-4f2414f4ed18 is DONE. 4.5 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'bigframes-dev.garrettwu_us_east5.museum_art_translate'" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# prompt: load the dataframe output to another Bigquery table\n", + "\n", + "# @title Save results to BigQuery\n", + "\n", + "output_df.to_gbq(\"bigframes-dev.garrettwu_us_east5.museum_art_translate\", if_exists=\"replace\") # replace with your table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/generative_ai/bq_dataframes_llm_code_generation.ipynb b/notebooks/generative_ai/bq_dataframes_llm_code_generation.ipynb index 527d3c4aaac..4f1329129e2 100644 --- a/notebooks/generative_ai/bq_dataframes_llm_code_generation.ipynb +++ b/notebooks/generative_ai/bq_dataframes_llm_code_generation.ipynb @@ -1,1305 +1,1304 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "ur8xi4C7S06n" - }, - "outputs": [], - "source": [ - "# Copyright 2022 Google LLC\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JAPoU8Sm5E6e" - }, - "source": [ - "# Use BigQuery DataFrames with Generative AI for code generation\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \"Colab Run in Colab\n", - " \n", - " \n", - " \n", - " \"GitHub\n", - " View on GitHub\n", - " \n", - " \n", - " \n", - " \"Vertex\n", - " Open in Vertex AI Workbench\n", - " \n", - " \n", - " \n", - " \"BQ\n", - " Open in BQ Studio\n", - " \n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "24743cf4a1e1" - }, - "source": [ - "**_NOTE_**: This notebook has been tested in the following environment:\n", - "\n", - "* Python version = 3.10" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tvgnzT1CKxrO" - }, - "source": [ - "## Overview\n", - "\n", - "Use this notebook to walk through an example use case of generating sample code by using BigQuery DataFrames and its integration with Generative AI support on Vertex AI.\n", - "\n", - "Learn more about [BigQuery DataFrames](https://cloud.google.com/python/docs/reference/bigframes/latest)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "d975e698c9a4" - }, - "source": [ - "### Objective\n", - "\n", - "In this tutorial, you create a CSV file containing sample code for calling a given set of APIs.\n", - "\n", - "The steps include:\n", - "\n", - "- Defining an LLM model in BigQuery DataFrames, specifically the [Gemini Model](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models), using `bigframes.ml.llm`.\n", - "- Creating a DataFrame by reading in data from Cloud Storage.\n", - "- Manipulating data in the DataFrame to build LLM prompts.\n", - "- Sending DataFrame prompts to the LLM model using the `predict` method.\n", - "- Creating and using a custom function to transform the output provided by the LLM model response.\n", - "- Exporting the resulting transformed DataFrame as a CSV file." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "08d289fa873f" - }, - "source": [ - "### Dataset\n", - "\n", - "This tutorial uses a dataset listing the names of various pandas DataFrame and Series APIs." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aed92deeb4a0" - }, - "source": [ - "### Costs\n", - "\n", - "This tutorial uses billable components of Google Cloud:\n", - "\n", - "* BigQuery\n", - "* Generative AI support on Vertex AI\n", - "* Cloud Functions\n", - "\n", - "Learn about [BigQuery compute pricing](https://cloud.google.com/bigquery/pricing#analysis_pricing_models),\n", - "[Generative AI support on Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing#generative_ai_models), and [Cloud Functions pricing](https://cloud.google.com/functions/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n", - "to generate a cost estimate based on your projected usage." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "i7EUnXsZhAGF" - }, - "source": [ - "## Installation\n", - "\n", - "Install the following packages, which are required to run this notebook:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "2b4ef9b72d43" - }, - "outputs": [], - "source": [ - "!pip install bigframes --upgrade --quiet" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BF1j6f9HApxa" - }, - "source": [ - "## Before you begin\n", - "\n", - "Complete the tasks in this section to set up your environment." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Wbr2aVtFQBcg" - }, - "source": [ - "### Set up your Google Cloud project\n", - "\n", - "**The following steps are required, regardless of your notebook environment.**\n", - "\n", - "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 credit towards your compute/storage costs.\n", - "\n", - "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", - "\n", - "3. [Click here](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com,bigqueryconnection.googleapis.com,cloudfunctions.googleapis.com,run.googleapis.com,artifactregistry.googleapis.com,cloudbuild.googleapis.com,cloudresourcemanager.googleapis.com) to enable the following APIs:\n", - "\n", - " * BigQuery API\n", - " * BigQuery Connection API\n", - " * Cloud Functions API\n", - " * Cloud Run API\n", - " * Artifact Registry API\n", - " * Cloud Build API\n", - " * Cloud Resource Manager API\n", - " * Vertex AI API\n", - "\n", - "4. If you are running this notebook locally, install the [Cloud SDK](https://cloud.google.com/sdk)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "WReHDGG5g0XY" - }, - "source": [ - "#### Set your project ID\n", - "\n", - "If you don't know your project ID, try the following:\n", - "* Run `gcloud config list`.\n", - "* Run `gcloud projects list`.\n", - "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)." - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "id": "oM1iC_MfAts1" - }, - "outputs": [ + "cells": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[1;31mERROR:\u001b[0m (gcloud.config.set) argument VALUE: Must be specified.\n", - "Usage: gcloud config set SECTION/PROPERTY VALUE [optional flags]\n", - " optional flags may be --help | --installation\n", - "\n", - "For detailed information on this command and its flags, run:\n", - " gcloud config set --help\n" - ] - } - ], - "source": [ - "PROJECT_ID = \"\" # @param {type:\"string\"}\n", - "\n", - "# Set the project id\n", - "! gcloud config set project {PROJECT_ID}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "region" - }, - "source": [ - "#### Set the region\n", - "\n", - "You can also change the `REGION` variable used by BigQuery. Learn more about [BigQuery regions](https://cloud.google.com/bigquery/docs/locations#supported_locations)." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "eF-Twtc4XGem" - }, - "outputs": [], - "source": [ - "REGION = \"US\" # @param {type: \"string\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sBCra4QMA2wR" - }, - "source": [ - "### Authenticate your Google Cloud account\n", - "\n", - "Depending on your Jupyter environment, you might have to manually authenticate. Follow the relevant instructions below." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "74ccc9e52986" - }, - "source": [ - "**Vertex AI Workbench**\n", - "\n", - "Do nothing, you are already authenticated." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "de775a3773ba" - }, - "source": [ - "**Local JupyterLab instance**\n", - "\n", - "Uncomment and run the following cell:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "id": "254614fa0c46" - }, - "outputs": [], - "source": [ - "# ! gcloud auth login" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ef21552ccea8" - }, - "source": [ - "**Colab**\n", - "\n", - "Uncomment and run the following cell:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "id": "603adbbf0532" - }, - "outputs": [], - "source": [ - "# from google.colab import auth\n", - "# auth.authenticate_user()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "960505627ddf" - }, - "source": [ - "### Import libraries" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "id": "PyQmSRbKA8r-" - }, - "outputs": [], - "source": [ - "import bigframes.pandas as bf\n", - "from google.cloud import bigquery\n", - "from google.cloud import bigquery_connection_v1 as bq_connection" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "init_aip:mbsdk,all" - }, - "source": [ - "### Set BigQuery DataFrames options" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "id": "NPPMuw2PXGeo" - }, - "outputs": [], - "source": [ - "# Note: The project option is not required in all environments.\n", - "# On BigQuery Studio, the project ID is automatically detected.\n", - "bf.options.bigquery.project = PROJECT_ID\n", - "\n", - "# Note: The location option is not required.\n", - "# It defaults to the location of the first table or query\n", - "# passed to read_gbq(). For APIs where a location can't be\n", - "# auto-detected, the location defaults to the \"US\" location.\n", - "bf.options.bigquery.location = REGION" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DTVtFlqeFbrU" - }, - "source": [ - "If you want to reset the location of the created DataFrame or Series objects, reset the session by executing `bf.close_session()`. After that, you can reuse `bf.options.bigquery.location` to specify another location." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6eytf4xQHzcF" - }, - "source": [ - "# Define the LLM model\n", - "\n", - "BigQuery DataFrames provides integration with [Gemini Models](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models) via Vertex AI.\n", - "\n", - "This section walks through a few steps required in order to use the model in your notebook." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qUjT8nw-jIXp" - }, - "source": [ - "## Define the model\n", - "\n", - "Use `bigframes.ml.llm` to define the model:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "sdjeXFwcHfl7" - }, - "outputs": [ + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "ur8xi4C7S06n" + }, + "outputs": [], + "source": [ + "# Copyright 2022 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, { - "data": { - "text/html": [ - "Query job 0ee1a08e-788e-4fc7-b061-52c23ab25d5a is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "JAPoU8Sm5E6e" + }, + "source": [ + "## Use BigQuery DataFrames with Generative AI for code generation\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \"Colab Run in Colab\n", + " \n", + " \n", + " \n", + " \"GitHub\n", + " View on GitHub\n", + " \n", + " \n", + " \n", + " \"Vertex\n", + " Open in Vertex AI Workbench\n", + " \n", + " \n", + " \n", + " \"BQ\n", + " Open in BQ Studio\n", + " \n", + "
    " ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from bigframes.ml.llm import GeminiTextGenerator\n", - "\n", - "model = GeminiTextGenerator(model_name=\"gemini-2.5-flash\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GbW0oCnU1s1N" - }, - "source": [ - "# Read data from Cloud Storage into BigQuery DataFrames\n", - "\n", - "You can create a BigQuery DataFrames DataFrame by reading data from any of the following locations:\n", - "\n", - "* A local data file\n", - "* Data stored in a BigQuery table\n", - "* A data file stored in Cloud Storage\n", - "* An in-memory pandas DataFrame\n", - "\n", - "In this tutorial, you create BigQuery DataFrames DataFrames by reading two CSV files stored in Cloud Storage, one containing a list of DataFrame API names and one containing a list of Series API names." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "id": "SchiTkQGIJog" - }, - "outputs": [], - "source": [ - "df_api = bf.read_csv(\"gs://cloud-samples-data/vertex-ai/bigframe/df.csv\")\n", - "series_api = bf.read_csv(\"gs://cloud-samples-data/vertex-ai/bigframe/series.csv\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7OBjw2nmQY3-" - }, - "source": [ - "Take a peek at a few rows of data for each file:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "id": "QCqgVCIsGGuv" - }, - "outputs": [ + }, { - "data": { - "text/html": [ - "Query job 48be241c-ee93-4dfa-a9e3-66b64c4b5150 is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "24743cf4a1e1" + }, + "source": [ + "**_NOTE_**: This notebook has been tested in the following environment:\n", + "\n", + "* Python version = 3.10" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job 6af9caa5-4f7a-48f0-a7df-d692ee063b7e is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "tvgnzT1CKxrO" + }, + "source": [ + "## Overview\n", + "\n", + "Use this notebook to walk through an example use case of generating sample code by using BigQuery DataFrames and its integration with Generative AI support on Vertex AI.\n", + "\n", + "Learn more about [BigQuery DataFrames](https://cloud.google.com/python/docs/reference/bigframes/latest)." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    API
    0values
    1dtypes
    \n", - "

    2 rows × 1 columns

    \n", - "
    [2 rows x 1 columns in total]" - ], - "text/plain": [ - " API\n", - "0 values\n", - "1 dtypes\n", - "\n", - "[2 rows x 1 columns]" + "cell_type": "markdown", + "metadata": { + "id": "d975e698c9a4" + }, + "source": [ + "### Objective\n", + "\n", + "In this tutorial, you create a CSV file containing sample code for calling a given set of APIs.\n", + "\n", + "The steps include:\n", + "\n", + "- Defining an LLM model in BigQuery DataFrames, specifically the [Gemini Model](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models), using `bigframes.ml.llm`.\n", + "- Creating a DataFrame by reading in data from Cloud Storage.\n", + "- Manipulating data in the DataFrame to build LLM prompts.\n", + "- Sending DataFrame prompts to the LLM model using the `predict` method.\n", + "- Creating and using a custom function to transform the output provided by the LLM model response.\n", + "- Exporting the resulting transformed DataFrame as a CSV file." ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_api.head(2)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "id": "BGJnZbgEGS5-" - }, - "outputs": [ + }, { - "data": { - "text/html": [ - "Query job 41e4f2e7-689a-45d9-bf92-4416f5560b81 is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "08d289fa873f" + }, + "source": [ + "### Dataset\n", + "\n", + "This tutorial uses a dataset listing the names of various pandas DataFrame and Series APIs." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job aae0b164-f786-4734-8c79-2af9805af0cf is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "aed92deeb4a0" + }, + "source": [ + "### Costs\n", + "\n", + "This tutorial uses billable components of Google Cloud:\n", + "\n", + "* BigQuery\n", + "* Generative AI support on Vertex AI\n", + "* Cloud Functions\n", + "\n", + "Learn about [BigQuery compute pricing](https://cloud.google.com/bigquery/pricing#analysis_pricing_models),\n", + "[Generative AI support on Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing#generative_ai_models), and [Cloud Functions pricing](https://cloud.google.com/functions/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n", + "to generate a cost estimate based on your projected usage." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    API
    0shape
    1size
    \n", - "

    2 rows × 1 columns

    \n", - "
    [2 rows x 1 columns in total]" - ], - "text/plain": [ - " API\n", - "0 shape\n", - "1 size\n", - "\n", - "[2 rows x 1 columns]" + "cell_type": "markdown", + "metadata": { + "id": "i7EUnXsZhAGF" + }, + "source": [ + "## Installation\n", + "\n", + "Install the following packages, which are required to run this notebook:" ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "series_api.head(2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "m3ZJEsi7SUKV" - }, - "source": [ - "# Generate code using the LLM model\n", - "\n", - "Prepare the prompts and send them to the LLM model for prediction." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9EMAqR37AfLS" - }, - "source": [ - "## Prompt design in BigQuery DataFrames\n", - "\n", - "Designing prompts for LLMs is a fast growing area and you can read more in [this documentation](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/introduction-prompt-design).\n", - "\n", - "For this tutorial, you use a simple prompt to ask the LLM model for sample code for each of the API methods (or rows) from the last step's DataFrames. The output is the new DataFrames `df_prompt` and `series_prompt`, which contain the full prompt text." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "id": "EDAaIwHpQCDZ" - }, - "outputs": [ + }, { - "data": { - "text/html": [ - "Query job 17f50c10-aa81-4023-b206-4ba59ddf2269 is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "2b4ef9b72d43" + }, + "outputs": [], + "source": [ + "!pip install bigframes --upgrade --quiet" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job d6d217aa-a623-4ea4-83fb-8f1b8bfb8e68 is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "BF1j6f9HApxa" + }, + "source": [ + "## Before you begin\n", + "\n", + "Complete the tasks in this section to set up your environment." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job a275a107-752e-46f8-be9f-9cb35eb6b0b9 is DONE. 132 Bytes processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "Wbr2aVtFQBcg" + }, + "source": [ + "### Set up your Google Cloud project\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 credit towards your compute/storage costs.\n", + "\n", + "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", + "\n", + "3. [Click here](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com,bigqueryconnection.googleapis.com,cloudfunctions.googleapis.com,run.googleapis.com,artifactregistry.googleapis.com,cloudbuild.googleapis.com,cloudresourcemanager.googleapis.com) to enable the following APIs:\n", + "\n", + " * BigQuery API\n", + " * BigQuery Connection API\n", + " * Cloud Functions API\n", + " * Cloud Run API\n", + " * Artifact Registry API\n", + " * Cloud Build API\n", + " * Cloud Resource Manager API\n", + " * Vertex AI API\n", + "\n", + "4. If you are running this notebook locally, install the [Cloud SDK](https://cloud.google.com/sdk)." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/plain": [ - "0 Generate Pandas sample code for DataFrame.values\n", - "1 Generate Pandas sample code for DataFrame.dtypes\n", - "Name: API, dtype: string" + "cell_type": "markdown", + "metadata": { + "id": "WReHDGG5g0XY" + }, + "source": [ + "#### Set your project ID\n", + "\n", + "If you don't know your project ID, try the following:\n", + "* Run `gcloud config list`.\n", + "* Run `gcloud projects list`.\n", + "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)." ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_prompt_prefix = \"Generate Pandas sample code for DataFrame.\"\n", - "series_prompt_prefix = \"Generate Pandas sample code for Series.\"\n", - "\n", - "df_prompt = (df_prompt_prefix + df_api['API'])\n", - "series_prompt = (series_prompt_prefix + series_api['API'])\n", - "\n", - "df_prompt.head(2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "rwPLjqW2Ajzh" - }, - "source": [ - "## Make predictions using the LLM model\n", - "\n", - "Use the BigQuery DataFrames DataFrame containing the full prompt text as the input to the `predict` method. The `predict` method calls the LLM model and returns its generated text output back to two new BigQuery DataFrames DataFrames, `df_pred` and `series_pred`.\n", - "\n", - "Note: The predictions might take a few minutes to run." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "id": "6i6HkFJZa8na" - }, - "outputs": [ + }, { - "data": { - "text/html": [ - "Query job 01f95d2d-901d-4edf-bd3a-245d17c31ef6 is DONE. 0 Bytes processed. Open Job" + "cell_type": "code", + "execution_count": 27, + "metadata": { + "id": "oM1iC_MfAts1" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;31mERROR:\u001b[0m (gcloud.config.set) argument VALUE: Must be specified.\n", + "Usage: gcloud config set SECTION/PROPERTY VALUE [optional flags]\n", + " optional flags may be --help | --installation\n", + "\n", + "For detailed information on this command and its flags, run:\n", + " gcloud config set --help\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "PROJECT_ID = \"\" # @param {type:\"string\"}\n", + "\n", + "# Set the project id\n", + "! gcloud config set project {PROJECT_ID}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "region" + }, + "source": [ + "#### Set the region\n", + "\n", + "You can also change the `REGION` variable used by BigQuery. Learn more about [BigQuery regions](https://cloud.google.com/bigquery/docs/locations#supported_locations)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "eF-Twtc4XGem" + }, + "outputs": [], + "source": [ + "REGION = \"US\" # @param {type: \"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sBCra4QMA2wR" + }, + "source": [ + "### Authenticate your Google Cloud account\n", + "\n", + "Depending on your Jupyter environment, you might have to manually authenticate. Follow the relevant instructions below." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "74ccc9e52986" + }, + "source": [ + "**Vertex AI Workbench**\n", + "\n", + "Do nothing, you are already authenticated." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "de775a3773ba" + }, + "source": [ + "**Local JupyterLab instance**\n", + "\n", + "Uncomment and run the following cell:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "254614fa0c46" + }, + "outputs": [], + "source": [ + "# ! gcloud auth login" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job 55927a6f-b023-479a-b9bf-826abde77111 is DONE. 584 Bytes processed. Open Job" + "cell_type": "markdown", + "metadata": { + "id": "ef21552ccea8" + }, + "source": [ + "**Colab**\n", + "\n", + "Uncomment and run the following cell:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "603adbbf0532" + }, + "outputs": [], + "source": [ + "# from google.colab import auth\n", + "# auth.authenticate_user()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "960505627ddf" + }, + "source": [ + "### Import libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "PyQmSRbKA8r-" + }, + "outputs": [], + "source": [ + "import bigframes.pandas as bf\n", + "from google.cloud import bigquery\n", + "from google.cloud import bigquery_connection_v1 as bq_connection" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "init_aip:mbsdk,all" + }, + "source": [ + "### Set BigQuery DataFrames options" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "NPPMuw2PXGeo" + }, + "outputs": [], + "source": [ + "# Note: The project option is not required in all environments.\n", + "# On BigQuery Studio, the project ID is automatically detected.\n", + "bf.options.bigquery.project = PROJECT_ID\n", + "\n", + "# Note: The location option is not required.\n", + "# It defaults to the location of the first table or query\n", + "# passed to read_gbq(). For APIs where a location can't be\n", + "# auto-detected, the location defaults to the \"US\" location.\n", + "bf.options.bigquery.location = REGION" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DTVtFlqeFbrU" + }, + "source": [ + "If you want to reset the location of the created DataFrame or Series objects, reset the session by executing `bf.close_session()`. After that, you can reuse `bf.options.bigquery.location` to specify another location." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6eytf4xQHzcF" + }, + "source": [ + "# Define the LLM model\n", + "\n", + "BigQuery DataFrames provides integration with [Gemini Models](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models) via Vertex AI.\n", + "\n", + "This section walks through a few steps required in order to use the model in your notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qUjT8nw-jIXp" + }, + "source": [ + "## Define the model\n", + "\n", + "Use `bigframes.ml.llm` to define the model:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "id": "sdjeXFwcHfl7" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 0ee1a08e-788e-4fc7-b061-52c23ab25d5a is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } ], - "text/plain": [ - "" + "source": [ + "from bigframes.ml.llm import GeminiTextGenerator\n", + "\n", + "model = GeminiTextGenerator(model_name=\"gemini-2.0-flash-001\")" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job 445eb0af-f643-40c5-9c1e-25aa3db8374a is DONE. 146 Bytes processed. Open Job" + "cell_type": "markdown", + "metadata": { + "id": "GbW0oCnU1s1N" + }, + "source": [ + "# Read data from Cloud Storage into BigQuery DataFrames\n", + "\n", + "You can create a BigQuery DataFrames DataFrame by reading data from any of the following locations:\n", + "\n", + "* A local data file\n", + "* Data stored in a BigQuery table\n", + "* A data file stored in Cloud Storage\n", + "* An in-memory pandas DataFrame\n", + "\n", + "In this tutorial, you create BigQuery DataFrames DataFrames by reading two CSV files stored in Cloud Storage, one containing a list of DataFrame API names and one containing a list of Series API names." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "id": "SchiTkQGIJog" + }, + "outputs": [], + "source": [ + "df_api = bf.read_csv(\"gs://cloud-samples-data/vertex-ai/bigframe/df.csv\")\n", + "series_api = bf.read_csv(\"gs://cloud-samples-data/vertex-ai/bigframe/series.csv\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7OBjw2nmQY3-" + }, + "source": [ + "Take a peek at a few rows of data for each file:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "id": "QCqgVCIsGGuv" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 48be241c-ee93-4dfa-a9e3-66b64c4b5150 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 6af9caa5-4f7a-48f0-a7df-d692ee063b7e is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    API
    0values
    1dtypes
    \n", + "

    2 rows × 1 columns

    \n", + "
    [2 rows x 1 columns in total]" + ], + "text/plain": [ + " API\n", + "0 values\n", + "1 dtypes\n", + "\n", + "[2 rows x 1 columns]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "df_api.head(2)" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job ddee268c-773a-4dcc-b14c-ebdd90c2c347 is DONE. 0 Bytes processed. Open Job" + "cell_type": "code", + "execution_count": 13, + "metadata": { + "id": "BGJnZbgEGS5-" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 41e4f2e7-689a-45d9-bf92-4416f5560b81 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job aae0b164-f786-4734-8c79-2af9805af0cf is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    API
    0shape
    1size
    \n", + "

    2 rows × 1 columns

    \n", + "
    [2 rows x 1 columns in total]" + ], + "text/plain": [ + " API\n", + "0 shape\n", + "1 size\n", + "\n", + "[2 rows x 1 columns]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "series_api.head(2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "m3ZJEsi7SUKV" + }, + "source": [ + "# Generate code using the LLM model\n", + "\n", + "Prepare the prompts and send them to the LLM model for prediction." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9EMAqR37AfLS" + }, + "source": [ + "## Prompt design in BigQuery DataFrames\n", + "\n", + "Designing prompts for LLMs is a fast growing area and you can read more in [this documentation](https://cloud.google.com/vertex-ai/docs/generative-ai/learn/introduction-prompt-design).\n", + "\n", + "For this tutorial, you use a simple prompt to ask the LLM model for sample code for each of the API methods (or rows) from the last step's DataFrames. The output is the new DataFrames `df_prompt` and `series_prompt`, which contain the full prompt text." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job d7f1eb26-28b2-44ba-8858-5cd4df8621bd is DONE. 904 Bytes processed. Open Job" + "cell_type": "code", + "execution_count": 14, + "metadata": { + "id": "EDAaIwHpQCDZ" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 17f50c10-aa81-4023-b206-4ba59ddf2269 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job d6d217aa-a623-4ea4-83fb-8f1b8bfb8e68 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job a275a107-752e-46f8-be9f-9cb35eb6b0b9 is DONE. 132 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "0 Generate Pandas sample code for DataFrame.values\n", + "1 Generate Pandas sample code for DataFrame.dtypes\n", + "Name: API, dtype: string" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "df_prompt_prefix = \"Generate Pandas sample code for DataFrame.\"\n", + "series_prompt_prefix = \"Generate Pandas sample code for Series.\"\n", + "\n", + "df_prompt = (df_prompt_prefix + df_api['API'])\n", + "series_prompt = (series_prompt_prefix + series_api['API'])\n", + "\n", + "df_prompt.head(2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rwPLjqW2Ajzh" + }, + "source": [ + "## Make predictions using the LLM model\n", + "\n", + "Use the BigQuery DataFrames DataFrame containing the full prompt text as the input to the `predict` method. The `predict` method calls the LLM model and returns its generated text output back to two new BigQuery DataFrames DataFrames, `df_pred` and `series_pred`.\n", + "\n", + "Note: The predictions might take a few minutes to run." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job f24d27a5-0e36-4fb5-953b-d09298f83af6 is DONE. 226 Bytes processed. Open Job" + "cell_type": "code", + "execution_count": 15, + "metadata": { + "id": "6i6HkFJZa8na" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 01f95d2d-901d-4edf-bd3a-245d17c31ef6 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 55927a6f-b023-479a-b9bf-826abde77111 is DONE. 584 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 445eb0af-f643-40c5-9c1e-25aa3db8374a is DONE. 146 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job ddee268c-773a-4dcc-b14c-ebdd90c2c347 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job d7f1eb26-28b2-44ba-8858-5cd4df8621bd is DONE. 904 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job f24d27a5-0e36-4fb5-953b-d09298f83af6 is DONE. 226 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } ], - "text/plain": [ - "" + "source": [ + "df_pred = model.predict(df_prompt.to_frame(), max_output_tokens=1024)\n", + "series_pred = model.predict(series_prompt.to_frame(), max_output_tokens=1024)" ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "df_pred = model.predict(df_prompt.to_frame(), max_output_tokens=1024)\n", - "series_pred = model.predict(series_prompt.to_frame(), max_output_tokens=1024)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "89cB8MW4UIdV" - }, - "source": [ - "Once the predictions are processed, take a look at the sample output from the LLM, which provides code samples for the API names listed in the DataFrames dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "id": "9A2gw6hP_2nX" - }, - "outputs": [ + }, { - "data": { - "text/html": [ - "Query job 65599c98-72ad-4088-8b09-f29bf05c164b is DONE. 21.8 kB processed. Open Job" + "cell_type": "markdown", + "metadata": { + "id": "89cB8MW4UIdV" + }, + "source": [ + "Once the predictions are processed, take a look at the sample output from the LLM, which provides code samples for the API names listed in the DataFrames dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "id": "9A2gw6hP_2nX" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 65599c98-72ad-4088-8b09-f29bf05c164b is DONE. 21.8 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "```python\n", + "import pandas as pd\n", + "\n", + "# Create a DataFrame\n", + "df = pd.DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]])\n", + "\n", + "# Get the values as a NumPy array\n", + "values = df.values\n", + "\n", + "# Print the values\n", + "print(values)\n", + "```\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "print(df_pred['ml_generate_text_llm_result'].iloc[0])" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "```python\n", - "import pandas as pd\n", - "\n", - "# Create a DataFrame\n", - "df = pd.DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]])\n", - "\n", - "# Get the values as a NumPy array\n", - "values = df.values\n", - "\n", - "# Print the values\n", - "print(values)\n", - "```\n" - ] - } - ], - "source": [ - "print(df_pred['ml_generate_text_llm_result'].iloc[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Fx4lsNqMorJ-" - }, - "source": [ - "# Manipulate LLM output using a remote function\n", - "\n", - "The output that the LLM provides often contains additional text beyond the code sample itself. Using BigQuery DataFrames, you can deploy custom Python functions that process and transform this output.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "d8L7SN03VByG" - }, - "source": [ - "Running the cell below creates a custom function that you can use to process the LLM output data in two ways:\n", - "1. Strip the LLM text output to include only the code block.\n", - "2. Substitute `import pandas as pd` with `import bigframes.pandas as bf` so that the resulting code block works with BigQuery DataFrames." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "id": "GskyyUQPowBT" - }, - "outputs": [], - "source": [ - "@bf.remote_function(cloud_function_service_account=\"default\")\n", - "def extract_code(text: str) -> str:\n", - " try:\n", - " res = text[text.find('\\n')+1:text.find('```', 3)]\n", - " res = res.replace(\"import pandas as pd\", \"import bigframes.pandas as bf\")\n", - " if \"import bigframes.pandas as bf\" not in res:\n", - " res = \"import bigframes.pandas as bf\\n\" + res\n", - " return res\n", - " except:\n", - " return \"\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "hVQAoqBUOJQf" - }, - "source": [ - "The custom function is deployed as a Cloud Function, and then integrated with BigQuery as a [remote function](https://cloud.google.com/bigquery/docs/remote-functions). Save both of the function names so that you can clean them up at the end of this notebook." - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "id": "PBlp-C-DOHRO" - }, - "outputs": [ + "cell_type": "markdown", + "metadata": { + "id": "Fx4lsNqMorJ-" + }, + "source": [ + "# Manipulate LLM output using a remote function\n", + "\n", + "The output that the LLM provides often contains additional text beyond the code sample itself. Using BigQuery DataFrames, you can deploy custom Python functions that process and transform this output.\n", + "\n" + ] + }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "Cloud Function Name projects/swast-scratch/locations/us-central1/functions/bigframes-6e7606963c3f06b8181b3cb9449a4363\n", - "Remote Function Name swast-scratch._63cfa399614a54153cc386c27d6c0c6fdb249f9e.bigframes_6e7606963c3f06b8181b3cb9449a4363\n" - ] - } - ], - "source": [ - "CLOUD_FUNCTION_NAME = format(extract_code.bigframes_cloud_function)\n", - "print(\"Cloud Function Name \" + CLOUD_FUNCTION_NAME)\n", - "REMOTE_FUNCTION_NAME = format(extract_code.bigframes_remote_function)\n", - "print(\"Remote Function Name \" + REMOTE_FUNCTION_NAME)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4FEucaiqVs3H" - }, - "source": [ - "Apply the custom function to each LLM output DataFrame to get the processed results:" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "id": "bsQ9cmoWo0Ps" - }, - "outputs": [ + "cell_type": "markdown", + "metadata": { + "id": "d8L7SN03VByG" + }, + "source": [ + "Running the cell below creates a custom function that you can use to process the LLM output data in two ways:\n", + "1. Strip the LLM text output to include only the code block.\n", + "2. Substitute `import pandas as pd` with `import bigframes.pandas as bf` so that the resulting code block works with BigQuery DataFrames." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "id": "GskyyUQPowBT" + }, + "outputs": [], + "source": [ + "@bf.remote_function(cloud_function_service_account=\"default\")\n", + "def extract_code(text: str) -> str:\n", + " try:\n", + " res = text[text.find('\\n')+1:text.find('```', 3)]\n", + " res = res.replace(\"import pandas as pd\", \"import bigframes.pandas as bf\")\n", + " if \"import bigframes.pandas as bf\" not in res:\n", + " res = \"import bigframes.pandas as bf\\n\" + res\n", + " return res\n", + " except:\n", + " return \"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hVQAoqBUOJQf" + }, + "source": [ + "The custom function is deployed as a Cloud Function, and then integrated with BigQuery as a [remote function](https://cloud.google.com/bigquery/docs/remote-functions). Save both of the function names so that you can clean them up at the end of this notebook." + ] + }, { - "data": { - "text/html": [ - "Query job 047903f8-ea67-430a-8281-8fb5a119b779 is DONE. 21.8 kB processed. Open Job" + "cell_type": "code", + "execution_count": 18, + "metadata": { + "id": "PBlp-C-DOHRO" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cloud Function Name projects/swast-scratch/locations/us-central1/functions/bigframes-6e7606963c3f06b8181b3cb9449a4363\n", + "Remote Function Name swast-scratch._63cfa399614a54153cc386c27d6c0c6fdb249f9e.bigframes_6e7606963c3f06b8181b3cb9449a4363\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "CLOUD_FUNCTION_NAME = format(extract_code.bigframes_cloud_function)\n", + "print(\"Cloud Function Name \" + CLOUD_FUNCTION_NAME)\n", + "REMOTE_FUNCTION_NAME = format(extract_code.bigframes_remote_function)\n", + "print(\"Remote Function Name \" + REMOTE_FUNCTION_NAME)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4FEucaiqVs3H" + }, + "source": [ + "Apply the custom function to each LLM output DataFrame to get the processed results:" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job 793df956-0b1a-46ba-bb5e-e428171f3bd0 is DONE. 26.3 kB processed. Open Job" + "cell_type": "code", + "execution_count": 19, + "metadata": { + "id": "bsQ9cmoWo0Ps" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 047903f8-ea67-430a-8281-8fb5a119b779 is DONE. 21.8 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 793df956-0b1a-46ba-bb5e-e428171f3bd0 is DONE. 26.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } ], - "text/plain": [ - "" + "source": [ + "df_code = df_pred.assign(code=df_pred['ml_generate_text_llm_result'].apply(extract_code))\n", + "series_code = series_pred.assign(code=series_pred['ml_generate_text_llm_result'].apply(extract_code))" ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "df_code = df_pred.assign(code=df_pred['ml_generate_text_llm_result'].apply(extract_code))\n", - "series_code = series_pred.assign(code=series_pred['ml_generate_text_llm_result'].apply(extract_code))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ujQVVuhfWA3y" - }, - "source": [ - "You can see the differences by inspecting the first row of data:" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "id": "7yWzjhGy_zcy" - }, - "outputs": [ + }, { - "data": { - "text/html": [ - "Query job 6974c2b7-2ed9-4564-a80b-57aef6959e19 is DONE. 22.8 kB processed. Open Job" + "cell_type": "markdown", + "metadata": { + "id": "ujQVVuhfWA3y" + }, + "source": [ + "You can see the differences by inspecting the first row of data:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "id": "7yWzjhGy_zcy" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 6974c2b7-2ed9-4564-a80b-57aef6959e19 is DONE. 22.8 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "import bigframes.pandas as bf\n", + "\n", + "# Create a DataFrame\n", + "df = pd.DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]])\n", + "\n", + "# Get the values as a NumPy array\n", + "values = df.values\n", + "\n", + "# Print the values\n", + "print(values)\n", + "\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "print(df_code['code'].iloc[0])" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "import bigframes.pandas as bf\n", - "\n", - "# Create a DataFrame\n", - "df = pd.DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]])\n", - "\n", - "# Get the values as a NumPy array\n", - "values = df.values\n", - "\n", - "# Print the values\n", - "print(values)\n", - "\n" - ] - } - ], - "source": [ - "print(df_code['code'].iloc[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GTRdUw-Ro5R1" - }, - "source": [ - "# Save the results to Cloud Storage\n", - "\n", - "BigQuery DataFrames lets you save a BigQuery DataFrames DataFrame as a CSV file in Cloud Storage for further use. Try that now with your processed LLM output data." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9DQ7eiQxPTi3" - }, - "source": [ - "Create a new Cloud Storage bucket with a unique name:" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "id": "-J5LHgS6LLZ0" - }, - "outputs": [ + "cell_type": "markdown", + "metadata": { + "id": "GTRdUw-Ro5R1" + }, + "source": [ + "# Save the results to Cloud Storage\n", + "\n", + "BigQuery DataFrames lets you save a BigQuery DataFrames DataFrame as a CSV file in Cloud Storage for further use. Try that now with your processed LLM output data." + ] + }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "Creating gs://code-samples-773ee0f2-e302-11ee-8298-4201c0a8181f/...\n" - ] - } - ], - "source": [ - "import uuid\n", - "BUCKET_ID = \"code-samples-\" + str(uuid.uuid1())\n", - "\n", - "!gcloud storage buckets create gs://{BUCKET_ID}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tyxZXj0UPYUv" - }, - "source": [ - "Use `to_csv` to write each BigQuery DataFrames DataFrame as a CSV file in the Cloud Storage bucket:" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "id": "Zs_b5L-4IvER" - }, - "outputs": [ + "cell_type": "markdown", + "metadata": { + "id": "9DQ7eiQxPTi3" + }, + "source": [ + "Create a new Cloud Storage bucket with a unique name:" + ] + }, { - "data": { - "text/html": [ - "Query job 81277037-032f-4557-a46e-1d39702f33d5 is DONE. 22.8 kB processed. Open Job" + "cell_type": "code", + "execution_count": 21, + "metadata": { + "id": "-J5LHgS6LLZ0" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating gs://code-samples-773ee0f2-e302-11ee-8298-4201c0a8181f/...\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "import uuid\n", + "BUCKET_ID = \"code-samples-\" + str(uuid.uuid1())\n", + "\n", + "!gcloud storage buckets create gs://{BUCKET_ID}" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job 8dc5a38c-ac16-44e7-83dd-4187380f780f is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "tyxZXj0UPYUv" + }, + "source": [ + "Use `to_csv` to write each BigQuery DataFrames DataFrame as a CSV file in the Cloud Storage bucket:" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job 9087a758-b1f9-4be7-889b-7761ef0ad966 is DONE. 27.7 kB processed. Open Job" + "cell_type": "code", + "execution_count": 22, + "metadata": { + "id": "Zs_b5L-4IvER" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 81277037-032f-4557-a46e-1d39702f33d5 is DONE. 22.8 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 8dc5a38c-ac16-44e7-83dd-4187380f780f is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 9087a758-b1f9-4be7-889b-7761ef0ad966 is DONE. 27.7 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 6126ea72-c6f7-43f0-8888-e1c2a464a8a4 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } ], - "text/plain": [ - "" + "source": [ + "df_code[[\"code\"]].to_csv(f\"gs://{BUCKET_ID}/df_code*.csv\")\n", + "series_code[[\"code\"]].to_csv(f\"gs://{BUCKET_ID}/series_code*.csv\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UDBtDlrTuuh8" + }, + "source": [ + "You can navigate to the Cloud Storage bucket browser to download the two files and view them.\n", + "\n", + "Run the following cell, and then follow the link to your Cloud Storage bucket browser:" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job 6126ea72-c6f7-43f0-8888-e1c2a464a8a4 is DONE. 0 Bytes processed. Open Job" + "cell_type": "code", + "execution_count": 23, + "metadata": { + "id": "PspCXu-qu_ND" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://console.developers.google.com/storage/browser/code-samples-773ee0f2-e302-11ee-8298-4201c0a8181f/\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "print(f'https://console.developers.google.com/storage/browser/{BUCKET_ID}/')" ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "df_code[[\"code\"]].to_csv(f\"gs://{BUCKET_ID}/df_code*.csv\")\n", - "series_code[[\"code\"]].to_csv(f\"gs://{BUCKET_ID}/series_code*.csv\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "UDBtDlrTuuh8" - }, - "source": [ - "You can navigate to the Cloud Storage bucket browser to download the two files and view them.\n", - "\n", - "Run the following cell, and then follow the link to your Cloud Storage bucket browser:" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "id": "PspCXu-qu_ND" - }, - "outputs": [ + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RGSvUk48RK20" + }, + "source": [ + "# Summary and next steps\n", + "\n", + "You've used BigQuery DataFrames' integration with LLM models (`bigframes.ml.llm`) to generate code samples, and have tranformed LLM output by creating and using a custom function in BigQuery DataFrames.\n", + "\n", + "Learn more about BigQuery DataFrames in the [documentation](https://cloud.google.com/python/docs/reference/bigframes/latest) and find more sample notebooks in the [GitHub repo](https://github.com/googleapis/python-bigquery-dataframes/tree/main/notebooks)." + ] + }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "https://console.developers.google.com/storage/browser/code-samples-773ee0f2-e302-11ee-8298-4201c0a8181f/\n" - ] + "cell_type": "markdown", + "metadata": { + "id": "TpV-iwP9qw9c" + }, + "source": [ + "## Cleaning up\n", + "\n", + "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n", + "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", + "\n", + "Otherwise, you can uncomment the remaining cells and run them to delete the individual resources you created in this tutorial:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bf.close_session()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "id": "yw7A461XLjvW" + }, + "outputs": [], + "source": [ + "# # Delete the BigQuery Connection\n", + "# from google.cloud import bigquery_connection_v1 as bq_connection\n", + "# client = bq_connection.ConnectionServiceClient()\n", + "# CONNECTION_ID = f\"projects/{PROJECT_ID}/locations/{REGION}/connections/{CONN_NAME}\"\n", + "# client.delete_connection(name=CONNECTION_ID)\n", + "# print(f\"Deleted connection '{CONNECTION_ID}'.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "id": "sx_vKniMq9ZX" + }, + "outputs": [], + "source": [ + "# # Delete the Cloud Function\n", + "# ! gcloud functions delete {CLOUD_FUNCTION_NAME} --quiet\n", + "# # Delete the Remote Function\n", + "# REMOTE_FUNCTION_NAME = REMOTE_FUNCTION_NAME.replace(PROJECT_ID + \".\", \"\")\n", + "# ! bq rm --routine --force=true {REMOTE_FUNCTION_NAME}" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "id": "iQFo6OUBLmi3" + }, + "outputs": [], + "source": [ + "# # Delete the Google Cloud Storage bucket and files\n", + "# ! gcloud storage rm gs://{BUCKET_ID} --recursive\n", + "# print(f\"Deleted bucket '{BUCKET_ID}'.\")" + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.1" } - ], - "source": [ - "print(f'https://console.developers.google.com/storage/browser/{BUCKET_ID}/')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RGSvUk48RK20" - }, - "source": [ - "# Summary and next steps\n", - "\n", - "You've used BigQuery DataFrames' integration with LLM models (`bigframes.ml.llm`) to generate code samples, and have tranformed LLM output by creating and using a custom function in BigQuery DataFrames.\n", - "\n", - "Learn more about BigQuery DataFrames in the [documentation](https://cloud.google.com/python/docs/reference/bigframes/latest) and find more sample notebooks in the [GitHub repo](https://github.com/googleapis/python-bigquery-dataframes/tree/main/notebooks)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TpV-iwP9qw9c" - }, - "source": [ - "## Cleaning up\n", - "\n", - "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n", - "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", - "\n", - "Otherwise, you can uncomment the remaining cells and run them to delete the individual resources you created in this tutorial:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "bf.close_session()" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "id": "yw7A461XLjvW" - }, - "outputs": [], - "source": [ - "# # Delete the BigQuery Connection\n", - "# from google.cloud import bigquery_connection_v1 as bq_connection\n", - "# client = bq_connection.ConnectionServiceClient()\n", - "# CONNECTION_ID = f\"projects/{PROJECT_ID}/locations/{REGION}/connections/{CONN_NAME}\"\n", - "# client.delete_connection(name=CONNECTION_ID)\n", - "# print(f\"Deleted connection '{CONNECTION_ID}'.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "id": "sx_vKniMq9ZX" - }, - "outputs": [], - "source": [ - "# # Delete the Cloud Function\n", - "# ! gcloud functions delete {CLOUD_FUNCTION_NAME} --quiet\n", - "# # Delete the Remote Function\n", - "# REMOTE_FUNCTION_NAME = REMOTE_FUNCTION_NAME.replace(PROJECT_ID + \".\", \"\")\n", - "# ! bq rm --routine --force=true {REMOTE_FUNCTION_NAME}" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": { - "id": "iQFo6OUBLmi3" - }, - "outputs": [], - "source": [ - "# # Delete the Google Cloud Storage bucket and files\n", - "# ! gcloud storage rm gs://{BUCKET_ID} --recursive\n", - "# print(f\"Deleted bucket '{BUCKET_ID}'.\")" - ] - } - ], - "metadata": { - "colab": { - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "venv (3.10.14)", - "language": "python", - "name": "python3" }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 0 + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/notebooks/generative_ai/bq_dataframes_llm_gemini_2.ipynb b/notebooks/generative_ai/bq_dataframes_llm_gemini_2.ipynb new file mode 100644 index 00000000000..1a9b5688975 --- /dev/null +++ b/notebooks/generative_ai/bq_dataframes_llm_gemini_2.ipynb @@ -0,0 +1,377 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Copyright 2024 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# BigFrames Gemini 2.0 Text Generation Simple Example" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: This feature is only available in bigframes >= 1.29.0" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Import packages" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import bigframes.pandas as bpd\n", + "from bigframes.ml import llm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create Gemini 2.0 experimental Model with model_name as \"gemini-2.0-flash-exp\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/garrettwu/src/bigframes/bigframes/ml/llm.py:803: PreviewWarning: Model gemini-2.0-flash-exp is subject to the \"Pre-GA Offerings Terms\" in the General Service Terms section of the\n", + " Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available \"as is\"\n", + " and might have limited support. For more information, see the launch stage descriptions\n", + " (https://cloud.google.com/products#product-launch-stages).\n", + " warnings.warn(\n", + "/usr/local/google/home/garrettwu/src/bigframes/bigframes/pandas/__init__.py:435: DefaultLocationWarning: No explicit location is set, so using location US for the session.\n", + " return global_session.get_global_session()\n" + ] + }, + { + "data": { + "text/html": [ + "Query job f673a2ea-023e-4771-84a2-fb81f808fa1b is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "model = llm.GeminiTextGenerator(model_name=\"gemini-2.0-flash-exp\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a simple DataFrame" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 2276ea5b-2e08-4ed6-af34-49a7d165d145 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    prompt
    0Tell me something about Gemini 2.0.
    \n", + "

    1 rows × 1 columns

    \n", + "
    [1 rows x 1 columns in total]" + ], + "text/plain": [ + " prompt\n", + "0 Tell me something about Gemini 2.0.\n", + "\n", + "[1 rows x 1 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = bpd.DataFrame({\"prompt\": [\"Tell me something about Gemini 2.0.\"]})\n", + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Make predictions" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 9ba21e96-6023-491e-8e83-f2e6fa7df0e7 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/garrettwu/src/bigframes/bigframes/core/__init__.py:109: PreviewWarning: Interpreting JSON column(s) as StringDtype. This behavior may change in future versions.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "text/html": [ + "Query job 933d45cc-4bc0-4bdf-b4b8-573da2d58be3 is DONE. 2 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 3dda9bc6-84b1-4f4a-8891-85d25d8848ce is DONE. 4.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    ml_generate_text_llm_resultml_generate_text_rai_resultml_generate_text_statusprompt
    0Alright, let's talk about Gemini 2.0! It's a b...<NA>Tell me something about Gemini 2.0.
    \n", + "

    1 rows × 4 columns

    \n", + "
    [1 rows x 4 columns in total]" + ], + "text/plain": [ + " ml_generate_text_llm_result \\\n", + "0 Alright, let's talk about Gemini 2.0! It's a b... \n", + "\n", + " ml_generate_text_rai_result ml_generate_text_status \\\n", + "0 \n", + "\n", + " prompt \n", + "0 Tell me something about Gemini 2.0. \n", + "\n", + "[1 rows x 4 columns]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result = model.predict(df)\n", + "result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Save the model" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Copy job 8e68af62-e7ab-475b-99c9-b79e8ba3c40b is DONE. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/garrettwu/src/bigframes/bigframes/ml/llm.py:803: PreviewWarning: Model gemini-2.0-flash-exp is subject to the \"Pre-GA Offerings Terms\" in the General Service Terms section of the\n", + " Service Specific Terms(https://cloud.google.com/terms/service-terms#1). Pre-GA products and features are available \"as is\"\n", + " and might have limited support. For more information, see the launch stage descriptions\n", + " (https://cloud.google.com/products#product-launch-stages).\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "text/html": [ + "Query job cae7f929-d8cb-4819-a644-ac832cdc0912 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "GeminiTextGenerator(connection_name='bigframes-dev.us.bigframes-rf-connection',\n", + " model_name='gemini-2.0-flash-exp',\n", + " session=)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.to_gbq(\"bigframes-dev.garrettwu.gemini_2_flash\", replace=True)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.15" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/generative_ai/bq_dataframes_llm_kmeans.ipynb b/notebooks/generative_ai/bq_dataframes_llm_kmeans.ipynb index 2d5bb46d95e..08891d2b445 100644 --- a/notebooks/generative_ai/bq_dataframes_llm_kmeans.ipynb +++ b/notebooks/generative_ai/bq_dataframes_llm_kmeans.ipynb @@ -26,7 +26,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Use BigQuery DataFrames to cluster and characterize complaints\n", + "## Use BigQuery DataFrames to cluster and characterize complaints\n", "\n", "\n", "\n", @@ -1593,7 +1593,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": { "id": "mL5P0_3X04dE" }, @@ -1614,7 +1614,7 @@ "source": [ "from bigframes.ml.llm import GeminiTextGenerator\n", "\n", - "q_a_model = GeminiTextGenerator(model_name=\"gemini-2.5-flash\")" + "q_a_model = GeminiTextGenerator(model_name=\"gemini-2.0-flash-001\")" ] }, { diff --git a/notebooks/generative_ai/bq_dataframes_llm_output_schema.ipynb b/notebooks/generative_ai/bq_dataframes_llm_output_schema.ipynb index b3e2e4ebc84..5399363e348 100644 --- a/notebooks/generative_ai/bq_dataframes_llm_output_schema.ipynb +++ b/notebooks/generative_ai/bq_dataframes_llm_output_schema.ipynb @@ -169,9 +169,9 @@ "metadata": {}, "outputs": [], "source": [ - "PROJECT_ID = \"\" # @param {type:\"string\"}\n", + "PROJECT = \"\" # replace with your project\n", "import bigframes\n", - "bigframes.options.bigquery.project = PROJECT_ID\n", + "bigframes.options.bigquery.project = PROJECT\n", "bigframes.options.display.progress_bar = None\n", "\n", "import bigframes.pandas as bpd\n", diff --git a/notebooks/generative_ai/bq_dataframes_llm_vector_search.ipynb b/notebooks/generative_ai/bq_dataframes_llm_vector_search.ipynb index c9fa39926a9..72651f19729 100644 --- a/notebooks/generative_ai/bq_dataframes_llm_vector_search.ipynb +++ b/notebooks/generative_ai/bq_dataframes_llm_vector_search.ipynb @@ -1,1790 +1,1790 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "TpJu6BBeooES" - }, - "outputs": [], - "source": [ - "# Copyright 2023 Google LLC\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EQbZKS7_ooET" - }, - "source": [ - "# Build a Vector Search application using BigQuery DataFrames (aka BigFrames)\n", - "\n", - "
    \n", - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \"Colab Run in Colab\n", - " \n", - " \n", - " \n", - " \"GitHub\n", - " View on GitHub\n", - " \n", - " \n", - " \n", - " \"Vertex\n", - " Open in Vertex AI Workbench\n", - " \n", - " \n", - " \n", - " \"BQ\n", - " Open in BQ Studio\n", - " \n", - "
    \n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vFMjpPBo9aVv" - }, - "source": [ - "**Author:** Sudipto Guha (Google)\n", - "\n", - "**Last updated:** March 16th 2025" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "SHQ3Gx-oooEU" - }, - "source": [ - "## Overview\n", - "\n", - "This notebook will guide you through a practical example of using [BigFrames](https://github.com/googleapis/python-bigquery-dataframes/issues) to perform [vector search](https://cloud.google.com/bigquery/docs/vector-search-intro) and analysis on a patent dataset within BigQuery. We will leverage Python and BigFrames to efficiently process, analyze, and gain insights from a large-scale dataset without moving data from BigQuery.\n", - "\n", - "Here's a breakdown of what we'll cover:\n", - "\n", - "1. **Data Ingestion and Embedding Generation:**\n", - "We will start by reading a public patent dataset directly from BigQuery into a BigFrames DataFrame.\n", - "We'll demonstrate how to use BigFrames' `TextEmbeddingGenerator` to create text embeddings for the patent abstracts. This process converts the textual data into numerical vectors that capture the semantic meaning of each abstract.\n", - "We'll show how BigFrames efficiently performs this embedding generation within BigQuery, avoiding data transfer to the client-side.\n", - "Finally, we'll store the generated embeddings back into a new BigQuery table for subsequent analysis.\n", - "\n", - "2. **Indexing and Similarity Search:**\n", - "Here we'll create a vector index using BigFrames to enable fast and scalable similarity searches.\n", - "We'll demonstrate how to create an IVF index for efficient approximate nearest neighbor searches.\n", - "We'll then perform a vector search using a sample query string to find patents that are semantically similar to the query. This showcases how vector search goes beyond keyword matching to find relevant results based on meaning.\n", - "\n", - "3. **AI-Powered Summarization with Retrieval Augmented Generation (RAG):**\n", - "To further enhance the analysis, we'll implement a RAG pipeline.\n", - "We'll retrieve the top most similar patents based on the vector search results from step 2.\n", - "We'll use BigFrames' `GeminiTextGenerator` to create a prompt for an LLM to generate a concise summary of the retrieved patents.\n", - "This demonstrates how to combine vector search with generative AI to extract and synthesize meaningful insights from complex patent data.\n", - "\n", - "\n", - "We will tie these pieces together in Python using BigQuery DataFrames. [Click here](https://cloud.google.com/bigquery/docs/dataframes-quickstart) to learn more about BigQuery DataFrames!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EHjmqb-0ooEU" - }, - "source": [ - "### Dataset\n", - "\n", - "This notebook uses the [BQ Patents Public Dataset](https://bigquery.cloud.google.com/dataset/patents-public-data:patentsview)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "AqdihIDJooEU" - }, - "source": [ - "### Costs\n", - "\n", - "This tutorial uses billable components of Google Cloud:\n", - "\n", - "* BigQuery (compute)\n", - "* BigQuery ML\n", - "* Generative AI support on Vertex AI\n", - "\n", - "Learn about [BigQuery compute pricing](https://cloud.google.com/bigquery/pricing#analysis_pricing_models), [Generative AI support on Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing#generative_ai_models),\n", - "and [BigQuery ML pricing](https://cloud.google.com/bigquery/pricing#bqml),\n", - "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n", - "to generate a cost estimate based on your projected usage." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GqLjnm1hsKGU" - }, - "source": [ - "## Setup & initialization\n", - "\n", - "Make sure you have the required roles and permissions listed below:\n", - "\n", - "For [Vector embedding generation](https://cloud.google.com/bigquery/docs/generate-text-embedding#required_roles)\n", - "\n", - "For [Vector Index creation](https://cloud.google.com/bigquery/docs/vector-index#roles_and_permissions)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Z-mvYJUCooEV" - }, - "source": [ - "## Before you begin\n", - "\n", - "Complete the tasks in this section to set up your environment." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xn-v3mSvooEV" - }, - "source": [ - "### Set up your Google Cloud project\n", - "\n", - "**The following steps are required, regardless of your notebook environment.**\n", - "\n", - "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 credit towards your compute/storage costs.\n", - "\n", - "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", - "\n", - "3. [Click here](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com,bigqueryconnection.googleapis.com,aiplatform.googleapis.com) to enable the following APIs:\n", - "\n", - " * BigQuery API\n", - " * BigQuery Connection API\n", - " * Vertex AI API\n", - "\n", - "4. If you are running this notebook locally, install the [Cloud SDK](https://cloud.google.com/sdk)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Ioydzb_8ooEV" - }, - "source": [ - "#### Set your project ID\n", - "\n", - "**If you don't know your project ID**, see the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "executionInfo": { - "elapsed": 2, - "status": "ok", - "timestamp": 1742191597773, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 - }, - "id": "b8bKCfIiooEV" - }, - "outputs": [], - "source": [ - "# set your project ID below\n", - "PROJECT_ID = \"bigframes-dev\" # @param {type:\"string\"}\n", - "\n", - "# set your region\n", - "REGION = \"US\" # @param {type: \"string\"}\n", - "\n", - "# Set the project id in gcloud\n", - "#! gcloud config set project {PROJECT_ID}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GbUgWr6LooEV" - }, - "source": [ - "#### Authenticate your Google Cloud account\n", - "\n", - "Depending on your Jupyter environment, you might have to manually authenticate. Follow the relevant instructions below." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "U7ChP8jUooEV" - }, - "source": [ - "**Vertex AI Workbench**\n", - "\n", - "Do nothing, you are already authenticated." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "VfHOYcZZooEW" - }, - "source": [ - "**Local JupyterLab instance**\n", - "\n", - "Uncomment and run the following cell:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "3cGhUVM0ooEW" - }, - "outputs": [], - "source": [ - "# ! gcloud auth login" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "AoHnXlg-ooEW" - }, - "source": [ - "**Colab**\n", - "\n", - "Uncomment and run the following cell:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "executionInfo": { - "elapsed": 2, - "status": "ok", - "timestamp": 1742191608487, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "TpJu6BBeooES" + }, + "outputs": [], + "source": [ + "# Copyright 2023 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] }, - "id": "j3lmnsh7ooEW", - "outputId": "eb68daf5-5558-487a-91d2-4b4f9e476da0" - }, - "outputs": [], - "source": [ - "# from google.colab import auth\n", - "# auth.authenticate_user()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "a9gsyttuooEW" - }, - "source": [ - "Now we are ready to use BigQuery DataFrames!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xckgWno6ouHY" - }, - "source": [ - "## Step 1: Data Ingestion and Embedding Generation" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Hjg9jDN-ooEW" - }, - "source": [ - "Install libraries" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "executionInfo": { - "elapsed": 947, - "status": "ok", - "timestamp": 1742195413800, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "markdown", + "metadata": { + "id": "EQbZKS7_ooET" + }, + "source": [ + "## Build a Vector Search application using BigQuery DataFrames (aka BigFrames)\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \"Colab Run in Colab\n", + " \n", + " \n", + " \n", + " \"GitHub\n", + " View on GitHub\n", + " \n", + " \n", + " \n", + " \"Vertex\n", + " Open in Vertex AI Workbench\n", + " \n", + " \n", + " \n", + " \"BQ\n", + " Open in BQ Studio\n", + " \n", + "
    \n" + ] }, - "id": "R7STCS8xB5d2" - }, - "outputs": [], - "source": [ - "import bigframes.pandas as bf\n", - "import bigframes.ml as bf_ml\n", - "import bigframes.bigquery as bf_bq\n", - "import bigframes.ml.llm as bf_llm\n", - "\n", - "\n", - "from google.cloud import bigquery\n", - "from google.cloud import storage\n", - "\n", - "# Construct a BigQuery client object.\n", - "client = bigquery.Client()\n", - "\n", - "import pandas as pd\n", - "from IPython.display import Image, display\n", - "from PIL import Image as PILImage\n", - "import io\n", - "\n", - "import json\n", - "from IPython.display import Markdown\n", - "\n", - "# Note: The project option is not required in all environments.\n", - "# On BigQuery Studio, the project ID is automatically detected.\n", - "bf.options.bigquery.project = PROJECT_ID\n", - "bf.options.bigquery.location = REGION\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "iOFF9hrvs5WE" - }, - "source": [ - "Partial ordering mode allows BigQuery DataFrames to push down many more row and column filters. On large clustered and partitioned tables, this can greatly reduce the number of bytes scanned and computation slots used. This [blog post](https://medium.com/google-cloud/introducing-partial-ordering-mode-for-bigquery-dataframes-bigframes-ec35841d95c0) goes over it in more detail." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "executionInfo": { - "elapsed": 2, - "status": "ok", - "timestamp": 1742191620533, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "markdown", + "metadata": { + "id": "vFMjpPBo9aVv" + }, + "source": [ + "**Author:** Sudipto Guha (Google)\n", + "\n", + "**Last updated:** March 16th 2025" + ] }, - "id": "9Gil1Oaas7KA" - }, - "outputs": [], - "source": [ - "bf.options.bigquery.ordering_mode = \"partial\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "XGaGyyZsooEW" - }, - "source": [ - "If you want to reset the location of the created DataFrame or Series objects, reset the session by executing `bf.close_session()`. After that, you can reuse `bf.options.bigquery.location` to specify another location." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "v6FGschEowht" - }, - "source": [ - "Data Input - read the data from a publicly available BigQuery dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" + { + "cell_type": "markdown", + "metadata": { + "id": "SHQ3Gx-oooEU" + }, + "source": [ + "## Overview\n", + "\n", + "This notebook will guide you through a practical example of using [BigFrames](https://github.com/googleapis/python-bigquery-dataframes/issues) to perform [vector search](https://cloud.google.com/bigquery/docs/vector-search-intro) and analysis on a patent dataset within BigQuery. We will leverage Python and BigFrames to efficiently process, analyze, and gain insights from a large-scale dataset without moving data from BigQuery.\n", + "\n", + "Here's a breakdown of what we'll cover:\n", + "\n", + "1. **Data Ingestion and Embedding Generation:**\n", + "We will start by reading a public patent dataset directly from BigQuery into a BigFrames DataFrame.\n", + "We'll demonstrate how to use BigFrames' `TextEmbeddingGenerator` to create text embeddings for the patent abstracts. This process converts the textual data into numerical vectors that capture the semantic meaning of each abstract.\n", + "We'll show how BigFrames efficiently performs this embedding generation within BigQuery, avoiding data transfer to the client-side.\n", + "Finally, we'll store the generated embeddings back into a new BigQuery table for subsequent analysis.\n", + "\n", + "2. **Indexing and Similarity Search:**\n", + "Here we'll create a vector index using BigFrames to enable fast and scalable similarity searches.\n", + "We'll demonstrate how to create an IVF index for efficient approximate nearest neighbor searches.\n", + "We'll then perform a vector search using a sample query string to find patents that are semantically similar to the query. This showcases how vector search goes beyond keyword matching to find relevant results based on meaning.\n", + "\n", + "3. **AI-Powered Summarization with Retrieval Augmented Generation (RAG):**\n", + "To further enhance the analysis, we'll implement a RAG pipeline.\n", + "We'll retrieve the top most similar patents based on the vector search results from step 2.\n", + "We'll use BigFrames' `GeminiTextGenerator` to create a prompt for an LLM to generate a concise summary of the retrieved patents.\n", + "This demonstrates how to combine vector search with generative AI to extract and synthesize meaningful insights from complex patent data.\n", + "\n", + "\n", + "We will tie these pieces together in Python using BigQuery DataFrames. [Click here](https://cloud.google.com/bigquery/docs/dataframes-quickstart) to learn more about BigQuery DataFrames!" + ] }, - "executionInfo": { - "elapsed": 468, - "status": "ok", - "timestamp": 1742192516923, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "markdown", + "metadata": { + "id": "EHjmqb-0ooEU" + }, + "source": [ + "### Dataset\n", + "\n", + "This notebook uses the [BQ Patents Public Dataset](https://bigquery.cloud.google.com/dataset/patents-public-data:patentsview)." + ] }, - "id": "zDSwoBo1CU3G", - "outputId": "83edbc2f-5a23-407b-8890-f968eb31be44" - }, - "outputs": [], - "source": [ - "publications = bf.read_gbq('patents-public-data.google_patents_research.publications')" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 + { + "cell_type": "markdown", + "metadata": { + "id": "AqdihIDJooEU" + }, + "source": [ + "### Costs\n", + "\n", + "This tutorial uses billable components of Google Cloud:\n", + "\n", + "* BigQuery (compute)\n", + "* BigQuery ML\n", + "* Generative AI support on Vertex AI\n", + "\n", + "Learn about [BigQuery compute pricing](https://cloud.google.com/bigquery/pricing#analysis_pricing_models), [Generative AI support on Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing#generative_ai_models),\n", + "and [BigQuery ML pricing](https://cloud.google.com/bigquery/pricing#bqml),\n", + "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n", + "to generate a cost estimate based on your projected usage." + ] }, - "executionInfo": { - "elapsed": 6697, - "status": "ok", - "timestamp": 1742192524632, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "markdown", + "metadata": { + "id": "GqLjnm1hsKGU" + }, + "source": [ + "## Setup & initialization\n", + "\n", + "Make sure you have the required roles and permissions listed below:\n", + "\n", + "For [Vector embedding generation](https://cloud.google.com/bigquery/docs/generate-text-embedding#required_roles)\n", + "\n", + "For [Vector Index creation](https://cloud.google.com/bigquery/docs/vector-index#roles_and_permissions)" + ] }, - "id": "tYDoaKgJChiq", - "outputId": "9174da29-a051-4a99-e38f-6a2b09cfe4e9" - }, - "outputs": [], - "source": [ - "## create patents base table (subset of 10k out of ~110M records)\n", - "\n", - "keep = (publications.embedding_v1.str.len() > 0) & (publications.title.str.len() > 0) & (publications.abstract.str.len() > 30)\n", - "\n", - "## Choose 10000 random rows to analyze\n", - "publications = publications[keep].peek(10000)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 556 + { + "cell_type": "markdown", + "metadata": { + "id": "Z-mvYJUCooEV" + }, + "source": [ + "## Before you begin\n", + "\n", + "Complete the tasks in this section to set up your environment." + ] }, - "executionInfo": { - "elapsed": 6, - "status": "ok", - "timestamp": 1742191801044, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "markdown", + "metadata": { + "id": "xn-v3mSvooEV" + }, + "source": [ + "### Set up your Google Cloud project\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 credit towards your compute/storage costs.\n", + "\n", + "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", + "\n", + "3. [Click here](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com,bigqueryconnection.googleapis.com,aiplatform.googleapis.com) to enable the following APIs:\n", + "\n", + " * BigQuery API\n", + " * BigQuery Connection API\n", + " * Vertex AI API\n", + "\n", + "4. If you are running this notebook locally, install the [Cloud SDK](https://cloud.google.com/sdk)." + ] }, - "id": "XmqdJInztzPl", - "outputId": "ae05f3a6-edeb-423a-c061-c416717e1ec5" - }, - "outputs": [ { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    publication_numbertitletitle_translatedabstractabstract_translatedcpccpc_lowcpc_inventive_lowtop_termssimilarurlcountrypublication_descriptioncited_byembedding_v1
    0WO-2007022924-B1Pharmaceutical compositions with melting point...FalseThe invention relates to the use of chemical f...False[{'code': 'A61K47/32', 'inventive': True, 'fir...['A61K47/32' 'A61K47/30' 'A61K47/00' 'A61K' 'A...['A61K47/32' 'A61K47/30' 'A61K47/00' 'A61K' 'A...['composition' 'mucosa' 'melting point' 'agent...[{'publication_number': 'WO-2007022924-B1', 'a...https://patents.google.com/patent/WO2007022924B1WIPO (PCT)Amended claims[][ 5.3550040e-02 -9.3632710e-02 1.4337189e-02 ...
    1WO-03043855-B1Convenience lighting for interior and exterior...FalseA lighting apparatus for a vehicle(21) include...False[{'code': 'B60Q1/247', 'inventive': True, 'fir...['B60Q1/247' 'B60Q1/24' 'B60Q1/02' 'B60Q1/00' ...['B60Q1/247' 'B60Q1/24' 'B60Q1/02' 'B60Q1/00' ...['vehicle' 'light' 'apparatus defined' 'pillar...[{'publication_number': 'WO-03043855-B1', 'app...https://patents.google.com/patent/WO2003043855B1WIPO (PCT)Amended claims[][ 0.00484032 -0.02695554 -0.20798226 -0.207528...
    2AU-2020396918-A2Shot detection and verification systemFalseA shot detection system for a projectile weapo...False[{'code': 'F41A19/01', 'inventive': True, 'fir...['F41A19/01' 'F41A19/00' 'F41A' 'F41' 'F' 'H04...['F41A19/01' 'F41A19/00' 'F41A' 'F41' 'F' 'H04...['interest' 'region' 'property' 'shot' 'test' ...[{'publication_number': 'US-2023228510-A1', 'a...https://patents.google.com/patent/AU2020396918A2AustraliaAmended post open to public inspection[][-1.49729420e-02 -2.27105440e-01 -2.68012730e-...
    3PL-347539-A1Concrete mix of increased fire resistanceFalseThe burning resistance of concrete containing ...False[{'code': 'Y02W30/91', 'inventive': False, 'fi...['Y02W30/91' 'Y02W30/50' 'Y02W30/00' 'Y02W' 'Y...['Y02W30/91' 'Y02W30/50' 'Y02W30/00' 'Y02W' 'Y...['fire resistance' 'concrete mix' 'increased f...[{'publication_number': 'DK-1564194-T3', 'appl...https://patents.google.com/patent/PL347539A1PolandApplication[][ 0.01849568 -0.05340371 -0.19257502 -0.174919...
    4AU-PS049302-A0Methods and systems (ap53)FalseA charging stand for charging a mobile phone, ...False[{'code': 'H02J7/00', 'inventive': True, 'firs...['H02J7/00' 'H02J' 'H02' 'H' 'H04B1/40' 'H04B1...['H02J7/00' 'H02J' 'H02' 'H' 'H04B1/40' 'H04B1...['connection pin' 'mobile phone' 'cartridge' '...[{'publication_number': 'AU-PS049302-A0', 'app...https://patents.google.com/patent/AUPS049302A0AustraliaApplication filed, as announced in the Gazette...[][ 0.00064732 -0.2136009 0.0040593 -0.024562...
    \n", - "
    " - ], - "text/plain": [ - " publication_number title \\\n", - "0 WO-2007022924-B1 Pharmaceutical compositions with melting point... \n", - "1 WO-03043855-B1 Convenience lighting for interior and exterior... \n", - "2 AU-2020396918-A2 Shot detection and verification system \n", - "3 PL-347539-A1 Concrete mix of increased fire resistance \n", - "4 AU-PS049302-A0 Methods and systems (ap53) \n", - "\n", - " title_translated abstract \\\n", - "0 False The invention relates to the use of chemical f... \n", - "1 False A lighting apparatus for a vehicle(21) include... \n", - "2 False A shot detection system for a projectile weapo... \n", - "3 False The burning resistance of concrete containing ... \n", - "4 False A charging stand for charging a mobile phone, ... \n", - "\n", - " abstract_translated cpc \\\n", - "0 False [{'code': 'A61K47/32', 'inventive': True, 'fir... \n", - "1 False [{'code': 'B60Q1/247', 'inventive': True, 'fir... \n", - "2 False [{'code': 'F41A19/01', 'inventive': True, 'fir... \n", - "3 False [{'code': 'Y02W30/91', 'inventive': False, 'fi... \n", - "4 False [{'code': 'H02J7/00', 'inventive': True, 'firs... \n", - "\n", - " cpc_low \\\n", - "0 ['A61K47/32' 'A61K47/30' 'A61K47/00' 'A61K' 'A... \n", - "1 ['B60Q1/247' 'B60Q1/24' 'B60Q1/02' 'B60Q1/00' ... \n", - "2 ['F41A19/01' 'F41A19/00' 'F41A' 'F41' 'F' 'H04... \n", - "3 ['Y02W30/91' 'Y02W30/50' 'Y02W30/00' 'Y02W' 'Y... \n", - "4 ['H02J7/00' 'H02J' 'H02' 'H' 'H04B1/40' 'H04B1... \n", - "\n", - " cpc_inventive_low \\\n", - "0 ['A61K47/32' 'A61K47/30' 'A61K47/00' 'A61K' 'A... \n", - "1 ['B60Q1/247' 'B60Q1/24' 'B60Q1/02' 'B60Q1/00' ... \n", - "2 ['F41A19/01' 'F41A19/00' 'F41A' 'F41' 'F' 'H04... \n", - "3 ['Y02W30/91' 'Y02W30/50' 'Y02W30/00' 'Y02W' 'Y... \n", - "4 ['H02J7/00' 'H02J' 'H02' 'H' 'H04B1/40' 'H04B1... \n", - "\n", - " top_terms \\\n", - "0 ['composition' 'mucosa' 'melting point' 'agent... \n", - "1 ['vehicle' 'light' 'apparatus defined' 'pillar... \n", - "2 ['interest' 'region' 'property' 'shot' 'test' ... \n", - "3 ['fire resistance' 'concrete mix' 'increased f... \n", - "4 ['connection pin' 'mobile phone' 'cartridge' '... \n", - "\n", - " similar \\\n", - "0 [{'publication_number': 'WO-2007022924-B1', 'a... \n", - "1 [{'publication_number': 'WO-03043855-B1', 'app... \n", - "2 [{'publication_number': 'US-2023228510-A1', 'a... \n", - "3 [{'publication_number': 'DK-1564194-T3', 'appl... \n", - "4 [{'publication_number': 'AU-PS049302-A0', 'app... \n", - "\n", - " url country \\\n", - "0 https://patents.google.com/patent/WO2007022924B1 WIPO (PCT) \n", - "1 https://patents.google.com/patent/WO2003043855B1 WIPO (PCT) \n", - "2 https://patents.google.com/patent/AU2020396918A2 Australia \n", - "3 https://patents.google.com/patent/PL347539A1 Poland \n", - "4 https://patents.google.com/patent/AUPS049302A0 Australia \n", - "\n", - " publication_description cited_by \\\n", - "0 Amended claims [] \n", - "1 Amended claims [] \n", - "2 Amended post open to public inspection [] \n", - "3 Application [] \n", - "4 Application filed, as announced in the Gazette... [] \n", - "\n", - " embedding_v1 \n", - "0 [ 5.3550040e-02 -9.3632710e-02 1.4337189e-02 ... \n", - "1 [ 0.00484032 -0.02695554 -0.20798226 -0.207528... \n", - "2 [-1.49729420e-02 -2.27105440e-01 -2.68012730e-... \n", - "3 [ 0.01849568 -0.05340371 -0.19257502 -0.174919... \n", - "4 [ 0.00064732 -0.2136009 0.0040593 -0.024562... " + "cell_type": "markdown", + "metadata": { + "id": "Ioydzb_8ooEV" + }, + "source": [ + "#### Set your project ID\n", + "\n", + "**If you don't know your project ID**, see the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)" ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "## take a look at the sample dataset\n", - "\n", - "publications.head(5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Wl2o-NYMoygb" - }, - "source": [ - "Generate the text embeddings" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 }, - "executionInfo": { - "elapsed": 4528, - "status": "ok", - "timestamp": 1742192047236, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "executionInfo": { + "elapsed": 2, + "status": "ok", + "timestamp": 1742191597773, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "b8bKCfIiooEV" + }, + "outputs": [], + "source": [ + "# set your project ID below\n", + "PROJECT_ID = \"bigframes-dev\" # @param {type:\"string\"}\n", + "\n", + "# set your region\n", + "REGION = \"US\" # @param {type: \"string\"}\n", + "\n", + "# Set the project id in gcloud\n", + "#! gcloud config set project {PROJECT_ID}" + ] }, - "id": "li38q8FzDDMu", - "outputId": "b8c1bd38-b484-4f71-bd38-927c8677d0c5" - }, - "outputs": [ { - "data": { - "text/html": [ - "Query job 0e9d9117-4981-4f5c-b785-ed831c08e7aa is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "GbUgWr6LooEV" + }, + "source": [ + "#### Authenticate your Google Cloud account\n", + "\n", + "Depending on your Jupyter environment, you might have to manually authenticate. Follow the relevant instructions below." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job fa4f1a54-85d4-4030-992e-fddda5edf3e3 is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "U7ChP8jUooEV" + }, + "source": [ + "**Vertex AI Workbench**\n", + "\n", + "Do nothing, you are already authenticated." ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from bigframes.ml.llm import TextEmbeddingGenerator\n", - "\n", - "text_model = TextEmbeddingGenerator(\n", - " model_name=\"text-embedding-005\",\n", - " # No connection id needed\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 139 }, - "executionInfo": { - "elapsed": 126632, - "status": "ok", - "timestamp": 1742192656608, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "markdown", + "metadata": { + "id": "VfHOYcZZooEW" + }, + "source": [ + "**Local JupyterLab instance**\n", + "\n", + "Uncomment and run the following cell:" + ] }, - "id": "b5HHZob_u61B", - "outputId": "c9ecc5fd-5d11-4fd8-f59b-9dce4e12e371" - }, - "outputs": [ { - "data": { - "text/html": [ - "Load job 70377d71-bb13-46af-80c1-71ef16bf2949 is DONE. Open Job" - ], - "text/plain": [ - "" + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "3cGhUVM0ooEW" + }, + "outputs": [], + "source": [ + "# ! gcloud auth login" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job cc3b609d-b6b7-404f-9447-c76d3a52698b is DONE. 9.5 MB processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "AoHnXlg-ooEW" + }, + "source": [ + "**Colab**\n", + "\n", + "Uncomment and run the following cell:" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/swast/src/github.com/googleapis/python-bigquery-dataframes-2/bigframes/core/array_value.py:109: PreviewWarning: JSON column interpretation as a custom PyArrow extention in\n", - "`db_dtypes` is a preview feature and subject to change.\n", - " warnings.warn(msg, bfe.PreviewWarning)\n" - ] - } - ], - "source": [ - "## rename abstract column to content as the desired column on which embedding will be generated\n", - "publications = publications[[\"publication_number\", \"title\", \"abstract\"]].rename(columns={'abstract': 'content'})\n", - "\n", - "## generate the embeddings\n", - "## takes ~2-3 mins to run\n", - "embedding = text_model.predict(publications)[[\"publication_number\", \"title\", \"content\", \"ml_generate_embedding_result\",\"ml_generate_embedding_status\"]]\n", - "\n", - "## filter out rows where the embedding generation failed. the embedding status value is empty if the embedding generation was successful\n", - "embedding = embedding[~embedding[\"ml_generate_embedding_status\"].isnull()]\n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 464 + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 2, + "status": "ok", + "timestamp": 1742191608487, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "j3lmnsh7ooEW", + "outputId": "eb68daf5-5558-487a-91d2-4b4f9e476da0" + }, + "outputs": [], + "source": [ + "# from google.colab import auth\n", + "# auth.authenticate_user()" + ] }, - "executionInfo": { - "elapsed": 6715, - "status": "ok", - "timestamp": 1742192727525, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "markdown", + "metadata": { + "id": "a9gsyttuooEW" + }, + "source": [ + "Now we are ready to use BigQuery DataFrames!" + ] }, - "id": "OIT5FbqAwqG5", - "outputId": "d04c994a-a0c8-44b0-e897-d871036eeb1f" - }, - "outputs": [ { - "data": { - "text/html": [ - "Query job 5b15fc4a-fa9a-4608-825f-be5af9953a38 is DONE. 71.0 MB processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "xckgWno6ouHY" + }, + "source": [ + "## Step 1: Data Ingestion and Embedding Generation" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    publication_numbertitlecontentml_generate_embedding_resultml_generate_embedding_status
    5611WO-2014005277-A1Resource management in a cloud computing envir...Technologies and implementations for managing ...[-2.92946529e-02 -1.24640828e-02 1.27173709e-...
    6895AU-2011325479-B27-([1,2,3]triazol-4-yl)-pyrrolo[2,3-b]pyrazine...Compounds of formula I, in which R[-6.45397678e-02 1.19616119e-02 -9.85191786e-...
    6IL-45347-A7h-indolizino(5,6,7-ij)isoquinoline derivative...Compounds of the formula:\\n[US3946019A][-3.82784344e-02 -2.31682733e-02 -4.35006060e-...
    5923WO-2005111625-A3Method to predict prostate cancerA method for predicting the probability or ris...[ 0.02480386 -0.01648765 0.03873815 -0.025998...
    6370US-7868678-B2Configurable differential linesEmbodiments related to configurable differenti...[ 2.71715336e-02 -1.93733890e-02 2.82729534e-...
    \n", - "

    5 rows × 5 columns

    \n", - "
    [5 rows x 5 columns in total]" - ], - "text/plain": [ - " publication_number title \\\n", - "5611 WO-2014005277-A1 Resource management in a cloud computing envir... \n", - "6895 AU-2011325479-B2 7-([1,2,3]triazol-4-yl)-pyrrolo[2,3-b]pyrazine... \n", - "6 IL-45347-A 7h-indolizino(5,6,7-ij)isoquinoline derivative... \n", - "5923 WO-2005111625-A3 Method to predict prostate cancer \n", - "6370 US-7868678-B2 Configurable differential lines \n", - "\n", - " content \\\n", - "5611 Technologies and implementations for managing ... \n", - "6895 Compounds of formula I, in which R \n", - "6 Compounds of the formula:\\n[US3946019A] \n", - "5923 A method for predicting the probability or ris... \n", - "6370 Embodiments related to configurable differenti... \n", - "\n", - " ml_generate_embedding_result \\\n", - "5611 [-2.92946529e-02 -1.24640828e-02 1.27173709e-... \n", - "6895 [-6.45397678e-02 1.19616119e-02 -9.85191786e-... \n", - "6 [-3.82784344e-02 -2.31682733e-02 -4.35006060e-... \n", - "5923 [ 0.02480386 -0.01648765 0.03873815 -0.025998... \n", - "6370 [ 2.71715336e-02 -1.93733890e-02 2.82729534e-... \n", - "\n", - " ml_generate_embedding_status \n", - "5611 \n", - "6895 \n", - "6 \n", - "5923 \n", - "6370 \n", - "\n", - "[5 rows x 5 columns]" + "cell_type": "markdown", + "metadata": { + "id": "Hjg9jDN-ooEW" + }, + "source": [ + "Install libraries" ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "embedding.head(5)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 53 }, - "executionInfo": { - "elapsed": 6590, - "status": "ok", - "timestamp": 1742192833667, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "executionInfo": { + "elapsed": 947, + "status": "ok", + "timestamp": 1742195413800, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "R7STCS8xB5d2" + }, + "outputs": [], + "source": [ + "import bigframes.pandas as bf\n", + "import bigframes.ml as bf_ml\n", + "import bigframes.bigquery as bf_bq\n", + "import bigframes.ml.llm as bf_llm\n", + "\n", + "\n", + "from google.cloud import bigquery\n", + "from google.cloud import storage\n", + "\n", + "# Construct a BigQuery client object.\n", + "client = bigquery.Client()\n", + "\n", + "import pandas as pd\n", + "from IPython.display import Image, display\n", + "from PIL import Image as PILImage\n", + "import io\n", + "\n", + "import json\n", + "from IPython.display import Markdown\n", + "\n", + "# Note: The project option is not required in all environments.\n", + "# On BigQuery Studio, the project ID is automatically detected.\n", + "bf.options.bigquery.project = PROJECT_ID\n", + "bf.options.bigquery.location = REGION\n", + "\n" + ] }, - "id": "GP3ZqX_bxLGq", - "outputId": "fb823ea2-e47c-415f-84d4-543dd3291e15" - }, - "outputs": [ { - "data": { - "text/html": [ - "Query job 06ce090b-e3f9-4252-b847-45c2a296ca61 is DONE. 70.9 MB processed. Open Job" - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "iOFF9hrvs5WE" + }, + "source": [ + "Partial ordering mode allows BigQuery DataFrames to push down many more row and column filters. On large clustered and partitioned tables, this can greatly reduce the number of bytes scanned and computation slots used. This [blog post](https://medium.com/google-cloud/introducing-partial-ordering-mode-for-bigquery-dataframes-bigframes-ec35841d95c0) goes over it in more detail." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/plain": [ - "'my_dataset.my_embeddings_table'" + "cell_type": "code", + "execution_count": 6, + "metadata": { + "executionInfo": { + "elapsed": 2, + "status": "ok", + "timestamp": 1742191620533, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "9Gil1Oaas7KA" + }, + "outputs": [], + "source": [ + "bf.options.bigquery.ordering_mode = \"partial\"" ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# store embeddings in a BQ table\n", - "DATASET_ID = \"my_dataset\" # @param {type:\"string\"}\n", - "TEXT_EMBEDDING_TABLE_ID = \"my_embeddings_table\" # @param {type:\"string\"}\n", - "embedding.to_gbq(f\"{DATASET_ID}.{TEXT_EMBEDDING_TABLE_ID}\", if_exists='replace')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "OUZ3NNbzo1Tb" - }, - "source": [ - "## Step 2: Indexing and Similarity Search" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mvJH2FCmynMm" - }, - "source": [ - "### [Create a Vector Index](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.bigquery#bigframes_bigquery_create_vector_index) using BigFrames\n", - "\n", - "\n", - "**Index Type**\n", - "\n", - "The algorithm to use to build the vector index.\n", - "The supported values are IVF and TREE_AH." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 }, - "executionInfo": { - "elapsed": 3882, - "status": "ok", - "timestamp": 1742193028877, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "markdown", + "metadata": { + "id": "XGaGyyZsooEW" + }, + "source": [ + "If you want to reset the location of the created DataFrame or Series objects, reset the session by executing `bf.close_session()`. After that, you can reuse `bf.options.bigquery.location` to specify another location." + ] }, - "id": "6SBVdv6gyU5A", - "outputId": "6583e113-de27-4b44-972d-c1cc061e3c76" - }, - "outputs": [], - "source": [ - "## create vector index (note only works of tables >5000 rows)\n", - "\n", - "bf_bq.create_vector_index(\n", - " table_id = f\"{DATASET_ID}.{TEXT_EMBEDDING_TABLE_ID}\",\n", - " column_name = \"ml_generate_embedding_result\",\n", - " replace= True,\n", - " index_name = \"bf_python_index\",\n", - " distance_type=\"cosine\",\n", - " index_type= \"ivf\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bo8mBbRLzCOA" - }, - "source": [ - "### Vector Search (semantic search) using Vector Index\n", - "\n", - "ANN (approx nearest neighbor) search using the created vector index" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "executionInfo": { - "elapsed": 639, - "status": "ok", - "timestamp": 1742194606771, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "markdown", + "metadata": { + "id": "v6FGschEowht" + }, + "source": [ + "Data Input - read the data from a publicly available BigQuery dataset" + ] }, - "id": "v19BJm_wzPdZ" - }, - "outputs": [], - "source": [ - "## Set variable for vector search\n", - "\n", - "TEXT_SEARCH_STRING = \"Chip assemblies employing solder bonds to back-side lands including an electrolytic nickel layer\" ## replace with whatever search string you want to use for the vector search\n", - "FRACTION_LISTS_TO_SEARCH = 0.01" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 121 + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 468, + "status": "ok", + "timestamp": 1742192516923, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "zDSwoBo1CU3G", + "outputId": "83edbc2f-5a23-407b-8890-f968eb31be44" + }, + "outputs": [], + "source": [ + "publications = bf.read_gbq('patents-public-data.google_patents_research.publications')" + ] }, - "executionInfo": { - "elapsed": 6927, - "status": "ok", - "timestamp": 1742194625774, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "executionInfo": { + "elapsed": 6697, + "status": "ok", + "timestamp": 1742192524632, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "tYDoaKgJChiq", + "outputId": "9174da29-a051-4a99-e38f-6a2b09cfe4e9" + }, + "outputs": [], + "source": [ + "## create patents base table (subset of 10k out of ~110M records)\n", + "\n", + "keep = (publications.embedding_v1.str.len() > 0) & (publications.title.str.len() > 0) & (publications.abstract.str.len() > 30)\n", + "\n", + "## Choose 10000 random rows to analyze\n", + "publications = publications[keep].peek(10000)" + ] }, - "id": "pAQY1ejpzPap", - "outputId": "485698ad-ac6e-4c93-844e-5d0f30aff13a" - }, - "outputs": [ { - "data": { - "text/html": [ - "Query job 016ad678-9609-4c78-8f07-3f9887ce67ac is DONE. 0 Bytes processed. Open Job" + "cell_type": "code", + "execution_count": 9, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 556 + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1742191801044, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "XmqdJInztzPl", + "outputId": "ae05f3a6-edeb-423a-c061-c416717e1ec5" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    publication_numbertitletitle_translatedabstractabstract_translatedcpccpc_lowcpc_inventive_lowtop_termssimilarurlcountrypublication_descriptioncited_byembedding_v1
    0WO-2007022924-B1Pharmaceutical compositions with melting point...FalseThe invention relates to the use of chemical f...False[{'code': 'A61K47/32', 'inventive': True, 'fir...['A61K47/32' 'A61K47/30' 'A61K47/00' 'A61K' 'A...['A61K47/32' 'A61K47/30' 'A61K47/00' 'A61K' 'A...['composition' 'mucosa' 'melting point' 'agent...[{'publication_number': 'WO-2007022924-B1', 'a...https://patents.google.com/patent/WO2007022924B1WIPO (PCT)Amended claims[][ 5.3550040e-02 -9.3632710e-02 1.4337189e-02 ...
    1WO-03043855-B1Convenience lighting for interior and exterior...FalseA lighting apparatus for a vehicle(21) include...False[{'code': 'B60Q1/247', 'inventive': True, 'fir...['B60Q1/247' 'B60Q1/24' 'B60Q1/02' 'B60Q1/00' ...['B60Q1/247' 'B60Q1/24' 'B60Q1/02' 'B60Q1/00' ...['vehicle' 'light' 'apparatus defined' 'pillar...[{'publication_number': 'WO-03043855-B1', 'app...https://patents.google.com/patent/WO2003043855B1WIPO (PCT)Amended claims[][ 0.00484032 -0.02695554 -0.20798226 -0.207528...
    2AU-2020396918-A2Shot detection and verification systemFalseA shot detection system for a projectile weapo...False[{'code': 'F41A19/01', 'inventive': True, 'fir...['F41A19/01' 'F41A19/00' 'F41A' 'F41' 'F' 'H04...['F41A19/01' 'F41A19/00' 'F41A' 'F41' 'F' 'H04...['interest' 'region' 'property' 'shot' 'test' ...[{'publication_number': 'US-2023228510-A1', 'a...https://patents.google.com/patent/AU2020396918A2AustraliaAmended post open to public inspection[][-1.49729420e-02 -2.27105440e-01 -2.68012730e-...
    3PL-347539-A1Concrete mix of increased fire resistanceFalseThe burning resistance of concrete containing ...False[{'code': 'Y02W30/91', 'inventive': False, 'fi...['Y02W30/91' 'Y02W30/50' 'Y02W30/00' 'Y02W' 'Y...['Y02W30/91' 'Y02W30/50' 'Y02W30/00' 'Y02W' 'Y...['fire resistance' 'concrete mix' 'increased f...[{'publication_number': 'DK-1564194-T3', 'appl...https://patents.google.com/patent/PL347539A1PolandApplication[][ 0.01849568 -0.05340371 -0.19257502 -0.174919...
    4AU-PS049302-A0Methods and systems (ap53)FalseA charging stand for charging a mobile phone, ...False[{'code': 'H02J7/00', 'inventive': True, 'firs...['H02J7/00' 'H02J' 'H02' 'H' 'H04B1/40' 'H04B1...['H02J7/00' 'H02J' 'H02' 'H' 'H04B1/40' 'H04B1...['connection pin' 'mobile phone' 'cartridge' '...[{'publication_number': 'AU-PS049302-A0', 'app...https://patents.google.com/patent/AUPS049302A0AustraliaApplication filed, as announced in the Gazette...[][ 0.00064732 -0.2136009 0.0040593 -0.024562...
    \n", + "
    " + ], + "text/plain": [ + " publication_number title \\\n", + "0 WO-2007022924-B1 Pharmaceutical compositions with melting point... \n", + "1 WO-03043855-B1 Convenience lighting for interior and exterior... \n", + "2 AU-2020396918-A2 Shot detection and verification system \n", + "3 PL-347539-A1 Concrete mix of increased fire resistance \n", + "4 AU-PS049302-A0 Methods and systems (ap53) \n", + "\n", + " title_translated abstract \\\n", + "0 False The invention relates to the use of chemical f... \n", + "1 False A lighting apparatus for a vehicle(21) include... \n", + "2 False A shot detection system for a projectile weapo... \n", + "3 False The burning resistance of concrete containing ... \n", + "4 False A charging stand for charging a mobile phone, ... \n", + "\n", + " abstract_translated cpc \\\n", + "0 False [{'code': 'A61K47/32', 'inventive': True, 'fir... \n", + "1 False [{'code': 'B60Q1/247', 'inventive': True, 'fir... \n", + "2 False [{'code': 'F41A19/01', 'inventive': True, 'fir... \n", + "3 False [{'code': 'Y02W30/91', 'inventive': False, 'fi... \n", + "4 False [{'code': 'H02J7/00', 'inventive': True, 'firs... \n", + "\n", + " cpc_low \\\n", + "0 ['A61K47/32' 'A61K47/30' 'A61K47/00' 'A61K' 'A... \n", + "1 ['B60Q1/247' 'B60Q1/24' 'B60Q1/02' 'B60Q1/00' ... \n", + "2 ['F41A19/01' 'F41A19/00' 'F41A' 'F41' 'F' 'H04... \n", + "3 ['Y02W30/91' 'Y02W30/50' 'Y02W30/00' 'Y02W' 'Y... \n", + "4 ['H02J7/00' 'H02J' 'H02' 'H' 'H04B1/40' 'H04B1... \n", + "\n", + " cpc_inventive_low \\\n", + "0 ['A61K47/32' 'A61K47/30' 'A61K47/00' 'A61K' 'A... \n", + "1 ['B60Q1/247' 'B60Q1/24' 'B60Q1/02' 'B60Q1/00' ... \n", + "2 ['F41A19/01' 'F41A19/00' 'F41A' 'F41' 'F' 'H04... \n", + "3 ['Y02W30/91' 'Y02W30/50' 'Y02W30/00' 'Y02W' 'Y... \n", + "4 ['H02J7/00' 'H02J' 'H02' 'H' 'H04B1/40' 'H04B1... \n", + "\n", + " top_terms \\\n", + "0 ['composition' 'mucosa' 'melting point' 'agent... \n", + "1 ['vehicle' 'light' 'apparatus defined' 'pillar... \n", + "2 ['interest' 'region' 'property' 'shot' 'test' ... \n", + "3 ['fire resistance' 'concrete mix' 'increased f... \n", + "4 ['connection pin' 'mobile phone' 'cartridge' '... \n", + "\n", + " similar \\\n", + "0 [{'publication_number': 'WO-2007022924-B1', 'a... \n", + "1 [{'publication_number': 'WO-03043855-B1', 'app... \n", + "2 [{'publication_number': 'US-2023228510-A1', 'a... \n", + "3 [{'publication_number': 'DK-1564194-T3', 'appl... \n", + "4 [{'publication_number': 'AU-PS049302-A0', 'app... \n", + "\n", + " url country \\\n", + "0 https://patents.google.com/patent/WO2007022924B1 WIPO (PCT) \n", + "1 https://patents.google.com/patent/WO2003043855B1 WIPO (PCT) \n", + "2 https://patents.google.com/patent/AU2020396918A2 Australia \n", + "3 https://patents.google.com/patent/PL347539A1 Poland \n", + "4 https://patents.google.com/patent/AUPS049302A0 Australia \n", + "\n", + " publication_description cited_by \\\n", + "0 Amended claims [] \n", + "1 Amended claims [] \n", + "2 Amended post open to public inspection [] \n", + "3 Application [] \n", + "4 Application filed, as announced in the Gazette... [] \n", + "\n", + " embedding_v1 \n", + "0 [ 5.3550040e-02 -9.3632710e-02 1.4337189e-02 ... \n", + "1 [ 0.00484032 -0.02695554 -0.20798226 -0.207528... \n", + "2 [-1.49729420e-02 -2.27105440e-01 -2.68012730e-... \n", + "3 [ 0.01849568 -0.05340371 -0.19257502 -0.174919... \n", + "4 [ 0.00064732 -0.2136009 0.0040593 -0.024562... " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "## take a look at the sample dataset\n", + "\n", + "publications.head(5)" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/swast/src/github.com/googleapis/python-bigquery-dataframes-2/bigframes/core/array_value.py:109: PreviewWarning: JSON column interpretation as a custom PyArrow extention in\n", - "`db_dtypes` is a preview feature and subject to change.\n", - " warnings.warn(msg, bfe.PreviewWarning)\n" - ] - } - ], - "source": [ - "# convert search string to dataframe\n", - "TEXT_SEARCH_DF = bf.DataFrame([TEXT_SEARCH_STRING], columns=['search_string'])\n", - "\n", - "#generate embedding of search query\n", - "search_query = bf.DataFrame(text_model.predict(TEXT_SEARCH_DF))" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 104 - }, - "executionInfo": { - "elapsed": 5110, - "status": "ok", - "timestamp": 1742194670801, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 - }, - "id": "sx0AGAdn5FYX", - "outputId": "551ebac3-594f-4303-ca97-5301dfee72bb" - }, - "outputs": [], - "source": [ - "## search the base table for the user's query\n", - "\n", - "vector_search_results = bf_bq.vector_search(\n", - " base_table=f\"{DATASET_ID}.{TEXT_EMBEDDING_TABLE_ID}\",\n", - " column_to_search=\"ml_generate_embedding_result\",\n", - " query=search_query,\n", - " distance_type=\"cosine\",\n", - " query_column_to_search=\"ml_generate_embedding_result\",\n", - " top_k=5,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 270 + "cell_type": "markdown", + "metadata": { + "id": "Wl2o-NYMoygb" + }, + "source": [ + "Generate the text embeddings" + ] }, - "executionInfo": { - "elapsed": 3511, - "status": "ok", - "timestamp": 1742195090670, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "executionInfo": { + "elapsed": 4528, + "status": "ok", + "timestamp": 1742192047236, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "li38q8FzDDMu", + "outputId": "b8c1bd38-b484-4f71-bd38-927c8677d0c5" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 0e9d9117-4981-4f5c-b785-ed831c08e7aa is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job fa4f1a54-85d4-4030-992e-fddda5edf3e3 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from bigframes.ml.llm import TextEmbeddingGenerator\n", + "\n", + "text_model = TextEmbeddingGenerator(\n", + " model_name=\"text-embedding-005\",\n", + " # No connection id needed\n", + ")" + ] }, - "id": "px1v4iJM5L0c", - "outputId": "d107b6e3-a362-42db-c0c2-084d02acd244" - }, - "outputs": [ { - "data": { - "text/html": [ - "Load job b6b88844-9ed7-4c92-8984-556414592f0b is DONE. Open Job" + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 139 + }, + "executionInfo": { + "elapsed": 126632, + "status": "ok", + "timestamp": 1742192656608, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "b5HHZob_u61B", + "outputId": "c9ecc5fd-5d11-4fd8-f59b-9dce4e12e371" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Load job 70377d71-bb13-46af-80c1-71ef16bf2949 is DONE. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job cc3b609d-b6b7-404f-9447-c76d3a52698b is DONE. 9.5 MB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/swast/src/github.com/googleapis/python-bigquery-dataframes-2/bigframes/core/array_value.py:109: PreviewWarning: JSON column interpretation as a custom PyArrow extention in\n", + "`db_dtypes` is a preview feature and subject to change.\n", + " warnings.warn(msg, bfe.PreviewWarning)\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "## rename abstract column to content as the desired column on which embedding will be generated\n", + "publications = publications[[\"publication_number\", \"title\", \"abstract\"]].rename(columns={'abstract': 'content'})\n", + "\n", + "## generate the embeddings\n", + "## takes ~2-3 mins to run\n", + "embedding = text_model.predict(publications)[[\"publication_number\", \"title\", \"content\", \"ml_generate_embedding_result\",\"ml_generate_embedding_status\"]]\n", + "\n", + "## filter out rows where the embedding generation failed. the embedding status value is empty if the embedding generation was successful\n", + "embedding = embedding[~embedding[\"ml_generate_embedding_status\"].isnull()]\n" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job aa95f59c-7229-4e76-bd2c-3a63deea3285 is DONE. 4.7 kB processed. Open Job" + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 464 + }, + "executionInfo": { + "elapsed": 6715, + "status": "ok", + "timestamp": 1742192727525, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "OIT5FbqAwqG5", + "outputId": "d04c994a-a0c8-44b0-e897-d871036eeb1f" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 5b15fc4a-fa9a-4608-825f-be5af9953a38 is DONE. 71.0 MB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    publication_numbertitlecontentml_generate_embedding_resultml_generate_embedding_status
    5611WO-2014005277-A1Resource management in a cloud computing envir...Technologies and implementations for managing ...[-2.92946529e-02 -1.24640828e-02 1.27173709e-...
    6895AU-2011325479-B27-([1,2,3]triazol-4-yl)-pyrrolo[2,3-b]pyrazine...Compounds of formula I, in which R[-6.45397678e-02 1.19616119e-02 -9.85191786e-...
    6IL-45347-A7h-indolizino(5,6,7-ij)isoquinoline derivative...Compounds of the formula:\\n[US3946019A][-3.82784344e-02 -2.31682733e-02 -4.35006060e-...
    5923WO-2005111625-A3Method to predict prostate cancerA method for predicting the probability or ris...[ 0.02480386 -0.01648765 0.03873815 -0.025998...
    6370US-7868678-B2Configurable differential linesEmbodiments related to configurable differenti...[ 2.71715336e-02 -1.93733890e-02 2.82729534e-...
    \n", + "

    5 rows × 5 columns

    \n", + "
    [5 rows x 5 columns in total]" + ], + "text/plain": [ + " publication_number title \\\n", + "5611 WO-2014005277-A1 Resource management in a cloud computing envir... \n", + "6895 AU-2011325479-B2 7-([1,2,3]triazol-4-yl)-pyrrolo[2,3-b]pyrazine... \n", + "6 IL-45347-A 7h-indolizino(5,6,7-ij)isoquinoline derivative... \n", + "5923 WO-2005111625-A3 Method to predict prostate cancer \n", + "6370 US-7868678-B2 Configurable differential lines \n", + "\n", + " content \\\n", + "5611 Technologies and implementations for managing ... \n", + "6895 Compounds of formula I, in which R \n", + "6 Compounds of the formula:\\n[US3946019A] \n", + "5923 A method for predicting the probability or ris... \n", + "6370 Embodiments related to configurable differenti... \n", + "\n", + " ml_generate_embedding_result \\\n", + "5611 [-2.92946529e-02 -1.24640828e-02 1.27173709e-... \n", + "6895 [-6.45397678e-02 1.19616119e-02 -9.85191786e-... \n", + "6 [-3.82784344e-02 -2.31682733e-02 -4.35006060e-... \n", + "5923 [ 0.02480386 -0.01648765 0.03873815 -0.025998... \n", + "6370 [ 2.71715336e-02 -1.93733890e-02 2.82729534e-... \n", + "\n", + " ml_generate_embedding_status \n", + "5611 \n", + "6895 \n", + "6 \n", + "5923 \n", + "6370 \n", + "\n", + "[5 rows x 5 columns]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "embedding.head(5)" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    querypublication_numbertitle (relevant match)abstract (relevant match)distance
    0Chip assemblies employing solder bonds to back...CN-103515336-AChip package, chip arrangement, circuit board ...A chip package is provided, the chip package i...0.287274
    0Chip assemblies employing solder bonds to back...US-9548145-B2Microelectronic assembly with multi-layer supp...A method of forming a microelectronic assembly...0.290519
    0Chip assemblies employing solder bonds to back...JP-2012074505-ASemiconductor mounting device substrate, semic...To provide a substrate for a semiconductor mou...0.294241
    0Chip assemblies employing solder bonds to back...US-2015380164-A1Ceramic electronic componentA ceramic electronic component includes an ele...0.295716
    0Chip assemblies employing solder bonds to back...US-2012153447-A1Microelectronic flip chip packages with solder...Processes of assembling microelectronic packag...0.300337
    \n", - "

    5 rows × 5 columns

    \n", - "
    [5 rows x 5 columns in total]" + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 53 + }, + "executionInfo": { + "elapsed": 6590, + "status": "ok", + "timestamp": 1742192833667, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "GP3ZqX_bxLGq", + "outputId": "fb823ea2-e47c-415f-84d4-543dd3291e15" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 06ce090b-e3f9-4252-b847-45c2a296ca61 is DONE. 70.9 MB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'my_dataset.my_embeddings_table'" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - " query publication_number \\\n", - "0 Chip assemblies employing solder bonds to back... CN-103515336-A \n", - "0 Chip assemblies employing solder bonds to back... US-9548145-B2 \n", - "0 Chip assemblies employing solder bonds to back... JP-2012074505-A \n", - "0 Chip assemblies employing solder bonds to back... US-2015380164-A1 \n", - "0 Chip assemblies employing solder bonds to back... US-2012153447-A1 \n", - "\n", - " title (relevant match) \\\n", - "0 Chip package, chip arrangement, circuit board ... \n", - "0 Microelectronic assembly with multi-layer supp... \n", - "0 Semiconductor mounting device substrate, semic... \n", - "0 Ceramic electronic component \n", - "0 Microelectronic flip chip packages with solder... \n", - "\n", - " abstract (relevant match) distance \n", - "0 A chip package is provided, the chip package i... 0.287274 \n", - "0 A method of forming a microelectronic assembly... 0.290519 \n", - "0 To provide a substrate for a semiconductor mou... 0.294241 \n", - "0 A ceramic electronic component includes an ele... 0.295716 \n", - "0 Processes of assembling microelectronic packag... 0.300337 \n", - "\n", - "[5 rows x 5 columns]" + "source": [ + "# store embeddings in a BQ table\n", + "DATASET_ID = \"my_dataset\" # @param {type:\"string\"}\n", + "TEXT_EMBEDDING_TABLE_ID = \"my_embeddings_table\" # @param {type:\"string\"}\n", + "embedding.to_gbq(f\"{DATASET_ID}.{TEXT_EMBEDDING_TABLE_ID}\", if_exists='replace')" ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "## View the returned results based on simalirity with the user's query\n", - "\n", - "vector_search_results[\n", - " [\n", - " 'content',\n", - " 'publication_number',\n", - " 'title',\n", - " 'content_1',\n", - " 'distance',\n", - " ]\n", - "].rename(columns={\n", - " 'content': 'query',\n", - " 'content_1':'abstract (relevant match)' ,\n", - " 'title':'title (relevant match)',\n", - "})" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "executionInfo": { - "elapsed": 1622, - "status": "ok", - "timestamp": 1742195139318, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 }, - "id": "5fb_O-ne5cvH" - }, - "outputs": [], - "source": [ - "## Brute force result (for comparison)\n", - "\n", - "\n", - "brute_force_result = bf_bq.vector_search(\n", - " base_table=f\"{DATASET_ID}.{TEXT_EMBEDDING_TABLE_ID}\",\n", - " column_to_search=\"ml_generate_embedding_result\",\n", - " query=search_query,\n", - " top_k=5,\n", - " distance_type=\"cosine\",\n", - " use_brute_force=True,\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "21rNsFMHo8hO" - }, - "source": [ - "## Step 3: AI-Powered Summarization with Retrieval Augmented Generation (RAG)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "K3pIQrzB7T_G" - }, - "source": [ - "Patent documents can be dense and time-consuming to digest. AI-Powered Patent Summarization utilizes Retrieval Augmented Generation (RAG) to streamline this process. By retrieving relevant patent information through vector search and then synthesizing it with a large language model, we can generate concise, human-readable summaries, saving valuable time and effort. The code sample below walks through how to set this up continuing with the same user query as the previous use case." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 + { + "cell_type": "markdown", + "metadata": { + "id": "OUZ3NNbzo1Tb" + }, + "source": [ + "## Step 2: Indexing and Similarity Search" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mvJH2FCmynMm" + }, + "source": [ + "### [Create a Vector Index](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.bigquery#bigframes_bigquery_create_vector_index) using BigFrames\n", + "\n", + "\n", + "**Index Type**\n", + "\n", + "The algorithm to use to build the vector index.\n", + "The supported values are IVF and TREE_AH." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "executionInfo": { + "elapsed": 3882, + "status": "ok", + "timestamp": 1742193028877, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "6SBVdv6gyU5A", + "outputId": "6583e113-de27-4b44-972d-c1cc061e3c76" + }, + "outputs": [], + "source": [ + "## create vector index (note only works of tables >5000 rows)\n", + "\n", + "bf_bq.create_vector_index(\n", + " table_id = f\"{DATASET_ID}.{TEXT_EMBEDDING_TABLE_ID}\",\n", + " column_name = \"ml_generate_embedding_result\",\n", + " replace= True,\n", + " index_name = \"bf_python_index\",\n", + " distance_type=\"cosine\",\n", + " index_type= \"ivf\"\n", + ")" + ] }, - "executionInfo": { - "elapsed": 4827, - "status": "ok", - "timestamp": 1742195565658, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "markdown", + "metadata": { + "id": "bo8mBbRLzCOA" + }, + "source": [ + "### Vector Search (semantic search) using Vector Index\n", + "\n", + "ANN (approx nearest neighbor) search using the created vector index" + ] }, - "id": "jb5rueqU7T5J", - "outputId": "43732836-ebae-4fb3-b28e-bfea51146c72" - }, - "outputs": [ { - "data": { - "text/html": [ - "Query job 3fabe659-f95b-49cb-b0c7-9d32b09177bf is DONE. 0 Bytes processed. Open Job" + "cell_type": "code", + "execution_count": 15, + "metadata": { + "executionInfo": { + "elapsed": 639, + "status": "ok", + "timestamp": 1742194606771, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "v19BJm_wzPdZ" + }, + "outputs": [], + "source": [ + "## Set variable for vector search\n", + "\n", + "TEXT_SEARCH_STRING = \"Chip assemblies employing solder bonds to back-side lands including an electrolytic nickel layer\" ## replace with whatever search string you want to use for the vector search\n", + "FRACTION_LISTS_TO_SEARCH = 0.01" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 121 + }, + "executionInfo": { + "elapsed": 6927, + "status": "ok", + "timestamp": 1742194625774, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "pAQY1ejpzPap", + "outputId": "485698ad-ac6e-4c93-844e-5d0f30aff13a" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 016ad678-9609-4c78-8f07-3f9887ce67ac is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/swast/src/github.com/googleapis/python-bigquery-dataframes-2/bigframes/core/array_value.py:109: PreviewWarning: JSON column interpretation as a custom PyArrow extention in\n", + "`db_dtypes` is a preview feature and subject to change.\n", + " warnings.warn(msg, bfe.PreviewWarning)\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "# convert search string to dataframe\n", + "TEXT_SEARCH_DF = bf.DataFrame([TEXT_SEARCH_STRING], columns=['search_string'])\n", + "\n", + "#generate embedding of search query\n", + "search_query = bf.DataFrame(text_model.predict(TEXT_SEARCH_DF))" ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "## gemini model\n", - "\n", - "llm_model = bf_llm.GeminiTextGenerator(model_name = \"gemini-2.5-flash\") ## replace with other model as needed" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "41e12JTf70sr" - }, - "source": [ - "We will use the same user query from Section 2, and pass the list of abstracts returned by the vector search into the prompt for the RAG application" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "executionInfo": { - "elapsed": 1474, - "status": "ok", - "timestamp": 1742195536109, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 }, - "id": "EyP-ZFJK8h-2" - }, - "outputs": [], - "source": [ - "TEMPERATURE = 0.4" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 72 + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 104 + }, + "executionInfo": { + "elapsed": 5110, + "status": "ok", + "timestamp": 1742194670801, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "sx0AGAdn5FYX", + "outputId": "551ebac3-594f-4303-ca97-5301dfee72bb" + }, + "outputs": [], + "source": [ + "## search the base table for the user's query\n", + "\n", + "vector_search_results = bf_bq.vector_search(\n", + " base_table=f\"{DATASET_ID}.{TEXT_EMBEDDING_TABLE_ID}\",\n", + " column_to_search=\"ml_generate_embedding_result\",\n", + " query=search_query,\n", + " distance_type=\"cosine\",\n", + " query_column_to_search=\"ml_generate_embedding_result\",\n", + " top_k=5,\n", + ")" + ] }, - "executionInfo": { - "elapsed": 3371, - "status": "ok", - "timestamp": 1742195421813, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 270 + }, + "executionInfo": { + "elapsed": 3511, + "status": "ok", + "timestamp": 1742195090670, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "px1v4iJM5L0c", + "outputId": "d107b6e3-a362-42db-c0c2-084d02acd244" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Load job b6b88844-9ed7-4c92-8984-556414592f0b is DONE. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job aa95f59c-7229-4e76-bd2c-3a63deea3285 is DONE. 4.7 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    querypublication_numbertitle (relevant match)abstract (relevant match)distance
    0Chip assemblies employing solder bonds to back...CN-103515336-AChip package, chip arrangement, circuit board ...A chip package is provided, the chip package i...0.287274
    0Chip assemblies employing solder bonds to back...US-9548145-B2Microelectronic assembly with multi-layer supp...A method of forming a microelectronic assembly...0.290519
    0Chip assemblies employing solder bonds to back...JP-2012074505-ASemiconductor mounting device substrate, semic...To provide a substrate for a semiconductor mou...0.294241
    0Chip assemblies employing solder bonds to back...US-2015380164-A1Ceramic electronic componentA ceramic electronic component includes an ele...0.295716
    0Chip assemblies employing solder bonds to back...US-2012153447-A1Microelectronic flip chip packages with solder...Processes of assembling microelectronic packag...0.300337
    \n", + "

    5 rows × 5 columns

    \n", + "
    [5 rows x 5 columns in total]" + ], + "text/plain": [ + " query publication_number \\\n", + "0 Chip assemblies employing solder bonds to back... CN-103515336-A \n", + "0 Chip assemblies employing solder bonds to back... US-9548145-B2 \n", + "0 Chip assemblies employing solder bonds to back... JP-2012074505-A \n", + "0 Chip assemblies employing solder bonds to back... US-2015380164-A1 \n", + "0 Chip assemblies employing solder bonds to back... US-2012153447-A1 \n", + "\n", + " title (relevant match) \\\n", + "0 Chip package, chip arrangement, circuit board ... \n", + "0 Microelectronic assembly with multi-layer supp... \n", + "0 Semiconductor mounting device substrate, semic... \n", + "0 Ceramic electronic component \n", + "0 Microelectronic flip chip packages with solder... \n", + "\n", + " abstract (relevant match) distance \n", + "0 A chip package is provided, the chip package i... 0.287274 \n", + "0 A method of forming a microelectronic assembly... 0.290519 \n", + "0 To provide a substrate for a semiconductor mou... 0.294241 \n", + "0 A ceramic electronic component includes an ele... 0.295716 \n", + "0 Processes of assembling microelectronic packag... 0.300337 \n", + "\n", + "[5 rows x 5 columns]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "## View the returned results based on simalirity with the user's query\n", + "\n", + "vector_search_results[\n", + " [\n", + " 'content',\n", + " 'publication_number',\n", + " 'title',\n", + " 'content_1',\n", + " 'distance',\n", + " ]\n", + "].rename(columns={\n", + " 'content': 'query',\n", + " 'content_1':'abstract (relevant match)' ,\n", + " 'title':'title (relevant match)',\n", + "})" + ] }, - "id": "eP99R6SV7Tug", - "outputId": "c34bc931-5be8-410e-ac1f-604df31ef533" - }, - "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "['{\"abstract\": \"A chip package is provided, the chip package including: a chip carrier; a chip disposed over and electrically connected to a chip carrier top side; an electrically insulating material disposed over and at least partially surrounding the chip; one or more electrically conductive contact regions formed over the electrically insulating material and in electrical connection with the chip; and another electrically insulating material disposed over a chip carrier bottom side. An electrically conductive contact region on the chip carrier bottom side is released from the further electrically insulating material.\"}', '{\"abstract\": \"A method of forming a microelectronic assembly includes positioning a support structure adjacent to an active region of a device but not extending onto the active region. The support structure has planar sections. Each planar section has a substantially uniform composition. The composition of at least one of the planar sections differs from the composition of at least one of the other planar sections. A lid is positioned in contact with the support structure and extends over the active region. The support structure is bonded to the device and to the lid.\"}', '{\"abstract\": \"To provide a substrate for a semiconductor mounting device capable of obtaining high reliability. In a semiconductor mounting device substrate of the present invention, a semiconductor chip can be surface-mounted by a flip chip connection method on a semiconductor chip mounting region of a first main surface of a multilayer wiring substrate. A plurality of second main surface side solder bumps 52 forming a plate-like component mounting region 53 are formed at a location immediately below the semiconductor chip 21 on the second main surface 13 of the multilayer wiring board 11. A plate-like component 101 mainly composed of an inorganic material is surface-mounted on the multilayer wiring board 11 by a flip chip connection method via a plurality of second main surface side solder bumps 52. A plurality of second main surface side solder bumps 52 are sealed by a second main surface side underfill 107 provided in the gap S <b> 2 between the second main surface 13 and the plate-like component 101. [Selection] Figure 1\"}', '{\"abstract\": \"A ceramic electronic component includes an electronic component body, an inner electrode, and an outer electrode. The outer electrode includes a fired electrode layer and first and second plated layers. The fired electrode layer is disposed on the electronic component body. The first plated layer is disposed on the fired electrode layer. The thickness of the first plated layer is about 3 \\\\u03bcm to about 8 \\\\u03bcm, for example. The first plated layer contains nickel. The second plated layer is disposed on the first plated layer. The thickness of the second plated layer is about 0.025 \\\\u03bcm to about 1 \\\\u03bcm, for example. The second plated layer contains lead.\"}', '{\"abstract\": \"Processes of assembling microelectronic packages with lead frames and/or other suitable substrates are described herein. In one embodiment, a method for fabricating a semiconductor assembly includes forming an attachment area and a non-attachment area on a lead finger of a lead frame. The attachment area is more wettable to the solder ball than the non-attachment area during reflow. The method also includes contacting a solder ball carried by a semiconductor die with the attachment area of the lead finger, reflowing the solder ball while the solder ball is in contact with the attachment area of the lead finger, and controllably collapsing the solder ball to establish an electrical connection between the semiconductor die and the lead finger of the lead frame.\"}']\n" - ] - } - ], - "source": [ - "# Extract strings into a list of JSON strings\n", - "json_strings = [json.dumps({'abstract': s}) for s in vector_search_results['content_1']]\n", - "ALL_ABSTRACTS = json_strings\n", - "\n", - "# Print the result (optional)\n", - "print(ALL_ABSTRACTS)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" + "cell_type": "code", + "execution_count": 19, + "metadata": { + "executionInfo": { + "elapsed": 1622, + "status": "ok", + "timestamp": 1742195139318, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "5fb_O-ne5cvH" + }, + "outputs": [], + "source": [ + "## Brute force result (for comparison)\n", + "\n", + "\n", + "brute_force_result = bf_bq.vector_search(\n", + " base_table=f\"{DATASET_ID}.{TEXT_EMBEDDING_TABLE_ID}\",\n", + " column_to_search=\"ml_generate_embedding_result\",\n", + " query=search_query,\n", + " top_k=5,\n", + " distance_type=\"cosine\",\n", + " use_brute_force=True,\n", + ")\n" + ] }, - "collapsed": true, - "executionInfo": { - "elapsed": 1620, - "status": "ok", - "timestamp": 1742195587180, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "markdown", + "metadata": { + "id": "21rNsFMHo8hO" + }, + "source": [ + "## Step 3: AI-Powered Summarization with Retrieval Augmented Generation (RAG)" + ] }, - "id": "kSNSi1GV8OAD", - "outputId": "37fbc822-1160-4fbd-c7d6-ecb4a16db394" - }, - "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "You are an expert patent analyst. I will provide you the abstracts of the top 5 patents in json format retrieved by a vector search based on a user's query.\n", - "Your task is to analyze these abstracts and generate a concise, coherent summary that encapsulates the core innovations and concepts shared among them.\n", - "\n", - "In your output, share the original user query.\n", - "Then output the concise, coherent summary that encapsulates the core innovations and concepts shared among the top 5 abstracts. The heading for this section should\n", - "be : Summary of the top 5 abstracts that are semantically closest to the user query.\n", - "\n", - "User Query: Chip assemblies employing solder bonds to back-side lands including an electrolytic nickel layer\n", - "Top 5 abstracts: ['{\"abstract\": \"A chip package is provided, the chip package including: a chip carrier; a chip disposed over and electrically connected to a chip carrier top side; an electrically insulating material disposed over and at least partially surrounding the chip; one or more electrically conductive contact regions formed over the electrically insulating material and in electrical connection with the chip; and another electrically insulating material disposed over a chip carrier bottom side. An electrically conductive contact region on the chip carrier bottom side is released from the further electrically insulating material.\"}', '{\"abstract\": \"A method of forming a microelectronic assembly includes positioning a support structure adjacent to an active region of a device but not extending onto the active region. The support structure has planar sections. Each planar section has a substantially uniform composition. The composition of at least one of the planar sections differs from the composition of at least one of the other planar sections. A lid is positioned in contact with the support structure and extends over the active region. The support structure is bonded to the device and to the lid.\"}', '{\"abstract\": \"To provide a substrate for a semiconductor mounting device capable of obtaining high reliability. In a semiconductor mounting device substrate of the present invention, a semiconductor chip can be surface-mounted by a flip chip connection method on a semiconductor chip mounting region of a first main surface of a multilayer wiring substrate. A plurality of second main surface side solder bumps 52 forming a plate-like component mounting region 53 are formed at a location immediately below the semiconductor chip 21 on the second main surface 13 of the multilayer wiring board 11. A plate-like component 101 mainly composed of an inorganic material is surface-mounted on the multilayer wiring board 11 by a flip chip connection method via a plurality of second main surface side solder bumps 52. A plurality of second main surface side solder bumps 52 are sealed by a second main surface side underfill 107 provided in the gap S <b> 2 between the second main surface 13 and the plate-like component 101. [Selection] Figure 1\"}', '{\"abstract\": \"A ceramic electronic component includes an electronic component body, an inner electrode, and an outer electrode. The outer electrode includes a fired electrode layer and first and second plated layers. The fired electrode layer is disposed on the electronic component body. The first plated layer is disposed on the fired electrode layer. The thickness of the first plated layer is about 3 \\\\u03bcm to about 8 \\\\u03bcm, for example. The first plated layer contains nickel. The second plated layer is disposed on the first plated layer. The thickness of the second plated layer is about 0.025 \\\\u03bcm to about 1 \\\\u03bcm, for example. The second plated layer contains lead.\"}', '{\"abstract\": \"Processes of assembling microelectronic packages with lead frames and/or other suitable substrates are described herein. In one embodiment, a method for fabricating a semiconductor assembly includes forming an attachment area and a non-attachment area on a lead finger of a lead frame. The attachment area is more wettable to the solder ball than the non-attachment area during reflow. The method also includes contacting a solder ball carried by a semiconductor die with the attachment area of the lead finger, reflowing the solder ball while the solder ball is in contact with the attachment area of the lead finger, and controllably collapsing the solder ball to establish an electrical connection between the semiconductor die and the lead finger of the lead frame.\"}']\n", - "\n", - "Instructions:\n", - "\n", - "Focus on identifying the common themes and key technological advancements described in the abstracts.\n", - "Synthesize the information into a clear and concise summary, approximately 150-200 words.\n", - "Avoid simply copying phrases from the abstracts. Instead, aim to provide a cohesive overview of the shared concepts.\n", - "Highlight the potential applications and benefits of the described inventions.\n", - "Maintain a professional and objective tone.\n", - "Do not mention the individual patents by number, focus on summarizing the shared concepts.\n", - "\n" - ] - } - ], - "source": [ - "## Setup the LLM prompt\n", - "\n", - "prompt = f\"\"\"\n", - "You are an expert patent analyst. I will provide you the abstracts of the top 5 patents in json format retrieved by a vector search based on a user's query.\n", - "Your task is to analyze these abstracts and generate a concise, coherent summary that encapsulates the core innovations and concepts shared among them.\n", - "\n", - "In your output, share the original user query.\n", - "Then output the concise, coherent summary that encapsulates the core innovations and concepts shared among the top 5 abstracts. The heading for this section should\n", - "be : Summary of the top 5 abstracts that are semantically closest to the user query.\n", - "\n", - "User Query: {TEXT_SEARCH_STRING}\n", - "Top 5 abstracts: {ALL_ABSTRACTS}\n", - "\n", - "Instructions:\n", - "\n", - "Focus on identifying the common themes and key technological advancements described in the abstracts.\n", - "Synthesize the information into a clear and concise summary, approximately 150-200 words.\n", - "Avoid simply copying phrases from the abstracts. Instead, aim to provide a cohesive overview of the shared concepts.\n", - "Highlight the potential applications and benefits of the described inventions.\n", - "Maintain a professional and objective tone.\n", - "Do not mention the individual patents by number, focus on summarizing the shared concepts.\n", - "\"\"\"\n", - "\n", - "print(prompt)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "executionInfo": { - "elapsed": 1, - "status": "ok", - "timestamp": 1742195567707, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + "cell_type": "markdown", + "metadata": { + "id": "K3pIQrzB7T_G" + }, + "source": [ + "Patent documents can be dense and time-consuming to digest. AI-Powered Patent Summarization utilizes Retrieval Augmented Generation (RAG) to streamline this process. By retrieving relevant patent information through vector search and then synthesizing it with a large language model, we can generate concise, human-readable summaries, saving valuable time and effort. The code sample below walks through how to set this up continuing with the same user query as the previous use case." + ] }, - "id": "njiQdfkT8Y7V" - }, - "outputs": [], - "source": [ - "## Define a function that will take the input propmpt and run the LLM\n", - "\n", - "def predict(prompt: str, temperature: float = TEMPERATURE) -> str:\n", - " # Create dataframe\n", - " input = bf.DataFrame(\n", - " {\n", - " \"prompt\": [prompt],\n", - " }\n", - " )\n", - "\n", - " # Return response\n", - " return llm_model.predict(input, temperature=temperature).ml_generate_text_llm_result.iloc[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 426 + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "executionInfo": { + "elapsed": 4827, + "status": "ok", + "timestamp": 1742195565658, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "jb5rueqU7T5J", + "outputId": "43732836-ebae-4fb3-b28e-bfea51146c72" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 3fabe659-f95b-49cb-b0c7-9d32b09177bf is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "## gemini model\n", + "\n", + "llm_model = bf_llm.GeminiTextGenerator(model_name = \"gemini-2.0-flash-001\") ## replace with other model as needed" + ] }, - "executionInfo": { - "elapsed": 14425, - "status": "ok", - "timestamp": 1742195608280, - "user": { - "displayName": "", - "userId": "" - }, - "user_tz": -480 + { + "cell_type": "markdown", + "metadata": { + "id": "41e12JTf70sr" + }, + "source": [ + "We will use the same user query from Section 2, and pass the list of abstracts returned by the vector search into the prompt for the RAG application" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "executionInfo": { + "elapsed": 1474, + "status": "ok", + "timestamp": 1742195536109, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "EyP-ZFJK8h-2" + }, + "outputs": [], + "source": [ + "TEMPERATURE = 0.4" + ] }, - "id": "OYYkVYbs8Y0P", - "outputId": "def839e3-3dee-4320-9cb5-cac855ddea6b" - }, - "outputs": [ { - "data": { - "text/html": [ - "Load job 34f3b649-6e45-46db-a6e5-405ae0a8bf69 is DONE. Open Job" + "cell_type": "code", + "execution_count": 22, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 72 + }, + "executionInfo": { + "elapsed": 3371, + "status": "ok", + "timestamp": 1742195421813, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "eP99R6SV7Tug", + "outputId": "c34bc931-5be8-410e-ac1f-604df31ef533" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['{\"abstract\": \"A chip package is provided, the chip package including: a chip carrier; a chip disposed over and electrically connected to a chip carrier top side; an electrically insulating material disposed over and at least partially surrounding the chip; one or more electrically conductive contact regions formed over the electrically insulating material and in electrical connection with the chip; and another electrically insulating material disposed over a chip carrier bottom side. An electrically conductive contact region on the chip carrier bottom side is released from the further electrically insulating material.\"}', '{\"abstract\": \"A method of forming a microelectronic assembly includes positioning a support structure adjacent to an active region of a device but not extending onto the active region. The support structure has planar sections. Each planar section has a substantially uniform composition. The composition of at least one of the planar sections differs from the composition of at least one of the other planar sections. A lid is positioned in contact with the support structure and extends over the active region. The support structure is bonded to the device and to the lid.\"}', '{\"abstract\": \"To provide a substrate for a semiconductor mounting device capable of obtaining high reliability. In a semiconductor mounting device substrate of the present invention, a semiconductor chip can be surface-mounted by a flip chip connection method on a semiconductor chip mounting region of a first main surface of a multilayer wiring substrate. A plurality of second main surface side solder bumps 52 forming a plate-like component mounting region 53 are formed at a location immediately below the semiconductor chip 21 on the second main surface 13 of the multilayer wiring board 11. A plate-like component 101 mainly composed of an inorganic material is surface-mounted on the multilayer wiring board 11 by a flip chip connection method via a plurality of second main surface side solder bumps 52. A plurality of second main surface side solder bumps 52 are sealed by a second main surface side underfill 107 provided in the gap S <b> 2 between the second main surface 13 and the plate-like component 101. [Selection] Figure 1\"}', '{\"abstract\": \"A ceramic electronic component includes an electronic component body, an inner electrode, and an outer electrode. The outer electrode includes a fired electrode layer and first and second plated layers. The fired electrode layer is disposed on the electronic component body. The first plated layer is disposed on the fired electrode layer. The thickness of the first plated layer is about 3 \\\\u03bcm to about 8 \\\\u03bcm, for example. The first plated layer contains nickel. The second plated layer is disposed on the first plated layer. The thickness of the second plated layer is about 0.025 \\\\u03bcm to about 1 \\\\u03bcm, for example. The second plated layer contains lead.\"}', '{\"abstract\": \"Processes of assembling microelectronic packages with lead frames and/or other suitable substrates are described herein. In one embodiment, a method for fabricating a semiconductor assembly includes forming an attachment area and a non-attachment area on a lead finger of a lead frame. The attachment area is more wettable to the solder ball than the non-attachment area during reflow. The method also includes contacting a solder ball carried by a semiconductor die with the attachment area of the lead finger, reflowing the solder ball while the solder ball is in contact with the attachment area of the lead finger, and controllably collapsing the solder ball to establish an electrical connection between the semiconductor die and the lead finger of the lead frame.\"}']\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "# Extract strings into a list of JSON strings\n", + "json_strings = [json.dumps({'abstract': s}) for s in vector_search_results['content_1']]\n", + "ALL_ABSTRACTS = json_strings\n", + "\n", + "# Print the result (optional)\n", + "print(ALL_ABSTRACTS)" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job a574725f-64ae-4a19-aac0-959bec0bffeb is DONE. 5.0 kB processed. Open Job" + "cell_type": "code", + "execution_count": 23, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "executionInfo": { + "elapsed": 1620, + "status": "ok", + "timestamp": 1742195587180, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "kSNSi1GV8OAD", + "outputId": "37fbc822-1160-4fbd-c7d6-ecb4a16db394" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "You are an expert patent analyst. I will provide you the abstracts of the top 5 patents in json format retrieved by a vector search based on a user's query.\n", + "Your task is to analyze these abstracts and generate a concise, coherent summary that encapsulates the core innovations and concepts shared among them.\n", + "\n", + "In your output, share the original user query.\n", + "Then output the concise, coherent summary that encapsulates the core innovations and concepts shared among the top 5 abstracts. The heading for this section should\n", + "be : Summary of the top 5 abstracts that are semantically closest to the user query.\n", + "\n", + "User Query: Chip assemblies employing solder bonds to back-side lands including an electrolytic nickel layer\n", + "Top 5 abstracts: ['{\"abstract\": \"A chip package is provided, the chip package including: a chip carrier; a chip disposed over and electrically connected to a chip carrier top side; an electrically insulating material disposed over and at least partially surrounding the chip; one or more electrically conductive contact regions formed over the electrically insulating material and in electrical connection with the chip; and another electrically insulating material disposed over a chip carrier bottom side. An electrically conductive contact region on the chip carrier bottom side is released from the further electrically insulating material.\"}', '{\"abstract\": \"A method of forming a microelectronic assembly includes positioning a support structure adjacent to an active region of a device but not extending onto the active region. The support structure has planar sections. Each planar section has a substantially uniform composition. The composition of at least one of the planar sections differs from the composition of at least one of the other planar sections. A lid is positioned in contact with the support structure and extends over the active region. The support structure is bonded to the device and to the lid.\"}', '{\"abstract\": \"To provide a substrate for a semiconductor mounting device capable of obtaining high reliability. In a semiconductor mounting device substrate of the present invention, a semiconductor chip can be surface-mounted by a flip chip connection method on a semiconductor chip mounting region of a first main surface of a multilayer wiring substrate. A plurality of second main surface side solder bumps 52 forming a plate-like component mounting region 53 are formed at a location immediately below the semiconductor chip 21 on the second main surface 13 of the multilayer wiring board 11. A plate-like component 101 mainly composed of an inorganic material is surface-mounted on the multilayer wiring board 11 by a flip chip connection method via a plurality of second main surface side solder bumps 52. A plurality of second main surface side solder bumps 52 are sealed by a second main surface side underfill 107 provided in the gap S <b> 2 between the second main surface 13 and the plate-like component 101. [Selection] Figure 1\"}', '{\"abstract\": \"A ceramic electronic component includes an electronic component body, an inner electrode, and an outer electrode. The outer electrode includes a fired electrode layer and first and second plated layers. The fired electrode layer is disposed on the electronic component body. The first plated layer is disposed on the fired electrode layer. The thickness of the first plated layer is about 3 \\\\u03bcm to about 8 \\\\u03bcm, for example. The first plated layer contains nickel. The second plated layer is disposed on the first plated layer. The thickness of the second plated layer is about 0.025 \\\\u03bcm to about 1 \\\\u03bcm, for example. The second plated layer contains lead.\"}', '{\"abstract\": \"Processes of assembling microelectronic packages with lead frames and/or other suitable substrates are described herein. In one embodiment, a method for fabricating a semiconductor assembly includes forming an attachment area and a non-attachment area on a lead finger of a lead frame. The attachment area is more wettable to the solder ball than the non-attachment area during reflow. The method also includes contacting a solder ball carried by a semiconductor die with the attachment area of the lead finger, reflowing the solder ball while the solder ball is in contact with the attachment area of the lead finger, and controllably collapsing the solder ball to establish an electrical connection between the semiconductor die and the lead finger of the lead frame.\"}']\n", + "\n", + "Instructions:\n", + "\n", + "Focus on identifying the common themes and key technological advancements described in the abstracts.\n", + "Synthesize the information into a clear and concise summary, approximately 150-200 words.\n", + "Avoid simply copying phrases from the abstracts. Instead, aim to provide a cohesive overview of the shared concepts.\n", + "Highlight the potential applications and benefits of the described inventions.\n", + "Maintain a professional and objective tone.\n", + "Do not mention the individual patents by number, focus on summarizing the shared concepts.\n", + "\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "## Setup the LLM prompt\n", + "\n", + "prompt = f\"\"\"\n", + "You are an expert patent analyst. I will provide you the abstracts of the top 5 patents in json format retrieved by a vector search based on a user's query.\n", + "Your task is to analyze these abstracts and generate a concise, coherent summary that encapsulates the core innovations and concepts shared among them.\n", + "\n", + "In your output, share the original user query.\n", + "Then output the concise, coherent summary that encapsulates the core innovations and concepts shared among the top 5 abstracts. The heading for this section should\n", + "be : Summary of the top 5 abstracts that are semantically closest to the user query.\n", + "\n", + "User Query: {TEXT_SEARCH_STRING}\n", + "Top 5 abstracts: {ALL_ABSTRACTS}\n", + "\n", + "Instructions:\n", + "\n", + "Focus on identifying the common themes and key technological advancements described in the abstracts.\n", + "Synthesize the information into a clear and concise summary, approximately 150-200 words.\n", + "Avoid simply copying phrases from the abstracts. Instead, aim to provide a cohesive overview of the shared concepts.\n", + "Highlight the potential applications and benefits of the described inventions.\n", + "Maintain a professional and objective tone.\n", + "Do not mention the individual patents by number, focus on summarizing the shared concepts.\n", + "\"\"\"\n", + "\n", + "print(prompt)" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/swast/src/github.com/googleapis/python-bigquery-dataframes-2/bigframes/core/array_value.py:109: PreviewWarning: JSON column interpretation as a custom PyArrow extention in\n", - "`db_dtypes` is a preview feature and subject to change.\n", - " warnings.warn(msg, bfe.PreviewWarning)\n" - ] + "cell_type": "code", + "execution_count": 24, + "metadata": { + "executionInfo": { + "elapsed": 1, + "status": "ok", + "timestamp": 1742195567707, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "njiQdfkT8Y7V" + }, + "outputs": [], + "source": [ + "## Define a function that will take the input propmpt and run the LLM\n", + "\n", + "def predict(prompt: str, temperature: float = TEMPERATURE) -> str:\n", + " # Create dataframe\n", + " input = bf.DataFrame(\n", + " {\n", + " \"prompt\": [prompt],\n", + " }\n", + " )\n", + "\n", + " # Return response\n", + " return llm_model.predict(input, temperature=temperature).ml_generate_text_llm_result.iloc[0]" + ] }, { - "data": { - "text/markdown": [ - "User Query: Chip assemblies employing solder bonds to back-side lands including an electrolytic nickel layer\n", - "\n", - "Summary of the top 5 abstracts that are semantically closest to the user query:\n", - "\n", - "The abstracts describe various aspects of microelectronic assembly and packaging, with a focus on enhancing reliability and electrical connectivity. A common theme is the use of solder bumps or balls for creating electrical connections between different components, such as semiconductor chips and substrates or lead frames. Several abstracts highlight methods for improving the solderability and wettability of contact regions, often involving the use of multiple layers with differing compositions. The use of electrically insulating materials to provide support and protection to the chip and electrical connections is also described. One abstract specifically mentions a nickel-containing plated layer as part of an outer electrode, suggesting its role in improving the electrical or mechanical properties of the connection. The innovations aim to improve the reliability and performance of microelectronic devices through optimized material selection, assembly processes, and structural designs.\n" + "cell_type": "code", + "execution_count": 25, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 426 + }, + "executionInfo": { + "elapsed": 14425, + "status": "ok", + "timestamp": 1742195608280, + "user": { + "displayName": "", + "userId": "" + }, + "user_tz": -480 + }, + "id": "OYYkVYbs8Y0P", + "outputId": "def839e3-3dee-4320-9cb5-cac855ddea6b" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Load job 34f3b649-6e45-46db-a6e5-405ae0a8bf69 is DONE. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job a574725f-64ae-4a19-aac0-959bec0bffeb is DONE. 5.0 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/swast/src/github.com/googleapis/python-bigquery-dataframes-2/bigframes/core/array_value.py:109: PreviewWarning: JSON column interpretation as a custom PyArrow extention in\n", + "`db_dtypes` is a preview feature and subject to change.\n", + " warnings.warn(msg, bfe.PreviewWarning)\n" + ] + }, + { + "data": { + "text/markdown": [ + "User Query: Chip assemblies employing solder bonds to back-side lands including an electrolytic nickel layer\n", + "\n", + "Summary of the top 5 abstracts that are semantically closest to the user query:\n", + "\n", + "The abstracts describe various aspects of microelectronic assembly and packaging, with a focus on enhancing reliability and electrical connectivity. A common theme is the use of solder bumps or balls for creating electrical connections between different components, such as semiconductor chips and substrates or lead frames. Several abstracts highlight methods for improving the solderability and wettability of contact regions, often involving the use of multiple layers with differing compositions. The use of electrically insulating materials to provide support and protection to the chip and electrical connections is also described. One abstract specifically mentions a nickel-containing plated layer as part of an outer electrode, suggesting its role in improving the electrical or mechanical properties of the connection. The innovations aim to improve the reliability and performance of microelectronic devices through optimized material selection, assembly processes, and structural designs.\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "# Invoke LLM with prompt\n", + "response = predict(prompt, temperature = TEMPERATURE)\n", + "\n", + "# Print results as Markdown\n", + "Markdown(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sy82XLDfooEb" + }, + "source": [ + "# Summary and next steps\n", + "\n", + "Ready to dive deeper and explore the endless possibilities? Start building your own vector search applications with BigFrames and BigQuery today! Check out our [documentation](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.bigquery#bigframes_bigquery_vector_search), explore our sample [notebooks](https://github.com/googleapis/python-bigquery-dataframes/tree/main/notebooks), and unleash the power of vector analytics on your data.\n", + "The BigFrames team would also love to hear from you. If you would like to reach out, please send an email to: bigframes-feedback@google.com or by filing an issue at the [open source BigFrames repository](https://github.com/googleapis/python-bigquery-dataframes/issues). To receive updates about BigFrames, subscribe to the BigFrames email list." ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" } - ], - "source": [ - "# Invoke LLM with prompt\n", - "response = predict(prompt, temperature = TEMPERATURE)\n", - "\n", - "# Print results as Markdown\n", - "Markdown(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sy82XLDfooEb" - }, - "source": [ - "# Summary and next steps\n", - "\n", - "Ready to dive deeper and explore the endless possibilities? Start building your own vector search applications with BigFrames and BigQuery today! Check out our [documentation](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.bigquery#bigframes_bigquery_vector_search), explore our sample [notebooks](https://github.com/googleapis/python-bigquery-dataframes/tree/main/notebooks), and unleash the power of vector analytics on your data.\n", - "The BigFrames team would also love to hear from you. If you would like to reach out, please send an email to: bigframes-feedback@google.com or by filing an issue at the [open source BigFrames repository](https://github.com/googleapis/python-bigquery-dataframes/issues). To receive updates about BigFrames, subscribe to the BigFrames email list." - ] - } - ], - "metadata": { - "colab": { - "name": "bq_dataframes_llm_kmeans", - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" + ], + "metadata": { + "colab": { + "name": "bq_dataframes_llm_kmeans", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.16" - } - }, - "nbformat": 4, - "nbformat_minor": 0 + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/notebooks/generative_ai/bq_dataframes_ml_drug_name_generation.ipynb b/notebooks/generative_ai/bq_dataframes_ml_drug_name_generation.ipynb index 93ac3f31c14..3220bbf6cdb 100644 --- a/notebooks/generative_ai/bq_dataframes_ml_drug_name_generation.ipynb +++ b/notebooks/generative_ai/bq_dataframes_ml_drug_name_generation.ipynb @@ -58,6 +58,17 @@ "" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "24743cf4a1e1" + }, + "source": [ + "**_NOTE_**: This notebook has been tested in the following environment:\n", + "\n", + "* Python version = 3.9" + ] + }, { "cell_type": "markdown", "metadata": { @@ -480,7 +491,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": { "id": "UW2fQ2k5Hsic" }, @@ -570,7 +581,7 @@ ], "source": [ "# Define the model\n", - "model = GeminiTextGenerator(model_name=\"gemini-2.5-flash\")\n", + "model = GeminiTextGenerator(model_name=\"gemini-2.0-flash-001\")\n", "\n", "# Invoke LLM with prompt\n", "response = predict(zero_shot_prompt, temperature = TEMPERATURE)\n", @@ -1568,21 +1579,12 @@ "provenance": [] }, "kernelspec": { - "display_name": "venv", - "language": "python", + "display_name": "Python 3", "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.1" + "version": "3.10.9" } }, "nbformat": 4, diff --git a/notebooks/generative_ai/large_language_models.ipynb b/notebooks/generative_ai/large_language_models.ipynb index 4ff9a9d3d23..1d7bc7f6ef1 100644 --- a/notebooks/generative_ai/large_language_models.ipynb +++ b/notebooks/generative_ai/large_language_models.ipynb @@ -16,12 +16,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Define the model" + "## Define the model" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -60,7 +60,7 @@ } ], "source": [ - "model = GeminiTextGenerator(model_name=\"gemini-2.5-flash\")" + "model = GeminiTextGenerator(model_name=\"gemini-2.0-flash-001\")" ] }, { diff --git a/notebooks/geo/geoseries.ipynb b/notebooks/geo/geoseries.ipynb index 1159b8d31de..953fc8f45fa 100644 --- a/notebooks/geo/geoseries.ipynb +++ b/notebooks/geo/geoseries.ipynb @@ -44,7 +44,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 1. Load the Counties table from the Census Bureau US Boundaries dataset" + "### 1. Load the Counties table from the Census Bureau US Boundaries dataset" ] }, { @@ -699,7 +699,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Reuse `five_geom` and `geom_obj` to find the difference between the geometry objects" + "#### Reuse `five_geom` and `geom_obj` to find the difference between the geometry objects" ] }, { @@ -902,7 +902,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Reuse `wkts_from_geo` and `geom_obj`" + "#### Reuse `wkts_from_geo` and `geom_obj`" ] }, { diff --git a/notebooks/getting_started/bq_dataframes_template.ipynb b/notebooks/getting_started/bq_dataframes_template.ipynb index 664a3a68d33..0970dcedc91 100644 --- a/notebooks/getting_started/bq_dataframes_template.ipynb +++ b/notebooks/getting_started/bq_dataframes_template.ipynb @@ -1305,13 +1305,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ "# from bigframes.ml.llm import GeminiTextGenerator\n", "\n", - "# model = GeminiTextGenerator(model_name=\"gemini-2.5-flash\")\n", + "# model = GeminiTextGenerator(model_name=\"gemini-2.0-flash-001\")\n", "\n", "# pred = model.predict(df)\n", "# pred" diff --git a/notebooks/getting_started/getting_started_bq_dataframes.ipynb b/notebooks/getting_started/getting_started_bq_dataframes.ipynb index f9fb950c534..fa88cf65bbb 100644 --- a/notebooks/getting_started/getting_started_bq_dataframes.ipynb +++ b/notebooks/getting_started/getting_started_bq_dataframes.ipynb @@ -1329,6 +1329,20 @@ "Running your own Python functions (or being able to bring your packages) and using them at scale is a challenge many data scientists face. BigQuery DataFrames makes it easy to deploy [remote functions](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.pandas#bigframes_pandas_remote_function) that run scalar Python functions at BigQuery scale. These functions are persisted as [BigQuery remote functions](https://cloud.google.com/bigquery/docs/remote-functions) that you can then re-use." ] }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Python 3.13 is not yet a supported runtime for remote functions.\n", + "# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n", + "if sys.version_info >= (3, 13, 0):\n", + " sys.exit(0)" + ] + }, { "cell_type": "markdown", "metadata": { diff --git a/notebooks/getting_started/magics.ipynb b/notebooks/getting_started/magics.ipynb deleted file mode 100644 index 1f2cf7a409b..00000000000 --- a/notebooks/getting_started/magics.ipynb +++ /dev/null @@ -1,406 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "91edcf7b", - "metadata": {}, - "source": [ - "# %%bqsql cell magics\n", - "\n", - "The BigQuery DataFrames (aka BigFrames) package provides a `%%bqsql` cell magics for Jupyter environments.\n", - "\n", - "To use it, first activate the extension:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "98cd0489", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext bigframes" - ] - }, - { - "cell_type": "markdown", - "id": "f18fdc63", - "metadata": {}, - "source": [ - "Now, use the magics by including SQL in the body." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "269c5862", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes. [Job bigframes-dev:US.job_UVe7FsupxF3CbYuLcLT7fpw9dozg details]\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1e2fb7b019754d31b11323a054f97f47", - "version_major": 2, - "version_minor": 1 - }, - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    stategenderyearnamenumber
    0HIF1999Ariana10
    1HIF2002Jordyn10
    2HIF2006Mya10
    3HIF2010Jordyn10
    4HIM1921Nobuo10
    5HIM1925Ralph10
    6HIM1926Hisao10
    7HIM1927Moses10
    8HIM1933Larry10
    9HIM1933Alfredo10
    \n", - "

    10 rows × 5 columns

    \n", - "
    [5552452 rows x 5 columns in total]" - ], - "text/plain": [ - "state gender year name number\n", - " HI F 1999 Ariana 10\n", - " HI F 2002 Jordyn 10\n", - " HI F 2006 Mya 10\n", - " HI F 2010 Jordyn 10\n", - " HI M 1921 Nobuo 10\n", - " HI M 1925 Ralph 10\n", - " HI M 1926 Hisao 10\n", - " HI M 1927 Moses 10\n", - " HI M 1933 Larry 10\n", - " HI M 1933 Alfredo 10\n", - "...\n", - "\n", - "[5552452 rows x 5 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%%bqsql\n", - "SELECT * FROM `bigquery-public-data.usa_names.usa_1910_2013`" - ] - }, - { - "cell_type": "markdown", - "id": "8771e10f", - "metadata": {}, - "source": [ - "The output DataFrame can be saved to a variable." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "30bb6327", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes. [Job bigframes-dev:US.c142adf3-cd95-42da-bbdc-c176b36b934f details]\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%%bqsql mydf\n", - "SELECT * FROM `bigquery-public-data.usa_names.usa_1910_2013`" - ] - }, - { - "cell_type": "markdown", - "id": "533e2e9e", - "metadata": {}, - "source": [ - "You can chain cells together using format strings. DataFrame objects are automatically turned into table expressions." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "6a8a8123", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " Query processed 88.1 MB in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c4889de9296440428de90defb5c58070", - "version_major": 2, - "version_minor": 1 - }, - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    total_countname
    0304036Tracy
    1293876Travis
    2203784Troy
    3150127Trevor
    496397Tristan
    589996Tracey
    665546Trinity
    750112Traci
    849657Trenton
    945692Trent
    \n", - "

    10 rows × 2 columns

    \n", - "
    [238 rows x 2 columns in total]" - ], - "text/plain": [ - " total_count name\n", - "0 304036 Tracy\n", - "1 293876 Travis\n", - "2 203784 Troy\n", - "3 150127 Trevor\n", - "4 96397 Tristan\n", - "5 89996 Tracey\n", - "6 65546 Trinity\n", - "7 50112 Traci\n", - "8 49657 Trenton\n", - "9 45692 Trent\n", - "...\n", - "\n", - "[238 rows x 2 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%%bqsql\n", - "SELECT sum(number) as total_count, name\n", - "FROM {mydf}\n", - "WHERE name LIKE 'Tr%'\n", - "GROUP BY name\n", - "ORDER BY total_count DESC" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d2a17078", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.18" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/getting_started/pandas_extensions.ipynb b/notebooks/getting_started/pandas_extensions.ipynb deleted file mode 100644 index c511eab9b4a..00000000000 --- a/notebooks/getting_started/pandas_extensions.ipynb +++ /dev/null @@ -1,160 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# BigQuery extension for pandas\n", - "\n", - "BigQuery DataFrames provides a pandas extension to execute BigQuery SQL scalar functions directly on pandas DataFrames." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import bigframes # This import registers the bigquery accessor." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "By default, BigQuery DataFrames selects a location to process data based on the\n", - "data location, but using a pandas object doesn't provide such informat. If\n", - "processing location is important to you, configure the location before using the\n", - "accessor." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import bigframes.pandas as bpd\n", - "\n", - "bpd.reset_session()\n", - "bpd.options.bigquery.location = \"US\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using `sql_scalar`\n", - "\n", - "The `bigquery.sql_scalar` method allows you to apply a SQL scalar function to a pandas DataFrame by converting it to BigFrames, executing the SQL in BigQuery, and returning the result as a pandas Series." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "0 2.0\n", - "1 3.0\n", - "2 4.0\n", - "dtype: Float64" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = pd.DataFrame({\"a\": [1.5, 2.5, 3.5]})\n", - "result = df.bigquery.sql_scalar(\"ROUND({0}, 0)\")\n", - "result" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also use multiple columns." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " Query processed 0 Bytes in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "0 11\n", - "1 22\n", - "2 33\n", - "dtype: Int64" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = pd.DataFrame({\"a\": [1, 2, 3], \"b\": [10, 20, 30]})\n", - "result = df.bigquery.sql_scalar(\"{a} + {b}\")\n", - "result" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.9" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/kaggle/describe-product-images-with-bigframes-multimodal.ipynb b/notebooks/kaggle/describe-product-images-with-bigframes-multimodal.ipynb index 1a7de9b837f..1c2e2b53a83 100644 --- a/notebooks/kaggle/describe-product-images-with-bigframes-multimodal.ipynb +++ b/notebooks/kaggle/describe-product-images-with-bigframes-multimodal.ipynb @@ -1,1131 +1 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "876eb80c", - "metadata": { - "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", - "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5" - }, - "source": [ - "# Describe product images with BigFrames multimodal DataFrames\n", - "\n", - "Based on notebook at https://github.com/googleapis/python-bigquery-dataframes/blob/main/notebooks/multimodal/multimodal_dataframe.ipynb\n", - "\n", - "This notebook is introducing BigFrames Multimodal features:\n", - "\n", - "1. Create Multimodal DataFrame\n", - "2. Combine unstructured data with structured data\n", - "3. Conduct image transformations\n", - "4. Use LLM models to ask questions and generate embeddings on images\n", - "5. PDF chunking function\n", - "\n", - "Install the bigframes package and upgrade other packages that are already included in Kaggle but have versions incompatible with bigframes." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "0506e15e", - "metadata": { - "trusted": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: bigframes in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (2.39.0)\n", - "Requirement already satisfied: google-cloud-automl in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (2.19.0)\n", - "Requirement already satisfied: google-cloud-translate in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (3.26.0)\n", - "Requirement already satisfied: google-ai-generativelanguage in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (0.11.0)\n", - "Requirement already satisfied: tensorflow in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (2.21.0)\n", - "Requirement already satisfied: cloudpickle>=2.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (3.1.2)\n", - "Requirement already satisfied: fsspec>=2023.3.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2026.1.0)\n", - "Requirement already satisfied: gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2026.1.0)\n", - "Requirement already satisfied: geopandas>=0.12.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.1.3)\n", - "Requirement already satisfied: google-auth<3.0,>=2.15.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.49.1)\n", - "Requirement already satisfied: google-cloud-bigquery>=3.36.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-cloud-bigquery[bqstorage,pandas]>=3.36.0->bigframes) (3.41.0)\n", - "Requirement already satisfied: google-cloud-bigquery-storage<3.0.0,>=2.30.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.37.0)\n", - "Requirement already satisfied: google-cloud-functions>=1.12.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.23.0)\n", - "Requirement already satisfied: google-cloud-bigquery-connection>=1.12.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.21.0)\n", - "Requirement already satisfied: google-cloud-resource-manager>=1.10.3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.17.0)\n", - "Requirement already satisfied: google-cloud-storage>=2.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (3.10.1)\n", - "Requirement already satisfied: google-crc32c<2.0.0,>=1.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.8.0)\n", - "Requirement already satisfied: grpc-google-iam-v1>=0.14.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (0.14.4)\n", - "Requirement already satisfied: numpy>=1.24.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.4.4)\n", - "Requirement already satisfied: pandas>=1.5.3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.3.3)\n", - "Requirement already satisfied: pandas-gbq>=0.26.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (0.34.1)\n", - "Requirement already satisfied: pyarrow>=15.0.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (21.0.0)\n", - "Requirement already satisfied: pydata-google-auth>=1.8.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.9.1)\n", - "Requirement already satisfied: requests>=2.27.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.33.1)\n", - "Requirement already satisfied: shapely>=1.8.5 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.1.2)\n", - "Requirement already satisfied: tabulate>=0.9 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (0.10.0)\n", - "Requirement already satisfied: humanize>=4.6.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (4.15.0)\n", - "Requirement already satisfied: matplotlib>=3.7.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (3.10.8)\n", - "Requirement already satisfied: db-dtypes>=1.4.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.5.1)\n", - "Requirement already satisfied: pyiceberg>=0.7.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (0.11.1)\n", - "Requirement already satisfied: atpublic<6,>=2.3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (5.1)\n", - "Requirement already satisfied: python-dateutil<3,>=2.8.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2022.7 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2026.1.post1)\n", - "Requirement already satisfied: toolz<2,>=0.11 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.1.0)\n", - "Requirement already satisfied: typing-extensions<5,>=4.5.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (4.15.0)\n", - "Requirement already satisfied: rich<14,>=12.4.4 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (13.9.4)\n", - "Requirement already satisfied: google-api-core<3.0.0,>=2.11.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-api-core[grpc]<3.0.0,>=2.11.0->google-cloud-automl) (2.30.2)\n", - "Requirement already satisfied: grpcio<2.0.0,>=1.33.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-cloud-automl) (1.80.0)\n", - "Requirement already satisfied: proto-plus<2.0.0,>=1.22.3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-cloud-automl) (1.27.2)\n", - "Requirement already satisfied: protobuf<8.0.0,>=4.25.8 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-cloud-automl) (6.33.6)\n", - "Requirement already satisfied: google-cloud-core<3.0.0,>=2.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-cloud-translate) (2.5.1)\n", - "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (2.4.0)\n", - "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (1.6.3)\n", - "Requirement already satisfied: flatbuffers>=25.9.23 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (25.12.19)\n", - "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (0.7.0)\n", - "Requirement already satisfied: google_pasta>=0.1.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (0.2.0)\n", - "Requirement already satisfied: libclang>=13.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (18.1.1)\n", - "Requirement already satisfied: opt_einsum>=2.3.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (3.4.0)\n", - "Requirement already satisfied: packaging in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (26.0)\n", - "Requirement already satisfied: setuptools in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (82.0.1)\n", - "Requirement already satisfied: six>=1.12.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (1.17.0)\n", - "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (3.3.0)\n", - "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (2.1.2)\n", - "Requirement already satisfied: keras>=3.12.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (3.14.0)\n", - "Requirement already satisfied: h5py<3.15.0,>=3.11.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (3.14.0)\n", - "Requirement already satisfied: ml_dtypes<1.0.0,>=0.5.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (0.5.4)\n", - "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from astunparse>=1.6.0->tensorflow) (0.47.0)\n", - "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (3.13.5)\n", - "Requirement already satisfied: decorator>4.1.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (5.2.1)\n", - "Requirement already satisfied: google-auth-oauthlib in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (1.3.1)\n", - "Requirement already satisfied: google-cloud-storage-control in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (1.11.0)\n", - "Requirement already satisfied: pyogrio>=0.7.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from geopandas>=0.12.2->bigframes) (0.12.1)\n", - "Requirement already satisfied: pyproj>=3.5.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from geopandas>=0.12.2->bigframes) (3.7.2)\n", - "Requirement already satisfied: googleapis-common-protos<2.0.0,>=1.63.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-api-core<3.0.0,>=2.11.0->google-api-core[grpc]<3.0.0,>=2.11.0->google-cloud-automl) (1.74.0)\n", - "Requirement already satisfied: grpcio-status<2.0.0,>=1.33.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-api-core[grpc]<3.0.0,>=2.11.0->google-cloud-automl) (1.80.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-auth<3.0,>=2.15.0->bigframes) (0.4.2)\n", - "Requirement already satisfied: cryptography>=38.0.3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-auth<3.0,>=2.15.0->bigframes) (46.0.7)\n", - "Requirement already satisfied: google-resumable-media<3.0.0,>=2.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-cloud-bigquery>=3.36.0->google-cloud-bigquery[bqstorage,pandas]>=3.36.0->bigframes) (2.8.2)\n", - "Requirement already satisfied: namex in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from keras>=3.12.0->tensorflow) (0.1.0)\n", - "Requirement already satisfied: optree in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from keras>=3.12.0->tensorflow) (0.19.0)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from matplotlib>=3.7.1->bigframes) (1.3.3)\n", - "Requirement already satisfied: cycler>=0.10 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from matplotlib>=3.7.1->bigframes) (0.12.1)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from matplotlib>=3.7.1->bigframes) (4.62.1)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from matplotlib>=3.7.1->bigframes) (1.5.0)\n", - "Requirement already satisfied: pillow>=8 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from matplotlib>=3.7.1->bigframes) (12.2.0)\n", - "Requirement already satisfied: pyparsing>=3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from matplotlib>=3.7.1->bigframes) (3.3.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pandas>=1.5.3->bigframes) (2026.1)\n", - "Requirement already satisfied: psutil>=5.9.8 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pandas-gbq>=0.26.1->bigframes) (7.2.2)\n", - "Requirement already satisfied: mmh3<6.0.0,>=4.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (5.2.1)\n", - "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (8.3.2)\n", - "Requirement already satisfied: strictyaml<2.0.0,>=1.7.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (1.7.3)\n", - "Requirement already satisfied: pydantic!=2.12.0,!=2.12.1,!=2.4.0,!=2.4.1,<3.0,>=2.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (2.12.5)\n", - "Requirement already satisfied: tenacity<10.0.0,>=8.2.3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (9.1.4)\n", - "Requirement already satisfied: pyroaring<2.0.0,>=1.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (1.0.4)\n", - "Requirement already satisfied: cachetools<7.0,>=5.5 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (6.2.6)\n", - "Requirement already satisfied: zstandard<1.0.0,>=0.13.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (0.25.0)\n", - "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from requests>=2.27.1->bigframes) (3.4.7)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from requests>=2.27.1->bigframes) (3.11)\n", - "Requirement already satisfied: urllib3<3,>=1.26 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from requests>=2.27.1->bigframes) (2.6.3)\n", - "Requirement already satisfied: certifi>=2023.5.7 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from requests>=2.27.1->bigframes) (2026.2.25)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from rich<14,>=12.4.4->bigframes) (4.0.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from rich<14,>=12.4.4->bigframes) (2.20.0)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (2.6.1)\n", - "Requirement already satisfied: aiosignal>=1.4.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (1.4.0)\n", - "Requirement already satisfied: attrs>=17.3.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (26.1.0)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (1.8.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (6.7.1)\n", - "Requirement already satisfied: propcache>=0.2.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (0.4.1)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (1.23.0)\n", - "Requirement already satisfied: cffi>=2.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from cryptography>=38.0.3->google-auth<3.0,>=2.15.0->bigframes) (2.0.0)\n", - "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-auth-oauthlib->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (2.0.0)\n", - "Requirement already satisfied: mdurl~=0.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from markdown-it-py>=2.2.0->rich<14,>=12.4.4->bigframes) (0.1.2)\n", - "Requirement already satisfied: pyasn1<0.7.0,>=0.6.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0,>=2.15.0->bigframes) (0.6.3)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pydantic!=2.12.0,!=2.12.1,!=2.4.0,!=2.4.1,<3.0,>=2.0->pyiceberg>=0.7.1->bigframes) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.41.5 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pydantic!=2.12.0,!=2.12.1,!=2.4.0,!=2.4.1,<3.0,>=2.0->pyiceberg>=0.7.1->bigframes) (2.41.5)\n", - "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pydantic!=2.12.0,!=2.12.1,!=2.4.0,!=2.4.1,<3.0,>=2.0->pyiceberg>=0.7.1->bigframes) (0.4.2)\n", - "Requirement already satisfied: pycparser in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from cffi>=2.0.0->cryptography>=38.0.3->google-auth<3.0,>=2.15.0->bigframes) (3.0)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (3.3.1)\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.1\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], - "source": [ - "%pip install --upgrade bigframes google-cloud-automl google-cloud-translate google-ai-generativelanguage tensorflow " - ] - }, - { - "cell_type": "markdown", - "id": "c749e07c", - "metadata": {}, - "source": [ - "**Important:** restart the kernel by going to \"Run -> Restart & clear cell outputs\" before continuing.\n", - "\n", - "Configure bigframes to use your GCP project. First, go to \"Add-ons -> Google Cloud SDK\" and click the \"Attach\" button. Then," - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "5e00777d", - "metadata": { - "execution": { - "iopub.execute_input": "2025-08-18T20:17:14.873201Z", - "iopub.status.busy": "2025-08-18T20:17:14.872905Z", - "iopub.status.idle": "2025-08-18T20:17:14.946971Z", - "shell.execute_reply": "2025-08-18T20:17:14.945996Z", - "shell.execute_reply.started": "2025-08-18T20:17:14.873171Z" - }, - "trusted": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Not running on Kaggle, skipping Kaggle secrets initialization.\n" - ] - } - ], - "source": [ - "try:\n", - " from kaggle_secrets import UserSecretsClient\n", - " user_secrets = UserSecretsClient()\n", - " user_credential = user_secrets.get_gcloud_credential()\n", - " user_secrets.set_tensorflow_credential(user_credential)\n", - " print(\"Successfully authenticated using Kaggle secrets.\")\n", - "except ImportError:\n", - " print(\"Not running on Kaggle, skipping Kaggle secrets initialization.\")\n", - "except Exception as e:\n", - " print(f\"Could not initialize Kaggle secrets: {e}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "b2e171de", - "metadata": { - "execution": { - "iopub.execute_input": "2025-08-18T20:17:25.574192Z", - "iopub.status.busy": "2025-08-18T20:17:25.573874Z", - "iopub.status.idle": "2025-08-18T20:17:45.102002Z", - "shell.execute_reply": "2025-08-18T20:17:45.101140Z", - "shell.execute_reply.started": "2025-08-18T20:17:25.574168Z" - }, - "trusted": true - }, - "outputs": [], - "source": [ - "PROJECT = \"bigframes-dev\" # replace with your project. \n", - "# Refer to https://cloud.google.com/bigquery/docs/multimodal-data-dataframes-tutorial#required_roles for your required permissions\n", - "\n", - "LOCATION = \"us\" # replace with your location.\n", - "DATASET_ID = \"bigframes_samples\" # replace with your dataset ID.\n", - "OUTPUT_BUCKET = \"bigframes_blob_test\" # replace with your GCS bucket. \n", - "\n", - "FULL_CONNECTION_ID = f\"{PROJECT}.{LOCATION}.bigframes-default-connection\"\n", - "\n", - "import bigframes\n", - "# Setup project\n", - "bigframes.options.bigquery.project = PROJECT\n", - "bigframes.options.bigquery.location = LOCATION\n", - "\n", - "# Display options\n", - "bigframes.options.display.blob_display_width = 300\n", - "bigframes.options.display.progress_bar = None\n", - "\n", - "import bigframes.pandas as bpd\n", - "import bigframes.bigquery as bbq\n", - "\n", - "def get_runtime_json_str(series, mode=\"R\", with_metadata=False):\n", - " \"\"\"Get runtime JSON from objectref.\"\"\"\n", - " s = bbq.obj.fetch_metadata(series) if with_metadata else series\n", - " runtime = bbq.obj.get_access_url(s, mode=mode)\n", - " return bbq.to_json_string(runtime)\n", - "\n", - "def get_metadata(series):\n", - " metadata_obj = bbq.obj.fetch_metadata(series)\n", - " return bbq.json_query(metadata_obj.struct.field(\"details\"), \"$.gcs_metadata\")\n", - "\n", - "def get_content_type(series):\n", - " return bbq.json_value(get_metadata(series), \"$.content_type\")\n", - "\n", - "def get_size(series):\n", - " return bbq.json_value(get_metadata(series), \"$.size\").astype(\"Int64\")\n", - "\n", - "def get_updated(series):\n", - " return bpd.to_datetime(bbq.json_value(get_metadata(series), \"$.updated\").astype(\"Int64\"), unit=\"us\", utc=True)\n", - "\n", - "from IPython.display import HTML, display\n", - "\n", - "def render_images(df):\n", - " \"\"\"Helper to display BigFrames DataFrame with rendered image previews.\"\"\"\n", - " import bigframes.pandas as bpd\n", - " import bigframes.bigquery as bbq\n", - " import bigframes\n", - " from bigframes import dtypes\n", - " import json\n", - " \n", - " if isinstance(df, bpd.Series):\n", - " df = df.to_frame()\n", - " \n", - " object_cols = [\n", - " col for col, dtype in zip(df.columns, df.dtypes)\n", - " if dtype == dtypes.OBJ_REF_DTYPE\n", - " ]\n", - " \n", - " if not object_cols:\n", - " display(df)\n", - " return\n", - "\n", - " limit = bigframes.options.display.max_rows or 10\n", - " view_df = df.head(limit)\n", - " \n", - " runtime_cols = {\n", - " col: get_runtime_json_str(view_df[col], mode=\"R\", with_metadata=False) \n", - " for col in object_cols\n", - " }\n", - " \n", - " pandas_json_df = bpd.DataFrame(runtime_cols).to_pandas()\n", - " final_pd = view_df.to_pandas()\n", - " \n", - " width = bigframes.options.display.blob_display_width or 300\n", - " IMAGE_EXTENSIONS = (\".png\", \".jpg\", \".jpeg\", \".gif\", \".webp\")\n", - " \n", - " def format_cell_html(raw_json):\n", - " if not raw_json:\n", - " return \"\"\n", - " try:\n", - " obj_rt = json.loads(raw_json)\n", - " if \"access_urls\" not in obj_rt:\n", - " err = obj_rt.get(\"errors\", [{\"message\": \"URL Generation Failed\"}])[0].get(\"message\")\n", - " return f'Error: {err}'\n", - " \n", - " uri = obj_rt.get(\"objectref\", {}).get(\"uri\", \"\")\n", - " url = obj_rt[\"access_urls\"][\"read_url\"]\n", - " \n", - " if uri and str(uri).lower().endswith(IMAGE_EXTENSIONS):\n", - " return f''\n", - " \n", - " return f'{uri if uri else \"view\"}'\n", - " except:\n", - " return \"Format Error\"\n", - "\n", - " for col in object_cols:\n", - " final_pd[col] = pandas_json_df[col].map(format_cell_html)\n", - " \n", - " display(HTML(final_pd.to_html(escape=False)))" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "d17afaf1", - "metadata": { - "execution": { - "iopub.execute_input": "2025-08-18T20:17:45.103530Z", - "iopub.status.busy": "2025-08-18T20:17:45.103249Z", - "iopub.status.idle": "2025-08-18T20:17:47.424586Z", - "shell.execute_reply": "2025-08-18T20:17:47.423762Z", - "shell.execute_reply.started": "2025-08-18T20:17:45.103499Z" - }, - "trusted": true - }, - "outputs": [], - "source": [ - "import gcsfs\n", - "import bigframes.bigquery as bbq\n", - "\n", - "# List files using gcsfs (public bucket)\n", - "fs = gcsfs.GCSFileSystem(anon=True)\n", - "uris = fs.glob(\"gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/images/*\")\n", - "\n", - "# Ensure URIs have gs:// prefix\n", - "uris = [u if u.startswith(\"gs://\") else f\"gs://{u}\" for u in uris]\n", - "\n", - "# Read the URIs into a BigQuery DataFrame using UNNEST\n", - "# We take the first 5 for this example\n", - "df_image = bpd.read_gbq(f\"SELECT uri FROM UNNEST({uris[:5]}) as uri\")\n", - "\n", - "# Create the object reference column\n", - "df_image['image'] = bbq.obj.make_ref(df_image['uri'], authorizer=FULL_CONNECTION_ID)\n", - "df_image = df_image[['image']]" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "3e84b922", - "metadata": { - "execution": { - "iopub.execute_input": "2025-08-18T20:17:47.425873Z", - "iopub.status.busy": "2025-08-18T20:17:47.425578Z", - "iopub.status.idle": "2025-08-18T20:18:07.919961Z", - "shell.execute_reply": "2025-08-18T20:18:07.918942Z", - "shell.execute_reply.started": "2025-08-18T20:17:47.425844Z" - }, - "trusted": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    image
    0
    1
    2
    3
    4
    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Take only the 5 images to deal with. Preview the content of the Mutimodal DataFrame\n", - "df_image = df_image.head(5)\n", - "render_images(df_image)" - ] - }, - { - "cell_type": "markdown", - "id": "b0eaa73c", - "metadata": {}, - "source": [ - "# 2. Combine unstructured data with structured data\n", - "\n", - "Now you can put more information into the table to describe the files. Such as author info from inputs, or other metadata from the gcs object itself." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "7d64fb54", - "metadata": { - "execution": { - "iopub.execute_input": "2025-08-18T20:18:07.922593Z", - "iopub.status.busy": "2025-08-18T20:18:07.921884Z", - "iopub.status.idle": "2025-08-18T20:18:35.549725Z", - "shell.execute_reply": "2025-08-18T20:18:35.548942Z", - "shell.execute_reply.started": "2025-08-18T20:18:07.922551Z" - }, - "trusted": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    imageauthorcontent_typesizeupdated
    0aliceimage/png7157662025-03-20 17:44:38+00:00
    1bobimage/png11674062025-03-20 17:44:38+00:00
    2bobimage/png11508922025-03-20 17:44:39+00:00
    3aliceimage/png17365332025-03-20 17:44:39+00:00
    4bobimage/png4397402025-03-20 17:44:39+00:00
    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Combine unstructured data with structured data\n", - "df_image[\"author\"] = [\"alice\", \"bob\", \"bob\", \"alice\", \"bob\"] # type: ignore\n", - "df_image[\"content_type\"] = get_content_type(df_image[\"image\"])\n", - "df_image[\"size\"] = get_size(df_image[\"image\"])\n", - "df_image[\"updated\"] = get_updated(df_image[\"image\"])\n", - "render_images(df_image)" - ] - }, - { - "cell_type": "markdown", - "id": "a23ef0e4", - "metadata": {}, - "source": [ - "Then you can filter the rows based on the structured data. And for different content types, you can display them respectively or together." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "ce102df0", - "metadata": { - "execution": { - "iopub.execute_input": "2025-08-18T20:18:55.300314Z", - "iopub.status.busy": "2025-08-18T20:18:55.299993Z", - "iopub.status.idle": "2025-08-18T20:19:09.154492Z", - "shell.execute_reply": "2025-08-18T20:19:09.153315Z", - "shell.execute_reply.started": "2025-08-18T20:18:55.300289Z" - }, - "trusted": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    imageauthorcontent_typesizeupdated
    0aliceimage/png7157662025-03-20 17:44:38+00:00
    3aliceimage/png17365332025-03-20 17:44:39+00:00
    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# filter images and display, you can also display audio and video types\n", - "filtered_df = df_image[df_image[\"author\"] == \"alice\"]\n", - "render_images(filtered_df)" - ] - }, - { - "cell_type": "markdown", - "id": "db2b3b12", - "metadata": {}, - "source": [ - "# 3. Conduct image transformations\n", - "\n", - "BigFrames Multimodal DataFrame provides image(and other) transformation functions. Such as image_blur, image_resize and image_normalize. The output can be saved to GCS folders or to BQ as bytes." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "283036f5", - "metadata": { - "execution": { - "iopub.execute_input": "2025-08-18T20:19:22.950652Z", - "iopub.status.busy": "2025-08-18T20:19:22.950277Z", - "iopub.status.idle": "2025-08-18T20:31:51.799997Z", - "shell.execute_reply": "2025-08-18T20:31:51.798840Z", - "shell.execute_reply.started": "2025-08-18T20:19:22.950625Z" - }, - "trusted": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/pandas/__init__.py:211: PreviewWarning: udf is in preview.\n", - " return global_session.with_default_session(\n", - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dataframe.py:4695: FunctionAxisOnePreviewWarning: DataFrame.apply with parameter axis=1 scenario is in preview.\n", - " warnings.warn(msg, category=bfe.FunctionAxisOnePreviewWarning)\n", - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    imageblurred
    0
    1
    2
    3
    4
    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "@bpd.udf(\n", - " input_types=[str, str, int, int],\n", - " output_type=str,\n", - " dataset=DATASET_ID,\n", - " name=\"image_blur_kaggle\",\n", - " bigquery_connection=FULL_CONNECTION_ID,\n", - " packages=[\"opencv-python-headless\", \"numpy\", \"requests\"],\n", - ")\n", - "def image_blur(src_rt: str, dst_rt: str, kx: int, ky: int) -> str:\n", - " import json\n", - " import cv2 as cv\n", - " import numpy as np\n", - " import requests\n", - " \n", - " src_obj = json.loads(src_rt)\n", - " if \"access_urls\" not in src_obj:\n", - " raise ValueError(f\"Missing 'access_urls' in source object. Response: {src_obj}\")\n", - " src_url = src_obj[\"access_urls\"][\"read_url\"]\n", - " \n", - " response = requests.get(src_url, timeout=30)\n", - " response.raise_for_status()\n", - " \n", - " img = cv.imdecode(np.frombuffer(response.content, np.uint8), cv.IMREAD_UNCHANGED)\n", - " if img is None:\n", - " raise ValueError(\"cv.imdecode failed\")\n", - " \n", - " img_blurred = cv.blur(img, ksize=(int(kx), int(ky)))\n", - " success, encoded = cv.imencode(\".jpeg\", img_blurred)\n", - " \n", - " if not success:\n", - " raise ValueError(\"cv.imencode failed\")\n", - " \n", - " if dst_rt: # GCS Output Mode\n", - " dst_obj = json.loads(dst_rt)\n", - " if \"access_urls\" not in dst_obj:\n", - " raise ValueError(f\"Missing 'access_urls' in destination object. Response: {dst_obj}\")\n", - " dst_url = dst_obj[\"access_urls\"][\"write_url\"]\n", - " \n", - " requests.put(dst_url, data=encoded.tobytes(), headers={\"Content-Type\": \"image/jpeg\"}, timeout=30).raise_for_status()\n", - " return dst_obj[\"objectref\"][\"uri\"]\n", - " return \"\"\n", - "\n", - "def apply_transformation(series, dst_folder, udf, *args, verbose=False):\n", - " import os\n", - " dst_folder = os.path.join(dst_folder, \"\")\n", - " metadata = bbq.obj.fetch_metadata(series)\n", - " current_uri = metadata.struct.field(\"uri\")\n", - " dst_uri = current_uri.str.replace(r\"^.*\\/(.*)$\", rf\"{dst_folder}\\1\", regex=True)\n", - " \n", - " # Bypass synchronous validation via JSON initialization\n", - " dst_blob_df = bpd.DataFrame({\"uri\": dst_uri})\n", - " dst_blob_df[\"authorizer\"] = FULL_CONNECTION_ID\n", - " dst_blob = bbq.obj.make_ref(bbq.to_json(bbq.struct(dst_blob_df)))\n", - "\n", - " df_transform = bpd.DataFrame({\n", - " \"src_rt\": get_runtime_json_str(series, mode=\"R\"),\n", - " \"dst_rt\": get_runtime_json_str(dst_blob, mode=\"RW\"),\n", - " })\n", - " res = df_transform[[\"src_rt\", \"dst_rt\"]].apply(udf, axis=1, args=args)\n", - " \n", - " if verbose:\n", - " return res\n", - " \n", - " res_df = bpd.DataFrame({\"uri\": res})\n", - " res_df[\"authorizer\"] = FULL_CONNECTION_ID\n", - " return bbq.obj.make_ref(bbq.to_json(bbq.struct(res_df)))\n", - "\n", - "# Apply Blur Transformation\n", - "df_image[\"blurred\"] = apply_transformation(\n", - " df_image[\"image\"], f\"gs://{OUTPUT_BUCKET}/image_blur_transformed/\",\n", - " image_blur, 20, 20\n", - ")\n", - "render_images(df_image[[\"image\", \"blurred\"]])" - ] - }, - { - "cell_type": "markdown", - "id": "2d68a468", - "metadata": {}, - "source": [ - "# 4. Use LLM models to ask questions and generate embeddings on images" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "662054a0", - "metadata": { - "execution": { - "iopub.execute_input": "2025-08-18T20:36:13.954686Z", - "iopub.status.busy": "2025-08-18T20:36:13.954340Z", - "iopub.status.idle": "2025-08-18T20:36:43.225449Z", - "shell.execute_reply": "2025-08-18T20:36:43.224579Z", - "shell.execute_reply.started": "2025-08-18T20:36:13.954661Z" - }, - "trusted": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/core/logging/log_adapter.py:183: FutureWarning: Since upgrading the default model can cause unintended breakages, the\n", - "default model will be removed in BigFrames 3.0. Please supply an\n", - "explicit model to avoid this message.\n", - " return method(*args, **kwargs)\n", - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/session/__init__.py:437: FutureWarning: You are using the BigFrames session default connection: bigframes-\n", - "default-connection, which can be different from the\n", - "BigQuery project default connection. This default\n", - "connection may change in the future.\n", - " warnings.warn(msg, category=FutureWarning)\n" - ] - } - ], - "source": [ - "from bigframes.ml import llm\n", - "gemini = llm.GeminiTextGenerator()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "a31730ff", - "metadata": { - "execution": { - "iopub.execute_input": "2025-08-18T20:36:43.227798Z", - "iopub.status.busy": "2025-08-18T20:36:43.227457Z", - "iopub.status.idle": "2025-08-18T20:37:25.238649Z", - "shell.execute_reply": "2025-08-18T20:37:25.237623Z", - "shell.execute_reply.started": "2025-08-18T20:36:43.227764Z" - }, - "trusted": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    ml_generate_text_llm_resultimage
    0Please provide me with the picture! I need to see the image to tell you what the item is and what color the picture is.\\n
    1To answer your question accurately, I need you to provide me with the picture you are referring to. Once you provide the picture, I can analyze it and tell you what item is in the picture and what color the picture is.
    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Ask the same question on the images\n", - "df_image = df_image.head(2)\n", - "answer = gemini.predict(df_image, prompt=[\"what item is it?\", \"what color is the picture?\"])\n", - "render_images(answer[[\"ml_generate_text_llm_result\", \"image\"]])" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "f5d2a1ed", - "metadata": { - "execution": { - "iopub.execute_input": "2025-08-18T20:37:25.239875Z", - "iopub.status.busy": "2025-08-18T20:37:25.239607Z", - "iopub.status.idle": "2025-08-18T20:37:25.263034Z", - "shell.execute_reply": "2025-08-18T20:37:25.262002Z", - "shell.execute_reply.started": "2025-08-18T20:37:25.239847Z" - }, - "trusted": true - }, - "outputs": [], - "source": [ - "# Ask different questions\n", - "df_image[\"question\"] = [\"what item is it?\", \"what color is the picture?\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "fb67bf8e", - "metadata": { - "execution": { - "iopub.execute_input": "2025-08-18T20:37:25.264585Z", - "iopub.status.busy": "2025-08-18T20:37:25.264072Z", - "iopub.status.idle": "2025-08-18T20:38:10.129667Z", - "shell.execute_reply": "2025-08-18T20:38:10.128677Z", - "shell.execute_reply.started": "2025-08-18T20:37:25.264518Z" - }, - "trusted": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    ml_generate_text_llm_resultimage
    0The item is a glass aquarium.
    1Dark brown
    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "answer_alt = gemini.predict(df_image, prompt=[df_image[\"question\"], df_image[\"image\"]])\n", - "render_images(answer_alt[[\"ml_generate_text_llm_result\", \"image\"]])" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "0cf33170", - "metadata": { - "execution": { - "iopub.execute_input": "2025-08-18T20:38:10.130851Z", - "iopub.status.busy": "2025-08-18T20:38:10.130617Z", - "iopub.status.idle": "2025-08-18T20:39:04.790416Z", - "shell.execute_reply": "2025-08-18T20:39:04.789398Z", - "shell.execute_reply.started": "2025-08-18T20:38:10.130833Z" - }, - "trusted": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/core/logging/log_adapter.py:183: FutureWarning: Since upgrading the default model can cause unintended breakages, the\n", - "default model will be removed in BigFrames 3.0. Please supply an\n", - "explicit model to avoid this message.\n", - " return method(*args, **kwargs)\n", - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/session/__init__.py:437: FutureWarning: You are using the BigFrames session default connection: bigframes-\n", - "default-connection, which can be different from the\n", - "BigQuery project default connection. This default\n", - "connection may change in the future.\n", - " warnings.warn(msg, category=FutureWarning)\n", - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] - }, - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    ml_generate_embedding_resultml_generate_embedding_statusml_generate_embedding_start_secml_generate_embedding_end_seccontent
    0[ 0.03416207 0.0419732 -0.0227391 ... -0.03...<NA><NA>{\"access_urls\":{\"expiry_time\":\"2026-05-02T03:3...
    1[ 0.01908903 0.0193082 -0.00221754 ... 0.00...<NA><NA>{\"access_urls\":{\"expiry_time\":\"2026-05-02T03:3...
    \n", - "

    2 rows × 5 columns

    \n", - "
    [2 rows x 5 columns in total]" - ], - "text/plain": [ - " ml_generate_embedding_result \\\n", - "0 [ 0.03416207 0.0419732 -0.0227391 ... -0.03... \n", - "1 [ 0.01908903 0.0193082 -0.00221754 ... 0.00... \n", - "\n", - " ml_generate_embedding_status ml_generate_embedding_start_sec \\\n", - "0 \n", - "1 \n", - "\n", - " ml_generate_embedding_end_sec \\\n", - "0 \n", - "1 \n", - "\n", - " content \n", - "0 {\"access_urls\":{\"expiry_time\":\"2026-05-02T03:3... \n", - "1 {\"access_urls\":{\"expiry_time\":\"2026-05-02T03:3... \n", - "\n", - "[2 rows x 5 columns]" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Generate embeddings.\n", - "embed_model = llm.MultimodalEmbeddingGenerator()\n", - "embeddings = embed_model.predict(df_image[\"image\"])\n", - "embeddings" - ] - } - ], - "metadata": { - "kaggle": { - "accelerator": "none", - "dataSources": [ - { - "databundleVersionId": 13391012, - "sourceId": 110281, - "sourceType": "competition" - } - ], - "dockerImageVersionId": 31089, - "isGpuEnabled": false, - "isInternetEnabled": true, - "language": "python", - "sourceType": "notebook" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.13" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} +{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":110281,"databundleVersionId":13391012,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Describe product images with BigFrames multimodal DataFrames\n\nBased on notebook at https://github.com/googleapis/python-bigquery-dataframes/blob/main/notebooks/multimodal/multimodal_dataframe.ipynb\n\nThis notebook is introducing BigFrames Multimodal features:\n\n1. Create Multimodal DataFrame\n2. Combine unstructured data with structured data\n3. Conduct image transformations\n4. Use LLM models to ask questions and generate embeddings on images\n5. PDF chunking function\n\nInstall the bigframes package and upgrade other packages that are already included in Kaggle but have versions incompatible with bigframes.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"%pip install --upgrade bigframes google-cloud-automl google-cloud-translate google-ai-generativelanguage tensorflow ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Important:** restart the kernel by going to \"Run -> Restart & clear cell outputs\" before continuing.\n\nConfigure bigframes to use your GCP project. First, go to \"Add-ons -> Google Cloud SDK\" and click the \"Attach\" button. Then,","metadata":{}},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nuser_credential = user_secrets.get_gcloud_credential()\nuser_secrets.set_tensorflow_credential(user_credential)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:17:14.872905Z","iopub.execute_input":"2025-08-18T20:17:14.873201Z","iopub.status.idle":"2025-08-18T20:17:14.946971Z","shell.execute_reply.started":"2025-08-18T20:17:14.873171Z","shell.execute_reply":"2025-08-18T20:17:14.945996Z"}},"outputs":[],"execution_count":2},{"cell_type":"code","source":"PROJECT = \"bigframes-dev\" # replace with your project. \n# Refer to https://cloud.google.com/bigquery/docs/multimodal-data-dataframes-tutorial#required_roles for your required permissions\n\nOUTPUT_BUCKET = \"bigframes_blob_test\" # replace with your GCS bucket. \n# The connection (or bigframes-default-connection of the project) must have read/write permission to the bucket. \n# Refer to https://cloud.google.com/bigquery/docs/multimodal-data-dataframes-tutorial#grant-permissions for setting up connection service account permissions.\n# In this Notebook it uses bigframes-default-connection by default. You can also bring in your own connections in each method.\n\nimport bigframes\n# Setup project\nbigframes.options.bigquery.project = PROJECT\n\n# Display options\nbigframes.options.display.blob_display_width = 300\nbigframes.options.display.progress_bar = None\n\nimport bigframes.pandas as bpd","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:17:25.573874Z","iopub.execute_input":"2025-08-18T20:17:25.574192Z","iopub.status.idle":"2025-08-18T20:17:45.102002Z","shell.execute_reply.started":"2025-08-18T20:17:25.574168Z","shell.execute_reply":"2025-08-18T20:17:45.101140Z"}},"outputs":[],"execution_count":3},{"cell_type":"code","source":"# Create blob columns from wildcard path.\ndf_image = bpd.from_glob_path(\n \"gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/images/*\", name=\"image\"\n)\n# Other ways are: from string uri column\n# df = bpd.DataFrame({\"uri\": [\"gs:///\", \"gs:///\"]})\n# df[\"blob_col\"] = df[\"uri\"].str.to_blob()\n\n# From an existing object table\n# df = bpd.read_gbq_object_table(\"\", name=\"blob_col\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:17:45.103249Z","iopub.execute_input":"2025-08-18T20:17:45.103530Z","iopub.status.idle":"2025-08-18T20:17:47.424586Z","shell.execute_reply.started":"2025-08-18T20:17:45.103499Z","shell.execute_reply":"2025-08-18T20:17:47.423762Z"}},"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.11/dist-packages/bigframes/core/global_session.py:103: DefaultLocationWarning: No explicit location is set, so using location US for the session.\n _global_session = bigframes.session.connect(\n","output_type":"stream"},{"name":"stdout","text":"Please ensure you have selected a BigQuery account in the Notebook Add-ons menu.\n","output_type":"stream"}],"execution_count":4},{"cell_type":"code","source":"# Take only the 5 images to deal with. Preview the content of the Mutimodal DataFrame\ndf_image = df_image.head(5)\ndf_image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:17:47.425578Z","iopub.execute_input":"2025-08-18T20:17:47.425873Z","iopub.status.idle":"2025-08-18T20:18:07.919961Z","shell.execute_reply.started":"2025-08-18T20:17:47.425844Z","shell.execute_reply":"2025-08-18T20:18:07.918942Z"}},"outputs":[{"execution_count":5,"output_type":"execute_result","data":{"text/plain":" image\n0 {'uri': 'gs://cloud-samples-data/bigquery/tuto...\n1 {'uri': 'gs://cloud-samples-data/bigquery/tuto...\n2 {'uri': 'gs://cloud-samples-data/bigquery/tuto...\n3 {'uri': 'gs://cloud-samples-data/bigquery/tuto...\n4 {'uri': 'gs://cloud-samples-data/bigquery/tuto...\n\n[5 rows x 1 columns]","text/html":"
    \n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    image
    0
    1
    2
    3
    4
    \n

    5 rows × 1 columns

    \n
    [5 rows x 1 columns in total]"},"metadata":{}}],"execution_count":5},{"cell_type":"markdown","source":"# 2. Combine unstructured data with structured data\n\nNow you can put more information into the table to describe the files. Such as author info from inputs, or other metadata from the gcs object itself.","metadata":{}},{"cell_type":"code","source":"# Combine unstructured data with structured data\ndf_image[\"author\"] = [\"alice\", \"bob\", \"bob\", \"alice\", \"bob\"] # type: ignore\ndf_image[\"content_type\"] = df_image[\"image\"].blob.content_type()\ndf_image[\"size\"] = df_image[\"image\"].blob.size()\ndf_image[\"updated\"] = df_image[\"image\"].blob.updated()\ndf_image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:18:07.921884Z","iopub.execute_input":"2025-08-18T20:18:07.922593Z","iopub.status.idle":"2025-08-18T20:18:35.549725Z","shell.execute_reply.started":"2025-08-18T20:18:07.922551Z","shell.execute_reply":"2025-08-18T20:18:35.548942Z"}},"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.11/dist-packages/bigframes/bigquery/_operations/json.py:124: UserWarning: The `json_extract` is deprecated and will be removed in a future\nversion. Use `json_query` instead.\n warnings.warn(bfe.format_message(msg), category=UserWarning)\n/usr/local/lib/python3.11/dist-packages/bigframes/bigquery/_operations/json.py:124: UserWarning: The `json_extract` is deprecated and will be removed in a future\nversion. Use `json_query` instead.\n warnings.warn(bfe.format_message(msg), category=UserWarning)\n/usr/local/lib/python3.11/dist-packages/bigframes/bigquery/_operations/json.py:124: UserWarning: The `json_extract` is deprecated and will be removed in a future\nversion. Use `json_query` instead.\n warnings.warn(bfe.format_message(msg), category=UserWarning)\n","output_type":"stream"},{"execution_count":6,"output_type":"execute_result","data":{"text/plain":" image author content_type \\\n0 {'uri': 'gs://cloud-samples-data/bigquery/tuto... alice image/png \n1 {'uri': 'gs://cloud-samples-data/bigquery/tuto... bob image/png \n2 {'uri': 'gs://cloud-samples-data/bigquery/tuto... bob image/png \n3 {'uri': 'gs://cloud-samples-data/bigquery/tuto... alice image/png \n4 {'uri': 'gs://cloud-samples-data/bigquery/tuto... bob image/png \n\n size updated \n0 1591240 2025-03-20 17:45:04+00:00 \n1 1182951 2025-03-20 17:45:02+00:00 \n2 1520884 2025-03-20 17:44:55+00:00 \n3 1235401 2025-03-20 17:45:19+00:00 \n4 1591923 2025-03-20 17:44:47+00:00 \n\n[5 rows x 5 columns]","text/html":"
    \n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    imageauthorcontent_typesizeupdated
    0aliceimage/png15912402025-03-20 17:45:04+00:00
    1bobimage/png11829512025-03-20 17:45:02+00:00
    2bobimage/png15208842025-03-20 17:44:55+00:00
    3aliceimage/png12354012025-03-20 17:45:19+00:00
    4bobimage/png15919232025-03-20 17:44:47+00:00
    \n

    5 rows × 5 columns

    \n
    [5 rows x 5 columns in total]"},"metadata":{}}],"execution_count":6},{"cell_type":"markdown","source":"Then you can filter the rows based on the structured data. And for different content types, you can display them respectively or together.","metadata":{}},{"cell_type":"code","source":"# filter images and display, you can also display audio and video types\ndf_image[df_image[\"author\"] == \"alice\"][\"image\"].blob.display()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:18:55.299993Z","iopub.execute_input":"2025-08-18T20:18:55.300314Z","iopub.status.idle":"2025-08-18T20:19:09.154492Z","shell.execute_reply.started":"2025-08-18T20:18:55.300289Z","shell.execute_reply":"2025-08-18T20:19:09.153315Z"}},"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.11/dist-packages/bigframes/bigquery/_operations/json.py:124: UserWarning: The `json_extract` is deprecated and will be removed in a future\nversion. Use `json_query` instead.\n warnings.warn(bfe.format_message(msg), category=UserWarning)\n","output_type":"stream"},{"output_type":"display_data","data":{"text/html":"","text/plain":""},"metadata":{}},{"output_type":"display_data","data":{"text/html":"","text/plain":""},"metadata":{}}],"execution_count":7},{"cell_type":"markdown","source":"# 3. Conduct image transformations\n\nBigFrames Multimodal DataFrame provides image(and other) transformation functions. Such as image_blur, image_resize and image_normalize. The output can be saved to GCS folders or to BQ as bytes.","metadata":{}},{"cell_type":"code","source":"df_image[\"blurred\"] = df_image[\"image\"].blob.image_blur(\n (20, 20), dst=f\"gs://{OUTPUT_BUCKET}/image_blur_transformed/\", engine=\"opencv\"\n)\ndf_image[\"resized\"] = df_image[\"image\"].blob.image_resize(\n (300, 200), dst=f\"gs://{OUTPUT_BUCKET}/image_resize_transformed/\", engine=\"opencv\"\n)\ndf_image[\"normalized\"] = df_image[\"image\"].blob.image_normalize(\n alpha=50.0,\n beta=150.0,\n norm_type=\"minmax\",\n dst=f\"gs://{OUTPUT_BUCKET}/image_normalize_transformed/\",\n engine=\"opencv\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:19:22.950277Z","iopub.execute_input":"2025-08-18T20:19:22.950652Z","iopub.status.idle":"2025-08-18T20:31:51.799997Z","shell.execute_reply.started":"2025-08-18T20:19:22.950625Z","shell.execute_reply":"2025-08-18T20:31:51.798840Z"}},"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.11/dist-packages/bigframes/core/log_adapter.py:175: FunctionAxisOnePreviewWarning: Blob Functions use bigframes DataFrame Managed function with axis=1 senario, which is a preview feature.\n return method(*args, **kwargs)\n/usr/local/lib/python3.11/dist-packages/bigframes/core/log_adapter.py:175: FunctionAxisOnePreviewWarning: Blob Functions use bigframes DataFrame Managed function with axis=1 senario, which is a preview feature.\n return method(*args, **kwargs)\n/usr/local/lib/python3.11/dist-packages/bigframes/core/log_adapter.py:175: FunctionAxisOnePreviewWarning: Blob Functions use bigframes DataFrame Managed function with axis=1 senario, which is a preview feature.\n return method(*args, **kwargs)\n","output_type":"stream"}],"execution_count":8},{"cell_type":"code","source":"# You can also chain functions together\ndf_image[\"blur_resized\"] = df_image[\"blurred\"].blob.image_resize((300, 200), dst=f\"gs://{OUTPUT_BUCKET}/image_blur_resize_transformed/\", engine=\"opencv\")\ndf_image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:31:51.802219Z","iopub.execute_input":"2025-08-18T20:31:51.802745Z","iopub.status.idle":"2025-08-18T20:36:13.953258Z","shell.execute_reply.started":"2025-08-18T20:31:51.802700Z","shell.execute_reply":"2025-08-18T20:36:13.951930Z"}},"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.11/dist-packages/bigframes/core/log_adapter.py:175: FunctionAxisOnePreviewWarning: Blob Functions use bigframes DataFrame Managed function with axis=1 senario, which is a preview feature.\n return method(*args, **kwargs)\n","output_type":"stream"},{"execution_count":9,"output_type":"execute_result","data":{"text/plain":" image author content_type \\\n0 {'uri': 'gs://cloud-samples-data/bigquery/tuto... alice image/png \n1 {'uri': 'gs://cloud-samples-data/bigquery/tuto... bob image/png \n2 {'uri': 'gs://cloud-samples-data/bigquery/tuto... bob image/png \n3 {'uri': 'gs://cloud-samples-data/bigquery/tuto... alice image/png \n4 {'uri': 'gs://cloud-samples-data/bigquery/tuto... bob image/png \n\n size updated \\\n0 1591240 2025-03-20 17:45:04+00:00 \n1 1182951 2025-03-20 17:45:02+00:00 \n2 1520884 2025-03-20 17:44:55+00:00 \n3 1235401 2025-03-20 17:45:19+00:00 \n4 1591923 2025-03-20 17:44:47+00:00 \n\n blurred \\\n0 {'uri': 'gs://bigframes_blob_test/image_blur_t... \n1 {'uri': 'gs://bigframes_blob_test/image_blur_t... \n2 {'uri': 'gs://bigframes_blob_test/image_blur_t... \n3 {'uri': 'gs://bigframes_blob_test/image_blur_t... \n4 {'uri': 'gs://bigframes_blob_test/image_blur_t... \n\n resized \\\n0 {'uri': 'gs://bigframes_blob_test/image_resize... \n1 {'uri': 'gs://bigframes_blob_test/image_resize... \n2 {'uri': 'gs://bigframes_blob_test/image_resize... \n3 {'uri': 'gs://bigframes_blob_test/image_resize... \n4 {'uri': 'gs://bigframes_blob_test/image_resize... \n\n normalized \\\n0 {'uri': 'gs://bigframes_blob_test/image_normal... \n1 {'uri': 'gs://bigframes_blob_test/image_normal... \n2 {'uri': 'gs://bigframes_blob_test/image_normal... \n3 {'uri': 'gs://bigframes_blob_test/image_normal... \n4 {'uri': 'gs://bigframes_blob_test/image_normal... \n\n blur_resized \n0 {'uri': 'gs://bigframes_blob_test/image_blur_r... \n1 {'uri': 'gs://bigframes_blob_test/image_blur_r... \n2 {'uri': 'gs://bigframes_blob_test/image_blur_r... \n3 {'uri': 'gs://bigframes_blob_test/image_blur_r... \n4 {'uri': 'gs://bigframes_blob_test/image_blur_r... \n\n[5 rows x 9 columns]","text/html":"
    \n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    imageauthorcontent_typesizeupdatedblurredresizednormalizedblur_resized
    0aliceimage/png15912402025-03-20 17:45:04+00:00
    1bobimage/png11829512025-03-20 17:45:02+00:00
    2bobimage/png15208842025-03-20 17:44:55+00:00
    3aliceimage/png12354012025-03-20 17:45:19+00:00
    4bobimage/png15919232025-03-20 17:44:47+00:00
    \n

    5 rows × 9 columns

    \n
    [5 rows x 9 columns in total]"},"metadata":{}}],"execution_count":9},{"cell_type":"markdown","source":"# 4. Use LLM models to ask questions and generate embeddings on images","metadata":{}},{"cell_type":"code","source":"from bigframes.ml import llm\ngemini = llm.GeminiTextGenerator()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:36:13.954340Z","iopub.execute_input":"2025-08-18T20:36:13.954686Z","iopub.status.idle":"2025-08-18T20:36:43.225449Z","shell.execute_reply.started":"2025-08-18T20:36:13.954661Z","shell.execute_reply":"2025-08-18T20:36:43.224579Z"}},"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.11/dist-packages/bigframes/core/log_adapter.py:175: FutureWarning: Since upgrading the default model can cause unintended breakages, the\ndefault model will be removed in BigFrames 3.0. Please supply an\nexplicit model to avoid this message.\n return method(*args, **kwargs)\n","output_type":"stream"}],"execution_count":10},{"cell_type":"code","source":"# Ask the same question on the images\ndf_image = df_image.head(2)\nanswer = gemini.predict(df_image, prompt=[\"what item is it?\", df_image[\"image\"]])\nanswer[[\"ml_generate_text_llm_result\", \"image\"]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:36:43.227457Z","iopub.execute_input":"2025-08-18T20:36:43.227798Z","iopub.status.idle":"2025-08-18T20:37:25.238649Z","shell.execute_reply.started":"2025-08-18T20:36:43.227764Z","shell.execute_reply":"2025-08-18T20:37:25.237623Z"}},"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.11/dist-packages/bigframes/core/array_value.py:108: PreviewWarning: JSON column interpretation as a custom PyArrow extention in\n`db_dtypes` is a preview feature and subject to change.\n warnings.warn(msg, bfe.PreviewWarning)\n","output_type":"stream"},{"execution_count":11,"output_type":"execute_result","data":{"text/plain":" ml_generate_text_llm_result \\\n0 The item is a tin of K9 Guard Dog Paw Balm. \n1 The item is a bottle of K9 Guard Dog Hot Spot ... \n\n image \n0 {'uri': 'gs://cloud-samples-data/bigquery/tuto... \n1 {'uri': 'gs://cloud-samples-data/bigquery/tuto... \n\n[2 rows x 2 columns]","text/html":"
    \n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ml_generate_text_llm_resultimage
    0The item is a tin of K9 Guard Dog Paw Balm.
    1The item is a bottle of K9 Guard Dog Hot Spot Spray.
    \n

    2 rows × 2 columns

    \n
    [2 rows x 2 columns in total]"},"metadata":{}}],"execution_count":11},{"cell_type":"code","source":"# Ask different questions\ndf_image[\"question\"] = [\"what item is it?\", \"what color is the picture?\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:37:25.239607Z","iopub.execute_input":"2025-08-18T20:37:25.239875Z","iopub.status.idle":"2025-08-18T20:37:25.263034Z","shell.execute_reply.started":"2025-08-18T20:37:25.239847Z","shell.execute_reply":"2025-08-18T20:37:25.262002Z"}},"outputs":[],"execution_count":12},{"cell_type":"code","source":"answer_alt = gemini.predict(df_image, prompt=[df_image[\"question\"], df_image[\"image\"]])\nanswer_alt[[\"ml_generate_text_llm_result\", \"image\"]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:37:25.264072Z","iopub.execute_input":"2025-08-18T20:37:25.264585Z","iopub.status.idle":"2025-08-18T20:38:10.129667Z","shell.execute_reply.started":"2025-08-18T20:37:25.264518Z","shell.execute_reply":"2025-08-18T20:38:10.128677Z"}},"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.11/dist-packages/bigframes/core/array_value.py:108: PreviewWarning: JSON column interpretation as a custom PyArrow extention in\n`db_dtypes` is a preview feature and subject to change.\n warnings.warn(msg, bfe.PreviewWarning)\n","output_type":"stream"},{"execution_count":13,"output_type":"execute_result","data":{"text/plain":" ml_generate_text_llm_result \\\n0 The item is a tin of K9 Guard Dog Paw Balm. \n1 The picture has colors such as white, gray, an... \n\n image \n0 {'uri': 'gs://cloud-samples-data/bigquery/tuto... \n1 {'uri': 'gs://cloud-samples-data/bigquery/tuto... \n\n[2 rows x 2 columns]","text/html":"
    \n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ml_generate_text_llm_resultimage
    0The item is a tin of K9 Guard Dog Paw Balm.
    1The picture has colors such as white, gray, and a light blue (cyan).
    \n

    2 rows × 2 columns

    \n
    [2 rows x 2 columns in total]"},"metadata":{}}],"execution_count":13},{"cell_type":"code","source":"# Generate embeddings.\nembed_model = llm.MultimodalEmbeddingGenerator()\nembeddings = embed_model.predict(df_image[\"image\"])\nembeddings","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T20:38:10.130617Z","iopub.execute_input":"2025-08-18T20:38:10.130851Z","iopub.status.idle":"2025-08-18T20:39:04.790416Z","shell.execute_reply.started":"2025-08-18T20:38:10.130833Z","shell.execute_reply":"2025-08-18T20:39:04.789398Z"}},"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.11/dist-packages/bigframes/core/log_adapter.py:175: FutureWarning: Since upgrading the default model can cause unintended breakages, the\ndefault model will be removed in BigFrames 3.0. Please supply an\nexplicit model to avoid this message.\n return method(*args, **kwargs)\n/usr/local/lib/python3.11/dist-packages/bigframes/core/array_value.py:108: PreviewWarning: JSON column interpretation as a custom PyArrow extention in\n`db_dtypes` is a preview feature and subject to change.\n warnings.warn(msg, bfe.PreviewWarning)\n","output_type":"stream"},{"execution_count":14,"output_type":"execute_result","data":{"text/plain":" ml_generate_embedding_result \\\n0 [ 0.00638822 0.01666385 0.00451817 ... -0.02... \n1 [ 0.00973672 0.02148364 0.00244308 ... 0.00... \n\n ml_generate_embedding_status ml_generate_embedding_start_sec \\\n0 \n1 \n\n ml_generate_embedding_end_sec \\\n0 \n1 \n\n content \n0 {\"access_urls\":{\"expiry_time\":\"2025-08-19T02:3... \n1 {\"access_urls\":{\"expiry_time\":\"2025-08-19T02:3... \n\n[2 rows x 5 columns]","text/html":"
    \n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ml_generate_embedding_resultml_generate_embedding_statusml_generate_embedding_start_secml_generate_embedding_end_seccontent
    0[ 0.00638822 0.01666385 0.00451817 ... -0.02...<NA><NA>{\"access_urls\":{\"expiry_time\":\"2025-08-19T02:3...
    1[ 0.00973672 0.02148364 0.00244308 ... 0.00...<NA><NA>{\"access_urls\":{\"expiry_time\":\"2025-08-19T02:3...
    \n

    2 rows × 5 columns

    \n
    [2 rows x 5 columns in total]"},"metadata":{}}],"execution_count":14},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} diff --git a/notebooks/kaggle/vector-search-with-bigframes-over-national-jukebox.ipynb b/notebooks/kaggle/vector-search-with-bigframes-over-national-jukebox.ipynb index 317ba0f1adb..fe2d567d1b3 100644 --- a/notebooks/kaggle/vector-search-with-bigframes-over-national-jukebox.ipynb +++ b/notebooks/kaggle/vector-search-with-bigframes-over-national-jukebox.ipynb @@ -1,8 +1,23 @@ { "cells": [ { - "id": "f4ece66a", "cell_type": "markdown", + "metadata": { + "@deathbeds/jupyterlab-fonts": { + "styles": { + "": { + "body[data-jp-deck-mode='presenting'] &": { + "zoom": "194%" + } + } + } + }, + "editable": true, + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, "source": [ "# Creating a searchable index of the National Jukebox\n", "\n", @@ -20,42 +35,42 @@ "To follow along, you'll need a Google Cloud project\n", "\n", "* Go to https://cloud.google.com/free to start a free trial." - ], + ] + }, + { + "cell_type": "markdown", "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { "": { "body[data-jp-deck-mode='presenting'] &": { - "zoom": "194%" + "z-index": "0", + "zoom": "216%" } } } }, - "editable": true, "slideshow": { - "slide_type": "subslide" - }, - "tags": [] + "slide_type": "slide" + } }, - "execution_count": null - }, - { - "id": "bc01a1d3", - "cell_type": "markdown", "source": [ "The National Jukebox is a project of the USA Library of Congress to provide access to thousands of acoustic sound recordings from the very earliest days of the commercial record industry.\n", "\n", "* Learn more at https://www.loc.gov/collections/national-jukebox/about-this-collection/\n", "\n", "\"recording" - ], + ] + }, + { + "cell_type": "markdown", "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { "": { "body[data-jp-deck-mode='presenting'] &": { "z-index": "0", - "zoom": "216%" + "zoom": "181%" } } } @@ -64,11 +79,6 @@ "slide_type": "slide" } }, - "execution_count": null - }, - { - "id": "4fc7c468", - "cell_type": "markdown", "source": [ "\n", "To search the National Jukebox, we combine powerful features of BigQuery:\n", @@ -86,32 +96,10 @@ "3. BigQuery DataFrames to use Python instead of SQL.\n", "\n", " https://cloud.google.com/bigquery/docs/bigquery-dataframes-introduction" - ], - "metadata": { - "@deathbeds/jupyterlab-fonts": { - "styles": { - "": { - "body[data-jp-deck-mode='presenting'] &": { - "z-index": "0", - "zoom": "181%" - } - } - } - }, - "slideshow": { - "slide_type": "slide" - } - }, - "execution_count": null + ] }, { - "id": "90f2e543", "cell_type": "markdown", - "source": [ - "## Getting started with BigQuery DataFrames (bigframes)\n", - "\n", - "Install the bigframes package." - ], "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -126,14 +114,15 @@ "slide_type": "slide" } }, - "execution_count": null + "source": [ + "## Getting started with BigQuery DataFrames (bigframes)\n", + "\n", + "Install the bigframes package." + ] }, { - "id": "56694cb4", "cell_type": "code", - "source": [ - "%pip install --upgrade bigframes google-cloud-automl google-cloud-translate google-ai-generativelanguage tensorflow " - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -153,17 +142,13 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "%pip install --upgrade bigframes google-cloud-automl google-cloud-translate google-ai-generativelanguage tensorflow " + ] }, { - "id": "fa84ad03", "cell_type": "markdown", - "source": [ - "**Important:** restart the kernel by going to \"Run -> Restart & clear cell outputs\" before continuing.\n", - "\n", - "Configure bigframes to use your GCP project. First, go to \"Add-ons -> Google Cloud SDK\" and click the \"Attach\" button. Then," - ], "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -176,17 +161,15 @@ } } }, - "execution_count": null + "source": [ + "**Important:** restart the kernel by going to \"Run -> Restart & clear cell outputs\" before continuing.\n", + "\n", + "Configure bigframes to use your GCP project. First, go to \"Add-ons -> Google Cloud SDK\" and click the \"Attach\" button. Then," + ] }, { - "id": "1fbd4f9e", "cell_type": "code", - "source": [ - "from kaggle_secrets import UserSecretsClient\n", - "user_secrets = UserSecretsClient()\n", - "user_credential = user_secrets.get_gcloud_credential()\n", - "user_secrets.set_tensorflow_credential(user_credential)" - ], + "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-08-14T15:53:08.494636Z", @@ -197,22 +180,17 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "from kaggle_secrets import UserSecretsClient\n", + "user_secrets = UserSecretsClient()\n", + "user_credential = user_secrets.get_gcloud_credential()\n", + "user_secrets.set_tensorflow_credential(user_credential)" + ] }, { - "id": "0b0b1cd8", "cell_type": "code", - "source": [ - "import bigframes._config\n", - "import bigframes.pandas as bpd\n", - "\n", - "PROJECT_ID = \"your-project-id\" # @param {type:\"string\"}\n", - "bpd.options.bigquery.location = \"US\"\n", - "\n", - "# Set to your GCP project ID.\n", - "bpd.options.bigquery.project = PROJECT_ID" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -232,17 +210,19 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "import bigframes._config\n", + "import bigframes.pandas as bpd\n", + "\n", + "bpd.options.bigquery.location = \"US\"\n", + "\n", + "# Set to your GCP project ID.\n", + "bpd.options.bigquery.project = \"swast-scratch\"" + ] }, { - "id": "32e58a7f", "cell_type": "markdown", - "source": [ - "## Reading data\n", - "\n", - "BigQuery DataFrames can read data from BigQuery, GCS, or even local sources. With `engine=\"bigquery\"`, BigQuery's distributed processing reads the file without it ever having to reach your local Python environment." - ], "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -257,19 +237,15 @@ "slide_type": "slide" } }, - "execution_count": null + "source": [ + "## Reading data\n", + "\n", + "BigQuery DataFrames can read data from BigQuery, GCS, or even local sources. With `engine=\"bigquery\"`, BigQuery's distributed processing reads the file without it ever having to reach your local Python environment." + ] }, { - "id": "e52aa9e8", "cell_type": "code", - "source": [ - "df = bpd.read_json(\n", - " \"gs://cloud-samples-data/third-party/usa-loc-national-jukebox/jukebox.jsonl\",\n", - " engine=\"bigquery\",\n", - " orient=\"records\",\n", - " lines=True,\n", - ")" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -289,16 +265,19 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "df = bpd.read_json(\n", + " \"gs://cloud-samples-data/third-party/usa-loc-national-jukebox/jukebox.jsonl\",\n", + " engine=\"bigquery\",\n", + " orient=\"records\",\n", + " lines=True,\n", + ")" + ] }, { - "id": "0c1fca97", "cell_type": "code", - "source": [ - "# Use `peek()` instead of `head()` to see arbitrary rows rather than the \"first\" rows.\n", - "df.peek()" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -321,15 +300,15 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "# Use `peek()` instead of `head()` to see arbitrary rows rather than the \"first\" rows.\n", + "df.peek()" + ] }, { - "id": "4a13e789", "cell_type": "code", - "source": [ - "df.shape" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -349,18 +328,14 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "df.shape" + ] }, { - "id": "26b8baba", "cell_type": "code", - "source": [ - "# For the purposes of a demo, select only a subset of rows.\n", - "df = df.sample(n=250)\n", - "df.cache()\n", - "df.shape" - ], + "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-08-14T15:55:55.448664Z", @@ -371,32 +346,17 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "# For the purposes of a demo, select only a subset of rows.\n", + "df = df.sample(n=250)\n", + "df.cache()\n", + "df.shape" + ] }, { - "id": "af84cb21", "cell_type": "code", - "source": [ - "# As a side effect of how I extracted the song information from the HTML DOM,\n", - "# we ended up with lists in places where we only expect one item.\n", - "#\n", - "# We can \"explode\" to flatten these lists.\n", - "flattened = df.explode([\n", - " \"Recording Repository\",\n", - " \"Recording Label\",\n", - " \"Recording Take Number\",\n", - " \"Recording Date\",\n", - " \"Recording Matrix Number\",\n", - " \"Recording Catalog Number\",\n", - " \"Media Size\",\n", - " \"Recording Location\",\n", - " \"Summary\",\n", - " \"Rights Advisory\",\n", - " \"Title\",\n", - "])\n", - "flattened.peek()" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -419,15 +379,31 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "# As a side effect of how I extracted the song information from the HTML DOM,\n", + "# we ended up with lists in places where we only expect one item.\n", + "#\n", + "# We can \"explode\" to flatten these lists.\n", + "flattened = df.explode([\n", + " \"Recording Repository\",\n", + " \"Recording Label\",\n", + " \"Recording Take Number\",\n", + " \"Recording Date\",\n", + " \"Recording Matrix Number\",\n", + " \"Recording Catalog Number\",\n", + " \"Media Size\",\n", + " \"Recording Location\",\n", + " \"Summary\",\n", + " \"Rights Advisory\",\n", + " \"Title\",\n", + "])\n", + "flattened.peek()" + ] }, { - "id": "085deffd", "cell_type": "code", - "source": [ - "flattened.shape" - ], + "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-08-14T15:56:06.546531Z", @@ -438,15 +414,13 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "flattened.shape" + ] }, { - "id": "f8e653ee", "cell_type": "markdown", - "source": [ - "To access unstructured data from BigQuery, create a URI pointing to a file in Google Cloud Storage (GCS). Then, construct a \"blob\" (also known as an \"Object Ref\" in BigQuery terms) so that BigQuery can read from GCS." - ], "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -463,14 +437,13 @@ }, "tags": [] }, - "execution_count": null + "source": [ + "To access unstructured data from BigQuery, create a URI pointing to a file in Google Cloud Storage (GCS). Then, construct a \"blob\" (also known as an \"Object Ref\" in BigQuery terms) so that BigQuery can read from GCS." + ] }, { - "id": "dbd1a844", "cell_type": "code", - "source": [ - "flattened = flattened.assign(**{\\n \"GCS Prefix\": \"gs://cloud-samples-data/third-party/usa-loc-national-jukebox/\",\\n \"GCS Stub\": flattened['URL'].str.extract(r'/(jukebox-[0-9]+)/'),\\n})\\nflattened[\"GCS URI\"] = flattened[\"GCS Prefix\"] + flattened[\"GCS Stub\"] + \".mp3\"" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -495,15 +468,18 @@ "tags": [], "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "flattened = flattened.assign(**{\n", + " \"GCS Prefix\": \"gs://cloud-samples-data/third-party/usa-loc-national-jukebox/\",\n", + " \"GCS Stub\": flattened['URL'].str.extract(r'/(jukebox-[0-9]+)/'),\n", + "})\n", + "flattened[\"GCS URI\"] = flattened[\"GCS Prefix\"] + flattened[\"GCS Stub\"] + \".mp3\"\n", + "flattened[\"GCS Blob\"] = flattened[\"GCS URI\"].str.to_blob()" + ] }, { - "id": "fae13ec5", "cell_type": "markdown", - "source": [ - "BigQuery (and BigQuery DataFrames) provide access to powerful models and multimodal capabilities. Here, we transcribe audio to text." - ], "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -520,35 +496,13 @@ }, "tags": [] }, - "execution_count": null + "source": [ + "BigQuery (and BigQuery DataFrames) provide access to powerful models and multimodal capabilities. Here, we transcribe audio to text." + ] }, { - "id": "f08f92b1", "cell_type": "code", - "source": [ - "import bigframes.bigquery as bbq\n", - "\n", - "# Replace with your own connection name.\n", - "CONNECTION_ID = 'your-project-id.your-location.your-connection' # @param {type:\"string\"}\n", - "\n", - "# Convert the audio URI to the runtime representation required by the model.\n", - "audio_ref = bbq.obj.make_ref(flattened[\"GCS URI\"], authorizer=CONNECTION_ID)\n", - "audio_metadata = bbq.obj.fetch_metadata(audio_ref)\n", - "audio_runtime = bbq.obj.get_access_url(audio_metadata, mode=\"R\")\n", - "\n", - "# Call GenAI model to perform audio transcription\n", - "raw_results = bbq.ai.generate(\n", - " prompt=(\"Transcribe the provided audio.\", audio_runtime),\n", - " endpoint=\"gemini-2.5-flash\"\n", - ")\n", - "\n", - "# Package result struct to contain 'content' and 'status' expected by downstream cells\n", - "transcription_df = bpd.DataFrame({\n", - " \"content\": raw_results.struct.field(\"result\"),\n", - " \"status\": raw_results.struct.field(\"status\")\n", - "})\n", - "flattened[\"Transcription\"] = bbq.struct(transcription_df)" - ], + "execution_count": null, "metadata": { "editable": true, "execution": { @@ -564,15 +518,17 @@ "tags": [], "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "flattened[\"Transcription\"] = flattened[\"GCS Blob\"].blob.audio_transcribe(\n", + " model_name=\"gemini-2.0-flash-001\",\n", + " verbose=True,\n", + ")\n", + "flattened[\"Transcription\"]" + ] }, { - "id": "30969ae1", "cell_type": "markdown", - "source": [ - "Sometimes the model has transient errors. Check the status column to see if there are errors." - ], "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -587,16 +543,13 @@ "slide_type": "slide" } }, - "execution_count": null + "source": [ + "Sometimes the model has transient errors. Check the status column to see if there are errors." + ] }, { - "id": "7d0dbc38", "cell_type": "code", - "source": [ - "print(f\"Successful rows: {(flattened['Transcription'].struct.field('status') == '').sum()}\")\n", - "print(f\"Failed rows: {(flattened['Transcription'].struct.field('status') != '').sum()}\")\n", - "flattened.shape" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -621,16 +574,16 @@ "tags": [], "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "print(f\"Successful rows: {(flattened['Transcription'].struct.field('status') == '').sum()}\")\n", + "print(f\"Failed rows: {(flattened['Transcription'].struct.field('status') != '').sum()}\")\n", + "flattened.shape" + ] }, { - "id": "6cddf53b", "cell_type": "code", - "source": [ - "# Show transcribed lyrics.\n", - "flattened[\"Transcription\"].struct.field(\"content\")" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -650,19 +603,15 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "# Show transcribed lyrics.\n", + "flattened[\"Transcription\"].struct.field(\"content\")" + ] }, { - "id": "ba0386cc", "cell_type": "code", - "source": [ - "# Find all instrumentatal songs\n", - "instrumental = flattened[flattened[\"Transcription\"].struct.field(\"content\") == \"\"]\n", - "print(instrumental.shape)\n", - "song = instrumental.peek(1)\n", - "song" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -685,22 +634,18 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "# Find all instrumentatal songs\n", + "instrumental = flattened[flattened[\"Transcription\"].struct.field(\"content\") == \"\"]\n", + "print(instrumental.shape)\n", + "song = instrumental.peek(1)\n", + "song" + ] }, { - "id": "61a883b2", "cell_type": "code", - "source": [ - "import gcsfs\n", - "import IPython.display\n", - "\n", - "fs = gcsfs.GCSFileSystem(project='bigframes-dev')\n", - "with fs.open(song[\"GCS URI\"].iloc[0]) as song_file:\n", - " song_bytes = song_file.read()\n", - "\n", - "IPython.display.Audio(song_bytes)" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -725,19 +670,20 @@ "tags": [], "trusted": true }, - "execution_count": null, - "outputs": [] - }, - { - "id": "e8a25c46", - "cell_type": "markdown", + "outputs": [], "source": [ - "## Creating a searchable index\n", + "import gcsfs\n", + "import IPython.display\n", "\n", - "To be able to search by semantics rather than just text, generate embeddings and then create an index to efficiently search these.\n", + "fs = gcsfs.GCSFileSystem(project='bigframes-dev')\n", + "with fs.open(song[\"GCS URI\"].iloc[0]) as song_file:\n", + " song_bytes = song_file.read()\n", "\n", - "See also, this example: https://github.com/googleapis/python-bigquery-dataframes/blob/main/notebooks/generative_ai/bq_dataframes_llm_vector_search.ipynb" - ], + "IPython.display.Audio(song_bytes)" + ] + }, + { + "cell_type": "markdown", "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -752,16 +698,17 @@ "slide_type": "slide" } }, - "execution_count": null + "source": [ + "## Creating a searchable index\n", + "\n", + "To be able to search by semantics rather than just text, generate embeddings and then create an index to efficiently search these.\n", + "\n", + "See also, this example: https://github.com/googleapis/python-bigquery-dataframes/blob/main/notebooks/generative_ai/bq_dataframes_llm_vector_search.ipynb" + ] }, { - "id": "ead0fa8c", "cell_type": "code", - "source": [ - "from bigframes.ml.llm import TextEmbeddingGenerator\n", - "\n", - "text_model = TextEmbeddingGenerator(model_name=\"text-multilingual-embedding-002\")" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -781,21 +728,16 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "from bigframes.ml.llm import TextEmbeddingGenerator\n", + "\n", + "text_model = TextEmbeddingGenerator(model_name=\"text-multilingual-embedding-002\")" + ] }, { - "id": "5ed7776d", "cell_type": "code", - "source": [ - "df_to_index = (\n", - " flattened\n", - " .assign(content=flattened[\"Transcription\"].struct.field(\"content\"))\n", - " [flattened[\"Transcription\"].struct.field(\"content\") != \"\"]\n", - ")\n", - "embedding = text_model.predict(df_to_index)\n", - "embedding.peek(1)" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -815,18 +757,20 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "df_to_index = (\n", + " flattened\n", + " .assign(content=flattened[\"Transcription\"].struct.field(\"content\"))\n", + " [flattened[\"Transcription\"].struct.field(\"content\") != \"\"]\n", + ")\n", + "embedding = text_model.predict(df_to_index)\n", + "embedding.peek(1)" + ] }, { - "id": "c96e9832", "cell_type": "code", - "source": [ - "# Check the status column to look for errors.\n", - "print(f\"Successful rows: {(embedding['ml_generate_embedding_status'] == '').sum()}\")\n", - "print(f\"Failed rows: {(embedding['ml_generate_embedding_status'] != '').sum()}\")\n", - "embedding.shape" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -851,15 +795,16 @@ "tags": [], "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "# Check the status column to look for errors.\n", + "print(f\"Successful rows: {(embedding['ml_generate_embedding_status'] == '').sum()}\")\n", + "print(f\"Failed rows: {(embedding['ml_generate_embedding_status'] != '').sum()}\")\n", + "embedding.shape" + ] }, { - "id": "0e2a5d7b", "cell_type": "markdown", - "source": [ - "We're now ready to save this to a table." - ], "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -871,15 +816,13 @@ } } }, - "execution_count": null + "source": [ + "We're now ready to save this to a table." + ] }, { - "id": "51819a0c", "cell_type": "code", - "source": [ - "embedding_table_id = f\"{bpd.options.bigquery.project}.kaggle.national_jukebox\"\n", - "embedding.to_gbq(embedding_table_id, if_exists=\"replace\")" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -899,20 +842,14 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "embedding_table_id = f\"{bpd.options.bigquery.project}.kaggle.national_jukebox\"\n", + "embedding.to_gbq(embedding_table_id, if_exists=\"replace\")" + ] }, { - "id": "5e16fb14", "cell_type": "markdown", - "source": [ - "## Searching the database\n", - "\n", - "To search by semantics, we:\n", - "\n", - "1. Turn our search string into an embedding using the same model as our index.\n", - "2. Find the closest matches to the search string." - ], "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -927,17 +864,18 @@ "slide_type": "slide" } }, - "execution_count": null + "source": [ + "## Searching the database\n", + "\n", + "To search by semantics, we:\n", + "\n", + "1. Turn our search string into an embedding using the same model as our index.\n", + "2. Find the closest matches to the search string." + ] }, { - "id": "1bad3317", "cell_type": "code", - "source": [ - "import bigframes.pandas as bpd\n", - "\n", - "df_written = bpd.read_gbq(embedding_table_id)\n", - "df_written.peek(1)" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -960,22 +898,17 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "import bigframes.pandas as bpd\n", + "\n", + "df_written = bpd.read_gbq(embedding_table_id)\n", + "df_written.peek(1)" + ] }, { - "id": "8aaaef1f", "cell_type": "code", - "source": [ - "from bigframes.ml.llm import TextEmbeddingGenerator\n", - "\n", - "search_string = \"walking home\"\n", - "\n", - "text_model = TextEmbeddingGenerator(model_name=\"text-multilingual-embedding-002\")\n", - "search_df = bpd.DataFrame([search_string], columns=['search_string'])\n", - "search_embedding = text_model.predict(search_df)\n", - "search_embedding" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -995,24 +928,21 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "from bigframes.ml.llm import TextEmbeddingGenerator\n", + "\n", + "search_string = \"walking home\"\n", + "\n", + "text_model = TextEmbeddingGenerator(model_name=\"text-multilingual-embedding-002\")\n", + "search_df = bpd.DataFrame([search_string], columns=['search_string'])\n", + "search_embedding = text_model.predict(search_df)\n", + "search_embedding" + ] }, { - "id": "908a2340", "cell_type": "code", - "source": [ - "import bigframes.bigquery as bbq\n", - "\n", - "vector_search_results = bbq.vector_search(\n", - " base_table=embedding_table_id,\n", - " column_to_search=\"ml_generate_embedding_result\",\n", - " query=search_embedding,\n", - " distance_type=\"COSINE\",\n", - " query_column_to_search=\"ml_generate_embedding_result\",\n", - " top_k=5,\n", - ")" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -1037,15 +967,23 @@ "tags": [], "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "import bigframes.bigquery as bbq\n", + "\n", + "vector_search_results = bbq.vector_search(\n", + " base_table=f\"swast-scratch.scipy2025.national_jukebox\",\n", + " column_to_search=\"ml_generate_embedding_result\",\n", + " query=search_embedding,\n", + " distance_type=\"COSINE\",\n", + " query_column_to_search=\"ml_generate_embedding_result\",\n", + " top_k=5,\n", + ")" + ] }, { - "id": "f84ebe70", "cell_type": "code", - "source": [ - "vector_search_results.dtypes" - ], + "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-08-14T16:05:50.566930Z", @@ -1056,16 +994,14 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "vector_search_results.dtypes" + ] }, { - "id": "eeff1c72", "cell_type": "code", - "source": [ - "results = vector_search_results[[\"Title\", \"Summary\", \"Names\", \"GCS URI\", \"Transcription\", \"distance\"]].sort_values(\"distance\").to_pandas()\n", - "results" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -1088,15 +1024,15 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "results = vector_search_results[[\"Title\", \"Summary\", \"Names\", \"GCS URI\", \"Transcription\", \"distance\"]].sort_values(\"distance\").to_pandas()\n", + "results" + ] }, { - "id": "7ec53675", "cell_type": "code", - "source": [ - "print(results[\"Transcription\"].struct.field(\"content\").iloc[0])" - ], + "execution_count": null, "metadata": { "@deathbeds/jupyterlab-fonts": { "styles": { @@ -1116,22 +1052,14 @@ }, "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "print(results[\"Transcription\"].struct.field(\"content\").iloc[0])" + ] }, { - "id": "a96552fb", "cell_type": "code", - "source": [ - "import gcsfs\n", - "import IPython.display\n", - "\n", - "fs = gcsfs.GCSFileSystem(project='bigframes-dev')\n", - "with fs.open(results[\"GCS URI\"].iloc[0]) as song_file:\n", - " song_bytes = song_file.read()\n", - "\n", - "IPython.display.Audio(song_bytes)" - ], + "execution_count": null, "metadata": { "editable": true, "execution": { @@ -1148,18 +1076,26 @@ "tags": [], "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "import gcsfs\n", + "import IPython.display\n", + "\n", + "fs = gcsfs.GCSFileSystem(project='bigframes-dev')\n", + "with fs.open(results[\"GCS URI\"].iloc[0]) as song_file:\n", + " song_bytes = song_file.read()\n", + "\n", + "IPython.display.Audio(song_bytes)" + ] }, { - "id": "72af7c7f", "cell_type": "code", - "source": [], + "execution_count": null, "metadata": { "trusted": true }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [] } ], "metadata": { @@ -1196,6 +1132,6 @@ "version": "3.11.13" } }, - "nbformat_minor": 4, - "nbformat": 4 -} \ No newline at end of file + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/location/regionalized.ipynb b/notebooks/location/regionalized.ipynb index b1e9e010d48..066cd181364 100644 --- a/notebooks/location/regionalized.ipynb +++ b/notebooks/location/regionalized.ipynb @@ -17,7 +17,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Infer location and set up data in that location if needed" + "### Infer location and set up data in that location if needed" ] }, { @@ -126,7 +126,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Set BigQuery DataFrames options" + "### Set BigQuery DataFrames options" ] }, { @@ -1339,12 +1339,26 @@ "# Using the Remote Functions" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Python 3.13 is not yet a supported runtime for remote functions.\n", + "# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n", + "if sys.version_info >= (3, 13, 0):\n", + " sys.exit(0)" + ] + }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "## BigQuery DataFrames gives you the ability to turn your custom scalar functions into a BigQuery remote function.", + "### BigQuery DataFrames gives you the ability to turn your custom scalar functions into a BigQuery remote function.\n", "\n", "It requires the GCP project to be set up appropriately and the user having sufficient privileges to use them. One can find more details on it via `help` command." ] @@ -1643,7 +1657,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Initialize a DataFrame from a BigQuery table" + "### Initialize a DataFrame from a BigQuery table" ] }, { diff --git a/notebooks/ml/bq_dataframes_ml_cross_validation.ipynb b/notebooks/ml/bq_dataframes_ml_cross_validation.ipynb index 3dc0eabf5a1..501bfc88d31 100644 --- a/notebooks/ml/bq_dataframes_ml_cross_validation.ipynb +++ b/notebooks/ml/bq_dataframes_ml_cross_validation.ipynb @@ -991,7 +991,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv (3.10.14)", + "display_name": "venv", "language": "python", "name": "python3" }, @@ -1005,7 +1005,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.10.15" } }, "nbformat": 4, diff --git a/notebooks/ml/bq_dataframes_ml_linear_regression.ipynb b/notebooks/ml/bq_dataframes_ml_linear_regression.ipynb index 210922eab94..00aa7a347cb 100644 --- a/notebooks/ml/bq_dataframes_ml_linear_regression.ipynb +++ b/notebooks/ml/bq_dataframes_ml_linear_regression.ipynb @@ -1,760 +1,760 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ur8xi4C7S06n" - }, - "outputs": [], - "source": [ - "# Copyright 2023 Google LLC\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JAPoU8Sm5E6e" - }, - "source": [ - "# Train a linear regression model with BigQuery DataFrames ML", - "\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \"Colab Run in Colab\n", - " \n", - " \n", - " \n", - " \"GitHub\n", - " View on GitHub\n", - " \n", - " \n", - " \n", - " \"Vertex\n", - " Open in Vertex AI Workbench\n", - " \n", - " \n", - " \n", - " \"BQ\n", - " Open in BQ Studio\n", - " \n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "24743cf4a1e1" - }, - "source": [ - "**_NOTE_**: This notebook has been tested in the following environment:\n", - "\n", - "* Python version = 3.10" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tvgnzT1CKxrO" - }, - "source": [ - "## Overview\n", - "\n", - "Use this notebook to learn how to train a linear regression model using BigQuery DataFrames ML. BigQuery DataFrames ML provides a provides a scikit-learn-like API for ML powered by the BigQuery engine.\n", - "\n", - "This example is adapted from the [BQML linear regression tutorial](https://cloud.google.com/bigquery-ml/docs/linear-regression-tutorial).\n", - "\n", - "Learn more about [BigQuery DataFrames](https://cloud.google.com/python/docs/reference/bigframes/latest)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "d975e698c9a4" - }, - "source": [ - "### Objective\n", - "\n", - "In this tutorial, you use BigQuery DataFrames to create a linear regression model that predicts the weight of an Adelie penguin based on the penguin's island of residence, culmen length and depth, flipper length, and sex.\n", - "\n", - "The steps include:\n", - "\n", - "- Creating a DataFrame from a BigQuery table.\n", - "- Cleaning and preparing data using pandas.\n", - "- Creating a linear regression model using `bigframes.ml`.\n", - "- Saving the ML model to BigQuery for future use." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "08d289fa873f" - }, - "source": [ - "### Dataset\n", - "\n", - "This tutorial uses the [```penguins``` table](https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=ml_datasets&t=penguins) (a BigQuery Public Dataset) which includes data on a set of penguins including species, island of residence, weight, culmen length and depth, flipper length, and sex." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aed92deeb4a0" - }, - "source": [ - "### Costs\n", - "\n", - "This tutorial uses billable components of Google Cloud:\n", - "\n", - "* BigQuery (compute)\n", - "* BigQuery ML\n", - "\n", - "Learn about [BigQuery compute pricing](https://cloud.google.com/bigquery/pricing#analysis_pricing_models)\n", - "and [BigQuery ML pricing](https://cloud.google.com/bigquery/pricing#bqml),\n", - "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n", - "to generate a cost estimate based on your projected usage." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "i7EUnXsZhAGF" - }, - "source": [ - "## Installation\n", - "\n", - "If you don't have [bigframes](https://pypi.org/project/bigframes/) package already installed, uncomment and execute the following cells to\n", - "\n", - "1. Install the package\n", - "1. Restart the notebook kernel (Jupyter or Colab) to work with the package" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "9O0Ka4W2MNF3" - }, - "outputs": [], - "source": [ - "# !pip install bigframes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "f200f10a1da3" - }, - "outputs": [], - "source": [ - "# Automatically restart kernel after installs so that your environment can access the new packages\n", - "# import IPython\n", - "\n", - "# app = IPython.Application.instance()\n", - "# app.kernel.do_shutdown(True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BF1j6f9HApxa" - }, - "source": [ - "## Before you begin\n", - "\n", - "Complete the tasks in this section to set up your environment." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oDfTjfACBvJk" - }, - "source": [ - "### Set up your Google Cloud project\n", - "\n", - "**The following steps are required, regardless of your notebook environment.**\n", - "\n", - "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 credit towards your compute/storage costs.\n", - "\n", - "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", - "\n", - "3. [Enable the BigQuery API](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com).\n", - "\n", - "4. If you are running this notebook locally, install the [Cloud SDK](https://cloud.google.com/sdk)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "WReHDGG5g0XY" - }, - "source": [ - "#### Set your project ID\n", - "\n", - "If you don't know your project ID, try the following:\n", - "* Run `gcloud config list`.\n", - "* Run `gcloud projects list`.\n", - "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "oM1iC_MfAts1" - }, - "outputs": [], - "source": [ - "PROJECT_ID = \"\" # @param {type:\"string\"}\n", - "\n", - "# Set the project id\n", - "! gcloud config set project {PROJECT_ID}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "region" - }, - "source": [ - "#### Set the region\n", - "\n", - "You can also change the `REGION` variable used by BigQuery. Learn more about [BigQuery regions](https://cloud.google.com/bigquery/docs/locations#supported_locations)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "eF-Twtc4XGem" - }, - "outputs": [], - "source": [ - "REGION = \"US\" # @param {type: \"string\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sBCra4QMA2wR" - }, - "source": [ - "### Authenticate your Google Cloud account\n", - "\n", - "Depending on your Jupyter environment, you might have to manually authenticate. Follow the relevant instructions below." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "74ccc9e52986" - }, - "source": [ - "**Vertex AI Workbench**\n", - "\n", - "Do nothing, you are already authenticated." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "de775a3773ba" - }, - "source": [ - "**Local JupyterLab instance**\n", - "\n", - "Uncomment and run the following cell:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "254614fa0c46" - }, - "outputs": [], - "source": [ - "# ! gcloud auth login" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ef21552ccea8" - }, - "source": [ - "**Colab**\n", - "\n", - "Uncomment and run the following cell:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "603adbbf0532" - }, - "outputs": [], - "source": [ - "# from google.colab import auth\n", - "# auth.authenticate_user()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "960505627ddf" - }, - "source": [ - "### Import libraries" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "PyQmSRbKA8r-" - }, - "outputs": [], - "source": [ - "import bigframes.pandas as bpd" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "init_aip:mbsdk,all" - }, - "source": [ - "### Set BigQuery DataFrames options" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NPPMuw2PXGeo" - }, - "outputs": [], - "source": [ - "# Note: The project option is not required in all environments.\n", - "# On BigQuery Studio, the project ID is automatically detected.\n", - "bpd.options.bigquery.project = PROJECT_ID\n", - "\n", - "# Note: The location option is not required.\n", - "# It defaults to the location of the first table or query\n", - "# passed to read_gbq(). For APIs where a location can't be\n", - "# auto-detected, the location defaults to the \"US\" location.\n", - "bpd.options.bigquery.location = REGION" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "D21CoOlfFTYI" - }, - "source": [ - "If you want to reset the location of the created DataFrame or Series objects, reset the session by executing `bpd.close_session()`. After that, you can reuse `bpd.options.bigquery.location` to specify another location." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9EMAqR37AfLS" - }, - "source": [ - "## Read a BigQuery table into a BigQuery DataFrames DataFrame\n", - "\n", - "Read the [```penguins``` table](https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=ml_datasets&t=penguins) into a BigQuery DataFrames DataFrame:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "EDAaIwHpQCDZ" - }, - "outputs": [], - "source": [ - "df = bpd.read_gbq(\"bigquery-public-data.ml_datasets.penguins\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DJu837YEXD7B" - }, - "source": [ - "Take a look at the DataFrame:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "_gPD0Zn1Stdb" - }, - "outputs": [], - "source": [ - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "rwPLjqW2Ajzh" - }, - "source": [ - "## Clean and prepare data\n", - "\n", - "You can use pandas as you normally would on the BigQuery DataFrames DataFrame, but calculations happen in the BigQuery query engine instead of your local environment.\n", - "\n", - "Because this model will focus on the Adelie Penguin species, you need to filter the data for only those rows representing Adelie penguins. Then you drop the `species` column because it is no longer needed.\n", - "\n", - "As these functions are applied, only the new DataFrame object `adelie_data` is modified. The source table and the original DataFrame object `df` don't change." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "6i6HkFJZa8na" - }, - "outputs": [], - "source": [ - "# Filter down to the data to the Adelie Penguin species\n", - "adelie_data = df[df.species == \"Adelie Penguin (Pygoscelis adeliae)\"]\n", - "\n", - "# Drop the species column\n", - "adelie_data = adelie_data.drop(columns=[\"species\"])\n", - "\n", - "# Take a look at the filtered DataFrame\n", - "adelie_data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jhK2OlyMbY4L" - }, - "source": [ - "Drop rows with `NULL` values in order to create a BigQuery DataFrames DataFrame for the training data:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "0am3hdlXZfxZ" - }, - "outputs": [], - "source": [ - "# Drop rows with nulls to get training data\n", - "training_data = adelie_data.dropna()\n", - "\n", - "# Take a peek at the training data\n", - "training_data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "M_-0X7NxYK5f" - }, - "source": [ - "Specify your feature (or input) columns and the label (or output) column:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "YKwCW7Nsavap" - }, - "outputs": [], - "source": [ - "feature_columns = training_data[['island', 'culmen_length_mm', 'culmen_depth_mm', 'flipper_length_mm', 'sex']]\n", - "label_columns = training_data[['body_mass_g']]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "CjyM7vZJZ0sQ" - }, - "source": [ - "There is a row within the `adelie_data` BigQuery DataFrames DataFrame that has a `NULL` value for the `body mass` column. `body mass` is the label column, which is the value that the model you are creating is trying to predict.\n", - "\n", - "Create a new BigQuery DataFrames DataFrame, `test_data`, for this row so that you can use it as test data on which to make a prediction later:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "wej78IDUaRW9" - }, - "outputs": [], - "source": [ - "test_data = adelie_data[adelie_data.body_mass_g.isnull()]\n", - "\n", - "test_data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Fx4lsNqMorJ-" - }, - "source": [ - "## Create the linear regression model\n", - "\n", - "BigQuery DataFrames ML lets you move from exploring data to creating machine learning models through its scikit-learn-like API, `bigframes.ml`. BigQuery DataFrames ML supports several types of [ML models](https://cloud.google.com/python/docs/reference/bigframes/latest#ml-capabilities).\n", - "\n", - "In this notebook, you create a linear regression model, a type of regression model that generates a continuous value from a linear combination of input features.\n", - "\n", - "When you create a model with BigQuery DataFrames ML, it is saved locally and limited to the BigQuery session. However, as you'll see in the next section, you can use `to_gbq` to save the model permanently to your BigQuery project." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EloGtMnverFF" - }, - "source": [ - "### Create the model using `bigframes.ml`\n", - "\n", - "When you pass the feature columns without transforms, BigQuery ML uses\n", - "[automatic preprocessing](https://cloud.google.com/bigquery/docs/auto-preprocessing) to encode string values and scale numeric values.\n", - "\n", - "BigQuery ML also [automatically splits the data for training and evaluation](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-create-glm#data_split_method), although for datasets with less than 500 rows (such as this one), all rows are used for training." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "GskyyUQPowBT" - }, - "outputs": [], - "source": [ - "from bigframes.ml.linear_model import LinearRegression\n", - "\n", - "model = LinearRegression()\n", - "\n", - "model.fit(feature_columns, label_columns)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "UGjeMPC2caKK" - }, - "source": [ - "### Score the model\n", - "\n", - "Check how the model performed by using the `score` method. More information on model scoring can be found [here](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-evaluate#mlevaluate_output)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "kGBJKafpo0dl" - }, - "outputs": [], - "source": [ - "model.score(feature_columns, label_columns)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "P2lUiZZ_cjri" - }, - "source": [ - "### Predict using the model\n", - "\n", - "Use the model to predict the body mass of the data row you saved earlier to the `test_data` DataFrame:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "bsQ9cmoWo0Ps" - }, - "outputs": [], - "source": [ - "model.predict(test_data)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GTRdUw-Ro5R1" - }, - "source": [ - "## Save the model in BigQuery\n", - "\n", - "The model is saved locally within this session. You can save the model permanently to BigQuery for use in future sessions, and to make the model sharable with others." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "K0mPaoGpcwwy" - }, - "source": [ - "Create a BigQuery dataset to house the model, adding a name for your dataset as the `DATASET_ID` variable:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZSP7gt13QrQt" - }, - "outputs": [], - "source": [ - "DATASET_ID = \"\" # @param {type:\"string\"}\n", - "\n", - "from google.cloud import bigquery\n", - "client = bigquery.Client(project=PROJECT_ID)\n", - "dataset = bigquery.Dataset(PROJECT_ID + \".\" + DATASET_ID)\n", - "dataset.location = REGION\n", - "dataset = client.create_dataset(dataset, exists_ok=True)\n", - "print(f\"Dataset {dataset.dataset_id} created.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zqAIWWgJczp-" - }, - "source": [ - "Save the model using the `to_gbq` method:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "QE_GD4Byo_jb" - }, - "outputs": [], - "source": [ - "model.to_gbq(DATASET_ID + \".penguin_weight\" , replace=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "f7uHacAy49rT" - }, - "source": [ - "You can view the saved model in the BigQuery console under the dataset you created in the first step. Run the following cell and follow the link to view your BigQuery console:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "qDBoiA_0488Z" - }, - "outputs": [], - "source": [ - "print(f'https://console.developers.google.com/bigquery?p={PROJECT_ID}')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "G_wjSfXpWTuy" - }, - "source": [ - "# Summary and next steps\n", - "\n", - "You've created a linear regression model using `bigframes.ml`.\n", - "\n", - "Learn more about BigQuery DataFrames in the [documentation](https://cloud.google.com/python/docs/reference/bigframes/latest) and find more sample notebooks in the [GitHub repo](https://github.com/googleapis/python-bigquery-dataframes/tree/main/notebooks)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TpV-iwP9qw9c" - }, - "source": [ - "## Cleaning up\n", - "\n", - "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n", - "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", - "\n", - "Otherwise, you can uncomment the remaining cells and run them to delete the individual resources you created in this tutorial:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "sx_vKniMq9ZX" - }, - "outputs": [], - "source": [ - "# # Delete the BigQuery dataset and associated ML model\n", - "# from google.cloud import bigquery\n", - "# client = bigquery.Client(project=PROJECT_ID)\n", - "# client.delete_dataset(\n", - "# DATASET_ID, delete_contents=True, not_found_ok=True\n", - "# )\n", - "# print(\"Deleted dataset '{}'.\".format(DATASET_ID))" - ] - } - ], - "metadata": { - "colab": { - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.0" - } - }, - "nbformat": 4, - "nbformat_minor": 0 + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ur8xi4C7S06n" + }, + "outputs": [], + "source": [ + "# Copyright 2023 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JAPoU8Sm5E6e" + }, + "source": [ + "## Train a linear regression model with BigQuery DataFrames ML\n", + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \"Colab Run in Colab\n", + " \n", + " \n", + " \n", + " \"GitHub\n", + " View on GitHub\n", + " \n", + " \n", + " \n", + " \"Vertex\n", + " Open in Vertex AI Workbench\n", + " \n", + " \n", + " \n", + " \"BQ\n", + " Open in BQ Studio\n", + " \n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "24743cf4a1e1" + }, + "source": [ + "**_NOTE_**: This notebook has been tested in the following environment:\n", + "\n", + "* Python version = 3.10" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tvgnzT1CKxrO" + }, + "source": [ + "## Overview\n", + "\n", + "Use this notebook to learn how to train a linear regression model using BigQuery DataFrames ML. BigQuery DataFrames ML provides a provides a scikit-learn-like API for ML powered by the BigQuery engine.\n", + "\n", + "This example is adapted from the [BQML linear regression tutorial](https://cloud.google.com/bigquery-ml/docs/linear-regression-tutorial).\n", + "\n", + "Learn more about [BigQuery DataFrames](https://cloud.google.com/python/docs/reference/bigframes/latest)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d975e698c9a4" + }, + "source": [ + "### Objective\n", + "\n", + "In this tutorial, you use BigQuery DataFrames to create a linear regression model that predicts the weight of an Adelie penguin based on the penguin's island of residence, culmen length and depth, flipper length, and sex.\n", + "\n", + "The steps include:\n", + "\n", + "- Creating a DataFrame from a BigQuery table.\n", + "- Cleaning and preparing data using pandas.\n", + "- Creating a linear regression model using `bigframes.ml`.\n", + "- Saving the ML model to BigQuery for future use." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "08d289fa873f" + }, + "source": [ + "### Dataset\n", + "\n", + "This tutorial uses the [```penguins``` table](https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=ml_datasets&t=penguins) (a BigQuery Public Dataset) which includes data on a set of penguins including species, island of residence, weight, culmen length and depth, flipper length, and sex." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aed92deeb4a0" + }, + "source": [ + "### Costs\n", + "\n", + "This tutorial uses billable components of Google Cloud:\n", + "\n", + "* BigQuery (compute)\n", + "* BigQuery ML\n", + "\n", + "Learn about [BigQuery compute pricing](https://cloud.google.com/bigquery/pricing#analysis_pricing_models)\n", + "and [BigQuery ML pricing](https://cloud.google.com/bigquery/pricing#bqml),\n", + "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n", + "to generate a cost estimate based on your projected usage." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "i7EUnXsZhAGF" + }, + "source": [ + "## Installation\n", + "\n", + "If you don't have [bigframes](https://pypi.org/project/bigframes/) package already installed, uncomment and execute the following cells to\n", + "\n", + "1. Install the package\n", + "1. Restart the notebook kernel (Jupyter or Colab) to work with the package" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9O0Ka4W2MNF3" + }, + "outputs": [], + "source": [ + "# !pip install bigframes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "f200f10a1da3" + }, + "outputs": [], + "source": [ + "# Automatically restart kernel after installs so that your environment can access the new packages\n", + "# import IPython\n", + "\n", + "# app = IPython.Application.instance()\n", + "# app.kernel.do_shutdown(True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BF1j6f9HApxa" + }, + "source": [ + "## Before you begin\n", + "\n", + "Complete the tasks in this section to set up your environment." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oDfTjfACBvJk" + }, + "source": [ + "### Set up your Google Cloud project\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 credit towards your compute/storage costs.\n", + "\n", + "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", + "\n", + "3. [Enable the BigQuery API](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com).\n", + "\n", + "4. If you are running this notebook locally, install the [Cloud SDK](https://cloud.google.com/sdk)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WReHDGG5g0XY" + }, + "source": [ + "#### Set your project ID\n", + "\n", + "If you don't know your project ID, try the following:\n", + "* Run `gcloud config list`.\n", + "* Run `gcloud projects list`.\n", + "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "oM1iC_MfAts1" + }, + "outputs": [], + "source": [ + "PROJECT_ID = \"\" # @param {type:\"string\"}\n", + "\n", + "# Set the project id\n", + "! gcloud config set project {PROJECT_ID}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "region" + }, + "source": [ + "#### Set the region\n", + "\n", + "You can also change the `REGION` variable used by BigQuery. Learn more about [BigQuery regions](https://cloud.google.com/bigquery/docs/locations#supported_locations)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eF-Twtc4XGem" + }, + "outputs": [], + "source": [ + "REGION = \"US\" # @param {type: \"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sBCra4QMA2wR" + }, + "source": [ + "### Authenticate your Google Cloud account\n", + "\n", + "Depending on your Jupyter environment, you might have to manually authenticate. Follow the relevant instructions below." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "74ccc9e52986" + }, + "source": [ + "**Vertex AI Workbench**\n", + "\n", + "Do nothing, you are already authenticated." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "de775a3773ba" + }, + "source": [ + "**Local JupyterLab instance**\n", + "\n", + "Uncomment and run the following cell:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "254614fa0c46" + }, + "outputs": [], + "source": [ + "# ! gcloud auth login" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ef21552ccea8" + }, + "source": [ + "**Colab**\n", + "\n", + "Uncomment and run the following cell:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "603adbbf0532" + }, + "outputs": [], + "source": [ + "# from google.colab import auth\n", + "# auth.authenticate_user()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "960505627ddf" + }, + "source": [ + "### Import libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PyQmSRbKA8r-" + }, + "outputs": [], + "source": [ + "import bigframes.pandas as bpd" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "init_aip:mbsdk,all" + }, + "source": [ + "### Set BigQuery DataFrames options" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NPPMuw2PXGeo" + }, + "outputs": [], + "source": [ + "# Note: The project option is not required in all environments.\n", + "# On BigQuery Studio, the project ID is automatically detected.\n", + "bpd.options.bigquery.project = PROJECT_ID\n", + "\n", + "# Note: The location option is not required.\n", + "# It defaults to the location of the first table or query\n", + "# passed to read_gbq(). For APIs where a location can't be\n", + "# auto-detected, the location defaults to the \"US\" location.\n", + "bpd.options.bigquery.location = REGION" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "D21CoOlfFTYI" + }, + "source": [ + "If you want to reset the location of the created DataFrame or Series objects, reset the session by executing `bpd.close_session()`. After that, you can reuse `bpd.options.bigquery.location` to specify another location." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9EMAqR37AfLS" + }, + "source": [ + "## Read a BigQuery table into a BigQuery DataFrames DataFrame\n", + "\n", + "Read the [```penguins``` table](https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=ml_datasets&t=penguins) into a BigQuery DataFrames DataFrame:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EDAaIwHpQCDZ" + }, + "outputs": [], + "source": [ + "df = bpd.read_gbq(\"bigquery-public-data.ml_datasets.penguins\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DJu837YEXD7B" + }, + "source": [ + "Take a look at the DataFrame:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_gPD0Zn1Stdb" + }, + "outputs": [], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rwPLjqW2Ajzh" + }, + "source": [ + "## Clean and prepare data\n", + "\n", + "You can use pandas as you normally would on the BigQuery DataFrames DataFrame, but calculations happen in the BigQuery query engine instead of your local environment.\n", + "\n", + "Because this model will focus on the Adelie Penguin species, you need to filter the data for only those rows representing Adelie penguins. Then you drop the `species` column because it is no longer needed.\n", + "\n", + "As these functions are applied, only the new DataFrame object `adelie_data` is modified. The source table and the original DataFrame object `df` don't change." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6i6HkFJZa8na" + }, + "outputs": [], + "source": [ + "# Filter down to the data to the Adelie Penguin species\n", + "adelie_data = df[df.species == \"Adelie Penguin (Pygoscelis adeliae)\"]\n", + "\n", + "# Drop the species column\n", + "adelie_data = adelie_data.drop(columns=[\"species\"])\n", + "\n", + "# Take a look at the filtered DataFrame\n", + "adelie_data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jhK2OlyMbY4L" + }, + "source": [ + "Drop rows with `NULL` values in order to create a BigQuery DataFrames DataFrame for the training data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "0am3hdlXZfxZ" + }, + "outputs": [], + "source": [ + "# Drop rows with nulls to get training data\n", + "training_data = adelie_data.dropna()\n", + "\n", + "# Take a peek at the training data\n", + "training_data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M_-0X7NxYK5f" + }, + "source": [ + "Specify your feature (or input) columns and the label (or output) column:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YKwCW7Nsavap" + }, + "outputs": [], + "source": [ + "feature_columns = training_data[['island', 'culmen_length_mm', 'culmen_depth_mm', 'flipper_length_mm', 'sex']]\n", + "label_columns = training_data[['body_mass_g']]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CjyM7vZJZ0sQ" + }, + "source": [ + "There is a row within the `adelie_data` BigQuery DataFrames DataFrame that has a `NULL` value for the `body mass` column. `body mass` is the label column, which is the value that the model you are creating is trying to predict.\n", + "\n", + "Create a new BigQuery DataFrames DataFrame, `test_data`, for this row so that you can use it as test data on which to make a prediction later:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wej78IDUaRW9" + }, + "outputs": [], + "source": [ + "test_data = adelie_data[adelie_data.body_mass_g.isnull()]\n", + "\n", + "test_data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Fx4lsNqMorJ-" + }, + "source": [ + "## Create the linear regression model\n", + "\n", + "BigQuery DataFrames ML lets you move from exploring data to creating machine learning models through its scikit-learn-like API, `bigframes.ml`. BigQuery DataFrames ML supports several types of [ML models](https://cloud.google.com/python/docs/reference/bigframes/latest#ml-capabilities).\n", + "\n", + "In this notebook, you create a linear regression model, a type of regression model that generates a continuous value from a linear combination of input features.\n", + "\n", + "When you create a model with BigQuery DataFrames ML, it is saved locally and limited to the BigQuery session. However, as you'll see in the next section, you can use `to_gbq` to save the model permanently to your BigQuery project." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EloGtMnverFF" + }, + "source": [ + "### Create the model using `bigframes.ml`\n", + "\n", + "When you pass the feature columns without transforms, BigQuery ML uses\n", + "[automatic preprocessing](https://cloud.google.com/bigquery/docs/auto-preprocessing) to encode string values and scale numeric values.\n", + "\n", + "BigQuery ML also [automatically splits the data for training and evaluation](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-create-glm#data_split_method), although for datasets with less than 500 rows (such as this one), all rows are used for training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GskyyUQPowBT" + }, + "outputs": [], + "source": [ + "from bigframes.ml.linear_model import LinearRegression\n", + "\n", + "model = LinearRegression()\n", + "\n", + "model.fit(feature_columns, label_columns)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UGjeMPC2caKK" + }, + "source": [ + "### Score the model\n", + "\n", + "Check how the model performed by using the `score` method. More information on model scoring can be found [here](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-evaluate#mlevaluate_output)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kGBJKafpo0dl" + }, + "outputs": [], + "source": [ + "model.score(feature_columns, label_columns)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P2lUiZZ_cjri" + }, + "source": [ + "### Predict using the model\n", + "\n", + "Use the model to predict the body mass of the data row you saved earlier to the `test_data` DataFrame:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bsQ9cmoWo0Ps" + }, + "outputs": [], + "source": [ + "model.predict(test_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GTRdUw-Ro5R1" + }, + "source": [ + "## Save the model in BigQuery\n", + "\n", + "The model is saved locally within this session. You can save the model permanently to BigQuery for use in future sessions, and to make the model sharable with others." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "K0mPaoGpcwwy" + }, + "source": [ + "Create a BigQuery dataset to house the model, adding a name for your dataset as the `DATASET_ID` variable:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZSP7gt13QrQt" + }, + "outputs": [], + "source": [ + "DATASET_ID = \"\" # @param {type:\"string\"}\n", + "\n", + "from google.cloud import bigquery\n", + "client = bigquery.Client(project=PROJECT_ID)\n", + "dataset = bigquery.Dataset(PROJECT_ID + \".\" + DATASET_ID)\n", + "dataset.location = REGION\n", + "dataset = client.create_dataset(dataset, exists_ok=True)\n", + "print(f\"Dataset {dataset.dataset_id} created.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zqAIWWgJczp-" + }, + "source": [ + "Save the model using the `to_gbq` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QE_GD4Byo_jb" + }, + "outputs": [], + "source": [ + "model.to_gbq(DATASET_ID + \".penguin_weight\" , replace=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f7uHacAy49rT" + }, + "source": [ + "You can view the saved model in the BigQuery console under the dataset you created in the first step. Run the following cell and follow the link to view your BigQuery console:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qDBoiA_0488Z" + }, + "outputs": [], + "source": [ + "print(f'https://console.developers.google.com/bigquery?p={PROJECT_ID}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "G_wjSfXpWTuy" + }, + "source": [ + "# Summary and next steps\n", + "\n", + "You've created a linear regression model using `bigframes.ml`.\n", + "\n", + "Learn more about BigQuery DataFrames in the [documentation](https://cloud.google.com/python/docs/reference/bigframes/latest) and find more sample notebooks in the [GitHub repo](https://github.com/googleapis/python-bigquery-dataframes/tree/main/notebooks)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TpV-iwP9qw9c" + }, + "source": [ + "## Cleaning up\n", + "\n", + "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n", + "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", + "\n", + "Otherwise, you can uncomment the remaining cells and run them to delete the individual resources you created in this tutorial:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sx_vKniMq9ZX" + }, + "outputs": [], + "source": [ + "# # Delete the BigQuery dataset and associated ML model\n", + "# from google.cloud import bigquery\n", + "# client = bigquery.Client(project=PROJECT_ID)\n", + "# client.delete_dataset(\n", + "# DATASET_ID, delete_contents=True, not_found_ok=True\n", + "# )\n", + "# print(\"Deleted dataset '{}'.\".format(DATASET_ID))" + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/notebooks/ml/bq_dataframes_ml_linear_regression_bbq.ipynb b/notebooks/ml/bq_dataframes_ml_linear_regression_bbq.ipynb index 396fde5a397..6be836c6f81 100644 --- a/notebooks/ml/bq_dataframes_ml_linear_regression_bbq.ipynb +++ b/notebooks/ml/bq_dataframes_ml_linear_regression_bbq.ipynb @@ -1,2637 +1,2637 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "ur8xi4C7S06n" - }, - "outputs": [], - "source": [ - "# Copyright 2023 Google LLC\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JAPoU8Sm5E6e" - }, - "source": [ - "# Train a linear regression model with BigQuery DataFrames ML", - "\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \"Colab Run in Colab\n", - " \n", - " \n", - " \n", - " \"GitHub\n", - " View on GitHub\n", - " \n", - " \n", - " \n", - " \"Vertex\n", - " Open in Vertex AI Workbench\n", - " \n", - " \n", - " \n", - " \"BQ\n", - " Open in BQ Studio\n", - " \n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "24743cf4a1e1" - }, - "source": [ - "**_NOTE_**: This notebook has been tested in the following environment:\n", - "\n", - "* Python version = 3.10" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tvgnzT1CKxrO" - }, - "source": [ - "## Overview\n", - "\n", - "Use this notebook to learn how to train a linear regression model using BigQuery ML and the `bigframes.bigquery` module.\n", - "\n", - "This example is adapted from the [BQML linear regression tutorial](https://cloud.google.com/bigquery-ml/docs/linear-regression-tutorial).\n", - "\n", - "Learn more about [BigQuery DataFrames](https://dataframes.bigquery.dev/)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "d975e698c9a4" - }, - "source": [ - "### Objective\n", - "\n", - "In this tutorial, you use BigQuery DataFrames to create a linear regression model that predicts the weight of an Adelie penguin based on the penguin's island of residence, culmen length and depth, flipper length, and sex.\n", - "\n", - "The steps include:\n", - "\n", - "- Creating a DataFrame from a BigQuery table.\n", - "- Cleaning and preparing data using pandas.\n", - "- Creating a linear regression model using `bigframes.ml`.\n", - "- Saving the ML model to BigQuery for future use." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "08d289fa873f" - }, - "source": [ - "### Dataset\n", - "\n", - "This tutorial uses the [```penguins``` table](https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=ml_datasets&t=penguins) (a BigQuery Public Dataset) which includes data on a set of penguins including species, island of residence, weight, culmen length and depth, flipper length, and sex." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aed92deeb4a0" - }, - "source": [ - "### Costs\n", - "\n", - "This tutorial uses billable components of Google Cloud:\n", - "\n", - "* BigQuery (compute)\n", - "* BigQuery ML\n", - "\n", - "Learn about [BigQuery compute pricing](https://cloud.google.com/bigquery/pricing#analysis_pricing_models)\n", - "and [BigQuery ML pricing](https://cloud.google.com/bigquery/pricing#bqml),\n", - "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n", - "to generate a cost estimate based on your projected usage." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "i7EUnXsZhAGF" - }, - "source": [ - "## Installation\n", - "\n", - "If you don't have [bigframes](https://pypi.org/project/bigframes/) package already installed, uncomment and execute the following cells to\n", - "\n", - "1. Install the package\n", - "1. Restart the notebook kernel (Jupyter or Colab) to work with the package" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "9O0Ka4W2MNF3" - }, - "outputs": [], - "source": [ - "# !pip install bigframes" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "f200f10a1da3" - }, - "outputs": [], - "source": [ - "# Automatically restart kernel after installs so that your environment can access the new packages\n", - "# import IPython\n", - "\n", - "# app = IPython.Application.instance()\n", - "# app.kernel.do_shutdown(True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BF1j6f9HApxa" - }, - "source": [ - "## Before you begin\n", - "\n", - "Complete the tasks in this section to set up your environment." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oDfTjfACBvJk" - }, - "source": [ - "### Set up your Google Cloud project\n", - "\n", - "**The following steps are required, regardless of your notebook environment.**\n", - "\n", - "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 credit towards your compute/storage costs.\n", - "\n", - "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", - "\n", - "3. [Enable the BigQuery API](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com).\n", - "\n", - "4. If you are running this notebook locally, install the [Cloud SDK](https://cloud.google.com/sdk)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "WReHDGG5g0XY" - }, - "source": [ - "#### Set your project ID\n", - "\n", - "If you don't know your project ID, try the following:\n", - "* Run `gcloud config list`.\n", - "* Run `gcloud projects list`.\n", - "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "oM1iC_MfAts1" - }, - "outputs": [ + "cells": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Updated property [core/project].\n" - ] - } - ], - "source": [ - "PROJECT_ID = \"\" # @param {type:\"string\"}\n", - "\n", - "# Set the project id\n", - "! gcloud config set project {PROJECT_ID}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "region" - }, - "source": [ - "#### Set the region\n", - "\n", - "You can also change the `REGION` variable used by BigQuery. Learn more about [BigQuery regions](https://cloud.google.com/bigquery/docs/locations#supported_locations)." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "eF-Twtc4XGem" - }, - "outputs": [], - "source": [ - "REGION = \"US\" # @param {type: \"string\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sBCra4QMA2wR" - }, - "source": [ - "### Authenticate your Google Cloud account\n", - "\n", - "Depending on your Jupyter environment, you might have to manually authenticate. Follow the relevant instructions below." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "74ccc9e52986" - }, - "source": [ - "**Vertex AI Workbench**\n", - "\n", - "Do nothing, you are already authenticated." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "de775a3773ba" - }, - "source": [ - "**Local JupyterLab instance**\n", - "\n", - "Uncomment and run the following cell:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "id": "254614fa0c46" - }, - "outputs": [], - "source": [ - "# ! gcloud auth login" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ef21552ccea8" - }, - "source": [ - "**Colab**\n", - "\n", - "Uncomment and run the following cell:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "id": "603adbbf0532" - }, - "outputs": [], - "source": [ - "# from google.colab import auth\n", - "# auth.authenticate_user()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "960505627ddf" - }, - "source": [ - "### Import libraries" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "id": "PyQmSRbKA8r-" - }, - "outputs": [], - "source": [ - "import bigframes.pandas as bpd" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "init_aip:mbsdk,all" - }, - "source": [ - "### Set BigQuery DataFrames options" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "id": "NPPMuw2PXGeo" - }, - "outputs": [], - "source": [ - "# Note: The project option is not required in all environments.\n", - "# On BigQuery Studio, the project ID is automatically detected.\n", - "bpd.options.bigquery.project = PROJECT_ID\n", - "\n", - "# Note: The location option is not required.\n", - "# It defaults to the location of the first table or query\n", - "# passed to read_gbq(). For APIs where a location can't be\n", - "# auto-detected, the location defaults to the \"US\" location.\n", - "bpd.options.bigquery.location = REGION\n", - "\n", - "# Recommended for performance. Disables pandas default ordering of all rows.\n", - "bpd.options.bigquery.ordering_mode = \"partial\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "D21CoOlfFTYI" - }, - "source": [ - "If you want to reset the location of the created DataFrame or Series objects, reset the session by executing `bpd.close_session()`. After that, you can reuse `bpd.options.bigquery.location` to specify another location." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9EMAqR37AfLS" - }, - "source": [ - "## Read a BigQuery table into a BigQuery DataFrames DataFrame\n", - "\n", - "Read the [```penguins``` table](https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=ml_datasets&t=penguins) into a BigQuery DataFrames DataFrame:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "id": "EDAaIwHpQCDZ" - }, - "outputs": [], - "source": [ - "df = bpd.read_gbq(\"bigquery-public-data.ml_datasets.penguins\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DJu837YEXD7B" - }, - "source": [ - "Take a look at the DataFrame:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "id": "_gPD0Zn1Stdb" - }, - "outputs": [ + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "ur8xi4C7S06n" + }, + "outputs": [], + "source": [ + "# Copyright 2023 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, { - "data": { - "text/html": [ - "✅ Completed. " - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "JAPoU8Sm5E6e" + }, + "source": [ + "## Train a linear regression model with BigQuery DataFrames ML\n", + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \"Colab Run in Colab\n", + " \n", + " \n", + " \n", + " \"GitHub\n", + " View on GitHub\n", + " \n", + " \n", + " \n", + " \"Vertex\n", + " Open in Vertex AI Workbench\n", + " \n", + " \n", + " \n", + " \"BQ\n", + " Open in BQ Studio\n", + " \n", + "
    " ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "species", - "rawType": "string", - "type": "string" - }, - { - "name": "island", - "rawType": "string", - "type": "string" - }, - { - "name": "culmen_length_mm", - "rawType": "Float64", - "type": "float" - }, - { - "name": "culmen_depth_mm", - "rawType": "Float64", - "type": "float" - }, - { - "name": "flipper_length_mm", - "rawType": "Float64", - "type": "float" - }, - { - "name": "body_mass_g", - "rawType": "Float64", - "type": "float" - }, - { - "name": "sex", - "rawType": "string", - "type": "string" - } - ], - "ref": "a652ba52-0445-4228-a2d5-baf837933515", - "rows": [ - [ - "0", - "Adelie Penguin (Pygoscelis adeliae)", - "Dream", - "36.6", - "18.4", - "184.0", - "3475.0", - "FEMALE" - ], - [ - "1", - "Adelie Penguin (Pygoscelis adeliae)", - "Dream", - "39.8", - "19.1", - "184.0", - "4650.0", - "MALE" - ], - [ - "2", - "Adelie Penguin (Pygoscelis adeliae)", - "Dream", - "40.9", - "18.9", - "184.0", - "3900.0", - "MALE" - ], - [ - "3", - "Chinstrap penguin (Pygoscelis antarctica)", - "Dream", - "46.5", - "17.9", - "192.0", - "3500.0", - "FEMALE" - ], - [ - "4", - "Adelie Penguin (Pygoscelis adeliae)", - "Dream", - "37.3", - "16.8", - "192.0", - "3000.0", - "FEMALE" - ] - ], - "shape": { - "columns": 7, - "rows": 5 - } + "cell_type": "markdown", + "metadata": { + "id": "24743cf4a1e1" }, - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    speciesislandculmen_length_mmculmen_depth_mmflipper_length_mmbody_mass_gsex
    0Adelie Penguin (Pygoscelis adeliae)Dream36.618.4184.03475.0FEMALE
    1Adelie Penguin (Pygoscelis adeliae)Dream39.819.1184.04650.0MALE
    2Adelie Penguin (Pygoscelis adeliae)Dream40.918.9184.03900.0MALE
    3Chinstrap penguin (Pygoscelis antarctica)Dream46.517.9192.03500.0FEMALE
    4Adelie Penguin (Pygoscelis adeliae)Dream37.316.8192.03000.0FEMALE
    \n", - "
    " - ], - "text/plain": [ - " species island culmen_length_mm \\\n", - "0 Adelie Penguin (Pygoscelis adeliae) Dream 36.6 \n", - "1 Adelie Penguin (Pygoscelis adeliae) Dream 39.8 \n", - "2 Adelie Penguin (Pygoscelis adeliae) Dream 40.9 \n", - "3 Chinstrap penguin (Pygoscelis antarctica) Dream 46.5 \n", - "4 Adelie Penguin (Pygoscelis adeliae) Dream 37.3 \n", - "\n", - " culmen_depth_mm flipper_length_mm body_mass_g sex \n", - "0 18.4 184.0 3475.0 FEMALE \n", - "1 19.1 184.0 4650.0 MALE \n", - "2 18.9 184.0 3900.0 MALE \n", - "3 17.9 192.0 3500.0 FEMALE \n", - "4 16.8 192.0 3000.0 FEMALE " + "source": [ + "**_NOTE_**: This notebook has been tested in the following environment:\n", + "\n", + "* Python version = 3.10" ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.peek()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "rwPLjqW2Ajzh" - }, - "source": [ - "## Clean and prepare data\n", - "\n", - "You can use pandas as you normally would on the BigQuery DataFrames DataFrame, but calculations happen in the BigQuery query engine instead of your local environment.\n", - "\n", - "Because this model will focus on the Adelie Penguin species, you need to filter the data for only those rows representing Adelie penguins. Then you drop the `species` column because it is no longer needed.\n", - "\n", - "As these functions are applied, only the new DataFrame object `adelie_data` is modified. The source table and the original DataFrame object `df` don't change." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "id": "6i6HkFJZa8na" - }, - "outputs": [ + }, { - "data": { - "text/html": [ - "✅ Completed. \n", - " Query processed 28.9 kB in 12 seconds of slot time. [Job bigframes-dev:US.bb256e8c-f2c7-4eff-b5f3-fcc6836110cf details]\n", - " " - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "tvgnzT1CKxrO" + }, + "source": [ + "## Overview\n", + "\n", + "Use this notebook to learn how to train a linear regression model using BigQuery ML and the `bigframes.bigquery` module.\n", + "\n", + "This example is adapted from the [BQML linear regression tutorial](https://cloud.google.com/bigquery-ml/docs/linear-regression-tutorial).\n", + "\n", + "Learn more about [BigQuery DataFrames](https://dataframes.bigquery.dev/)." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. \n", - " Query processed 8.4 kB in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "d975e698c9a4" + }, + "source": [ + "### Objective\n", + "\n", + "In this tutorial, you use BigQuery DataFrames to create a linear regression model that predicts the weight of an Adelie penguin based on the penguin's island of residence, culmen length and depth, flipper length, and sex.\n", + "\n", + "The steps include:\n", + "\n", + "- Creating a DataFrame from a BigQuery table.\n", + "- Cleaning and preparing data using pandas.\n", + "- Creating a linear regression model using `bigframes.ml`.\n", + "- Saving the ML model to BigQuery for future use." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "08d289fa873f" + }, + "source": [ + "### Dataset\n", + "\n", + "This tutorial uses the [```penguins``` table](https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=ml_datasets&t=penguins) (a BigQuery Public Dataset) which includes data on a set of penguins including species, island of residence, weight, culmen length and depth, flipper length, and sex." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    islandculmen_length_mmculmen_depth_mmflipper_length_mmbody_mass_gsex
    0Dream36.618.4184.03475.0FEMALE
    1Dream39.819.1184.04650.0MALE
    2Dream40.918.9184.03900.0MALE
    3Dream37.316.8192.03000.0FEMALE
    4Dream43.218.5192.04100.0MALE
    5Dream40.220.1200.03975.0MALE
    6Dream40.818.9208.04300.0MALE
    7Dream39.018.7185.03650.0MALE
    8Dream37.016.9185.03000.0FEMALE
    9Dream34.017.1185.03400.0FEMALE
    \n", - "

    10 rows × 6 columns

    \n", - "
    [152 rows x 6 columns in total]" - ], - "text/plain": [ - "island culmen_length_mm culmen_depth_mm flipper_length_mm body_mass_g \\\n", - " Dream 36.6 18.4 184.0 3475.0 \n", - " Dream 39.8 19.1 184.0 4650.0 \n", - " Dream 40.9 18.9 184.0 3900.0 \n", - " Dream 37.3 16.8 192.0 3000.0 \n", - " Dream 43.2 18.5 192.0 4100.0 \n", - " Dream 40.2 20.1 200.0 3975.0 \n", - " Dream 40.8 18.9 208.0 4300.0 \n", - " Dream 39.0 18.7 185.0 3650.0 \n", - " Dream 37.0 16.9 185.0 3000.0 \n", - " Dream 34.0 17.1 185.0 3400.0 \n", - "\n", - " sex \n", - "FEMALE \n", - " MALE \n", - " MALE \n", - "FEMALE \n", - " MALE \n", - " MALE \n", - " MALE \n", - " MALE \n", - "FEMALE \n", - "FEMALE \n", - "...\n", - "\n", - "[152 rows x 6 columns]" + "cell_type": "markdown", + "metadata": { + "id": "aed92deeb4a0" + }, + "source": [ + "### Costs\n", + "\n", + "This tutorial uses billable components of Google Cloud:\n", + "\n", + "* BigQuery (compute)\n", + "* BigQuery ML\n", + "\n", + "Learn about [BigQuery compute pricing](https://cloud.google.com/bigquery/pricing#analysis_pricing_models)\n", + "and [BigQuery ML pricing](https://cloud.google.com/bigquery/pricing#bqml),\n", + "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n", + "to generate a cost estimate based on your projected usage." ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Filter down to the data to the Adelie Penguin species\n", - "adelie_data = df[df.species == \"Adelie Penguin (Pygoscelis adeliae)\"]\n", - "\n", - "# Drop the species column\n", - "adelie_data = adelie_data.drop(columns=[\"species\"])\n", - "\n", - "# Take a look at the filtered DataFrame\n", - "adelie_data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jhK2OlyMbY4L" - }, - "source": [ - "Drop rows with `NULL` values in order to create a BigQuery DataFrames DataFrame for the training data:" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "id": "0am3hdlXZfxZ" - }, - "outputs": [ + }, { - "data": { - "text/html": [ - "Starting." - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "i7EUnXsZhAGF" + }, + "source": [ + "## Installation\n", + "\n", + "If you don't have [bigframes](https://pypi.org/project/bigframes/) package already installed, uncomment and execute the following cells to\n", + "\n", + "1. Install the package\n", + "1. Restart the notebook kernel (Jupyter or Colab) to work with the package" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. \n", - " Query processed 8.1 kB in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "9O0Ka4W2MNF3" + }, + "outputs": [], + "source": [ + "# !pip install bigframes" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " - ], - "text/plain": [ - "" + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "f200f10a1da3" + }, + "outputs": [], + "source": [ + "# Automatically restart kernel after installs so that your environment can access the new packages\n", + "# import IPython\n", + "\n", + "# app = IPython.Application.instance()\n", + "# app.kernel.do_shutdown(True)" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    islandculmen_length_mmculmen_depth_mmflipper_length_mmbody_mass_gsex
    0Dream36.618.4184.03475.0FEMALE
    1Dream39.819.1184.04650.0MALE
    2Dream40.918.9184.03900.0MALE
    3Dream37.316.8192.03000.0FEMALE
    4Dream43.218.5192.04100.0MALE
    5Dream40.220.1200.03975.0MALE
    6Dream40.818.9208.04300.0MALE
    7Dream39.018.7185.03650.0MALE
    8Dream37.016.9185.03000.0FEMALE
    9Dream34.017.1185.03400.0FEMALE
    \n", - "

    10 rows × 6 columns

    \n", - "
    [146 rows x 6 columns in total]" - ], - "text/plain": [ - "island culmen_length_mm culmen_depth_mm flipper_length_mm body_mass_g \\\n", - " Dream 36.6 18.4 184.0 3475.0 \n", - " Dream 39.8 19.1 184.0 4650.0 \n", - " Dream 40.9 18.9 184.0 3900.0 \n", - " Dream 37.3 16.8 192.0 3000.0 \n", - " Dream 43.2 18.5 192.0 4100.0 \n", - " Dream 40.2 20.1 200.0 3975.0 \n", - " Dream 40.8 18.9 208.0 4300.0 \n", - " Dream 39.0 18.7 185.0 3650.0 \n", - " Dream 37.0 16.9 185.0 3000.0 \n", - " Dream 34.0 17.1 185.0 3400.0 \n", - "\n", - " sex \n", - "FEMALE \n", - " MALE \n", - " MALE \n", - "FEMALE \n", - " MALE \n", - " MALE \n", - " MALE \n", - " MALE \n", - "FEMALE \n", - "FEMALE \n", - "...\n", - "\n", - "[146 rows x 6 columns]" + "cell_type": "markdown", + "metadata": { + "id": "BF1j6f9HApxa" + }, + "source": [ + "## Before you begin\n", + "\n", + "Complete the tasks in this section to set up your environment." ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Drop rows with nulls to get training data\n", - "training_data = adelie_data.dropna()\n", - "\n", - "# Take a peek at the training data\n", - "training_data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Fx4lsNqMorJ-" - }, - "source": [ - "## Create the linear regression model\n", - "\n", - "In this notebook, you create a linear regression model, a type of regression model that generates a continuous value from a linear combination of input features." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create a BigQuery dataset to house the model, adding a name for your dataset as the `DATASET_ID` variable:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ + }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "Dataset bqml_tutorial created.\n" - ] - } - ], - "source": [ - "DATASET_ID = \"bqml_tutorial\" # @param {type:\"string\"}\n", - "\n", - "from google.cloud import bigquery\n", - "client = bigquery.Client(project=PROJECT_ID)\n", - "dataset = bigquery.Dataset(PROJECT_ID + \".\" + DATASET_ID)\n", - "dataset.location = REGION\n", - "dataset = client.create_dataset(dataset, exists_ok=True)\n", - "print(f\"Dataset {dataset.dataset_id} created.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EloGtMnverFF" - }, - "source": [ - "### Create the model using `bigframes.bigquery.ml.create_model`\n", - "\n", - "When you pass the feature columns without transforms, BigQuery ML uses\n", - "[automatic preprocessing](https://cloud.google.com/bigquery/docs/auto-preprocessing) to encode string values and scale numeric values.\n", - "\n", - "BigQuery ML also [automatically splits the data for training and evaluation](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-create-glm#data_split_method), although for datasets with less than 500 rows (such as this one), all rows are used for training." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "id": "GskyyUQPowBT" - }, - "outputs": [ + "cell_type": "markdown", + "metadata": { + "id": "oDfTjfACBvJk" + }, + "source": [ + "### Set up your Google Cloud project\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 credit towards your compute/storage costs.\n", + "\n", + "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", + "\n", + "3. [Enable the BigQuery API](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com).\n", + "\n", + "4. If you are running this notebook locally, install the [Cloud SDK](https://cloud.google.com/sdk)." + ] + }, { - "data": { - "text/html": [ - "\n", - " Query started with request ID bigframes-dev:US.a33b3628-730b-46e8-ad17-c78bb48619ce.
    SQL
    CREATE OR REPLACE MODEL `bigframes-dev.bqml_tutorial.penguin_weight`\n",
-       "OPTIONS(model_type = 'LINEAR_REG')\n",
-       "AS SELECT\n",
-       "`bfuid_col_3` AS `island`,\n",
-       "`bfuid_col_4` AS `culmen_length_mm`,\n",
-       "`bfuid_col_5` AS `culmen_depth_mm`,\n",
-       "`bfuid_col_6` AS `flipper_length_mm`,\n",
-       "`bfuid_col_7` AS `label`,\n",
-       "`bfuid_col_8` AS `sex`\n",
-       "FROM\n",
-       "(SELECT\n",
-       "  `t0`.`bfuid_col_3`,\n",
-       "  `t0`.`bfuid_col_4`,\n",
-       "  `t0`.`bfuid_col_5`,\n",
-       "  `t0`.`bfuid_col_6`,\n",
-       "  `t0`.`bfuid_col_7`,\n",
-       "  `t0`.`bfuid_col_8`\n",
-       "FROM `bigframes-dev._63cfa399614a54153cc386c27d6c0c6fdb249f9e._e154f0aa_5b29_492a_b464_a77c5f5a3dbd_bqdf_60fa3196-5a3e-45ae-898e-c2b473bfa1e9` AS `t0`)\n",
-       "
    \n", - " " - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "WReHDGG5g0XY" + }, + "source": [ + "#### Set your project ID\n", + "\n", + "If you don't know your project ID, try the following:\n", + "* Run `gcloud config list`.\n", + "* Run `gcloud projects list`.\n", + "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "object", - "type": "string" - }, + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "oM1iC_MfAts1" + }, + "outputs": [ { - "name": "0", - "rawType": "object", - "type": "unknown" + "name": "stdout", + "output_type": "stream", + "text": [ + "Updated property [core/project].\n" + ] } - ], - "ref": "851c170c-08a5-4c06-8c0b-4547dbde3f18", - "rows": [ - [ - "etag", - "P3XS+g0ZZM19ywL+hdwUmQ==" - ], - [ - "modelReference", - "{'projectId': 'bigframes-dev', 'datasetId': 'bqml_tutorial', 'modelId': 'penguin_weight'}" - ], - [ - "creationTime", - "1764779445166" - ], - [ - "lastModifiedTime", - "1764779445237" - ], - [ - "modelType", - "LINEAR_REGRESSION" - ], - [ - "trainingRuns", - "[{'trainingOptions': {'lossType': 'MEAN_SQUARED_LOSS', 'l2Regularization': 0, 'inputLabelColumns': ['label'], 'dataSplitMethod': 'AUTO_SPLIT', 'optimizationStrategy': 'NORMAL_EQUATION', 'calculatePValues': False, 'enableGlobalExplain': False, 'categoryEncodingMethod': 'ONE_HOT_ENCODING', 'fitIntercept': True, 'standardizeFeatures': True}, 'trainingStartTime': '1764779429690', 'results': [{'index': 0, 'durationMs': '3104', 'trainingLoss': 78553.60163372214}], 'evaluationMetrics': {'regressionMetrics': {'meanAbsoluteError': 223.87876300779865, 'meanSquaredError': 78553.60163372215, 'meanSquaredLogError': 0.005614202871872688, 'medianAbsoluteError': 181.33091105963013, 'rSquared': 0.6239507555914934}}, 'startTime': '2025-12-03T16:30:29.690Z'}]" - ], - [ - "featureColumns", - "[{'name': 'island', 'type': {'typeKind': 'STRING'}}, {'name': 'culmen_length_mm', 'type': {'typeKind': 'FLOAT64'}}, {'name': 'culmen_depth_mm', 'type': {'typeKind': 'FLOAT64'}}, {'name': 'flipper_length_mm', 'type': {'typeKind': 'FLOAT64'}}, {'name': 'sex', 'type': {'typeKind': 'STRING'}}]" - ], - [ - "labelColumns", - "[{'name': 'predicted_label', 'type': {'typeKind': 'FLOAT64'}}]" - ], - [ - "location", - "US" - ] - ], - "shape": { - "columns": 1, - "rows": 9 - } + ], + "source": [ + "PROJECT_ID = \"\" # @param {type:\"string\"}\n", + "\n", + "# Set the project id\n", + "! gcloud config set project {PROJECT_ID}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "region" }, - "text/plain": [ - "etag P3XS+g0ZZM19ywL+hdwUmQ==\n", - "modelReference {'projectId': 'bigframes-dev', 'datasetId': 'b...\n", - "creationTime 1764779445166\n", - "lastModifiedTime 1764779445237\n", - "modelType LINEAR_REGRESSION\n", - "trainingRuns [{'trainingOptions': {'lossType': 'MEAN_SQUARE...\n", - "featureColumns [{'name': 'island', 'type': {'typeKind': 'STRI...\n", - "labelColumns [{'name': 'predicted_label', 'type': {'typeKin...\n", - "location US\n", - "dtype: object" + "source": [ + "#### Set the region\n", + "\n", + "You can also change the `REGION` variable used by BigQuery. Learn more about [BigQuery regions](https://cloud.google.com/bigquery/docs/locations#supported_locations)." ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import bigframes.bigquery as bbq\n", - "\n", - "model_name = f\"{PROJECT_ID}.{DATASET_ID}.penguin_weight\"\n", - "model_metadata = bbq.ml.create_model(\n", - " model_name,\n", - " replace=True,\n", - " options={\n", - " \"model_type\": \"LINEAR_REG\",\n", - " },\n", - " training_data=training_data.rename(columns={\"body_mass_g\": \"label\"})\n", - ")\n", - "model_metadata" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GskyyUQPowBT" - }, - "source": [ - "### Evaluate the model\n", - "\n", - "Check how the model performed by using the `evalutate` function. More information on model evaluation can be found [here](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-evaluate#mlevaluate_output)." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "id": "kGBJKafpo0dl" - }, - "outputs": [ + }, { - "data": { - "text/html": [ - "✅ Completed. \n", - " Query processed 0 Bytes in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "eF-Twtc4XGem" + }, + "outputs": [], + "source": [ + "REGION = \"US\" # @param {type: \"string\"}" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "sBCra4QMA2wR" + }, + "source": [ + "### Authenticate your Google Cloud account\n", + "\n", + "Depending on your Jupyter environment, you might have to manually authenticate. Follow the relevant instructions below." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "74ccc9e52986" + }, + "source": [ + "**Vertex AI Workbench**\n", + "\n", + "Do nothing, you are already authenticated." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    mean_absolute_errormean_squared_errormean_squared_log_errormedian_absolute_errorr2_scoreexplained_variance
    0223.87876378553.6016340.005614181.3309110.6239510.623951
    \n", - "

    1 rows × 6 columns

    \n", - "
    [1 rows x 6 columns in total]" - ], - "text/plain": [ - " mean_absolute_error mean_squared_error mean_squared_log_error \\\n", - "0 223.878763 78553.601634 0.005614 \n", - "\n", - " median_absolute_error r2_score explained_variance \n", - "0 181.330911 0.623951 0.623951 \n", - "\n", - "[1 rows x 6 columns]" + "cell_type": "markdown", + "metadata": { + "id": "de775a3773ba" + }, + "source": [ + "**Local JupyterLab instance**\n", + "\n", + "Uncomment and run the following cell:" ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "bbq.ml.evaluate(model_name)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "P2lUiZZ_cjri" - }, - "source": [ - "### Use the model to predict outcomes\n", - "\n", - "Now that you have evaluated your model, the next step is to use it to predict an\n", - "outcome. You can run `bigframes.bigquery.ml.predict` function on the model to\n", - "predict the body mass in grams of all penguins that reside on the Biscoe\n", - "Islands." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "id": "bsQ9cmoWo0Ps" - }, - "outputs": [ + }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/swast/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py:182: TimeTravelCacheWarning: Reading cached table from 2025-12-03 16:30:18.272882+00:00 to avoid\n", - "incompatibilies with previous reads of this table. To read the latest\n", - "version, set `use_cache=False` or close the current session with\n", - "Session.close() or bigframes.pandas.close_session().\n", - " return method(*args, **kwargs)\n" - ] + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "254614fa0c46" + }, + "outputs": [], + "source": [ + "# ! gcloud auth login" + ] }, { - "data": { - "text/html": [ - "✅ Completed. \n", - " Query processed 29.3 kB in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "ef21552ccea8" + }, + "source": [ + "**Colab**\n", + "\n", + "Uncomment and run the following cell:" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " - ], - "text/plain": [ - "" + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "603adbbf0532" + }, + "outputs": [], + "source": [ + "# from google.colab import auth\n", + "# auth.authenticate_user()" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "960505627ddf" + }, + "source": [ + "### Import libraries" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    predicted_labelspeciesislandculmen_length_mmculmen_depth_mmflipper_length_mmbody_mass_gsex
    03945.010052Gentoo penguin (Pygoscelis papua)Biscoe<NA><NA><NA><NA><NA>
    13914.916297Adelie Penguin (Pygoscelis adeliae)Biscoe39.718.9184.03550.0MALE
    23278.611224Adelie Penguin (Pygoscelis adeliae)Biscoe36.417.1184.02850.0FEMALE
    34006.367355Adelie Penguin (Pygoscelis adeliae)Biscoe41.618.0192.03950.0MALE
    43417.610478Adelie Penguin (Pygoscelis adeliae)Biscoe35.017.9192.03725.0FEMALE
    54009.612421Adelie Penguin (Pygoscelis adeliae)Biscoe41.118.2192.04050.0MALE
    64231.330911Adelie Penguin (Pygoscelis adeliae)Biscoe42.019.5200.04050.0MALE
    73554.308906Gentoo penguin (Pygoscelis papua)Biscoe43.813.9208.04300.0FEMALE
    83550.677455Gentoo penguin (Pygoscelis papua)Biscoe43.314.0208.04575.0FEMALE
    93537.882543Gentoo penguin (Pygoscelis papua)Biscoe44.013.6208.04350.0FEMALE
    \n", - "

    10 rows × 8 columns

    \n", - "
    [168 rows x 8 columns in total]" - ], - "text/plain": [ - " predicted_label species island \\\n", - "0 3945.010052 Gentoo penguin (Pygoscelis papua) Biscoe \n", - "1 3914.916297 Adelie Penguin (Pygoscelis adeliae) Biscoe \n", - "2 3278.611224 Adelie Penguin (Pygoscelis adeliae) Biscoe \n", - "3 4006.367355 Adelie Penguin (Pygoscelis adeliae) Biscoe \n", - "4 3417.610478 Adelie Penguin (Pygoscelis adeliae) Biscoe \n", - "5 4009.612421 Adelie Penguin (Pygoscelis adeliae) Biscoe \n", - "6 4231.330911 Adelie Penguin (Pygoscelis adeliae) Biscoe \n", - "7 3554.308906 Gentoo penguin (Pygoscelis papua) Biscoe \n", - "8 3550.677455 Gentoo penguin (Pygoscelis papua) Biscoe \n", - "9 3537.882543 Gentoo penguin (Pygoscelis papua) Biscoe \n", - "\n", - " culmen_length_mm culmen_depth_mm flipper_length_mm body_mass_g sex \n", - "0 \n", - "1 39.7 18.9 184.0 3550.0 MALE \n", - "2 36.4 17.1 184.0 2850.0 FEMALE \n", - "3 41.6 18.0 192.0 3950.0 MALE \n", - "4 35.0 17.9 192.0 3725.0 FEMALE \n", - "5 41.1 18.2 192.0 4050.0 MALE \n", - "6 42.0 19.5 200.0 4050.0 MALE \n", - "7 43.8 13.9 208.0 4300.0 FEMALE \n", - "8 43.3 14.0 208.0 4575.0 FEMALE \n", - "9 44.0 13.6 208.0 4350.0 FEMALE \n", - "...\n", - "\n", - "[168 rows x 8 columns]" + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "PyQmSRbKA8r-" + }, + "outputs": [], + "source": [ + "import bigframes.pandas as bpd" ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = bpd.read_gbq(\"bigquery-public-data.ml_datasets.penguins\")\n", - "biscoe = df[df[\"island\"].str.contains(\"Biscoe\")]\n", - "bbq.ml.predict(model_name, biscoe)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GTRdUw-Ro5R1" - }, - "source": [ - "### Explain the prediction results\n", - "\n", - "To understand why the model is generating these prediction results, you can use the `explain_predict` function." - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ + }, { - "data": { - "text/html": [ - "\n", - " Query started with request ID bigframes-dev:US.161bba69-c852-4916-a2df-bb5b309be6e4.
    SQL
    SELECT * FROM ML.EXPLAIN_PREDICT(MODEL `bigframes-dev.bqml_tutorial.penguin_weight`, (SELECT\n",
-       "`bfuid_col_22` AS `species`,\n",
-       "`bfuid_col_23` AS `island`,\n",
-       "`bfuid_col_24` AS `culmen_length_mm`,\n",
-       "`bfuid_col_25` AS `culmen_depth_mm`,\n",
-       "`bfuid_col_26` AS `flipper_length_mm`,\n",
-       "`bfuid_col_27` AS `body_mass_g`,\n",
-       "`bfuid_col_28` AS `sex`\n",
-       "FROM\n",
-       "(SELECT\n",
-       "  `t0`.`species`,\n",
-       "  `t0`.`island`,\n",
-       "  `t0`.`culmen_length_mm`,\n",
-       "  `t0`.`culmen_depth_mm`,\n",
-       "  `t0`.`flipper_length_mm`,\n",
-       "  `t0`.`body_mass_g`,\n",
-       "  `t0`.`sex`,\n",
-       "  `t0`.`species` AS `bfuid_col_22`,\n",
-       "  `t0`.`island` AS `bfuid_col_23`,\n",
-       "  `t0`.`culmen_length_mm` AS `bfuid_col_24`,\n",
-       "  `t0`.`culmen_depth_mm` AS `bfuid_col_25`,\n",
-       "  `t0`.`flipper_length_mm` AS `bfuid_col_26`,\n",
-       "  `t0`.`body_mass_g` AS `bfuid_col_27`,\n",
-       "  `t0`.`sex` AS `bfuid_col_28`,\n",
-       "  regexp_contains(`t0`.`island`, 'Biscoe') AS `bfuid_col_29`\n",
-       "FROM (\n",
-       "  SELECT\n",
-       "    `species`,\n",
-       "    `island`,\n",
-       "    `culmen_length_mm`,\n",
-       "    `culmen_depth_mm`,\n",
-       "    `flipper_length_mm`,\n",
-       "    `body_mass_g`,\n",
-       "    `sex`\n",
-       "  FROM `bigquery-public-data.ml_datasets.penguins` FOR SYSTEM_TIME AS OF TIMESTAMP('2025-12-03T16:30:18.272882+00:00')\n",
-       ") AS `t0`\n",
-       "WHERE\n",
-       "  regexp_contains(`t0`.`island`, 'Biscoe'))), STRUCT(3 AS top_k_features))\n",
-       "
    \n", - " " - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "init_aip:mbsdk,all" + }, + "source": [ + "### Set BigQuery DataFrames options" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " - ], - "text/plain": [ - "" + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "NPPMuw2PXGeo" + }, + "outputs": [], + "source": [ + "# Note: The project option is not required in all environments.\n", + "# On BigQuery Studio, the project ID is automatically detected.\n", + "bpd.options.bigquery.project = PROJECT_ID\n", + "\n", + "# Note: The location option is not required.\n", + "# It defaults to the location of the first table or query\n", + "# passed to read_gbq(). For APIs where a location can't be\n", + "# auto-detected, the location defaults to the \"US\" location.\n", + "bpd.options.bigquery.location = REGION\n", + "\n", + "# Recommended for performance. Disables pandas default ordering of all rows.\n", + "bpd.options.bigquery.ordering_mode = \"partial\"" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "D21CoOlfFTYI" + }, + "source": [ + "If you want to reset the location of the created DataFrame or Series objects, reset the session by executing `bpd.close_session()`. After that, you can reuse `bpd.options.bigquery.location` to specify another location." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    predicted_labeltop_feature_attributionsbaseline_prediction_valueprediction_valueapproximation_errorspeciesislandculmen_length_mmculmen_depth_mmflipper_length_mmbody_mass_gsex
    03945.010052[{'feature': 'island', 'attribution': 0.0}\n", - " {'...3945.0100523945.0100520.0Gentoo penguin (Pygoscelis papua)Biscoe<NA><NA><NA><NA><NA>
    13914.916297[{'feature': 'flipper_length_mm', 'attribution...3945.0100523914.9162970.0Adelie Penguin (Pygoscelis adeliae)Biscoe39.718.9184.03550.0MALE
    23278.611224[{'feature': 'sex', 'attribution': -443.175184...3945.0100523278.6112240.0Adelie Penguin (Pygoscelis adeliae)Biscoe36.417.1184.02850.0FEMALE
    34006.367355[{'feature': 'culmen_length_mm', 'attribution'...3945.0100524006.3673550.0Adelie Penguin (Pygoscelis adeliae)Biscoe41.618.0192.03950.0MALE
    43417.610478[{'feature': 'sex', 'attribution': -443.175184...3945.0100523417.6104780.0Adelie Penguin (Pygoscelis adeliae)Biscoe35.017.9192.03725.0FEMALE
    54009.612421[{'feature': 'culmen_length_mm', 'attribution'...3945.0100524009.6124210.0Adelie Penguin (Pygoscelis adeliae)Biscoe41.118.2192.04050.0MALE
    64231.330911[{'feature': 'flipper_length_mm', 'attribution...3945.0100524231.3309110.0Adelie Penguin (Pygoscelis adeliae)Biscoe42.019.5200.04050.0MALE
    73554.308906[{'feature': 'sex', 'attribution': -443.175184...3945.0100523554.3089060.0Gentoo penguin (Pygoscelis papua)Biscoe43.813.9208.04300.0FEMALE
    83550.677455[{'feature': 'sex', 'attribution': -443.175184...3945.0100523550.6774550.0Gentoo penguin (Pygoscelis papua)Biscoe43.314.0208.04575.0FEMALE
    93537.882543[{'feature': 'sex', 'attribution': -443.175184...3945.0100523537.8825430.0Gentoo penguin (Pygoscelis papua)Biscoe44.013.6208.04350.0FEMALE
    \n", - "

    10 rows × 12 columns

    \n", - "
    [168 rows x 12 columns in total]" - ], - "text/plain": [ - " predicted_label top_feature_attributions \\\n", - "0 3945.010052 [{'feature': 'island', 'attribution': 0.0}\n", - " {'... \n", - "1 3914.916297 [{'feature': 'flipper_length_mm', 'attribution... \n", - "2 3278.611224 [{'feature': 'sex', 'attribution': -443.175184... \n", - "3 4006.367355 [{'feature': 'culmen_length_mm', 'attribution'... \n", - "4 3417.610478 [{'feature': 'sex', 'attribution': -443.175184... \n", - "5 4009.612421 [{'feature': 'culmen_length_mm', 'attribution'... \n", - "6 4231.330911 [{'feature': 'flipper_length_mm', 'attribution... \n", - "7 3554.308906 [{'feature': 'sex', 'attribution': -443.175184... \n", - "8 3550.677455 [{'feature': 'sex', 'attribution': -443.175184... \n", - "9 3537.882543 [{'feature': 'sex', 'attribution': -443.175184... \n", - "\n", - " baseline_prediction_value prediction_value approximation_error \\\n", - "0 3945.010052 3945.010052 0.0 \n", - "1 3945.010052 3914.916297 0.0 \n", - "2 3945.010052 3278.611224 0.0 \n", - "3 3945.010052 4006.367355 0.0 \n", - "4 3945.010052 3417.610478 0.0 \n", - "5 3945.010052 4009.612421 0.0 \n", - "6 3945.010052 4231.330911 0.0 \n", - "7 3945.010052 3554.308906 0.0 \n", - "8 3945.010052 3550.677455 0.0 \n", - "9 3945.010052 3537.882543 0.0 \n", - "\n", - " species island culmen_length_mm \\\n", - "0 Gentoo penguin (Pygoscelis papua) Biscoe \n", - "1 Adelie Penguin (Pygoscelis adeliae) Biscoe 39.7 \n", - "2 Adelie Penguin (Pygoscelis adeliae) Biscoe 36.4 \n", - "3 Adelie Penguin (Pygoscelis adeliae) Biscoe 41.6 \n", - "4 Adelie Penguin (Pygoscelis adeliae) Biscoe 35.0 \n", - "5 Adelie Penguin (Pygoscelis adeliae) Biscoe 41.1 \n", - "6 Adelie Penguin (Pygoscelis adeliae) Biscoe 42.0 \n", - "7 Gentoo penguin (Pygoscelis papua) Biscoe 43.8 \n", - "8 Gentoo penguin (Pygoscelis papua) Biscoe 43.3 \n", - "9 Gentoo penguin (Pygoscelis papua) Biscoe 44.0 \n", - "\n", - " culmen_depth_mm flipper_length_mm body_mass_g sex \n", - "0 \n", - "1 18.9 184.0 3550.0 MALE \n", - "2 17.1 184.0 2850.0 FEMALE \n", - "3 18.0 192.0 3950.0 MALE \n", - "4 17.9 192.0 3725.0 FEMALE \n", - "5 18.2 192.0 4050.0 MALE \n", - "6 19.5 200.0 4050.0 MALE \n", - "7 13.9 208.0 4300.0 FEMALE \n", - "8 14.0 208.0 4575.0 FEMALE \n", - "9 13.6 208.0 4350.0 FEMALE \n", - "...\n", - "\n", - "[168 rows x 12 columns]" + "cell_type": "markdown", + "metadata": { + "id": "9EMAqR37AfLS" + }, + "source": [ + "## Read a BigQuery table into a BigQuery DataFrames DataFrame\n", + "\n", + "Read the [```penguins``` table](https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=ml_datasets&t=penguins) into a BigQuery DataFrames DataFrame:" ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "bbq.ml.explain_predict(model_name, biscoe, top_k_features=3)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "K0mPaoGpcwwy" - }, - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Globally explain the model\n", - "\n", - "To know which features are generally the most important to determine penguin\n", - "weight, you can use the `global_explain` function. In order to use\n", - "`global_explain`, you must retrain the model with the `enable_global_explain`\n", - "option set to `True`." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "id": "ZSP7gt13QrQt" - }, - "outputs": [ + }, { - "data": { - "text/html": [ - "✅ Completed. \n", - " Query processed 6.9 kB in 53 seconds of slot time. [Job bigframes-dev:US.job_welN8ErlZ_sTG7oOEULsWUgmIg7l details]\n", - " " - ], - "text/plain": [ - "" + "cell_type": "code", + "execution_count": 10, + "metadata": { + "id": "EDAaIwHpQCDZ" + }, + "outputs": [], + "source": [ + "df = bpd.read_gbq(\"bigquery-public-data.ml_datasets.penguins\")" ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "model_name = f\"{PROJECT_ID}.{DATASET_ID}.penguin_weight_with_global_explain\"\n", - "model_metadata = bbq.ml.create_model(\n", - " model_name,\n", - " replace=True,\n", - " options={\n", - " \"model_type\": \"LINEAR_REG\",\n", - " \"input_label_cols\": [\"body_mass_g\"],\n", - " \"enable_global_explain\": True,\n", - " },\n", - " training_data=training_data,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ + }, { - "data": { - "text/html": [ - "✅ Completed. \n", - " Query processed 0 Bytes in a moment of slot time.\n", - " " - ], - "text/plain": [ - "" + "cell_type": "markdown", + "metadata": { + "id": "DJu837YEXD7B" + }, + "source": [ + "Take a look at the DataFrame:" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " + "cell_type": "code", + "execution_count": 11, + "metadata": { + "id": "_gPD0Zn1Stdb" + }, + "outputs": [ + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "species", + "rawType": "string", + "type": "string" + }, + { + "name": "island", + "rawType": "string", + "type": "string" + }, + { + "name": "culmen_length_mm", + "rawType": "Float64", + "type": "float" + }, + { + "name": "culmen_depth_mm", + "rawType": "Float64", + "type": "float" + }, + { + "name": "flipper_length_mm", + "rawType": "Float64", + "type": "float" + }, + { + "name": "body_mass_g", + "rawType": "Float64", + "type": "float" + }, + { + "name": "sex", + "rawType": "string", + "type": "string" + } + ], + "ref": "a652ba52-0445-4228-a2d5-baf837933515", + "rows": [ + [ + "0", + "Adelie Penguin (Pygoscelis adeliae)", + "Dream", + "36.6", + "18.4", + "184.0", + "3475.0", + "FEMALE" + ], + [ + "1", + "Adelie Penguin (Pygoscelis adeliae)", + "Dream", + "39.8", + "19.1", + "184.0", + "4650.0", + "MALE" + ], + [ + "2", + "Adelie Penguin (Pygoscelis adeliae)", + "Dream", + "40.9", + "18.9", + "184.0", + "3900.0", + "MALE" + ], + [ + "3", + "Chinstrap penguin (Pygoscelis antarctica)", + "Dream", + "46.5", + "17.9", + "192.0", + "3500.0", + "FEMALE" + ], + [ + "4", + "Adelie Penguin (Pygoscelis adeliae)", + "Dream", + "37.3", + "16.8", + "192.0", + "3000.0", + "FEMALE" + ] + ], + "shape": { + "columns": 7, + "rows": 5 + } + }, + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    speciesislandculmen_length_mmculmen_depth_mmflipper_length_mmbody_mass_gsex
    0Adelie Penguin (Pygoscelis adeliae)Dream36.618.4184.03475.0FEMALE
    1Adelie Penguin (Pygoscelis adeliae)Dream39.819.1184.04650.0MALE
    2Adelie Penguin (Pygoscelis adeliae)Dream40.918.9184.03900.0MALE
    3Chinstrap penguin (Pygoscelis antarctica)Dream46.517.9192.03500.0FEMALE
    4Adelie Penguin (Pygoscelis adeliae)Dream37.316.8192.03000.0FEMALE
    \n", + "
    " + ], + "text/plain": [ + " species island culmen_length_mm \\\n", + "0 Adelie Penguin (Pygoscelis adeliae) Dream 36.6 \n", + "1 Adelie Penguin (Pygoscelis adeliae) Dream 39.8 \n", + "2 Adelie Penguin (Pygoscelis adeliae) Dream 40.9 \n", + "3 Chinstrap penguin (Pygoscelis antarctica) Dream 46.5 \n", + "4 Adelie Penguin (Pygoscelis adeliae) Dream 37.3 \n", + "\n", + " culmen_depth_mm flipper_length_mm body_mass_g sex \n", + "0 18.4 184.0 3475.0 FEMALE \n", + "1 19.1 184.0 4650.0 MALE \n", + "2 18.9 184.0 3900.0 MALE \n", + "3 17.9 192.0 3500.0 FEMALE \n", + "4 16.8 192.0 3000.0 FEMALE " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "df.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rwPLjqW2Ajzh" + }, + "source": [ + "## Clean and prepare data\n", + "\n", + "You can use pandas as you normally would on the BigQuery DataFrames DataFrame, but calculations happen in the BigQuery query engine instead of your local environment.\n", + "\n", + "Because this model will focus on the Adelie Penguin species, you need to filter the data for only those rows representing Adelie penguins. Then you drop the `species` column because it is no longer needed.\n", + "\n", + "As these functions are applied, only the new DataFrame object `adelie_data` is modified. The source table and the original DataFrame object `df` don't change." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " + "cell_type": "code", + "execution_count": 12, + "metadata": { + "id": "6i6HkFJZa8na" + }, + "outputs": [ + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 28.9 kB in 12 seconds of slot time. [Job bigframes-dev:US.bb256e8c-f2c7-4eff-b5f3-fcc6836110cf details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 8.4 kB in a moment of slot time.\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    islandculmen_length_mmculmen_depth_mmflipper_length_mmbody_mass_gsex
    0Dream36.618.4184.03475.0FEMALE
    1Dream39.819.1184.04650.0MALE
    2Dream40.918.9184.03900.0MALE
    3Dream37.316.8192.03000.0FEMALE
    4Dream43.218.5192.04100.0MALE
    5Dream40.220.1200.03975.0MALE
    6Dream40.818.9208.04300.0MALE
    7Dream39.018.7185.03650.0MALE
    8Dream37.016.9185.03000.0FEMALE
    9Dream34.017.1185.03400.0FEMALE
    \n", + "

    10 rows × 6 columns

    \n", + "
    [152 rows x 6 columns in total]" + ], + "text/plain": [ + "island culmen_length_mm culmen_depth_mm flipper_length_mm body_mass_g \\\n", + " Dream 36.6 18.4 184.0 3475.0 \n", + " Dream 39.8 19.1 184.0 4650.0 \n", + " Dream 40.9 18.9 184.0 3900.0 \n", + " Dream 37.3 16.8 192.0 3000.0 \n", + " Dream 43.2 18.5 192.0 4100.0 \n", + " Dream 40.2 20.1 200.0 3975.0 \n", + " Dream 40.8 18.9 208.0 4300.0 \n", + " Dream 39.0 18.7 185.0 3650.0 \n", + " Dream 37.0 16.9 185.0 3000.0 \n", + " Dream 34.0 17.1 185.0 3400.0 \n", + "\n", + " sex \n", + "FEMALE \n", + " MALE \n", + " MALE \n", + "FEMALE \n", + " MALE \n", + " MALE \n", + " MALE \n", + " MALE \n", + "FEMALE \n", + "FEMALE \n", + "...\n", + "\n", + "[152 rows x 6 columns]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "# Filter down to the data to the Adelie Penguin species\n", + "adelie_data = df[df.species == \"Adelie Penguin (Pygoscelis adeliae)\"]\n", + "\n", + "# Drop the species column\n", + "adelie_data = adelie_data.drop(columns=[\"species\"])\n", + "\n", + "# Take a look at the filtered DataFrame\n", + "adelie_data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jhK2OlyMbY4L" + }, + "source": [ + "Drop rows with `NULL` values in order to create a BigQuery DataFrames DataFrame for the training data:" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    featureattribution
    0sex221.587592
    1flipper_length_mm71.311846
    2culmen_depth_mm66.17986
    3culmen_length_mm45.443363
    4island17.258076
    \n", - "

    5 rows × 2 columns

    \n", - "
    [5 rows x 2 columns in total]" + "cell_type": "code", + "execution_count": 13, + "metadata": { + "id": "0am3hdlXZfxZ" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Starting." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 8.1 kB in a moment of slot time.\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    islandculmen_length_mmculmen_depth_mmflipper_length_mmbody_mass_gsex
    0Dream36.618.4184.03475.0FEMALE
    1Dream39.819.1184.04650.0MALE
    2Dream40.918.9184.03900.0MALE
    3Dream37.316.8192.03000.0FEMALE
    4Dream43.218.5192.04100.0MALE
    5Dream40.220.1200.03975.0MALE
    6Dream40.818.9208.04300.0MALE
    7Dream39.018.7185.03650.0MALE
    8Dream37.016.9185.03000.0FEMALE
    9Dream34.017.1185.03400.0FEMALE
    \n", + "

    10 rows × 6 columns

    \n", + "
    [146 rows x 6 columns in total]" + ], + "text/plain": [ + "island culmen_length_mm culmen_depth_mm flipper_length_mm body_mass_g \\\n", + " Dream 36.6 18.4 184.0 3475.0 \n", + " Dream 39.8 19.1 184.0 4650.0 \n", + " Dream 40.9 18.9 184.0 3900.0 \n", + " Dream 37.3 16.8 192.0 3000.0 \n", + " Dream 43.2 18.5 192.0 4100.0 \n", + " Dream 40.2 20.1 200.0 3975.0 \n", + " Dream 40.8 18.9 208.0 4300.0 \n", + " Dream 39.0 18.7 185.0 3650.0 \n", + " Dream 37.0 16.9 185.0 3000.0 \n", + " Dream 34.0 17.1 185.0 3400.0 \n", + "\n", + " sex \n", + "FEMALE \n", + " MALE \n", + " MALE \n", + "FEMALE \n", + " MALE \n", + " MALE \n", + " MALE \n", + " MALE \n", + "FEMALE \n", + "FEMALE \n", + "...\n", + "\n", + "[146 rows x 6 columns]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - " feature attribution\n", - "0 sex 221.587592\n", - "1 flipper_length_mm 71.311846\n", - "2 culmen_depth_mm 66.17986\n", - "3 culmen_length_mm 45.443363\n", - "4 island 17.258076\n", - "\n", - "[5 rows x 2 columns]" + "source": [ + "# Drop rows with nulls to get training data\n", + "training_data = adelie_data.dropna()\n", + "\n", + "# Take a peek at the training data\n", + "training_data" ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "bbq.ml.global_explain(model_name)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Compatibility with pandas\n", - "\n", - "The functions in `bigframes.bigquery.ml` can accept pandas DataFrames as well. Use the `to_pandas()` method on the results of methods like `predict()` to get a pandas DataFrame back." - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ + }, { - "data": { - "text/html": [ - "\n", - " Query started with request ID bigframes-dev:US.18d9027b-7d55-42c9-ad1b-dabccdda80dc.
    SQL
    SELECT * FROM ML.PREDICT(MODEL `bigframes-dev.bqml_tutorial.penguin_weight_with_global_explain`, (SELECT\n",
-       "`column_0` AS `sex`,\n",
-       "`column_1` AS `flipper_length_mm`,\n",
-       "`column_2` AS `culmen_depth_mm`,\n",
-       "`column_3` AS `culmen_length_mm`,\n",
-       "`column_4` AS `island`\n",
-       "FROM\n",
-       "(SELECT\n",
-       "  *\n",
-       "FROM (\n",
-       "  SELECT\n",
-       "    *\n",
-       "  FROM UNNEST(ARRAY<STRUCT<`column_0` STRING, `column_1` INT64, `column_2` INT64, `column_3` INT64, `column_4` STRING>>[STRUCT('MALE', 180, 15, 40, 'Biscoe'), STRUCT('FEMALE', 190, 16, 41, 'Biscoe'), STRUCT('MALE', 200, 17, 42, 'Dream'), STRUCT('FEMALE', 210, 18, 43, 'Dream')]) AS `column_0`\n",
-       ") AS `t0`)))\n",
-       "
    \n", - " " + "cell_type": "markdown", + "metadata": { + "id": "Fx4lsNqMorJ-" + }, + "source": [ + "## Create the linear regression model\n", + "\n", + "In this notebook, you create a linear regression model, a type of regression model that generates a continuous value from a linear combination of input features." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create a BigQuery dataset to house the model, adding a name for your dataset as the `DATASET_ID` variable:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataset bqml_tutorial created.\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "DATASET_ID = \"bqml_tutorial\" # @param {type:\"string\"}\n", + "\n", + "from google.cloud import bigquery\n", + "client = bigquery.Client(project=PROJECT_ID)\n", + "dataset = bigquery.Dataset(PROJECT_ID + \".\" + DATASET_ID)\n", + "dataset.location = REGION\n", + "dataset = client.create_dataset(dataset, exists_ok=True)\n", + "print(f\"Dataset {dataset.dataset_id} created.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EloGtMnverFF" + }, + "source": [ + "### Create the model using `bigframes.bigquery.ml.create_model`\n", + "\n", + "When you pass the feature columns without transforms, BigQuery ML uses\n", + "[automatic preprocessing](https://cloud.google.com/bigquery/docs/auto-preprocessing) to encode string values and scale numeric values.\n", + "\n", + "BigQuery ML also [automatically splits the data for training and evaluation](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-create-glm#data_split_method), although for datasets with less than 500 rows (such as this one), all rows are used for training." ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " + "cell_type": "code", + "execution_count": 15, + "metadata": { + "id": "GskyyUQPowBT" + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " Query started with request ID bigframes-dev:US.a33b3628-730b-46e8-ad17-c78bb48619ce.
    SQL
    CREATE OR REPLACE MODEL `bigframes-dev.bqml_tutorial.penguin_weight`\n",
+              "OPTIONS(model_type = 'LINEAR_REG')\n",
+              "AS SELECT\n",
+              "`bfuid_col_3` AS `island`,\n",
+              "`bfuid_col_4` AS `culmen_length_mm`,\n",
+              "`bfuid_col_5` AS `culmen_depth_mm`,\n",
+              "`bfuid_col_6` AS `flipper_length_mm`,\n",
+              "`bfuid_col_7` AS `label`,\n",
+              "`bfuid_col_8` AS `sex`\n",
+              "FROM\n",
+              "(SELECT\n",
+              "  `t0`.`bfuid_col_3`,\n",
+              "  `t0`.`bfuid_col_4`,\n",
+              "  `t0`.`bfuid_col_5`,\n",
+              "  `t0`.`bfuid_col_6`,\n",
+              "  `t0`.`bfuid_col_7`,\n",
+              "  `t0`.`bfuid_col_8`\n",
+              "FROM `bigframes-dev._63cfa399614a54153cc386c27d6c0c6fdb249f9e._e154f0aa_5b29_492a_b464_a77c5f5a3dbd_bqdf_60fa3196-5a3e-45ae-898e-c2b473bfa1e9` AS `t0`)\n",
+              "
    \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "object", + "type": "string" + }, + { + "name": "0", + "rawType": "object", + "type": "unknown" + } + ], + "ref": "851c170c-08a5-4c06-8c0b-4547dbde3f18", + "rows": [ + [ + "etag", + "P3XS+g0ZZM19ywL+hdwUmQ==" + ], + [ + "modelReference", + "{'projectId': 'bigframes-dev', 'datasetId': 'bqml_tutorial', 'modelId': 'penguin_weight'}" + ], + [ + "creationTime", + "1764779445166" + ], + [ + "lastModifiedTime", + "1764779445237" + ], + [ + "modelType", + "LINEAR_REGRESSION" + ], + [ + "trainingRuns", + "[{'trainingOptions': {'lossType': 'MEAN_SQUARED_LOSS', 'l2Regularization': 0, 'inputLabelColumns': ['label'], 'dataSplitMethod': 'AUTO_SPLIT', 'optimizationStrategy': 'NORMAL_EQUATION', 'calculatePValues': False, 'enableGlobalExplain': False, 'categoryEncodingMethod': 'ONE_HOT_ENCODING', 'fitIntercept': True, 'standardizeFeatures': True}, 'trainingStartTime': '1764779429690', 'results': [{'index': 0, 'durationMs': '3104', 'trainingLoss': 78553.60163372214}], 'evaluationMetrics': {'regressionMetrics': {'meanAbsoluteError': 223.87876300779865, 'meanSquaredError': 78553.60163372215, 'meanSquaredLogError': 0.005614202871872688, 'medianAbsoluteError': 181.33091105963013, 'rSquared': 0.6239507555914934}}, 'startTime': '2025-12-03T16:30:29.690Z'}]" + ], + [ + "featureColumns", + "[{'name': 'island', 'type': {'typeKind': 'STRING'}}, {'name': 'culmen_length_mm', 'type': {'typeKind': 'FLOAT64'}}, {'name': 'culmen_depth_mm', 'type': {'typeKind': 'FLOAT64'}}, {'name': 'flipper_length_mm', 'type': {'typeKind': 'FLOAT64'}}, {'name': 'sex', 'type': {'typeKind': 'STRING'}}]" + ], + [ + "labelColumns", + "[{'name': 'predicted_label', 'type': {'typeKind': 'FLOAT64'}}]" + ], + [ + "location", + "US" + ] + ], + "shape": { + "columns": 1, + "rows": 9 + } + }, + "text/plain": [ + "etag P3XS+g0ZZM19ywL+hdwUmQ==\n", + "modelReference {'projectId': 'bigframes-dev', 'datasetId': 'b...\n", + "creationTime 1764779445166\n", + "lastModifiedTime 1764779445237\n", + "modelType LINEAR_REGRESSION\n", + "trainingRuns [{'trainingOptions': {'lossType': 'MEAN_SQUARE...\n", + "featureColumns [{'name': 'island', 'type': {'typeKind': 'STRI...\n", + "labelColumns [{'name': 'predicted_label', 'type': {'typeKin...\n", + "location US\n", + "dtype: object" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "import bigframes.bigquery as bbq\n", + "\n", + "model_name = f\"{PROJECT_ID}.{DATASET_ID}.penguin_weight\"\n", + "model_metadata = bbq.ml.create_model(\n", + " model_name,\n", + " replace=True,\n", + " options={\n", + " \"model_type\": \"LINEAR_REG\",\n", + " },\n", + " training_data=training_data.rename(columns={\"body_mass_g\": \"label\"})\n", + ")\n", + "model_metadata" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ + "cell_type": "markdown", + "metadata": { + "id": "GskyyUQPowBT" + }, + "source": [ + "### Evaluate the model\n", + "\n", + "Check how the model performed by using the `evalutate` function. More information on model evaluation can be found [here](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-evaluate#mlevaluate_output)." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "id": "kGBJKafpo0dl" + }, + "outputs": [ { - "name": "index", - "rawType": "Int64", - "type": "integer" + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 0 Bytes in a moment of slot time.\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { - "name": "predicted_body_mass_g", - "rawType": "Float64", - "type": "float" + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { - "name": "sex", - "rawType": "string", - "type": "string" + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { - "name": "flipper_length_mm", - "rawType": "Int64", - "type": "integer" + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    mean_absolute_errormean_squared_errormean_squared_log_errormedian_absolute_errorr2_scoreexplained_variance
    0223.87876378553.6016340.005614181.3309110.6239510.623951
    \n", + "

    1 rows × 6 columns

    \n", + "
    [1 rows x 6 columns in total]" + ], + "text/plain": [ + " mean_absolute_error mean_squared_error mean_squared_log_error \\\n", + "0 223.878763 78553.601634 0.005614 \n", + "\n", + " median_absolute_error r2_score explained_variance \n", + "0 181.330911 0.623951 0.623951 \n", + "\n", + "[1 rows x 6 columns]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bbq.ml.evaluate(model_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P2lUiZZ_cjri" + }, + "source": [ + "### Use the model to predict outcomes\n", + "\n", + "Now that you have evaluated your model, the next step is to use it to predict an\n", + "outcome. You can run `bigframes.bigquery.ml.predict` function on the model to\n", + "predict the body mass in grams of all penguins that reside on the Biscoe\n", + "Islands." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "id": "bsQ9cmoWo0Ps" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/swast/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py:182: TimeTravelCacheWarning: Reading cached table from 2025-12-03 16:30:18.272882+00:00 to avoid\n", + "incompatibilies with previous reads of this table. To read the latest\n", + "version, set `use_cache=False` or close the current session with\n", + "Session.close() or bigframes.pandas.close_session().\n", + " return method(*args, **kwargs)\n" + ] }, { - "name": "culmen_depth_mm", - "rawType": "Int64", - "type": "integer" + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 29.3 kB in a moment of slot time.\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { - "name": "culmen_length_mm", - "rawType": "Int64", - "type": "integer" + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { - "name": "island", - "rawType": "string", - "type": "string" + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    predicted_labelspeciesislandculmen_length_mmculmen_depth_mmflipper_length_mmbody_mass_gsex
    03945.010052Gentoo penguin (Pygoscelis papua)Biscoe<NA><NA><NA><NA><NA>
    13914.916297Adelie Penguin (Pygoscelis adeliae)Biscoe39.718.9184.03550.0MALE
    23278.611224Adelie Penguin (Pygoscelis adeliae)Biscoe36.417.1184.02850.0FEMALE
    34006.367355Adelie Penguin (Pygoscelis adeliae)Biscoe41.618.0192.03950.0MALE
    43417.610478Adelie Penguin (Pygoscelis adeliae)Biscoe35.017.9192.03725.0FEMALE
    54009.612421Adelie Penguin (Pygoscelis adeliae)Biscoe41.118.2192.04050.0MALE
    64231.330911Adelie Penguin (Pygoscelis adeliae)Biscoe42.019.5200.04050.0MALE
    73554.308906Gentoo penguin (Pygoscelis papua)Biscoe43.813.9208.04300.0FEMALE
    83550.677455Gentoo penguin (Pygoscelis papua)Biscoe43.314.0208.04575.0FEMALE
    93537.882543Gentoo penguin (Pygoscelis papua)Biscoe44.013.6208.04350.0FEMALE
    \n", + "

    10 rows × 8 columns

    \n", + "
    [168 rows x 8 columns in total]" + ], + "text/plain": [ + " predicted_label species island \\\n", + "0 3945.010052 Gentoo penguin (Pygoscelis papua) Biscoe \n", + "1 3914.916297 Adelie Penguin (Pygoscelis adeliae) Biscoe \n", + "2 3278.611224 Adelie Penguin (Pygoscelis adeliae) Biscoe \n", + "3 4006.367355 Adelie Penguin (Pygoscelis adeliae) Biscoe \n", + "4 3417.610478 Adelie Penguin (Pygoscelis adeliae) Biscoe \n", + "5 4009.612421 Adelie Penguin (Pygoscelis adeliae) Biscoe \n", + "6 4231.330911 Adelie Penguin (Pygoscelis adeliae) Biscoe \n", + "7 3554.308906 Gentoo penguin (Pygoscelis papua) Biscoe \n", + "8 3550.677455 Gentoo penguin (Pygoscelis papua) Biscoe \n", + "9 3537.882543 Gentoo penguin (Pygoscelis papua) Biscoe \n", + "\n", + " culmen_length_mm culmen_depth_mm flipper_length_mm body_mass_g sex \n", + "0 \n", + "1 39.7 18.9 184.0 3550.0 MALE \n", + "2 36.4 17.1 184.0 2850.0 FEMALE \n", + "3 41.6 18.0 192.0 3950.0 MALE \n", + "4 35.0 17.9 192.0 3725.0 FEMALE \n", + "5 41.1 18.2 192.0 4050.0 MALE \n", + "6 42.0 19.5 200.0 4050.0 MALE \n", + "7 43.8 13.9 208.0 4300.0 FEMALE \n", + "8 43.3 14.0 208.0 4575.0 FEMALE \n", + "9 44.0 13.6 208.0 4350.0 FEMALE \n", + "...\n", + "\n", + "[168 rows x 8 columns]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" } - ], - "ref": "01d67015-64b6-463e-8c16-e8ac1363ff67", - "rows": [ - [ - "0", - "3596.332210728767", - "MALE", - "180", - "15", - "40", - "Biscoe" - ], - [ - "1", - "3384.6999176328636", - "FEMALE", - "190", - "16", - "41", - "Biscoe" - ], - [ - "2", - "4049.581795919061", - "MALE", - "200", - "17", - "42", - "Dream" - ], - [ - "3", - "3837.9495028231568", - "FEMALE", - "210", - "18", - "43", - "Dream" - ] - ], - "shape": { - "columns": 6, - "rows": 4 - } + ], + "source": [ + "df = bpd.read_gbq(\"bigquery-public-data.ml_datasets.penguins\")\n", + "biscoe = df[df[\"island\"].str.contains(\"Biscoe\")]\n", + "bbq.ml.predict(model_name, biscoe)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GTRdUw-Ro5R1" }, - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    predicted_body_mass_gsexflipper_length_mmculmen_depth_mmculmen_length_mmisland
    03596.332211MALE1801540Biscoe
    13384.699918FEMALE1901641Biscoe
    24049.581796MALE2001742Dream
    33837.949503FEMALE2101843Dream
    \n", - "
    " + "source": [ + "### Explain the prediction results\n", + "\n", + "To understand why the model is generating these prediction results, you can use the `explain_predict` function." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " Query started with request ID bigframes-dev:US.161bba69-c852-4916-a2df-bb5b309be6e4.
    SQL
    SELECT * FROM ML.EXPLAIN_PREDICT(MODEL `bigframes-dev.bqml_tutorial.penguin_weight`, (SELECT\n",
+              "`bfuid_col_22` AS `species`,\n",
+              "`bfuid_col_23` AS `island`,\n",
+              "`bfuid_col_24` AS `culmen_length_mm`,\n",
+              "`bfuid_col_25` AS `culmen_depth_mm`,\n",
+              "`bfuid_col_26` AS `flipper_length_mm`,\n",
+              "`bfuid_col_27` AS `body_mass_g`,\n",
+              "`bfuid_col_28` AS `sex`\n",
+              "FROM\n",
+              "(SELECT\n",
+              "  `t0`.`species`,\n",
+              "  `t0`.`island`,\n",
+              "  `t0`.`culmen_length_mm`,\n",
+              "  `t0`.`culmen_depth_mm`,\n",
+              "  `t0`.`flipper_length_mm`,\n",
+              "  `t0`.`body_mass_g`,\n",
+              "  `t0`.`sex`,\n",
+              "  `t0`.`species` AS `bfuid_col_22`,\n",
+              "  `t0`.`island` AS `bfuid_col_23`,\n",
+              "  `t0`.`culmen_length_mm` AS `bfuid_col_24`,\n",
+              "  `t0`.`culmen_depth_mm` AS `bfuid_col_25`,\n",
+              "  `t0`.`flipper_length_mm` AS `bfuid_col_26`,\n",
+              "  `t0`.`body_mass_g` AS `bfuid_col_27`,\n",
+              "  `t0`.`sex` AS `bfuid_col_28`,\n",
+              "  regexp_contains(`t0`.`island`, 'Biscoe') AS `bfuid_col_29`\n",
+              "FROM (\n",
+              "  SELECT\n",
+              "    `species`,\n",
+              "    `island`,\n",
+              "    `culmen_length_mm`,\n",
+              "    `culmen_depth_mm`,\n",
+              "    `flipper_length_mm`,\n",
+              "    `body_mass_g`,\n",
+              "    `sex`\n",
+              "  FROM `bigquery-public-data.ml_datasets.penguins` FOR SYSTEM_TIME AS OF TIMESTAMP('2025-12-03T16:30:18.272882+00:00')\n",
+              ") AS `t0`\n",
+              "WHERE\n",
+              "  regexp_contains(`t0`.`island`, 'Biscoe'))), STRUCT(3 AS top_k_features))\n",
+              "
    \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    predicted_labeltop_feature_attributionsbaseline_prediction_valueprediction_valueapproximation_errorspeciesislandculmen_length_mmculmen_depth_mmflipper_length_mmbody_mass_gsex
    03945.010052[{'feature': 'island', 'attribution': 0.0}\n", + " {'...3945.0100523945.0100520.0Gentoo penguin (Pygoscelis papua)Biscoe<NA><NA><NA><NA><NA>
    13914.916297[{'feature': 'flipper_length_mm', 'attribution...3945.0100523914.9162970.0Adelie Penguin (Pygoscelis adeliae)Biscoe39.718.9184.03550.0MALE
    23278.611224[{'feature': 'sex', 'attribution': -443.175184...3945.0100523278.6112240.0Adelie Penguin (Pygoscelis adeliae)Biscoe36.417.1184.02850.0FEMALE
    34006.367355[{'feature': 'culmen_length_mm', 'attribution'...3945.0100524006.3673550.0Adelie Penguin (Pygoscelis adeliae)Biscoe41.618.0192.03950.0MALE
    43417.610478[{'feature': 'sex', 'attribution': -443.175184...3945.0100523417.6104780.0Adelie Penguin (Pygoscelis adeliae)Biscoe35.017.9192.03725.0FEMALE
    54009.612421[{'feature': 'culmen_length_mm', 'attribution'...3945.0100524009.6124210.0Adelie Penguin (Pygoscelis adeliae)Biscoe41.118.2192.04050.0MALE
    64231.330911[{'feature': 'flipper_length_mm', 'attribution...3945.0100524231.3309110.0Adelie Penguin (Pygoscelis adeliae)Biscoe42.019.5200.04050.0MALE
    73554.308906[{'feature': 'sex', 'attribution': -443.175184...3945.0100523554.3089060.0Gentoo penguin (Pygoscelis papua)Biscoe43.813.9208.04300.0FEMALE
    83550.677455[{'feature': 'sex', 'attribution': -443.175184...3945.0100523550.6774550.0Gentoo penguin (Pygoscelis papua)Biscoe43.314.0208.04575.0FEMALE
    93537.882543[{'feature': 'sex', 'attribution': -443.175184...3945.0100523537.8825430.0Gentoo penguin (Pygoscelis papua)Biscoe44.013.6208.04350.0FEMALE
    \n", + "

    10 rows × 12 columns

    \n", + "
    [168 rows x 12 columns in total]" + ], + "text/plain": [ + " predicted_label top_feature_attributions \\\n", + "0 3945.010052 [{'feature': 'island', 'attribution': 0.0}\n", + " {'... \n", + "1 3914.916297 [{'feature': 'flipper_length_mm', 'attribution... \n", + "2 3278.611224 [{'feature': 'sex', 'attribution': -443.175184... \n", + "3 4006.367355 [{'feature': 'culmen_length_mm', 'attribution'... \n", + "4 3417.610478 [{'feature': 'sex', 'attribution': -443.175184... \n", + "5 4009.612421 [{'feature': 'culmen_length_mm', 'attribution'... \n", + "6 4231.330911 [{'feature': 'flipper_length_mm', 'attribution... \n", + "7 3554.308906 [{'feature': 'sex', 'attribution': -443.175184... \n", + "8 3550.677455 [{'feature': 'sex', 'attribution': -443.175184... \n", + "9 3537.882543 [{'feature': 'sex', 'attribution': -443.175184... \n", + "\n", + " baseline_prediction_value prediction_value approximation_error \\\n", + "0 3945.010052 3945.010052 0.0 \n", + "1 3945.010052 3914.916297 0.0 \n", + "2 3945.010052 3278.611224 0.0 \n", + "3 3945.010052 4006.367355 0.0 \n", + "4 3945.010052 3417.610478 0.0 \n", + "5 3945.010052 4009.612421 0.0 \n", + "6 3945.010052 4231.330911 0.0 \n", + "7 3945.010052 3554.308906 0.0 \n", + "8 3945.010052 3550.677455 0.0 \n", + "9 3945.010052 3537.882543 0.0 \n", + "\n", + " species island culmen_length_mm \\\n", + "0 Gentoo penguin (Pygoscelis papua) Biscoe \n", + "1 Adelie Penguin (Pygoscelis adeliae) Biscoe 39.7 \n", + "2 Adelie Penguin (Pygoscelis adeliae) Biscoe 36.4 \n", + "3 Adelie Penguin (Pygoscelis adeliae) Biscoe 41.6 \n", + "4 Adelie Penguin (Pygoscelis adeliae) Biscoe 35.0 \n", + "5 Adelie Penguin (Pygoscelis adeliae) Biscoe 41.1 \n", + "6 Adelie Penguin (Pygoscelis adeliae) Biscoe 42.0 \n", + "7 Gentoo penguin (Pygoscelis papua) Biscoe 43.8 \n", + "8 Gentoo penguin (Pygoscelis papua) Biscoe 43.3 \n", + "9 Gentoo penguin (Pygoscelis papua) Biscoe 44.0 \n", + "\n", + " culmen_depth_mm flipper_length_mm body_mass_g sex \n", + "0 \n", + "1 18.9 184.0 3550.0 MALE \n", + "2 17.1 184.0 2850.0 FEMALE \n", + "3 18.0 192.0 3950.0 MALE \n", + "4 17.9 192.0 3725.0 FEMALE \n", + "5 18.2 192.0 4050.0 MALE \n", + "6 19.5 200.0 4050.0 MALE \n", + "7 13.9 208.0 4300.0 FEMALE \n", + "8 14.0 208.0 4575.0 FEMALE \n", + "9 13.6 208.0 4350.0 FEMALE \n", + "...\n", + "\n", + "[168 rows x 12 columns]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - " predicted_body_mass_g sex flipper_length_mm culmen_depth_mm \\\n", - "0 3596.332211 MALE 180 15 \n", - "1 3384.699918 FEMALE 190 16 \n", - "2 4049.581796 MALE 200 17 \n", - "3 3837.949503 FEMALE 210 18 \n", - "\n", - " culmen_length_mm island \n", - "0 40 Biscoe \n", - "1 41 Biscoe \n", - "2 42 Dream \n", - "3 43 Dream " + "source": [ + "bbq.ml.explain_predict(model_name, biscoe, top_k_features=3)" ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "predict_df = pd.DataFrame({\n", - " \"sex\": [\"MALE\", \"FEMALE\", \"MALE\", \"FEMALE\"],\n", - " \"flipper_length_mm\": [180, 190, 200, 210],\n", - " \"culmen_depth_mm\": [15, 16, 17, 18],\n", - " \"culmen_length_mm\": [40, 41, 42, 43],\n", - " \"island\": [\"Biscoe\", \"Biscoe\", \"Dream\", \"Dream\"],\n", - "})\n", - "bbq.ml.predict(model_metadata, predict_df).to_pandas()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Compatibility with `bigframes.ml`\n", - "\n", - "The models created with `bigframes.bigquery.ml` can be used with the scikit-learn-like `bigframes.ml` modules by using the `read_gbq_model` method.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ + }, + { + "cell_type": "markdown", + "metadata": { + "id": "K0mPaoGpcwwy" + }, + "source": [] + }, { - "data": { - "text/plain": [ - "LinearRegression(enable_global_explain=True,\n", - " optimize_strategy='NORMAL_EQUATION')" + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Globally explain the model\n", + "\n", + "To know which features are generally the most important to determine penguin\n", + "weight, you can use the `global_explain` function. In order to use\n", + "`global_explain`, you must retrain the model with the `enable_global_explain`\n", + "option set to `True`." ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model = bpd.read_gbq_model(model_name)\n", - "model" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ + }, { - "data": { - "text/html": [ - "✅ Completed. \n", - " Query processed 7.3 kB in a moment of slot time. [Job bigframes-dev:US.f2f86927-bbd1-431d-b89e-3d6a064268d7 details]\n", - " " + "cell_type": "code", + "execution_count": 19, + "metadata": { + "id": "ZSP7gt13QrQt" + }, + "outputs": [ + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 6.9 kB in 53 seconds of slot time. [Job bigframes-dev:US.job_welN8ErlZ_sTG7oOEULsWUgmIg7l details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } ], - "text/plain": [ - "" + "source": [ + "model_name = f\"{PROJECT_ID}.{DATASET_ID}.penguin_weight_with_global_explain\"\n", + "model_metadata = bbq.ml.create_model(\n", + " model_name,\n", + " replace=True,\n", + " options={\n", + " \"model_type\": \"LINEAR_REG\",\n", + " \"input_label_cols\": [\"body_mass_g\"],\n", + " \"enable_global_explain\": True,\n", + " },\n", + " training_data=training_data,\n", + ")" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 0 Bytes in a moment of slot time.\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    featureattribution
    0sex221.587592
    1flipper_length_mm71.311846
    2culmen_depth_mm66.17986
    3culmen_length_mm45.443363
    4island17.258076
    \n", + "

    5 rows × 2 columns

    \n", + "
    [5 rows x 2 columns in total]" + ], + "text/plain": [ + " feature attribution\n", + "0 sex 221.587592\n", + "1 flipper_length_mm 71.311846\n", + "2 culmen_depth_mm 66.17986\n", + "3 culmen_length_mm 45.443363\n", + "4 island 17.258076\n", + "\n", + "[5 rows x 2 columns]" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "bbq.ml.global_explain(model_name)" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "✅ Completed. " + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compatibility with pandas\n", + "\n", + "The functions in `bigframes.bigquery.ml` can accept pandas DataFrames as well. Use the `to_pandas()` method on the results of methods like `predict()` to get a pandas DataFrame back." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " Query started with request ID bigframes-dev:US.18d9027b-7d55-42c9-ad1b-dabccdda80dc.
    SQL
    SELECT * FROM ML.PREDICT(MODEL `bigframes-dev.bqml_tutorial.penguin_weight_with_global_explain`, (SELECT\n",
+              "`column_0` AS `sex`,\n",
+              "`column_1` AS `flipper_length_mm`,\n",
+              "`column_2` AS `culmen_depth_mm`,\n",
+              "`column_3` AS `culmen_length_mm`,\n",
+              "`column_4` AS `island`\n",
+              "FROM\n",
+              "(SELECT\n",
+              "  *\n",
+              "FROM (\n",
+              "  SELECT\n",
+              "    *\n",
+              "  FROM UNNEST(ARRAY<STRUCT<`column_0` STRING, `column_1` INT64, `column_2` INT64, `column_3` INT64, `column_4` STRING>>[STRUCT('MALE', 180, 15, 40, 'Biscoe'), STRUCT('FEMALE', 190, 16, 41, 'Biscoe'), STRUCT('MALE', 200, 17, 42, 'Dream'), STRUCT('FEMALE', 210, 18, 43, 'Dream')]) AS `column_0`\n",
+              ") AS `t0`)))\n",
+              "
    \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "Int64", + "type": "integer" + }, + { + "name": "predicted_body_mass_g", + "rawType": "Float64", + "type": "float" + }, + { + "name": "sex", + "rawType": "string", + "type": "string" + }, + { + "name": "flipper_length_mm", + "rawType": "Int64", + "type": "integer" + }, + { + "name": "culmen_depth_mm", + "rawType": "Int64", + "type": "integer" + }, + { + "name": "culmen_length_mm", + "rawType": "Int64", + "type": "integer" + }, + { + "name": "island", + "rawType": "string", + "type": "string" + } + ], + "ref": "01d67015-64b6-463e-8c16-e8ac1363ff67", + "rows": [ + [ + "0", + "3596.332210728767", + "MALE", + "180", + "15", + "40", + "Biscoe" + ], + [ + "1", + "3384.6999176328636", + "FEMALE", + "190", + "16", + "41", + "Biscoe" + ], + [ + "2", + "4049.581795919061", + "MALE", + "200", + "17", + "42", + "Dream" + ], + [ + "3", + "3837.9495028231568", + "FEMALE", + "210", + "18", + "43", + "Dream" + ] + ], + "shape": { + "columns": 6, + "rows": 4 + } + }, + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    predicted_body_mass_gsexflipper_length_mmculmen_depth_mmculmen_length_mmisland
    03596.332211MALE1801540Biscoe
    13384.699918FEMALE1901641Biscoe
    24049.581796MALE2001742Dream
    33837.949503FEMALE2101843Dream
    \n", + "
    " + ], + "text/plain": [ + " predicted_body_mass_g sex flipper_length_mm culmen_depth_mm \\\n", + "0 3596.332211 MALE 180 15 \n", + "1 3384.699918 FEMALE 190 16 \n", + "2 4049.581796 MALE 200 17 \n", + "3 3837.949503 FEMALE 210 18 \n", + "\n", + " culmen_length_mm island \n", + "0 40 Biscoe \n", + "1 41 Biscoe \n", + "2 42 Dream \n", + "3 43 Dream " + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "import pandas as pd\n", + "\n", + "predict_df = pd.DataFrame({\n", + " \"sex\": [\"MALE\", \"FEMALE\", \"MALE\", \"FEMALE\"],\n", + " \"flipper_length_mm\": [180, 190, 200, 210],\n", + " \"culmen_depth_mm\": [15, 16, 17, 18],\n", + " \"culmen_length_mm\": [40, 41, 42, 43],\n", + " \"island\": [\"Biscoe\", \"Biscoe\", \"Dream\", \"Dream\"],\n", + "})\n", + "bbq.ml.predict(model_metadata, predict_df).to_pandas()" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    mean_absolute_errormean_squared_errormean_squared_log_errormedian_absolute_errorr2_scoreexplained_variance
    0223.87876378553.6016340.005614181.3309110.6239510.623951
    \n", - "

    1 rows × 6 columns

    \n", - "
    [1 rows x 6 columns in total]" + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Compatibility with `bigframes.ml`\n", + "\n", + "The models created with `bigframes.bigquery.ml` can be used with the scikit-learn-like `bigframes.ml` modules by using the `read_gbq_model` method.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LinearRegression(enable_global_explain=True,\n", + " optimize_strategy='NORMAL_EQUATION')" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - " mean_absolute_error mean_squared_error mean_squared_log_error \\\n", - " 223.878763 78553.601634 0.005614 \n", - "\n", - " median_absolute_error r2_score explained_variance \n", - " 181.330911 0.623951 0.623951 \n", - "\n", - "[1 rows x 6 columns]" + "source": [ + "model = bpd.read_gbq_model(model_name)\n", + "model" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 7.3 kB in a moment of slot time. [Job bigframes-dev:US.f2f86927-bbd1-431d-b89e-3d6a064268d7 details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    mean_absolute_errormean_squared_errormean_squared_log_errormedian_absolute_errorr2_scoreexplained_variance
    0223.87876378553.6016340.005614181.3309110.6239510.623951
    \n", + "

    1 rows × 6 columns

    \n", + "
    [1 rows x 6 columns in total]" + ], + "text/plain": [ + " mean_absolute_error mean_squared_error mean_squared_log_error \\\n", + " 223.878763 78553.601634 0.005614 \n", + "\n", + " median_absolute_error r2_score explained_variance \n", + " 181.330911 0.623951 0.623951 \n", + "\n", + "[1 rows x 6 columns]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = training_data[[\"sex\", \"flipper_length_mm\", \"culmen_depth_mm\", \"culmen_length_mm\", \"island\"]]\n", + "y = training_data[[\"body_mass_g\"]]\n", + "model.score(X, y)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "G_wjSfXpWTuy" + }, + "source": [ + "# Summary and next steps\n", + "\n", + "You've created a linear regression model using `bigframes.bigquery.ml`.\n", + "\n", + "Learn more about BigQuery DataFrames in the [documentation](https://dataframes.bigquery.dev/) and find more sample notebooks in the [GitHub repo](https://github.com/googleapis/python-bigquery-dataframes/tree/main/notebooks)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TpV-iwP9qw9c" + }, + "source": [ + "## Cleaning up\n", + "\n", + "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n", + "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", + "\n", + "Otherwise, you can uncomment the remaining cells and run them to delete the individual resources you created in this tutorial:" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "id": "sx_vKniMq9ZX" + }, + "outputs": [], + "source": [ + "# # Delete the BigQuery dataset and associated ML model\n", + "# from google.cloud import bigquery\n", + "# client = bigquery.Client(project=PROJECT_ID)\n", + "# client.delete_dataset(\n", + "# DATASET_ID, delete_contents=True, not_found_ok=True\n", + "# )\n", + "# print(\"Deleted dataset '{}'.\".format(DATASET_ID))" ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" } - ], - "source": [ - "X = training_data[[\"sex\", \"flipper_length_mm\", \"culmen_depth_mm\", \"culmen_length_mm\", \"island\"]]\n", - "y = training_data[[\"body_mass_g\"]]\n", - "model.score(X, y)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "G_wjSfXpWTuy" - }, - "source": [ - "# Summary and next steps\n", - "\n", - "You've created a linear regression model using `bigframes.bigquery.ml`.\n", - "\n", - "Learn more about BigQuery DataFrames in the [documentation](https://dataframes.bigquery.dev/) and find more sample notebooks in the [GitHub repo](https://github.com/googleapis/python-bigquery-dataframes/tree/main/notebooks)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TpV-iwP9qw9c" - }, - "source": [ - "## Cleaning up\n", - "\n", - "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n", - "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", - "\n", - "Otherwise, you can uncomment the remaining cells and run them to delete the individual resources you created in this tutorial:" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "id": "sx_vKniMq9ZX" - }, - "outputs": [], - "source": [ - "# # Delete the BigQuery dataset and associated ML model\n", - "# from google.cloud import bigquery\n", - "# client = bigquery.Client(project=PROJECT_ID)\n", - "# client.delete_dataset(\n", - "# DATASET_ID, delete_contents=True, not_found_ok=True\n", - "# )\n", - "# print(\"Deleted dataset '{}'.\".format(DATASET_ID))" - ] - } - ], - "metadata": { - "colab": { - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.9" + } }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.9" - } - }, - "nbformat": 4, - "nbformat_minor": 0 + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/notebooks/ml/bq_dataframes_ml_linear_regression_big.ipynb b/notebooks/ml/bq_dataframes_ml_linear_regression_big.ipynb index d286f5ce31d..5c016f9157d 100644 --- a/notebooks/ml/bq_dataframes_ml_linear_regression_big.ipynb +++ b/notebooks/ml/bq_dataframes_ml_linear_regression_big.ipynb @@ -1,1064 +1,1064 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ur8xi4C7S06n" - }, - "outputs": [], - "source": [ - "# Copyright 2025 Google LLC\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JAPoU8Sm5E6e" - }, - "source": [ - "# Train a linear regression model with BigQuery DataFrames ML", - "\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \"Colab Run in Colab\n", - " \n", - " \n", - " \n", - " \"GitHub\n", - " View on GitHub\n", - " \n", - " \n", - " \n", - " \"Vertex\n", - " Open in Vertex AI Workbench\n", - " \n", - " \n", - " \n", - " \"BQ\n", - " Open in BQ Studio\n", - " \n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "24743cf4a1e1" - }, - "source": [ - "**_NOTE_**: This notebook has been tested in the following environment:\n", - "\n", - "* Python version = 3.11" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tvgnzT1CKxrO" - }, - "source": [ - "## Overview\n", - "\n", - "This notebook demonstrates training a linear regression model on Big Data using BigQuery DataFrames ML. BigQuery DataFrames ML provides a provides a scikit-learn-like API for ML powered by the BigQuery engine.\n", - "\n", - "Learn more about [BigQuery DataFrames](https://cloud.google.com/python/docs/reference/bigframes/latest)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "d975e698c9a4" - }, - "source": [ - "### Objective\n", - "\n", - "In this tutorial, we use BigQuery DataFrames to create a linear regression model that predicts the levels of Ozone in the atmosphere.\n", - "\n", - "The steps include:\n", - "\n", - "- Creating a DataFrame from the BigQuery table.\n", - "- Cleaning and preparing data using `bigframes.pandas` module.\n", - "- Creating a linear regression model using `bigframes.ml` module.\n", - "- Saving the ML model to BigQuery for future use.\n", - "\n", - "\n", - "Let's formally define our problem as: **Train a linear regression model to predict the level of ozone in the atmosphere given the measurements of other constituents and properties of the atmosphere.**" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "08d289fa873f" - }, - "source": [ - "### Dataset\n", - "\n", - "In this tutorial we are going to use the [`bigquery-public-data.epa_historical_air_quality`](https://console.cloud.google.com/marketplace/product/epa/historical-air-quality) dataset. To quote the description of the dataset:\n", - "\n", - "\"The United States Environmental Protection Agency (EPA) protects both public health and the environment by establishing the standards for national air quality. The EPA provides annual summary data as well as hourly and daily data in the categories of criteria gases, particulates, meteorological, and toxics.\"\n", - "\n", - "There are several tables capturing data about the constituents of the atmosphere, see them in the [BigQuery cloud console](https://pantheon.corp.google.com/bigquery?p=bigquery-public-data&d=epa_historical_air_quality&page=dataset). Most tables carry 10's of GBs of data, but that is not an issue with BigQuery DataFrames as the data is efficiently processed at BigQuery without transferring them to the client." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aed92deeb4a0" - }, - "source": [ - "### Costs\n", - "\n", - "This tutorial uses billable components of Google Cloud:\n", - "\n", - "* BigQuery (compute)\n", - "* BigQuery ML\n", - "\n", - "Learn about [BigQuery compute pricing](https://cloud.google.com/bigquery/pricing#analysis_pricing_models)\n", - "and [BigQuery ML pricing](https://cloud.google.com/bigquery/pricing#bqml),\n", - "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n", - "to generate a cost estimate based on your projected usage." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "i7EUnXsZhAGF" - }, - "source": [ - "## Installation\n", - "\n", - "If you don't have [bigframes](https://pypi.org/project/bigframes/) package already installed, uncomment and execute the following cells to\n", - "\n", - "1. Install the package\n", - "1. Restart the notebook kernel (Jupyter or Colab) to work with the package" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "9O0Ka4W2MNF3" - }, - "outputs": [], - "source": [ - "# !pip install bigframes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "f200f10a1da3" - }, - "outputs": [], - "source": [ - "# Automatically restart kernel after installs so that your environment can access the new packages\n", - "\n", - "# import IPython\n", - "#\n", - "# app = IPython.Application.instance()\n", - "# app.kernel.do_shutdown(True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BF1j6f9HApxa" - }, - "source": [ - "## Before you begin\n", - "\n", - "Complete the tasks in this section to set up your environment." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oDfTjfACBvJk" - }, - "source": [ - "### Set up your Google Cloud project\n", - "\n", - "**The following steps are required, regardless of your notebook environment.**\n", - "\n", - "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 credit towards your compute/storage costs.\n", - "\n", - "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", - "\n", - "3. [Enable the BigQuery API](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com).\n", - "\n", - "4. If you are running this notebook locally, install the [Cloud SDK](https://cloud.google.com/sdk)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "WReHDGG5g0XY" - }, - "source": [ - "#### Set your project ID\n", - "\n", - "If you don't know your project ID, try the following:\n", - "* Run `gcloud config list`.\n", - "* Run `gcloud projects list`.\n", - "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "oM1iC_MfAts1" - }, - "outputs": [], - "source": [ - "PROJECT_ID = \"\" # @param {type:\"string\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "region" - }, - "source": [ - "#### Set the BigQuery location\n", - "\n", - "You can also change the `LOCATION` variable used by BigQuery. Learn more about [BigQuery locations](https://cloud.google.com/bigquery/docs/locations#supported_locations)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "eF-Twtc4XGem" - }, - "outputs": [], - "source": [ - "LOCATION = \"US\" # @param {type: \"string\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sBCra4QMA2wR" - }, - "source": [ - "### Set up APIs, IAM permissions and Authentication\n", - "\n", - "Follow the instructions at https://cloud.google.com/bigquery/docs/use-bigquery-dataframes#permissions.\n", - "\n", - "Depending on your notebook environment, you might have to manually authenticate. Follow the relevant instructions below." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "74ccc9e52986" - }, - "source": [ - "**Vertex AI Workbench**\n", - "\n", - "Do nothing, you are already authenticated." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "de775a3773ba" - }, - "source": [ - "**Local JupyterLab instance**\n", - "\n", - "Uncomment and run the following cell:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "254614fa0c46" - }, - "outputs": [], - "source": [ - "# ! gcloud auth login\n", - "# ! gcloud auth application-default login" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ef21552ccea8" - }, - "source": [ - "**Colab**\n", - "\n", - "Uncomment and run the following cell:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "603adbbf0532" - }, - "outputs": [], - "source": [ - "# from google.colab import auth\n", - "# auth.authenticate_user()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "960505627ddf" - }, - "source": [ - "### Import libraries" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "PyQmSRbKA8r-" - }, - "outputs": [], - "source": [ - "import bigframes.pandas as bpd" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "init_aip:mbsdk,all" - }, - "source": [ - "### Set BigQuery DataFrames options" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NPPMuw2PXGeo" - }, - "outputs": [], - "source": [ - "# NOTE: The project option is not required in all environments.\n", - "# On BigQuery Studio, the project ID is automatically detected.\n", - "bpd.options.bigquery.project = PROJECT_ID\n", - "\n", - "# NOTE: The location option is not required.\n", - "# It defaults to the location of the first table or query\n", - "# passed to read_gbq(). For APIs where a location can't be\n", - "# auto-detected, the location defaults to the \"US\" location.\n", - "bpd.options.bigquery.location = LOCATION\n", - "\n", - "# NOTE: For a machine learning model the order of the data is\n", - "# not important. So let's relax the ordering_mode to accept\n", - "# partial ordering. This allows BigQuery DataFrames to run cost\n", - "# and performance optimized jobs at the BigQuery engine.\n", - "bpd.options.bigquery.ordering_mode = \"partial\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "D21CoOlfFTYI" - }, - "source": [ - "If you want to reset the location of the created DataFrame or Series objects, reset the session by executing `bpd.close_session()`. After that, you can reuse `bpd.options.bigquery.location` to specify another location." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9EMAqR37AfLS" - }, - "source": [ - "## Read data in BigQuery tables as DataFrame\n", - "\n", - "Let's read the tables in the dataset to construct a BigQuery DataFrames DataFrame. We will combine measurements of various parameters of the atmosphere from multiple tables to represent a consolidated dataframe to use for our model training and prediction. We have daily and hourly versions of the data available, but since we want to create a model that is dynamic so that it can capture the variance throughout the day, we would choose the hourly version.\n", - "\n", - "Note that we would use the pandas APIs as we normally would on the BigQuery DataFrames DataFrame, but calculations happen in the BigQuery query engine instead of the local environment." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataset = \"bigquery-public-data.epa_historical_air_quality\"\n", - "hourly_summary_tables = [\n", - " \"co_hourly_summary\",\n", - " \"hap_hourly_summary\",\n", - " \"no2_hourly_summary\",\n", - " \"nonoxnoy_hourly_summary\",\n", - " \"o3_hourly_summary\",\n", - " \"pm10_hourly_summary\",\n", - " \"pm25_frm_hourly_summary\",\n", - " \"pm25_nonfrm_hourly_summary\",\n", - " \"pm25_speciation_hourly_summary\",\n", - " \"pressure_hourly_summary\",\n", - " \"rh_and_dp_hourly_summary\",\n", - " \"so2_hourly_summary\",\n", - " \"temperature_hourly_summary\",\n", - " \"voc_hourly_summary\",\n", - " \"wind_hourly_summary\",\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's pick index columns - to identify a measurement of the atmospheric parameter, param column - to identify which param the measurement pertains to, and value column - the column containing the measurement itself." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "index_columns = [\"state_name\", \"county_name\", \"site_num\", \"date_local\", \"time_local\"]\n", - "param_column = \"parameter_name\"\n", - "value_column = \"sample_measurement\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's observe how much data each table contains:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for table in hourly_summary_tables:\n", - " # get the bigframes global session\n", - " bigframes_session = bpd.get_global_session()\n", - "\n", - " # get the bigquery table info\n", - " table_info = bigframes_session.bqclient.get_table(f\"{dataset}.{table}\")\n", - "\n", - " # read the table as a dataframe\n", - " df = bpd.read_gbq(f\"{dataset}.{table}\")\n", - "\n", - " # print metadata about the table\n", - " print(\n", - " f\"{table}: \"\n", - " f\"{round(table_info.num_bytes/1_000_000_000, 1)} GB, \"\n", - " f\"{round(table_info.num_rows/1_000_000, 1)} million rows, \"\n", - " f\"{df[param_column].nunique()} params\"\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's be mindful that the rows in each table may contain duplicates, which may introdude bias in any model trained on the raw data. We will make sure to drop the duplicates when we use the data for model training.\n", - "\n", - "Since we want to predict ozone level, we obviously pick the `o3` table. Let's also pick the tables about other gases - `co`, `no2` and `so2`. Let's also pick `pressure` and `temperature` tables as they seem fundamental indicators for the atmosphere. Note that each of these tables capture measurements for a single parameter (i.e. the column `parameter_name` has a single unique value).\n", - "\n", - "We are also interested in the nonoxny and wind tables, but they capture multiple parameters (i.e. the column `parameter_name` has a more than one unique values). We will include their measurements in later step, as they require extar processing to separate out the measurements for the individual parameters.\n", - "\n", - "We skip the other tables in this exercise for either they have very little or fragmented data or they seem uninteresting for the purpose of predicting ozone levels. You can take this as a separate exercise to train a linear regression model by including those parameters. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's maintain an array of dtaframes, one for each parameter, and eventually combine them into a single dataframe." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "params_dfs = []" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's process the tables with single parameter measurements first." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "EDAaIwHpQCDZ" - }, - "outputs": [], - "source": [ - "table_param_dict = {\n", - " \"co_hourly_summary\" : \"co\",\n", - " \"no2_hourly_summary\" : \"no2\",\n", - " \"o3_hourly_summary\" : \"o3\",\n", - " \"pressure_hourly_summary\" : \"pressure\",\n", - " \"so2_hourly_summary\" : \"so2\",\n", - " \"temperature_hourly_summary\" : \"temperature\",\n", - "}\n", - "\n", - "for table, param in table_param_dict.items():\n", - " param_df = bpd.read_gbq(\n", - " f\"{dataset}.{table}\",\n", - " columns=index_columns + [value_column]\n", - " )\n", - " param_df = param_df\\\n", - " .sort_values(index_columns)\\\n", - " .drop_duplicates(index_columns)\\\n", - " .set_index(index_columns)\\\n", - " .rename(columns={value_column : param})\n", - " params_dfs.append(param_df)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The nonoxnoy table captures measurements for 3 parameters. Let's analyze how many instances of each parameter it contains." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nonoxnoy_table = f\"{dataset}.nonoxnoy_hourly_summary\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "bpd.read_gbq(nonoxnoy_table, columns=[param_column]).value_counts()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We see that the NOy data is significantly sparse as compared to NO and NOx, so we skip that and include NO and NOx data." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "no_df = bpd.read_gbq(\n", - " nonoxnoy_table,\n", - " columns=index_columns + [value_column],\n", - " filters=[(param_column, \"==\", \"Nitric oxide (NO)\")]\n", - ")\n", - "no_df = no_df\\\n", - " .sort_values(index_columns)\\\n", - " .drop_duplicates(index_columns)\\\n", - " .set_index(index_columns)\\\n", - " .rename(columns={value_column: \"no_\"})\n", - "params_dfs.append(no_df)\n", - "\n", - "nox_df = bpd.read_gbq(\n", - " nonoxnoy_table,\n", - " columns=index_columns + [value_column],\n", - " filters=[(param_column, \"==\", \"Oxides of nitrogen (NOx)\")]\n", - ")\n", - "nox_df = nox_df\\\n", - " .sort_values(index_columns)\\\n", - " .drop_duplicates(index_columns)\\\n", - " .set_index(index_columns)\\\n", - " .rename(columns={value_column: \"nox\"})\n", - "params_dfs.append(nox_df)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The wind table captures measurements for 2 parameters. Let's analyze how many instances of each parameter it contains." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "wind_table = f\"{dataset}.wind_hourly_summary\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "bpd.read_gbq(wind_table, columns=[param_column]).value_counts()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's include the data for wind speed and wind direction." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "wind_speed_df = bpd.read_gbq(\n", - " wind_table,\n", - " columns=index_columns + [value_column],\n", - " filters=[(param_column, \"==\", \"Wind Speed - Resultant\")]\n", - ")\n", - "wind_speed_df = wind_speed_df\\\n", - " .sort_values(index_columns)\\\n", - " .drop_duplicates(index_columns)\\\n", - " .set_index(index_columns)\\\n", - " .rename(columns={value_column: \"wind_speed\"})\n", - "params_dfs.append(wind_speed_df)\n", - "\n", - "wind_dir_df = bpd.read_gbq(\n", - " wind_table,\n", - " columns=index_columns + [value_column],\n", - " filters=[(param_column, \"==\", \"Wind Direction - Resultant\")]\n", - ")\n", - "wind_dir_df = wind_dir_df\\\n", - " .sort_values(index_columns)\\\n", - " .drop_duplicates(index_columns)\\\n", - " .set_index(index_columns)\\\n", - " .rename(columns={value_column: \"wind_dir\"})\n", - "params_dfs.append(wind_dir_df)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's observe each individual parameter and number of data points for each parameter." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for param_df in params_dfs:\n", - " print(f\"{param_df.columns.values}: {len(param_df)}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's combine data from all parameters into a single DataFrame. The measurements for each parameter may not be available for every (state, county, site, date, time) identifier, we will consider only those identifiers for which measurements of all parameters are available. To achieve this we will combine the measurements via \"inner\" join.\n", - "\n", - "We will also materialize this combined data via `cache` method for efficient reuse in the subsequent steps." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df = bpd.concat(params_dfs, axis=1, join=\"inner\").cache()\n", - "df.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "rwPLjqW2Ajzh" - }, - "source": [ - "## Clean and prepare data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's temporarily bring the index columns as dataframe columns for further processing on the index values for the purpose of data preparation.\n", - "We will reconstruct the index back at the time of the model training." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df = df.reset_index()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Observe the years from which we have consolidated data so far." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df[\"date_local\"].dt.year.value_counts().sort_index().to_pandas()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this tutorial we would train a model from the past data to predict ozone levels for the future data. Let's define the cut-off year as 2020. We will pretend that the data before 2020 has known ozone levels, and the 2020 onwards the ozone levels are unknown, which we will predict using our model.\n", - "\n", - "We should further separate the known data into training and test sets. The model would be trained on the training set and then evaluated on the test set to make sure the model generalizes beyond the training data. We could use [train_test_split](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.ml.model_selection#bigframes_ml_model_selection_train_test_split) method to randomly split the training and test data, but we leave that for you to try out. In this exercise, let's split based on another cutoff year 2017 - the known data before 2017 would be training data and 2017 onwards would be the test data. This way we stay with the idea that the model is trained on past data and then used to predict the future values." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "6i6HkFJZa8na" - }, - "outputs": [], - "source": [ - "train_data_filter = (df.date_local.dt.year < 2017)\n", - "test_data_filter = (df.date_local.dt.year >= 2017) & (df.date_local.dt.year < 2020)\n", - "predict_data_filter = (df.date_local.dt.year >= 2020)\n", - "\n", - "df_train = df[train_data_filter].set_index(index_columns)\n", - "df_test = df[test_data_filter].set_index(index_columns)\n", - "df_predict = df[predict_data_filter].set_index(index_columns)\n", - "\n", - "df_train.shape, df_test.shape, df_predict.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "M_-0X7NxYK5f" - }, - "source": [ - "Prepare your feature (or input) columns and the target (or output) column for the purpose of model training and evaluation:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "YKwCW7Nsavap" - }, - "outputs": [], - "source": [ - "X_train = df_train.drop(columns=\"o3\")\n", - "y_train = df_train[\"o3\"]\n", - "\n", - "X_test = df_test.drop(columns=\"o3\")\n", - "y_test = df_test[\"o3\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Prepare the unknown data for prediction." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "wej78IDUaRW9" - }, - "outputs": [], - "source": [ - "X_predict = df_predict.drop(columns=\"o3\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Fx4lsNqMorJ-" - }, - "source": [ - "## Create the linear regression model\n", - "\n", - "BigQuery DataFrames ML lets you seamlessly transition from exploring data to creating machine learning models through its scikit-learn-like API, `bigframes.ml`. BigQuery DataFrames ML supports several types of [ML models](https://cloud.google.com/python/docs/reference/bigframes/latest#ml-capabilities).\n", - "\n", - "In this notebook, you create a [`LinearRegression`](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.ml.linear_model.LinearRegression) model, a type of regression model that generates a continuous value from a linear combination of input features.\n", - "\n", - "When you create a model with BigQuery DataFrames ML, it is saved in an internal location and limited to the BigQuery DataFrames session. However, as you'll see in the next section, you can use `to_gbq` to save the model permanently to your BigQuery project." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EloGtMnverFF" - }, - "source": [ - "### Create the model using `bigframes.ml`\n", - "\n", - "Please note that BigQuery DataFrames ML is backed by BigQuery ML, which uses\n", - "[automatic preprocessing](https://cloud.google.com/bigquery/docs/auto-preprocessing) to encode string values and scale numeric values when you pass the feature columns without transforms.\n", - "\n", - "BigQuery ML also [automatically splits the data for training and evaluation](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-create-glm#data_split_method), although for datasets with less than 500 rows (such as this one), all rows are used for training." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "GskyyUQPowBT" - }, - "outputs": [], - "source": [ - "from bigframes.ml.linear_model import LinearRegression\n", - "\n", - "model = LinearRegression()\n", - "\n", - "model.fit(X_train, y_train)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "UGjeMPC2caKK" - }, - "source": [ - "### Score the model\n", - "\n", - "Check how the model performs by using the [`score`](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.ml.linear_model.LinearRegression#bigframes_ml_linear_model_LinearRegression_score) method. More information on BigQuery ML model scoring can be found [here](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-evaluate#mlevaluate_output)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "kGBJKafpo0dl" - }, - "outputs": [], - "source": [ - "# On the training data\n", - "model.score(X_train, y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# On the test data\n", - "model.score(X_test, y_test)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "P2lUiZZ_cjri" - }, - "source": [ - "### Predict using the model\n", - "\n", - "Use the model to predict the levels of ozone. The predicted levels are returned in the column `predicted_o3`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "bsQ9cmoWo0Ps" - }, - "outputs": [], - "source": [ - "df_pred = model.predict(X_predict)\n", - "df_pred.peek()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GTRdUw-Ro5R1" - }, - "source": [ - "## Save the model in BigQuery\n", - "\n", - "The model is saved locally within this session. You can save the model permanently to BigQuery for use in future sessions, and to make the model sharable with others." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "K0mPaoGpcwwy" - }, - "source": [ - "Create a BigQuery dataset to house the model, adding a name for your dataset as the `DATASET_ID` variable:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZSP7gt13QrQt" - }, - "outputs": [], - "source": [ - "DATASET_ID = \"\" # @param {type:\"string\"}\n", - "\n", - "if not DATASET_ID:\n", - " raise ValueError(\"Please define the DATASET_ID\")\n", - "\n", - "client = bpd.get_global_session().bqclient\n", - "dataset = client.create_dataset(DATASET_ID, exists_ok=True)\n", - "print(f\"Dataset {dataset.dataset_id} created.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zqAIWWgJczp-" - }, - "source": [ - "Save the model using the `to_gbq` method:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "QE_GD4Byo_jb" - }, - "outputs": [], - "source": [ - "model.to_gbq(DATASET_ID + \".o3_lr_model\" , replace=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "f7uHacAy49rT" - }, - "source": [ - "You can view the saved model in the BigQuery console under the dataset you created in the first step. Run the following cell and follow the link to view your BigQuery console:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "qDBoiA_0488Z" - }, - "outputs": [], - "source": [ - "print(f'https://console.cloud.google.com/bigquery?ws=!1m5!1m4!5m3!1s{PROJECT_ID}!2s{DATASET_ID}!3so3_lr_model')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "G_wjSfXpWTuy" - }, - "source": [ - "# Summary and next steps\n", - "\n", - "You've created a linear regression model using `bigframes.ml`.\n", - "\n", - "Learn more about BigQuery DataFrames in the [documentation](https://cloud.google.com/python/docs/reference/bigframes/latest) and find more sample notebooks in the [GitHub repo](https://github.com/googleapis/python-bigquery-dataframes/tree/main/notebooks)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TpV-iwP9qw9c" - }, - "source": [ - "## Cleaning up\n", - "\n", - "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n", - "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", - "\n", - "Otherwise, you can uncomment the remaining cells and run them to delete the individual resources you created in this tutorial:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "sx_vKniMq9ZX" - }, - "outputs": [], - "source": [ - "# # Delete the BigQuery dataset and associated ML model\n", - "# client.delete_dataset(DATASET_ID, delete_contents=True, not_found_ok=True)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.0" - } - }, - "nbformat": 4, - "nbformat_minor": 0 + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ur8xi4C7S06n" + }, + "outputs": [], + "source": [ + "# Copyright 2025 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JAPoU8Sm5E6e" + }, + "source": [ + "## Train a linear regression model with BigQuery DataFrames ML\n", + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \"Colab Run in Colab\n", + " \n", + " \n", + " \n", + " \"GitHub\n", + " View on GitHub\n", + " \n", + " \n", + " \n", + " \"Vertex\n", + " Open in Vertex AI Workbench\n", + " \n", + " \n", + " \n", + " \"BQ\n", + " Open in BQ Studio\n", + " \n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "24743cf4a1e1" + }, + "source": [ + "**_NOTE_**: This notebook has been tested in the following environment:\n", + "\n", + "* Python version = 3.11" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tvgnzT1CKxrO" + }, + "source": [ + "## Overview\n", + "\n", + "This notebook demonstrates training a linear regression model on Big Data using BigQuery DataFrames ML. BigQuery DataFrames ML provides a provides a scikit-learn-like API for ML powered by the BigQuery engine.\n", + "\n", + "Learn more about [BigQuery DataFrames](https://cloud.google.com/python/docs/reference/bigframes/latest)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d975e698c9a4" + }, + "source": [ + "### Objective\n", + "\n", + "In this tutorial, we use BigQuery DataFrames to create a linear regression model that predicts the levels of Ozone in the atmosphere.\n", + "\n", + "The steps include:\n", + "\n", + "- Creating a DataFrame from the BigQuery table.\n", + "- Cleaning and preparing data using `bigframes.pandas` module.\n", + "- Creating a linear regression model using `bigframes.ml` module.\n", + "- Saving the ML model to BigQuery for future use.\n", + "\n", + "\n", + "Let's formally define our problem as: **Train a linear regression model to predict the level of ozone in the atmosphere given the measurements of other constituents and properties of the atmosphere.**" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "08d289fa873f" + }, + "source": [ + "### Dataset\n", + "\n", + "In this tutorial we are going to use the [`bigquery-public-data.epa_historical_air_quality`](https://console.cloud.google.com/marketplace/product/epa/historical-air-quality) dataset. To quote the description of the dataset:\n", + "\n", + "\"The United States Environmental Protection Agency (EPA) protects both public health and the environment by establishing the standards for national air quality. The EPA provides annual summary data as well as hourly and daily data in the categories of criteria gases, particulates, meteorological, and toxics.\"\n", + "\n", + "There are several tables capturing data about the constituents of the atmosphere, see them in the [BigQuery cloud console](https://pantheon.corp.google.com/bigquery?p=bigquery-public-data&d=epa_historical_air_quality&page=dataset). Most tables carry 10's of GBs of data, but that is not an issue with BigQuery DataFrames as the data is efficiently processed at BigQuery without transferring them to the client." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aed92deeb4a0" + }, + "source": [ + "### Costs\n", + "\n", + "This tutorial uses billable components of Google Cloud:\n", + "\n", + "* BigQuery (compute)\n", + "* BigQuery ML\n", + "\n", + "Learn about [BigQuery compute pricing](https://cloud.google.com/bigquery/pricing#analysis_pricing_models)\n", + "and [BigQuery ML pricing](https://cloud.google.com/bigquery/pricing#bqml),\n", + "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n", + "to generate a cost estimate based on your projected usage." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "i7EUnXsZhAGF" + }, + "source": [ + "## Installation\n", + "\n", + "If you don't have [bigframes](https://pypi.org/project/bigframes/) package already installed, uncomment and execute the following cells to\n", + "\n", + "1. Install the package\n", + "1. Restart the notebook kernel (Jupyter or Colab) to work with the package" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9O0Ka4W2MNF3" + }, + "outputs": [], + "source": [ + "# !pip install bigframes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "f200f10a1da3" + }, + "outputs": [], + "source": [ + "# Automatically restart kernel after installs so that your environment can access the new packages\n", + "\n", + "# import IPython\n", + "#\n", + "# app = IPython.Application.instance()\n", + "# app.kernel.do_shutdown(True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BF1j6f9HApxa" + }, + "source": [ + "## Before you begin\n", + "\n", + "Complete the tasks in this section to set up your environment." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oDfTjfACBvJk" + }, + "source": [ + "### Set up your Google Cloud project\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 credit towards your compute/storage costs.\n", + "\n", + "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", + "\n", + "3. [Enable the BigQuery API](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com).\n", + "\n", + "4. If you are running this notebook locally, install the [Cloud SDK](https://cloud.google.com/sdk)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WReHDGG5g0XY" + }, + "source": [ + "#### Set your project ID\n", + "\n", + "If you don't know your project ID, try the following:\n", + "* Run `gcloud config list`.\n", + "* Run `gcloud projects list`.\n", + "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "oM1iC_MfAts1" + }, + "outputs": [], + "source": [ + "PROJECT_ID = \"\" # @param {type:\"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "region" + }, + "source": [ + "#### Set the BigQuery location\n", + "\n", + "You can also change the `LOCATION` variable used by BigQuery. Learn more about [BigQuery locations](https://cloud.google.com/bigquery/docs/locations#supported_locations)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eF-Twtc4XGem" + }, + "outputs": [], + "source": [ + "LOCATION = \"US\" # @param {type: \"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sBCra4QMA2wR" + }, + "source": [ + "### Set up APIs, IAM permissions and Authentication\n", + "\n", + "Follow the instructions at https://cloud.google.com/bigquery/docs/use-bigquery-dataframes#permissions.\n", + "\n", + "Depending on your notebook environment, you might have to manually authenticate. Follow the relevant instructions below." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "74ccc9e52986" + }, + "source": [ + "**Vertex AI Workbench**\n", + "\n", + "Do nothing, you are already authenticated." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "de775a3773ba" + }, + "source": [ + "**Local JupyterLab instance**\n", + "\n", + "Uncomment and run the following cell:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "254614fa0c46" + }, + "outputs": [], + "source": [ + "# ! gcloud auth login\n", + "# ! gcloud auth application-default login" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ef21552ccea8" + }, + "source": [ + "**Colab**\n", + "\n", + "Uncomment and run the following cell:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "603adbbf0532" + }, + "outputs": [], + "source": [ + "# from google.colab import auth\n", + "# auth.authenticate_user()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "960505627ddf" + }, + "source": [ + "### Import libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PyQmSRbKA8r-" + }, + "outputs": [], + "source": [ + "import bigframes.pandas as bpd" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "init_aip:mbsdk,all" + }, + "source": [ + "### Set BigQuery DataFrames options" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NPPMuw2PXGeo" + }, + "outputs": [], + "source": [ + "# NOTE: The project option is not required in all environments.\n", + "# On BigQuery Studio, the project ID is automatically detected.\n", + "bpd.options.bigquery.project = PROJECT_ID\n", + "\n", + "# NOTE: The location option is not required.\n", + "# It defaults to the location of the first table or query\n", + "# passed to read_gbq(). For APIs where a location can't be\n", + "# auto-detected, the location defaults to the \"US\" location.\n", + "bpd.options.bigquery.location = LOCATION\n", + "\n", + "# NOTE: For a machine learning model the order of the data is\n", + "# not important. So let's relax the ordering_mode to accept\n", + "# partial ordering. This allows BigQuery DataFrames to run cost\n", + "# and performance optimized jobs at the BigQuery engine.\n", + "bpd.options.bigquery.ordering_mode = \"partial\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "D21CoOlfFTYI" + }, + "source": [ + "If you want to reset the location of the created DataFrame or Series objects, reset the session by executing `bpd.close_session()`. After that, you can reuse `bpd.options.bigquery.location` to specify another location." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9EMAqR37AfLS" + }, + "source": [ + "## Read data in BigQuery tables as DataFrame\n", + "\n", + "Let's read the tables in the dataset to construct a BigQuery DataFrames DataFrame. We will combine measurements of various parameters of the atmosphere from multiple tables to represent a consolidated dataframe to use for our model training and prediction. We have daily and hourly versions of the data available, but since we want to create a model that is dynamic so that it can capture the variance throughout the day, we would choose the hourly version.\n", + "\n", + "Note that we would use the pandas APIs as we normally would on the BigQuery DataFrames DataFrame, but calculations happen in the BigQuery query engine instead of the local environment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = \"bigquery-public-data.epa_historical_air_quality\"\n", + "hourly_summary_tables = [\n", + " \"co_hourly_summary\",\n", + " \"hap_hourly_summary\",\n", + " \"no2_hourly_summary\",\n", + " \"nonoxnoy_hourly_summary\",\n", + " \"o3_hourly_summary\",\n", + " \"pm10_hourly_summary\",\n", + " \"pm25_frm_hourly_summary\",\n", + " \"pm25_nonfrm_hourly_summary\",\n", + " \"pm25_speciation_hourly_summary\",\n", + " \"pressure_hourly_summary\",\n", + " \"rh_and_dp_hourly_summary\",\n", + " \"so2_hourly_summary\",\n", + " \"temperature_hourly_summary\",\n", + " \"voc_hourly_summary\",\n", + " \"wind_hourly_summary\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's pick index columns - to identify a measurement of the atmospheric parameter, param column - to identify which param the measurement pertains to, and value column - the column containing the measurement itself." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index_columns = [\"state_name\", \"county_name\", \"site_num\", \"date_local\", \"time_local\"]\n", + "param_column = \"parameter_name\"\n", + "value_column = \"sample_measurement\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's observe how much data each table contains:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for table in hourly_summary_tables:\n", + " # get the bigframes global session\n", + " bigframes_session = bpd.get_global_session()\n", + "\n", + " # get the bigquery table info\n", + " table_info = bigframes_session.bqclient.get_table(f\"{dataset}.{table}\")\n", + "\n", + " # read the table as a dataframe\n", + " df = bpd.read_gbq(f\"{dataset}.{table}\")\n", + "\n", + " # print metadata about the table\n", + " print(\n", + " f\"{table}: \"\n", + " f\"{round(table_info.num_bytes/1_000_000_000, 1)} GB, \"\n", + " f\"{round(table_info.num_rows/1_000_000, 1)} million rows, \"\n", + " f\"{df[param_column].nunique()} params\"\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's be mindful that the rows in each table may contain duplicates, which may introdude bias in any model trained on the raw data. We will make sure to drop the duplicates when we use the data for model training.\n", + "\n", + "Since we want to predict ozone level, we obviously pick the `o3` table. Let's also pick the tables about other gases - `co`, `no2` and `so2`. Let's also pick `pressure` and `temperature` tables as they seem fundamental indicators for the atmosphere. Note that each of these tables capture measurements for a single parameter (i.e. the column `parameter_name` has a single unique value).\n", + "\n", + "We are also interested in the nonoxny and wind tables, but they capture multiple parameters (i.e. the column `parameter_name` has a more than one unique values). We will include their measurements in later step, as they require extar processing to separate out the measurements for the individual parameters.\n", + "\n", + "We skip the other tables in this exercise for either they have very little or fragmented data or they seem uninteresting for the purpose of predicting ozone levels. You can take this as a separate exercise to train a linear regression model by including those parameters. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's maintain an array of dtaframes, one for each parameter, and eventually combine them into a single dataframe." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "params_dfs = []" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's process the tables with single parameter measurements first." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EDAaIwHpQCDZ" + }, + "outputs": [], + "source": [ + "table_param_dict = {\n", + " \"co_hourly_summary\" : \"co\",\n", + " \"no2_hourly_summary\" : \"no2\",\n", + " \"o3_hourly_summary\" : \"o3\",\n", + " \"pressure_hourly_summary\" : \"pressure\",\n", + " \"so2_hourly_summary\" : \"so2\",\n", + " \"temperature_hourly_summary\" : \"temperature\",\n", + "}\n", + "\n", + "for table, param in table_param_dict.items():\n", + " param_df = bpd.read_gbq(\n", + " f\"{dataset}.{table}\",\n", + " columns=index_columns + [value_column]\n", + " )\n", + " param_df = param_df\\\n", + " .sort_values(index_columns)\\\n", + " .drop_duplicates(index_columns)\\\n", + " .set_index(index_columns)\\\n", + " .rename(columns={value_column : param})\n", + " params_dfs.append(param_df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The nonoxnoy table captures measurements for 3 parameters. Let's analyze how many instances of each parameter it contains." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nonoxnoy_table = f\"{dataset}.nonoxnoy_hourly_summary\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bpd.read_gbq(nonoxnoy_table, columns=[param_column]).value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see that the NOy data is significantly sparse as compared to NO and NOx, so we skip that and include NO and NOx data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "no_df = bpd.read_gbq(\n", + " nonoxnoy_table,\n", + " columns=index_columns + [value_column],\n", + " filters=[(param_column, \"==\", \"Nitric oxide (NO)\")]\n", + ")\n", + "no_df = no_df\\\n", + " .sort_values(index_columns)\\\n", + " .drop_duplicates(index_columns)\\\n", + " .set_index(index_columns)\\\n", + " .rename(columns={value_column: \"no_\"})\n", + "params_dfs.append(no_df)\n", + "\n", + "nox_df = bpd.read_gbq(\n", + " nonoxnoy_table,\n", + " columns=index_columns + [value_column],\n", + " filters=[(param_column, \"==\", \"Oxides of nitrogen (NOx)\")]\n", + ")\n", + "nox_df = nox_df\\\n", + " .sort_values(index_columns)\\\n", + " .drop_duplicates(index_columns)\\\n", + " .set_index(index_columns)\\\n", + " .rename(columns={value_column: \"nox\"})\n", + "params_dfs.append(nox_df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The wind table captures measurements for 2 parameters. Let's analyze how many instances of each parameter it contains." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wind_table = f\"{dataset}.wind_hourly_summary\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bpd.read_gbq(wind_table, columns=[param_column]).value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's include the data for wind speed and wind direction." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wind_speed_df = bpd.read_gbq(\n", + " wind_table,\n", + " columns=index_columns + [value_column],\n", + " filters=[(param_column, \"==\", \"Wind Speed - Resultant\")]\n", + ")\n", + "wind_speed_df = wind_speed_df\\\n", + " .sort_values(index_columns)\\\n", + " .drop_duplicates(index_columns)\\\n", + " .set_index(index_columns)\\\n", + " .rename(columns={value_column: \"wind_speed\"})\n", + "params_dfs.append(wind_speed_df)\n", + "\n", + "wind_dir_df = bpd.read_gbq(\n", + " wind_table,\n", + " columns=index_columns + [value_column],\n", + " filters=[(param_column, \"==\", \"Wind Direction - Resultant\")]\n", + ")\n", + "wind_dir_df = wind_dir_df\\\n", + " .sort_values(index_columns)\\\n", + " .drop_duplicates(index_columns)\\\n", + " .set_index(index_columns)\\\n", + " .rename(columns={value_column: \"wind_dir\"})\n", + "params_dfs.append(wind_dir_df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's observe each individual parameter and number of data points for each parameter." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for param_df in params_dfs:\n", + " print(f\"{param_df.columns.values}: {len(param_df)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's combine data from all parameters into a single DataFrame. The measurements for each parameter may not be available for every (state, county, site, date, time) identifier, we will consider only those identifiers for which measurements of all parameters are available. To achieve this we will combine the measurements via \"inner\" join.\n", + "\n", + "We will also materialize this combined data via `cache` method for efficient reuse in the subsequent steps." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = bpd.concat(params_dfs, axis=1, join=\"inner\").cache()\n", + "df.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rwPLjqW2Ajzh" + }, + "source": [ + "## Clean and prepare data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's temporarily bring the index columns as dataframe columns for further processing on the index values for the purpose of data preparation.\n", + "We will reconstruct the index back at the time of the model training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = df.reset_index()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Observe the years from which we have consolidated data so far." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"date_local\"].dt.year.value_counts().sort_index().to_pandas()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this tutorial we would train a model from the past data to predict ozone levels for the future data. Let's define the cut-off year as 2020. We will pretend that the data before 2020 has known ozone levels, and the 2020 onwards the ozone levels are unknown, which we will predict using our model.\n", + "\n", + "We should further separate the known data into training and test sets. The model would be trained on the training set and then evaluated on the test set to make sure the model generalizes beyond the training data. We could use [train_test_split](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.ml.model_selection#bigframes_ml_model_selection_train_test_split) method to randomly split the training and test data, but we leave that for you to try out. In this exercise, let's split based on another cutoff year 2017 - the known data before 2017 would be training data and 2017 onwards would be the test data. This way we stay with the idea that the model is trained on past data and then used to predict the future values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6i6HkFJZa8na" + }, + "outputs": [], + "source": [ + "train_data_filter = (df.date_local.dt.year < 2017)\n", + "test_data_filter = (df.date_local.dt.year >= 2017) & (df.date_local.dt.year < 2020)\n", + "predict_data_filter = (df.date_local.dt.year >= 2020)\n", + "\n", + "df_train = df[train_data_filter].set_index(index_columns)\n", + "df_test = df[test_data_filter].set_index(index_columns)\n", + "df_predict = df[predict_data_filter].set_index(index_columns)\n", + "\n", + "df_train.shape, df_test.shape, df_predict.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M_-0X7NxYK5f" + }, + "source": [ + "Prepare your feature (or input) columns and the target (or output) column for the purpose of model training and evaluation:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YKwCW7Nsavap" + }, + "outputs": [], + "source": [ + "X_train = df_train.drop(columns=\"o3\")\n", + "y_train = df_train[\"o3\"]\n", + "\n", + "X_test = df_test.drop(columns=\"o3\")\n", + "y_test = df_test[\"o3\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Prepare the unknown data for prediction." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wej78IDUaRW9" + }, + "outputs": [], + "source": [ + "X_predict = df_predict.drop(columns=\"o3\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Fx4lsNqMorJ-" + }, + "source": [ + "## Create the linear regression model\n", + "\n", + "BigQuery DataFrames ML lets you seamlessly transition from exploring data to creating machine learning models through its scikit-learn-like API, `bigframes.ml`. BigQuery DataFrames ML supports several types of [ML models](https://cloud.google.com/python/docs/reference/bigframes/latest#ml-capabilities).\n", + "\n", + "In this notebook, you create a [`LinearRegression`](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.ml.linear_model.LinearRegression) model, a type of regression model that generates a continuous value from a linear combination of input features.\n", + "\n", + "When you create a model with BigQuery DataFrames ML, it is saved in an internal location and limited to the BigQuery DataFrames session. However, as you'll see in the next section, you can use `to_gbq` to save the model permanently to your BigQuery project." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EloGtMnverFF" + }, + "source": [ + "### Create the model using `bigframes.ml`\n", + "\n", + "Please note that BigQuery DataFrames ML is backed by BigQuery ML, which uses\n", + "[automatic preprocessing](https://cloud.google.com/bigquery/docs/auto-preprocessing) to encode string values and scale numeric values when you pass the feature columns without transforms.\n", + "\n", + "BigQuery ML also [automatically splits the data for training and evaluation](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-create-glm#data_split_method), although for datasets with less than 500 rows (such as this one), all rows are used for training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GskyyUQPowBT" + }, + "outputs": [], + "source": [ + "from bigframes.ml.linear_model import LinearRegression\n", + "\n", + "model = LinearRegression()\n", + "\n", + "model.fit(X_train, y_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UGjeMPC2caKK" + }, + "source": [ + "### Score the model\n", + "\n", + "Check how the model performs by using the [`score`](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.ml.linear_model.LinearRegression#bigframes_ml_linear_model_LinearRegression_score) method. More information on BigQuery ML model scoring can be found [here](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-evaluate#mlevaluate_output)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kGBJKafpo0dl" + }, + "outputs": [], + "source": [ + "# On the training data\n", + "model.score(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# On the test data\n", + "model.score(X_test, y_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P2lUiZZ_cjri" + }, + "source": [ + "### Predict using the model\n", + "\n", + "Use the model to predict the levels of ozone. The predicted levels are returned in the column `predicted_o3`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bsQ9cmoWo0Ps" + }, + "outputs": [], + "source": [ + "df_pred = model.predict(X_predict)\n", + "df_pred.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GTRdUw-Ro5R1" + }, + "source": [ + "## Save the model in BigQuery\n", + "\n", + "The model is saved locally within this session. You can save the model permanently to BigQuery for use in future sessions, and to make the model sharable with others." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "K0mPaoGpcwwy" + }, + "source": [ + "Create a BigQuery dataset to house the model, adding a name for your dataset as the `DATASET_ID` variable:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZSP7gt13QrQt" + }, + "outputs": [], + "source": [ + "DATASET_ID = \"\" # @param {type:\"string\"}\n", + "\n", + "if not DATASET_ID:\n", + " raise ValueError(\"Please define the DATASET_ID\")\n", + "\n", + "client = bpd.get_global_session().bqclient\n", + "dataset = client.create_dataset(DATASET_ID, exists_ok=True)\n", + "print(f\"Dataset {dataset.dataset_id} created.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zqAIWWgJczp-" + }, + "source": [ + "Save the model using the `to_gbq` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QE_GD4Byo_jb" + }, + "outputs": [], + "source": [ + "model.to_gbq(DATASET_ID + \".o3_lr_model\" , replace=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f7uHacAy49rT" + }, + "source": [ + "You can view the saved model in the BigQuery console under the dataset you created in the first step. Run the following cell and follow the link to view your BigQuery console:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qDBoiA_0488Z" + }, + "outputs": [], + "source": [ + "print(f'https://console.cloud.google.com/bigquery?ws=!1m5!1m4!5m3!1s{PROJECT_ID}!2s{DATASET_ID}!3so3_lr_model')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "G_wjSfXpWTuy" + }, + "source": [ + "# Summary and next steps\n", + "\n", + "You've created a linear regression model using `bigframes.ml`.\n", + "\n", + "Learn more about BigQuery DataFrames in the [documentation](https://cloud.google.com/python/docs/reference/bigframes/latest) and find more sample notebooks in the [GitHub repo](https://github.com/googleapis/python-bigquery-dataframes/tree/main/notebooks)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TpV-iwP9qw9c" + }, + "source": [ + "## Cleaning up\n", + "\n", + "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n", + "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", + "\n", + "Otherwise, you can uncomment the remaining cells and run them to delete the individual resources you created in this tutorial:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sx_vKniMq9ZX" + }, + "outputs": [], + "source": [ + "# # Delete the BigQuery dataset and associated ML model\n", + "# client.delete_dataset(DATASET_ID, delete_contents=True, not_found_ok=True)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/notebooks/ml/timeseries_analysis.ipynb b/notebooks/ml/timeseries_analysis.ipynb index 3b227460230..01c5a20efa3 100644 --- a/notebooks/ml/timeseries_analysis.ipynb +++ b/notebooks/ml/timeseries_analysis.ipynb @@ -12,14 +12,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "c0b2db75", "metadata": {}, "outputs": [], "source": [ "import bigframes.pandas as bpd\n", "from bigframes.ml import forecasting\n", - "bpd.options.display.render_mode = \"anywidget\"" + "bpd.options.display.repr_mode = \"anywidget\"" ] }, { @@ -27,7 +27,7 @@ "id": "0eba46b9", "metadata": {}, "source": [ - "## 1. Data Loading and Preprocessing", + "### 1. Data Loading and Preprocessing\n", "\n", "The first step is to load the San Francisco bikeshare dataset from BigQuery. We then preprocess the data by filtering for trips made by 'Subscriber' type users from 2018 onwards. This ensures we are working with a relevant and consistent subset of the data. Finally, we aggregate the trip data by the hour to create a time series of trip counts." ] @@ -1113,7 +1113,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv (3.13.0)", + "display_name": "venv", "language": "python", "name": "python3" }, diff --git a/notebooks/multimodal/multimodal_dataframe.ipynb b/notebooks/multimodal/multimodal_dataframe.ipynb index cd363db6f36..0822ee4c2db 100644 --- a/notebooks/multimodal/multimodal_dataframe.ipynb +++ b/notebooks/multimodal/multimodal_dataframe.ipynb @@ -1,1114 +1,1575 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "9edad7a6", - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2025 Google LLC\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "id": "816ab253", - "metadata": { - "id": "YOrUAvz6DMw-" - }, - "source": [ - "# BigFrames Multimodal DataFrame\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \"Colab Run in Colab\n", - " \n", - " \n", - " \n", - " \"GitHub\n", - " View on GitHub\n", - " \n", - " \n", - " \n", - " \"BQ\n", - " Open in BQ Studio\n", - " \n", - "
    \n" - ] - }, - { - "cell_type": "markdown", - "id": "77d821d4", - "metadata": {}, - "source": [ - "This notebook is introducing BigFrames Multimodal features:\n", - "1. Create Multimodal DataFrame\n", - "2. Combine unstructured data with structured data\n", - "3. Conduct image transformations\n", - "4. Use LLM models to ask questions and generate embeddings on images\n", - "5. PDF chunking function\n", - "6. Transcribe audio\n", - "7. Extract EXIF metadata from images" - ] - }, - { - "cell_type": "markdown", - "id": "75ab1c13", - "metadata": { - "id": "PEAJQQ6AFg-n" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "markdown", - "id": "750954c4", - "metadata": {}, - "source": [ - "Install the latest bigframes package if bigframes version < 2.4.0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2a6fafb1", - "metadata": {}, - "outputs": [], - "source": [ - "# !pip install bigframes --upgrade" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "df561d04", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Copyright 2025 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] }, - "id": "bGyhLnfEeB0X", - "outputId": "83ac8b64-3f44-4d43-d089-28a5026cbb42" - }, - "outputs": [], - "source": [ - "PROJECT = \"bigframes-dev\" # replace with your project. \n", - "# Refer to https://cloud.google.com/bigquery/docs/multimodal-data-dataframes-tutorial#required_roles for your required permissions\n", - "\n", - "LOCATION = \"us\" # replace with your location.\n", - "\n", - "# Dataset where the UDF will be created.\n", - "DATASET_ID = \"bigframes_samples\" # replace with your dataset ID.\n", - "\n", - "OUTPUT_BUCKET = \"bigframes_blob_test\" # replace with your GCS bucket. \n", - "# The connection (or bigframes-default-connection of the project) must have read/write permission to the bucket. \n", - "# Refer to https://cloud.google.com/bigquery/docs/multimodal-data-dataframes-tutorial#grant-permissions for setting up connection service account permissions.\n", - "# In this Notebook it uses bigframes-default-connection by default. You can also bring in your own connections in each method.\n", - "\n", - "FULL_CONNECTION_ID = f\"{PROJECT}.{LOCATION}.bigframes-default-connection\"\n", - "\n", - "import bigframes\n", - "# Setup project\n", - "bigframes.options.bigquery.project = PROJECT\n", - "bigframes.options.bigquery.location = LOCATION\n", - "\n", - "# Display options\n", - "bigframes.options.display.blob_display_width = 300\n", - "bigframes.options.display.progress_bar = None\n", - "\n", - "import bigframes.pandas as bpd\n", - "import bigframes.bigquery as bbq" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "id": "35bd6e6e", - "metadata": {}, - "outputs": [], - "source": [ - "import bigframes.bigquery as bbq\n", - "\n", - "def get_runtime_json_str(series, mode=\"R\", with_metadata=False):\n", - " \"\"\"\n", - " Get the runtime (contains signed URL to access gcs data) and apply the\n", - " ToJSONSTring transformation.\n", - " \n", - " Args:\n", - " series: bigframes.series.Series to operate on.\n", - " mode: \"R\" for read, \"RW\" for read/write.\n", - " with_metadata: Whether to fetch and include blob metadata.\n", - " \"\"\"\n", - " # 1. Optionally fetch metadata\n", - " s = (\n", - " bbq.obj.fetch_metadata(series)\n", - " if with_metadata\n", - " else series\n", - " )\n", - " \n", - " # 2. Retrieve the access URL runtime object\n", - " runtime = bbq.obj.get_access_url(s, mode=mode)\n", - " \n", - " # 3. Convert the runtime object to a JSON string\n", - " return bbq.to_json_string(runtime)\n", - "\n", - "def get_metadata(series):\n", - " # Fetch metadata and extract GCS metadata from the details JSON field\n", - " metadata_obj = bbq.obj.fetch_metadata(series)\n", - " return bbq.json_query(metadata_obj.struct.field(\"details\"), \"$.gcs_metadata\")\n", - "\n", - "def get_content_type(series):\n", - " return bbq.json_value(get_metadata(series), \"$.content_type\")\n", - "\n", - "def get_size(series):\n", - " return bbq.json_value(get_metadata(series), \"$.size\").astype(\"Int64\")\n", - "\n", - "def get_updated(series):\n", - " return bpd.to_datetime(bbq.json_value(get_metadata(series), \"$.updated\").astype(\"Int64\"), unit=\"us\", utc=True)\n", - "\n", - "from IPython.display import HTML, display\n", - "\n", - "def render_images(df):\n", - " \"\"\"Helper to display BigFrames DataFrame with rendered image previews.\"\"\"\n", - " import bigframes.pandas as bpd\n", - " import bigframes.bigquery as bbq\n", - " import bigframes\n", - " from bigframes import dtypes\n", - " import json\n", - " \n", - " if isinstance(df, bpd.Series):\n", - " df = df.to_frame()\n", - " \n", - " # 1. Auto-detect columns holding ObjectRefs\n", - " object_cols = [\n", - " col for col, dtype in zip(df.columns, df.dtypes)\n", - " if dtype == dtypes.OBJ_REF_DTYPE\n", - " ]\n", - " \n", - " if not object_cols:\n", - " display(df)\n", - " return\n", - "\n", - " limit = bigframes.options.display.max_rows or 10\n", - " view_df = df.head(limit)\n", - " \n", - " # 2. Bulk-fetch access runtime URLs ONLY (disable with_metadata to bypass potential \n", - " # race conditions on new files where BigQuery may error before async writes finalize)\n", - " runtime_cols = {\n", - " col: get_runtime_json_str(view_df[col], mode=\"R\", with_metadata=False) \n", - " for col in object_cols\n", - " }\n", - " \n", - " pandas_json_df = bpd.DataFrame(runtime_cols).to_pandas()\n", - " final_pd = view_df.to_pandas()\n", - " \n", - " width = bigframes.options.display.blob_display_width or 300\n", - " IMAGE_EXTENSIONS = (\".png\", \".jpg\", \".jpeg\", \".gif\", \".webp\")\n", - " \n", - " def format_cell_html(raw_json):\n", - " if not raw_json:\n", - " return \"\"\n", - " try:\n", - " obj_rt = json.loads(raw_json)\n", - " \n", - " if \"access_urls\" not in obj_rt:\n", - " err = obj_rt.get(\"errors\", [{\"message\": \"URL Generation Failed\"}])[0].get(\"message\")\n", - " return f'Error: {err}'\n", - " \n", - " uri = obj_rt.get(\"objectref\", {}).get(\"uri\", \"\")\n", - " url = obj_rt[\"access_urls\"][\"read_url\"]\n", - " \n", - " # Safely infer type from extension to guarantee immediate display availability\n", - " if uri and str(uri).lower().endswith(IMAGE_EXTENSIONS):\n", - " return f''\n", - " \n", - " return f'{uri if uri else \"view\"}'\n", - " except:\n", - " return \"Format Error\"\n", - "\n", - " for col in object_cols:\n", - " final_pd[col] = pandas_json_df[col].map(format_cell_html)\n", - " \n", - " display(HTML(final_pd.to_html(escape=False)))" - ] - }, - { - "cell_type": "markdown", - "id": "be9ce892", - "metadata": { - "id": "ifKOq7VZGtZy" - }, - "source": [ - "To create a Multimodal DataFrame, you can use `bigframes.bigquery.obj.make_ref` on a series of URIs. You can get the URIs from a BigQuery table or by listing them from Cloud Storage.\n", - "\n", - "In this example, we use `gcsfs` to list the files from Cloud Storage, and then use `read_gbq` to load them into a BigQuery DataFrame before creating the object reference." - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "id": "871d02f4", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" + { + "cell_type": "markdown", + "metadata": { + "id": "YOrUAvz6DMw-" + }, + "source": [ + "# BigFrames Multimodal DataFrame\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \"Colab Run in Colab\n", + " \n", + " \n", + " \n", + " \"GitHub\n", + " View on GitHub\n", + " \n", + " \n", + " \n", + " \"BQ\n", + " Open in BQ Studio\n", + " \n", + "
    \n" + ] }, - "id": "fx6YcZJbeYru", - "outputId": "d707954a-0dd0-4c50-b7bf-36b140cf76cf" - }, - "outputs": [], - "source": [ - "import gcsfs\n", - "import bigframes.bigquery as bbq\n", - "\n", - "# List files using gcsfs (public bucket)\n", - "fs = gcsfs.GCSFileSystem(anon=True)\n", - "uris = fs.glob(\"gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/images/*\")\n", - "\n", - "# Ensure URIs have gs:// prefix\n", - "uris = [u if u.startswith(\"gs://\") else f\"gs://{u}\" for u in uris]\n", - "\n", - "# Read the URIs into a BigQuery DataFrame using UNNEST\n", - "# We take the first 5 for this example\n", - "df_image = bpd.read_gbq(f\"SELECT uri FROM UNNEST({uris[:5]}) as uri\")\n", - "\n", - "# Create the object reference column\n", - "df_image['image'] = bbq.obj.make_ref(df_image['uri'], authorizer=FULL_CONNECTION_ID)\n", - "df_image = df_image[['image']]" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "id": "2e0436b0", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 487 + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook is introducing BigFrames Multimodal features:\n", + "1. Create Multimodal DataFrame\n", + "2. Combine unstructured data with structured data\n", + "3. Conduct image transformations\n", + "4. Use LLM models to ask questions and generate embeddings on images\n", + "5. PDF chunking function\n", + "6. Transcribe audio" + ] }, - "id": "HhCb8jRsLe9B", - "outputId": "03081cf9-3a22-42c9-b38f-649f592fdada" - }, - "outputs": [ { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] + "cell_type": "markdown", + "metadata": { + "id": "PEAJQQ6AFg-n" + }, + "source": [ + "### Setup" + ] }, { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    image
    0
    1
    2
    3
    4
    " + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Install the latest bigframes package if bigframes version < 2.4.0" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install bigframes --upgrade" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "bGyhLnfEeB0X", + "outputId": "83ac8b64-3f44-4d43-d089-28a5026cbb42" + }, + "outputs": [], + "source": [ + "PROJECT = \"bigframes-dev\" # replace with your project. \n", + "# Refer to https://cloud.google.com/bigquery/docs/multimodal-data-dataframes-tutorial#required_roles for your required permissions\n", + "\n", + "OUTPUT_BUCKET = \"bigframes_blob_test\" # replace with your GCS bucket. \n", + "# The connection (or bigframes-default-connection of the project) must have read/write permission to the bucket. \n", + "# Refer to https://cloud.google.com/bigquery/docs/multimodal-data-dataframes-tutorial#grant-permissions for setting up connection service account permissions.\n", + "# In this Notebook it uses bigframes-default-connection by default. You can also bring in your own connections in each method.\n", + "\n", + "import bigframes\n", + "# Setup project\n", + "bigframes.options.bigquery.project = PROJECT\n", + "\n", + "# Display options\n", + "bigframes.options.display.blob_display_width = 300\n", + "bigframes.options.display.progress_bar = None\n", + "\n", + "import bigframes.pandas as bpd" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ifKOq7VZGtZy" + }, + "source": [ + "### 1. Create Multimodal DataFrame\n", + "There are several ways to create Multimodal DataFrame. The easiest way is from the wildcard paths." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "fx6YcZJbeYru", + "outputId": "d707954a-0dd0-4c50-b7bf-36b140cf76cf" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/global_session.py:113: DefaultLocationWarning: No explicit location is set, so using location US for the session.\n", + " _global_session = bigframes.session.connect(\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + } ], - "text/plain": [ - "" + "source": [ + "# Create blob columns from wildcard path.\n", + "df_image = bpd.from_glob_path(\n", + " \"gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/images/*\", name=\"image\"\n", + ")\n", + "# Other ways are: from string uri column\n", + "# df = bpd.DataFrame({\"uri\": [\"gs:///\", \"gs:///\"]})\n", + "# df[\"blob_col\"] = df[\"uri\"].str.to_blob()\n", + "\n", + "# From an existing object table\n", + "# df = bpd.read_gbq_object_table(\"\", name=\"blob_col\")" ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Take only the 5 images to deal with. Preview the content of the Mutimodal DataFrame\n", - "df_image = df_image.head(5)\n", - "render_images(df_image)" - ] - }, - { - "cell_type": "markdown", - "id": "429b0117", - "metadata": { - "id": "b6RRZb3qPi_T" - }, - "source": [ - "### 2. Combine unstructured data with structured data" - ] - }, - { - "cell_type": "markdown", - "id": "991fa065", - "metadata": { - "id": "4YJCdmLtR-qu" - }, - "source": [ - "Now you can put more information into the table to describe the files. Such as author info from inputs, or other metadata from the gcs object itself." - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "08722ec5", - "metadata": { - "id": "YYYVn7NDH0Me" - }, - "outputs": [ + }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 487 + }, + "id": "HhCb8jRsLe9B", + "outputId": "03081cf9-3a22-42c9-b38f-649f592fdada" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    image
    0
    1
    2
    3
    4
    \n", + "

    5 rows × 1 columns

    \n", + "
    [5 rows x 1 columns in total]" + ], + "text/plain": [ + " image\n", + "0 {'uri': 'gs://cloud-samples-data/bigquery/tuto...\n", + "1 {'uri': 'gs://cloud-samples-data/bigquery/tuto...\n", + "2 {'uri': 'gs://cloud-samples-data/bigquery/tuto...\n", + "3 {'uri': 'gs://cloud-samples-data/bigquery/tuto...\n", + "4 {'uri': 'gs://cloud-samples-data/bigquery/tuto...\n", + "\n", + "[5 rows x 1 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Take only the 5 images to deal with. Preview the content of the Mutimodal DataFrame\n", + "df_image = df_image.head(5)\n", + "df_image" + ] }, { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    imageauthorcontent_typesizeupdated
    0aliceimage/png7157662025-03-20 17:44:38+00:00
    1bobimage/png11674062025-03-20 17:44:38+00:00
    2bobimage/png11508922025-03-20 17:44:39+00:00
    3aliceimage/png17365332025-03-20 17:44:39+00:00
    4bobimage/png4397402025-03-20 17:44:39+00:00
    " + "cell_type": "markdown", + "metadata": { + "id": "b6RRZb3qPi_T" + }, + "source": [ + "### 2. Combine unstructured data with structured data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4YJCdmLtR-qu" + }, + "source": [ + "Now you can put more information into the table to describe the files. Such as author info from inputs, or other metadata from the gcs object itself." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "YYYVn7NDH0Me" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/bigquery/_operations/json.py:121: UserWarning: The `json_extract` is deprecated and will be removed in a future\n", + "version. Use `json_query` instead.\n", + " warnings.warn(bfe.format_message(msg), category=UserWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/bigquery/_operations/json.py:121: UserWarning: The `json_extract` is deprecated and will be removed in a future\n", + "version. Use `json_query` instead.\n", + " warnings.warn(bfe.format_message(msg), category=UserWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/bigquery/_operations/json.py:121: UserWarning: The `json_extract` is deprecated and will be removed in a future\n", + "version. Use `json_query` instead.\n", + " warnings.warn(bfe.format_message(msg), category=UserWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    imageauthorcontent_typesizeupdated
    0aliceimage/png15912402025-03-20 17:45:04+00:00
    1bobimage/png11829512025-03-20 17:45:02+00:00
    2bobimage/png15208842025-03-20 17:44:55+00:00
    3aliceimage/png12354012025-03-20 17:45:19+00:00
    4bobimage/png15919232025-03-20 17:44:47+00:00
    \n", + "

    5 rows × 5 columns

    \n", + "
    [5 rows x 5 columns in total]" + ], + "text/plain": [ + " image author content_type \\\n", + "0 {'uri': 'gs://cloud-samples-data/bigquery/tuto... alice image/png \n", + "1 {'uri': 'gs://cloud-samples-data/bigquery/tuto... bob image/png \n", + "2 {'uri': 'gs://cloud-samples-data/bigquery/tuto... bob image/png \n", + "3 {'uri': 'gs://cloud-samples-data/bigquery/tuto... alice image/png \n", + "4 {'uri': 'gs://cloud-samples-data/bigquery/tuto... bob image/png \n", + "\n", + " size updated \n", + "0 1591240 2025-03-20 17:45:04+00:00 \n", + "1 1182951 2025-03-20 17:45:02+00:00 \n", + "2 1520884 2025-03-20 17:44:55+00:00 \n", + "3 1235401 2025-03-20 17:45:19+00:00 \n", + "4 1591923 2025-03-20 17:44:47+00:00 \n", + "\n", + "[5 rows x 5 columns]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - "" + "source": [ + "# Combine unstructured data with structured data\n", + "df_image[\"author\"] = [\"alice\", \"bob\", \"bob\", \"alice\", \"bob\"] # type: ignore\n", + "df_image[\"content_type\"] = df_image[\"image\"].blob.content_type()\n", + "df_image[\"size\"] = df_image[\"image\"].blob.size()\n", + "df_image[\"updated\"] = df_image[\"image\"].blob.updated()\n", + "df_image" ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Combine unstructured data with structured data\n", - "df_image = df_image.head(5)\n", - "df_image[\"author\"] = [\"alice\", \"bob\", \"bob\", \"alice\", \"bob\"] # type: ignore\n", - "df_image[\"content_type\"] = get_content_type(df_image[\"image\"])\n", - "df_image[\"size\"] = get_size(df_image[\"image\"])\n", - "df_image[\"updated\"] = get_updated(df_image[\"image\"])\n", - "render_images(df_image)" - ] - }, - { - "cell_type": "markdown", - "id": "f90826f6", - "metadata": {}, - "source": [ - "### 3. Conduct image transformations" - ] - }, - { - "cell_type": "markdown", - "id": "e24c9f8c", - "metadata": {}, - "source": [ - "This section demonstrates how to perform image transformations like blur, resize, and normalize using custom BigQuery Python UDFs and the `opencv-python` library." - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "id": "db665049", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 487 }, - "id": "HhCb8jRsLe9B", - "outputId": "03081cf9-3a22-42c9-b38f-649f592fdada" - }, - "outputs": [ { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/pandas/__init__.py:211: PreviewWarning: udf is in preview.\n", - " return global_session.with_default_session(\n", - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dataframe.py:4695: FunctionAxisOnePreviewWarning: DataFrame.apply with parameter axis=1 scenario is in preview.\n", - " warnings.warn(msg, category=bfe.FunctionAxisOnePreviewWarning)\n", - "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", - "instead of using `db_dtypes` in the future when available in pandas\n", - "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", - " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" - ] + "cell_type": "markdown", + "metadata": { + "id": "NUd4Kog_QLRS" + }, + "source": [ + "Then you can filter the rows based on the structured data. And for different content types, you can display them respectively or together." + ] }, { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    imageblurred
    0
    1
    2
    3
    4
    " + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 75 + }, + "id": "UGuAk9PNDRF3", + "outputId": "73feb33d-4a05-48fb-96e5-3c48c2a456f3" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/bigquery/_operations/json.py:121: UserWarning: The `json_extract` is deprecated and will be removed in a future\n", + "version. Use `json_query` instead.\n", + " warnings.warn(bfe.format_message(msg), category=UserWarning)\n" + ] + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } ], - "text/plain": [ - "" + "source": [ + "# filter images and display, you can also display audio and video types\n", + "df_image[df_image[\"author\"] == \"alice\"][\"image\"].blob.display()" ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Construct the canonical connection ID\n", - "FULL_CONNECTION_ID = f\"{PROJECT}.{LOCATION}.bigframes-default-connection\"\n", - "\n", - "@bpd.udf(\n", - " input_types=[str, str, int, int],\n", - " output_type=str,\n", - " dataset=DATASET_ID,\n", - " name=\"image_blur_v2\",\n", - " bigquery_connection=FULL_CONNECTION_ID,\n", - " packages=[\"opencv-python-headless\", \"numpy\", \"requests\"],\n", - ")\n", - "def image_blur(src_rt: str, dst_rt: str, kx: int, ky: int) -> str:\n", - " import json\n", - " import cv2 as cv\n", - " import numpy as np\n", - " import requests\n", - " import base64\n", - "\n", - " src_obj = json.loads(src_rt)\n", - " if \"access_urls\" not in src_obj:\n", - " raise ValueError(f\"Missing 'access_urls' in source object. Response: {src_obj}\")\n", - " src_url = src_obj[\"access_urls\"][\"read_url\"]\n", - " \n", - " response = requests.get(src_url, timeout=30)\n", - " response.raise_for_status()\n", - " \n", - " img = cv.imdecode(np.frombuffer(response.content, np.uint8), cv.IMREAD_UNCHANGED)\n", - " if img is None:\n", - " raise ValueError(\"cv.imdecode failed\")\n", - " \n", - " kx, ky = int(kx), int(ky)\n", - " img_blurred = cv.blur(img, ksize=(kx, ky))\n", - " \n", - " success, encoded = cv.imencode(\".jpeg\", img_blurred)\n", - " if not success:\n", - " raise ValueError(\"cv.imencode failed\")\n", - " \n", - " # Handle two output modes\n", - " if dst_rt: # GCS/Series output mode\n", - " dst_obj = json.loads(dst_rt)\n", - " if \"access_urls\" not in dst_obj:\n", - " raise ValueError(f\"Missing 'access_urls' in destination object. Verify authorizer permissions. Response: {dst_obj}\")\n", - " dst_url = dst_obj[\"access_urls\"][\"write_url\"]\n", - " \n", - " requests.put(dst_url, data=encoded.tobytes(), headers={\"Content-Type\": \"image/jpeg\"}, timeout=30).raise_for_status()\n", - " \n", - " uri = dst_obj[\"objectref\"][\"uri\"]\n", - " return uri\n", - " \n", - " else: # BigQuery bytes output mode \n", - " image_bytes = encoded.tobytes()\n", - " return base64.b64encode(image_bytes).decode()\n", - "\n", - "def apply_transformation(series, dst_folder, udf, *args, verbose=False):\n", - " import os\n", - " dst_folder = os.path.join(dst_folder, \"\")\n", - " # Fetch metadata to get the URI\n", - " metadata = bbq.obj.fetch_metadata(series)\n", - " current_uri = metadata.struct.field(\"uri\")\n", - " dst_uri = current_uri.str.replace(r\"^.*\\/(.*)$\", rf\"{dst_folder}\\1\", regex=True)\n", - " \n", - " # To avoid synchronous 404 validation checks on files that don't exist yet, \n", - " # bypass the validator by explicitly constructing an objectref JSON.\n", - " dst_blob_df = bpd.DataFrame({\"uri\": dst_uri})\n", - " dst_blob_df[\"authorizer\"] = FULL_CONNECTION_ID\n", - " dst_blob = bbq.obj.make_ref(bbq.to_json(bbq.struct(dst_blob_df)))\n", - "\n", - " df_transform = bpd.DataFrame({\n", - " \"src_rt\": get_runtime_json_str(series, mode=\"R\"),\n", - " \"dst_rt\": get_runtime_json_str(dst_blob, mode=\"RW\"),\n", - " })\n", - " res = df_transform[[\"src_rt\", \"dst_rt\"]].apply(\n", - " udf, axis=1, args=args\n", - " )\n", - " \n", - " if verbose:\n", - " return res\n", - " \n", - " # Final return MUST also use JSON bypass to eliminate temporary 404 validation \n", - " # errors from embedded ObjectRefs during fused query execution pipelines.\n", - " res_df = bpd.DataFrame({\"uri\": res})\n", - " res_df[\"authorizer\"] = FULL_CONNECTION_ID\n", - " return bbq.obj.make_ref(bbq.to_json(bbq.struct(res_df)))\n", - "\n", - "# Apply transformations\n", - "df_image[\"blurred\"] = apply_transformation(\n", - " df_image[\"image\"], f\"gs://{OUTPUT_BUCKET}/image_blur_transformed/\",\n", - " image_blur, 20, 20\n", - ")\n", - "render_images(df_image[[\"image\", \"blurred\"]])" - ] - }, - { - "cell_type": "markdown", - "id": "11fcc6ec", - "metadata": { - "id": "Euk5saeVVdTP" - }, - "source": [ - "### 4. Use LLM models to ask questions and generate embeddings on images" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "793b2f45", - "metadata": { - "id": "mRUGfcaFVW-3" - }, - "outputs": [], - "source": [ - "from bigframes.ml import llm\n", - "gemini = llm.GeminiTextGenerator()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13d7cb93", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 657 }, - "id": "DNFP7CbjWdR9", - "outputId": "3f90a062-0abc-4bce-f53c-db57b06a14b9" - }, - "outputs": [], - "source": [ - "# Ask the same question on the images\n", - "answer = gemini.predict(df_image, prompt=[\"what item is it?\", \"what color is the picture?\"])\n", - "render_images(answer[[\"ml_generate_text_llm_result\", \"image\"]])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "68857305", - "metadata": { - "id": "IG3J3HsKhyBY" - }, - "outputs": [], - "source": [ - "# Ask different questions\n", - "df_image[\"question\"] = [\n", - " \"what item is it?\",\n", - " \"what color is the picture?\",\n", - " \"what is the product name?\",\n", - " \"is it for pets?\",\n", - " \"what is the weight of the product?\",\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "829afc69", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 657 + { + "cell_type": "markdown", + "metadata": { + "id": "1IJuakwJTZey" + }, + "source": [ + "### 3. Conduct image transformations\n", + "BigFrames Multimodal DataFrame provides image(and other) transformation functions. Such as image_blur, image_resize and image_normalize. The output can be saved to GCS folders or to BQ as bytes." + ] }, - "id": "qKOb765IiVuD", - "outputId": "731bafad-ea29-463f-c8c1-cb7acfd70e5d" - }, - "outputs": [], - "source": [ - "answer_alt = gemini.predict(df_image, prompt=[df_image[\"question\"], df_image[\"image\"]])\n", - "render_images(answer_alt[[\"ml_generate_text_llm_result\", \"image\"]])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e75df430", - "metadata": { + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "VWsl5BBPJ6N7", + "outputId": "45d2356e-322b-4982-cfa7-42d034dc4344" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py:182: FunctionAxisOnePreviewWarning: Blob Functions use bigframes DataFrame Managed function with axis=1 senario, which is a preview feature.\n", + " return method(*args, **kwargs)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py:182: FunctionAxisOnePreviewWarning: Blob Functions use bigframes DataFrame Managed function with axis=1 senario, which is a preview feature.\n", + " return method(*args, **kwargs)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py:182: FunctionAxisOnePreviewWarning: Blob Functions use bigframes DataFrame Managed function with axis=1 senario, which is a preview feature.\n", + " return method(*args, **kwargs)\n" + ] + } + ], + "source": [ + "df_image[\"blurred\"] = df_image[\"image\"].blob.image_blur(\n", + " (20, 20), dst=f\"gs://{OUTPUT_BUCKET}/image_blur_transformed/\", engine=\"opencv\"\n", + ")\n", + "df_image[\"resized\"] = df_image[\"image\"].blob.image_resize(\n", + " (300, 200), dst=f\"gs://{OUTPUT_BUCKET}/image_resize_transformed/\", engine=\"opencv\"\n", + ")\n", + "df_image[\"normalized\"] = df_image[\"image\"].blob.image_normalize(\n", + " alpha=50.0,\n", + " beta=150.0,\n", + " norm_type=\"minmax\",\n", + " dst=f\"gs://{OUTPUT_BUCKET}/image_normalize_transformed/\",\n", + " engine=\"opencv\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "rWCAGC8w64vU", + "outputId": "d7d456f0-8b56-492c-fe1b-967e9664d813" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py:182: FunctionAxisOnePreviewWarning: Blob Functions use bigframes DataFrame Managed function with axis=1 senario, which is a preview feature.\n", + " return method(*args, **kwargs)\n" + ] + } + ], + "source": [ + "# You can also chain functions together\n", + "df_image[\"blur_resized\"] = df_image[\"blurred\"].blob.image_resize((300, 200), dst=f\"gs://{OUTPUT_BUCKET}/image_blur_resize_transformed/\", engine=\"opencv\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using `verbose` mode for detailed output\\n\n", + "\\n\n", + "All multimodal functions support a `verbose` parameter, which defaults to `False`.\\n\n", + "\\n\n", + "* When `verbose=False` (the default), the function will only return the main content of the result (e.g., the transformed image, the extracted text).\\n\n", + "* When `verbose=True`, the function returns a `STRUCT` containing two fields:\\n\n", + " * `content`: The main result of the operation.\\n\n", + " * `status`: An informational field. If the operation is successful, this will be empty. If an error occurs during the processing of a specific row, this field will contain the error message, allowing the overall job to complete without failing.\\n\n", + "\\n\n", + "Using `verbose=True` is highly recommended for debugging and for workflows where you need to handle potential failures on a row-by-row basis. Let's see it in action with the `image_blur` function." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py:182: FunctionAxisOnePreviewWarning: Blob Functions use bigframes DataFrame Managed function with axis=1 senario, which is a preview feature.\n", + " return method(*args, **kwargs)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    blurred_verbose
    0{'status': '', 'content': {'uri': 'gs://bigfra...
    1{'status': '', 'content': {'uri': 'gs://bigfra...
    2{'status': '', 'content': {'uri': 'gs://bigfra...
    3{'status': '', 'content': {'uri': 'gs://bigfra...
    4{'status': '', 'content': {'uri': 'gs://bigfra...
    \n", + "

    5 rows × 1 columns

    \n", + "
    [5 rows x 1 columns in total]" + ], + "text/plain": [ + " blurred_verbose\n", + "0 {'status': '', 'content': {'uri': 'gs://bigfra...\n", + "1 {'status': '', 'content': {'uri': 'gs://bigfra...\n", + "2 {'status': '', 'content': {'uri': 'gs://bigfra...\n", + "3 {'status': '', 'content': {'uri': 'gs://bigfra...\n", + "4 {'status': '', 'content': {'uri': 'gs://bigfra...\n", + "\n", + "[5 rows x 1 columns]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_image[\"blurred_verbose\"] = df_image[\"image\"].blob.image_blur(\n", + " (20, 20), dst=f\"gs://{OUTPUT_BUCKET}/image_blur_transformed_verbose/\", engine=\"opencv\", verbose=True\n", + ")\n", + "df_image[[\"blurred_verbose\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 605 + }, + "id": "6NGK6GYSU44B", + "outputId": "859101c1-2ee4-4f9a-e250-e8947127420a" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    imageauthorcontent_typesizeupdatedblurredresizednormalizedblur_resizedblurred_verbose
    0aliceimage/png15912402025-03-20 17:45:04+00:00{'status': '', 'content': {'uri': 'gs://bigframes_blob_test/image_blur_transformed_verbose/k9-guard-dog-paw-balm.png', 'version': None, 'authorizer': 'bigframes-dev.us.bigframes-default-connection', 'details': None}}
    1bobimage/png11829512025-03-20 17:45:02+00:00{'status': '', 'content': {'uri': 'gs://bigframes_blob_test/image_blur_transformed_verbose/k9-guard-dog-hot-spot-spray.png', 'version': None, 'authorizer': 'bigframes-dev.us.bigframes-default-connection', 'details': None}}
    2bobimage/png15208842025-03-20 17:44:55+00:00{'status': '', 'content': {'uri': 'gs://bigframes_blob_test/image_blur_transformed_verbose/fluffy-buns-chinchilla-food-variety-pack.png', 'version': None, 'authorizer': 'bigframes-dev.us.bigframes-default-connection', 'details': None}}
    3aliceimage/png12354012025-03-20 17:45:19+00:00{'status': '', 'content': {'uri': 'gs://bigframes_blob_test/image_blur_transformed_verbose/purrfect-perch-cat-scratcher.png', 'version': None, 'authorizer': 'bigframes-dev.us.bigframes-default-connection', 'details': None}}
    4bobimage/png15919232025-03-20 17:44:47+00:00{'status': '', 'content': {'uri': 'gs://bigframes_blob_test/image_blur_transformed_verbose/chirpy-seed-deluxe-bird-food.png', 'version': None, 'authorizer': 'bigframes-dev.us.bigframes-default-connection', 'details': None}}
    \n", + "

    5 rows × 10 columns

    \n", + "
    [5 rows x 10 columns in total]" + ], + "text/plain": [ + " image author content_type \\\n", + "0 {'uri': 'gs://cloud-samples-data/bigquery/tuto... alice image/png \n", + "1 {'uri': 'gs://cloud-samples-data/bigquery/tuto... bob image/png \n", + "2 {'uri': 'gs://cloud-samples-data/bigquery/tuto... bob image/png \n", + "3 {'uri': 'gs://cloud-samples-data/bigquery/tuto... alice image/png \n", + "4 {'uri': 'gs://cloud-samples-data/bigquery/tuto... bob image/png \n", + "\n", + " size updated \\\n", + "0 1591240 2025-03-20 17:45:04+00:00 \n", + "1 1182951 2025-03-20 17:45:02+00:00 \n", + "2 1520884 2025-03-20 17:44:55+00:00 \n", + "3 1235401 2025-03-20 17:45:19+00:00 \n", + "4 1591923 2025-03-20 17:44:47+00:00 \n", + "\n", + " blurred \\\n", + "0 {'uri': 'gs://bigframes_blob_test/image_blur_t... \n", + "1 {'uri': 'gs://bigframes_blob_test/image_blur_t... \n", + "2 {'uri': 'gs://bigframes_blob_test/image_blur_t... \n", + "3 {'uri': 'gs://bigframes_blob_test/image_blur_t... \n", + "4 {'uri': 'gs://bigframes_blob_test/image_blur_t... \n", + "\n", + " resized \\\n", + "0 {'uri': 'gs://bigframes_blob_test/image_resize... \n", + "1 {'uri': 'gs://bigframes_blob_test/image_resize... \n", + "2 {'uri': 'gs://bigframes_blob_test/image_resize... \n", + "3 {'uri': 'gs://bigframes_blob_test/image_resize... \n", + "4 {'uri': 'gs://bigframes_blob_test/image_resize... \n", + "\n", + " normalized \\\n", + "0 {'uri': 'gs://bigframes_blob_test/image_normal... \n", + "1 {'uri': 'gs://bigframes_blob_test/image_normal... \n", + "2 {'uri': 'gs://bigframes_blob_test/image_normal... \n", + "3 {'uri': 'gs://bigframes_blob_test/image_normal... \n", + "4 {'uri': 'gs://bigframes_blob_test/image_normal... \n", + "\n", + " blur_resized \\\n", + "0 {'uri': 'gs://bigframes_blob_test/image_blur_r... \n", + "1 {'uri': 'gs://bigframes_blob_test/image_blur_r... \n", + "2 {'uri': 'gs://bigframes_blob_test/image_blur_r... \n", + "3 {'uri': 'gs://bigframes_blob_test/image_blur_r... \n", + "4 {'uri': 'gs://bigframes_blob_test/image_blur_r... \n", + "\n", + " blurred_verbose \n", + "0 {'status': '', 'content': {'uri': 'gs://bigfra... \n", + "1 {'status': '', 'content': {'uri': 'gs://bigfra... \n", + "2 {'status': '', 'content': {'uri': 'gs://bigfra... \n", + "3 {'status': '', 'content': {'uri': 'gs://bigfra... \n", + "4 {'status': '', 'content': {'uri': 'gs://bigfra... \n", + "\n", + "[5 rows x 10 columns]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_image" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Euk5saeVVdTP" + }, + "source": [ + "### 4. Use LLM models to ask questions and generate embeddings on images" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "id": "mRUGfcaFVW-3" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py:182: FutureWarning: Since upgrading the default model can cause unintended breakages, the\n", + "default model will be removed in BigFrames 3.0. Please supply an\n", + "explicit model to avoid this message.\n", + " return method(*args, **kwargs)\n" + ] + } + ], + "source": [ + "from bigframes.ml import llm\n", + "gemini = llm.GeminiTextGenerator()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 657 + }, + "id": "DNFP7CbjWdR9", + "outputId": "3f90a062-0abc-4bce-f53c-db57b06a14b9" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    ml_generate_text_llm_resultimage
    0The item is a tin of K9 Guard dog paw balm.
    1The item is K9 Guard Dog Hot Spot Spray.
    \n", + "

    2 rows × 2 columns

    \n", + "
    [2 rows x 2 columns in total]" + ], + "text/plain": [ + " ml_generate_text_llm_result \\\n", + "0 The item is a tin of K9 Guard dog paw balm. \n", + "1 The item is K9 Guard Dog Hot Spot Spray. \n", + "\n", + " image \n", + "0 {'uri': 'gs://cloud-samples-data/bigquery/tuto... \n", + "1 {'uri': 'gs://cloud-samples-data/bigquery/tuto... \n", + "\n", + "[2 rows x 2 columns]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Ask the same question on the images\n", + "df_image = df_image.head(2)\n", + "answer = gemini.predict(df_image, prompt=[\"what item is it?\", df_image[\"image\"]])\n", + "answer[[\"ml_generate_text_llm_result\", \"image\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "id": "IG3J3HsKhyBY" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + } + ], + "source": [ + "# Ask different questions\n", + "df_image[\"question\"] = [\"what item is it?\", \"what color is the picture?\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 657 + }, + "id": "qKOb765IiVuD", + "outputId": "731bafad-ea29-463f-c8c1-cb7acfd70e5d" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    ml_generate_text_llm_resultimage
    0The item is a tin of K9Guard Dog Paw Balm.
    1The bottle is mostly white, with a light blue accents. The background is a light gray. There are also black and green elements on the bottle's label.
    \n", + "

    2 rows × 2 columns

    \n", + "
    [2 rows x 2 columns in total]" + ], + "text/plain": [ + " ml_generate_text_llm_result \\\n", + "0 The item is a tin of K9Guard Dog Paw Balm. \n", + "1 The bottle is mostly white, with a light blue ... \n", + "\n", + " image \n", + "0 {'uri': 'gs://cloud-samples-data/bigquery/tuto... \n", + "1 {'uri': 'gs://cloud-samples-data/bigquery/tuto... \n", + "\n", + "[2 rows x 2 columns]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "answer_alt = gemini.predict(df_image, prompt=[df_image[\"question\"], df_image[\"image\"]])\n", + "answer_alt[[\"ml_generate_text_llm_result\", \"image\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 300 + }, + "id": "KATVv2CO5RT1", + "outputId": "6ec01f27-70b6-4f69-c545-e5e3c879480c" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py:182: FutureWarning: Since upgrading the default model can cause unintended breakages, the\n", + "default model will be removed in BigFrames 3.0. Please supply an\n", + "explicit model to avoid this message.\n", + " return method(*args, **kwargs)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    ml_generate_embedding_resultml_generate_embedding_statusml_generate_embedding_start_secml_generate_embedding_end_seccontent
    0[ 0.00638842 0.01666344 0.00451782 ... -0.02...<NA><NA>{\"access_urls\":{\"expiry_time\":\"2025-10-25T00:2...
    1[ 0.00973689 0.02148374 0.00244311 ... 0.00...<NA><NA>{\"access_urls\":{\"expiry_time\":\"2025-10-25T00:2...
    \n", + "

    2 rows × 5 columns

    \n", + "
    [2 rows x 5 columns in total]" + ], + "text/plain": [ + " ml_generate_embedding_result \\\n", + "0 [ 0.00638842 0.01666344 0.00451782 ... -0.02... \n", + "1 [ 0.00973689 0.02148374 0.00244311 ... 0.00... \n", + "\n", + " ml_generate_embedding_status ml_generate_embedding_start_sec \\\n", + "0 \n", + "1 \n", + "\n", + " ml_generate_embedding_end_sec \\\n", + "0 \n", + "1 \n", + "\n", + " content \n", + "0 {\"access_urls\":{\"expiry_time\":\"2025-10-25T00:2... \n", + "1 {\"access_urls\":{\"expiry_time\":\"2025-10-25T00:2... \n", + "\n", + "[2 rows x 5 columns]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Generate embeddings.\n", + "embed_model = llm.MultimodalEmbeddingGenerator()\n", + "embeddings = embed_model.predict(df_image[\"image\"])\n", + "embeddings" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iRUi8AjG7cIf" + }, + "source": [ + "### 5. PDF chunking function" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "id": "oDDuYtUm5Yiy" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + } + ], + "source": [ + "df_pdf = bpd.from_glob_path(\"gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/documents/*\", name=\"pdf\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7jLpMYaj7nj8", + "outputId": "06d5456f-580f-4693-adff-2605104b056c" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py:182: FunctionAxisOnePreviewWarning: Blob Functions use bigframes DataFrame Managed function with axis=1 senario, which is a preview feature.\n", + " return method(*args, **kwargs)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/bigquery/_operations/json.py:239: UserWarning: The `json_extract_string_array` is deprecated and will be removed in a\n", + "future version. Use `json_value_array` instead.\n", + " warnings.warn(bfe.format_message(msg), category=UserWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/bigquery/_operations/json.py:239: UserWarning: The `json_extract_string_array` is deprecated and will be removed in a\n", + "future version. Use `json_value_array` instead.\n", + " warnings.warn(bfe.format_message(msg), category=UserWarning)\n" + ] + } + ], + "source": [ + "df_pdf[\"chunked\"] = df_pdf[\"pdf\"].blob.pdf_chunk(engine=\"pypdf\")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py:182: FunctionAxisOnePreviewWarning: Blob Functions use bigframes DataFrame Managed function with axis=1 senario, which is a preview feature.\n", + " return method(*args, **kwargs)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/bigquery/_operations/json.py:239: UserWarning: The `json_extract_string_array` is deprecated and will be removed in a\n", + "future version. Use `json_value_array` instead.\n", + " warnings.warn(bfe.format_message(msg), category=UserWarning)\n" + ] + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    chunked_verbose
    0{'status': '', 'content': array([\"CritterCuisi...
    \n", + "

    1 rows × 1 columns

    \n", + "
    [1 rows x 1 columns in total]" + ], + "text/plain": [ + " chunked_verbose\n", + "0 {'status': '', 'content': array([\"CritterCuisi...\n", + "\n", + "[1 rows x 1 columns]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_pdf[\"chunked_verbose\"] = df_pdf[\"pdf\"].blob.pdf_chunk(engine=\"pypdf\", verbose=True)\n", + "df_pdf[[\"chunked_verbose\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "id": "kaPvJATN7zlw" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + }, + { + "data": { + "text/plain": [ + "0 CritterCuisine Pro 5000 - Automatic Pet Feeder...\n", + "0 on a level, stable surface to prevent tipping....\n", + "0 included)\\nto maintain the schedule during pow...\n", + "0 digits for Meal 1 will flash.\\n\u0000. Use the UP/D...\n", + "0 paperclip) for 5\\nseconds. This will reset all...\n", + "0 unit with a damp cloth. Do not immerse the bas...\n", + "0 continues,\\ncontact customer support.\\nE2: Foo...\n", + "Name: chunked, dtype: string" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "chunked = df_pdf[\"chunked\"].explode()\n", + "chunked" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6. Audio transcribe function" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + } + ], + "source": [ + "audio_gcs_path = \"gs://bigframes_blob_test/audio/*\"\n", + "df = bpd.from_glob_path(audio_gcs_path, name=\"audio\")" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + }, + { + "data": { + "text/plain": [ + "0 Now, as all books, not primarily intended as p...\n", + "Name: transcribed_content, dtype: string" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "transcribed_series = df['audio'].blob.audio_transcribe(model_name=\"gemini-2.0-flash-001\", verbose=False)\n", + "transcribed_series" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", + "/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", + "instead of using `db_dtypes` in the future when available in pandas\n", + "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", + " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" + ] + }, + { + "data": { + "text/plain": [ + "0 {'status': '', 'content': 'Now, as all books, ...\n", + "Name: transcription_results, dtype: struct[pyarrow]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "transcribed_series_verbose = df['audio'].blob.audio_transcribe(model_name=\"gemini-2.0-flash-001\", verbose=True)\n", + "transcribed_series_verbose" + ] + } + ], + "metadata": { "colab": { - "base_uri": "https://localhost:8080/", - "height": 300 + "provenance": [] }, - "id": "KATVv2CO5RT1", - "outputId": "6ec01f27-70b6-4f69-c545-e5e3c879480c" - }, - "outputs": [], - "source": [ - "# Generate embeddings.\n", - "embed_model = llm.MultimodalEmbeddingGenerator()\n", - "embeddings = embed_model.predict(df_image[\"image\"])\n", - "embeddings" - ] - }, - { - "cell_type": "markdown", - "id": "23892b0e", - "metadata": { - "id": "iRUi8AjG7cIf" - }, - "source": [ - "### 5. PDF extraction and chunking function\n", - "\n", - "This section demonstrates how to extract text and chunk text from PDF files using custom BigQuery Python UDFs and the `pypdf` library." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "136a18b8", - "metadata": {}, - "outputs": [], - "source": [ - "# Construct the canonical connection ID\n", - "FULL_CONNECTION_ID = f\"{PROJECT}.{LOCATION}.bigframes-default-connection\"\n", - "\n", - "@bpd.udf(\n", - " input_types=[str],\n", - " output_type=str,\n", - " dataset=DATASET_ID,\n", - " name=\"pdf_extract\",\n", - " bigquery_connection=FULL_CONNECTION_ID,\n", - " packages=[\"pypdf\", \"requests\", \"cryptography\"],\n", - ")\n", - "def pdf_extract(src_obj_ref_rt: str) -> str:\n", - " import io\n", - " import json\n", - " from pypdf import PdfReader\n", - " import requests\n", - " src_obj_ref_rt_json = json.loads(src_obj_ref_rt)\n", - " src_url = src_obj_ref_rt_json[\"access_urls\"][\"read_url\"]\n", - " response = requests.get(src_url, timeout=30, stream=True)\n", - " response.raise_for_status()\n", - " pdf_bytes = response.content\n", - " pdf_file = io.BytesIO(pdf_bytes)\n", - " reader = PdfReader(pdf_file, strict=False)\n", - " all_text = \"\"\n", - " for page in reader.pages:\n", - " page_extract_text = page.extract_text()\n", - " if page_extract_text:\n", - " all_text += page_extract_text\n", - " return all_text\n", - "\n", - "@bpd.udf(\n", - " input_types=[str, int, int],\n", - " output_type=list[str],\n", - " dataset=DATASET_ID,\n", - " name=\"pdf_chunk\",\n", - " bigquery_connection=FULL_CONNECTION_ID,\n", - " packages=[\"pypdf\", \"requests\", \"cryptography\"],\n", - ")\n", - "def pdf_chunk(src_obj_ref_rt: str, chunk_size: int, overlap_size: int) -> list[str]:\n", - " import io\n", - " import json\n", - " from pypdf import PdfReader\n", - " import requests\n", - " src_obj_ref_rt_json = json.loads(src_obj_ref_rt)\n", - " src_url = src_obj_ref_rt_json[\"access_urls\"][\"read_url\"]\n", - " response = requests.get(src_url, timeout=30, stream=True)\n", - " response.raise_for_status()\n", - " pdf_bytes = response.content\n", - " pdf_file = io.BytesIO(pdf_bytes)\n", - " reader = PdfReader(pdf_file, strict=False)\n", - " all_text_chunks = []\n", - " curr_chunk = \"\"\n", - " for page in reader.pages:\n", - " page_text = page.extract_text()\n", - " if page_text:\n", - " curr_chunk += page_text\n", - " while len(curr_chunk) >= chunk_size:\n", - " split_idx = curr_chunk.rfind(\" \", 0, chunk_size)\n", - " if split_idx == -1:\n", - " split_idx = chunk_size\n", - " actual_chunk = curr_chunk[:split_idx]\n", - " all_text_chunks.append(actual_chunk)\n", - " overlap = curr_chunk[split_idx + 1 : split_idx + 1 + overlap_size]\n", - " curr_chunk = overlap + curr_chunk[split_idx + 1 + overlap_size :]\n", - " if curr_chunk:\n", - " all_text_chunks.append(curr_chunk)\n", - " return all_text_chunks" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "234a5f86", - "metadata": {}, - "outputs": [], - "source": [ - "import gcsfs\n", - "import bigframes.bigquery as bbq\n", - "\n", - "# List files using gcsfs\n", - "fs = gcsfs.GCSFileSystem(anon=True)\n", - "uris = fs.glob(\"gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/documents/*\")\n", - "\n", - "# Ensure URIs have gs:// prefix\n", - "uris = [u if u.startswith(\"gs://\") else f\"gs://{u}\" for u in uris]\n", - "\n", - "# Read the URIs into a BigQuery DataFrame\n", - "df_pdf = bpd.read_gbq(f\"SELECT uri FROM UNNEST({uris[:5]}) as uri\")\n", - "\n", - "# Create the object reference column\n", - "df_pdf['pdf'] = bbq.obj.make_ref(df_pdf['uri'], authorizer=FULL_CONNECTION_ID)\n", - "df_pdf = df_pdf[['pdf']]\n", - "\n", - "# Generate a JSON string containing the runtime information (including signed read URLs)\n", - "access_urls = get_runtime_json_str(df_pdf[\"pdf\"], mode=\"R\")\n", - "\n", - "# Apply PDF extraction\n", - "df_pdf[\"extracted_text\"] = access_urls.apply(pdf_extract)\n", - "\n", - "# Apply PDF chunking\n", - "df_pdf[\"chunked\"] = access_urls.apply(pdf_chunk, args=(2000, 200))\n", - "\n", - "df_pdf[[\"extracted_text\", \"chunked\"]]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d80effbe", - "metadata": {}, - "outputs": [], - "source": [ - "# Explode the chunks to see each chunk as a separate row\n", - "chunked = df_pdf[\"chunked\"].explode()\n", - "chunked" - ] - }, - { - "cell_type": "markdown", - "id": "118cf1c7", - "metadata": {}, - "source": [ - "### 6. Audio transcribe" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1794c54f", - "metadata": {}, - "outputs": [], - "source": [ - "import gcsfs\n", - "import bigframes.bigquery as bbq\n", - "\n", - "audio_gcs_path = \"gs://bigframes_blob_test/audio/*\"\n", - "\n", - "# List files using gcsfs\n", - "fs = gcsfs.GCSFileSystem()\n", - "uris = fs.glob(audio_gcs_path)\n", - "\n", - "# Ensure URIs have gs:// prefix\n", - "uris = [u if u.startswith(\"gs://\") else f\"gs://{u}\" for u in uris]\n", - "\n", - "# Read the URIs into a BigQuery DataFrame\n", - "# If the bucket is empty or doesn't exist, this will result in an empty DataFrame\n", - "if not uris:\n", - " # Fallback to a dummy list or just let it be empty\n", - " uris = [\"gs://bigframes_blob_test/audio/dummy.mp3\"]\n", - "\n", - "df = bpd.read_gbq(f\"SELECT uri FROM UNNEST({uris[:5]}) as uri\")\n", - "\n", - "# Create the object reference column\n", - "df['audio'] = bbq.obj.make_ref(df['uri'], authorizer=FULL_CONNECTION_ID)\n", - "df = df[['audio']]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c9f9d484", - "metadata": {}, - "outputs": [], - "source": [ - "# The audio_transcribe function is a convenience wrapper around bigframes.bigquery.ai.generate.\n", - "# Here's how to perform the same operation directly:\n", - "\n", - "audio_series = df[\"audio\"]\n", - "prompt_text = (\n", - " \"**Task:** Transcribe the provided audio. **Instructions:** - Your response \"\n", - " \"must contain only the verbatim transcription of the audio. - Do not include \"\n", - " \"any introductory text, summaries, or conversational filler in your response. \"\n", - " \"The output should begin directly with the first word of the audio.\"\n", - ")\n", - "\n", - "# Convert the audio series to the runtime representation required by the model.\n", - "# This involves fetching metadata and getting a signed access URL.\n", - "audio_metadata = bbq.obj.fetch_metadata(audio_series)\n", - "audio_runtime = bbq.obj.get_access_url(audio_metadata, mode=\"R\")\n", - "\n", - "transcribed_results = bbq.ai.generate(\n", - " prompt=(prompt_text, audio_runtime),\n", - " endpoint=\"gemini-2.5-flash\",\n", - " model_params={\"generationConfig\": {\"temperature\": 0.0}},\n", - ")\n", - "\n", - "transcribed_series = transcribed_results.struct.field(\"result\").rename(\"transcribed_content\")\n", - "transcribed_series" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7209a62a", - "metadata": {}, - "outputs": [], - "source": [ - "# To get verbose results (including status), we can extract both fields from the result struct.\n", - "transcribed_content_series = transcribed_results.struct.field(\"result\")\n", - "transcribed_status_series = transcribed_results.struct.field(\"status\")\n", - "\n", - "transcribed_series_verbose = bpd.DataFrame(\n", - " {\n", - " \"status\": transcribed_status_series,\n", - " \"content\": transcribed_content_series,\n", - " }\n", - ")\n", - "# Package as a struct for consistent display\n", - "transcribed_series_verbose = bbq.struct(transcribed_series_verbose).rename(\"transcription_results\")\n", - "transcribed_series_verbose" - ] - }, - { - "cell_type": "markdown", - "id": "c8351cc3", - "metadata": {}, - "source": [ - "### 7. Extract EXIF metadata from images" - ] - }, - { - "cell_type": "markdown", - "id": "e59670b9", - "metadata": {}, - "source": [ - "This section demonstrates how to extract EXIF metadata from images using a custom BigQuery Python UDF and the `Pillow` library." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fda362f4", - "metadata": {}, - "outputs": [], - "source": [ - "# Construct the canonical connection ID\n", - "FULL_CONNECTION_ID = f\"{PROJECT}.{LOCATION}.bigframes-default-connection\"\n", - "\n", - "@bpd.udf(\n", - " input_types=[str],\n", - " output_type=str,\n", - " dataset=DATASET_ID,\n", - " name=\"extract_exif\",\n", - " bigquery_connection=FULL_CONNECTION_ID,\n", - " packages=[\"pillow\", \"requests\"],\n", - " max_batching_rows=8192,\n", - " container_cpu=0.33,\n", - " container_memory=\"512Mi\"\n", - ")\n", - "def extract_exif(src_obj_ref_rt: str) -> str:\n", - " import io\n", - " import json\n", - " from PIL import ExifTags, Image\n", - " import requests\n", - " src_obj_ref_rt_json = json.loads(src_obj_ref_rt)\n", - " src_url = src_obj_ref_rt_json[\"access_urls\"][\"read_url\"]\n", - " response = requests.get(src_url, timeout=30)\n", - " bts = response.content\n", - " image = Image.open(io.BytesIO(bts))\n", - " exif_data = image.getexif()\n", - " exif_dict = {}\n", - " if exif_data:\n", - " for tag, value in exif_data.items():\n", - " tag_name = ExifTags.TAGS.get(tag, tag)\n", - " exif_dict[tag_name] = value\n", - " return json.dumps(exif_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "40bb6bc9", - "metadata": {}, - "outputs": [], - "source": [ - "import gcsfs\n", - "import bigframes.bigquery as bbq\n", - "\n", - "# Create a Multimodal DataFrame from the sample image URIs\n", - "fs = gcsfs.GCSFileSystem()\n", - "uris = fs.glob(\"gs://bigframes_blob_test/images_exif/*\")\n", - "\n", - "# Ensure URIs have gs:// prefix\n", - "uris = [u if u.startswith(\"gs://\") else f\"gs://{u}\" for u in uris]\n", - "\n", - "if not uris:\n", - " uris = [\"gs://bigframes_blob_test/images_exif/dummy.jpg\"]\n", - "\n", - "exif_image_df = bpd.read_gbq(f\"SELECT uri FROM UNNEST({uris[:5]}) as uri\")\n", - "exif_image_df['blob_col'] = bbq.obj.make_ref(exif_image_df['uri'], authorizer=FULL_CONNECTION_ID)\n", - "exif_image_df = exif_image_df[['blob_col']]\n", - "\n", - "# Generate a JSON string containing the runtime information (including signed read URLs)\n", - "# This allows the UDF to download the images from Google Cloud Storage\n", - "access_urls = get_runtime_json_str(exif_image_df[\"blob_col\"], mode=\"R\")\n", - "\n", - "# Apply the BigQuery Python UDF to the runtime JSON strings\n", - "# We cast to string to ensure the input matches the UDF's signature\n", - "exif_json = access_urls.astype(str).apply(extract_exif)\n", - "\n", - "# Parse the resulting JSON strings back into a structured JSON type for easier access\n", - "exif_data = bbq.parse_json(exif_json)\n", - "\n", - "exif_data" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "venv (3.13.0)", - "language": "python", - "name": "python3" + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.18" + } }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.0" - } - }, - "nbformat": 4, - "nbformat_minor": 0 + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/notebooks/remote_functions/remote_function.ipynb b/notebooks/remote_functions/remote_function.ipynb index a70d05ae062..e2bc88ecae7 100644 --- a/notebooks/remote_functions/remote_function.ipynb +++ b/notebooks/remote_functions/remote_function.ipynb @@ -1,11 +1,18 @@ { "cells": [ { - "cell_type": "markdown", - "id": "title-cell", + "cell_type": "code", + "execution_count": null, + "id": "bcff4fc4", "metadata": {}, + "outputs": [], "source": [ - "# Remote Functions" + "import sys\n", + "\n", + "# Python 3.13 is not yet a supported runtime for remote functions.\n", + "# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n", + "if sys.version_info >= (3, 13, 0):\n", + " sys.exit(0)" ] }, { diff --git a/notebooks/remote_functions/remote_function_usecases.ipynb b/notebooks/remote_functions/remote_function_usecases.ipynb index e3a94160ad9..03ae6520952 100644 --- a/notebooks/remote_functions/remote_function_usecases.ipynb +++ b/notebooks/remote_functions/remote_function_usecases.ipynb @@ -21,6 +21,20 @@ "# limitations under the License." ] }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Python 3.13 is not yet a supported runtime for remote functions.\n", + "# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n", + "if sys.version_info >= (3, 13, 0):\n", + " sys.exit(0)" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/notebooks/remote_functions/remote_function_vertex_claude_model.ipynb b/notebooks/remote_functions/remote_function_vertex_claude_model.ipynb index dfc993072cf..9792c90205c 100644 --- a/notebooks/remote_functions/remote_function_vertex_claude_model.ipynb +++ b/notebooks/remote_functions/remote_function_vertex_claude_model.ipynb @@ -28,6 +28,20 @@ "" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Python 3.13 is not yet a supported runtime for remote functions.\n", + "# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n", + "if sys.version_info >= (3, 13, 0):\n", + " sys.exit(0)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -94,11 +108,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "PROJECT_ID = \"bigframes-dev\" # @param {type:\"string\"}\n", + "PROJECT = \"bigframes-dev\" # replace with your project\n", "LOCATION = \"us-east5\"" ] }, @@ -115,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -123,7 +137,7 @@ "# and location\n", "\n", "import bigframes.pandas as bpd\n", - "bpd.options.bigquery.project = PROJECT_ID\n", + "bpd.options.bigquery.project = PROJECT\n", "bpd.options.bigquery.location = LOCATION" ] }, @@ -141,6 +155,30 @@ "execution_count": 4, "metadata": {}, "outputs": [ + { + "data": { + "text/html": [ + "Query job c4c27713-51c8-4293-8454-5c904df79318 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 0b1b71d8-8546-45f2-b403-707161fe4002 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "data": { "text/html": [ @@ -229,15 +267,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "\n", - " Query processed 0 Bytes in a moment of slot time. [Job bigframes-dev:us-east5.9bc70627-6891-44a4-b7d7-8a28e213cdec details]\n", - " " + "Query job 488a116f-44b2-4ff7-9f95-bd36473dab0f is DONE. 0 Bytes processed. Open Job" ], "text/plain": [ "" @@ -255,7 +291,7 @@ ")\n", "def anthropic_transformer(message: str) -> str:\n", " from anthropic import AnthropicVertex\n", - " client = AnthropicVertex(region=LOCATION, project_id=PROJECT_ID)\n", + " client = AnthropicVertex(region=LOCATION, project_id=PROJECT)\n", "\n", " message = client.messages.create(\n", " max_tokens=1024,\n", @@ -265,7 +301,7 @@ " \"content\": message,\n", " }\n", " ],\n", - " model=\"claude-3-haiku@20240307\",\n", + " model=\"claude-3-5-sonnet@20240620\",\n", " )\n", " content_text = message.content[0].text if message.content else \"\"\n", " return content_text" @@ -273,16 +309,16 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'bigframes-dev._e9a5162ae4daa9f50fda3f95febaa9781131f3b8.bigframes_sessionc10c73_49262141176cbf70037559ae84e834d3'" + "'bigframes-dev._b52b272a35b88e236e1f96fbe3f560c83a8fee85.bigframes_session265649_de1176dd4c57f40ba959503af3981682'" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -294,16 +330,16 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'projects/bigframes-dev/locations/us-east5/functions/bigframes-sessionc10c73-49262141176cbf70037559ae84e834d3'" + "'projects/bigframes-dev/locations/us-east5/functions/bigframes-session265649-de1176dd4c57f40ba959503af3981682'" ] }, - "execution_count": 7, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -315,40 +351,49 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "\n", - " Query started with request ID bigframes-dev:us-east5.821579f4-63ea-4072-a3ce-318e43768432.
    SQL
    SELECT\n",
-       "`bfuid_col_3` AS `bfuid_col_3`,\n",
-       "`bfuid_col_4` AS `bfuid_col_4`,\n",
-       "`bfuid_col_5` AS `bfuid_col_5`\n",
-       "FROM\n",
-       "(SELECT\n",
-       "  `t1`.`bfuid_col_3`,\n",
-       "  `t1`.`bfuid_col_4`,\n",
-       "  `t1`.`bfuid_col_5`,\n",
-       "  `t1`.`bfuid_col_6` AS `bfuid_col_7`\n",
-       "FROM (\n",
-       "  SELECT\n",
-       "    `t0`.`level_0`,\n",
-       "    `t0`.`column_0`,\n",
-       "    `t0`.`bfuid_col_6`,\n",
-       "    `t0`.`level_0` AS `bfuid_col_3`,\n",
-       "    `t0`.`column_0` AS `bfuid_col_4`,\n",
-       "    `bigframes-dev._e9a5162ae4daa9f50fda3f95febaa9781131f3b8.bigframes_sessionc10c73_49262141176cbf70037559ae84e834d3`(`t0`.`column_0`) AS `bfuid_col_5`\n",
-       "  FROM (\n",
-       "    SELECT\n",
-       "      *\n",
-       "    FROM UNNEST(ARRAY<STRUCT<`level_0` INT64, `column_0` STRING, `bfuid_col_6` INT64>>[STRUCT(0, 'What is the capital of France?', 0), STRUCT(1, 'Explain the concept of photosynthesis in simple terms.', 1), STRUCT(2, 'Write a haiku about artificial intelligence.', 2)]) AS `level_0`\n",
-       "  ) AS `t0`\n",
-       ") AS `t1`)\n",
-       "ORDER BY `bfuid_col_7` ASC NULLS LAST\n",
-       "LIMIT 10
    \n", - " " + "Query job 58b230a8-6536-4bac-ab02-dcf574692dd6 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 46d6a1e9-426a-4615-8eb5-98d34d08ec07 is DONE. 1.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job cf8fcbaa-b233-47cd-b4e3-60876b24879f is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 2620a544-d1df-4b30-bec4-4221e79ddf0f is DONE. 1.4 kB processed. Open Job" ], "text/plain": [ "" @@ -391,12 +436,13 @@ " \n", " 1\n", " Explain the concept of photosynthesis in simpl...\n", - " Photosynthesis is the process by which plants ...\n", + " Photosynthesis is the process plants use to ma...\n", " \n", " \n", " 2\n", " Write a haiku about artificial intelligence.\n", - " Here is a haiku about artificial intelligence:...\n", + " Here's a haiku about artificial intelligence:\n", + "...\n", " \n", " \n", "\n", @@ -411,13 +457,14 @@ "\n", " answers \n", "0 The capital of France is Paris. \n", - "1 Photosynthesis is the process by which plants ... \n", - "2 Here is a haiku about artificial intelligence:... \n", + "1 Photosynthesis is the process plants use to ma... \n", + "2 Here's a haiku about artificial intelligence:\n", + "... \n", "\n", "[3 rows x 2 columns]" ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -437,22 +484,9 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "Session sessionc10c73 closed." - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "bpd.close_session()" ] @@ -460,7 +494,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv (3.14.2)", + "display_name": "venv", "language": "python", "name": "python3" }, @@ -474,7 +508,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.14.2" + "version": "3.10.9" } }, "nbformat": 4, diff --git a/notebooks/streaming/streaming_dataframe.ipynb b/notebooks/streaming/streaming_dataframe.ipynb index e3dafa98195..b7da0cfd077 100644 --- a/notebooks/streaming/streaming_dataframe.ipynb +++ b/notebooks/streaming/streaming_dataframe.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# BigFrames StreamingDataFrame", + "### BigFrames StreamingDataFrame\n", "bigframes.streaming.StreamingDataFrame is a special DataFrame type that allows simple operations and can create streaming jobs to process real-time data and reverse ETL output to Bigtable and Pub/Sub using [BigQuery continuous queries](https://cloud.google.com/bigquery/docs/continuous-queries-introduction).\n", "\n", "In this notebook, we will:\n", @@ -97,7 +97,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Create, select, filter and preview", + "### Create, select, filter and preview\n", "Create the StreamingDataFrame from a BigQuery table, select certain columns, filter rows and preview the output" ] }, diff --git a/notebooks/visualization/bq_dataframes_covid_line_graphs.ipynb b/notebooks/visualization/bq_dataframes_covid_line_graphs.ipynb index b28df7b0d7d..d69aecd8c30 100644 --- a/notebooks/visualization/bq_dataframes_covid_line_graphs.ipynb +++ b/notebooks/visualization/bq_dataframes_covid_line_graphs.ipynb @@ -1,648 +1,648 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "9GIt_orUtNvA" - }, - "outputs": [], - "source": [ - "# Copyright 2023 Google LLC\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "h7AT6h2ItNvD" - }, - "source": [ - "# Use BigQuery DataFrames to visualize COVID-19 data", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \"Colab Run in Colab\n", - " \n", - " \n", - " \n", - " \"GitHub\n", - " View on GitHub\n", - " \n", - " \n", - " \n", - " \"BQ\n", - " Open in BQ Studio\n", - " \n", - "
    " - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "n-MFJQxLtNvE" - }, - "source": [ - "## Overview\n", - "\n", - "The goal of this notebook is to demonstrate creating line graphs from a ~20 million-row BigQuery dataset using BigQuery DataFrames. We will first create a plain line graph using matplotlip, then we will downsample and download our data to create a graph with a line of best fit using seaborn.\n", - "\n", - "If you're like me, during 2020 (and/or later years) you often found yourself looking at charts like [these](https://health.google.com/covid-19/open-data/explorer/statistics) visualizing COVID-19 cases over time. For our first graph, we're going to recreate one of those charts by filtering, summing, and then graphing COVID-19 data from the United States. BigQuery DataFrame's default integration with matplotlib will get us a satisfying result for this first graph.\n", - "\n", - "For our second graph, though, we want to use a scatterplot with a line of best fit, something that matplotlib will not do for us automatically. So, we'll demonstrate how to downsample our data and use seaborn to make our plot. Our second graph will be of symptom-related search trends against new cases of COVID-19, so we'll see if searches for things like \"cough\" and \"fever\" are more common in the places and times where more new cases of COVID-19 occur." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "ffqBzbNztNvF" - }, - "source": [ - "### Dataset\n", - "\n", - "This notebook uses the [BigQuery COVID-19 Open Data](https://pantheon.corp.google.com/marketplace/product/bigquery-public-datasets/covid19-open-data). In this dataset, each row represents a new observation of the COVID-19 situation in a particular time and place. We will use the \"new_confirmed\" column, which contains the number of new COVID-19 cases at each observation, along with the \"search_trends_cough\", \"search_trends_fever\", and \"search_trends_bruise\" columns, which are [Google Trends](https://trends.google.com/trends/) data for searches related to cough, fever, and bruises. In the first section of the notebook, we will also use the \"country_code\" and \"date\" columns to compile one data point per day for a particular country." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Nf__tMR-tNvF" - }, - "source": [ - "### Costs\n", - "\n", - "This tutorial uses billable components of Google Cloud:\n", - "\n", - "* BigQuery (compute)\n", - "\n", - "Learn about [BigQuery compute pricing](https://cloud.google.com/bigquery/pricing#analysis_pricing_models),\n", - "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n", - "to generate a cost estimate based on your projected usage." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7_rsbkCktNvG" - }, - "source": [ - "## Before you begin\n", - "\n", - "### Set up your Google Cloud project\n", - "\n", - "**The following steps are required, regardless of your notebook environment.**\n", - "\n", - "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n", - "\n", - "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", - "\n", - "3. [Enable the BigQuery API](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com).\n", - "\n", - "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "XZKC6iMFxmMG" - }, - "source": [ - "#### Set your project ID\n", - "\n", - "**If you don't know your project ID**, try the following:\n", - "* Run `gcloud config list`.\n", - "* Run `gcloud projects list`.\n", - "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "4aooKMmnxrWF" - }, - "outputs": [], - "source": [ - "PROJECT_ID = \"\" # @param {type:\"string\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "pv5A8Tm-yC1U" - }, - "source": [ - "#### Set the region\n", - "\n", - "You can also change the `REGION` variable used by BigQuery. Learn more about [BigQuery regions](https://cloud.google.com/bigquery/docs/locations#supported_locations)." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "bk03Rt_HyGx-" - }, - "outputs": [], - "source": [ - "REGION = \"US\" # @param {type: \"string\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "B9RWxD1btNvK" - }, - "source": [ - "Now we are ready to use BigQuery DataFrames!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "wJ0gXezj2w1t" - }, - "source": [ - "## Visualization #1: Cases over time in the US" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "xckgWno6ouHY" - }, - "source": [ - "### Set up project and filter data" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "-uiY0hh4tNvK" - }, - "source": [ - "First, let's do project setup. We use options to tell BigQuery DataFrames what project and what region to use for our cloud computing." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "id": "R7STCS8xB5d2" - }, - "outputs": [], - "source": [ - "import bigframes.pandas as bpd\n", - "\n", - "# Note: The project option is not required in all environments.\n", - "# On BigQuery Studio, the project ID is automatically detected.\n", - "bpd.options.bigquery.project = PROJECT_ID\n", - "\n", - "# Note: The location option is not required.\n", - "# It defaults to the location of the first table or query\n", - "# passed to read_gbq(). For APIs where a location can't be\n", - "# auto-detected, the location defaults to the \"US\" location.\n", - "bpd.options.bigquery.location = REGION\n", - "# Improves performance by avoiding generating total row ordering\n", - "bpd.options.bigquery.ordering_mode = \"partial\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "v6FGschEowht" - }, - "source": [ - "Next, we read the data from a publicly available BigQuery dataset. This will take ~1 minute." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "zDSwoBo1CU3G" - }, - "outputs": [], - "source": [ - "all_data = bpd.read_gbq(\"bigquery-public-data.covid19_open_data.covid19_open_data\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9qV2y3iHp13y" - }, - "source": [ - "Using pandas syntax, we will select from our all_data input dataframe only those rows where the country_code is US. This is called row filtering." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "id": "UjMT_qhjf8Fu" - }, - "outputs": [], - "source": [ - "usa_data = all_data[all_data[\"country_code\"] == \"US\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "IYCUayWkwq8c" - }, - "source": [ - "We're only concerned with the date and the total number of confirmed cases for now, so select just those two columns as well." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "id": "IaoUf57ZwrJ8" - }, - "outputs": [], - "source": [ - "usa_data = usa_data[[\"date\", \"new_confirmed\"]]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "94oqNRnDvGkr" - }, - "source": [ - "### Sum data" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "TNCQWZW83U0b" - }, - "source": [ - "`usa_data.groupby(\"date\")` will give us a groupby object that lets us perform operations on groups of rows with the same date. We call sum on that object to get the sum for each day. This process might be familiar to pandas users." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "id": "tYDoaKgJChiq" - }, - "outputs": [], - "source": [ - "# numeric_only = True because we don't want to sum dates\n", - "new_cases_usa = usa_data.groupby(\"date\").sum(numeric_only = True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3jcwFPgK5BLh" - }, - "source": [ - "### Line graph" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8GvJAgnH5Nzi" - }, - "source": [ - "BigQuery DataFrames implements some plotting methods with the matplotlib backend. Use `DataFrame.plot.line()` to draw a simple line graph." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "id": "gFbCgfFC2gHw" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHkCAYAAADCag6yAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAfvpJREFUeJzt3Xd8U1X/B/BP0r0HUAq07L3LbgEBZYoKD4/ggwMcoD6KgjhBxcdZFBFQ+KGigqCIojJERRApyKbMsoplldVBoXsn5/dHaXpvmqRJm/Qml8/79eqL9OYmPYemud98z/ecoxFCCBARERGphFbpBhARERHZE4MbIiIiUhUGN0RERKQqDG6IiIhIVRjcEBERkaowuCEiIiJVYXBDREREqsLghoiIiFSFwQ0RERGpCoMbIiIiUpVbOrjZvn077r77bjRs2BAajQZr1661+TmEEPjwww/RunVreHl5oVGjRnj33Xft31giIiKyirvSDVBSXl4eunTpgkcffRRjxoyp1nNMnToVmzZtwocffohOnTrh+vXruH79up1bSkRERNbScOPMMhqNBmvWrMHo0aMNx4qKivDqq6/iu+++Q2ZmJjp27Ij3338fAwcOBACcPHkSnTt3xrFjx9CmTRtlGk5EREQyt/SwVFWmTJmC3bt3Y9WqVTh69CjGjh2L4cOH459//gEA/PLLL2jevDk2bNiAZs2aoWnTppg0aRIzN0RERApicGNGcnIyli5ditWrV6N///5o0aIFXnjhBfTr1w9Lly4FAJw9exYXLlzA6tWrsXz5cixbtgwHDhzAvffeq3DriYiIbl23dM2NJQkJCdDpdGjdurXseFFREerUqQMA0Ov1KCoqwvLlyw3nffnll+jevTsSExM5VEVERKQABjdm5Obmws3NDQcOHICbm5vsPn9/fwBAgwYN4O7uLguA2rVrB6As88PghoiIqPYxuDEjKioKOp0OaWlp6N+/v8lz+vbti9LSUpw5cwYtWrQAAJw+fRoA0KRJk1prKxEREVW4pWdL5ebmIikpCUBZMPPRRx9h0KBBCA0NRePGjfHggw9i586dmDt3LqKiopCeno4tW7agc+fOGDlyJPR6PXr27Al/f3/Mnz8fer0eTz/9NAIDA7Fp0yaFe0dERHRruqWDm7i4OAwaNKjS8YkTJ2LZsmUoKSnBO++8g+XLl+Py5cuoW7cu+vTpgzfffBOdOnUCAFy5cgXPPPMMNm3aBD8/P4wYMQJz585FaGhobXeHiIiIcIsHN0RERKQ+nApOREREqnLLFRTr9XpcuXIFAQEB0Gg0SjeHiIiIrCCEQE5ODho2bAit1nJu5pYLbq5cuYLIyEilm0FERETVcPHiRURERFg855YLbgICAgCU/ecEBgYq3BoiIiKyRnZ2NiIjIw3XcUtuueCmfCgqMDCQwQ0REZGLsaakhAXFREREpCoMboiIiEhVGNwQERGRqtxyNTfW0ul0KCkpUboZpFIeHh6VNmQlIiL7YHBjRAiBlJQUZGZmKt0UUrng4GCEh4dzvSUiIjtjcGOkPLAJCwuDr68vLzxkd0II5OfnIy0tDQDQoEEDhVtERKQuDG4kdDqdIbCpU6eO0s0hFfPx8QEApKWlISwsjENURER2xIJiifIaG19fX4VbQreC8tcZa7uIiOyLwY0JHIqi2sDXGRGRYzC4ISIiIlVxmuBm9uzZ0Gg0mDZtmsXzVq9ejbZt28Lb2xudOnXCb7/9VjsNJCIiIpfgFMHN/v378dlnn6Fz584Wz9u1axfGjx+Pxx57DIcOHcLo0aMxevRoHDt2rJZaSs5i586d6NSpEzw8PDB69GjExcVBo9E41RT+pk2bYv78+Uo3g4jolqN4cJObm4sHHngAS5YsQUhIiMVzFyxYgOHDh+PFF19Eu3bt8Pbbb6Nbt25YuHBhLbWWnMX06dPRtWtXnDt3DsuWLUNMTAyuXr2KoKAgpZtGREQKUzy4efrppzFy5EgMHjy4ynN3795d6bxhw4Zh9+7dZh9TVFSE7Oxs2Re5vjNnzuD2229HREQEgoOD4enpaXFBPJ1OB71eX8utJCJr6fUCM9ckYOXeZKWbQiqgaHCzatUqHDx4ELGxsVadn5KSgvr168uO1a9fHykpKWYfExsbi6CgIMNXZGSkTW0UQiC/uFSRLyGE1e0cOHAgnn32Wbz00ksIDQ1FeHg4/ve//xnuz8zMxKRJk1CvXj0EBgbi9ttvx5EjRwAAWVlZcHNzQ3x8PABAr9cjNDQUffr0MTz+m2++sfr/7tKlSxg/fjxCQ0Ph5+eHHj16YO/evYb7Fy9ejBYtWsDT0xNt2rTBihUrZI/XaDT44osv8K9//Qu+vr5o1aoV1q9fDwA4f/48NBoNMjIy8Oijj0Kj0WDZsmWVhqWWLVuG4OBgrF+/Hu3bt4eXlxeSk5PRtGlTvPPOO5gwYQL8/f3RpEkTrF+/Hunp6Rg1ahT8/f3RuXNnw/9FuR07dqB///7w8fFBZGQknn32WeTl5RnuT0tLw9133w0fHx80a9YM3377rVX/V0RUZtvpdKzcm4yZaxKUbgqpgGKL+F28eBFTp07F5s2b4e3t7bCfM2PGDEyfPt3wfXZ2tk0BTkGJDu1n/eGIplXpxFvD4Otp/a/o66+/xvTp07F3717s3r0bDz/8MPr27YshQ4Zg7Nix8PHxwe+//46goCB89tlnuOOOO3D69GmEhoaia9euiIuLQ48ePZCQkACNRoNDhw4hNzcX/v7+2LZtGwYMGFBlG3JzczFgwAA0atQI69evR3h4OA4ePGjImqxZswZTp07F/PnzMXjwYGzYsAGPPPIIIiIiMGjQIMPzvPnmm/jggw8wZ84cfPLJJ3jggQdw4cIFREZG4urVq2jTpg3eeust3HfffQgKCpIFT+Xy8/Px/vvv44svvkCdOnUQFhYGAJg3bx7ee+89vP7665g3bx4eeughxMTE4NFHH8WcOXPw8ssvY8KECTh+/Dg0Gg3OnDmD4cOH45133sFXX32F9PR0TJkyBVOmTMHSpUsBAA8//DCuXLmCrVu3wsPDA88++6xhBWIiqlpWAdd7IvtRLLg5cOAA0tLS0K1bN8MxnU6H7du3Y+HChSgqKqq0amt4eDhSU1Nlx1JTUxEeHm7253h5ecHLy8u+jXdSnTt3xhtvvAEAaNWqFRYuXIgtW7bAx8cH+/btQ1pamuH/4sMPP8TatWvx448/4vHHH8fAgQMRFxeHF154AXFxcRgyZAhOnTqFHTt2YPjw4YiLi8NLL71UZRtWrlyJ9PR07N+/H6GhoQCAli1bGu7/8MMP8fDDD+Opp54CUFY7s2fPHnz44Yey4Obhhx/G+PHjAQDvvfcePv74Y+zbtw/Dhw83DD8FBQVZ/N2XlJTg//7v/9ClSxfZ8TvvvBNPPPEEAGDWrFlYvHgxevbsibFjxwIAXn75ZURHRxteW7GxsXjggQcMM/latWqFjz/+GAMGDMDixYuRnJyM33//Hfv27UPPnj0BAF9++SXatWtX5f8XEZXhsk9kT4oFN3fccQcSEuTpx0ceeQRt27bFyy+/bHI5+ujoaGzZskU2XXzz5s2Ijo52WDt9PNxw4q1hDnv+qn62LYxnmzVo0ABpaWk4cuQIcnNzK20pUVBQgDNnzgAABgwYgC+//BI6nQ7btm3D0KFDER4ejri4OHTu3BlJSUkYOHBglW04fPgwoqKiDIGNsZMnT+Lxxx+XHevbty8WLFhgti9+fn4IDAy0ORPi6elpcgae9Fj5MGenTp0qHUtLS0N4eDiOHDmCo0ePyoaahBDQ6/U4d+4cTp8+DXd3d3Tv3t1wf9u2bREcHGxTe4mIyD4UC24CAgLQsWNH2TE/Pz/UqVPHcHzChAlo1KiRoSZn6tSpGDBgAObOnYuRI0di1apViI+Px+eff+6wdmo0GpuGhpTk4eEh+16j0UCv1yM3NxcNGjRAXFxcpceUX4Bvu+025OTk4ODBg9i+fTvee+89hIeHY/bs2ejSpQsaNmyIVq1aVdmG8j2TaspcX2zh4+NjssBY+tzl95s6Vv7zcnNz8cQTT+DZZ5+t9FyNGzfG6dOnbWoXERE5llNftZOTk6HVVtQ8x8TEYOXKlXjttdcwc+ZMtGrVCmvXrq0UJJFct27dkJKSAnd3dzRt2tTkOcHBwejcuTMWLlwIDw8PtG3bFmFhYbjvvvuwYcMGq+ptgLKsyBdffIHr16+bzN60a9cOO3fuxMSJEw3Hdu7cifbt21erb7WhW7duOHHihGx4Tapt27YoLS3FgQMHDMNSiYmJTrXmDhHRrcSpghvjzIKpTMPYsWMNtRFkncGDByM6OhqjR4/GBx98gNatW+PKlSv49ddf8a9//Qs9evQAUDbj6pNPPsG9994LAAgNDUW7du3w/fffY9GiRVb9rPHjx+O9997D6NGjERsbiwYNGuDQoUNo2LAhoqOj8eKLL2LcuHGIiorC4MGD8csvv+Dnn3/Gn3/+6bD+19TLL7+MPn36YMqUKZg0aRL8/Pxw4sQJbN68GQsXLkSbNm0wfPhwPPHEE1i8eDHc3d0xbdo0u2WxiIjINoqvc0OOp9Fo8Ntvv+G2227DI488gtatW+M///kPLly4IJtaP2DAAOh0OlltzcCBAysds8TT0xObNm1CWFgY7rzzTnTq1AmzZ8821FCNHj0aCxYswIcffogOHTrgs88+w9KlS61+fiV07twZ27Ztw+nTp9G/f39ERUVh1qxZaNiwoeGcpUuXomHDhhgwYADGjBmDxx9/3DA7i4iIapdG2LKYigpkZ2cjKCgIWVlZCAwMlN1XWFiIc+fOoVmzZg6dnk4E8PVGJLX+yBU8+90hAMD52SMVbg05I0vXb2PM3BAREZGqMLghm7z33nvw9/c3+TVixAilm0dERORcBcXk/J588kmMGzfO5H0soCWi6uIafmRPDG7IJqGhoWYX6CMiInIGHJYy4RarsSaF8HVGVIHbL5A9MbiRKF+lNj8/X+GW0K2g/HVmvBozERHVDIelJNzc3BAcHGzYw8jX19fk8v1ENSGEQH5+PtLS0hAcHGxyHzUiIqo+BjdGyneZtnWTRiJbBQcHW9zVnIiIqofBjRGNRoMGDRogLCwMJSUlSjeHVMrDw4MZGyIJDedLkR0xuDHDzc2NFx8iIiIXxIJiIiIiUhUGN0RERKQqDG6IiEhxnJhK9sTghoiIiFSFwQ0RESmOiRuyJwY3REREpCoMboiIiEhVGNwQEZHiWFBM9sTghoiIiFSFwQ0RETkBpm7IfhjcEBERkaowuCEiIiJVYXBDRERORQihdBPIxTG4ISIixUlnSzG2oZpicENERE6FsQ3VFIMbIiIiUhUGN0RE5FRYc0M1xeCGiIgUJ13lhqEN1RSDGyIicipM3FBNKRrcLF68GJ07d0ZgYCACAwMRHR2N33//3ez5y5Ytg0ajkX15e3vXYouJiMgRNJLpUoK5G6ohdyV/eEREBGbPno1WrVpBCIGvv/4ao0aNwqFDh9ChQweTjwkMDERiYqLhew13WyMiIiIJRYObu+++W/b9u+++i8WLF2PPnj1mgxuNRoPw8PDaaB4RESmAw1JUU05Tc6PT6bBq1Srk5eUhOjra7Hm5ublo0qQJIiMjMWrUKBw/ftzi8xYVFSE7O1v2RUREzoU5eLInxYObhIQE+Pv7w8vLC08++STWrFmD9u3bmzy3TZs2+Oqrr7Bu3Tp888030Ov1iImJwaVLl8w+f2xsLIKCggxfkZGRjuoKERHZATM3VFMaofCCAsXFxUhOTkZWVhZ+/PFHfPHFF9i2bZvZAEeqpKQE7dq1w/jx4/H222+bPKeoqAhFRUWG77OzsxEZGYmsrCwEBgbarR9ERFR9W06m4rGv4wEAJ94aBl9PRasmyAllZ2cjKCjIquu34q8eT09PtGzZEgDQvXt37N+/HwsWLMBnn31W5WM9PDwQFRWFpKQks+d4eXnBy8vLbu0lIiIi56b4sJQxvV4vy7RYotPpkJCQgAYNGji4VUREVFs4LEU1pWjmZsaMGRgxYgQaN26MnJwcrFy5EnFxcfjjjz8AABMmTECjRo0QGxsLAHjrrbfQp08ftGzZEpmZmZgzZw4uXLiASZMmKdkNIiKyI8Y2VFOKBjdpaWmYMGECrl69iqCgIHTu3Bl//PEHhgwZAgBITk6GVluRXLpx4wYmT56MlJQUhISEoHv37ti1a5dV9TlEROS8pEuWcW8pqinFC4prmy0FSUREVDv+OpWKR5eVFRQf/d9QBHp7KNwicja2XL+druaGiIiIqCYY3BARkVO5tcYTyBEY3BARkeI00jWKGdxQDTG4ISIi5cliG0Y3VDMMboiIyKlwWIpqisENERERqQqDGyIicipM3FBNMbghIiLFSUpuuIgf1RiDGyIicioMbaimGNwQEZHipAENEzdUUwxuiIiISFUY3BARkfKE9CZTN1QzDG6IiMi5MLahGmJwQ0REipNmaxjbUE0xuCEiIqfCgmKqKQY3RETkVFhzQzXF4IaIiBTHbA3ZE4MbIiJyKgx0qKYY3BARkeKEbCo4Uc0wuCEiIqfCvaWophjcEBGR4rj9AtkTgxsiIiJSFQY3RESkOA5FkT0xuCEiIqfCOIdqisENEREpTlZzw/lSVEMMboiIyKkMmBOH5bvPK90McmEMboiISHHGQ1Gz1h1XpiGkCgxuiIiISFUY3BARkRNgnQ3ZD4MbIiJySlNWHsTp1Bylm0EuiMENERE5pQ1Hr2LcZ7uVbga5IEWDm8WLF6Nz584IDAxEYGAgoqOj8fvvv1t8zOrVq9G2bVt4e3ujU6dO+O2332qptURE5Cjm1rbJzC+p3YaQKiga3ERERGD27Nk4cOAA4uPjcfvtt2PUqFE4ftx0lfyuXbswfvx4PPbYYzh06BBGjx6N0aNH49ixY7XcciIiInJWGuFka16HhoZizpw5eOyxxyrdd9999yEvLw8bNmwwHOvTpw+6du2KTz/91OTzFRUVoaioyPB9dnY2IiMjkZWVhcDAQPt3gIiIbPZbwlU89e1Bk/ednz2ylltDzig7OxtBQUFWXb+dpuZGp9Nh1apVyMvLQ3R0tMlzdu/ejcGDB8uODRs2DLt3mx+TjY2NRVBQkOErMjLSru0mIiLH2vHPNbyz4QSKS/VKN4VchLvSDUhISEB0dDQKCwvh7++PNWvWoH379ibPTUlJQf369WXH6tevj5SUFLPPP2PGDEyfPt3wfXnmhoiInIelMYQHv9wLAAgP8sak/s1rqUXkyhQPbtq0aYPDhw8jKysLP/74IyZOnIht27aZDXBs5eXlBS8vL7s8FxERKefSjQKlm0AuQvHgxtPTEy1btgQAdO/eHfv378eCBQvw2WefVTo3PDwcqampsmOpqakIDw+vlbYSEZFjcLNMsienqbkpp9frZQXAUtHR0diyZYvs2ObNm83W6BARkfNKySrE2E93Yf2RK6ynIbtSNHMzY8YMjBgxAo0bN0ZOTg5WrlyJuLg4/PHHHwCACRMmoFGjRoiNjQUATJ06FQMGDMDcuXMxcuRIrFq1CvHx8fj888+V7AYREVXD27+ewP7zN7D//A2rzneyyb3kxBQNbtLS0jBhwgRcvXoVQUFB6Ny5M/744w8MGTIEAJCcnAyttiK5FBMTg5UrV+K1117DzJkz0apVK6xduxYdO3ZUqgtERFRN2QVcoI8cQ9Hg5ssvv7R4f1xcXKVjY8eOxdixYx3UIiIiqi1ajcam8zU2nk+3LqeruSEioluDrbEKh6XIWgxuiIhIEbZmboisxeCGiIgUYWtow7wNWYvBDRERKYI1NOQoDG6IiEgRWsY25CAMboiISBG2FxQ7ph2kPgxuiIhIESwoJkdhcENERIpgbEOOwuCGiIgUwYJichQGN0REpAgOS5GjMLghIiJF2L7ODSuKyToMboiISBGcCk6OwuCGiIgUYeuwFKeCk7UY3BARkTKYuSEHYXBDRESKYEExOQqDGyIiUgQ3ziRHYXBDRESKYOaGHIXBDRERKcLW2IahEFmLwQ0RESnC1hWKOSxF1mJwQ0REiuA6N+QoDG6IiEgRtg5LcZ0bshaDGyIiUgQLislRGNwQEZEiGNyQozC4ISIiF8FxKbIOgxsiIlIEMzfkKAxuiIhIEZwtRY7C4IaIiBTBxA05CoMbIiJShK2L+BFZi8ENEREpguvckKMwuCEiIkWwoJgcRdHgJjY2Fj179kRAQADCwsIwevRoJCYmWnzMsmXLoNFoZF/e3t611GIiIrIXW0MbZm7IWooGN9u2bcPTTz+NPXv2YPPmzSgpKcHQoUORl5dn8XGBgYG4evWq4evChQu11GIiIrIXZm7IUdyV/OEbN26Ufb9s2TKEhYXhwIEDuO2228w+TqPRIDw83NHNIyIiB+JUcHIUp6q5ycrKAgCEhoZaPC83NxdNmjRBZGQkRo0ahePHj5s9t6ioCNnZ2bIvIiJyAjZmbgRXKCYrOU1wo9frMW3aNPTt2xcdO3Y0e16bNm3w1VdfYd26dfjmm2+g1+sRExODS5cumTw/NjYWQUFBhq/IyEhHdYGIiGzAzA05itMEN08//TSOHTuGVatWWTwvOjoaEyZMQNeuXTFgwAD8/PPPqFevHj777DOT58+YMQNZWVmGr4sXLzqi+UREZCPW3JCjKFpzU27KlCnYsGEDtm/fjoiICJse6+HhgaioKCQlJZm838vLC15eXvZoJhER2RFDG3IURTM3QghMmTIFa9aswV9//YVmzZrZ/Bw6nQ4JCQlo0KCBA1pIRESOwsQNOYqimZunn34aK1euxLp16xAQEICUlBQAQFBQEHx8fAAAEyZMQKNGjRAbGwsAeOutt9CnTx+0bNkSmZmZmDNnDi5cuIBJkyYp1g8iIrKdrdsvcJ0bspaiwc3ixYsBAAMHDpQdX7p0KR5++GEAQHJyMrTaigTTjRs3MHnyZKSkpCAkJATdu3fHrl270L59+9pqNhER2YFgtEIOomhwY80LOy4uTvb9vHnzMG/ePAe1iIiIagtjG3IUu9TcZGZm2uNpiIjoFmJrbMNYiKxlc3Dz/vvv4/vvvzd8P27cONSpUweNGjXCkSNH7No4IiIiIlvZHNx8+umnhoXwNm/ejM2bN+P333/HiBEj8OKLL9q9gUREpE4cliJHsbnmJiUlxRDcbNiwAePGjcPQoUPRtGlT9O7d2+4NJCIideJ2CuQoNmduQkJCDKv8bty4EYMHDwZQVhys0+ns2zoiIiIiG9mcuRkzZgzuv/9+tGrVChkZGRgxYgQA4NChQ2jZsqXdG0hEROpk67AUh7HIWjYHN/PmzUPTpk1x8eJFfPDBB/D39wcAXL16FU899ZTdG0hEROrEWIUcxebgxsPDAy+88EKl488995xdGkRERGQKa3TIWtVa52bFihXo168fGjZsiAsXLgAA5s+fj3Xr1tm1cUREpGIcZyIHsTm4Wbx4MaZPn44RI0YgMzPTUEQcHByM+fPn27t9RESkUgxtyFFsDm4++eQTLFmyBK+++irc3NwMx3v06IGEhAS7No6IiIjIVjYHN+fOnUNUVFSl415eXsjLy7NLo4iISP04KkWOYnNw06xZMxw+fLjS8Y0bN6Jdu3b2aBMREd0CbC4QZjBEVrJ5ttT06dPx9NNPo7CwEEII7Nu3D9999x1iY2PxxRdfOKKNRESkQszckKPYHNxMmjQJPj4+eO2115Cfn4/7778fDRs2xIIFC/Cf//zHEW0kIiIisprNwQ0APPDAA3jggQeQn5+P3NxchIWF2btdRESkcqYSN10jg3H4YqbV5xOZYnPNTUFBAfLz8wEAvr6+KCgowPz587Fp0ya7N46IiNTL1LCURlP77SD1sTm4GTVqFJYvXw4AyMzMRK9evTB37lyMGjUKixcvtnsDiYjo1mEpthEs0iEr2RzcHDx4EP379wcA/PjjjwgPD8eFCxewfPlyfPzxx3ZvIBERqZOp2VIapm7IDmwObvLz8xEQEAAA2LRpE8aMGQOtVos+ffoYtmIgIiKqkolEjJaxDdmBzcFNy5YtsXbtWly8eBF//PEHhg4dCgBIS0tDYGCg3RtIRES3Do3FgSki69gc3MyaNQsvvPACmjZtit69eyM6OhpAWRbH1MrFREREppisoGFsQ3Zg81Twe++9F/369cPVq1fRpUsXw/E77rgD//rXv+zaOCIiUi9TBcKMbcgeqrXOTXh4OMLDw2XHevXqZZcGERHRrUtroaCYc6XIWtUKbuLj4/HDDz8gOTkZxcXFsvt+/vlnuzSMiIjUjevckKPYXHOzatUqxMTE4OTJk1izZg1KSkpw/Phx/PXXXwgKCnJEG4mISIXKY5umdXwNxywFN1zmhqxlc3Dz3nvvYd68efjll1/g6emJBQsW4NSpUxg3bhwaN27siDYSEZEKlQcr0rVtOFuK7MHm4ObMmTMYOXIkAMDT0xN5eXnQaDR47rnn8Pnnn9u9gUREpG7SbA2HpcgebA5uQkJCkJOTAwBo1KgRjh07BqBsK4byPaeIiIiqUr5CsTSesbRC8fojV7gFA1nF5uDmtttuw+bNmwEAY8eOxdSpUzF58mSMHz8ed9xxh90bSERE6lQep2hlw1KW7TqT4bgGkWrYPFtq4cKFKCwsBAC8+uqr8PDwwK5du/Dvf/8br732mt0bSERE6iYLbqqIbi5e5wgBVc3mzE1oaCgaNmxY9mCtFq+88grWr1+PuXPnIiQkxKbnio2NRc+ePREQEICwsDCMHj0aiYmJVT5u9erVaNu2Lby9vdGpUyf89ttvtnaDiIichKzmpopzOShF1rA6uLly5QpeeOEFZGdnV7ovKysLL774IlJTU2364du2bcPTTz+NPXv2YPPmzSgpKcHQoUORl5dn9jG7du3C+PHj8dhjj+HQoUMYPXo0Ro8ebaj9ISIi12ByheIqUjcsuSFrWB3cfPTRR8jOzja5OWZQUBBycnLw0Ucf2fTDN27ciIcffhgdOnRAly5dsGzZMiQnJ+PAgQNmH7NgwQIMHz4cL774Itq1a4e3334b3bp1w8KFC2362URE5BzM1dyY2iFcMHdDVrA6uNm4cSMmTJhg9v4JEyZgw4YNNWpMVlYWgLKhL3N2796NwYMHy44NGzYMu3fvNnl+UVERsrOzZV9ERKS88jBFK7kSyda84bxwqiarg5tz585ZXKQvIiIC58+fr3ZD9Ho9pk2bhr59+6Jjx45mz0tJSUH9+vVlx+rXr4+UlBST58fGxiIoKMjwFRkZWe02EhGR/RgW8YPpgmJToQ2HpcgaVgc3Pj4+FoOX8+fPw8fHp9oNefrpp3Hs2DGsWrWq2s9hyowZM5CVlWX4unjxol2fn4iIasZcQKPRcFE/qh6rg5vevXtjxYoVZu9fvnx5tXcGnzJlCjZs2ICtW7ciIiLC4rnh4eGVCpdTU1Mr7VJezsvLC4GBgbIvIiJSnmERPzNTwTUmNmNg4oasYXVw88ILL2Dp0qV44YUXZMFFamoqnn/+eSxbtgwvvPCCTT9cCIEpU6ZgzZo1+Ouvv9CsWbMqHxMdHY0tW7bIjm3evBnR0dE2/WwiIlJWxbBUBW1VqRqOS5EVrF7Eb9CgQVi0aBGmTp2KefPmITAwEBqNBllZWfDw8MAnn3yC22+/3aYf/vTTT2PlypVYt24dAgICDHUzQUFBhiGuCRMmoFGjRoiNjQUATJ06FQMGDMDcuXMxcuRIrFq1CvHx8dzXiojIxRgKim3YW4qhDVnDphWKn3jiCdx111344YcfkJSUBCEEWrdujXvvvbfK4SRTFi9eDAAYOHCg7PjSpUvx8MMPAwCSk5OhlZTSx8TEYOXKlXjttdcwc+ZMtGrVCmvXrrVYhExERM5Ly13Byc5s3n6hUaNGeO655+zyw63ZAC0uLq7SsbFjx2Ls2LF2aQMRESnDMCwlqyI2c9voMUSW2Lz9AhERkX2U7wpufuNM47VuuCs4WYPBDRERKUoav1RVUMzQhqzB4IaIiBRhaliKk6XIHhjcEBGRIsoDFXN7S5lcodihLSK1sDm4mTVrFrZu3YrCwkJHtIeIiG4xWjP7SXF1Yqoum4Ob3bt34+6770ZwcDD69++P1157DX/++ScKCgoc0T4iIlKpihWKK44ZBzTG8c3plBzMXJOAlCx+wCbzbA5uNm/ejMzMTGzZsgV33nkn4uPjMWbMGAQHB6Nfv36OaCMREalQRc2N9evcfB9/ESv3JuPZ7w45smnk4mxe5wYA3N3d0bdvX9SrVw+hoaEICAjA2rVrcerUKXu3j4iIVM54s8yK4xqYq7I5cTXboW0i12Zz5ubzzz/H/fffj0aNGiEmJgYbN25Ev379EB8fj/T0dEe0kYiIVMjk9guKtITUxubMzZNPPol69erh+eefx1NPPQV/f39HtIuIiFTO1LBUlRtnGh7LeVNkns2Zm59//hkPPPAAVq1ahXr16iEmJgYzZ87Epk2bkJ+f74g2EhGRCpUXFFvaOJMzpqg6bM7cjB49GqNHjwYAZGVl4e+//8bq1atx1113QavVcoo4ERHZyPT0bwY2VF3VKijOyMjAtm3bEBcXh7i4OBw/fhwhISHo37+/vdtHRERqZWrjTCurbjgoRZbYHNx06tQJJ0+eREhICG677TZMnjwZAwYMQOfOnR3RPiIiUimTBcXM1pAdVKugeMCAAejYsaMj2kNERLcYc9svWMJ6YrLE5uDm6aefBgAUFxfj3LlzaNGiBdzdqzW6RUREt7DyGU/m6mw0sLzWDZE5Ns+WKigowGOPPQZfX1906NABycnJAIBnnnkGs2fPtnsDiYhIfVKzC5FXrANQvangRJbYHNy88sorOHLkCOLi4uDt7W04PnjwYHz//fd2bRwREalPSlYher+3BZtPpAIwvxO4xkKgI5jNIQtsHk9au3Ytvv/+e/Tp00f2wuvQoQPOnDlj18YREZH67D2XIfteY2ZXcEtYc0OW2Jy5SU9PR1hYWKXjeXl5Vr8oiYjo1uXj4Sb7Xmvm0mHpisLYhiyxObjp0aMHfv31V8P35QHNF198gejoaPu1jIiIVMm7UnBjoeaGn5mpGmwelnrvvfcwYsQInDhxAqWlpViwYAFOnDiBXbt2Ydu2bY5oIxERqZi5XcGJqsvmzE2/fv1w+PBhlJaWolOnTti0aRPCwsKwe/dudO/e3RFtJCIiFSnR6WXfa6qxzg3HpciSai1Q06JFCyxZssTebSEioltA5eDG9G0OSVF12Zy5ISIiqokSnTztYq6g2BJOBSdLrM7caLXaKmdDaTQalJaW1rhRRESkXpUyNzA/FZzJG6oOq4ObNWvWmL1v9+7d+Pjjj6HX682eQ0REBFQObrQcQyA7szq4GTVqVKVjiYmJeOWVV/DLL7/ggQcewFtvvWXXxhERkfoUGw1LmSsotrjODUelyIJqxctXrlzB5MmT0alTJ5SWluLw4cP4+uuv0aRJE3u3j4iIVKak1HhYisi+bApusrKy8PLLL6Nly5Y4fvw4tmzZgl9++QUdO3Z0VPuIiEhlLM2WshYTN2SJ1cNSH3zwAd5//32Eh4fju+++MzlMRUREVJVKNTeyueCSmxoNF/WjarE6uHnllVfg4+ODli1b4uuvv8bXX39t8ryff/7Z6h++fft2zJkzBwcOHMDVq1exZs0ajB492uz5cXFxGDRoUKXjV69eRXh4uNU/l4iIlFOp5sbMeQxsqLqsDm4mTJhg940x8/Ly0KVLFzz66KMYM2aM1Y9LTExEYGCg4XtTG3kSEZFzsrxCsbW7gnNgisyzOrhZtmyZ3X/4iBEjMGLECJsfFxYWhuDgYLu3h4iIHK+oxMKwFJEduOTqAl27dkWDBg0wZMgQ7Ny50+K5RUVFyM7Oln0REZEyNh5LwVc7z8mOmd1+wQLmbcgSlwpuGjRogE8//RQ//fQTfvrpJ0RGRmLgwIE4ePCg2cfExsYiKCjI8BUZGVmLLSYiIqknvzlQ6ZjZmhtYP0xFJFWtjTOV0qZNG7Rp08bwfUxMDM6cOYN58+ZhxYoVJh8zY8YMTJ8+3fB9dnY2AxwiIiei1VZjV3AiC1wquDGlV69e2LFjh9n7vby84OXlVYstIiIiW1QnoGE9MVniUsNSphw+fBgNGjRQuhlERFRNstlSTN2QHSiaucnNzUVSUpLh+3PnzuHw4cMIDQ1F48aNMWPGDFy+fBnLly8HAMyfPx/NmjVDhw4dUFhYiC+++AJ//fUXNm3apFQXiIiohrRmAhp7Lz9Ctw5Fg5v4+HjZonzltTETJ07EsmXLcPXqVSQnJxvuLy4uxvPPP4/Lly/D19cXnTt3xp9//mlyYT8iInIN8gWKGdBQzSka3AwcONDiQkzGa+u89NJLeOmllxzcKiIiqk2WAhomb6g6XL7mhoiIXJvZYSmj7+sHcnIIWYfBDRER1Rp3E5GMuYJijUYe4AT5eDiwZaQmDG6IiKjWuJkMbsyfLy1c4DYNZC0GN0REVGtMZW60Gi7iR/bF4IaIiGqNycyN2bPlpcacGk7WYnBDRES1psphKQsBDEMbshaDGyIiqjVu2sqXHUsZGel9Jh5KZBJfKkREVGtsrbmRroXGBf7IWgxuiIio1tgyW8r4OEtuyFoMboiIqNZUVVBcOaDhTCqyHYMbIiKqNaaCG2vXr+FsKbIWgxsiIqo1psITSxtnWjmRikiGwQ0REdUaU1slm8vIaIzOZ2xD1mJwQ0REtUYvKoc35jbONMZhKbIWgxsiIqo1On3l4MZiQbHktrVBEBGDGyIiqjVN6vhWOubmVnEpkmZ2jLM8XOeGrMXghoiIao2fp3ulY16S4KZUVxHQFJfqjdI6jmwZqQmDGyIiqjXloUuAV0WQ4+lecSkq0ekNt0uNhrA4LEXWYnBDRES1pnw7Ba0kUvFwkwY3QnK7ItABOCxF1mNwQ0REtaa8jEa6mJ80IyMNaKSBDsB1bsh6DG6IiKjWlBcJyzbLlAQtpXq98UNMnkdkCYMbIiKqNeW5GHn9TMU3xaWmlvkrfwyjG7IOgxsiIqo1poalpIwzNwxnqDoY3BARUa2paljKuIhYiisUk7UY3BARUa2TZm6kIYtxEbEUQxuyFoMbIiKqNeWZG3PDUpYyN8YPOZOea7d2kbowuCEiolpTXnMjDVSkw02lljI3RsNSr605Zte2kXowuCEiolpjKnMjDVmKjRfu05g+DwAKSnT2bh6pBIMbIiKqNRWZGzOzpXR6eLiZvs/4IcYbaxKVY3BDRES1pmKdG3OL+AnZdgxSxsNSloaw6NbG4IaIiGqNMDUsJYlZikv1cDdTbGx8lJkbMkfR4Gb79u24++670bBhQ2g0Gqxdu7bKx8TFxaFbt27w8vJCy5YtsWzZMoe3k4iI7MMwLGV2ET8h2yVcynhYSqdncEOmKRrc5OXloUuXLli0aJFV5587dw4jR47EoEGDcPjwYUybNg2TJk3CH3/84eCWEhGRPRgKiqWzpSQ5mRKdHgPbhAEAwgK8ZAGN8a7gDG7IHHclf/iIESMwYsQIq8//9NNP0axZM8ydOxcA0K5dO+zYsQPz5s3DsGHDHNVMIiKyE1M1N9KYpVQn8L97OqBteACGdwzHXZ/sMNynNfo4ruOwFJnhUjU3u3fvxuDBg2XHhg0bht27d5t9TFFREbKzs2VfRESkDFPDUhoAnjeLiDs1CoK/lzsm9W+OiBBf2WONMzcXMvJRXGp+0T+6dblUcJOSkoL69evLjtWvXx/Z2dkoKCgw+ZjY2FgEBQUZviIjI2ujqUREZIKhoNiogOb3af3x34Et8N6YTuYfbKJM58cDl+zZPFIJlwpuqmPGjBnIysoyfF28eFHpJhER3bLKB5Lks6U0aFHPHy8Pb4tQP0+zjzVVgpyRW2TfBpIqKFpzY6vw8HCkpqbKjqWmpiIwMBA+Pj4mH+Pl5QUvL6/aaB4REVXBsCu4mRWKjUnvM7XwX1gg39+pMpfK3ERHR2PLli2yY5s3b0Z0dLRCLSIiIluY2lvKWqYWNX75pwT8/U96zRpFqqNocJObm4vDhw/j8OHDAMqmeh8+fBjJyckAyoaUJkyYYDj/ySefxNmzZ/HSSy/h1KlT+L//+z/88MMPeO6555RoPhER2ah89rabmRWKLTF32kNf7qtZo0h1FA1u4uPjERUVhaioKADA9OnTERUVhVmzZgEArl69agh0AKBZs2b49ddfsXnzZnTp0gVz587FF198wWngREQuQpgcljIf3ci3aahGuoduSYrW3AwcONDwQjfF1OrDAwcOxKFDhxzYKiIicjRrMzfubrZneIhcquaGiIhcm97E3lLm9pIqu6/iMmUpw0MkxeCG6BZy6UY++r3/F5ZsP6t0U+gWVZ6sl2ZhpNkZYx7M3FA1MLghuoW8vzERl24U4N3fTirdFLpFmc7cmL8UebhJMzdE1mFwQ3QLuZ7HBc9IWYZF/CRpGEuZG3dJcGNqnRsiUxjcEN1C8ot1SjeBbnUm9paynLnhsBTZjsEN0S0kv4jBDdW+dYcv477PduNablHFsJS1mRstgxuynUttv0BENZNfUqp0E+gWNHXVYQDA7N9PGYalpMkai7OlpDU3jG7ISszcEN1CCiTDUjq9+TWmiBxhV9I1FJaUvQa1ssyN+UuRp5mC4pZh/nZvH6kHMzdEt4Bjl7Pg7qaR1dwUlOjg78W3AKo9V7IKDbetXufGTM2NpccQ8Z2NSOVyi0px1yc7Kh3PLyplcEOKkYYm1VnEjzOnyBIOSxGp3I28YpPH8zhzihQkHRS1draUNAZyY+aGLGBwQ+TClu48h9XxFy2eozVzEcgrYnExKUcv2VfQ2nVupAXF5l7XRACHpYhc1pXMArz5ywkAwJhuEWY/yerNFA5zzRtSknTPZEtZGA8z9zG2IUuYuSFyUTmFFZmXvGLzWZhind7kcUuPIXI0acztYWG2lLmCYjfW3JAFDG6IXFSJJGj562Qa+ry3Bfcu3oWiUnlGplRnJnNTpMPp1BxczSpwaDuJTBGS1I2lLIyHme0XjIelvt+fbL/GkctjcEPkonIlNTOvrT2GlOxCxF+4gVNXc2TnlZjJ3JxKycbQedsxfP7fDm0nkSnSYSlLi/OZ2zjTOHPz8k8J9moaqQCDGyIXJS0IlgY60lqaEp0e3+83XXD888HLAICsghLZp2hSnl4vVP870VvZP3PbL3C2FFnC4IbIhZTq9Pjwj0TsOnNNFtBIFUi2WPh+/0Ws2HNBdn/5xeJyZsVwFIuLnUdxqR7D5m/HpK/jlW6KQ1m7QDZnS1F1MLghciHf7b+IhVuTcP+SvWaDmw1Hr2LEgr9x4ko2dp/NqHR//UDvSsekxclU+7IKSvDQl3vx04FLOHDhBv5Jy8WWU2kAyrbMyMgtUriF9idgXXTTv1Vdw215QbG9W0RqwuCGyIUcuZhpuP3qmmMmz/n54GWcvJqNKd8dRD1/r0r3hweZCm5K7NZGst0nW/7B3/9cw/Orj1Sqker93p/o/s6fuG5mMUZXZe2oW9+WdbFyUm/snnG7bIViDkuRJQxuiFxIrg0Zlms5RcgqqBy0hJvI3GQzc6MoaeBSqq8Ibkp0esPvRhrYqoG1NTcAENOyLhoE+cgyN9x+gSxhcEPkQnKKbMuwZOZX/rTfKMSn8vMyc6MoneRCXyKZut/mtd8Nt//77QG8se4YsvJLMHNNAuLPX6/VNtpbdTal5/YLZC0GN0Qu5ExantXnajQak5mb6OZ1Kh1jzY2ySiVXeum6RNIAoLBEj693X8Dsjaewcm8y7v10d2020e6qMxtMugcVC4rJEgY3RC7iWm4RUrILZccs7aYMAJkmgps+kuCmS0QQAAY3Sjh5NRtjP92FvWczZFtkmFuXqNyxy1mOblqtqM5Md+kmmlyhmCxhcEPkIi5kVM7amJr5JJWVXzm48fF0w2/P9scPT0SjRZg/ACCbw1K1buJX+7D//A3c9/keWeamquAmQSXBTXW4STI3HJYiSxjcEDkhIUSl6b/pOZWnA5ua+SRlnLn5d7cIAED7hoHo1SwUgd4eAFhzo4Q0ye9Tmrl58cejSjSn1k2+rTnq+nvh8duaW/0YaebGVEHx6dScSsfo1sTghsgJzd10Gt3f+RNbE9MMx6oKbhoaBTq5RaXQGVVtzh3XRfZ9gLc7AA5LKU2nwtWI03OKkJSWa/b+sAAv7Jt5B2be2c7q55QOw5raa/PJFQdsaiOpF4MbIie0cGsSAODtDSfw1LcHMP/P07h0o/IGl9Jp3ff3biy7zziwMaU8uFkdfwm93v0Ti+PO1KTZVE2ZJoYPXV3Pd//E4I+24XJmgckhJI3G9qJgNzfpsFTly9fZa3l4/ocjtjeWVIfBDZGTkda/nE3Pw28JKZj/5z/4bPvZSuc2kGRrujcJtflnBdwclioo0SEtpwjvbzyFSzfyq9FqqonD1VzD5uL1fKfag2rtocvYfjpddizhUiZ8PdwAAHUli0pWZ52aqjI3APDTwUs2Py+pD4MbIichhECJTo+L1y0HF9K6gzr+nobbXSODbf6Z5ZkbqdRs9S317yw2HL2Cuz/ZgeQM+wSQ/T/Yis9NBL1KuJCRh2nfH8aEr/bJjpfohGHYTfrarU45sDQDxNlSZIlTBDeLFi1C06ZN4e3tjd69e2Pfvn1mz122bBk0Go3sy9vbclElkSt4+aej6PHOn0i4ZP1smK6RIYbbPp5u+PTB7nhtpOkahvG9Glc6Vl5QLKXGfYycxZSVh5BwOQuvrk2w23PG/n7Kbs9VE9dyKxaMlGaTnvnukGFjVg9puqUasYk0c8N1bsiSyh/batn333+P6dOn49NPP0Xv3r0xf/58DBs2DImJiQgLCzP5mMDAQCQmJhq+1zCCJxX4Ib4snT7vz9NWP6ZZXT/89N9oQ7p/eMdwFJbo8M6vJw3nDGpTD1Nub4XON9e0kTKVuVHbHkbOyNymp9XhLNd4aVal1Ey9l3sVs51s+RnM3JAlimduPvroI0yePBmPPPII2rdvj08//RS+vr746quvzD5Go9EgPDzc8FW/fv1abDGR/UmLf6saFnr29lYAgKHty1733ZuEokkdP8P9nkbFCCG+nujeJET+qfmmAFOZGwY3DmfPC7PPzXoWJej1AuuPXEFyRr6sT+bW6vGQFAFX53+AKxSTtRQNboqLi3HgwAEMHjzYcEyr1WLw4MHYvdv80uK5ublo0qQJIiMjMWrUKBw/ftzsuUVFRcjOzpZ9ETkbW7IlwzqG4++XBmHRA91M3m/8pi/9tGws0ETm5lpuEYQQsrVXyL7suQBdXrEOf55Itdvz2WLt4ct49rtDuG3OVlmfikpMBzfS12J1Mu7Sn8GNM8kSRYOba9euQafTVcq81K9fHykpKSYf06ZNG3z11VdYt24dvvnmG+j1esTExODSJdMV8rGxsQgKCjJ8RUZG2r0fRDVlag0bcwK9PRAZ6msyE2OKpYuAqcxNVkEJ3vn1JDq/uanK4maqHnuvrjtpebxdn89a+8/fMHm8sFRn8rj0pWgp6DbHzYrZUkSAEwxL2So6OhoTJkxA165dMWDAAPz888+oV68ePvvsM5Pnz5gxA1lZWYavixcv1nKLiaqWmiPfMyrEt3LQUS7Qx7ZSOUvpe2+Pym8BBy/cwJc7ziG3qBS/H7tq088i6zhq64Cz6bk4ebX2stPS2U96SRGxuUykdNa68fCpNdytrLmxZo0nUjdFg5u6devCzc0NqanylGpqairCw8Oteg4PDw9ERUUhKSnJ5P1eXl4IDAyUfRE5m7Pp8n2jejY1vWaNu1Zjc42FpeuoqaGB85Jpyu4mFkqjmnNEcJOVX4Lb527DiAV/m9xTzBGk2cNxn1WUEoz8eIdNj7WWm5nZUtL1c4Cq9+dSwtn0XJP7w5FjKPrO5enpie7du2PLli2GY3q9Hlu2bEF0dLRVz6HT6ZCQkIAGDRo4qplEDpeUJt8Tp1k9P5PnBft62FyrMLS9dR8UTDG1qzjV3NXMwqpPuql5XdOvBWNd3tpkuH05s/Jq1vZy7HIW7lm4A7uSrsmGlsqne1sizdxUJ8Azl7mpK1nvCXC+4Ca/uBS3z92GAXPiUOpkbVMrxT+WTZ8+HUuWLMHXX3+NkydP4r///S/y8vLwyCOPAAAmTJiAGTNmGM5/6623sGnTJpw9exYHDx7Egw8+iAsXLmDSpElKdYGoxo5fkQ8lhAWYXrvJx9P2mTG3ta5n1Xlt6gdUOpaZz5lTjpBowwaPvl62/86LHXgBnfR1PI5eysL9X+yVzX6yRk0Hi7RmMjfGWaBSnXMNS2VI1gAqKKk6CKSaU3ydm/vuuw/p6emYNWsWUlJS0LVrV2zcuNFQZJycnAyt5A/oxo0bmDx5MlJSUhASEoLu3btj165daN++vVJdIKqRzPxiJFyWL9xX198TXSODKy3LX2DFp2Mpa1Yt/t/d7bH6wCW8emc73P/FXqO2MXNTU0IIFJXq4V3NKdvVqU1xZObiuiTgrU5RcE3It1+ouG3cDmfL3EgVluhh5rML2ZHimRsAmDJlCi5cuICioiLs3bsXvXv3NtwXFxeHZcuWGb6fN2+e4dyUlBT8+uuviIqKUqDVRPaRmJIDIeS7enu4afHNpN5YOam37FzpKrDWsCbz/3DfZvj12f5oKhn+aBtelsW5wcxNjU1eHo8ub26yabp/qzB/w21Pd9Nv05aGq0p0esQlpuGtX07Y5UL/44FLGDhnK5LScmQBRnXqZmrC3CJ+xhmkDzclwplIfweFzNzUCqcIbohuVT/sv4i5m8tWJI4M9TUcbxseAH8vd/RsJi8sLl+4z1q2rAUSLJmh9Z+eZUsmZLHmpsb+PJmGolI9fjlyxerHSLM85gIIc0EPACzdeR4PL92Pr3aewzd7LljfWDNeWH0E5zPy8eKPR+UZExvrZmq6yae5Rfw83OXtKF/t21kUldovuEnLLsSkr/cjLjGtps1SNcWHpYhuVVkFJXjpp6OG78ODvPHn9NuQllOE5vXKPrlLLx4dGwXivTGdbPoZttQe+3q649MHuwMA6gWUFWgm39x1+lpuMeoFeFl6OFXB0vTktuEBOJVSUYcjjRm8zAQx5o4DwGbJon7nr9Vsho60ADansFQWbLnXcuZGmqCRjkQ5+6y+YklwU9Oam3d/O4k/T6bhz5NpOD97pOy+K5kFWLk3GQ9FN0H9QPNjX3P+OIXreSV4718dVbt9EYMbIoWcSc+VfR8e6I2WYQFoGVZR2Ct94/nvgJaVprxWRWPjIvfDO4bL2paZX4JmM34DACz4T1eM6trIpuejCnoLWYs6RrN9pMxlaLzcravhqUlxsU4vMPijbYbvS3V6WeamppkYW0mDGDfZ8JhzX6ClmRtb6+aMZVgYmh6/ZA8uZOTjn7QcPHN7Kxy5lIn7ezWWvY8UluiwaOsZAMATtzVHkzq+KNbprX49uQrnDneJVOxMmjy4sfRJq7raNqg8A8oawT6VFxHcmXStps255UizHtLNTAGgSZ2KYUg3C5kHcwXFloalpL7bd7Fai9rp9QIXr+fL1j0q0QlZNtFSwGZKTWMhc+vcuFLmprDUfLB5NasAr689hrNGH3ykLBWmX7j5uzqUnIm7PtmBV9ccQ8tXf8fesxm4Z+EOvLY2ASlZFcsQFJXq8eraY+j21mZcuqGu1cid+xVBpGJnjBbuMxfclH/oimocbPVzr326Lx7t2wwvDmtTrbYFmQhu8mr4ifNWZOlC5i35pFypdkVTddGupWEpY499vd+q8y5nFhgufhOX7sPAD+Mq3X9VcnHcmZRhdRsAQFfD6MZXshSCtJ7M1KytjU60unaRZDsK48xNblEp9p27Dr1eYPzne7BizwW8uuaY2efyNVoOIruwBAeTb8j2gkuTbOei0wvc9/keHL2UhW/2JOO/3x403JdTWIKVe5ORV6zDku1nq90/Z8RhKSKFVBqWCjI95HT49aHIKihBw2Afq5+7a2SwVdPAzTFVS2HL/ldUxtIQhJdk6wtLhd/mMjTST/CNgn0sLtwXl5iOtJxCs+snAWXDFX1n/wUASHp3BP7+p+pM3bbT6VWeI1XTzVgbBvtgcv9m8PFwk2W0TAWAT35zEM/c3hITY5raPJxrL5czC/DQl3tRX/L/nm1UpP/wV/sQf+EGYsd0MmTJTltYB8nPaN2jexfvwunUXHwy3rpZw9LtOXIKSw23i51sbaCaYuaGSCHWDksF+XqgsWQIo7YF3Nw5/FougxtbWZoZYylzI/1OeuGWBjrS26b2CDNWbCGLBMjXNMorsk+Wro6fJ+r4VdQT1TRzAwCvjmyP6UPbyIqLzdXcfPJXEqauOlTjn1ld7/56AmfT87D7bEWGq3wSQXm9UvyFss1Hv99fse9hRKgvsgtLMHl5fKVZdj4eFTmJ/4tLwunUsveRZ76zvZ/ZhRW/c2deG6g6GNwQKSA9pwjnjPaZsfSpWgkrJ/fGy8PbYs1TMQCYuakO4+BGmkGQZm7cLBTESoMYLzO3rVkgcGtiOqb/cFh2QZOSxlfmdvU2R7rfWfsGFfv3abUaWW2MPTe0lA9Lmb+U2Tp0Zi9CCPyWkGLyvpd+PIK+s/+S/S7cZf9Pesz9IxGbT6Time8OQacXeGJFPN799YQskP1gY83W81m5N9lwu7hUj6tZBViy/azZ14gr4bAUkQJ2nbkGIcqmAIf6eaJRsI/VBaK1JaZFXcS0qGvYhDGnsBR//5OO06m5mBjdpNanAbsi42m/Pp7S4MRCzY2E9D4vdzfkoGwoQfp6kQYXWg1gKoZ4fW1ZHUfr+gGYEN0Evp7yt/9SyYOs2SdKqmldP8Nwh7RdQgDSl4k9J1dJg5vqrOLsaH+eNL8OTfk6PI8uraiFkhZnl+oEjlyqWLV877kM/HG8bHr/kwNa2K2Ne89dN9wuLtXj4a/2IzE1B/+k5eCDe7tI2qPH7N9PoXfzOhhi41pbSnG+VwSRyv1y5AqmrjoMAOgSEYyVk/tgztgulh+koEAfd8PF46Ev9+HtDSewbNd5ZRvlIoxrbqRDUdJP4JY2kZQHN1VnbhqH+lpc32j276fQftYf+HjLP7Lj0v2Y8otLjR9mkbQvXrLgRsiCEHtmbmqymGBtsFQ3U658SAoADiZnGm6fSsmRbb2y92xFEFJg4+/GWueu5Rn2PPvdKOO07vAVfLHjHCYvj3fIz3YEBjdEtWj76XTZ2HiDYOcaijJFo9FUWsBvk2SROKos4VIWjl/JqpS5kQYh0syNm1E0Is1+SId15EGENFCSPJdWY9WGlh9tPo3iUj0eW7Yfnd74A3skdSG7z9g2lCPNnHhJ2iIgz7DYo+amnJ9XReapqiziL0eu4PW1x+waXFWlunuJmbJAEog6atVw2WaumrIFAZ9YEY89ZzMqFavnFpUir8gxQZa9MLghqkU7jNaKkRZbOrO6RovMXb5hfmbOrS6nsAR3L9yBkR/vkM1GAeTDR9JAxXgqs5+n6SGrQMkUfdmwlOR8jUYDa9duXL77PLacSkNOUalstWzjNXmq4iUL2irapRdClmGp6WwpKX9JcCMtKG4p2Zer3DPfHcKKPRew9tBlu/38qjhqYcHa2BIlp7AUr609hj+Op+I/n++R3VdUqkNM7BYM+jCuVoNFWzG4IapFxqnqIF/XCG5CjYKwK1kFsrU7qIJ0jRHjGSzSImLpJ3vjJfB9JRduaYYiRPJ6kQ1LGQ0FSZ+tqYWZdrYGMeZIf740i6PXy4Mbe07JlgY30kX8/DzNZ0zSHFwUL4TA+Wt50OmFzXVL1rqRXzvFvn+dqqgZkr6ejl/JRnZhKdJyipCR57yTDBjcEDnY1awC3L9kD/44noIDkjH2+oFeGHFzuwNn104yA8bbQwshmL0xp6jE/JRaHzMZDuPP+L6S8+pIAoKGkmFMczU3Qsj3FDMuHK6J5vVM70QuzdxIh9EE5G1Z9EAUejUNxbdGu91Xh7+3dFhKUpdkYTjI1hWVrVWq0+PSjXysO3wFAz+MQ4uZv8mG+exJWotTFXuVIkkDquNXKtbJSc7Ix+Tl8Vhvw6awtYWzpYgcbOFfSdh1JgO7JHUM3z/eB72ahbrMpnVPDmyBhMtZ6NU0FD8fuoxz1/KQLtng05ysghKTqx1bUqLTY84fibiSWYB593U1u0Kvs8qzUPDp7WG6TkajKQtWyvcgkq5CW1eSNWtdv2I7DQ9JcGP8fyTdU0y+qq/pmVTWMl4dt5w0cyMdjRECCPCu+P23DAvAD09GV78BEtLMjawtFoIbRw2jPPnNQfx5Ul6HFpcoX+DQXauRzUirDb6e7si9WRvjptVUu/9f7TxnuH1MMotrxZ4L2HwiFZtPpGJgm3oI9Lbtb92RXOtdg255Or3AjJ8T8L/1xyGEwLbT6Vi0NcmuY/n2diFDvmdL/1Z10bt5HZcJbAAg0NsDKx7rjWfuaIV6NzMJaTlFWPDnP9h0PAVbE9Pw78W7DL8XoGxn6i5vbqo0K6dcblEpfoi/iNOpOUhKy8Gkr+Nx8mo2Pt9+Fp9vP4sNR6/itg+2oukrv6L/B3+5zN43xivQSuuVpHU20otzbmGpYbFEQD4sJc3ctJQEk9JaHOnsprIi3oqfLx0KC7Qx0DTmYyZwkK3Zo5UPkUU3r1Ojn2mOdPhJujGlt4UlFRwV3BgHNqZ4uGktzopzBGmgZy4wtdWGoxVZmut5FZt4HrucZep0xTBzQy6hRKeHVqPBR5sT8d2+soWnmtTxxZu/nABQtt1Am/AAZOYXo0U9f8UDh1KdHkcvZ6FLRDCOX5H/0fdx0Jt9bakbUHaxXrbrvGGYzdNNi2KdHgcu3MDDMU3RtK4fZvycAKBsVs6zd7SSPceNvGJEvb0ZQNmbbr0AL1zIyMeWU6mytVDK9zG6eL0Am46n4tF+zRzdvRozLiL283LHtZs7OfuYydxculEAf8l50gu3dMfwMMkq1tILuvRnCiFkr39pDUyAt7thJWJPd22VqxYbM5cVkc38ksQWegE8NagF/jqVavfXvXSGlLT+y1LmxlHDUtZwd9NALyqyJ+V/M44kDWh8Pd1krxM/TzfDfnG2ZJWke8xdkgxNO9sinwxuyOmlZBViyLxt6BIRLBvHLg9sACAuMQ1PrjiAnKJSLJnQQ/GFpl5dcwzfx1/E04NaVCoAHB3VSKFW2Ud55kZaPyR9k160NQnFOr3F7RqOSj7l5RfrDNktS9eeq1kFKNHp4aaRr3rrbHKMVnf1k9S8SC+80jqRSzcKMLxjOJbtOg8fDzdZEBPi6wk/TzcUlOgQEVKxv5h0i4QcybRcvZDX8EhnVZUNGxTcvF0RTPl6ullVAGsucyPNSEmHxAQEAr09sOm5AVU+d01IM7eWtqL45K8kPDe4tSKvHw+3slq18qDUz8sNxfllt2s6XGguOJEGN2Wvw4q/SR9Pd0Og4uvphuxC26d2n7tWscp6ek4Rtp9Ox9e7zuOdf3VEgyDr98JzBA5LkVMTQmDIR9uQU1iKHUnXzH66WPL3OcMb/O4zGSjR6bHu8OVamTZprKhUh+/jy/aJWbT1TKX7G9mwAaYzCjOzB1a51QcuYd1heYHh6EU7cSY9F2nZhdh2Oh3JRltPWFJ+Qf3lyFV0eXMTer77Z6VsmDMxfs1JNzqUrhfkrtWgT/NQAMDIzg3w8vC2eGVEW/z6bD/8K6oRRnZugLdGdYCbVoPdM+/AwdeHyIKjLpFBhtvSgEpAHt14GGVuKm5XDFFJh8jahlfU9RjzMTO0Ic3cSLMjjh4tfqxfM7QND8BdnRsajlW1vsy+89ct3m8ra3cfd9NqZAGttCDaXP1QgJnjxqTPJSX9fRn/7qQF6fYoOk/PKcKEr/Zhy6k0vLn+RNUPcDBmbsgpFRTrsGLPeTQM9pF9KgWAEF8Pi9Mh/0nLwfu/n8IXO87hni4NMXdcF+w7dx0tw/zNbk5pD7vPZGD7P+mymUXl6vp74lpuMe7tHuGwn19bWplYRwQAwgO9kZJdaPK+wxcz8b/1x3Hgwg2bp8h2jgjC3nPXDc+dX6zDsp3n8eaoDjidmot2DQJkF1cl6PUCi7YmoXvTEGw/LV/LSDqVW7oGy/W8Ynw+oQe2JaZjcLv68PF0ky2tv+j+bobb0kLNHS8Pwpn0PMS0qGs4Jh1uyCksNZu5MRfQ+Hu7G6ZJ+xldUKWFqNZkbqQL9QkHDwO9fld7AECSZBPaqoKbr3acw4yfE/D5Q93Rqr75QM6S/OJSfLAxEXd3aYgnvzlo9rwW9fxwJr0ikJdPWZcHN6YyJ/7e7pXe/0zx83SXbXxaTpq58TKqRZKuwyPP8LjJhp6slSr523eGTXaZuSGnI4TAxKX78N5vpzBlpXydEDetBise641ezco+8b49qkOlx//9zzV8saOsun/9kSto9erveOCLvXh46X6Lb7aFJTpcsCKjkJiSgytGK3amZhdi/JI9WBx3Bs+a2J3312f7Y9XjffDevzpV+fzOThq8dWscbLjd+2YWwpy//7lmMbAZ16Mi8Hv29paG210igyudezD5BgbP3YbRi3bipR+PVrq/tq09fBlzN5/G/Uv2VsoMSIMF6SfkvCIdAr09cHeXhmYzIqZEhPhiQOt6AMqK0wHgPz0jDfdn5pfIhl2Ma27KBfpIsjhmFsQD5L8Xc+2U96viYlxbi7xJs2NV7TO16UQqzl3Lw/Orj1T75725vmwLkn8v3mXxPOOMjHQ0zM/MWkbmHm8pi2Mu8yPdQdx47zppzZL09yptiy373Un3wrLl9ewozNyQ01l7+DL2nZNfIF4c1gb3dGmIwhIdWtUPwJcTe2D/+esY2DoMOr1A7O+nMGNEW8zdfLpSQWe5k1ezcexyNjpFBJm8f+aaBPx88DJWPd7HbPHjxev5GDZ/OwK93XHkjaF477eTSErLRYKFmQL3dGmI+oHeDs0a1aaIEB/c3jYMuUWleHd0R4z8ZAcCvNxxV+eGlYajqqLRVNTZtJDMBIpqHGK43Vny+yrPDkk/Dcefr6j9qU2L487gYPINLLq/GxJTzO8jJC0O9nDT4M17OuCng5fwQJ/GNW7D5w/1wLErWejWOES2IF/ZHlYVhcPlpBmgAK+K236y4EZ+QZNmQsxlbqQzt6QzaGprEqN0kckSK4t0j16yfWhz4V//IMjXExuPm97t25hx0CL97zDOnJniLxtGNJ/FkQZ3Ur6y155RcKM1nbnx96rI4gV4uSOjtOz3WdVUcmn9jT23nqguBjfkNLILS7DlZCrWHqp8gezeJASRoRUrrQZ4e+D2tmVFww/3bYaHopvCTavBX4np2H66bH0J6boh5d785TgSU3LQq1koXhreFmsOXYaXuxbP3tEKPx8sW5r94y3/oFvjEGw/nY7bWtdDUlouHl8Rj84RQTh3Lf9mW0ux5WQalvx9DlVx9j1YbKXRaPDVwz0N3x+ZNRRaLVBYooe3hxaFJXq8PbqjYRfqMd0aGf5vGwZ540pWRfq6Z5NQQ6Yj2LfiYttaUvfRrG7FwnFtGwRUGvq6klWAg8k3sPVUGro1CcGgNmF27K157288BQD4LeFqpdeZlHHgMDGmKSbGNLVLG3w83dCzaVnGTPp/Wz/Qy/D/JM1kBBpdLMvJV/s1Wi3Zs+rgRpopysgrlgWttUE6LGlpnSFj/1t/HGN7ROBGXgl6NQuFp7vWMNusVKfH9B+OoFvj4LL3mC/34u9/yoYcLWU0GgX7GPZikma0yv4/Kv5TZMGNFZkbb083eLhpUHJzg9PyvzXAfObH8rCUNHMjXYZAnsXJuBmsBnq7G8oBqiqANvc6qU0MbshpTFl5yBCYGOsSEWzxseXrRwxoXc/wHK+ObIdZ644DKEvbr9p/0bAL75ZTadgiWV7873/SZc/1f3FJmP/nP7i/d2PEnUrDlaxC2bRHAJhkYYfcMVGN0LSuH+b9eRrPGE2DVpvyFLSXuxvWT+kHHw83+Hi6GYKbYR3CDcFN54hgXMmq+NTbrUmIIbiRXjDq+Hnimdtb4mpWIdpLhsHqmVi+XwhgzP+VDQ/4ebrh6P+GOXw9EensnJTswkpbUYQFeBk+/Qb7VGQVjPeQsqcvH+6J6T8cwfNDWt8saC/LTEgvfNJ1bqT1N9Lbbkabbkov0OaGG6QBUUZuMRoF+1T6e6kt5jK3pizbdd6ww/1DfZrg14SruJ5XjNfvao/mdf2w/sgVrD9yBf/uHmEIbABYnEIf5ONhCG68jGZuSQM+aXZDWn8jnSJuHHR6umlRoiuf4eSOwpLiSufJAyDzmRvpn4ivbD0c00FXoE9FraOvpzuKSnWGn2OMwQ3RTUcuZlYKbN7/dye8/FMCnh/S2uox3InRTZBbWIq+Leugc0Qw9p27jobBPmhRzw+r9l80+7iDyZmG25duFBjeyFbuTbapH2+P7ogD56/jrdEd4XezQNSWcWtXV76CrvTiL63hMF5n5Nk7WqK4VI+RncNxObMiI+PlrsXzQ9sYvh/Uph62JqZjQnRTrD5wyXC8eV0/nJWkw/OKdYg/fx0r9yUjxNcTb9zd3q5rHh2+mAkvdy0aSqa55haWVrqQN63jZwhu2jaQrCrswNWW2zUIxO9T+wMoq0nafHPndun0ceneVAFmsjjywRP5hcrcIoBuWo1h2CIixAeP9G2GF1YfwVAFlmSQzlazZf2WFXsuGG6/veGEYSYbAOyvYoaVNGsmzUDKsyVC9vqXvi/I6rIkU8SlwYUGGni6aw3FvtLfi/Tx3h5uKNGVBXjSzI2lndONh6UMt828RrzctdAAhp9jzBne8xjc3IJ0eoHcolIEeLnj233JqOvniRGdGsjOKSzRYfWBS8gvKsXjtzVHblEpPNy0DhlLLS7V4+0N8qmDfZqHYlyPSAxqEyabPlsVdzctpg6uyJQsvDnjJKewBAu3JiE1qwgPRTfBlzcLjqWfsMtJx45N6d+qriH46deyrmGn77dGdcBDfZrgoT5NDOd6ujvveiyOpNVqMK5HBI5czEJMi7qY3L8Zlu06j2dub4WCEh3+/ucaYlrUga+nO2bdXTbjxc+rYs8a44Dks4d6ID23qPI0etkeSmVrtdwn2cX43u4R6NjIdI2VrbLySzB60U4AwJbnK9ZtWbrzXKXZJdJgvJPk59dWge1Tg1riyKVMDO/YAE3qVAzrSYvBpRcrac2GcRulfZHOlHvzng54Y31ZZtRNq8EvU/ph0dYkPD+0NZrV9UPb8ACTO3Q7WvO6/oatD2qy3cGesxUBzaPLzGdpASDU39MQ3Jjb3BSQh43S+/y95Fmc8plP0uCibIuOiloq6e9MGpD4eFQs1me89YY55gqKA2S3K4I2rVYDLw8tzK3bZ23dkyMxuLnF6PQCD36xF7uNNnX74N7OyC4owY6ka5h6Ryss2PKP4Q0ixNcT7288hVA/T/w+tb/sE0BiSg5C/DwQFlC9YtlZ645h+e6yT0xuWg02PXcbMvOL0bp+ADQaTZVrqlgrwNsDG6b0R15xKTzdtfgh/iJCfD3xzWO98fzqw9hfRVHqzDvb4r3fTuGJAc3xUJ8mGPzRNnRqFITPHuqOez/dDQ83DR7s3cTic9xqPri3i+H2zDvb4bkhreHr6Y6PxnXF6gMXMbZ7pOz8tuGB+PTB7rLNIct5umsNgc1rI9vhnV9P4uPxUVj4V8XWDoPahuHXo/I1RxJTchDg7Y4Ab49KO5vbKi2nIrN0WlJAbGrarHSYJizAC72bheJyZgGaWNih2578vdzx7aQ+AMouNB0bBSLQ2wOt6lcEG9I6IekFzTgekF4gpXtbSfvi5+WO9g0DseiBiunr9goqrfXbs/3xy9Er+O/AFrK9kKRa1/fH6dRck/fVRB2/ig9gQbLMjfzDoLmMprk1b6TDVVqNRpYRqRfgZeiLLLiRrW0jzfwY0ZgrKJbcNjO7TgPA08LyC7aufO0IDG5uMZ9uO1MpsAEgm05rvOHbSz+V3ZeRV4w5mxKx4chVtAkPwIvD2uDuT3YgxM8Tf780CAeTbyArvwTDO4ZDo9HIloE/lHwDoX6eaFLHD4cvZuJsei7ahgcaAhugbAuFFlVsxFgTQb4ehjeePTPugLeHG9y0GvzwRDQKSnRIyy7Cm78cx9ab/R/TrRG0Gg2eHNACLcP8cXvb+ogM9YGXuxsOvDbE8Phfn+kHAE69aq7SNBqNYSy/XoAXnhrY0uR5w63YJf2xfs0wOqoR6vp7oZ6/F+7/Yg+eGdRSlu3x8Shb0bd8uq+nuxbrnu4LoCzAr86FVzrccayKRQSDfaV1NlqserwPSvVCkU1APdy0+GVK2WtUo9Hgrs4NsO/cdQzvGI45fySWHZdc+rILSwz/f0Dl4Y//9IzEmfRc9GtZFz88EY1DyTfQV7LmjlLaNwxE+4aV15iSMld4W1N1JIGzdKNYaTBSqheymhvpa0HarshQXySmlgXP0qUTjEqh0Lp+AHYmZVR6vI+Z2W2VsliSxkiDMGnNjZ+s6Fk+JGlcTyRVxMwN1aa//0k3vJmV+2hcFyzY8k+lzR0B03vPfLbtLADgcmYB/rpZkJueU4S2r280nPPc4NZoFOKD/60/ju5NQnB/78b47zcHEOjjgY//E4XJy+MrzS7xdNPiucGt7dJPa0j/aMsvvE3rumNI+3BDcDPtjtZoLPl0Kk2xSx/PoKZ2aTQa1L1ZWBzdog6OvDEUAV7u+GrnecM5zw9tLZsaXVyqx4gFfwMoS8///FRfdDWxfo4l0inO3xvVbzUI8jbsgwUA7RrIF4fTaDSV1o+pTdLAb+H93aDTC9kwxeXMir//U1dzMLZHhOGDR58WZcsitL6Z9Zn9786Gc3s1CzWsOeVMyrN75VnoctIMR3TzOoYPeuWLbFbFeFG+ctL9v6SBjvT9MzO/RFboHiYZbpe2S5oRS5VkC3MKS2UrUUuzaNJsS5/mdXDqZmZROrpraV8taaG7dOmCAFktjzyYsbRwJjM3VCvSc4rw2Nf7Des6DGlfHzEt6uDyjQKM6toILer5491fT6Jfq7ro07wO1h2+jJZh/hjeMRxj/m8XMvNLMP8/XfHOrydw8XrVMyDm/XnacHvb6XRsu1konJlfgglf7ZOd2zDIGysn94Gfl7tNtTWOcl/PSOj0erRvGCgLbMh5la/dMrJTA8zbfBp9mofKApcODQNx/EpFPY9eAHP+OIXCEj2a1/XD/+7pgHs/3Y2TV7Px4rA2eHqQPKv0T2oOEi5nyd6wjS+EIb6esuBmTLcIzP/zH9lML2diPJusuFSPTo2CkHA5Cz2ahmDmne3g4abFsA7hCPT2wIm3himSdaqux/o1w+1tw9C0jh8+3XbG8GFKOvzSIKhi+LN5XX9cy7VcNOzhppEVVI/qWrGuk3QNK+nt06ny9Y9ua1UXWxPTEeTjgZiWFdkuaXAjbVd6dkVRy4WMfNlwZ2SIfGmMch0aBmLh/VH46cAl3NmxAcKDvBH720k80rcZJvdvjglf7cMTtzXH5pMVs0XNZZGk7ZIGMxqN0X5iRlP/GdxQrfhg4ylDYOPj4YaXhrWRLTveJTIYPzwZbfhe+kls6wsDkVdUijr+XujQMBC7zmTg7s4N8fiKeENRbfkLu0eTEDSt64cfb85m8XTXop6/F67lFskyNeGB3ijVC2TkFeGNezqgqWQdE6W5aTV4KLqp0s2gaggP8sbuGbfD02h9o6cHtcRT38qXyC9P5x+4cAO7zmQYpu7O+SMRd7QLw0s/HkWPJqEY3ysSQ+Ztr/JnS2fIAGWLyu179Q7ZcvvOaO7YLvhyxzlMGdQKvl5uWL77Av7TMxLeHm6GrQ0A++w9VJs0Gg2a3xzibhTsY5hRJ60TkWZbIkJ8sO982e3yZSMAoG/LOobXivGwknRjyOgWFYt+5haVYvqQ1pj352k81q8Zjl7KQu7Nta6m3N4KDYJ98MRtzdGkjh9mjGgLrUYjG46vF+CFSf2a4Ysd5/D80Nb482Qqlvx9Do/2bYajlzIRf+EG2tQPQJ/moejfqi4aBvmge5OKRS/Dg7zRv1U9w35bg9qEydZ+OjxrKNy0GsNECEC+FpKfmdlSxsNQ0oLoIB8P2fYPDG5uWrRoEebMmYOUlBR06dIFn3zyCXr16mX2/NWrV+P111/H+fPn0apVK7z//vu48847a7HFrmPR1iT8eLAs2HhxWBs8OaCFTWuAeHu4GWZIRYT4YlyPsk8LXz/SCynZhWgQ5I3sglKk5RQaAqb7ezfGwQs3cF/PSAR4e0CIsjeFb/ZewLXcYtzTpSGa1fVDUanO5d40ybmVf4L1cnfDkgk9oNMLDG1f3zCT6r1/dcLMNQmyx1w22kpj+Pyy4aujl7LMFqYaK9HpsWRCDzyxIh7je5WtPOwKr+1/d4/AvyX7nU0fUntDw7WloSS4kQ7f1JWsmRQhWSBUOiNPOvNJCPkwj3RYqUGQD8b3aoxfj17BgNb10CDIGxNjmiLIxwPz7uuKycvj8eKwNujeJEQWiDxxcy8xaaFxyzB/3NOlIZ65oxWCfDzKFqZsG4ZujUNQUKzD3M2JGNI+HO5uWqx4rLfhcbe1rofTKTmy1b1NKX//f3l4W/zr/3ahe5MQdJO0SZo5kmaEpFmj3MJSWaDYMMhHFtxwthSA77//HtOnT8enn36K3r17Y/78+Rg2bBgSExMRFlZ5pdFdu3Zh/PjxiI2NxV133YWVK1di9OjROHjwIDp27KhAD5zTgQs38OYvxw0Zm4djmlZKt9eEVqtBw5tvAtJCXQDo1jgE3SR/YBqNBhoNMMEoI+IKb/7kuoZI1lj5Y9ptuJCRj97NQ/H6umN2mZIt3Si0fqA3hrSvjz0z70Cob81mZZF9PTmgBXYkXUP/VnXRMqwiY91Isv5PG0kmW/peZrytxkN9muBQciZahvnjX1GNkHw9H83q+iHUzxOxYzrh7VEdDLNJywuLh7SvjyOzhsqe15hWq8Gf0wcg+XoeOjQMkj3ew01r2CTV28MN74w2vT/d0od7QqupvIyCOVGNQ7Bnxh0I8HaHn5c7PhkfhYJiHQa1CcOwDvWx5+x1xLSog4Ft6iEuMR2t6wdgXI8I/BB/CWN7RCIzv2JotmfTEJy4WjH0W+wEwY1GOHrb1ir07t0bPXv2xMKFCwEAer0ekZGReOaZZ/DKK69UOv++++5DXl4eNmzYYDjWp08fdO3aFZ9++mmVPy87OxtBQUHIyspCYKBjxsPLMxWi/LbhOCBQkdo0/Avz50PI74fknPLnK7+jsESPcxl52HDkimyhM1N1BES3qm/2XMBrN1dPLvfisDYID/TG86uPoFndsuGCx1ccAAD8MqUf7l64AwDw9aO98Niy/SjVC4zu2hCTb2uOT7YkYergViZ3gyfncCY99+aqyfkY/FHZMOPJt4aj3ayyiRAHXhuMd387iXWHr+CPaf2xaOsZrDl0GW+P6oDk6/lY8vc5DGpTD0sf6YUd/1xDeJCXLFBSI51ewE2rgV4vcDmzAJGhvtDrBfaczUDHiCCUlOox4+cExLSog46NgjD9hyNoGx6ATSdS0TY8ABun3Wb3Ntly/VY0uCkuLoavry9+/PFHjB492nB84sSJyMzMxLp16yo9pnHjxpg+fTqmTZtmOPbGG29g7dq1OHKk8i6vRUVFKCqqKMrKzs5GZGSk3YObAxduVLlDbG37d7cIPDGguayqnojKNlF95aejhp2Mz8XeCY1Gg/3nr6NJHV/U8/fCsl3nEezrgX9FReDAhRu4nFmAe7o0xK6ka9iamIanBrZESA3XzqHa9/c/ZQW9nSOCcfF6PrIKStCxURCEEMgv1sHPyx06vcDB5BvoGhkMN40GvyZcRY+mIbI6G6ps37nrGPfZbgBl+wH+9N8Yuz6/LcGNouMC165dg06nQ/368iW669evj1OnTpl8TEpKisnzU1JM79IaGxuLN9980z4NdkIaTcXiTJ7uWjQM9kHHhkGYGNME3Zs43xRNImfQrkGgbOZLeSq/fBNKAHikbzPDbWmtREzLurKZLuRa+reqZ7gdGeqL8qUkNRqNoZjWTauRvRbu7tKwNpvoslqF+SPEt2wPKoUHhZSvuXG0GTNmYPr06YbvyzM39tapURD2vzrYUHCmwc1aE5QHIBVRSHlAUul+VBSslR+TnlvxvNaPqxKRac4wo4NITUL8PLFn5h3ILihVdF0nQOHgpm7dunBzc0NqaqrseGpqKsLDTa9UGh4ebtP5Xl5e8PJy/Popnu5ap1inhYisc3/vxth77jp6O+EidESuysvdDfUClN8VXNFFGDw9PdG9e3ds2bLFcEyv12PLli2Ijo42+Zjo6GjZ+QCwefNms+cTEZlyT5eGWPd0Xyx9pKfSTSEiO1N8WGr69OmYOHEievTogV69emH+/PnIy8vDI488AgCYMGECGjVqhNjYWADA1KlTMWDAAMydOxcjR47EqlWrEB8fj88//1zJbhCRi9FoNOhi4xYMROQaFA9u7rvvPqSnp2PWrFlISUlB165dsXHjRkPRcHJyMrSSVT5jYmKwcuVKvPbaa5g5cyZatWqFtWvXco0bIiIiAuAE69zUttpY54aIiIjsy5brt3NvfEJERERkIwY3REREpCoMboiIiEhVGNwQERGRqjC4ISIiIlVhcENERESqwuCGiIiIVIXBDREREakKgxsiIiJSFQY3REREpCoMboiIiEhVFN84s7aVb6WVnZ2tcEuIiIjIWuXXbWu2xLzlgpucnBwAQGRkpMItISIiIlvl5OQgKCjI4jm33K7ger0eV65cQUBAADQajV2fOzs7G5GRkbh48aLqdhxXc98A9s9VqbVf5dg/16TWfpVTqn9CCOTk5KBhw4bQai1X1dxymRutVouIiAiH/ozAwEBVvqABdfcNYP9clVr7VY79c01q7Vc5JfpXVcamHAuKiYiISFUY3BAREZGqMLixIy8vL7zxxhvw8vJSuil2p+a+Aeyfq1Jrv8qxf65Jrf0q5wr9u+UKiomIiEjdmLkhIiIiVWFwQ0RERKrC4IaIiIhUhcENERERqQqDGyIiIlIVBjdERESkKgxunIRer1e6CQ6RmpqKK1euKN0MqgG1rhZx8eJFnD59WulmUDXxPZMsYXCjsKysLABle16p7Y/10KFD6NWrF06dOqV0Uxzi/PnzWLJkCT7++GP8/vvvSjfH7q5fvw4A0Gg0qgtwDh06hB49eiAhIUHppjhEUlIS5syZg5dffhkrVqzAtWvXlG6S3fA903XV6numIMUcP35cBAUFiXfffddwTKfTKdgi+zl8+LDw8/MTU6dOVbopDnH06FERFhYmBg0aJAYOHCi0Wq146KGHxN69e5Vuml0cP35cuLu7y35/er1euQbZUflr87nnnlO6KQ6RkJAg6tSpI0aMGCHGjBkjPD09xe233y7Wr1+vdNNqjO+Zrqu23zMZ3Cjk4sWLIioqSrRu3VqEhoaK2NhYw32u/sd67NgxERAQIF555RUhhBClpaXi0KFDYufOneLYsWMKt67mrl27Jrp06SJeffVVw7HffvtNaLVacffdd4u//vpLwdbV3OXLl0WvXr1Et27dhJ+fn5g2bZrhPlcPcE6ePCl8fX3FzJkzhRBClJSUiG3btom1a9eKnTt3Kty6mrtx44aIiYkx9E+IsmDHzc1NdO/eXSxfvlzB1tUM3zNdlxLvmQxuFKDT6cT8+fPFmDFjxF9//SVmz54tAgMDVfHHWlhYKKKiokSDBg3E1atXhRBCjB49WkRFRYnQ0FDh5+cnPvjgA4VbWTNJSUmie/fu4vjx40Kv14uioiJx5coV0aFDBxEeHi7GjBkjrl+/rnQzq0Wv14tvvvlGjB07VuzcuVOsXLlSeHl5ybIcrhrgFBUViVGjRomwsDCxb98+IYQQd999t+jSpYsICwsTHh4e4tlnnxXp6ekKt7T60tLSRFRUlIiLixM6nU7k5eWJkpIS0b9/f9G1a1cxZMgQcfz4caWbaTO+Z/I901YMbhRy+vRpsXLlSiGEENevXxexsbGq+WPdunWraNOmjfjPf/4junXrJoYOHSr+/vtvsX//fvHxxx8LjUYjFi9erHQzq+3QoUNCo9GILVu2GI4lJSWJ4cOHi2+//VZoNBrx+eefK9jCmrlw4YJYt26d4ftvv/1WeHl5qSKDs3//fjF06FAxfPhw0bZtWzF8+HBx4MABcf78ebF+/Xrh4eEhXnvtNaWbWW1nzpwR3t7e4ocffjAcO3/+vOjdu7f49ttvRXBwsHjrrbcUbGH18T2T75m2YHCjIOkFIj09vdKnkdLSUrF+/XqX+SQp7c/WrVtFeHi4GDBggLhy5YrsvOeff1506tRJZGRkuORFsqSkRDz00EOiZcuWYuHCheK7774TISEh4qmnnhJCCDFt2jTxn//8R5SUlLhk/4SQ/y5LS0srZXBKSkrEN998IxISEpRqYrXt379fxMTEiCFDhohz587J7luwYIGoV6+euHz5ssv+7p577jnh5eUl3njjDfHxxx+LoKAg8cQTTwghhJgzZ47o27evyMvLc8n+8T2T75nWcndsuTKVu3LlCi5fvoyMjAwMHjwYWq0WWq0WpaWlcHd3R926dfHoo48CAN577z0IIZCRkYEFCxYgOTlZ4dZbJu3bHXfcAQAYOHAgNmzYgBMnTqBevXqy8729veHr64uQkBBoNBolmmwTaf+GDBkCd3d3vPzyy1i0aBHeeOMNhIeH46mnnsI777wDoGw2x40bN+Du7hp/XhcvXsTJkyeRnp6OIUOGIDg4GJ6enobXppubG8aOHQsAeOSRRwAAOp0OixcvRlJSkpJNr5K0b4MHD0ZQUBB69OiBzz77DImJiYiIiABQNt1do9FAo9GgQYMGqFOnjku8No1/d6GhoXjrrbcQGBiI5cuXo379+pg+fTpmzZoFoGIGnK+vr5LNtgrfMyvwPbMa7BIikUVHjhwRkZGRon379sLd3V1ERUWJxYsXi5ycHCFE2aeNcunp6SI2NlZoNBoREhIi9u/fr1SzrWKqb4sWLRJZWVlCCCGKi4srPebJJ58Ujz76qCgqKnL6TyHG/evatav4/PPPRX5+vhBCiEuXLsk+Zen1ejFhwgTx8ssvC71e7xL9q1+/vujWrZvw9PQUHTp0EC+++KK4ceOGEEL+2iwtLRUrVqxwqdemcd+ef/55kZGRIYQw/dqcOnWquPfee0VeXl5tN9dmxv1r166dePnllw2/u/T0dMPtco8//riYNGmSKC4udurXJt8z5fieaTsGNw6Wnp5ueNM5d+6cSEtLE+PHjxe9e/cW06ZNE9nZ2UII+VjxQw89JAIDA52+8M/avpW7cuWKeP3110VISIjT900I8/3r2bOnmDZtmsjMzJSdf+bMGTFz5kwRHBwsTpw4oVCrrZeZmSm6detmuOAXFBSIGTNmiJiYGDFq1ChDEFB+IdHpdOKxxx4TgYGBTt8/a/tW7uzZs+L1118XwcHBLjE7xVz/oqOjxT333COuXbsmhKgY9vjnn3/ESy+9JAIDA52+f3zPrMD3zOpjcONgCQkJomnTpuLIkSOGY0VFRWLWrFmiV69e4tVXXxUFBQVCiLI3ohUrVoj69euLAwcOKNVkq9nSt3379omxY8eKiIgIcejQIYVabBtb+peeni6efPJJ0aZNG3Hw4EGlmmyTc+fOiebNm4u4uDjDsaKiIvHVV1+J6Oho8cADDxjebPV6vfjtt99Es2bNnP6TsRC29S0hIUHcc889omnTpi7z2rTUvz59+oj777/f0L+MjAzx2muviR49erjEa5PvmXzPtAcGNw6WmJgomjVrJn755RchRFlhVfm/L774oujatavYvn274fyzZ8+K8+fPK9JWW9nSt4sXL4rVq1eLpKQkxdprK1t/d2fOnBGXLl1SpK3VkZ6eLjp27Cg++eQTIUTFp3ydTicWLVokunXrJlsXJSUlxTBV1dnZ0rf8/HyxZcsWcfbsWcXaaytbf3eXL18WqampirTVVnzP5HumPTC4cbDCwkLRo0cPcddddxnS++W/cL1eLzp16iQmTJhg+N6VWNO3hx56SMkm1ogtvztXVFxcLP7973+LmJgYkxeHoUOHipEjRyrQspqzpm933nmnAi2zDzX/7vieyfdMe+DeUg6k1+vh5eWFpUuXYvv27fjvf/8LAHB3dzfMzrjnnnuQlpYGAC5RBV/O2r6lp6cr3NLqsfV352qEEPDw8MD//d//4cyZM3j22WeRlpYm20Pq7rvvxrVr11BYWKhgS21nbd8yMjJcrm+Aun93fM/ke6a9MLhxIK1WC51Oh44dO+Lrr7/Gd999hwkTJiA1NdVwzrlz5xASEgKdTqdgS22n5r4B6u+fRqNBcXExwsLCsHHjRuzduxcPPvgg4uPjDf05fPgw6tSpA63Wtd4m1Nw3QN39U/PfnZr7Bjhf/zRCqGy7XydSvh5Dbm4uioqKcPjwYdx///1o0qQJQkNDUadOHaxbtw67d+9Gp06dlG6uTdTcN0D9/dPpdHBzc0NGRgaKi4tRUFCAESNGwN/fH6WlpWjevDm2bNmCHTt2oHPnzko31yZq7hug7v6p+e9OzX0DnK9/rhXWOynj+FAIYfhFnz9/Hq1bt8b+/ftxxx134Pjx47jzzjvRqFEjhIWFYd++fU79QlZz3wD198+U8ovj+fPn0blzZ2zZsgXNmzfH/v37MW3aNAwZMgQ9e/bE/v37Xe7iqOa+Aerun5r/7tTcN8A5+8fMTQ0lJibi22+/RXJyMvr164d+/fqhbdu2AIDk5GR069YNo0ePxpIlS6DX6+Hm5mYYf9Tr9U6dNlZz3wD19y81NRVZWVlo3bp1pfsuXbqETp06YezYsfjss88ghHD6/kipuW+Auvt37tw5/PHHHzh9+jRGjBiBqKgo1K1bF0DZisvdunXDqFGjXPLvTs19A1ysf7VQtKxax48fF0FBQYZZC7179xYRERFi8+bNQoiyfWqmTZtWqaK//HtnrvRXc9+EUH//Tpw4IRo3bizGjRtnctG2NWvWiOeff97p+2GKmvsmhLr7d/ToUdGwYUMxYsQI0apVK9GmTRvx/vvvi9LSUlFcXCwWLlwonnvuOZf8u1Nz34Rwvf4xuKmm0tJS8eCDD4oHHnjAcOzQoUNi0qRJws3NTWzatMlwnqtRc9+EUH//Ll++LGJiYkSXLl1Er169xGOPPVZpg0tTS7y7AjX3TQh19+/8+fOiVatWYubMmYY+vPLKK6Jly5aGhd2MV7B1FWrumxCu2T/nzoE5Mb1ej4sXLyIyMtJwrGvXrnjvvfcwefJkjBo1Cnv27IGbm5uCraweNfcNUH//Tp06hYCAAHz99dd46qmncOjQIcyfPx/Hjh0znOPh4aFgC6tPzX0D1Ns/nU6HdevWISoqCs8884xheGLatGkoLi7G6dOnAQBBQUFKNrNa1Nw3wHX7x+Cmmjw8PNCxY0ds27YNN27cMByvV68eZs6ciTvvvBNvv/02srOzFWxl9ai5b4D6+xcTE4M33ngDXbp0wcSJEzFlyhTDRTIhIcFwnrhZbqfX65Vqqs3U3DdAvf1zc3NDUFAQ+vbti/DwcMMHB41Gg+zsbMNu5VLCRcpB1dw3wIX7p2TayNV9//33IioqSsydO7fShmfLli0TDRs2FMnJyQq1rmbU3Dch1N8/4/HtZcuWiW7dusmGOd58803ZHjCuQs19E0L9/ROioo8FBQWibdu2Yu/evYb71q1bp4q/PTX2TQjX6Z+70sGVq7hy5QoOHjyI4uJiNG7cGD169MC4ceMQFxeHJUuWwMfHB/fddx9CQ0MBAD179oSvry9ycnIUbnnV1Nw34NbqX5MmTdC9e3doNBqIspo6aLVaTJw4EQDw8ccfY8GCBcjOzsaPP/6Ie++9V+HWW6bmvgHq7p+pvzugYjo7ULbwm1arNaw0PHPmTCxduhR79+5VrN3WUHPfAJX0T8nIylUcPXpUNG/eXPTq1UvUrVtX9OjRQ3z33XeG+x9++GHRqVMnMW3aNJGUlCTS09PFSy+9JFq3bi2uXbumYMurpua+CXFr9m/16tWyc3Q6neH2l19+KTw8PERQUJDT7zSs5r4Joe7+WdM3IYS4ceOGqFevnti5c6d4++23hbe3t9PvOq/mvgmhnv4xuKlCUlKSiIiIEC+99JLIzMwU8fHxYuLEieLRRx8VhYWFhvPefPNN0b9/f6HRaET37t1FeHi4Q7Zxtyc1902IW7t/paWlsuENvV4vSktLxbPPPitCQkJMTjF2JmrumxDq7p8tfcvJyRFRUVFi4MCBwtvbW8THxyvY8qqpuW9CqKt/DG4sKCoqEtOnTxfjxo0TRUVFhuNffvmlqFOnTqVP9teuXRO///672LFjh7h48WJtN9cmau6bEOyfqazTvn37hEajcapPV6aouW9CqLt/tvYtMzNTNGnSRISGhorDhw/XdnNtoua+CaG+/rHmxgK9Xo+IiAi0a9cOnp6ehpUWY2Ji4O/vj5KSEsN5Wq0WderUwfDhwxVutXXU3DeA/Svvn1TPnj1x/fp1BAcH136DbaDmvgHq7p+tfQsKCsLkyZPx73//27A6uLNSc98AFfZPsbDKRZw9e9Zwuzwld/XqVdGyZUtZVbgrDGMYU3PfhGD/ykn75+yroJZTc9+EUHf/rO2bs2ehTFFz34RQV/+4zo2Rq1evYt++fdi4cSP0ej2aNWsGoKxKvLwqPCsrS7Y+yqxZs3DHHXcgIyPDOeb3m6HmvgHsH1B1/8rPczZq7hug7v5Vt29Dhw51+r87NfcNUHn/FAurnNCRI0dEkyZNROvWrUVQUJBo27atWLlypcjIyBBCVESyiYmJol69euL69evi7bffFj4+Pk5XTGVMzX0Tgv1z5f6puW9CqLt/7Jtr9k0I9fePwc1NaWlpom3btmLmzJnizJkz4vLly+K+++4T7dq1E2+88YZIS0sznJuamiqioqLEfffdJzw9PZ3+F63mvgnB/rly/9TcNyHU3T/2rYyr9U0I9fdPCAY3BsePHxdNmzat9It7+eWXRadOncQHH3wg8vLyhBBlu/ZqNBrh4+Pj9OtNCKHuvgnB/rly/9TcNyHU3T/2zTX7JoT6+ycEa24MSkpKUFpaivz8fABAQUEBAGD27NkYNGgQFi9ejKSkJABASEgInnrqKRw8eBBdu3ZVqslWU3PfAPbPlfun5r4B6u4f++aafQPU3z8A0AjhzBVBtatXr17w9/fHX3/9BQAoKiqCl5cXgLKpmC1btsR3330HACgsLIS3t7dibbWVmvsGsH+u3D819w1Qd//YN9fsG6D+/t2ymZu8vDzk5OTIdn7+7LPPcPz4cdx///0AAC8vL5SWlgIAbrvtNuTl5RnOdeZftJr7BrB/gOv2T819A9TdP/bNNfsGqL9/ptySwc2JEycwZswYDBgwAO3atcO3334LAGjXrh0WLFiAzZs3Y+zYsSgpKYFWW/ZflJaWBj8/P5SWljr19Dc19w1g/1y5f2ruG6Du/rFvrtk3QP39M0uhWh/FHD9+XNSpU0c899xz4ttvvxXTp08XHh4ehsWy8vLyxPr160VERIRo27atGD16tBg3bpzw8/MTCQkJCrfeMjX3TQj2z5X7p+a+CaHu/rFvrtk3IdTfP0tuqZqb69evY/z48Wjbti0WLFhgOD5o0CB06tQJH3/8seFYTk4O3nnnHVy/fh3e3t7473//i/bt2yvRbKuouW8A++fK/VNz3wB19499K+NqfQPU37+q3FJ7S5WUlCAzMxP33nsvgIp9hZo1a4br168DAETZ9HgEBATg/fffl53nzNTcN4D9A1y3f2ruG6Du/rFvrtk3QP39q4rr98AG9evXxzfffIP+/fsDKFtiGgAaNWpk+GVqNBpotVpZ4ZWzLnsupea+Aewf4Lr9U3PfAHX3j31zzb4B6u9fVW6p4AYAWrVqBaAsOvXw8ABQFr2mpaUZzomNjcUXX3xhqBx3lV+2mvsGsH+A6/ZPzX0D1N0/9s01+waov3+W3FLDUlJarVa2GV15JDtr1iy88847OHToENzdXfO/R819A9g/V+6fmvsGqLt/7Jtr9g1Qf/9MueUyN1LltdTu7u6IjIzEhx9+iA8++ADx8fHo0qWLwq2rGTX3DWD/XJma+waou3/sm+tSe/+MqStUs1F59Orh4YElS5YgMDAQO3bsQLdu3RRuWc2puW8A++fK1Nw3QN39Y99cl9r7V4kDppe7nP379wuNRiOOHz+udFPsTs19E4L9c2Vq7psQ6u4f++a61N6/crfUOjeW5OXlwc/PT+lmOISa+wawf65MzX0D1N0/9s11qb1/ADfOJCIiIpW5pQuKiYiISH0Y3BAREZGqMLghIiIiVWFwQ0RERKrC4IaIiIhUhcENERERqQqDGyJyGQMHDsS0adOUbgYROTkGN0SkSnFxcdBoNMjMzFS6KURUyxjcEBERkaowuCEip5SXl4cJEybA398fDRo0wNy5c2X3r1ixAj169EBAQADCw8Nx//33Iy0tDQBw/vx5DBo0CAAQEhICjUaDhx9+GACg1+sRGxuLZs2awcfHB126dMGPP/5Yq30jIsdicENETunFF1/Etm3bsG7dOmzatAlxcXE4ePCg4f6SkhK8/fbbOHLkCNauXYvz588bApjIyEj89NNPAIDExERcvXoVCxYsAADExsZi+fLl+PTTT3H8+HE899xzePDBB7Ft27Za7yMROQb3liIip5Obm4s6dergm2++wdixYwEA169fR0REBB5//HHMnz+/0mPi4+PRs2dP5OTkwN/fH3FxcRg0aBBu3LiB4OBgAEBRURFCQ0Px559/Ijo62vDYSZMmIT8/HytXrqyN7hGRg7kr3QAiImNnzpxBcXExevfubTgWGhqKNm3aGL4/cOAA/ve//+HIkSO4ceMG9Ho9ACA5ORnt27c3+bxJSUnIz8/HkCFDZMeLi4sRFRXlgJ4QkRIY3BCRy8nLy8OwYcMwbNgwfPvtt6hXrx6Sk5MxbNgwFBcXm31cbm4uAODXX39Fo0aNZPd5eXk5tM1EVHsY3BCR02nRogU8PDywd+9eNG7cGABw48YNnD59GgMGDMCpU6eQkZGB2bNnIzIyEkDZsJSUp6cnAECn0xmOtW/fHl5eXkhOTsaAAQNqqTdEVNsY3BCR0/H398djjz2GF198EXXq1EFYWBheffVVaLVlcyAaN24MT09PfPLJJ3jyySdx7NgxvP3227LnaNKkCTQaDTZs2IA777wTPj4+CAgIwAsvvIDnnnsOer0e/fr1Q1ZWFnbu3InAwEBMnDhRie4SkZ1xthQROaU5c+agf//+uPvuuzF48GD069cP3bt3BwDUq1cPy5Ytw+rVq9G+fXvMnj0bH374oezxjRo1wptvvolXXnkF9evXx5QpUwAAb7/9Nl5//XXExsaiXbt2GD58OH799Vc0a9as1vtIRI7B2VJERESkKszcEBERkaowuCEiIiJVYXBDREREqsLghoiIiFSFwQ0RERGpCoMbIiIiUhUGN0RERKQqDG6IiIhIVRjcEBERkaowuCEiIiJVYXBDREREqvL/fw7LsBqgXnMAAAAASUVORK5CYII=", - "text/plain": [ - "
    " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "new_cases_usa.plot.line(\n", - " rot=45,\n", - " ylabel=\"New Cases\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sM5-HFDx70RG" - }, - "source": [ - "## Visualization #2: Symptom-related searches compared to new cases" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "se1b6Vf4XB9_" - }, - "source": [ - "### Filter data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Wl2o-NYMoygb" - }, - "source": [ - "We're curious if searches for symptoms like \"cough\" and \"fever\" went up in the same times and places that new COVID-19 cases occured, compared to non-symptoms like \"bruise.\" Let's plot searches vs. new cases to see if it looks like there's a correlation." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "olfnCzyg8jYi" - }, - "source": [ - "First, we select the new cases column and the search trends we're interested in." - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": { - "id": "LqqHzjty8jk0" - }, - "outputs": [], - "source": [ - "regional_data = all_data[all_data[\"aggregation_level\"] == 1] # get only region level data,\n", - "symptom_data = regional_data[[\"location_key\", \"new_confirmed\", \"search_trends_cough\", \"search_trends_fever\", \"search_trends_bruise\", \"population\", \"date\"]]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "b3DlJX-k9SPk" - }, - "source": [ - "Not all rows have data for all of these columns, so let's select only the rows that do. Finally, lets add a new column capturing new confirmed cases as a percentage of area population." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "g4MeM8Oe9Q6X" - }, - "outputs": [], - "source": [ - "symptom_data = symptom_data.dropna()\n", - "symptom_data = symptom_data[symptom_data[\"new_confirmed\"] > 0]\n", - "symptom_data[\"new_cases_percent_of_pop\"] = (symptom_data[\"new_confirmed\"] / symptom_data[\"population\"]) * 100\n", - "\n", - "\n", - "# remove impossible data points\n", - "symptom_data = symptom_data[(symptom_data[\"new_cases_percent_of_pop\"] >= 0)]\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# group data up by week\n", - "weekly_data = symptom_data.groupby([symptom_data.location_key, symptom_data.date.dt.isocalendar().week]).agg({\"new_cases_percent_of_pop\": \"sum\", \"search_trends_cough\": \"mean\", \"search_trends_fever\": \"mean\", \"search_trends_bruise\": \"mean\"})" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "IlXt__om9QYI" - }, - "source": [ - "We want to use a line of best fit to make the correlation stand out. Matplotlib does not include a feature for lines of best fit, but seaborn, which is built on matplotlib, does.\n", - "\n", - "BigQuery DataFrames does not currently integrate with seaborn by default. So we will demonstrate how to downsample and download a DataFrame, and use seaborn on the downloaded data." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "T9Hub_EAXWvY" - }, - "source": [ - "### Graph with lines of best fit" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "hoQ9TPgUPJnN" - }, - "source": [ - "We will now use seaborn to make the plots with the lines of best fit for cough, fever, and bruise. Note that since we're working with a local pandas dataframe, you could use any other Python library or technique you're familiar with, but we'll stick to seaborn for this notebook.\n", - "\n", - "Seaborn will take a few minutes to calculate the lines. Since cough and fever are symptoms of COVID-19, but bruising isn't, we expect the slope of the line of best fit to be positive in the first two graphs, but not the third, indicating that there is a correlation between new COVID-19 cases and cough- and fever-related searches." - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": { - "id": "EG7qM3R18bOb" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 59, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAsz5JREFUeJzs/XeQZPd5341+Tu7ck/PMJgCLBbBYLLEgGCVSlEVRDKIYAPnVK8tSlZOqLMv0lSW6JNlyWWLJrlLR9vWVS657JbkcXoCkSNGiKVqiKAYxYReZ2AV2sWly6BxOPuf+caYbE3pmZ2Z7Znpmfp8qFLlzZrp/3bvTz/c84ftIYRiGCAQCgUAgEOwR8n4fQCAQCAQCwdFCiA+BQCAQCAR7ihAfAoFAIBAI9hQhPgQCgUAgEOwpQnwIBAKBQCDYU4T4EAgEAoFAsKcI8SEQCAQCgWBPEeJDIBAIBALBnqLu9wHWEgQBMzMzpNNpJEna7+MIBAKBQCDYAmEYUqlUGBkZQZY3z210nPiYmZlhfHx8v48hEAgEAoFgB0xOTjI2Nrbp93Sc+Ein00B0+Ewms8+nEQgEAoFAsBXK5TLj4+PNOL4ZHSc+GqWWTCYjxIdAIBAIBAeMrbRMiIZTgUAgEAgEe4oQHwKBQCAQCPYUIT4EAoFAIBDsKUJ8CAQCgUAg2FOE+BAIBAKBQLCnCPEhEAgEAoFgTxHiQyAQCAQCwZ4ixIdAIBAIBII9RYgPgUAgEAgEe4oQHwKBQCAQCPYUIT4EAoFAIBDsKR2320UgaBCGIYsVm6rtkTJU+tPGlnYGCAQCgaCzEeJD0LEsVmxenCrhByGKLPHwWJaBTGy/jyUQCASCu0SUXQQdS9X28IOQka44fhBStb39PpJAIBAI2oAQH4KOJWWoKLLETNFEkSVShkjUCQQCwWFAfJoLOpb+tMHDY9lVPR8CgUAgOPgI8SHoWCRJYiATY2C/DyIQCASCtiLKLgKBQCAQCPYUIT4EAoFAIBDsKaLsIugohLeHQCAQHH6E+BB0FMLbQyAQCA4/ouwi6CiEt4dAIBAcfoT4EHQUwttDIBAIDj/ik13QUQhvD4FAIDj8CPEh6CiEt4dAIBAcfoT4OKCIqRCBQCAQHFSE+DigiKkQgUAgEBxURMPpAUVMhQgEAoHgoCLExwFFTIUIBAKB4KAiItYBRUyFCAQCgeCgIsTHAeWoToWIRluBQCA4+AjxIThQiEZbgUAgOPiIng/BgUI02goEAsHBR4gPwYFCNNoKBALBwUd8cgsOFKLRViAQCA4+QnwIDhRHtdFWIBAIDhOi7CIQCAQCgWBPEZkPQUchRmkFAoHg8LPtzMc3vvENPvjBDzIyMoIkSXzhC1/Y8Hv/4T/8h0iSxKc//em7OKLgKNEYpb06X+XFqRKLFXu/jyQQCASCNrNt8VGr1Th37hz/6T/9p02/7/Of/zzf/e53GRkZ2fHhBEcPMUorEAgEh59tl13e97738b73vW/T75menuYf/+N/zFe+8hXe//737/hwgqOHGKUVCASCw0/bP9mDIOBnf/Zn+ZVf+RUefPDBO36/bdvY9hup9XK53O4jCQ4QYpRWIBAIDj9tn3b53d/9XVRV5Zd+6Ze29P2f+tSnyGazzf/Gx8fbfSTBAaIxSnuyP8VAJiaaTQUCgeAQ0lbxcenSJf79v//3/NEf/dGWg8YnP/lJSqVS87/Jycl2HkkgEAgEAkGH0Vbx8c1vfpOFhQUmJiZQVRVVVbl16xb/7J/9M44fP97yZwzDIJPJrPrvMBKGIQtli+uLVRbKFmEY7veR2sphf30CgUAgaB9t7fn42Z/9WX70R3901dfe+9738rM/+7P8/M//fDuf6sDRSdtY7+SlsROvjU56fQKBQCDobLYtPqrVKteuXWv++caNGzz//PP09PQwMTFBb2/vqu/XNI2hoSFOnz5996c9wKwcIZ0pmlRtb98swu8kFHYiJDrp9QkEAoGgs9l22eXixYucP3+e8+fPA/CJT3yC8+fP85u/+ZttP9xhopNGSO/kpbETr41Oen0CgUAg6Gy2HSHe9a53bauef/Pmze0+xaGkk0ZI7yQUdiIkOun1CQQCgaA1puNTtb19/4wWt6d7RCdtY72TUNiJkOik1ycQCASC1dieT77mYDo+urr/O2WF+DiC3EkoCCEhEAgEhwPXDyjUnI5bVSHEh0AgEAgEhww/CCnUHSqW15HWB0J8CAQCgUBwSAiCkJLpUjJdgg4UHQ2E+BAIBAKB4IAThiFl06NoOvhB54qOBkJ8CAQCgUBwgKlYLsW6i+sH+32ULSPEh0AgEAgEB5C645GvOTjewREdDYT4EAgEAoHgAGG50dis5fr7fZQdI8SHQCAQCAQHAMcLKNQdah02NrsThPgQCAQCgaCD8fyAQt2lYrn7fZS2IcSHQCAQCAQdiL9ibLYTvTruBiE+BAKBQCDoIMLwDdFxEMZmd4IQH3tAGIYsVuxVu1IkSdrvYwkEAoGgwyhbLsWaixccvAmW7SDExx6wWLF5caqEH4QossTDY1kGMrH9PpZAIBAIOoSaHY3NHiSvjrtBiI89oGp7+EHIcFeMyzNlLs+WAUQGRCAQCI44luuTqznYB3hsdicI8bEHpAwVRZa4PFNmqmAiIeH6JZEBEQgEgiOK7fkUai515+CPze4EIT72gP60wcNjWS7PlpGQuH84zWzJomp7Ym29QCAQHCFcP/LqqFpHU3Q0kPf7AEcBSZLoTxv0pw3cIODybBlFjjIiAoFAIDj8+EFIrmozVTCPvPAAkfnYMxYrNtMFE02WcXyfka44/Wljv48lEAgEgl3koKy432uE+NgjqrZHEMKZkQwzRZOYphzIZlMxNiwQCAR3JgxDypZHsX4wVtzvNUJ87BGNptOZookiSwe25CLGhgUCgWBzqrZH4QiNze6EgxkBDyCNptOVGYODSGNseKQrzkzRFE2zAoFAsIzp+OTrR29sdicI8bFHSJLEQCZ24AP1YcngCAQCQbuwXJ9C3cF0hOjYKiJyCLZFJ2dwRD+KQCDYS1w/oFBzqB6CFfd7jRAfgm3RyRkc0Y8iEAj2As8PKJouFcs7dNtm9wrh8yE4NKzsR/GDUNyNCASCthIEIfmaw1TBpHwI19zvJSLzITg0iH4UgUCwG4RhSNn0KJpibLZdiE9nwaGhk/tRBALBwaRiuRTrrhibbTNCfAgODZ3cjyIQCA4WdSdace94QnTsBkJ8CAQCgUCwjOX65GsOlvDq2FWE+BAIBALBkcfxom2zNdGovicI8SEQCASCI4vnBxTqLhXL3e+j7All0+VLL81yY6nOf/k7j+6bF5IQHx2MMM0SCASC3cEPQop1h/IR8eqYK1l89tIU//vlWSw36mP5/o08j5/s3ZfzCPHRwdzJNEuIE4FAINgeYfjGivujMDb72nyFp56Z5OuvLbL25f7//uaGEB+C9dxpiZtw9BQIBIKtU7ZcijUXLzjcEyxhGPL9m3mevjjFc7eL665n4xp/923H+TtvPbb3h1tGiI8O5k6mWWLDrEAgENyZmh2NzR52rw7XD/jalQWeujjFjaXauuvD2Rgff3SMDz0ywj0D6X044RsI8dHB3Mk0Szh6CgQCwcZYrk+udvhX3Fdtjz97cZY/eXaKpaqz7vrpoTRPXhjnnff2ocgSurr/m1VEtOpA1vZynOhLtuzlEI6eAoFAsB7b8ynUXOrO4R6bXazYfO7ZKb704iw1Z73AesvJHp58bJyHR7Md1w8oxEcHstVejt129BQNrQKB4CDh+pFXR9U63KLj+mKVpy9O8dUrC+uaZjVF4kfPDPLxC2Mc703u0wnvjBAfHUin9HKIhlaBQHAQ8IOQQt051CvuwzDkuckiTz8zyfdvFtZdTxoKHzo3wkfOj9Kb6vws+LYLP9/4xjf44Ac/yMjICJIk8YUvfKF5zXVdfvVXf5WzZ8+STCYZGRnh7/ydv8PMzEw7z3zo6ZReDrGiXiAQdDJBEFKoOUzm64d2xb0fhPzVlQX+4X97lv/XZ15cJzwG0gb/6F2neOrvv4W/986TB0J4wA4yH7VajXPnzvELv/ALfOQjH1l1rV6v8+yzz/Ibv/EbnDt3jkKhwD/5J/+ED33oQ1y8eLFthz7sdEovR6eIIIFAIFhJGIaULY9i/fCuuDddny+/NMtnL00zV7bWXb+nP8WTj43xw/f1oyr730C6XaTwLqSiJEl8/vOf58Mf/vCG3/PMM8/w5je/mVu3bjExMXHHxyyXy2SzWUqlEplMZqdHaxtHue/hbl/7UX7vBALB7lC1PQqHeGw2X3P4/HPTfPGFGSotelcuHOvmycfGedNE144/T3VVZqw7cbdHXcd24veu38qWSiUkSaKrq6vlddu2sW27+edyubzbR9oWR7nv4W4bWo/yeycQCNqL6fjkavahXXF/O1fn6UuT/MUr87j+6pyAIku8+3Q/T14Y59RAap9O2F52VXxYlsWv/uqv8rf/9t/eUAV96lOf4rd+67d28xh3Rac0fx5ExHsnEAjuFsv1KdQdzBajpAedMAx5ebrMUxcn+fbruXXXE7rC+88O89E3jR66G7ddEx+u6/LEE08QhiG///u/v+H3ffKTn+QTn/hE88/lcpnx8fHdOta2EX0PO0e8dwKBYKc4XkCx7hzKRnc/CPmb15d4+plJXpmtrLvem9L56PlRPvDwCKnY4fzc3JVX1RAet27d4q/+6q82rf0YhoFhdG53bjubP49aD0SnNM4KBIKDQ2PFfdU+fGOztuvzlVfm+eylKaYK5rrrx3oTPHFhnPfcP9ARLqS7SdvFR0N4XL16la997Wv09u7Pxrx20arvYaci4qj1QOy2CZpAIDg8BEFI0XQpmy7BIRMdpbrLn74wzReem6FouuuunxvL8uRj47z5RA/yIb4hXcm2xUe1WuXatWvNP9+4cYPnn3+enp4ehoeH+djHPsazzz7Ln/3Zn+H7PnNzcwD09PSg63r7Tr6P7FREiB4IgUAgWE0YhpRNj6J5+MZmZ4omn7k0xZ+/PIe9plFWluCH7u3nicfGuH9o/yc795pti4+LFy/y7ne/u/nnRr/Gz/3cz/Gv/tW/4otf/CIAjzzyyKqf+9rXvsa73vWunZ+0g9ipiNitHoijVs4RCASHg4rlUqy7h25s9vJs1ET6ratLrNVTMVXmxx8a4mOPjjHSFd+fA3YA245+73rXuzatwx22Gl0rdioidqsH4qiVcwQCwcGm7kQr7g/T2GwQhnzvep6nLk7y4lRp3fWuuMZPnR/lQ4+MkI1r+3DCzuJwttHuMjsVEbvVAyHKOQKB4CBguT75moN1iFbcO17AVy/P8/TFKW7l6+uuj3XHeeLCGH/rzCCGpuzDCTsTIT6W2U7pYqciYqflkTv9nBhpFQgEnYzjRdtma4dobLZiufyvF2b5k+emydecddcfGM7w5GPjvO1UL4osyuBrEVFqmb0oXez0Oe70c1vNxIjeEIFAsJc0xmYr1voJj4PKfNnis5em+N8vzWGuyeBIwNtO9fLkY+M8NJrdnwMeEIT4WGYvShc7fY47/dxWMzGiN0QgEOwFfhBSrDuUD9GK+2sLVZ56ZpKvvbqwrolUUyTe+2DURDrR0/6dKYcRIT6W2YvSxU6fo11nE70hAoFgNwnDkJIZTbAcBq+OMAy5eKvA089Mcul2cd31dEzlQ+dG+Knzo/QkD4eVxF4hxMcye+HGudFz3Kkc0q6zid4QgUCwW5Qtl2LNxQsO/gSL5wf89WuLPPXMJK8v1tZdH8rE+NijY7zv7BBx0US6I0T0WWYv3Dg3eo47lUPadTZhdy4QCNpNzY7GZg+DV0fd8fjSi7N87tlpFir2uuv3DqR48rFxfvi+ftFEepcI8dEB7EU5RDSbCgSCdmK5Prmag30IxmaXqjZ/8uw0/+vFGWr2+tfz5hM9PHlhjEfGu8TnZpsQ4qMD2ItyiGg2FQgE7cD2fAo1l7pz8MdmbyzVePriJF+9vIC3potUlSXec2aAJy6Mc6IvuU8nPLwI8dEB7EU5RDSbCgSCu8H1I6+OqnWwRUcYhrw4VeKpi5N893p+3fWkrvCBh4f5yJvGRGl6FxHiowPYi34T0WwqEAh2gh+EFOoOlQM+NusHId+8ushTF6d4da6y7npfSuejbxrjAw8PkxSfj7uOeIePCJtlV0Q/iEAgWEsQRGOzpQO+4t50ff785Tk+e2mK2ZK17vrJviRPPDbOu0/3oynyPpzwaCLExxFhs+yK6AcRCAQNwjCkbHkU6wd7xX2h7vCF56b50+dnKLcoFb1poosnHxvnwrFucbO1DwjxIRD9IAKBAIg+CwoHfGx2qlDnMxen+Mor8+u25soSvOv0AE9cGOO+wfQ+nVAAQnwIiPpBZAkuz5RxfJ/xnjhhGIq7AYHgiGA6PrmafaBX3P9gpsRTz0zxN9eWWJuviWkyP3F2mI+9aYyhrMjqdgJCfOwSB6mPoj9tMNodZ6FioykyM0WTvpQhSi8CwSHHcn0KdQfTOZheHUEY8u1rOZ66OMkPZsrrrvckdT5yfpQPnhsmHdP24YSdRWPYoBPeCyE+domD1EchSRIxTaEvZXRM6eUgiTeB4KDheAHFukP1gK64d7yA//PKHE9fnGKqYK67PtGT4IkLY/zomUF09Wg3kUqSREJXSBkqCV3pmM9RIT52iYPWR7GTUdzdFAgHSbwJBAeFxor7qn0wx2bLpsufvjDDF56bplB3110/O5rlycfGeMvJXuQOCbL7habIZGIaqZjakVbwQnzsEp3iq7FVgbATo7OFssW3ri01f+Yd9/QxmI235dwHTbwJBJ1MEIQUl8dmD6LomC2ZfPbSNF9+aRZrTV+KBLzz3j6efGycM8OZ/TlghyBLEklDJR1TiXX4wjshPnaJTlnittUMwk6Mzm7n67y+WKMrrjFfrjHRk2ib+OgU8SYQHGTCMKRsehTNgzk2++pchacvTvL11xZZe3xdlfnxB4f4+KNjjHa353PnoBJfLqukDLVjyip3Qnyi7xJ74Vq6FfYmg9D+f+ydIt4EgoNKxXIpHMAV92EY8r0beZ6+OMnzk6V117NxjQ8/MsJPPjJCV0LfhxN2Bqosk46ppGLqgTRHE+KjBYep2XE3MwgTPQlO9iWp2R4n+5JM9CTa9tidIt4EgoNG3YlW3B+0sVnXD/jq5QWevjjJzVx93fWRrhgff3Sc9z442PElhd1CkiSSukI6phHXD/Z7IMRHCw5Ts+NuZhAGMjF+6L5+kZ0QCDoAy/XJ1xysA7bivmp7/NkLM3zuuWlyVWfd9TPDaZ68MM7b7+nryMbJvcDQorJK2lCRD8l7IMRHCw5Ts+NuZhBEdkIg2H8cL9o2WztgY7MLZYvPPTvNl16apd7CZ+StJ3t58rExzo5mD2zm+W5oZKpTMRVDPdhZjlYcGfGxnVLKVkoVe1WaOUwlIIFA0D48PyB/AFfcv75Y5emLU/zVlYV1TbCaIvG3zgzy8QtjHOtN7tMJ9w9JkohrCulYZ3ly7AZHRnxsp5SylVLFXpVm7vQ8QpwIBEcLPwgp1h3KB2jFfRiGPHu7yFPPTHLxVmHd9ZSh8qFzw/zU+VF6U0evfKspy82jhop6AJtHd8KRER/bKaVspZywV6WZOz3PYepPEQgEGxOG0Yr7Yv3grLj3g5C/fnWRpy5Ocm2huu76QNrgY4+O8RNnh0joRyYcAQfLk2M3ODJ/2+2e+tgrH4o7Pc9h6k8RCAStKVsuxQM0Nms6Pl96aZbPXppioWKvu35Pf4onHxvjh+/rPzJ3+g1iy2WVg+TJsRscGfHRrqmPRpmjYrmMdMUwVJl0TNu1SY87nVuYcQkEh5eaHY3NHpQV9/maw+efm+aLL8xQadGLcuFYN08+Ns6bJrqOVOBVZZlULMpyHERPjt3gyESqdk1m7HWZ407n7gQzLtF3IhC0F9Pxydcd7AMyNnsrV+MzF6f4i8vzuP7qkpAiS/zI/QM88egYpwZS+3TCvecweXLsBkdGfGyHzYJpp5U5OmHctZUg608bQpAIBNvE9nwKNZe60/kTLGEY8tJ0iaeemeI713Prrid0hfefHeajbxo9Un1oDU+ORlZa0BohPlqwWXajU8sc+5l9aCXIANEIKxBsEdePvDoOwtisH4T8zbUlnro4yeXZyrrrvSmdj54f5QPnRjrm83G3OeyeHLvB0fiXsU02y250QpmjFfs59dJKkHVahkgg6ET8IKRQd6gcgLFZy/X5yg/m+eylKaaL5rrrx3sTPHFhnPecGTgyfQ0JXT0Snhy7gRAfK2hkD3LVqKF0uhCiKvIq9d4JZY5W7Gew30iQyRJcninj+D7jPXHCMBS/oAIB0Yr70vKK+04fmy3WHb7w/Ax/+vwMJdNdd/2R8SxPXBjn8RM9R+L3+yh6cuwGQnysoJE98IIASYrSh8d6kx2T3diM/SwHtRJk/WmD0e44CxUbTZGZKZr0pQxRehEcacIwpGx5FOudv+J+umDymUtT/PkP5tYtqZMl+OH7+nniwjinh9L7dMK946h7cuwGQnysoJE9GO1KMFM06T1AwbLTykGSJBHTFPpShii9CAREny+FAzA2e3m2zFPPTPLNq0uslUcxVeZ9Z4f52KOjDGfj+3K+vaThyZHUD89Ct05BiI8VdGoz6VboxHLQZu+nGM8VHBUOwor7IAz57vUcTz0zxUvTpXXXuxMaHz4/yofOjZCNa/twwr1DeHLsDQcnuu4B7cgeiKD6Bpu9n8IWXnDYsVyfQt3BbLGxtVNwvIC/vDzP0xenuJ2vr7s+1h3niQtj/NgDQ+jq4Q3EDU+OVEw9cjbv+4V4l1fQjuzBToPqYRQtm72fYhpGcFhxvIBi3WmOnHciFcvlf70wy588N02+5qy7/tBIhicujPO2e3qRD/jn0Gboyw7VwpNj7xHio83sNKgetUzAQS5xCQSt8PyAQt2lanfu2Oxc2eJzl6b40kuzWO7qMpAEvP2ePp58bIwHR7L7c8A9QJHfaB4Vnhz7h/jEbzM7Dap3kwk4iFmTTmuQFQh2ShCEFJfHZjtVdFydr/D0xSm+9uoCa4dsNEXivQ8O8fFHxxjvSezPAfeAhB6ZgCWFJ0dHsG3x8Y1vfIN/9+/+HZcuXWJ2dpbPf/7zfPjDH25eD8OQf/kv/yX/5b/8F4rFIm9/+9v5/d//fe699952nrtj2WlQTeoKVdvl2dsmKSP6BdkqBzFr0okNsgLBdgjDkLLpUTQ7c2w2DEMu3irw1DOTPHu7uO56Jqbyk4+M8OHzo3Qn9L0/4B4gPDk6l22Lj1qtxrlz5/iFX/gFPvKRj6y7/m//7b/lP/yH/8Af//Efc+LECX7jN36D9773vbzyyivEYp0dENvB3QTVMATC5f/dBqJ/Yv85iNknwc6pWC6FDl1x7/oBX7uywNMXp7i+VFt3fTgb42OPjvHjDw0RP4SeFbIkkTAUMjFNeHJ0MNsWH+973/t43/ve1/JaGIZ8+tOf5td//df5yZ/8SQD+63/9rwwODvKFL3yBn/7pn7670x5AWgUlYN3Xao5POqZxeijDTNGkto0OedE/sf8cxOyTYPt08thszfb4sxdn+ZNnp1ms2uuunx5M8+RjY7zz3v5D2VwpPDkOFm2NUjdu3GBubo4f/dEfbX4tm83y+OOP853vfKel+LBtG9t+4xelXC6380j7TqugBOuXrt2NgBD9E/uPyD4dbizXJ19zsDpwxf1ixeZPnp3iz16cbXnT8viJHp58bJxzY9lDl41reHKkDPVQjwIfRtoqPubm5gAYHBxc9fXBwcHmtbV86lOf4rd+67faeYyOYqONr2u/dqIvuWMBIfon9h+RfTqcOF60bbbWgWOzN5ZqPH1xkq9eXsBb03OiyhLvOTPAExfGOdGX3KcT7g6SJJHQleWFbuL37KCy739zn/zkJ/nEJz7R/HO5XGZ8fHwfT9ReNgpKa78mBMTBRmSfDheeH5DvwBX3YRjy/GSRpy5O8f0b+XXXk4bCBx8e4SNvGqUvdbj+DeqqTNrQSMWEJ8dhoK3iY2hoCID5+XmGh4ebX5+fn+eRRx5p+TOGYWAYh+uXZCUbBaW1X9tJw6JocuwchHg8HPhBSLHuUO6wFfd+EPL11xZ5+uIkr81X113vTxl87NFRfuLsMMlDlHUTnhyHl7b+Kz1x4gRDQ0N89atfbYqNcrnM9773Pf7RP/pH7XyqtrHbAXyjoLT2awtla9sNi1E/SZFc1cELQs5PdHFmOLPt8wsRIzjqhOEbK+47aWzWdH2+/NIsn700zVzZWnf9ZH+SJy+M8+7T/YdqlFR4chx+ti0+qtUq165da/75xo0bPP/88/T09DAxMcEv//Iv82/+zb/h3nvvbY7ajoyMrPIC6SQ6ZUphJw2LVdsjV3Wo2B5LFZswDHe0tr5T3gOBYD8oWy7FDhubzdccvvD8NF98foZyi9LPoxNdPPHYOBeOdR+a4Cw8OY4W2xYfFy9e5N3vfnfzz41+jZ/7uZ/jj/7oj/jn//yfU6vV+Pt//+9TLBZ5xzvewZ//+Z93rMfHXk8pbJRl2EnDYspQ8YKQpYpNf9pAV5QdnV9MagiOInUnEu+dtOJ+Ml/nM5em+MoP5nD91RkYWYJ3nx7giQtj3DuY3qcTthfhyXF0kcJOKmwSlWmy2SylUolMJrPrz7eTcsduPN92Sx9hGLJQtnhhqsjrC1V6EgY9KZ1z413bPv9evwcCwX7SiWOzL0+XeOriJN++lmPtB3JMk3n/2WE++ugYQ4fk9zKmRRtkU8KT41Cxnfh9eDqTdsheTynsNMuwVpyEYchL02WCMOofmehJcKw3uaPzi0kNwVGg08ZmgzDk29dyPHVxkh/MrPc36knqfOT8KB88N0w6pu3DCduL8OQQrOTIi4+9nlLYqLxyp76LtdezcRU/CBntSjBTNOndQa9HAzGpITjMNLbNVix3v48CgO36/J9X5vnMpSmmCua668d6EjxxYYz3nBk88EFaeHIINkL8a9hjNsoybJYRCcOQW7ka04U6x/tSmE505yZMrQSCjem0bbMl0+WLz8/w+eemKZrrhdDDY1mevDDO4yd7kA94E6nw5BDcCRGx9piNsgybNZwuVmxu5+vMV2zmKzYn+5Kcn+hCkiRRKhEI1tBp22ZnSyafuTjFn788h7VmJ4wswTvu7ePJC+OcGd79HrfdRJakZllFNI8K7oQQHx3CZn0XVdsjaag8fqKHm7kax3oTq0osDcv2RpOq8O0QHFUqlkux7nbEBMurcxWeemaSb1xdZK0GMlSZH39wiI89OsZod3x/Dtgm4rpCOqYJTw7BthDio0PYrO8iZaiosozlBox2RY2lkiRtOKWyU9+O/RAtQigJ2kGnbJsNwpDv38jz9MVJnp8srbuejWv81PkRfvLcKNnEwW0i1RSZ1LLzqPDkEOwEIT4OANvtE9nORM3K4G+5PtMFkyBkQ9HSbrEgDM4Ed4PtRWOzZottrnuJ4wV89coCT1+c5Fauvu76aFecj18Y470PDGIc0JKEJEkkDYW0oRHXD+ZrEHQOQnzskL28Y99un8h2DMtWBv+lqo0my5wZyWwoWtotFoTBmWAnuH5AoeY0S477RdX2+LMXZvjcc9Pkqs666w8Mp3nisXHefqrvwDZeGlo0rSI8OQTtRIiPHdKuIHw3ImajjMh2fDtWBv9i3cHx/U1FS7vFglhFL9gOfhBSqDtU9nnx20LZ4nPPTvOll2apt8i6vO1UL09eGOeh0e3vWuoEGr+L6Zh24Md9BZ2J+KTfIe0KwncjYjbKiGzHt2Nl8O9N6Yx0xSP3wQ1ES7vFgjA4E2yFIHhj8Vuwj6Lj9YUqT12c5GuvLq6bpNEUib/1wCBPPDrORG9in064cxqeHClDJSGaRwW7jBAfO6RdQXi/yw6tgv9mHzrtFgvC4EywGWEYUrY8SvX9W/wWhiGXbhV46uIUl24V1l1Px1Q+dG6Enzo/Sk9S34cT3h2aIpOJCU8Owd4ixMcO2SwIb6eUsp9lh52UfIRYEOwVVdujUNu/xW+eH/D11xZ56pkpri1W110fzBh8/NEx3vfQ8IFrwJQlieTytIrw5BDsB0J87JDNgvB2Sin7WXY4zJMmYoT34GK5Prmag71Pi9/qjseXXprjc5emWKjY667fM5Dipx8b54fv6z9wmYL4clklZaji90GwrwjxsUW2E8y2U0q5UyZhN4PobpV8OiHwH2ZhdVixPZ9CzaXu7M8ES65q8yfPTfO/XphtOUXz5uPdPPHYOOfHuw5U4G54cqRiKprw5BB0CEdGfGw1IDZW1d/OR7P6Ez2JbRt3tbOUcrdBdLPXvVsln04I/PvdSyPYOp4fkK87VK39ER23cjWevjjFX16ex/VXN5EqssSP3D/AkxfGONmf2pfz7QRJkkguO48etJKQ4GhwZMTHVgPiYsXmW9eWeH2xBsDJviQ/dF//toJZO0spdxtEN3rdYRgShiHZuEoYhiQNtbn1825t2jsh8IsR3s7HD0KKdYfyPozNhmHIi9Mlnnpmku9ez6+7ntAVPvDwMB9909iBmsASnhyCg8KR+UTeakCs2h5V26MrrgESteU/byeYrSyl3G0J4m6D6Eave7Fi89J0GT8IqVgukgQpQ9u2TXur19cJgV+M8HYuYRiNzRbrez826wch37q2xFPPTHJlrrLuem9K56NvGuMDDw8fGMEqPDkEB5GD8dvVBrYaEBvNWPPlNzIfjeC1k2DWKoD3p40tC5K7DaIbve6VouTZWyZIcN/gG86m/WHIrVyN6UKd430pTMfbsuNpOwP/VsTbRt8jpnI6j7LlUqzt/dis5fr8+ctzfObSFLMla931E31Jnrgwxo/cP3Ag+iKEJ4fgoHNkxMdWA2J/2uAd9/Qx0ROZBE30JO4qmLXKPABb7om42yC60eteKUqShooksUqgLFZsbufrzFds5it2U4Rt5fUNZGJtC/xbyb50Qo+JYHP2a/Fbse7whedm+MLz05Rb9JQ8Mt7Fk4+N8ebjPQcigAtPDsFh4ciIj60GcUmSGMzGGcy2Z811q8zDXvRErM0GnOhLrvpwXSlKkssNaTXHbwqUG0s1kobK4yd6uJmrcaw30dLLJFe1qdou08UQVZbbnqreynvVCT0mgtZYbrT4zdrjsdnpgsnTlyb5yg/m1wkeWYIfvq+fJx8b577B9J6eaycITw7BYeTIiI/9YqPMw273RNwpG3AnMZYyVFRZxnIDRrsSHOtdLV4aj+/5AWEIvUmdY73JtvdWbKVc1gk9JoLVOF5Aoe5Q2+PFb6/MlHnq4iTfurrE2m6SmCrzE2eH+dijYwxlOz8zJjw5BIcZ8SndZlr1H6wN8lspAW3W67CVPoi7zQbc6YyNxx/tTizvhTF2pdSxlfdqP5tLO8HTpJPYj7HZIAz5zus5nr44yUvT5XXXuxMaP3V+lA+dGyET1/bsXDtBeHIIjgpCfLSZrfQfbKUEtNnjbOU5tpMN2EnD5lYevx2BeSvv1Va+Z7dEgug3idiPsVnHC/iLV+b5zKWppi/PSsa743z8wjg/9sBgR0+BCE8OwVFEiI8dsFkgu5uMw8rHzVVtvCAqeax9nDs9x0oPD3ijaXYjdhJAt5Jt2I3AvFMRsVsi4aj3m+zHttmK5fLFF2b4k2enKdTdddfPjmZ44sI4bz3Vi9zBWShjeXt02hCeHIKjhxAfO2CzQHY3/QcrH7fhvdHqce70HCs9PBRZQpKkTQP0TgLoVrINuxGYdyoidkskHNV+k8a22WLdWbdafreYK1l89tIU//vlWSx3dROpBLzj3j6evDDOAyOZPTnPTmj8G0nFVAxVZDkER5ej8UnZZjYLZBtlBLbbpzFdCOlN6fSmjHWZhb6UzkhXZALWnzboS+kbPs5WAu1uBdB2PO7a961iuTsSEbv1Gg+zmdlG/2YrVmQQtlfbZl+br/DUM5N8/bVF1uocXZV574ODfPzRMca6E3tynu0iSRLxZedR4ckhEEQI8bGGrYiEzQLZRhmB7fZpqIrMsd5ky7v6parDTNHCD0JmihZ9a5o97xRo177GvpS+KwG0HYF5sWLzwmSRQs3F8X2O9yVR5NYZod0+SysOs5nZ2n+z9w6kUBRpT7w6wjDk4q0CTz0zybO3i+uuZ2IqH35klJ88P0J3Ql//AB2ArsqkDeHJIRC0QogPVgdjy/WZKZr4ARuKhJ0Esq1kI7b6uHd6rFaPs/Y1ThdMgnD1a2x3AG1HYK7aHoWaS8V2WVxeb/6mY93EluvlWxURh1kk7BaNf2d9KYPX5itoisR4z+5mF1w/4GtXFnj64hTXl2rrrg9nY3z80TF+/KGhjvS8UOQ3PDlEWUUg2BghPlh9h7dYsdAUmQdGshuKhJ0Esq2k/bf6uHd6rFaPs1C2mq9xqWqjyTJnRjId3ySZMlQc32exYtOXNtAUmZimHKgNowcVQ5WpWC7zZQtZlkjou/dxUbM9/uzFWT737BRLVWfd9fuH0jz52DjvuKev47IIoqwiEGwfIT5YnUko1V3cIOjo3oC7zbwU6w6O77f1Ne7WKGt/2uBNx7qRJAlVluhN6UemqXO/aHh1WK7Psd4kdccjoav0JNvvkbFYsfncs1N86cVZas56F9S3nOzhycfGeXg023FBXZRVBIKdIz7FWZ1J6E5qjHTFqNkexbrLzaUqYRgykInd1YdfO9P+d5t56U3pjHTFm6WLvpTOfMlseiVM9CS2/Xp3a5RVkiTODGfoSxkblpGEuVd7WOvVIUmR2Oul/T0VN5ZqPH1xkq9eXsBb00WqKRI/emaQj18Y43hvsu3PfTeIsopA0B6E+GB9JiEMQ67MVXh9sbHZ1uSH7uvfcjDtxMDYKlvSONNC2eKbV5eaNfZT/UneeW//trbv7qbfxZ3KSEfZ3KsdBEEYbZvd5RX3YRjy3GSRp5+Z5Ps3C+uuJw2FD50b4SPnR+lNdc7UkNggKxC0HyE+WB/cri9WqdoeXXENkKjZrdfJb0Qnul5uli2p2h4126MrrgMh1eXXC1vfvrvXfhdH3dyrHTS8Okr13V1x7wchX39tkaeemeTqQnXd9YG0wUcfHeP9Z4d2ta9ku4iyikCwe3TOb/ous51sRGOZ03y5kflovU5+I+42MLYrc7LR46z9elJXSBoq85XlzEcque3tu3vtd3FUzb3aRTRF5OyqV4fp+Hz55Vk+e2maubK17vo9/SmefGyMH76vH7VD9pgIE7CDTydmngXrOTKf2BtlI4Ig4MpcpWnYdf9Qmv60wTvu6WNieazwTvbka7nbwNiuzMlGj7P669H44kRPnExMpSuhrdpOu9XXsdejrIfZ3Gs3qTse+Zqzq14d+ZrD55+b5osvzFBpsWDuwrFunnxsnDdNdHVEUBBllcNFJ2aeBes5MuKjYrnkqw6ZuEq+6lKxXAYyMa7MVfjyS3O4ftDcIvnASJbBbJzBbHzLj79SbSd1hbOjGWqOv6PA2K6SwkaPs/Lrr8yUmCtZ9KdjKLLM8b5U8xe1kwO88O3YHpbrU6g7mC0mStrF7Vydz1ya4v+8Mofrr+4dUWSJd5/u58kL45wa6IwxaV2VSce05s2C4HAgSrIHgyMjPmwvYLJQx12KRMZDY9H+h8WKjesHnB7K8OpcuWlktV1aqe2VXhTbLfu0o6Sw0eOs/LoXhOiK0vIXtVMDfCemVTvxTBBtfi3UHWr27qy4D8OQl6fLPHVxkm+/nlt3Pa4pfODhYT76ptGOuPsUZZXDjyjJHgyOzN+KocqMdcfJJjRKdRdjecV2/7Jx1atzZTRF3vHd/Z3U9nZSge3IOGy22Xbl44/3xJkumHvyi9quAN14L70goGZ7TPQkmqWi/Qr4nZbqbXh1VFuUPdqBH4T8zetLPP3MJK/MVtZd703q/NT5UT50boRUbH8/ZhpllXRMJa6Jssphp5MztoI3ODLiIx3T6E0Z+EFIb8ogHYsMk+4fSgOs6vnYCXdS29tJBbYj47DZZtuVjx+G4ToPjd2iXQG68V7GNYUXp0pULY+S6e1rwO+UVO9ar452Y7s+X3llns9emmKqYK67fqw3wRMXxnnP/QPo6v42kYqyytGkUzO2gtUcGfGxkRqW5chKfbcev8Fm4uROGYEwDFkoW9syAdtKMGy1YG43SwftCtCN9/JmLprOOd6XwnL9fa3t7lWqd7MJppK5e14dpbrLn74wzReem6FouuuunxvL8uRj47z5RA/yPmYWRFlFIDgYtP0T0vd9/tW/+lf8t//235ibm2NkZIS/+3f/Lr/+679+qNOdd1Lbm4mTO2UEFis237q2tML0LNnS9Gzt8jhZ2nz769rnHemKNbfl7kbpoF0BuvFeZuMqSb2O6Xioiryvtd29SvWu/Ts7O5ohpqu75tUxUzT5zKUp/vzlOew1EzKyBO+8t58nHxvj/qFM2597q4iyikBw8Gj7p/Xv/u7v8vu///v88R//MQ8++CAXL17k53/+58lms/zSL/1Su59uy2wU4BsBu2K52F6AsZyqbfdd/51MvhoZgelCnVu52qog1jD9upPp2doR2tHu+KbbX9dmIhYr9q6WDtoVoBvvZX/a4FhvsiNqu3uV6l35d3Z9scq1xSrD25jK2ipX5so89cwU37y6yBr3cwxV5scfGuLjj44x0tX+594qxvK/bVFWEQgOHm0XH9/+9rf5yZ/8Sd7//vcDcPz4cf7n//yffP/732/3U22LjVL+jYCdq9pMFUzGuxP0pPQ97R9YmRGo2h41xyNfc5siaSumZ2EYcitXY7pY53hvEtP1N9z+2hBcuapN1XaZLoaoctRsO1O0dq100O4Avdnjder0yd2SMlQ8P+ClqSIBtDX4B2HI967neeriJC9OldZd74przSbSbKL9S+a2girLJA2FdEzb954SgUCwc9ouPt72trfxB3/wB7z22mvcd999vPDCC3zrW9/i937v91p+v23b2PYb463lcrndRwI2Tvk3REk2oXFjqUYmruIH4Z72D6zMCOSqNrmas0oknehLNk3PwjAkaahULLf5s5IksVixuZWrM1+2mS/bnOrf2JW1OS3iB4RhNJlwrDdJX0rfs+bT3WY3p0/2S9hYro8XBAxkYqRiats2zTpewFcvz/P0pSlu5errro91x/n4o2P82AODGNre91FIkkRSV5qvWSAQHHza/pv8a7/2a5TLZe6//34URcH3fX77t3+bn/mZn2n5/Z/61Kf4rd/6rXYfYx19KZ2RrlhzqqUvFW3qbIiSXNVBlSWm8iYxXSJpKIRhuGEJpp0BaOUdfMpQKZneKpEkSVLT9GyjhWqNczx+opebS9VNXVkbgmu0O7G85dZoBuatZiY2e/07fW/a+Z7u5vTJXo/VrvXqaNem2arl8cUXZvj8c9Pkas666w8MZ3jysXHedqp3X8oaoqwiEBxe2i4+nn76af77f//v/I//8T948MEHef755/nlX/5lRkZG+Lmf+7l13//JT36ST3ziE80/l8tlxsfH230sFis2l2fLVG2PpapNb1JnMBtvZh0qlstod5ybSzVML+C713NMdCc3LMHsVgBa2xfRl9JZKFvNP1cst2VQTRkqqiJjuT6j3ZHvxVZNzJK6suo5thL0N3v9d+qv2eh52vme7ub0yV6N1Xp+QKHuNrNc7WK+bPG5Z6f40otzmO56x9O3n+rlycfGeWj07qfAtosqy6RikeAQZRWB4PDSdvHxK7/yK/zar/0aP/3TPw3A2bNnuXXrFp/61Kdaig/DMDCM3U/v387XeX2xRldcY75cY6Insco+XZIkDFWmL20QhiG3l2qYKZdcNWxasa9kJ6OsWwnqa/sY1mY6RrpiLYPqdpo5135vEAR88+oSNdsjaai8896+O1rLb/b679Rfs5G4aGdQ383pk90eq/WDaGy2ZLpt9eq4tlDl6YuT/NWVhXVNpJoi8WMPDPHxC2PNnUZbJQxD8jWXuuM1S0HbyViJsopAcPRo+296vV5HllffsSiKQrCLK7u3QhiG1G0f3w+xvYAgCFgoW9zK1biVq5PUFWZLFrbvY7sB+ZrDtQXoSuicHVt/B9gIQNPFOrXlXo21AqMdd/JrA7Khyi2D6lrR0vAGaSV81n7vMzdyXF+qkY1pXF8qoSkSbz3V19JvZOUoryK3HuW9U3/NRuKinUF9N6dPdkvYhGFI2fQomg7+WnVwF4956VaBpy5OcelWYd31TEzlQ4+M8OFHRulJ7qyUk6+5vDpfIQhCZFni9GCa3tSdH8vQovHYlK4ii7KKQHCkaLv4+OAHP8hv//ZvMzExwYMPPshzzz3H7/3e7/ELv/AL7X6qbZE0VGQJSqZDQldx/JAXp4pcmSszUzC5fzjLYsUiGVMhDLmnP8X9w2kqlt+0Yl9JIwDdytWoWh65qrPOZXNlsJ0q1Hj+dgFDU+hL6fQmdepusO09L+mYtqWgupnwWZuRCZZtyit1h6mSRV9KI2loq8olC2WL5ycLvDhVIq4qDGRiPDiaIa6r6wLwRsH5TuLioNgi74awqVguhVr7vDo8P+CvX1vkqWcmm/4wKxnKxPjYo2O87+wQ8btsIq07HkEQMpCOsVCxqDvehj0poqwiEAhgF8THf/yP/5Hf+I3f4Bd/8RdZWFhgZGSEf/AP/gG/+Zu/2e6n2hYxTeH0ULq528UPQnJVB8fzuZGrc3WuzGB3go+eH2Wh4uD6AZIsoSjRivB02VqXPehPG9zK1ajZHv3pGKaz2n9jZbCdK1lM5k10VcbxA8a64ox2J3Ztz0vV9vCCgLimcDNXIxOLGmhrjo/peFyerTTLLANpHUWWWKw4SFLIeHdi1cRPw+TsW1eXuJmrM9odY6nmcqI/yYOjXeuee6PgfKfXspOgvpXSVrsaWXdjyqXdK+7rjseXXprjc5emWGixJPG+wRRPXhjnh+7rb1sTZ2I5c7FQsZBlaV3ppFFWScc04rpwHRUIBLsgPtLpNJ/+9Kf59Kc/3e6HvivW7nYZyMSYLlpMFWw8P6DmBkzl6zx3u8TZsQyj3YnIzGuDrAZEQfl2vs58xWa+Yq/z31gZbC3XY65kc3oow/duLFGoOTx2onfT3oaVwS6pR+LhxlJtS4EvZajUbK/p1xAEIbfzJumYxvXFCnNlm9GuBPOVGqoMpwfT3DeY4vJcmZLpkorpq8olVdsjaSjENAnHDbC97a9m342MwVZKW+1qZG1nQ6zt+eRr7Vtxv1S1+ZNnp/lfL85Qs9c/5ptP9PDkhTEeGe9q+1hwT1Lj9GB6Vc8HRII/JcoqAoGgBUemu6s/bXB2NNPcj9KT0HhkPMurMyWCICQb13D8gGLdZrQ7wZnhDDeWauRr7oY9CtXlzMHjJ3q4matxrHf1eOvKYGu5PtcWarw6Vyahq3Qn9Tv2NqwMdlXbJQwjEdWw1ZYkacO78P60wURPgqrlcbwvxY2lKNNxeijDtfkKrh8AUV9BwlBJxWS8IODh0a5VW2KBVeOOcU0laajcN5RqNibup6HX2j6SxmTIWofYdjSytuNxXD+gUHOotmnF/c1cjaefmeIvL8/jrekTUWWJ95wZ4IkL45zoS7bl+VohSVJz/FeUVQQCwVY4MuKjsdW1ZHrL0wQeZ0czPHaql8sLFUqmR09KpzthEFveD3GnHoWUoaLKMpYbMNq1+Xjryu25rXo+VtII5pdny+SqNmdGMjx324QQTg9lmCma3MrVmCyYzSD7jnv61k3vHOtNUjI9TNcjBEzX5wfTRWKaTHdCw/F9TvYleHg0iyzLmwqZd9zTx3h3nGLdpSuhcaw3ecfR2r1g7d+R7QXcWHOWVn+POxFMd9MQ285ts2EY8uJUiacuTvLd6/l115O6wgfPjfBT50f3pG9GlFUEAsF2OTLio2k/XqhzvC+F6XjUHJ8zQ2neeqKPhaqF64X0prQtj69upx9jO9tzG8E8X3WYLNQp2x6e7xNTFaaLdVRZplh3eX2hiirLXJmtkI6p/K01m25XNsVWTI+EGpKv2xiazERPEtcPODO8eQYFWGVy1or9XCe/9u+gbDrkqw6ZuEq+GnlknOxPrft72olg2kn/TTu3zfpByDevLvHUxUlenausP1/K4KOPjvL+s8Mk92DJniirCASCnXJkxMdG/RlhGDLRm0BXJRRF5tHjPeuMvU70JZtryxfK1roldMd7EyxWbC7ejO5CV6683+4d9sodLRPdcaYKEpO5KuPdSZK60rRCv7lUpe4E1BybiuXx+mKNRyr2qgDaKPtUba9ZPnr2Vh4keGAky0zRpO74vDRd3rYh2Er2ep18qyWAjde9VLWZLNRxlwI0ReahsUzLXpOdCKbt9qy0a4LFdH3+/OU5PntpitmSte76yb4kT1wY4933D6Apu1vqaJRV0jF1159LIBAcXo6M+FjbnzHREycMQ24u1ZgpmbhuQLcR+ZH85SvzXFuo0psy6EnqnBvvYiATW5eRGOuO05syGOmKcXm23HLl/ULZ2paB18odLdcXa1QttzlNAHKzWTYMQwYzOrdyHqeH0nTHtQ0D6EpxkDRUJOkNfw5gR4ZgK9nrdfKbLQE0VJmx7nhzqqnVmPTa96TdgqldEyyFusMXnpvmT5+foWyt7xF5dKKLJx4b58Kx7l3tsZGkaN1A2hBlFYFA0B6OjPhI6go122O+bJEyoqbJl6bLXJktcWW2zD39aW7lTHJVh0LdIVdzOTOUQUJqBuTG3XImruIuBWQTGn7wRoZg5cr7RuPjd6/neHm6zHA2xnwlakrdTHys3NFy8UaOhKbQm9ZZrNgYqtwMkgOZGD98eoDnbhdR5ajhb6MAulIcJJeDR83xm5mfklnetiHYSvZ6nfxmSwDXTjWlY60Xr+2GYFo7wbJT58/JfJ3PXpriK6/MrxMwsgTvOj3AkxfGuHcwfddn3gxRVhEIBLvFkREfAGEIhNH/ThXqzJVtdFUmCMH2AhwvwJRCepMGjg9zJZO+FUG9cbecr7poikyp7tKbMuhPGyxV7VUr7xuNj5P5OgsVk0xsa2/1yh0tx/qSQIgfQFxTOT/RtcrR9MxwZktbaO+0ev7hNT0fK1/r2ibNhbLVnBhaWV7aC1YuAdQUmfJyk/BG4807fU+2y0YTLNt1/nx5usRTz0zy7ddzrO0OiWkyP3F2mI89OsbQLjbzakokcFOirCIQCHaRIyM+ao5POqZxeijDKzMlri/WqNg+VcuhK6GRiakMpA1qjstcyUKWQo71JnnTse5mAFu5hO6hsUwzExGGIePdcdIxla54NAnSuEt/aKyLxapDSMip/uQd92ZslqVY23fRjgC6HUOwxYrNN68ucX0pElmn+pO8897+OzZqbtarsR3hsvL9PzuWXfU4d3o9u4HnRzb8t/ORxf7a7MZWnD+DMOTb13I8fXGSl2fK656jK6HxsTeN8cFzwxtmce4WWZJIiLKKQCDYQ46M+Fh5J+8FIT0JgwdG4txcrDCUjdGd1CnUHabyISPZGLIs80P39TenQVY2YKZjGieXA+dC2VrRsClzvC8VZQPKFoosYTk+Z0ezHOtd7Z2xEXsZPFfSqsG0cY5GxuO713O8MlMkqWuklntMtrJQLwxDXpour+uV2e5IbvO92aMx3o1YOTa7VLE3zG5s5PwZhiFzJZu/uDzHV34w37KJdCgT4/ETPXzw3DAn+1O78jpiy7tVkqKsIhAI9pgjIz5W3smP98SZLphYrs9Id4K4rvDafJVC3Wax4nBmOEPN8lms2CxW7OZd/wuTRQo1F8f3edOxbs4MZzbsjWiVOdiN8kS7sgprG0xXmphZrs8rMyVemi5zO28iUWe8N8HDo10t+0zWPlZ2uTdjba/MXo7ktoMgCCmaLmXzjbHZzbIbrZw/y6bL//PM5IZOpGeG0jx6rJt7BlKoikw2vrNlbxshyioCgaATODLiYyW9ycjkq+b4WK7PpZt5XpuvYjoetwp1JgtVFEkhCAO8gKaIKNRcKrbLYsVGkiT6UsaGUxN7lcHYygTISjYaoV0rom7n601DtqWqTaFuM5KNkVm2bT833sVbTva2zOSsfSygZa9MuyZMdtthteHVUTLdddtmN9trstL5c65k8f/+2ut8+aVZrDVNpBLwznv7ePKxce4fSq9rUr1bxLSKQCDoNI6M+FgoW3zr2tIqR9CT/SmuL1ax/BDHD7iVMylaLt3xOBXLj6YXqg4VyyUdixxBFys2fWkDVY4C9om+5I6Mp9oVLLcyAbKSjUZo14ooeGMEt1h3UCSJoulSd3wG0wb3DqY3bDZd+1gTPQkkSVrVK7O2V+Nu3qvdclgNw5Cy5VGqb+zVsdFekwavzVd46plJvv7aImt0C6os8aZj3fzfj0/w0OgbBnQNwXK3iJX1AoGgUzky4uN2vs7rizW64hrz5RoTPdHIa8pQiasyuizTm9LwggBFUajaNt+5kWM4U2e4y+Dt90TNpwCmF+D6AZYbpc23k+EIw5DLs2WevVVAVxS6k1rTR2Tt921FoKyeAJGYzNdJGCrjyz4ma3+mIVaGu2JcnilzeTZqcuxbzpas7NNojOD2pDRGumJcX6xyY6mGHwa8MlOmN6m3HBveqOS0W8vcdsNhNcp02cyV7E1HZVdmNxqEYcj3buT579+7zQ9aNJFmYirvfXCIH7qvj6FMvC3ZjQaN7Fs6pondKgKBoGM5MuIjIqRiuRTrNoWaQxiG9KV0jvclubFYxQtCNAVyFQs/CPG8kNmSyYtTRU4PZTgznCEMQ77x2iJF1+OVmdKGAbj5jC2aL5+7XWSqYDbv/FsFy416TNYGv5UTIKPdcW4sVtFkmemCSV/KWBeoG2Ll8kyZqYKJhITrl5pBvXGOlSO4luszXTCpWB7zFSfajLu0sWdJO0tOWxEW7TQMMx2ffN3Bdn1yVWdbo7KuH/BXVxZ4+uIUN5YnglbSm9T5qfOj/NT5EeJ6+371JEkivpzlSOjKno0+CwQCwU45MuJjvDtOTJN5ba5CXFcpmVHvBkQbZ3VNwfYCTvanmSrUCG03KsZLMpXGVEcmRt3xqdg+XXGN60t1jvXWGczGN8xUtGq+VGWJvuUm1pXGYSvZqMek0fy6bipluQRSs/1Vgbp/zbkaGY7Ls2UkJO4fTjNbstYF9ZUC4vpilSCEvnSMYKaM4wco8s7uqle+TytHiTcaK96KsGiHYZjt+RSWey0abGVUFqK/qz97cZY/eXaKpaqz7npfUmeiN8FbT/UwnE1QdwLa0UeqKTKZmEbSUFBF82hHsJ8bngWCg8SRER+SJKHJMilDYzAba/ZFAPgBHOtN8PpiFVWRMTSZ7rhBKqbh+AGZmEpSV1goW0wX6ixWLDwvwPaD5obSrU7DAM2757imrDIOW0nKUFv2mAAbliFaBerN7N1dv8RsybpjtqDxuBIwmo38TIaz8Tt6lrRipRirWC6SBEldZaZoYvs+PQmD3pTOw2NRKWorwuJuMi2uH1CoO1Rb2Jdv1kzaeC2fe3aKP3txlrqzfnLlnv4kx/uSGApoqspEd2Q+t5GI2QqyJJE0ot0qMU00j3Ya+7nhWSA4SBwZ8VFzfHqTMXRVYbFi44c0yyBV22WpYhNXFaqmQ0xV8P2QmCZxsi/JWFec77y+RKEeCYtCzcH2fPqSseb20K1OwzSaL+90Z9SfNnjT8s6Olfbpm5UhWgXqizfzXF+q0RXXV9m7bydb0J82ODua4VauRndCoyuhNT1LVi7g28pd3srzP3vLBAn6UjGuLlQJCdEUpfl9A9ydsNjsLnQrK+43aia9vljl6YtTfPXKwrrpF02R+FtnBvnYo2OkDZWZkknZdKPyleejyPI6EbMVGp4cKUMVd9IdzH5ueBYIDhJHRnykDJXu5eBhqDLnJ7roS+lcni2zULYIw5BsXKVQt7H9kKLpoSoyARLPT5aoOz4ly+P8WIbhbJxTAynimkJMUwjDEMv1mSnVWara9KUMCnWbW7kajx7rXhXk+1J6y9T8WjazT9+oDLF5oF4dJLcT1CVJQpIkypZPSPS/kiSxVHW2fZfXasndzaUquirTndAiEagpzUzT3aSvW92F9qeNLa+4X9lM2ujVefriJN+/WVj3vQld4b0PDPG33zxGX/qN96A3bbTc8bIVGhtkU4YqmkcPCHu14VkgOOgcmd+MvpTOaHccTZFQFRldkbgyV+G528VmILqdr7NQsSiaLnFNpS9lMF0wCUM4M5IlP11krmyTTagUajbTro8EmI7HTNEipWtMOnWmiyb9KYNbuTrHepOrnEK3MunSoJVA2G5/w0RPglP9kd37qdSd7d0brM0aVCx33R0dtN6Iuxmt7ONv5+skDQU/iMog5ye6gI3LS1utq6+9C50rW1husK0V934Q8tevLvLUxUmuLVTXXe9L6bz5eA/nJ7qI6yqStF4ktJqI2QhJkkjojebRI/PreWjYqw3PAsFB58h8ui1WbC7PlpktmeRrLqcH07h+gOkFxHSFSzfz1B2PuKbi+QG6olA2PVJxBUL4wXSJnoTO4yd6cIKQr70yT950mcyb3MrXONaT4s0ne7B8D9sJuHCiF9NZbT++WLG3NOmyks1sz7fCQCbGO+/t3/aH4dqswUhXrOUd3Xbv8loJqoFMrLkPpyFIrsxVyFVtzoxkmC1a697HrWRcGneh1xermK5PT1LHM7YmPEzH53+/PMtnL00xX7bXXb9nIMWTF8Y52Z9gumDdsSn1TuiqTNrQSMXUps+K4OCxX+sRBIKDxpERHw2fD98PmC6a3DeYQl/uL7CkaBV7V0JjpmAhSyxbW8vcP5Tm5ECa64s1zo5l+dEzg3zz6hK6pnAiGaXwTcfD8X1mSxbD2ThhGE3QKLKE6Xg8cyMHREJCkbnjpMtK7raBbacfhmuzBoYqt7yja8dd3sozLpQtXpwqka86TBUaDbqr3VArlku+6pCJq+SrLhXLXfWeNATbUsVCUySyCZURfWt+Gvmaw588O8UXX5hdt6UW4LHj3Tx5YZzzE11IUuSvMivbGzalboZoHhUIBEeVIyM+GuiqgixJLFYshjJxBtIGuiIxmTeZr5hU7Kjkoqug6xpVO8DxQs6Nd3N2NMNS1cFyPUzPY7pQR9NkHhzu403Huokt9yoATev2V2bKzS2wfcsmVZbrkU2oG066rGS7DWztGvVbW7tOx7SWIqbdd3mN13v/cBqAwazBmeHMqvfJ9gImC3XcpQBNkXloLLPqMWaKJt95PUfd8bfkzwFwO1fn6UuT/MUr87j+6l4QRZb4kfsHeOLCGKfWLHm7k8NpK+K6Eu1XEc2jAoHgiHJkxMd4d5z+lB6l8odS3DuQouYE+GHIzaU6i1Ub0w0IQjAUiVRcRw6jlemlus2DI2kWK1bUfGp5GIrCaHecnqTB4yd7WhqAXV+sUrM9uuI6EFJzPFQ5Sq8njainZO3PBEHAlblKc6FdT0LbVmljq5mSO4mU/apdJ3WFiuUyV4oaUu8fSq87v6HKjHXHySY0SnUXY7kZ0/MDCnWXawtVao5HTFWYKtZJG0pLd9Jo226Jp56Z4jvXc+vOktAV3nP/AO8+3c94T7KlsNhqP4cqy9G0iljoJhAIBEdHfACEIUhIpAyNnoSOJPnoqsQrsyWmCiaaIpM0FGpuQDFXIwhBlsAnRFEkcjWXqXwdWZKQkXjsZDeOH2K6rfsIUoZK0lCZr0SZj3RMoTcR48xIhpmiSa2FN8SVuQpffmkO14/u6n/8ocFtiYCtZkruJFJ2q3a9VlzdP5RGXmNYJkmAtPy/LUjHNHpTBn4Q0psySBpqJBJNlzAMSegqpuPz6lwFgIRmMdKVaGY//CDkb64t8dQzk1xe/p6V9KZ0Pnp+lLed6mOqaFK1fV6dr2wpg7L6dUgkdYV0TCx0EwgEgpUcGfFxO1/n5rKgmCzUSRkKPSmD713PsVSz0VSJsuUyEU9wfDjOUs1hoezgeD4V0+W1uSoly6VseZSX77YVTaI/FSOpvzHVspL+tME77+3jWG80YZLQFWaK1qZZjMWKjesHnB7K8OpcmaWqw4OjXatszxsjqI0ST9X2sL0AQ5WxvQBZouVzrMx25Ko2nh8w2p1gpmhSsdzmY+2mM+NacQXwwMgbS9Uih1ON+wY3FmgrLeUb/TUrTb56khrD2Rg122OsO7F83SPlKvz5D+b57KUpppcN31Yy3hPn/3rzBD9y/wCaIjOZr2/J4XQt+vLivEbpSiAQCASrOTLio2i6TBXqmE6A7fmMdMd4aKyL3qROXzKyJ7+5VOOxEz2896EhvvTCDLOlJQIkcnWHnqRKTFWQ41HZJKZJDKWMllMtDSRJYjAbbzqKhmFIfzq2aRajP22gKTKvzpXRFJn+ZZ+IhmiwXJ+Zookf0HQI9f1IUI11x+lN6Yx0xZrBOAzD5oK5ldmOqh0F7oZIsb2AG3vgzNgQV/cNpnnudoEXp4pN2/hGpqBquzx724wyRy0yBpIkEdNkpgseZctdt/RNkiRGuhJUbB/bC7C8gC+9OMtXXpmnZLrrHu/0YJrHjnfzo2cGmOhNNr9+J4fTlSjyG82jhiqyHAKBQLAZR0Z8dMU1sjEdVfbpVQ0SWjRh8LZ7+pgt2SxWTVIxBV2RCMOQs2MZZssWsiQThAFvPdlDzQm4ulBFU2SO9cTJxHUs10dV7jy1AlsrZdw/FDVarixLrBQNixULTZF5YCTbdAgdTMdwlwKyCQ0/IDJEM6PyS8ks8/Dy864syUwXQ3qTenOSZKWPx3Sxzq1cbVeyIA1x9dztAoW6Q8X2eXGqtMbHAwiX/3cNdccjX3OYLVqbLn3rSWpkYipfeH6ab13L4XirS2OyBG852cu5sS6GszFkWSJprO7p2EozaXy5rJIUC90EAoFgyxwZ8XGsN8nZsSzXFipoqsxQNkbKUDnem+BHzrh86cUZ8jWXF6fL5E2Pd5/u5+339Dd3orzjnl4kSeJ2vg7AWFeMfN1lqerQnzbou0MvwFanUGRZXlWGgNV9HKW6ixsEqxxCy6aHpsiU6i69qSib0qrvo9HM+eytOkEIPQmtpXNqzfaoWh75mtv2LEhDXL04VaRi+7z5eDdzJbu5BO9WrsZcyaQ/HcPzA6q2xyBRaSVfc7DcKKOzdulbzXGhGn19umDy5R/M8a2rS6zVLzFV5n1nh/nYo6MMZWLkay41x8XxwuZjNLIoGzWTastic7PmUbFgTCAQCDbmyIiPvpTOvYMp/CAgG9d5+6neZtA1VBkFiUxcZzBtUHeiYP9D9/WvCx6NEspC2WK2VMUPQmaKVsv19StpZC+8IKBme0z0JDjWm2zarW8WpFaOvXYnNUa746vGequ2x0NjGYzlXoMwjDIerS3YoWJ75KtRuaJs+U3b8UZja65qk6s6u7KfoiGu+lIGL06VmCvZUclCV7g8W+avX1vghckSsgSj3QnuH04zX7aorfHcWFsSsdyAv3xlmm9eXeLWskBcSXdC48PnR/nQuRGy8TcyGL0pHaowVWhkUayWjaXbbR4VC8YEAoFgY46M+LgyV+Frry4up9BtHhzNMNwtsVC2uJ2vYwcB82UL0/E40ZdEVeRmU2cYhlxfrDabOtMxjYrl4gUBcU3hZq5GNr753W3FcslVbUJCLs+VqVouJdMlpincytVRZFBliWO9yebSNkmSmj0b2Xj0VzXRk2AgE2teW7nEriGmFsrWqu9vfL3RzHnPgMrzVpFsXGtu9x3IxJoloZShUjK9tu2naJUFWDvKG4Yhz94qMFUwsb2AuCZTrDncytXJtNg/3yiJlEyHi7cK/H++do2ZkrXu+8a64zxxYZwfe2AQXZUJw5Bc1VlVSlmbRVnZWGpokSdH2ojEzlYRC8YEAoFgY46M+Li2UGW6aDKSjTNdNLm2UOXB0S7KpkOuajPeFaNu+4xmdc6Od2E6Llfn/WZjZqHmcDtfZ6IvyYneBCNdcWq2x4tTJYBVEy+NiZRGiWaiJ5q4mCqYLFas5QV1XdzI1ZlcqlJzApK6RMH0ONZTozdl8OBIhuN9qWUvivLyHTTkas6yiFDXXIvuroHm12QJksYb35/UFRRZYqlq4/gB1xYrDGfj65o6d+rxsVGpYaMswMr+l+uLVXRFIRvXeH2xFi3t26DBMwxDbudNvvTSDH95eYFivXUT6c88PsHb7ulFXiEI8zV3Xa/I2ixKOqaSjUdW5zttHhULxgQCgWBjjswnYlyLnE1LpossScSX7axnSxbfv5GnVHex/YDBlM53Xs8RhCFvOdVHZXkdes3xmS/bpGIqGUPlRF+S8e44c0WLvrSB74dNm+/Fis23ri3x+mLk73GyL8lET5zx7gSj3XEuz5aZLNSZL9vkay5ly6ViunhBSNLQeH2pTs32KJnRuvfZksXxvhSzxTpzJau56VZTJGw3cgOdLVnrlr1dnimzUIm27CqyxNnRDA+PZbm5pFC3fWSpdVPn2sbYleO9m/UvbFRaarWUbm0WIKkr6KpESlcZyURTOxM9CUaWy1wN5soW/+07t/jLyws4/uomUgk4P9HFR86P8dZTPS3P2CrLMdYd5/RgmiAMGUgbHOtNrPMe2S5iwZhAIBBszJERHw+PZZkqmBRqDt1JnYfHss1yStl08cOQXNXihekitgdBGLBQcXhkogtNkalaNt1JjZrl4QUh6ZjWHOO8sVRDU2TOelHmoWpHo7ddcQ2QqNkekiTRk9Lx/ICzo1k0RSKuaqR0k8uzHn0pHdP1CYKAIAzpS8co1FxydYuK5TNfsUnHVHoTBnFd5cXpEgldxnYj9dCT0tcte3N8H02Rm0G/5vic7E9RtT1Guz2GszGuzFa4MldBkqQ7ioo79S80Sg1xTeHFqRJVKxJQGy2la2RK5ssWrhcw2hWnK6lxfqKbuuM1zxKGIdcWqjx1cYq/fnWBYI1g0hSJ9z44xMcfHWP8Dlt7oywHXFuo4Ich4z1xepI6x3qjUlu72I8FY53S5Nop5xAIBJ3LkREfA5kYbznV2xxhbWQoFio2ZdulUHMxHZ+aVaU7ZXBvfxJVkZjojnPvYJpnbxWw3ADHD+hP6YRhiK5ILW2+k7pCEIbczNVQFYnjPQmCICSmyXgyTPRm6I6rfPNqjtv5AEOTGO+OEyAR0xRShoYEOL5Pd0LngeE4N3M1hrMxJCRuLFao2R7j3Smqls9ARueBkey6ZW/jPZHoWBv0U4aKLMH3r+e5kavQX4oxma9zfqKLvpTRLNM0gsZW+xcapYabuSjjc7wvheX6Gy6lu5Wr8d3reRwvaJZAJnqS5KoOC1UH3w949naRZ28XeWm6tO754prC4yd7+L8fP8bJ/uS6663oSWoMZWJUTY/umI7nh7h+2FbhsV90SpNrp5xDIBB0LkdGfCxVHWaK1qrplKrtMd6d4ERvkppVIpHUKdYdKqbDrYLMPf0p+tKx5cVmMW4uVfnBTJnZkknZ8jgznF5l852OvTFFkdI1RrKRwFmq2rwwVWSuZEfBr+pw/1CKmuthez5hCMW6zfnjfTx+rAs3lJpupTNFE8sNGO1KcHY002w0vV0wuZkz0ZeNyABuLNWawb3Re9J4nSuDfn/aYLQ7zg9mSlh+VNbJVx3KpstgNkbK0FBkGOmKpmqiDb2tXVNX0ig1ZOMqSb2O6XioirxuKZ3p+ORqNi9MFpku1hnrSmB5frPRs2w5PHurwPdu5Fs6kfandN5zZpB3n+6jJxnb0jI3iMSK5fqoskRP0lhVrurkZtCtZhI6pcm1U84hEAg6lyMjPkp1m5cmiwRhgCzJHOuJkU0Y9KYM7h3KsFSxcfyQ7qRGQlfx/YDxngSm67FUdRjIxLiVq7FYdcjGNK4vlVBluG/ojRHXlVMlmXj05y88N81i1SZZcVioWGhKlrpbQ1MkZCnaMzNVNKm7PldmyxzvTTLSFTWBJvWwOWK6csrl1ECKQt1tZlxqtsdsaf2d5kap/8ghVGEkm4gaT+dr3DOQJAijyZf7BjO8MlNirmTRn44tj73Gl7MyG/cvNJ6vP21wrDe5TvTYXuTVYTo+uWUxmKs65KoOEz0JJCQ+c3GSpy9Okas56x7/VH+SC8d7uH8wjabK9CRjd9y1oinLC92W97+8vlgjV7WZKkSiZmW5qlPZaiahU5pcO+UcAoGgczkynwqX5yr89WsLWK5PTFM4NZjkg+cynBvv4nhvnIG0wQu3CpieT0pXMHSVt57qw3KDpgFWoe5QrDtYjs9CxWKqqJOK6ZwdjVa6NzIPjamSl6eLlC2XuCZzbaGKH4a4no/phhTrDiXL5dp8iWLN5Z7BNAtlm7++Ms+9AxnydRtDVRjpiqMqctP0CtYvVpMkacM7zY3umlOGSndSo2TqDKRdehIG2UQ09TFTNHH9AMsJCMOQQt3jZH+Sk2vWyW/EWtHj+gGFmt1siAWWR10Vzo11cWWuzDM38/zHr12jZq/f5fLmEz38xENDxFSZparD0HJGaaNdK7K00upcZrFic7tWj/bZBAFnRqK/r8GssZzVekNMdWK/wlYzCZ3S5Nop5xAIBJ3LkREfsyUTPwgZ7k6wVLaYLZnNJsvFikXZdOlJ6yiSxPHeJLIsYbo+qhy5WS5WbEp1D11RmCuZqAroisyVuRJ+4C/bsNOcKjk7mmGmUCMTU4hpKprqkDUkcnUbCYmZYp3Zko2hq8iWz0LZQlUkbuXruEEkdFK6yqmBNJbrrwo4az/cgyDgVq7Os7ci19OVo7Mb3TX3pw3OjXdxsj/Z9C9p3KHWHJ+kofDd6zmWJm00Reahscy233PPDyiaLhUrmtpZSUJXWao5fPPqIi9OldY1kaqyxI+eGeTjF8bIxDRena+Qq7rMlSMvj66kvmrXShiGzX02A8uOs5IU+bg0Xn9jF85s0aI3FQmPtRmETuxX2GomYT+aXDv5HAKBoHM5MuKjO64jyxKFio0sS3QvG1ctVmy+8VoUAFMxlYSuEBKl62UJHhyOvDauzEXeEO863c93ry9xK1fj29dy+GHIUsVmNJtgvDdBvhqN5qZjGglDI5PQmSnW6UlqvPVUH7eWquTrDjUnYLJg8cBQKrpT1xVODab59rUlrs5X6UsZeEHIzaUqo92JpshotY5+vmRuuIZ+o7vmZoBY7g1Zebd/oi9JGIaMdyfWNdNuBd8PeH2xynzZJqYpq5a+hWHIC1Mlnnpmku/dyK/72bgm82MPDvF/vXmc/nQU9BvbZU8ORE2lfWmdk/0pepJa0+rcdDxuLNXxg5D5st16n00hjOzSl/fZtLoj78R+BZFJEAgEh40jIz7ODKc50Z9sjtqeGY52jFRtjyCAdEwlCGChbFOpLWIHEqbjcu1ED31Jg4VK5MkBMNoVp+Z4LJYja/C5YtSAmqs7zSyBtBwoHp3oplx3kYC5okkQgu0F1CyTuuNStl3ScY2TfUkSetSHEdejLMpIV4wHRjJNx9PLs+WW6+g3W0O/lbvmVnf7a0s7K5tpNyIMQ8qmx+uLFV6ZXW3k1ZXQ+MZrizx1cZLX5qvrfja7/B68+Xg3471JZOkNsdMwAVus2GSTGqcGUkz0Jkgbb1idF+pOS9Gw8vWritw0gmucd61/SSf2K4hMgkAgOGzs/yfrHhHXFE70JhnvTqDKkclYsLygrVCzsJyAhCHTk9SYLVoU6g6FmsdscYaTgymGUjGuLlSQCPnh0/2ULZeFikPc0PC9yJ78kfGuZpYgWg3v8cpMEQh5ZKKbiuVRd3xMJ6BQcwiCkJLposkyY91xBjMx4ppKyXSp2S5nx7p49Fh30/Bq5Tr651eso2/0mLQKmFu5a251t3+iL7mtu+2K5VKsu7h+QMV6w8hrqlDnT5+f5qtXFphtYX/enzJ4/EQ3E70JinWXk/0pbC9Y1c/RsFL3goDBdIxjvQmUNaOxG4mGzV5/K9ElsgwCgUCw++yK+JienuZXf/VX+fKXv0y9Xueee+7hD//wD7lw4cJuPN2WmC3bvDxTwnR84rrCo8e6cQL43vUclheiqxKn+hMslBxuLJapOyFj3QZLNY+XJvNc0zQsL/L5GOmKMdGToGi6qJJMfzpFJq4jITWzBGEYUrHdZQdTn9cXI5+OvpRONq4zV6ozX7HoThp4Ychkvs5jx3tIGirfuLqIqsrMlUwWyhayHO2ZkSWwXZ+/fm2euu0z1hPnxalS07m0ETD7Uvq6O/rN+hYi34+Q71xbZLFqUzYdEprMYDZ+x36Hmu1RqDurVtYndJW66/M/n7nF928UMN31TaTHehJ86JERehMa3clIZMwUrWisV5Gb/RyRkNAY70mib1L62Ug0bJY1aFliWWP7LhAIBIL203bxUSgUePvb38673/1uvvzlL9Pf38/Vq1fp7u5u91Nti1zVxvNDBtMx8vVon0sQguuHPHq8l2dv5bk8U+FW3sQNJOqOy818iO36aIpE2fQZ70nQmzSYLproqsJAKkbN8bh3MM29AynqbtAMfNcXq9Rsj8G0QW/SIK7LnOpPUbY8Xl+sEkoSYQhT+Tq9SZ0buTq383UkSaJiechI/M21HPNli8FMjFRMo2I6dCc1XD/A0BTuHUjh+GHTubQRMFc2WW6labI/HbmmXpmvUKg5TBVNqo7H+8+OLDfk2quEzWLF5upClYrl0pc06Flu7oSoP+Ppi5P8n1fmcf3VXaSyBA+NZjk3miUEYoqCqiqMdCXoSWqMdCWiKRhNIabJmI5PX0qnJ6lvOHHSql8F2NLESieWWAQCgeAo0PZP29/93d9lfHycP/zDP2x+7cSJE+1+mm0T16PdLlXbR5YkYpqMIkfW58/eyhMSYrk+ITDeHcf2AkzbRVVluhM6JdOlbgdUbI+B0MBcduW03GjS5PRQhpP9kbV3Yx/LjcUauZpDXFd47FgP58a7AMjE1EiABCGvzJZJx2CpYnNlrhK5b1oeuZrNTDHKQoz1JviR04PMlwOycZ1z4z1870aO2wWT0a7EuqBZtT08PyCuq9xcqjY37sL6oNz42lShjucHHOtLUjE98lWnORq7Usj0p3Qu3S5wbSHq2xjvTnDheA+zJZOnLk7y7Ws51q6LiWsK7394iLed7KVq+/SnDV5fqK5qHJUkiaFsjHRMpe54vDJTwQ+i97HRPNqKVqWTxpk9PxqTPtabWLUpuIEosQgEAsH+0Hbx8cUvfpH3vve9fPzjH+frX/86o6Oj/OIv/iJ/7+/9vZbfb9s2tm03/1wul9t9JABGsjHShkqhZtOdNIhrCnXb41hPkorl0J+N8bzjkZ+r4vk+EiGj3Uls36fu+KRiCkNZg9GuGN0JDc/zKVs+fSkDywm4PBuduz9tRJmHyQJly8VQJXqTGg+OpJvGX/c4PiES3QmdpaqDJkPJCpgpmMS1yLBsqWIz2h1jIGlgeT43c7XlTbZgOh4n+5Jk4irZeLTdNgzDZmBNGSpV2+PFZUvyVD7auAtsGKgrdvQ6K0s14rrSNN9qlCb60wbX5qvkqjaFutMsLb00XeJzz003xchKepI6Hzk/ygfPDZOOaeSqDq/OV1ioRGPFMS2aKErHNDJxjdjysr98rXXzaCtalU4gWq4X7cApMlc2eW2+yvmJLs4MZ5rvk2jkFAgEgv2h7eLj+vXr/P7v/z6f+MQn+Bf/4l/wzDPP8Eu/9Evous7P/dzPrfv+T33qU/zWb/1Wu4+xDtMN6ErqDHXFsVyfQt0lpqvcO5TixakCs4U6EiE9SY3+VBLTDQj8gLIjo8kyfSmdEAlVlsnXXR4czpCJh1hOQNF0mC2aLFZsJnri3Fyq8+JUkYWyje1FHiCpmNa0Rrdcn6VqZJI1lDVYqtrEDZmYruAFIcd6E9iuR9ny0VWJsZ4Uw9kYXXGNpKES0xRsL2C6YJKvuZTMcjM70BAimiKR0GUeGsliecG6jbdrA/Wbj3dHXhxhyPG+JA+PRs2XXhBQsVyuzlUoWQ5dCR3X9XlussjluSoVy1v3Xh/rSfDEhTHec2YQTZHI11wm83USmsLpgRQzJRPHC/D9kNcXqwRhyLHeJIYqNw3QVpZDkrqy4VbdjUoniixxc6lKzfHQFZnJfL1pN7/fvh0CgUBw1Gm7+AiCgAsXLvA7v/M7AJw/f56XX36Z//yf/3NL8fHJT36ST3ziE80/l8tlxsfH232sN1g2u4ovT6O8OFVipmCyWLWIaRKuHxIEcOFYN0PZGK/NVZgumthewNWFGgEhCV3hZF9k9X11voLlecR0hVfnKswW6zw/WeTGYhUngIyhEgQhU/k6cV3Fcn2mC3VUWcLxAsa646iShOWHvL5YwfUDjvcmuWcwzWS+zmAmxom+ZNPHIl+zOdWfoiuh4Ycho10Jpot1buVqVG0Py/Wb+2BsN2Sh7LTceLs2UM+VbE72pZr9IX4Qkqs5WG5AJh6ZfE0XLb57I8+1hVrLJtKHR7M8+dg4j5/sQQLyNZfpQi3KiixnOH74vn7uT2QwtBq263PxRp6FikXJdHl4rKtpgLayHBKG4YY9LBuVTho7ZuquR7HmMZAx0BVlS+6vAoFAINhd2i4+hoeHeeCBB1Z97cyZM3zuc59r+f2GYWAYu19rTy7fIZcsl4SuMtoVp+74zBZNhjIG04U6rhfi+j75uoPtBzw4kiVfc7m+VGe+ZON6PrmKRU1XmS7USRoaFdul7vh8/Upk3Z6IqUwWTPwwpGJ6KDIsVm3++rUFCqZLfrnRdbwnScWK9rK8PFOkbPmoShT4RrtiJA2NfM0lrincWKyyVHcpVm1+MFvmG68uMNaT4P6hNIRQczyqlke+5rK4XNIYysRYrFoYWuS4unbj7dpA3fhab1KnUHMomS7BslArmy7fu57jldkK3horUgl4YDjDO+/r4z33DzZ3rTRKLDcWK1xfqnN6IE2xHrmdHutN8rJd4uLNAvm6y0A6Rm65x2SVAdryc1xfrG5YhmlVOmm4qfYkdc6OdnFzqYquRHbyK/tjOtHNVCAQCI4CbRcfb3/723n11VdXfe21117j2LFj7X6qbWGoMsNdcTRZwvUD6rZH0fTI110cz0eWJXI1B9OJTMe+9/oSYRC5fE70JMhVbdJxDdv1kaUAP4S5kkk6pnJ6OMP1pVpUGljylqdcYhTrkSiwHQ8vCPB8sNyApKHw+kKVuutSdwIm8yZuAClD4dXZCp4fEtMUana02n6hZDFXsZgrmyxUXHRFomi6JA2Nh8e76UUnV3UY6YpTrDvcztd4caqEpsgMZeLNu/mN7vIHMjH6lw3CpoqRDT3Aq3MVnnpmkq9fXWwkjJroqsw77unl4dEu7h/KsFCxmt4cQRhybaHC7aUqcU1Fk6BqR2KmUHN49Fg3Ez0JZosmg5kYpuPjBeGG0ybbnUpZKSokQo73pZp9K30rFtF1opvpQUVkkQQCwXZou/j4p//0n/K2t72N3/md3+GJJ57g+9//Pn/wB3/AH/zBH7T7qbaF7QXMFk3qjocEKJKEIoEfBDw4kiGpK1ydrzJVNCnUbeqOzPdvFjFUlfHuJLmqRcX2sN0ASZZZKJkYqkzd9bg8W6ZmOSQMFU2RqNoe8xUTQ5UYysZxPB93ub8haSjcP9zLS9NFbi7VmCs52J5PQFT+8EOXhbJFfyZOX1pnMl9HkcFQFTJxnbmSjaKqIEnYrkd3QsP2Ai4uVbk2H5Vt8jWbuh1wrC+B6/ncytWW/6uTNBTqjs9EzxsTIFXbo1Bz8YJokdz3buR5+uIkz0+W1r2PcU3hPff3896HBjFUlfmyxULFQpYlMjGNbFxjKl/n6nyV6ZKJ60Ur7Ku2TzahUbZclqoOx3qTFOsuhZqL4/ucn+jacNqkL6Uz0hVr2sr33WGT7UpRcXmmzGLVoS9lMFO0VvV8iFHb9iGySAKBYDu0/dP2scce4/Of/zyf/OQn+df/+l9z4sQJPv3pT/MzP/Mz7X6qbVFbDkjZmMZ82aZmuzw83sN81cHxQ4aycQo1lxtLFSwnZKBXR5UjcXLPYJKBjMa3ri7x6lwFRQXPD3H9gKLpcStXp267hBJ0xzV0RcHzo7HdpWq0SyauhRgJDc8PeWmqwGLFpeYEmJ6PCvghOJ7HcDbFyb4UXhBiOR4xTeGewSRThTq6EqOUcajYPoocLXKZK9vMlSzmyjalukvZclFlCV2VmCqYkaOq61NzPGaKNmeG08wUTWaLJi9OFTnZn2KiJ4EXhPzl5Xn+n+9PMrm8bn4lKUPhVF+SR8a7GO6KU6h5yLLHUDZGNq4xkDKI6wol0+VmroaqSLz1ZB9XZov0pwx0TaY/HcMPIjfUk/0pzo13belOeanqMFO08INwnYBoxUpR4fg+miK3zG6IUdv2IbJIAoFgO+zKrd4HPvABPvCBD+zGQ7cBCV2VURS5ObJ6rDdBEATkqzY9yRi5epWlqkMYQt32eO5WkWsLJRYqDgEhMVkiHlPRVYWYEpCNafQmdVzPR5KjPSUJ3WC6aAEhCU1FkaM+CAgpWRK5qo3lhVE5Q4a0pjCYNehK6tRcD5DoSxm4fsBi2cZQVSa6de4fSUd9Ktk4hCGvTBeZK1tkDB3PC7iZr5HUFfK1gO6EymAmxq28ia5I5OsO37+RR1dlbNdnpmhx6VYeJJnvXc+Tqznr3q2TfUl6kxqaLNOV1HC8gJrjcc9AmrLpcqwnQVdCb2ZWUoZKrurgBSFzJZP+dJz7hlK8Nlfl5lIdTZE5O5bd1pjrdgPbSlEx3hP9TKvshhi1bR8iiyQQCLbDkfmEiKkSs8U6+Wo0/XFmsJ9UXKc3pTPRk2CxYi1nClxSukJaV+hKqORrFi/MlJhdnngZ705guT6LZRtVkZkumJTqLklD4dxENwMpnb94ZZ7rizUsL6AvpRPXleXyRtTbEPgeM26A6QTIEshIDHXFeOLCONcXa1EvSdxgrCtOoR5lMi6c6MV0PHpTOif60uSqNq/OVVis2ixWoqyAtGy/3p3QsN2A4a4Ej5/o4U+fm8b2fEa74jhegO1Edu9X5qssLTfAruX0YJqPPTrKPQNJvne9QMl0qDs+3QmVTExluhgJDdcPeHGqxHSxznzZ5vETvQxnY4x2x7DcgLimkImp1LpidCX1DTfkbtYzsN3AtlJUNMZrRXZjdxFZJIFAsB2OjPi4MldlrmwhSRJzZYvXF+sc71fw/MihtGa7UU+HFzVe1l0fFJlCzaVs+yQNjbJpslS1uXcgTUJXSWoKKV0httz7UTEdzo+mOTeeRZZgvmItB0oJSWK5idTDCcJoWZwUoMmRv0dSV3hhsojth6TiBqoSlU2Gu2P0p2JYro+qyIx3x0kaLktVC9f30ZSoD2OyUCcmS2iqzHAmsnRPxRReni7h+gFBEJKvOXTFdV7JVXjudnGdE6kqS5wb7+JtJ3sZzMY4PRht/j3Zn8R0YiiyxLHeBHNli/mSFZWy/ADT9elK6MyVLG4uVRntTjDSFWuWSqZLFqoir9p9s5aFssU3ry5Rsz2Shso77+1jMBsH7i6wbTQNI5oj24vIIgkEgu1wZMRHwXQIfOjL6CyVbRaqFuO9qWUXzBJ10+ZWrkax7uCFAXgSvh8QShKe51MLojX2hiKTjMkYikQ6plF3q5RNj6SuMlO0+N7NIklDpSdlkKvZVG2XdEzjvoEUZ4bTTBdtnrmZo2756LKE7QfoCvSlDXJVB0WRONWXoGj69CQ1HhrJNqdbXC/gz16YYa5skTA0anY0rVO1oqmaVCaGD/hhyFhXfNmIrI4qQzoe41vXllr2c+iKxHvODPILbz8OSNQcFz8AWQpRZYnjvQkkSWKiJ0HV9pgpWsR0lVtLkYeHqkioUpS9OTOc5nhfiorlNksl08WQ3mS0o8X2ItMyYFXQv52vc32pRldcZ75S41hvoik+dhLYGgKjYrnYXoChRs6xjV01ojlSIBAI9o8jIz6Gs3EURWK2aKIoEjFFpWq7zBZrUTbCDymYDpYXRmUIWSIIJdKGjKpoLFUckrqM7ftMLtUJJZnbOZOi6WC6Hgk9gefLvDRVZLQ7juf5KEh4AZRNj2uLVe4byhDXVSZ6U+QqDpoqo8oSkixRMV2ySYNS3WWh6hDXVFRF4up8hbiucHW+xmLF5vXFCn4YMpKNM5Q1GMoavFKxCMOQIAwIwhDX8ZkrmxTrkTh5db5CyVzvRNoV13jseDcffHiYB0azKLJMQlewHJ+rC1WuLdSYKpiMdyeay+PSMQ0vCFmq2Mt7WHzimort+qiSzLHeZDOQN0olqiw37d1vtFhhv1ixmSma1ByXbHx9VmQnNARGrmqveg2NDIofhAxnY1yZrayyxhcZEIFAINh9joz4ODOU4rETPRSqNiXbIxtXCENI6BoyJpdnSzhuSDauUazZkZdH4JOMxRiMKSzWPGquj+kEuH6IH4YkVIWYriLLMktVB9sLCAEvBN8Po+93o4bU2bLF119doCcZIwxCVEUicAK6EzFkKZpcMZabYC9Pl6gt93fIwGA2wWLVRldlZEnCD2G2ZJLUFU71p5mM1zG9gLmSRdzQuFW0mC/bLC6faS2DGYM3H+/hTRPdaKrMeG+SvpRBylBRFZnri9Vo7JaQxarFaHcML4gs2k/0JTk/0RXZxDsBJdMhCELuHUyTNjRqTuR82qpUcmOp9kY2pFBfNQLs+QEKMq7vc6o/yURP4o5/p5uVTxoCI5vQuLFUIxNX8YOw+b2KLHFltsJkoU5IiOuHIgMiEAgEe8SRER+OD0EAphfgB9CbjhPXVQxVYqakEYQSfhAFU0NTuW8wSUxXcL2Qct1DCkLCQMIPQ6xlUyzXC1BVhfHuBJIUUqi5GJpK3fYICIjpKhXbxvJ8Yq5MxfIwfZNrc2VMN8APQ1JuiKFKGKrEbMlkumRStz38QKLmRN8zU3KWd54oOH50/v60jipLvL5Qxnaj7ENd8XD9kBemyuucSAGO9yb48COjeEG0b+ZkX5KpgknZdHG8ACX+RoNnzY78S0qmx+XZCg+PZUkZKpIkcWY4Q1/KoGK53F9Kt3QQbVUqWdk4WrW9yJnV9pgv27z5eA+yJDOYNTgznNlSX8dm5ZPGc+WqDpoiUza9ps18Qxhdni0TEnJmJMNs0Vo3RbORuBE9IwKBQHB3HBnx0eiLUBVwTJ/buRoPjHY1g5Uqw0A6hul6GJrCQCbGWHeCxYpDTIXXF6vU3Mj91PYDejMxug0VLwzRVehKxKhaPhXLJQwjvw9JAl2OzMx0VSZfd6haHqbrM5iKkatH9uphqKArEoYqYzkhXgCaIlO1XTRJYrg3hapI9KdUMgmDmYKJF4Rcni1h++D6AWXLo2r765pIJeDBkQz39CfpzxgMZAyCIMDxAl6aKZKvOtRdj6mC2dz62p82ov4O0+VNEz0UajYTPQn6UvqqBW8n+1Oc7E9x32B6S82gK7MhuapNrhaZf82XbW7lqqRiGgld2frf6SYjuI3nqlguZ8eyq3o+GsIIwPVDZotWyymajcSN6BkRCASCu+PIiI9i3Wa6VIcQHD8gG1d5eCxLX0onV7W5dDNPyfIZ605E22+zMXrTMTRFJl+zUBSJlK6gyDKaDCe64wykYxRMj3RMiTbUZmIYmoTthZiOR75qY6gKigxhKJPUFYIgxPZ8luo2VcslHdM5N5amWPcp1m16kjpFcznToUeZBNsP8AKJgWyC0a44QRhyY6HKbNnGdAPc9ZUVZAmO9cQ53pukO6GTiasktGiqpicZZ7ZoUqi5VO1IlOWqzqqtr8d6k5TMKLiP9SQ51ptkqeq0DLorx1o3ywiszIakDJWSGQmxU/1J0oZKefkcJdPbUkDfbAS3+VybPMadpmg2EjfCUEsgEAjujiMjPkqmR9ly8fwQP4hq/I3geO9A1A/y0lQJRZW5MJ7l9HAWPwhRlTQvT+bpS8WwHJ+a4zLWneAtp/oAmC/bxFSZ1xaq9CZVKrZHvmLjBlFJBknCcn1kOSRf96nbHoaqUrEd0jEDVQZJkulORqO3QSiRjav0pXQuHO+CUMIJYKFkkavYXLyR4/pSnaoTtCytyBKkdIWkIfOW4z1oioSqyjw81sVrC1WKpkvVjizP7xlMcyNX5cZSlRN9KYp1h1u5Gv1pY8OeDc8PiOsqN5eqZOOrBcZ2MgJrH79iuVxbqG0roG9lBHczQXSnKZqNxI0w1BIIBIK748h8auqqTNrQCAipmj5zJZPFis1AJkbdDbhvMMO58R5uLlWI62q0SbbqULMdTCdACn0UBRKawkA6FmUo6i7S8kRL3fGoWB6W6+F5IYau4AVQsxxScY1MTGOxYmGoMgohrhcy0qVHS+RUmftHstiuzzevLtKXjvHosW4eGM4wW7LIVR1uLFb4+tUchbrb0hTMUCWSeuRb4ocBVSvg2ckCJ/pSZOMal2fLlC2f00MZTCcqe1QtF0kiesylKmNdcW7n682JlVY9G1Xb48XpIjXHo+5GnhxnhjNIkkTFcslVbbIJjVzVoWK5G4qPVoF/uwF9KyO4d1Mi2UjcCEMtgUAguDuOjPi4ZyBFNqExuVQjk9SJa2ozODamPCzXJ6Gr/GCmxGtzFZbqDrbjI0mQiatMdMcpWS5Vy+XybIWuuErSkCnWXRK6Sq5qkU3oVEwH23PRpKjPYLQrRtX26YprdCV1nrtZIGe6FCaLpAyVt5zsRpUlXpiroioKg+kYhbrL1fkKC2Wbr7wyx4vTZVx/vepQZehJqPQldUw3YKnqEPghqiJTNB38wKfmyCgK6KrKq3NlTvYluWcgxWvzFYYycWp2Fcv1uH8oHTXJWi5hGHI7XwdgoidBf9ogDEM0RSIMoSumU6x5PHur0CzV2F7AVMHkxlKtaaMOWzP12m5A38zHY+Vj302JZCNxIwy19g7R3CsQHE6OjPjoTer0JjWuL4SUaw5zZQvLjcZCV25NvbVk8/J0icl8jYrlk4lHO1ykUGK+bJOrWQQhXJkvM5KN8fZ7+hnOxkgbKhctD9fxcfyQuKLghxKu73M7b2K6Hv3JGEsVC8v10GTw/Kj/ZDJXI2lohASMdcVZqlhcum1yK1dnumi2zHRIy/8RgucFpOM6tm8jKxBTFBRJwg8kbB8sz2O0O8ZbT/Xz8lQRVZaI6wpF0+XaQpWkrmK6HtMFi9PDGWwv4PnJJV5frAHRfpczw2muzFWYLZkslC0SusrxviS6ojQDuqHKjHcnyMRVyqaHocqEYcjl2TLP3Y6etzel8/BY17rsw9qAHobhqubWtUFnMx+PlY8tSiQHG9HcKxAcTo7MJ/Fkvs5Uvo7l+dhIzFdNqst3+Jdny/z1qwtULI+XJgtM5ev4AbhBiOeHQOTKaXk+S7XInTMMwPVBvZEnrsk4fkiuahHXZUzHx5GgavnEdBnbDfCCAAUIkZAVCSkEQ5NJqjLzFQdlvoLlhcwWba7MVZiv2C1fhy5HoiMMIZSiHg/LC3D9gPsGkqSKCkEIZdMlZqiMd8dx/Wib71zJjBxRHQ/rdoHFqsNSxSLbn2K0O85YT7w5IVK1PbriGiBRsz2uLVR5fbFGNqYiK1Lk8Gpoq8Zr0zGNnpSOH4T0pHTSMY3Fis2ztwpMFUz6lrMZW8k+3CnobObjsfKxRYnkYCOaewWCw8mRER83cya3ChZlK3L6zFVVinWXH0wX+f9+6zovTZWRgULdpur4GArIIXh+QDqmoWtg+9JysAcFsF2P1+YqQEhvysDxAzQ/8pZwvBBJhoodeYyEgB3YGLJETFMiE68wJBlTycYUSnWPF2fKLTfLSoCuRLtXErqK5fo4fkAQRAJEV2R6Ehq9KQPTCyiZHif6kxiagqHK3DOQ5MKxLp69XWS2UEdVJGZKNroaiaHpQp1TgwM8fqIHgHwtMg4r1KOpm5N9SeJaNAIrSTL9KYNHxrq4ZzC9rhfi7Ghm2abe5eZSFQBNlptOpvHliZtGViO5PFpbc/xVGY47BZ3NfDxWvXd7XCIRZYL2IjJXAsHh5Mj8JuuqRNZQ8Xwf1w+JKVAyHb7+2iIXbxUo1aNg5wcBQQhVP3IqDb2AuOsThBKOF9Iw0vCBihMi46PKoJsObkDTgExXJLwwZGWbhuMFGLrCyb4krh9QsX08P+Q7N0tUbb/1ueVoekVRJRw3pGq7JA0VRQbLCQhDSMZUZoomCxWHpKGiSjJnR7spmjZLVZuy5aEuW8vPVRwUKSRXiRpDHx7roma7dMV1bufrTBZM4lpULhlK68R1jfEuI1p4Zyg4XuRyOtodX3dWSZKQJInbeZPrS1HJpj+tk9I10oaGocqcn+gCaGY1qnbki5KOaasyHHcKOpv5eGyV3RAKokzQXkTmSiA4nBwZ8XGqP0lXQmWubKKpCt1JnVt5k9mSiR9Ekxau7xMEIEmR8AAIfKjaPr2KhhZNzq4iANwASnWfAJpiI1hWKbICuiYREgW7VEzB9kJuFUzyNa/1uCwQ0yQ8PyQTV9BkhbgmgaFQrNsogKrJWE6AIoEmSeSqHql4iCrLhIS8vlhlvmxRMl1C4Op8hVP9SUa6YziOj67IJAwZxwtQJYmi6fCD6RI38yYjXQavz9eigC9blCyXkunSFdOJxWSGs3FmilHviyJLnB19Y9rl9YUqt5aqqJJM0lCQgeN9CXqX7dvX2qw/e9uEEE4PZVZlOO4UdLbi43EndkMoiDJBexHNvQLB4eTIiA9JkkgYGn0pg2xcZziTIK7JDGUMfjBTxPHeyDzYK0y7PMD1fVKxGFU7JAyjksvKPEUIWOEbTaDB8vWYstwUSkha14jpErKscGmy1LKJNGMoxDWZiuUiEU2s9CcN+jMxbNcjCCUGMjrTxTrFmotPJIbyposiS8iShh8EpGIadcdjqlCn6vgMpA0qtkfJ9KISUkzjLSd7GMjEeH2xhu0FWI5PV9xgoWJzO1+lbPoc70lQdX16EhqeHzLWE0eSJPwgWr7XCLC383VKpke+6nBlvrzcMxI978Nj2VXL5uCNVPp0oU4QhJiOz+WZ8h3t2bfKVjMauyEURJlAIBAI7syR+WRcWt4Ue/9wlsWKjaHJDGZizJVNDEXBU6OSSbgsIlZqg4QelSHmK3a0I2aD5wjX/JwiR5kTWQI7CFkouOvszyEqqwxldABqtosEZAyNkCgDMJiOkTMdpnJ1ZAk8PyQkpCuuYtoeCnCsN0kYhnQndMa64wRByK1cnbrrU3c8srGoJ2SiN4EiS7zlVB8xTUFXVWKazPdu5FksmwykdeJanFfnKsQNhboXULP9VX0V/WmDmaLVDLAAfhCSiatoisyjE90s1WzGuxO85WTvuqxFI6txK1ejYrl4QchMqc5Idw99KX3Lf6cbiYxGRsMLAmq2x0RPgmO9yXUiZDeEwsrJqf60sa3XIxAIBEeFIyM+FFmiZDqU6i6qIvPweJZTA2mevVXA9EIqlo8URhmLhkBQpeVsxnJAszxaioeNMN1loeJAlEN5AwnQFIgvi6D+lIYmK5RtF1m2SMdUdFUlbqjEdIUhxaBU93EcB9MN8Hyo45PQZU72Zbh/NMOV2TLZuEbZ8pgtWrhBQGy5wfP0cJrzE108ONLFldkKS1WH/rSBIoPp+pzsSxAEITdzNcqWS0yNfu5Eb5KzoxlScb3ZV9GX0ulLGc2gH4YhJbNMvuqiKzKSJHH/UHbDMkYjq1G1Pa4v1pAkCcsNuLlU477BNAOZGEEQcGWu0gzi9w+lkWV51eNsVDapWC75qkNAwOXZChXTbWnZvhv9BEtVh5mihR+EzBStpgeKQCAQCN7gyIgPXZHoTur0pgyCMGQwEyOuqyhKtFzMbeWlsSw+6o6Pu03h0Si/rEWRQFMkDFkiHVfxAuhNGvSkdCzXJ4WGkwio2z66EuL5QRQ8LZe67UaTLq6Pqig4no9mqMR0iclcDVmS6E5o3MxFUyZjXQaKotKfUHnbPf10p/RVa+Rt18P2Q2p25FRaM10Wqw5zJYuupEpXQuet9/Q1HUyhdbYB4OHlno+HxjJbbv5MGSpeELK0LDBWeoZcmavw5ZfmcP0ATYlExwMj2VU/v1HZxPYCJgt1FqsWJdPjTRPdLcdwd6OfQPR8CAQCwZ05MuJDlmX60zG64hpF00WW5cgu3PLx/NaTJkEA3QmNmuPS+js2Zq1QUSRQlWgsViLKxPQkNEAmFVNIaTIKkTApWxKW6+EDsiQRLPuNpAwFy3GRJAlVliIvEdvjVt6kJ6GiKApLNZeqHVBY7gPxfI/uZAZZlhjJxqlZleaY78szFfI1i5ShU1sWEz0JAz+IplQShkpMU1YJj40Mw7bT/LnSnfRYb4IgCDBUdVXPx2LFxvUDTg9leHWuzGIL35ONyiaGKjPWHWe0O8bl2TLFms1oT7ItZZU79ZMclJ4PMRIsEAj2k878ZNwFJnoSnOpPUrU9TqWSzRXxg5kYcU3Bdr01hZGoBON4HrIkIxMQsL3sB0STK3EtGiWtmi6qLCFLEpoqcc9AGtsPWKxELp1126U7YbBUNglQUKWQqUKd2SIkDJ3RnhiD2Th+uLyPJYCaH6KYDgOZGLoqEYQho90xVEVmIG0wW7Ii87GYRs32uJ2v8/J0icuzZTQZMnGDR49lePZWHtf3cYOoDGN5PkldwXJ9ri9Wm+WVnRiGrWV1uQQePd5DTFPWeYZoisyrc2U0RaYvpa9zPN2obJJe7m/xgoCHx7pW9XzcLXeakDkoo6FiJFggEOwnR0Z89KcNzgxnWKzY9KV0wjDk4s08FdNBksJ1wqOB6URtpAGRkNiqANFkSOoKtucThETZFilqzIzHFDRZYqpoYnkB+apF3Qmomj6SJFH3QhzPxfVlTCdAV8G1bIJ8wOnBFA+ODPLXry4wV3ZY9kylZrk8cKKXwWyMuuNTs8tYbkA2rhFXNVQ52kEzW7JIGyp1x0eRJEJCrsyW6UpovPlED4YafV9XQiNpqEwXzOZIbTauoivKKsOwrd7Zr7zTXqpYLFUtuhI6uarLib4kJ/tTq77//qE0QLPnoyehtQyWrcomrQRAu+7q71RWOSijoaI8JBAI9pMjIz5WNgJenq0gSTBTqPPCZAmzVcPHMu6K/7+V0ktXTGE4a1AyfSzXIWkoxFSFkukRhpGJWc3yiGsqxZpDwXSpOR6EEh4wW7KIqRKqIuP7Ufur7UZOpkEQ9ac8eqyb2YqN4xVwAgldCrlvMMOP3D9ATFPI12w0WcLxAnRN4eHxLCf6U9xcqqKrMo4fMF+xGO+Oc7w3ygrcO5he1dTZEGczJZPjvUnM5T043UkNoGkY1ioj0SrQr7zTni7UuLpQJQQSuspDo5l13y/L8qoej+uL1S0Hy60KgJ2UHg5KWeVOHJbXIRAIDiZH5hOnse49JOS1uTI9SQNDUyhaLqaz0fDs1tFkGMnqPH6il5Sh8txkiWJdwg2i8VnL/f+3d+cxkl3l4fe/d69ba3dX79PTPYs9M8bjcWy8xGbTC7xE/lkkefOKkMiRDM5f0ZCYoEQsUWRQAoZIiYgAESCR+SNYBCUYEiSHGAJ2/BLD2GbAxvuMPXvvXdutuvt5/6jununZe6Z6amb6+Ugtu2uqu57ylO957jnPeU6KZWqkabvTqR+HzHtgmhBHkGoKywBD0zB0nSSFdj2ITsWLsCwD29RJleK5w1VqXkjGtjDihJ1jvfw/N28giBWtKGa2ETFUdHnTaImjlRYDizMESik2lXMcmW9SzJiM92UZLGYY7XHJWMbyDhiAF4/VePrAApNVn6maz9aBPDdt7GGirK0YrM93+v7EO+1Xpqq0wpShooMft7fDnstaDJYXsvRwpSyrnMvV8j6EEFemdZN8+FHCzw/Os2/GI05grDfDaE8G2zBWXcdxIkcH19YZ6ckSLZ6H4scpcw2fRGnUWgFx0i44DSK1PHuiL/YLSSMouCb1IMYyQUejlLGwzfYJcgXHJE0VBdfCjxIWmhGvTDdIgbde28+cF/DO7YOMlDI8/vIclgHzXkR/wT5loB4sZti5oUQzSIjSlFaYsNAMOTDXZN6LlgdggGcPLFDxQnqyFpauLScqmqatmFE43+n7E5MH09ApZS3K+QyVVnheSyJrMVheyNLDlbKsci5Xy/sQQlyZ1k3y0fBjpmoBC15AkipMXbF9KM94b4b5RkikFM1o9TMgarEjWd0PMXWD2XrEwYUqlWa8XB9i0O4ZcuKyTUr7P75pgB/GZG2d3oxFmEDGNujNO5SzFj1ZC03XiGPwzZgdw0V0YN+sx7wXMt6XY9twkal6yN7DC4RxewblmuERrh3Kk3fMFUsjOcdk23CeBS8mTNpdSE/sVtpYnIWwdB3XNpis+kz0twt0T0wSlpYs5hoBjSDiSKXd2n2pMPXk5YwTk4ex3gwvHK3RDBO2LP7uc1mLwVKWHoQQojvWzdW26kdUWgHzzZAgVvhRwlStSU/WQdMh8i9s6SVJYKDXxtB0bEun0gxoRfGKwtQUCNVSq/U2Rbvt2FIjs1TBnBeQcyx2bSzR49qM9rgM5i0ylslco4UXKtJUUQtjhksuAwWHX99SZsdwgSdemaE3a7Oh1+WVqTrHFpoMFzNkLX15e6xtGJRcg1zGwjaN5ULO54/WTxmAdV2j0ozRNI2MtbK5FxxfsoiT9uF25Zy9vKPkTMsZS8mDUoqBQqbrU/6y9CCEEN2xbpKPUsbC0AyCKEWhESt4baZFFMcEcXLG3S7nkgBHqz6OaWAa4PkJYXI88dAAx2wnKUod73NqLvVwT6GZghani4fPRRyd96BPo+BY7J/1mKy2yNkmEFHKmEz0Z9nYm6XSinBMnTRNOVxpcWi+wZFKC6VSXp3ROFIN2smEpogTGCw41FoRrhPRn2+3SC/n7NMOwJv6szSjeLnY1AtXltsuLVls6M1ytNKifEInzytlR8iFxiE9MoQQ4uKsm+QjnzHJ2u2lBJ32ttljNZ+5RkjrQjOPRV7Urimxjfb36oQikqVll56cSZykNCNFELcPZtNon4i79DwWvz9cCYjRsXSdvYcX8PwE2zKwDI3hokvesXj2YIW6HzFdC3h5ssZP9s1QC1JqLZ+JvhyDeZvJetDufKrD5v4807UA19IouFlGe1yOLDQ5ON9cceLs0iA6Uc5RbcX4UYqpayv6fQwUnLMuWXR6OeNyG+ylR4YQQlycdZN8ZCyDGzf20IoSDs77LLQigkZ07h88TwnQOmFyYKkniAFYls5EOUcQp7wx56FrijO9tKmDHydMVnxUCkGUkrUNTEOnN9c+U8XQNWqtiDiF12YavDZdY9aL6c1a1P2IVhRztBqw0AyxDI04Vsw2ArYNF7hhQw9+lCzPSHhhvKLYdGkQPXFJwo+SFf0+do2Vzrpkcbo/u5gEotOD/cUmM9IjQwghLs66ST7iJOXFyQa/PFIniE+t79AXl0EuftNtW0p7ZsM2IIkV816IaxvkbBMvTLDihFgdn/HQAV2HrGMykHNoRCleGJN1DHpdi1akcC2drG0yVHBwbZMgTsnYBq0wptZqMlcPyTo6m8pZdowUOTjXxA9TygUHy9C4ZaKPN0/0MtsIaQQxc42AOS887SB64pLE/pnGKUWpZ2rwdfLPwvG27M8eWMA2DHpzFjdu7DnvBOJcg/1qk4mLTWakUFUIIS7OurhqekHM//sPTy3v5FiiAWO9LjeM5tnz+jwLzaRjyQe06zpSBa1IMesF2L5OX9YiThIaamWnVMcA19IY783i2jqanpAmioxlUs7ZJEoxWMxQdE029LpcO5jjuaNVkhjGemyKTh91P0KhsWO4yG9cP8KcF644h2WinEPX9eXEIO+YVFvxOQfRix1sZ+oBPz9Y4fBCa3mGZDWzBed6/dUmExc7c3G2WZ/LbYlICCEuR+si+cg5Jndu7eO/XpgG2ksHt27q5f/sHObVqRo/emmG2WZyUf0+TlawwDI1aq126/a6n+JYECUhcaJQtGc7DK2dhGzsy6I0jb68hWOY1P0mqQaVZkgriunLZdjUb6GUjmub3L6ljB8lHKv61PyEwUKGGzf2EKeKmyd6l2cm+vPOGXdzLA2idT8iiFPqfrT8+IkD5smD7fl2NV3SCGJMXaN/cSeMY+qrSmDOtStltcnExSZTZytUlXoQIYQ4t3WRfAC8/Zoy//3SDBmr3cTrprECw8UMB2bqTNeCjiYesLiVNgbLgDiBWIGVKkKlyJgGTlaj1kowF/8GojjFNA2GSxlGenK8Ntug3oqwDYNEpeh6yC8OLeCYGnmnHwA/Usx6IV4QE8aK6zeU+LXxXvrz9oq77839udP26Fj687xj8vps7YwD5smD7XTNX9UAm3dMynkbANcyuGm8Z1XbWs+1K2W1ycRabrGVehAhhDi3dZN8HK60KDjt4+yrfsxkLWChGfHyVI2w05kH7R0w2uKBdEviVGGaOq04xjZ1MrZOOWcTJoqsbTDa6+JaJi8cqWItDqIpUPdj4lThWMby7pggTnl9ts7RBZ/BogO6hmMZDBYzy8lBnLZbl594qqumaUzXfP7n1Vm8xaZj430uSaoY6cnw4tEaLx6roRa37HhhcsrsxmoH2PZg37NmSxGrTSbWcquv1IMIIcS5rZsro9eKsQyDrGMwU/MXT3J1mPM6t+PlRCcPrRbtnSw5WydRBrbR3m6botB1jQ19LtsGCwRxiqFrjPZkqfkhKOhzbcbKWTaUXEqZdsGqY+psLhfwgoSKF1LMtJdD4Hhy4FoGvzxc4VilxStTDW4a7+G6kSIH55vsn/XocW2m6h7FjImh67x4tMbhhRYaGjP1AE2DvGOtmN1Qqt2gbabuU21G9Oascw6wnRzsz1RTcTn0DQFpXLaeSH2PEBdu3SQft27p46dvzDPfjLAMA13TOFbxKWctDM7vxNrzpXF8t8uSiPYyjB4kGIZOELd7g6DFOKbBvpkm/TmHzQMFhnqyvDHbYKI/y7WDBWrNiIMLTWa9kL68Tc420DSNsT6XybqPa0dM9Gcp59rJx9Ld9xtzHl6QYBk6h+abpGl72uRopYUXRJQy7RNqe7IWm/rzvHC0Sr0VU8gY7J/xyDkG/XmHN+Y8Su7xg+SOVlpYhk6Upoz2tBOSE3uArOUF+MTOqo0gZqK8clan2y6nREisLanvEeLCrZvk466dw+ybbvKTfdPkbAvHhHrQ7m8xXLSotGKakepI7YeinXic/LsU4EUKc7EhWcbUSNHIOSaOobGhN8um/iwLXvsMl5snetk+lOcn++aYavjknOOFmgMFh80DOVpxsqIL6VS1xYE5jyRNcE0dS4e5esCm/hxBpJZ3vxi6TpQmbB3IMVHOMVjMMNsIeOZAhdnDAWGckmDx09fnAcjZTSbKucVZFZZPzG2GCc8dOXO9CHT2DnF5Vsc2+eWRKl4YU23Fq77wy12ruFhS3yPEhVs3ycecF6Hr4FgG042AomOyc6wHTUsZ7snx+nSVfbMtvCAh6MB+2zMlMTpg6KCZEIYK3dRQKmWsL8vODUVc2yRV7b6oDT/ipck6b8x66Og4ps5U3efgfJPBYuakLqQ6QZzy84Oz7J/18IIIQ9PIuxZ+HIDScCwNU9e4bqSIhsZQyeG6keLy0oBj6oz1upSyFhUvJGPpVFsxm/rztMJ4eaA+saYBOOMFeGmAPzDncXC+Sc4xMXX9ou4Ql2d1ZhsAbCrn8KN01Rd+uWsVF0vqe4S4cGv+f8tnP/tZPv7xj3P//ffz+c9/fq1f7ox+cbjCnjfmWfBiGn5MIWMyUHRIkoQgDlDoNDuUeJyNqYNpaJRdi9hpz5CM9rhMlHNM10MSFbD3YIUgTii6FkMFhyBS+HHCy1N1RksOB4rN5aWGE+sL6n6EF8T0uDZJklL1I27d3Eet5TJcyjBQcDhaaXGs6tOXt7lupLhiwC1kLMp5hyRV9BcyjPZkOFrx8aME09CXZwhOfE2lFNVW7bQX4KUB/shCk6l6wO2b+2iFCQfmvAuecVh6/ZJrkp9v0oqS5dN0V0PuWsXFkvoeIS7cmiYfe/bs4Stf+Qq7du1ay5c5L1M1nzkvRCkwjPZ228G8w4uTNX51tMbrsx4XeLDtWS2dB6sBjgVZu721NWebZC2DYtZmy0CONFUcnGtScE0OznvkHRvDSJmsBiQqpeEnpGnCjtESecc8Y5fRnGMyVffw44SsbVJvJZTzx2c4zqfvx4n9PE5+/um6l+7StNP+zqUBflN/nql6wBtz3mKH19O3dD8fS68/UHCWl4Eu5MIvd63iYkl9jxAXbs2uuI1Gg3vuuYevfe1r/PVf//Vavcx5GyxkcAyNyZpPkkIYJcx5AUcWWiSpIko6m3notM91MYz2PwdLLpAwUsqybTDP04cWqIcxfpLiWjoF1yJWcKjiESaKKE6YrSdsLbv0ZDOMlDQOzOukyfFZiJMNFBzedm0/E+UsSilyjknGMihkrPManE93MT3XxfVsF+ClAb4VxmzpzzFRzgKcsaX7alzshV/uWoUQonvWLPnYvXs3d999N+9+97vPmnwEQUAQBMvf12q1NYlnrNelN+8w70W4WQPT1Nk3XWey4nNsoUkUd67Zhw5kLY1EKWxDR9c1FpoBPVmbZpjw/NEatVbCSNHBdSw292cZ7c3iWgZP7Z+j6BjkHINUQT5jU29FmIZOwbEY7ckuH+x2Mk3TGCq5DJXc08Z1puZga1V8eboBfqYenFdL97V2scmLFKwKIcSFW5Mr/ze/+U2effZZ9uzZc87nPvjgg3zqU59aizBWcG2T7YMFbMNYLJRUGIaOY+v4seI0Z81dsJR2Q7E4gSRJMRbXXrwgwmtFWKZGM1JkbIOsY1FyHSqtmJcmG6Bp9OczXDOU50ilRZy2iynH+7I4tsmWgdwFF0aeqc5hrYovTzfAXy0zDlKwKoQQF04/91NW59ChQ9x///184xvfIJM598X44x//ONVqdfnr0KFDnQ4JaBdTXjNUYLiUoSdrsWOkwEgpQ8GxKbkmeodvWv2kvePFNAGt3V696ifMtxIqrRilKVpBhB+n+FFMmiiCOGFjXxbd0Jistton2JZc6kHC4YUWtWZEmLRnaJRSTNd89s80mK75yx1Jz+ZMdQ4nJiVJqk45gO9sVhvHUkKyZSDPYDFzxc4WXMx/MyGEWO86PvPxzDPPMD09zc0337z8WJIkPPHEE3zxi18kCAIMw1j+M8dxcJy1v/sdKDi89ZoyxYxJK2r3twDQNZ05z2e6EUCHx48EaC42UNWXvjSIErBNHV03sHQNxzLZNlxkphFyeL6FYxq4tkGva3F4wSOMEgYH8gwXbQ7NeczUg3YtRRSTphqGrnHDhiIAB+ebAIz3ZU8Z3M8067Ca4suTlxuUUufs83E1koJVIYS4cB2/Yr7rXe/iueeeW/HYBz/4QXbs2MFHP/rRFYnHpaRpGrquo+s6GUtjshZyw4Yiv3PzGBlTY7LW4nAlvPDfz9l7eyw9J+fouJaJYWpsH8ox1pul4YdMVloMFixSZXHDWC/NMKLeinlxsk6SKp4/WmWyZmPoGkXXxvMjJgby/PrmMkcrLQ7ONzk432TfjAfAlv4cb982cNYD4pasZink5OWGkmuuyy2rV8vykRBCdEPHk49CocDOnTtXPJbL5SiXy6c8fikppTgw53FkwaOUtTk838QLIm7f3Edv1iaKLq7o42yLDQrImOCaOn35xYZetoFlGiilkc86xEqxa2MvfpTgRwmWYeBYKf2FDJsH8vx0/xwLno9umGwfLrIviPH8aPHOGxa8kDdmPUyt3THVC+LzTgRWU3x5ct0IsC5nAGSbpRBCXLj1MVLAYqfNJvtnmxycmyVKUl6davCrY1X2TTXwwuSssxfnQ6O9rVbXIUqP/y4dcC2dG8d6iFNFzY/J2gamplHO29y2qZeXjtWJk5TRHhfH1ClkLGbqPq9NexxeaJLPWOwcLfL80RovTdbozzncsqmP0R4XP0r41ZEqNT9mpu4zWHDZuaG4JonAycsN431ZtDP0+egU2VkihBCdcblcTy9J8vHjH//4UrzMWS39h37TSJFD8x6mDmGS8uwbFfw4Rjc0rFQRJReegCjaO11srd1CPU4XvzfaBa9DxQxhotC0iKl6C8swcC2dH7w4xUvHqowUXHaO9fCO7e3lkv68jaZpvDpVZ64RMlSwcS2Dct7m2qECO4YL6Lq+fKjbTeM9/PLQApv6s7z1mvI5E4EL+RCebrlB07Q1nQGQnSVCCNEZl8v1dN3MfOQdE3Nxz2tv1mbOC0mUwjQ0htwM8/UQTcVYeooXXfjrpItf/QWHWivCT1J0XccydOIUFpohx6rtotKCa6EUvHikyv65JofnWxyr+UyUswyVXDRNoz/v4Jjtc1scU+fWxYZhJyYJecfECxP2z3pkbIucY6Lr+jkTiQv5EHZjuUFaoQshRGdcLtfTdZN8LN2x1/2IkZLDU/vnmfcCMoaOacDWwRzzXsB0PSCKEyJ14TMgcQILzQBd03AtgzSFUsZGociYBsWMRW/OppyzUECoFGkKC0FCmLQPYbt96/knB+1W41m8MF4+4fZ8PlCXy4fwXGRniRBCdMblcj1dN1fxE88E8aOEQsYkY2nMNUL8KKG/4HB4oUUQp2Qcjci/8OoPXQM0yGdMerM2NT+mN2fS8GN6shYberNM1Vs4lsFwKYOtayRpSilrUHAsLKM9Y9EIYuIkxbVN3phtUHKPL3OcvGQy3pddccLt0gfqbEsrl8uH8FxkZ4kQQnTG5XI9vTxHmzU0Uw/Ye6hKtRWTsXSaQcpk3efArEctSNrdTi+i7gMgUpCGUCwZjPZkcL0YTdOotCLQoBmlWLrBUN7FNnTesX0QDQ1N0xjtyXDtUAFg+QC5Xx6ptr+fb59mO1jMnDIrcsOG4mk/UGebPblcPoTnIjtLhBCiMy6X6+m6Sz4aQYypa/QXHF6bqhOmKT2uzWtJgyhOiZN2zcbFULT7lR2rh4yUXHaMFDB1jfmjNWyjfdBaT8FlQ2+Gaivh1zf38eaJPmbqAQMFhx3DBZRSKKWwDI2srbNztIQfp8tLIycvmXhh0u4aepr3e6allcvlQyiEEGJ9WXfJR94xKedtAMbLWaZrPnONgJ6cRaV1cU3GNFYmLi0/Zb4ZkXMjGn7MQjOi5FqEiWK2GbL3UIUoUZSyBhv7coz1uhQyFpqmMVMPeO5IDT9KCSLFdC2kL28vL42c75LJ2Z53uWy5EkIIsb6su+SjvdTQQyOIaYUxP90/x1TVJ4wSLFMjitWqZz4MIGdBkEBwwg/HQJwkOIZOoZih4kfkHIM8GkXHJO8YvDLV4Kn9c/x0/wLbhwuU88eXQpJUcd1ou236UMnhupHi8tLI+S6ZnO15F7LbRRIWIYQQF2vdJR8nLjXsn2mQsXTiVOFHCVp6Yce7WCagaSSpQmfl7Ee1GbPQDOnPO5RzNqauM1DMEEUpr043OFr1cSyDehCwbbiwfEjZ0ozFsYpPOd9OPM6nVfrZ3u/JLmS3y+WyR1wIIcSVa90lHyfKOyYvT9Z5eapOI1Q0QrXqLqcmoFJItPbP6os/rwFZS0M3dRxToy9ns22ogB8njPW4BLFitmFR92Mypo4XaszWffrzzvKMwloXg17IbpcrZXvupSSzQUIIsTrrOvkYKDiUXBvb1Mk5Og3/7AfEnU7e0QmSFIVGuviTOav9e3pyFhPlHP2FTLt9uxdhGTr9hQxTtQCUxlAxw2DRZutgnutHi2zqzx/vGrrGxaAXkuBcKdtzLyWZDRJCiNVZ1yOHpmncurmPpw8scGDew7E0wujMNR/LZ7cYx/9dI8UyNKJEUc5boBTlvE0YK1zbBAVJCkNFh+3DBWqtGNvQ0TQouCbb3QLXj6xMOi7l+19tgnOlbM+9lGQ2SAghVmddJx8Ad24tU2lF/OBXxzi40KLeDFloRVSaCclJz9UAxwLL0HEsHVM3cC2NMIZaK8I0NPpzGXYM52kEitGSzWszTSqeT5oqhksZ+vMZdF0j71hsGypytNKiv5C5Yu6UZXvuqWQ2SAghVmfdXyU1TWNzOcu2oQK6rlHNmGgLLeLYpxquXICxNXBNA9PQ2NKfpy9n0woTXpys4zomlqGjUMRKQ9cVlVZCLYjJ2jbTjZAwStg1VkIpRbVVk8HqKiGzQUIIsTrretRTSvGTfXP829OHODDfpBlEpArSVJHNWDTCcMXsh6/AjBUqTnEtk768w4vH6qQKbFMnbxuUcxl2bSgRpYoDM3VU2q4HUWlK0bUYLGZQSrHrNMfQS+HilUlmg4QQYnXWdfIxUw94+vU59s96VFsRUZrQClNMHaJYnbLsAhDEKamC549W0XUouQb0Zak1Q0CjN2fi2iYFQ2O6ZhEreGPWo7/gUM63k4wzDVZXSuGiJElCCCEuxrpOPtqDp41tanhhQtbSiZOIhq9I1el3vkQKXAOSJGG2EWEZoNAo5RzKeYt37hjiupEi+2c8LB2uHymQKujL24yUzp5IXCmFi51MkiSREUKI9WddJx95x2S87LJrQy9R0j5LxQsigjghSY8nHgYsz4LoQJxCrDRKGZNKK2S0J8vbtw+glGKomGGhGXF4ocV0I2S6HjJccrhmIE/Rtc8ZT7cKF1eTBHQySbpSZnuEEEJ0zrpOPgYKDr823sumssvmgSyPvzzJkXltRbOwdPGfFmCYkLNNwjghjBMOV1qYps5g0aEna3N0ocnjr8zghwk1P6aUsYhiRdExlw+L2z/TOOPgvlS4WPcjgjil7kfLj6/1bMBqkoBOJklXymyPEEKIzlnXycdS7cVsI2D/TJOqn6K09lZa0wRdQV/OJEwUGduk2gwxdY1C3mHOC4mShIGCzXUjBfqyFk/t93h1ysO1DOa9kJ6sxa6xXkZKGVpRynNHamcd3JfiAXj9Es8GrCYJ6OTuDtmmKoQQ649c6Wnf9TfDmM3lPK0wptqMcSydkmuxfTBPlEIpazBZCzk832SmEWAbBo5tUS5k2NyfB2CqGtIMEmp+hEoUGhYLzZANPe3E4XwH927MBqwmCejk7g7ZpiqEEOuPJB+0B0DXMnl9roGh6QwUHPKOQStK0HTYNphjrDcHKuVHL88QRgn9eYMwSig4BuN9WX5xuELNb9eLVFohG3tc3rF9EKUUm/pzjPdlz7u3RzdmA7qVBMg2VSGEWH8k+QC2D+W5eaJEPQhxDI3ZesCcF+EFCXknoC/n8lyzypGFFq/PeSz4EUkzwrV0DP14LYZj6hQyFkGU4NomkzWfrQN5Jsq59uB+mt4ep9ONRECSACGEEJeKJB/AnBdR8xPKuQyaBvtnPUCj4JrU/ZijCx4J0AwTVJpiGzqxpujLZYgSxaGFFr1Zm/G+9rJNxjK4Y2sfUaIwdQ2l2vtmzndwl0RACCHE1UySD9o1Fqau4do6M5MBug5BlJBFZ2PZZcdwiYOVJkGUUPUTKo0Aw9CJ3IR0cT/uRDnHzg1FJqstco6JpmkEcYq/WGi664RiUmhvbZ2u+RycbwIw3pdlsJg5466Wpa2wSzthHFNfXo7xwkR6ZAghhLhiSPJBu8ainLeZafj05CxGSg6zXkSPa/Gbv7aBawZy/H/75vlFOo+hgWWZmLpGolIGC85y4vD2bQPLycF0zWeqFnDdaJFjFf+UotGZesD/vDrLc0eqREnKtUN5/s/OEYZK7mljXNoKO98IObTQZKzXxdA1NA3yjiU9MoQQQlwxJPlgqcaih5JrYWgalWbE1qESBcdkQ2+W4Z4sb99mMFdv9/XI2gapSsnbJjdu7FmesRgsHj+dtj/vEKdVjlX80xaNNoKYyWoLL4xRKbwy2WDnaPOMycfSDpiiaxLNppSyFlNVHzSWT8eVHhlCCCGuBJJ8cLzGYqDgkHNMfn6wgqlrlPM2OdtYXh45UvXxgpg4UcRJylCPy41jPadd6ji5aLQ/bzNd85dnRhp+RCtKqbciiq6FYxpnjXFpB8x8I8IydKrNaHF5B+mRIYQQ4ooio9UJNE3jupEi/XlnOWlI05RHfzXJq1MNDkzXMHSNgmPQ8BNMUqZrLdI05XDFB1bWbpxYNDpd8/nl4SpzjYDDCy3Gel368zaQJ2uZDBYzjPdlzxjbid1Pd44Vz1jzIYQQQlzuJPk4yclJw57X53hlskEYp2DoxFHCTBCRJBr7Zlv881OH2NCXoRWmNMOE4aLD27cN0J93ViQFjSAmTlMUipm6z4Zel+FShp0bSpTzzjmTh5OXdYQQQogrlSQfZ7C0u+RopUUcp/hxTNWL0A0DC9AUGJrOdKNFnCS4Trub6UIzYL4ZMlpqJxfNMGFjr0uYKPZPN3hxqkbFi0nUPLdvLvPmiZwkFEIIIdYVST4WnXyqq1KK547U8MMUXQcvaLdcz6U6mmZS89uJhm1pWHrEwfkms15IOW9TbYQcWWhxx9Z+jlVbHKu08KOUaiuk4oWM9WRRQNGVpRIhhBDrjyQfi04+1bXkmiSp4rrRIq/PNvCCmC3lPC9N1tA1yGVMSBL6cjZRFBGnUG9F+GFMT9ZC90IWvIANvTlumejljbkmQ0WHmUZI0bUxDI3erC19OYQQQqw7knwsOvEwtyOVJgvNkJl6QLUZUcxa9McuQ6UMsUrJWSYzXoAXJJi6xoFaxHTNR9M0/ChhupZSzjs0g5h5L+T12QZRApZrMlpyKWZMhkpnLzAVQgghrlaSfCzKOya6Bi8erTHn+dimTqJgthGwdSBHf86hFSVs7s/TDGJqfsTRVotqKwTaBaEZ26DSTLA0cGyz3ZMjToiShLG+HNePlCi41vIZMCcvuZy89CMdS4UQQlyNJPlYNFBw2NDrMl0PSJTiwFyT3pyNHyYcXmiydbDA5myONE355eHachfTFLB0jZJrYRkaqbLYsNgorNaK6M05FF2HvG0zUMywZSB/xhhOXvqRjqVCCCGuRpJ8nMBb3A67sS/HkQWfqarPcCnDZDUkUXV6XJtS1qLSCllohpiGzq2by8w3WqQJBKmiN0oZLTnYloGhafTmHFphQpgk52wCduLSj3QsFUIIcbWS5GPRTD3gwFyTqVrAsYUmhg6tIObQXJNGGNIKYyYtn3LWpifn8Otb+vnf1+cI45QtA0U29mZpRjG9WZtKM2Sk5KJpMO9FxKnipvGeFcssp1tiWepiKh1LhRBCXM1kdFu0lATcvrnM/+6bZs4LSFLFZLVJmqZMGhETfVn68g5Zy6AvZ/E2s5++rM21QwX6shbPH62TpIoNvRY3bCiiadoZ6zdm6gG/OFRhwYsIk4SbJ3rZMVxY0ZJdtuEKIYS4Gq375GNpBmKuEeCFMWgQJ4pWmNKXtQmj9hbZjG3iRwmkivE+F8fU6c8fP9EWQNf1U5KNMy2bNIKYBS+iHkTM1AM0TaM/76zorio6T4p6hRCi+9Z98jFd8/mfV2dp+BE1P2K8L8tw0eVIpUUrTrEsA8cySJXCCyKaQcKxShNdNyhkLKqtGrtOaH1+volD3jEJk4SZekB/wcHUNanxuASkqFcIIbpv3ScfB+eb7J/10FTKs4cqvHisSn/Oote10ICxTX0M5E2e3DdPGMNU3SdIEnpcm1s2l2mFMY0gZmCVd9QDBYebJ3rRNG35BF2p8Vh7UtQrhBDd1/HR7sEHH+Tb3/42L730Eq7rcuedd/K5z32O7du3d/qlOupYLeBwxafq6bxwNMbSdbYMFrh9MI+pa+i6Tilrsn/Woz9noQ/o/PT1Obb058g75nndUZ885b9juLDiBF2p8Vh7UtQrhBDd1/Er7+OPP87u3bu59dZbieOYT3ziE7znPe/hhRdeIJfLdfrlLtp4X5atAzkmKx6OoYGmMeeFoGmkaDiWzo7hAq5toAFZy2C8nOX/2j7AgfkmE+UsAwWH12e9c95RnylBkTvvS2eg4EhRrxBCdFnHk4///M//XPH917/+dQYHB3nmmWd4+9vf3umXu2iDxQxvu3aAvGOQKo3nDlfQ0ShlbRRQ92O29udwLZMFL2T7UIHRngxBrNjQk2WinEPTtPO6o5Yp/+7Tlupzuh2IEEKsY2s+51ytVgHo6+s77Z8HQUAQBMvf12q1tQ5phaXB6P9+0zBjvVm+98sjPPnKHEGcYOo624by/Np4Lzcv7mTJ2QYAXpisuHM+nztqmfIXQgghQFNKqbX65Wma8pu/+ZtUKhWefPLJ0z7nk5/8JJ/61KdOebxarVIsFtcqtDNKkoSf7JvjxWM1erM2b72mzHBPtiPbMWWbpxBCiKtVrVajVCqd1/i9psnHH/3RH/Hoo4/y5JNPMjY2dtrnnG7mY+PGjV1LPoQQQgixeqtJPtZs3v9DH/oQ3/ve93jiiSfOmHgAOI6D40jRnxBCCLFedDz5UErxx3/8xzzyyCP8+Mc/ZvPmzZ1+CSGEEEJcwTqefOzevZuHH36Y7373uxQKBSYnJwEolUq4rtvplxNCCCHEFabjNR9nKqB86KGH+MAHPnDOn1/NmpEQQgghLg9drflYw/pVIYQQQlwF9G4HIIQQQoj1RZIPIYQQQlxSknwIIYQQ4pKS5EMIIYQQl5QkH0IIIYS4pCT5EEIIIcQlJcmHEEIIIS6py+5M96U+IbVarcuRCCGEEOJ8LY3b59Pv67JLPur1OgAbN27sciRCCCGEWK16vU6pVDrrczreXv1ipWnK0aNHKRQKZ2zVfqFqtRobN27k0KFDV2Xrdnl/VzZ5f1e+q/09yvu7sq31+1NKUa/XGR0dRdfPXtVx2c186LrO2NjYmr5GsVi8Kj9YS+T9Xdnk/V35rvb3KO/vyraW7+9cMx5LpOBUCCGEEJeUJB9CCCGEuKTWVfLhOA4PPPAAjuN0O5Q1Ie/vyibv78p3tb9HeX9Xtsvp/V12BadCCCGEuLqtq5kPIYQQQnSfJB9CCCGEuKQk+RBCCCHEJSXJhxBCCCEuqXWTfHzpS19i06ZNZDIZbr/9dn72s591O6SOeeKJJ3jve9/L6Ogomqbxne98p9shddSDDz7IrbfeSqFQYHBwkN/+7d/m5Zdf7nZYHfPlL3+ZXbt2LTf+ueOOO3j00Ue7Hdaa+exnP4umaXz4wx/udigd8clPfhJN01Z87dixo9thddSRI0f4gz/4A8rlMq7rcsMNN/D00093O6yO2bRp0yl/h5qmsXv37m6HdtGSJOEv//Iv2bx5M67rsnXrVv7qr/7qvM5fWUvrIvn4l3/5Fz7ykY/wwAMP8Oyzz3LjjTfyG7/xG0xPT3c7tI7wPI8bb7yRL33pS90OZU08/vjj7N69m6eeeorHHnuMKIp4z3veg+d53Q6tI8bGxvjsZz/LM888w9NPP8073/lOfuu3fotf/epX3Q6t4/bs2cNXvvIVdu3a1e1QOur666/n2LFjy19PPvlkt0PqmIWFBd7ylrdgWRaPPvooL7zwAn/7t39Lb29vt0PrmD179qz4+3vssccAeN/73tflyC7e5z73Ob785S/zxS9+kRdffJHPfe5z/M3f/A1f+MIXuhuYWgduu+02tXv37uXvkyRRo6Oj6sEHH+xiVGsDUI888ki3w1hT09PTClCPP/54t0NZM729veof//Efux1GR9XrdXXttdeqxx57TL3jHe9Q999/f7dD6ogHHnhA3Xjjjd0OY8189KMfVW9961u7HcYldf/996utW7eqNE27HcpFu/vuu9V999234rHf+Z3fUffcc0+XImq76mc+wjDkmWee4d3vfvfyY7qu8+53v5v//d//7WJk4kJVq1UA+vr6uhxJ5yVJwje/+U08z+OOO+7odjgdtXv3bu6+++4V/y9eLV599VVGR0fZsmUL99xzDwcPHux2SB3z7//+79xyyy28733vY3BwkJtuuomvfe1r3Q5rzYRhyD//8z9z3333dfxw02648847+eEPf8grr7wCwC9+8QuefPJJ7rrrrq7GddkdLNdps7OzJEnC0NDQiseHhoZ46aWXuhSVuFBpmvLhD3+Yt7zlLezcubPb4XTMc889xx133IHv++TzeR555BHe9KY3dTusjvnmN7/Js88+y549e7odSsfdfvvtfP3rX2f79u0cO3aMT33qU7ztbW/j+eefp1AodDu8i7Z//36+/OUv85GPfIRPfOIT7Nmzhz/5kz/Btm3uvffebofXcd/5zneoVCp84AMf6HYoHfGxj32MWq3Gjh07MAyDJEn49Kc/zT333NPVuK765ENcXXbv3s3zzz9/Va2pA2zfvp29e/dSrVb513/9V+69914ef/zxqyIBOXToEPfffz+PPfYYmUym2+F03Il3kLt27eL2229nYmKCb33rW/zhH/5hFyPrjDRNueWWW/jMZz4DwE033cTzzz/PP/zDP1yVycc//dM/cddddzE6OtrtUDriW9/6Ft/4xjd4+OGHuf7669m7dy8f/vCHGR0d7erf31WffPT392MYBlNTUysen5qaYnh4uEtRiQvxoQ99iO9973s88cQTjI2NdTucjrJtm2uuuQaAN7/5zezZs4e///u/5ytf+UqXI7t4zzzzDNPT09x8883LjyVJwhNPPMEXv/hFgiDAMIwuRthZPT09bNu2jddee63boXTEyMjIKUnwddddx7/92791KaK1c+DAAX7wgx/w7W9/u9uhdMyf//mf87GPfYzf+73fA+CGG27gwIEDPPjgg11NPq76mg/btnnzm9/MD3/4w+XH0jTlhz/84VW3pn61UkrxoQ99iEceeYT//u//ZvPmzd0Oac2laUoQBN0OoyPe9a538dxzz7F3797lr1tuuYV77rmHvXv3XlWJB0Cj0WDfvn2MjIx0O5SOeMtb3nLK1vZXXnmFiYmJLkW0dh566CEGBwe5++67ux1KxzSbTXR95VBvGAZpmnYporarfuYD4CMf+Qj33nsvt9xyC7fddhuf//zn8TyPD37wg90OrSMajcaKu6zXX3+dvXv30tfXx/j4eBcj64zdu3fz8MMP893vfpdCocDk5CQApVIJ13W7HN3F+/jHP85dd93F+Pg49Xqdhx9+mB//+Md8//vf73ZoHVEoFE6pz8nlcpTL5auibufP/uzPeO9738vExARHjx7lgQcewDAMfv/3f7/boXXEn/7pn3LnnXfymc98ht/93d/lZz/7GV/96lf56le/2u3QOipNUx566CHuvfdeTPPqGRrf+9738ulPf5rx8XGuv/56fv7zn/N3f/d33Hfffd0NrKt7bS6hL3zhC2p8fFzZtq1uu+029dRTT3U7pI750Y9+pIBTvu69995uh9YRp3tvgHrooYe6HVpH3HfffWpiYkLZtq0GBgbUu971LvVf//Vf3Q5rTV1NW23f//73q5GREWXbttqwYYN6//vfr1577bVuh9VR//Ef/6F27typHMdRO3bsUF/96le7HVLHff/731eAevnll7sdSkfVajV1//33q/HxcZXJZNSWLVvUX/zFX6ggCLoal6ZUl9ucCSGEEGJdueprPoQQQghxeZHkQwghhBCXlCQfQgghhLikJPkQQgghxCUlyYcQQgghLilJPoQQQghxSUnyIYQQQohLSpIPIYQQQlxSknwIIYQQ4pKS5EMIIYQQl5QkH0IIIYS4pCT5EEIIIcQl9f8DvBq4eqmKlScAAAAASUVORK5CYII=", - "text/plain": [ - "
    " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import seaborn as sns\n", - "\n", - "# draw the graph. This might take ~30 seconds.\n", - "sns.regplot(x=\"new_cases_percent_of_pop\", y=\"search_trends_cough\", data=weekly_data, scatter_kws={'alpha': 0.2, \"s\" :5})" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": { - "id": "5nVy61rEGaM4" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 62, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGeCAYAAAA0WWMxAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAArzVJREFUeJzs/XmMnOl1349+3r32qt437rORM+SMJMvWYkmWYtkaymtyE8O5RqDYQBLAAWxHQGwrsA07sK04fxhGcgM7zgWcBNkQ3Fz7l5ufZrxKtmRLsuSRNMMZkjMckk2y96qu/d3f97l/vFU13c3qjey9ng9Ay+yu7nq6OF3n+5zzPecoQgiBRCKRSCQSyQGhHvYBJBKJRCKRDBZSfEgkEolEIjlQpPiQSCQSiURyoEjxIZFIJBKJ5ECR4kMikUgkEsmBIsWHRCKRSCSSA0WKD4lEIpFIJAeKFB8SiUQikUgOFCk+JBKJRCKRHCj6YR9gI3EcMz8/Tz6fR1GUwz6ORCKRSCSSHSCEoNlsMj09japuk9sQu+TP//zPxfd///eLqakpAYjf//3f733O933xsz/7s+Ly5csik8mIqakp8Q/+wT8Qc3NzO/7+9+/fF4D8I//IP/KP/CP/yD/H8M/9+/e3jfW7zny0221eeOEFfuInfoK/83f+zrrP2bbNK6+8wi/+4i/ywgsvUK1W+emf/ml+8Ad/kK9//es7+v75fB6A+/fvUygUdns8iUQikUgkh0Cj0eD06dO9OL4VyuMsllMUhd///d/nh3/4hzd9zNe+9jW+4zu+g9nZWc6cObPt92w0GhSLRer1uhQfEolEIpEcE3YTv/fd81Gv11EUhVKp1PfznufheV7v741GY7+PJJFIJBKJ5BDZ124X13X5uZ/7Of7+3//7m6qgz372sxSLxd6f06dP7+eRJBKJRCKRHDL7Jj6CIOBHfuRHEELw27/925s+7jOf+Qz1er335/79+/t1JIlEIpFIJEeAfSm7dIXH7Owsf/Znf7Zl7ceyLCzL2o9jSCQSiUQiOYLsufjoCo+33nqLz3/+84yMjOz1U0gkEolEIjnG7Fp8tFotbt261fv7nTt3+OY3v8nw8DBTU1P83b/7d3nllVf4P//n/xBFEYuLiwAMDw9jmubenVwikUgkEsmxZNettl/4whf42Mc+9tDHP/WpT/HLv/zLnD9/vu/Xff7zn+ejH/3ott9fttpKJBKJRHL82NdW249+9KNspVceY2yIRCKRSCSSAUAulpNIJBKJRHKgSPEhkUgkEonkQJHiQyKRSCQSyYGy7+PVJZKtEEKw0vRoeSE5S2csb6EoymEfSyKRSCT7iBQfkkNlpenx6oM6USzQVIXnTxUZL6QO+1gSiUQi2Udk2UVyqLS8kCgWTJfSRLGg5YWHfSSJRCKR7DNSfEgOlZylo6kK8zUHTVXIWTIZJ5FIJCcd+U4vOVTG8hbPnyqu83xIJBKJ5GQjxYfkUFEUhfFCivHDPohEIpFIDgxZdpFIJBKJRHKgSPEhkUgkEonkQJHiQyKRSCQSyYEixYdEIpFIJJIDRYoPiUQikUgkB4oUHxKJRCKRSA4UKT4kEolEIpEcKHLOh2RfkAvjJBKJRLIZUnxI9gW5ME4ikUgkmyHLLpJ9QS6Mk0gkEslmSPEh2RfkwjiJRCKRbIaMCJJ9QS6Mk0gkEslmSPEh2RfkwjiJRCKRbIYsu0gkEolEIjlQpPiQSCQSiURyoEjxIZFIJBKJ5ECRno8TiBzwJZFIJJKjjBQfJxA54EsikUgkRxlZdjmByAFfEolEIjnKSPFxApEDviQSiURylJFR6QQiB3xJJBKJ5CgjxccJRA74kkgkEslRRpZdJBKJRCKRHCgy8yGR7AGyvVkikUh2jhQfEskeINubJRKJZOfIsovkSCOEYLnhcnulxXLDRQhx2Efqi2xvlkgkkp0jMx+SI81xySjI9maJRCLZOfIdUnKkWZtRmK85tLzwSHbxyPZmiUQi2TlSfEiONMcloyDbmyUSiWTnHM13comkg8woSCQSyclDig/JkUZmFCQSieTkIbtdJBKJRCKRHChSfEgkEolEIjlQBr7sIidTSiQSiURysAy8+DgucyQkEolEIjkpDHzZRU6mlEgkEonkYBl48XFc5khIJBKJRHJSGPhIK+dISCQSiURysAy8+JBzJCQSiUQiOVgGvuwikUgkEonkYJHiQyKRSCQSyYEixYdEIpFIJJIDRYoPiUQikUgkB4oUHxKJRCKRSA4UKT4kEolEIpEcKFJ8SCQSiUQiOVCk+JBIJBKJRHKgSPEhkUgkEonkQNm1+PiLv/gLfuAHfoDp6WkUReEP/uAP1n1eCMEv/dIvMTU1RTqd5uMf/zhvvfXWXp332CKEYLnhcnulxXLDRQhx2EeSSCQSieRQ2LX4aLfbvPDCC/y7f/fv+n7+X//rf82/+Tf/ht/5nd/hq1/9Ktlslk984hO4rvvYhz3OrDQ9Xn1Q562lFq8+qLPS9A77SBKJRCKRHAq73u1y9epVrl692vdzQgh+67d+i1/4hV/gh37ohwD4z//5PzMxMcEf/MEf8KM/+qOPd9pjTMsLiWLBdCnNfM2h5YVyn4xEIpFIBpI99XzcuXOHxcVFPv7xj/c+ViwWed/73seXv/zlvl/jeR6NRmPdn5OGEAI3iCi3PN6Yr6OpkLMGfqefRCKRSAaUPRUfi4uLAExMTKz7+MTERO9zG/nsZz9LsVjs/Tl9+vReHulIsNL0mKs6GKpKEMVMl9KM5a3DPtaJRnpsJBKJ5Ohy6N0un/nMZ6jX670/9+/fP+wj7TktLyQWcGm6wFg+RcrQUBTlsI91LHhUESE9NhKJRHJ02VPxMTk5CcDS0tK6jy8tLfU+txHLsigUCuv+nDRylo6mKszXHDRVkSWXXfCoImKtxyaKBS0v3OeTSiQSiWSn7Kn4OH/+PJOTk/zpn/5p72ONRoOvfvWrfOADH9jLpzpWjOUtnj9V5KmJHM+fKsqSyy54VBEhBZ9EIpEcXXb9jtxqtbh161bv73fu3OGb3/wmw8PDnDlzhp/5mZ/hV3/1V3nqqac4f/48v/iLv8j09DQ//MM/vJfnPlYoisJ4ISW7Wx6BRxURXcHX8kJyli4Fn0QikRwhdi0+vv71r/Oxj32s9/dPf/rTAHzqU5/iP/7H/8jP/uzP0m63+cf/+B9Tq9X40Ic+xMsvv0wqldq7U0sGhkcVEVLwSSQSydFFEUesDaDRaFAsFqnX6yfS/yGRSCQSyWEgRFK6doKI8fzeJwR2E79lIVwikUgkkhNMHAsabkDDCQnjGFM/9EZXKT4kEolEIjmJhFFM3QlouiHx0SpySPEhkUgkEslJwgsj6k5A24uO7IBFKT5I6mArTW+dqVEOAZNIJBLJccLxI2qOj+NHh32UbZHig3cGWUWxQFMVnj9VZLwgu3MGHSlKJRLJUadrIq07AX4YH/ZxdowUH8iNs5L+SFEqkUiOKnEsaLqJ6Ajj4yM6ukjxgZyGKemPFKUSieSoEUYxDTek4QRHzkS6G2SURU7DlPRHilKJRHJU8MOYmuMfaRPpbpDvpshpmJL+SFEqkUgOG8dPOlds/2Qtx5TiQyLZBClKJRLJYdE1kXrB0e9ceRSk+JBIJBKJ5AgQx4Kml/g5guj4mUh3gxQfEolEIpEcIlEsOpNIA6L4+Ps5doIUHxKJRCKRHAJ+mIw/b3nhiTCR7gYpPiQSiUQiOUDcoDv+/GSZSHeDFB+PwHGffHkUzn8UziCRSCQHSdsLqZ1gE+lukOLjETjuky+PwvmPwhkkEolkvxFC9IaCnXQT6W5QD/sAh40QguWGy+2VFssNd0d1t7WTL6M4mat/nDgK5z8KZ5BIJJL9IooF1bbPvVWbSsuTwmMDA5/5WGl6fOt+jWo7wI8i3nN2iEtThS1LAEdh8uXjlC2Owvl3cgZZmpFIJMeNIIo7nSuDZyLdDQMvPlpeSLUd0PQCVpoeiqIwmrO2LAEchcmXj1O2OArn38kZZGlGIpEcF6SJdHcMvPjIWTp+FLHS9BjNW+iqsu0CsaMw+fJxlp4dhfPv5AxysZtEIjnqtDuTSF1pIt0VAy8+xvIW7zk7hKIo6KrCSM48FgvEsqZGywt45Z5DztLJmtphH2nPOQrlIYlEItmIEMkk0rotTaSPysC/myuKwqWpAqM569gtEBMCEJ3/PYEchfKQRCKRdIliQdMNqDuDM4l0vxh48QFHowyxW9p+RD5l8MxkgfmaQ9s/eSm/4/jvIpFITh5dE2nLDYlP6m3vgJHi45giSxISiUSyv7hBRKMz/lyyt8iIdUyRJQmJRCLZH2w/MZE6JzCjfFSQ4uOYIksSEolEsncIkQw7rEkT6YEgxYdEIpFIBpY4FjTcgIYTEsaDIzqEEIc6tFGKD4lEIpEMHOGaSaSDZCK9U27zR68v8vpCg//rn34ITT0cASLFh0QikUgGBi/sTiKNBmb8ecsL+fyNZV66tsiNxWbv4198a4WPPnM4xXspPiQSiURy4nH8iJrjD4yJVAjBqw/qfO7aIn/x5gpe+HBJ6X+9MifFh2TvOazFbHIhnEQiOQp0TaR1J8DvE3xPIitNjz98fZGXX19kvub2fczFyTw/9v6z/OAL0wd8uncYePHRL1ACJyJ4HtZiNrkQTiKRHCZxLGi6iegYBBOpH8Z8+XaFl64t8vW7q/QbvlpI6Xz82QmuXp7k0lSBU0OZgz/oGgZefPQLlMCxCZ5bZRkOazGbXAgnkUgOgzCKabghDScYCBPp7ZUWL11b5I/fWKLhPjwITQG+/dwQL16e4oNPjGDq6sEfchMGXnz0C5TAsQmeW2UZDmsK6mFOX5UlH4lk8BgkE2nLDfmzm4l59OYa8+hapoopXrw8ySeenTiyF+eBFx+bBcrjMrp8qyzDYU1B3fi8ozmT5YZ7IIJAlnwkksHB8RPRYfsne/x5LATful/jpWuL/MVb5b7+FVNX+chTo1y9PMkLp0uoR/zSdXSj6gGxWYDu97GjeKveKstwWFNQNz7vcsPtCQJVgZmhNClD25fXUJZ8JJKTT9dE6gUnu3Nlpenx8uuLvHxtkYV6f/PoMxN5Xrw8yXdfHCeXOj4h/ficdJ/YLED3+9hh3aq3Ej3HYcfLWkFwfb7BctNjNGfty2soF+5JJCeTOBY0vcTPcZLHn/fMo68t8PXZ6qbm0e95doIXL0/yxFju4A+5B8h35g1sZ+AMo5i0qXO33KKYPpjsx1ai5zCyG7vNAK0VBH4UYWjqvmUmjoMYk0gkO6drIm26AVG/SHxCuL3S4nPXFvmTY2gefRSk+NjAdgbOlhfy6lw9+fuqzdmR7L5nP45aKWG3GaC1guD0cPIz7FdmQi7ck0hOBn6YjD9veeGJNZH2zKOvLXJzaXPz6NXLk3zvETaPPgpSfGxgOwPn2ZEMbT/k3EgWJ4gORAgcVClhpxmN3YqhtYJACMFozpKZCYlE0hc3iKjZJ9dEuhPzqKWrfOTpMV58buJYmEcfBSk+NrCdgfPsSJa6E+IGMbqqHoinYK9LCZuJjJ1mNB5HDMnMhEQi6UfbC6mdYBPpcsPlD19f4uXXNzePXpxMzKN/6+L4iferneyfbgdsDMSjOXPLQH8YnoK9DtibiYydZjSkr0IikewFQojeULCTaCL1w5i/ervcmTxapV/xqGsevXp5kgvH1Dz6KAy8+NgsEG8W6E/Czb3pBlRaHsWMQaXl03QDxgupHWc0TsJrIJFIDo8oFjScgMYJNZG+vdLipdcW+ZPr/c2jqgLvPTfMJy9P8oEnRjC0420efRQGXnwcNTPnQeCFMQ+qDnfKbQxN5UpnpLzMaEgkkv0kiGJq9sk0kbbckD+9scxL1xZ4c6nV9zHTpa55dHLg318HXnx0b/tzNZu2F1JpeUdmgNh+Yekqp4cyFNI6DSfE6rRsyYyGRCLZD9ygO/78ZJlIYyH45v0aL+/APPrJy5NcOVU8kebRR2HgxUf3tj9badNyQyotn7oTcmWmgKIoR2qa6V6RTxkM50yiWDCcM8mnjMM+kkQiOYG0O5NI3RNmIl1quPzRDsyjn7wyyUefOfnm0Udh4F+R7m2/5YWstoNe+eXeqk3dCbft/NjNwK2jMp5dllckEsl+IUQyibRunywT6U7Mo8W0wfd2Jo+eH80e+BmPEwMvPrpsNFvCzjbb7mbg1lFZenbSyitHRdRJJIPMSTWRvr2cTB790y3Mo99xfpgXL0/ygQuDaR59FKT46LAxGyCEoO40tu382I1hdRDNrQfBURF1EskgEkTJJNKme3JMpE034M9uLPO51xZ5a7m/eXSmlObq5Um+59kJmT1+BAZWfPS7LY8XUoyt+fh0KYWlq+RTxqb/ce1m4JZcerY/SFEnkRw8bhDR6Iw/PwnEQvDNezU+d22RL761QhA9LKRSHfPo1SuTPD9TlBnWx2Bgo99mt+XH2Vuy1j/RT9xIr8X+IEWdRHJw2H5IzT45JtLFhssfXlvk5dcXWWp4fR9zaSrP1ctTfOyZMbLy/WVPGNhXsXtbniqmuLHQ5PpCA6C3OXG7W/RGcXF+NLtOBW86vOwEeS2OClLUSST7ixCClpeIjpNgIvXDmL+8VeZz1xZ5Zba/ebSUNvj4s+NcvTwlzaP7wMCKj+5t+cZCk/tVG4EgiATTpdSObtHbZUhkKeDgOGkGWonkqBDHgoYb0HBCwvj4i463lpq8dG2RP72xTFOaRw+VgRUf3dvy9YUGAsGl6QILNRdLV3d0i95OXOxVKUB2ckgkkoMmXGMijY+5ibThBJ3Jo4vc2sY8+r3PTTCak5nTg2BgxUf3tgwQRIKFmoumKuRTxo5u0duJi70qBchODolEclB4YUTdDmj70bHuXImF4JXZKi9dW+RLt8qbmke/65kxXrwszaOHwcCKjy79RMJOsg3biYu9KgXI8o1EItlvbD+ZROr4x9tEuthwefnaIi9fW2S52d88+mzHPPpRaR49VAb+le8nEpYb7rbZhoPyGchODolEsh90TaR1J+i7k+S44IcxX3yrzMvXFnjlXm1T8+j3PDvB1SuTnBuR5tGjwJ5HsiiK+OVf/mX+y3/5LywuLjI9Pc0//If/kF/4hV84Nmmto5RtOKqdHNKLIpEcT+JY0HQT0XGcTaRvLTU7k0eX+84aURV43/kRrl6e5P0XhtGlebTHUXiv3nPx8Ru/8Rv89m//Nv/pP/0nnnvuOb7+9a/z4z/+4xSLRX7qp35qr59uz1gbTN0gQlXYVbZhv4LxUe3kkF4UieR4cRJMpA0n4E+uL/PytUVurfQ3j54aSvPic9I8uhFVUchYGhlTJ2Noh32cvRcff/VXf8UP/dAP8X3f930AnDt3jv/+3/87f/3Xf73XT7WnrA+mMDOUJmVoO842DFowPkrZIYlEsjle2F1nfzxNpLsxj169PMkVaR7toasqGUsja+qkDPVIvS57Lj4++MEP8ru/+7u8+eabPP3003zrW9/iS1/6Er/5m7/Z9/Ge5+F57xiDGo3GXh9pR2wMpilD48JY7pG//qQH48SLAm/M1wljwenhNEKII/Uft0QyyDh+Ijps/3iOP1+su7z8+tbm0eemC1y9PMlHnxkjY0o/HIChqWQtnYypkToCGY7N2PN/rZ//+Z+n0Whw8eJFNE0jiiJ+7dd+jR/7sR/r+/jPfvaz/Mqv/MpeH2PXPK6xc9CMoWN5i+lSmsW6i6lpzFUdRnPWic72SCRHHSEEbT+iZvvH0kS6E/PoUOadtfVnpXkUgJSRZDcylnZsBqPteYT8n//zf/Jf/+t/5b/9t//Gc889xze/+U1+5md+hunpaT71qU899PjPfOYzfPrTn+79vdFocPr06b0+1pYIIRBCUEwnL8eZ4cyWO1r63e6PqjF0v1AUhZShMZZPPVa2Z+PrO5ozKbd8aWSVSHZB10TacI/f+HMhBG8tt5LJo9uYRz95ZZL3nZfmUUVRSBtar6SiqcfvPXLPxcc//+f/nJ//+Z/nR3/0RwG4cuUKs7OzfPazn+0rPizLwrION1CvND1em2v0/BqKovQC3kn3cjyOUXYvsj0bX9/pUor5mnskX2/Z4SM5aoRRTMMNezupjhN1J+BPry/z0rUF3l5p933MqaHO5NFnJxgZcPOopiqkzURspA0N9RgKjrXsufiwbRtVXa9KNU0jPqItXUIIZitt5mo250ayOEG07ga/Uy/HcRUpj3Puvcj2bHx9V5rekfXOHNd/Y8nJww+TzpWWFx4rE2kUC165V+Wl1xb5y7c3MY8aKh99epxPXpnkuenCQAv8o2wYfVz2XHz8wA/8AL/2a7/GmTNneO655/jGN77Bb/7mb/ITP/ETe/1Ue8JK02O2YrPU8FhqeDwxll13g9/p7f64Gk4f59x70Qa88fUdy1vM19wj6Z05rv/GkpODG0TU7ONnIl2oO/zhtSVefn1r8+gnL0/yXQNuHj0uhtHHZc//hf/tv/23/OIv/iI/+ZM/yfLyMtPT0/yTf/JP+KVf+qW9fqo9oXtrf9/5Ee6WW+v8HrDz2/1uShBHKX2/2bkP6owbX9/RnMlozjqS3plBMxVLjg7dSaRecHzGn3tBxJc6a+u/ca/W9zFd8+jVy1OcGckc7AGPEJahkTWTGRymPhh+FkUcsZxdo9GgWCxSr9cpFAr7/nw7GaW+E3YTrPfqOfeCzc692RmPknA6aAb5Z5ccPEIIGm5Iwzk+JtKeefS1ZG39ZubR919IJo8Osnk03REbWVM7Ma/BbuL3wF/d9qpLZTcliL1K3+9FMNzs3JudcZB9D0d12qzkZBHFgoYT0DhGJtLEPLrES9cWNzWPnu6aR5+bZDhrHvAJD59uh0q2M2X0OHao7CUDIz42C9SHEVD2Kn2/n0JgszPut+9BZhckg8pxM5F2zaOfe22Rv9rCPPqxZ8a5enkwzaOqopAxNTJWMtL8uHeo7CUDIz6O0o19r7It+ykENjvjfvsejtK/k0RyELhBd/z58TCRztccXn59kT+8tsRKq7959PJ0gatXpvjo02OkzZNrmuyHrqqkzWQ1x0nrUNlLBkZ8NN2A1ZZPIa2z2gpousGhBbW9yrbspxDY7Iz7PUxNdpRIBoV2x0TqHgMTqRdEfPFWmc+9tsg379f6PmYoY/CJ5yZ58fIkZ4YHyzxqaCoZUyNr6Se6Q2UvGRjx4YUx96s2QTnG0FQun0rMMMc5zX8YU1X3u0y1naDa6t/rOP9bSgYDIQRNL6RuH30TqRCCN5dafO7aAn92Y5m297BIUhX4wIURXhxA86ipq72R5pYuBcduGRjxYekqp4bSFDMGdTvA6rQzHec0/0k0QG4nqLb69zrO/5aSk81xMpHW7YA/ubHES68tcrvc3zx6ZjjDi53Jo4NkHj2OO1SOKgMjPvIpg5GcRRQLRnIW+ZQBrE/zz1VtZittmm6AF8ZYuko+ZezJDXq7W3m/zwNH/ibfPfdevWbbCaqtyjKyZCM5agRRYiJtukfbRBrFgr+ZrfK5awv81a0KYR+BlDY0PnYxWVv/7NRgmEdPwg6Vo8rAiI+dGChbXkjbD7m90uZB1eH0UIbhnLknN+jtbuX9Pg8c+Zt899yVlrfmNTOYLqVJGdqei6atyjJyCJjkqOAGEY1O58pRZifm0SszBV68PDjmUdmhcjAMzLvzTgyUlZZHpe0jhKA+5zOaNVhtsSfm1O1u5f0+DxzKTX433onuuYsZgzvlNoW0TqXls1h3Gcun9lw0bVWWGbTNwpKjh+2H1OyjbSJ1g4gvvlXmpWubm0eHs2Zvbf0gmEc1VUkGflkaaUMbiKzOYTMw4mMz1oqSnKVTd0LuVNqs2j5iBUoZs2dO3Q0bA3jW1La8la+/tSdvEG0vpOUFzNUEuqoe2E1+N96J7rkrLR9DU2k4IWEsMDVtX0TTVmWZk+iBkRx9joOJVAjBzaUmL11b5M+uL9P2HxZHmqrw/gvDncmjIye+xCA7VA6XgRcfa+nenHUViAWnhtM0nLBnTt0NGwP4lZnClrfytbd2N4iYqzpEsUAIGMmanB3JHthNfjfeie65m27AlVNFLF3FC2Pmqo4sfzwCsmPn+BDHgoYbdAT30RQddTvgj68v8fK1rc2jVy9P8j0DYB6VHSpHBxkV1tC9OQMEkaDaTm4yXhgjhNhVENgYwNt+xIWx3KZBfO2t/fZKi1jAzFCG+ZrDSM7a930za9mNd6J37jXnE0Ic2eVwRx3ZsXP0CdeYSOMjaCKNYsHXZ1d56dri1ubRZ8a4euXkm0dlh8rRRIoPHg7SozmTmaE0y00PQ1OZrzmM7lIAPI75cS/Hr3/rfo1qO8CPIt5zdohLO3ijeVzvxH6XP05ydkB27BxdvDCibge0/ehIdq7M1RxevrbIH76+SLnl933MlZkiVztr69MntNQgO1SOB1J80P+2mTI0RnPWuiAwtoug9zgBfC/Hr1fbAU0vYKXpoSjKjkTUUfdOnOTsgOzYOXrYfjKJ1Onjkzhs3CDiL94q8/K1Bb55v973MSNZk+95doKrlyc5fULNo7JD5fgh39nof9vsFwR2E/QeJ4Dv5fh1P4pYaXqM5i10VTkRN+mTnB2QHTtHAyEErc74cz88Wn4OIQQ3FhPz6OdvbG4e/cCFET55ZZJvPzd8Im//skPleCPFB93bJrw+V6PqBCiK4PmZIldmCrT9qBcE7pTbxyrojeUt3nN2CEVR0FWFkZx5Im7SJzk7cNSzTiedOBY03UR0HDUTac32+ePry7z02gJ3K3bfx5wdznD1SmIeHcqcPPOo7FA5OZycd+3HYCyflFfeWmqy1PBpdsxkH35qjAtjud7j9iroHZRnQVEULk0VtjV/HjcPhcwOSPaao2oijWLB1+4m5tEvv93fPJoxNT76zBifvDzFpan8kf7dfRRkh8rJRIoPkiCdMjQyps50SQOSlOvGzMZeBb2D9Czs5CZ93DwUMjsg2Su8sLvO/miZSLvm0ZdfX6SyiXn0+VOJefQjT58886jsUDn5SPHRIWfpZC2dpWbSC/9ELvtQZmOvgt5R8yzs13mOW0ZFMjg4fiI6bP/ojD93g4i/eHOFl64t8q0Hm5tHP/FcMnn01NDJMY+u7VDJGNpAbccdVKT4IAmSQgjODKcppHSK6USI3C23mK20OTOcYbyQOpD9JIfBfp3nuGVUJCeflhdSs/0jYyJdax79sxvL2ANkHpUdKoPNwIiPrbbGzlba3Fu1yVo6mqIQxPDFW2UW6y5ZU+fCWI6PPD3GWN56rJv82g2w06XUug2w+81WWYj98lActQyPZDCJ42T8ecM5OuPPa7bPH7+xxEvXFjc3j45k+OTlST5+gsyjskNF0mVgxMdWW2PnqjZLTY/3nR9mqe7x+nydxYZLGEMhpXZ2rIS9xz/qTf5RMgF7VbrY6rn3y0Nx1DI8Jx1Z5lpPFIuOiTQg6mPUPIzzdM2jf/V2pe+ZMqbG37o4ztXLk1ycPBnmUV1VyVqyQ0WynoGJBlttjT03mmOp6XG30kZTFLKmzmQhxfXFJqaa3EBylv7YN/lH+fq9Kl0cRhZCdqUcLLLMleCHSedKywuPhIn0QdVOJo++sbSpefSFNebRkxCgDU0layUZDtmhIunHwIiPzW7hCoKbC3VqLRcRx5wbzWCmNfJpHUtXeWIsxwunS73A+Tg3+UfJBOyVaHicLMSj3qhlV8rBMuhlLjfodq4cvonUWWMefXUz82jO5MXnJnnxuUlmhtIHfMK9xzI0smbSNWg+wjJOyWAxMOJjs1t4xtK5udRivubgL7cpN30uTRe4cqrImWcSN3nb70wJzZmPdZN/lEzATkTDTsTB42Qh5I36eDCoZa52ZxKpGxzu+HMhBNcXOpNHb25uHv3gEyNcvXz8zaPJiIIkwyE7VCS7ZTDendj8Fh7FAkNTmComt0VNS94gRnJJAO8XdB/1NvkomYCtRENXdKw1zOqq2lccPE4W4iBv1NK38OgMUplLiMREWrcP30RaXWMend3EPHpuJMPVK1N8z6VxSsfYPKoqCmlTS6aMmrrsUJE8MgMjPjZjNJe8EczV2jhBRBSnyVr6nng89oKtREM3I7HWMOsG8Z6f8yBv1DLL8ugMQpkrigVNN6DuHK6JNIoFf32nM3n0dn/zaLZjHn3xmJtHNTURHFlTJ2PKDhXJ3jDw4mMka/L0RI60odIOIp6dzHNpKt8TJUc5jd0VR2sNszOlzJ6f8yBv1EdB8EmOHkFn/HnrkMef31+1efn1Rf7o9SUq7S3Mo1em+MhTo8fWPKqram8lfdo8nj+D5GhztKLpIWAHMTNDWS6M5fi/X1vg1lKbWCgMZwxUVaWYTl6iM8OZI5fG7mYkHD/kwmiWM8NpcimDphsA7FnJYrsb9V6WSgbVtyDpjxtENDqdK4eF40f8ecc8+tpcf/PoaM7kE8fcPNrtUMmY2rEVTZLjw8C/s3eD3Vdvr3JruU0pYzBbtYljwbmxLFGcZD8URTly6caNGQkhBK/NNXZUslgrGLKdm83aDb7AjgXFXpZKBsm3INkc2w+p2YdnIhVC8MZCo7O2fgWnzzl0VeGDTybm0feePZ7mUdmhIjksBl58dIPdnZUmKUNFQdDwQm4uN8inDZ6dLm6a/j9sc6SiKL3g3PJCKi2PMIqZGcpsW7JYKxhaXoAQkE8ZDw1g24mg2MtSySD4FiT9EUJ0xp8fnol0tZ2YR1++tsjsan/z6PnRLFcvT/I9lyYoZowDPuHj0e1QyZg6WVN2qEgOj4EXH91g98EnR7m+2GSp4XJ2OMNUMUMYi176P2tqLDfcdUJjJzf+/RYo/UTETkoWawXDK/ccEPDMZOGhAWw7ERSyVCJ5HOJY0HADGk5IGB+86IhiwVfvVHjptUW+cmd1c/PopXE+eXmKpydyRy4LuhVKd4dKJ8NxHDM0kpOHjBIdLk0V+DvvOcXX7lQQisJoVufsSIbJvMli0+fLb5dZbQdMFVMYutYrDWwXoPe7e2PtGeZqgpGsyUjO2rRk0RVDlZZHywuYq4lOyeZh0bJTQbEfpZLDzipJ9p+wYyJtHpKJ9N5qMnn0j95YYnUT8+i7The5enmKDx8z86jsUJEcdQZGfGwXzFRV5TufHGU4a/KNezV0VcENIhabPl+9vcpKy6XuhLz43CSqKnrfZ7sA3U+gjO25QRPemK8TxoIzwxnOj2b7fr9kCFKDb9yroSmgayojWZN3ny4BD3s+dioo9qNUIltuTy5eGFG3A9p+dODjzx0/4gtvrvDytQVem2v0fUzPPHp5kpnS8TGPru1QSRmqFBySI83AiI/lhsuXbpV7wfRDT44yUVz/xpLUQzVGc1ZPLDyo2gRRzMXJAl++XeHWUpMXzgz1AvJ2AbqfQNlrg+Z0Kc1i3cXUNOaqDqM5q+/3W2l6vDJb5UHVYTRvkbeSYWobX4cuh+m9kC23Jw/bTyaROn0mf+4nuzGPfvLyFN92dujYlCZkh4rkuDIw4uPeqs3bK21KaYOlRpszw5l1QbdfOUJXVU4NZZiruizUXWZKaa6cKvL8qWIvW7FdgB7NmUyXUqw0PcbyFqM5k7sV+51SSdVmttJ+5CxIVzCN5VPbBuqWF2JqWs+vkja0I+vPkD6Sk0HXRFp3AvzwYP0cq22fP+qYR++dIPOo7FCRnAQG8B1963bRMIoRIhk+dnYky0jWYDhr9sTDxck8qrrzX/hyy2e+5hJGMW/MN2h7IVlLR1XoCYW2H7LaDh45C7LTQJ2zdIayyRuspau8+0zpsfwZu/Vl7ObxsuX2eHNYJtIoFnzldoWXO5NH+w1BzVoa331xgquXJ4+FeVR2qEhOIgMjPs4MZ7gwmqXtdQdyZdZ9vpvm77apjqwpXTw7XXzk5+1+37Sp8+pcnbYfMlNKMzOUJmVoVFoelbb/WOWFnQbqsbzFC6dLj+01WbtTZrZik7N0dK3/Tpm17KbctJ8tt9LMun8clol0Z+bREp+8MsmHnjz65lHZoSI56QyM+BgvpPjI02ObBuisqdF0A16Zdchaem/w1mbsNIB1sxJ3yy0Azo1kcYOYlKFxYSxHztKpOyFzNZt2Z1bHbgPiTgN193Fdw+udcvuRgm9vp0zNZqnh8b7zI7hB9JBw2vgaNd3gSCyok2bWvccLu+vsD85E6vgRX7i5zEvXFrk23988OpazePHyBJ94bpLpAzCPCiFYbQfYfkjG1BnOGjv+3ZIdKpJBYmDEx04CdPK7Lmh6AbOVdm+IV783gZ0GsG5WopjWya3aOEGErqq90kj387OVNi03pNLyqTvhvgbExw2+vZ0yI1mWGh53yy1mhh7eKbPxeaZLqSOxoO4gzKyDkl1x/Iia4x+YiVQIwevzjd7aejd4uKRjaArf+cQoV69M8p4zB2seXW0H3FxqEscCVVV4ZiLPSG7zLbayQ0UyqAyM+NiOpM3UYDRn8dU7q1ynScONekHrUW/xvWxD3uLsSPahzEv38y0v8X0cRFbgcYNvb6dMEPHEWFLCOjuSfSibtPF5LF09EgvqDsLMehKyK5sJqMMwke7EPHphLDGPfvzSBMX04ZhHbT8kjgXj+RTLTRfbDxlhvfiQHSoSiRQfPXrlkUobgHOjuXWlhMe9xW+XeTnI7o7Hfa6NHpPRnEm55T9Uxtn4PPmUcWDtu1v9jAdhZj0JrcIb/5u/PFMgbejUneBATKRhFPPVztr6r2xhHv34xQmuXpnkqfHDN49mTB1VVVhuuqiqQsZM/rsz9STbKTtUJJIEKT46rC2PZE0bxw/RNbU3Vv36QoPVls/FqTwLdXfdLT5ragghuL3SeuQU+0F2d+z2ufrdgNfulCm3POaqDrFg3S3/MDtW+gmkjePx9zMTcRJahbsCaqKQ4tZyk7eWWgeysfVexealawv80RtLVO2g72Pec6bEi5cn+fCTo1hHKHswnDV4ZiKP7YcMZ01OD2fIWjqG7FCRSNZx/N4RH4ONQbR7Y98YVNeWR4QQvPqgTqXl8aDqADCcM8mnjF4w3W3XRz8OcqHabp+rXwkB3lk8V255GKrKpenCulv+fvxMO/VSbHzu5YZ7oGWQk9AqbGoqLS9k6UENVVX2tURg+yFfuLnC515b5I2F/ubR8bzFi89N8onLE0xtMhjvMEk6VHTGCimyskNFItmSgRIf/Uon8zWXKBaoCswMpbF0FS+MsXQVIQSzlTZzNZuzwxkEgomixaWpwrrFctt1fewFh2lg7FdCgHcWz9VsHz+KDt1IutufYT+F3nHezuv4SeeKF0acGc6s69zYS4QQXJtLzKNfePPomUd3gtptibV0MoaGesTOJ5EcVQZKfGwMQCtNr/f36/MNlpsemgpvLrUYzhpkLZ04ElTsgKWGxxNjWS5NFR7qmtiu62Mv2E8DoxCC5YbbM/KdGc4wXkj1xM1mJYTux0ZyJtOlZG7JYRpJt+IklEH2EyEEbT9KhGTHRKooCiM58yHD5ONSaXn80RtLvHRtsZdN3MiFsSyfvDzJdx+ieXQzdFVNWmItjbQhW2IlkkdhoN6BNwagsbzFfM1lvubgRxGGpiIQzNUc/CBpIZwppfnAk2PMlpOR7GsD6067PvaC/by5LzdcPndtgTcXW1i6xnPTBb7rmbFedqfh+KQMlSCMMHSVhuOTTxlcmSmsW0Z3EG/C6/8NwQ2iHXltTkIZZD+IY0HTDWm4AUG0fybSrnn0c68t8tU7x8M8uhZDU8mYGllLlx0qEskeMFDio58JcTRn0fJCTg8nQf3GYoMgjKm6AbYXsdL0Waq7zAwlwmLtG2K/gPaob5jblVX2+ua+9vluLTV5c7GJHwrCOO4ZM4F1fpdiyqDuBpwaSjOSS372C2O5Xf8sj8Pa19wNor5G134c5zLIfhBGMQ037LWM7xezlTYvXVvkj7cwj777TIlPXk4mjx4l86ipq8nAL0vD0o/OuSSSk8DAiI/NAmL3BixEklUoWBqOH1Nuujw5miNn6kwWUz2fx1r2MqBtV1bZ65v72ue7XW4RxgJFhaYboqqJ2OlmW4oZgzvlNoYGQRRTzBhEsdg0+7KfJaK1r/ntlRaxgKliihsLTa53jIondaDXXuCHyfjzlhfu2yTS42weTRlaT3DIDhWJZP8YGPGxWUBc+3FVgelSihdOF3l7WWM4YzGcM9f5PLo8yu2+39d0z3Z9oUGl5XFpusBCzX0osO/1zX1tGadqe5wfgVgINE3lI0+N9s6mqQqVlo+hqQRRkn6u2wEjOWvT7Mt+mzvXbiBuuj73Km1mV9uc9XIEUczzp0rHbqDXfuMGETU7Gfu9H3TNo5+7tsCf31zB7TN8zNAUPvTkKC9ePjrmUUVRSBtab8roUTiTRDIIDIz46BcQx7rdLFWbc6M5FmsOy02PkZzJeCHFmeEMZ4YzfWd4PMrtfquW1dWW3zPfbRXY+/EoQmhtGWcka3JqKEMUi97m3m5W6PlTRZpuwJVTRUxNwY8Elq6uazXe6nvvh7lz7QbihhOy1HRQUFAQVDqt0/tZXjlOo9O7k0i9YH/Gn1daHn/4+hL/92sLLNTdvo95YizL1ctTfPzSOIUjYB6VHSonm+P0+znIDIz46BcQV5oe91ZtlpoeS02PvKUxlDVJ6Sq3lh10NXmTmq+5D/kKHuV2v1XL6sWpPMC6Vt6dsp0Q2mxI2FrvxHzNIYwF1xcatL2wZ5wdL6R2nUXYb3Pn2g3ESw2XUsYkbST/nmlD3/dOlqM+Oj2OBU0vpOHsj4k0jGK+fHuVl64t8Nd3VvuaR3OWzndfGueTlyd5aiK/52fYLVpn2qjsUDn5HPXfT0nCwIiPfgHxTrlN1tL5jnNDXJuvkzU1HD/i8zeXWW76rDQ95usOI5kUF6fzXJ9v9HwFWVNDVeD6fAM/ijg9nEYIseWb2lYtqwt1d9MSz3ZsJ4Q2+2Vc652IYkgbGq8+qNNyw8dabrff5s61r2PWSgJKHCtYusq7z5T2vZPlqI5Oj2LRWWe/PybSu5U2L722yJ9c728eVYDzo1k+8dwEP/SumUMfIy47VAaTo/r7KVnPwIiPfgExZ+noqspSw8MLBFZOZ7XtoSnwZGfdfRBH+FHEG3N13lxqUW56rDQ9PvTkCDNDaZabHoamMl9zGM1tPbJ7s4zA42YJsqZG0w14ZbYTjM31b7Tb/TJut9fmqLH2dez+rAfZ8nvUZobsp4m07SXm0ZeuLfDGQrPvY0ZzJldmirzrdImRnMUzE/lDEx6yQ0Vy1H4/Jf0Z6H+VbhC7vtBAQeHiVJ4bC00EsNTwKLc8nhrP8u4zJW4tt3qll2tzDQxNYaqYQlOhmDGotHyabtATH5vVHftlBPYiS6AogNL53w1s98u42V6bo/pLe9hts/1E5GHUmd0gmUTa9vbWRCqE4LW5Oi9dW9zWPPrJK1O863SRmh3u2yTU7ZAdKpK1yJk+x4OjGV0OiG4QAwiiOgt1l6GswVTJQlGSMkUhbTCas7D9iJtLLWw/YrnpcH81ERwPak6nEyRmKGP0jJgHWXdMbv0GT08ku1Xa/npz4Xa/jN3XYeNem8P4pX3UIH6Qwb+f+DnI3TFtL6S2DybScsvjj15PJo/O1fpPHn1yPMfVy5N898X15tH9mIS6GbJDRbIVh305keyMgRUf3WDVdAPcIKKQSkxoZ4YzNN2A+ZpLMWNQt5N09pnhDE+MZblbbjOWT3H5VIm7K8kY9tGcxfWFJvM1B1V9Z9vt2lJH0w0QQmw6wvxxfg43iFhputTtgKGs8VDGYqtfxn5B+zDNWY8q2g7bZHYQ7cUNd+9NpEEU85VtzKP5lM53Xxzn6iGaR2WHikRystgX8TE3N8fP/dzP8dJLL2HbNk8++SS/93u/x3vf+979eLpHohusutM7Tw9lGM6ZKErSTvqg6nQGa6lcOVXkwliKDz81xpnhDLMVG9ePyKUM8mkj8R5YOufH8j2vxMZShxfGfONemdvlxFfxxFiWDz819kgBcq1gcIOIuZqdZF/imJmh9CN0ytSotHzCWPDuMyUuTRV6n9ssk/Coc0622iEDjx7ED9tktl915igWNJyAxh6bSLvm0T9+Y4ma0988+m1nh7h6eZLvfHL0UDwcskNFIjm57Ln4qFarfOd3ficf+9jHeOmllxgbG+Ott95iaGhor5/qsegGq0JapzbnM5o1WG1B0w2wdJXTQxkKaZ2GE2Lpai97MJozyXbadE8PpxnJmtyvOg95JTaWOppuUpsvpU0gmQ76qAFy7S1/peliaCrPTheZrzmkdvAmvVY4VFoe5aZHy49Yabg0HJ92R0zN1xyiOAkCV2YKKIrS+3kSX0Bj13NOvnSrzNsriQC7MJrlI0+vF2CPGsQP22S213Xm/TCRtr2Qz3fMo9c3MY9OFlK8eHmC731ukslDyIDJDhWJZDDY83fo3/iN3+D06dP83u/9Xu9j58+f3+uneWy6wWp2xWah7rDUdMlbOlMli6cn8gznTKJYMJwzyafeqW2XWz7zNZcoFizUPcbyKd57bnidV2I0Z/adZJq1dJaancxHLrsuQO4mk7D2ll+3A4I43lXQXSteWl7Aqu2zUHNRFbizEpAxNTRVXSdq7q3a1J2wJzaKaf2R5py0vJBS2gAU2n0E2KMG8cM2me1VnXmvTaRCCF6dq/PyNubRjzw1xtXLk7zrTAn1gDMMskNFIhk89lx8/O///b/5xCc+wd/7e3+PP//zP2dmZoaf/Mmf5B/9o3/U9/Ge5+F5Xu/vjUb/XRB7TS9YuT5pU8ULBJW2z6sPajw9kd80kLW8kDCKSZs6d8stiul3fBLdwNPPfDiWt/jwU6OcHckA9DbkdkXHbKXNvVWbbKf9d6tMwtpb/lDWYGZod+vs14qXuZpgNGcxX3Oo2QF+LChlLTw/XidqgHViA9h1piFnJQPAlhrvZD5240/ZisMyme2V0XWnJlIhBKvtYF1nSb/nW2l6/PEbW5tHn57I8eJzk3z3pfF1AvsgkB0qEslgs+fi4/bt2/z2b/82n/70p/kX/+Jf8LWvfY2f+qmfwjRNPvWpTz30+M9+9rP8yq/8yl4f4yE2M1bODGXIpwwMLdnt0nRDbiw2uTRV4Pxo9qE39u7CtVfn6snfV23OjmTXCYXNBMpEMc3EhiVaXaHyYLXNnYrNpak8Kuq6tt2NjOZMpkvJXpq149BXmh53yu2H9sZsDIxrxYuuqpwbydDN7L+x0KDW9pguZdaJGiEEdafRExtnhjPryjA7ET1jeYsPPTnKmeH1Auw48zhGVyGSSaR1e+cm0tV2wM2lJnEsUFWFZybyjOSSLpMgivny7QovvbbI1+72N48WUjofvzTB1cuTPDH+8Ebi/WJth0rG0NCl4JBIBpo9Fx9xHPPe976XX//1Xwfg3e9+N9euXeN3fud3+oqPz3zmM3z605/u/b3RaHD69Om9PtamQeLMcIYnx3K8+qCOHURoCizWXIJI9A0kSTtqhrYfcm4ki9NnGNdOBEqXbhZiKGvx1btVvDBiLJfiuZk8S3WnrzlzbelnvuYymksC+GZ7Yzb+zBvFy3DGoOFGhFHMlZkiZ0cyvfHqXfElhOB5RaHpBnhhTMsLyaeMvgJtMxRF6SvAHoWdZhz2uwX3UYyuj2Mitf2QOBaM51MsN11sP6RRDnj52vbm0U9emeSDTxyceVR2qEgkks3Yc/ExNTXFs88+u+5jly5d4n/9r//V9/GWZWFZ+3/73SxIjBdSvO/CCF4UU254hDGMFyxWmj5vzNcptzxMLelWaXth5wankjU17laSLMPGiaI7EShdulmIWttjPJ+MV1c6fojrC82+3TFb7YiZLqWZq9rMVtrYfsRqy+fiVJ6FukvTTQLTbKXNbMUmZ+nM11xGsua6MtNozqTc8tdlUdbORLnzGDf9vRICO8047HcL7m6MrkEUd8afP7qJNGPqqKrCvdU2ry80+M9fmeXWcqvvY7vm0U88N8nEAZlHZYeKRCLZCXsuPr7zO7+TmzdvrvvYm2++ydmzZ/f6qXbFVkHC9iPSusbZ0SxvLDT46u0KXhhzu6LhBzFThRQLDZeImKxpMJIxUFUFVVHoF0O680JmKzZ3O/tjugJlYwAezZm96aK5tNHzfCiKsml3zFY7YrozRRbqDm0vZLWdzBcZyVt4YcydBzVuLCblk/edHwElGVJ2YSy3pWelG7Afp6V1s7beRwlQ252j+zpfX2isE2B73YK7E6OrG0Q0Op0rj4MQgvurbf74jSW+/HYFv0+p5jDMo7qqkrVkh4pEItk5ey4+/tk/+2d88IMf5Nd//df5kR/5Ef76r/+a3/3d3+V3f/d39/qpdsVmQaK72fZOxWa50/HS8AKCSGDoKvN1h+GMTtsLMHSVKBLMVR3OjGZ4z9nhvhNFu/Qbeb52HXzLC3sljm87O7SuY0YIwWzF7tsds9l4724ppe7AStOlmLGIhE/KTAysTTdIAn8kqLQDvnJ7lfeeG3rott50A1ZbPoW0zmoroOH4AL25IqrCI7W0Jq29Pk0vpNz0EEJsuw9nM7bLOGyc4wIwnDP3vAV3K6Or7YfU7GSI3eOw0vT4w9cXefn1ReZr/dfWPzWe45NXJvlbFw/GPGpoam+pn+xQkUgku2XPxce3f/u38/u///t85jOf4V/+y3/J+fPn+a3f+i1+7Md+bK+faldsFiS6A8IuTeXxwoh3nS5RbnrM1VwsPdlc2/ZiFEVhvu5gaD5DaQMhkgCsKsnN9vZKa10pYbOR590be9rUeXWuTttfv0G2ez4hRN/umM1+lpWm1/OBlJse7SCiRNLeO11KM15IJZ0SnbbaM0MZCimtr+nTC2PuV22CcoyhqUwPpbhbcTqZEHbdXdMlZ+mEnfON5S1MTXvkTMR2GYfu63xpOhmYNlG0uDRV6D1uv7wgj2Ii7UcQxXz57Qqfu7bI14+IedQyNHKmTtrUDn1jrUQiOd7syySm7//+7+f7v//79+Nb7zk5S0dTFJpOiB9GvDHXIJdSmSxYFFI6xlSBU8UUq22DtKEwVcqQT+k8MZZjNJ/CDaJ1w7i6ImKzm3lvg2w5qdOfG8niBvFDQXi35sy1ZYia7aOoYBkqT+Syve4SgJShoqoKQSSYLCZZF0VR1gXjlhswM5SilDGp2wFhFK8rcaQMjQtjuw92Y3mLd58pIYTA1LS+o+B3ynattd3XeaHmMpJLhMfaDMteloAg8ds03YCGExLGjy467pTbfO61Bf7k+jL1Tcyj7z2XTB7db/OooiikjCTDITtUJBLJXjIwu1363XS7H49FzP3VFvcqNg3X5/RQliszBTRNJWcm49bn6x6xolBzQoazFudGc4wXUtxeaRHFPOQ92OxmvnaDbG7VxgkidPXxN8iuFTvDWZMrp4qdWQpJSvz2SotKy2OykOLCaI67lTbnRjOM5kyWGy53yy1en2+gKhDFUEjrKCiMdMoi8zWXuZpNuzMV9VGyBYqicGmqwGjO2vdhYDvJjOxFCahrIm25IfEjmkhbXsjnbyzzuWuL3FzsP3l0qpjixecm+d7nJvbVPKoqCmlTS6aMdsytEolEstcMjPjY2PVwZaZApe3zjXs17lfaXJtvstz0iEXMg6qDqas8KRQiAWesDLqmMJXL0HADCunEKAqbew82u5k/ygbZnZQINgbbbsfK2uFlrc7NXFMVspbOmeEM5ZbPqw/q3Fio8/pCkyfHskSx4NRQmicncmRNDSEEbS+kavsIAZWWv65UtBsOahjYTjIjj1MC2omJdKuBYEIIXn1Q53PXFvmLN1fw+kweNXWVjzw1youXJ3nX6f0zj2pqIjiypk7GlB0qEolk/xkY8bGxO+Leqs3NxSYPqg52EOH4EQpgKEkQqtsBY4UUi3WHctMljAUPajZZy6DhhJRbfk9E7Gas90YhsZM5GTtpF90YbLsdK3NVm6Wmx/vOD1OzfbwwImPpPRNs93UZzaeI5xv4UYymqgxlTS6M5VhuuL0dLkt1h1U7oJQxCWLBuZH0I7et7vf8je141BJQ2wupOzszkfYbCBYLsa159JmJPC921tbnUvvzK6qram8lfdqUhlGJRHKwDIz4yJoaTTfglVmHrKUzlNExNY2xvMWdlYCpksVSTWAHMTrJTXB+1Wa8aDFTTLPYcDtGzTRRJGg4PkKIdUPAdhJAdzp3Ym1wLjddyi2XUsZMSgVbTD/t0hUV50ZzLDU97lba6KrKSDbFpel3TLBJ5gYajk/O1FEVhQujmZ5PZK1oe2OuzqtzNQxNw9QVLk3meXKi/5nXZl/6CYz9nr+xHTspAXV/noYbJMJUUwl3MRSsOxBsOGPyxVsr/M+v3efafH1z8+izHfPoI/hpdkK3QyVjarIlViKRHCoDIz6SLoSkhTRGkDFzDGWTlsSnJ3I8M5njb2ar3Fu1iYWglE5S5NPFFE0vZLHu8dZyi5VWsgsmk9Jw/Zg7lYeHgG183rUBudmZarndnIy1wXm+ZnN/NSkFGZrKlc700n4/Y/e53CBCU8HxQy6MZjk7kiFr6cxVnXUlorG8xXQpzULN4dJkActQeHb6HSGwtqwEMcPZZPHeg6pNuKGbY6OgmC6leh04ezkvZK+yJtuVZhbqDl+9vUrbi0Bh3SjznbDS9Hj59UW+eb+G3acd+x3z6BQffGJkX8yj3R0qskNFIpEcJQZGfNyvOqw0fUppg5Wmj+1HvHC6RMsLcfywV3c3NJWWG3BrpY0XxYzkTVbbPlEcUXd8IAZS3FpsYOr6Q0PAxoRgueH2MiIZU2O+5hILegF5JxMx1wbnhbrNcNbgyfE8DSfE6hNEhBBcX2jwymwVU9MoZXRODWceaondeNNPOho0xgvpdd0s3WC+tqyUtTTi2xWqtk8pYz4ktDYKipWmt6nA6OeV2amo2C5r8rjipLvO/tZyi6YbrhtlPsJ68bHR12FqCp9/c4WXtjGPXr08yfc+O7Hn2Z61O1Sypt5bCiiRSCRHiYERH0IIbC8iikTP3NcNyK89qHG73MYLImYrbYSAfNqg4QS8vdRE11WCUFBt+wQRjORAoKKqUOsM4OoOAVtpenzpVpm3V5KMSD6lM5KxeqUOS1cfMoYuN9wtl7/lUwY5K8nEDOfMvkOkVpoe37hX40HV6f1cT0680xK7VUDeamDX2uzAuZEMw1lz3UK7tWz8PmN5i/ma2/f79vPK7LQUs13W5FFLOo6frLO3/cREmjaSbo/lpovaGRu+kdV2wPXFBrdX2nzjXpU3FhoE0cN1la559JNXprgyU6Bmh9h+0nGz2WbanSJ3qEgkkuPGwIiPtKFSbXtU2x5DWYu0ofaC1P2qTauz90TXVHRVwQsjFBSCWGApClXXp5Q1KaQMohgsI1kJrygKpYzRW8R2p9ym5YWU0gag4AUBlbbLK7PJMLOcpfc1hm4MlOsyDh1DYNej0c/U2vJCdFVhtBPELX19++76gJy0BnezIt0R79uZZlVV5dnp/iUf6N9xs5mnol/JY6elmO2mm+62pNPqmEg3rrMfzho8M5Ff162yluWGy//8+gM+f3OZqv3wTA5IynEfenKUjz0zzunhNIqiUGn5m26m3Slyh4pEIjnODIz4mK+7NL2QlGnQ9ELm6y7ZlEkUCy5PF7mx0KDS8jp7WFRW2xGWnqSwz49kGc1bPDmRp9zySBkqacNAoKCpam/mBySBMWfpLDXaCCFI6WoS3NyAQkqn3PJ622mTEept5qptSlmLWtujmF6/yG3txNNu5gJ4qJSQs3RGciYCgR9qmJrK3XILIcRDy+jemK+zWHcZy6d6bcc7DV47MZWuzTJs/Bn6ZXnW/gz9RMVm+3A2E0s7WfYWx4KmG9JwN59EqigKIzlzXanFD2P+6u0KL19b4Gt3q/Szn3bNox+8MIIbxsSx4EEtMTqP5My+m2k3lnP6YWhqMn9D7lCRSCTHnIERH44fEccCXYeaHbJYd3jX6SFUBd5cauKHMaWMiaGrtNyQrCUQioKhqVwYz+GFMZWWz1g+xVjOJBYwM5R56GY9lrf4zidGyKd0Fusuiw2Hhh1S9wIWUFBQGM1ZTBTTrDQ9Zis2t8s2K3dWGc+nyVpJFmVjmWC7UkKSdSgl22y9iLuVNndXbZ4Yc/jwU2PrAnIYJ+2la9uO6064ozLFbkyl231t/5+h//6dfl+3WTZjNGf29tyM5a3eTBaAcM1m2d0MBXt7pcVL1xb5kzeWaLgPz/ZQgBdOl/jBF6Z6k0fvr9rMVuyHREZ3M+1W5ZwuskNFIpGcRAZGfIzkLCIhuFNuo+sqNTsJIDNDaV6fr1NImahpaPkRKTNmJJflzEiW4ZzFVDFFIW2uW/r22lxjU4+EqqooKDScgPmqQyjg3qrNVD7FYiNZZDdRTPe+36WpPLW2x0hGY7Xl8Ve3VpgZStpdu1mSbuZiqpTi+nyD6wsN4OEMiO0nM0uKaRNFoWeEPT+a7QX208Np5qoOc1W70xkT4gYxl6YLLNS23vy6G1Ppdl/bb6T8Zvt3ul83V7OZrbS3NJOWW35PEM3XXEZzFsWMQd0JaHvRjtfZt9yQP72xzEvXFnhzqf/a+q3Mo5uJjO3KOVZnMm3G1GWHikQiOZEMjPiYKqa4PF1kte2Ts3SKaYO2H5EyNKaKaXKWzmzFJhIxGSNNxlRpuiEzpTSFdNLZ0e1kma20iUXM0BqvR5duKeXmUoO6G+AEMbYfkdZ1Tg1lMI13gknO0tE1laYXEQm4u+pSd3yGcyajuTYXRrN85Omxdbtirs83eFB1Ej9KVO9lAdZuca20fWIBGUvrGWHXBvbuKPHZSpu2nwiP7ubXkZy1ZefJ2gyKqiTeg5WmS90Oth3UtZNyyHZf1/ZCWm7IajvYNNOyVqzcKbe4U27veIx7LATfvF/j5WuL/MVbZfw+k0ctXeXDOzCPbiYyNpZzujtUMqZO1pQ7VCQSyclnYMRHPmUwXrSoOQGRgKz1TqDseiXyKY2a47NUbxEJheGMzvsuDPfS9itNjy++VeZ2+Z3ZHudGc+tu3t1SylzVZbXtc2Y4g6YqqIrCeN7qpdBvr7TImhpXZgroKhALDE3hlftVhtIGpbSZBNoNu2KuLzRQUHhmKseNhWYvA9KdH3JpuoAQgrSZlFX6ba3tCpGWlwTxqWIKBWXd5tfNSh1rSyNuEDFXszE0lSCOmRlKM5a3NhUuu50G22Xt11VaHpWWv2WmJWtqeGHEqw9qCGCquL2fZanh8kevL/Hy64ss1DeZPDqZ55OXJ/nYxfHefztbmUf7eUbW/htkOjtUMrIlViKRDBgDIz6EEIgY0kYyvfTSVK4X/LpeifurDqstr2cSDCKdt5ea3BjLcWmqQMsLaXvhQ7M9NnZs5Cyd918Y4Su3y+iqynQpxfmxHFPFFF4YP7QF99JUgXLLZ7HmoAiFxYaDGwouTxce2hUjhKDc8vjiWyustpIOiyASTBUtmm7A4qxNLODCWLaXldnMTNqd+rpYT8yQFyfz2w4BW5tBub3SIo6ToWRr54Ns1sHzqHtd1n5dztKpO2Hf7EkUCxpOgBNETBXTFNMhGUMDIbi/aj+0XyUxj5b53GuL/M1sf/NoMW3w8UvjXL082XeT727Mo3KHyu457DH8EolkfxgY8XG/6lBu+0wWs9QcHyeIex0n0N3Z4REKQdsN8EJBxtS5X3X5m7urjHbKEVlLZ6nZyXx0Shpr6ZZSAJ4az+MGIV4guLPSYjhrYunqQ1twM4aaTF9t+6AIJgspdE2jkE7KH0KIdW+4QoAXxERxjKWrzFVtojgCkg6OB1WHlhtwb9Xhw0+NMlFMb/q6KAqgwMb38+7Y9Tfm64Sx4PRwuneObkCotDxaXsBcTazbzPs400u3o1/2pDsUrOWFvX/PbsahX2aiZvuJefR6f/OoqsC3nxvm6uVJPvDECMYWZZDtzKNrd6ikDFUGzl1y2GP4JRLJ/jAw4iPZzBoQRTFu+I7psPvmdqfS5m7ZZrHu0e4EpCCK8MOIuZrDbKXNt50d4sNPjXJ2JNl7cmb4nZX0TTfAC2NMLekAsXSVkZzJ7ZXEHDlXc1Hv1XjX6SItL+CVe04iZkyNe6s2K00fQ1dxQ8FoPk3VDlise7ymNni+c/OHzqyPlM6TE3k+f32JL9xcZqqUxvZDhrMWo3mL1+YbFNMmt8ttznRmS3TPZ+kq+ZTBWN7qzA0xeHrinV0vXcbyidH2zcUmsRC8PldnJGv2unRefVAnjGKEgJGsyZnhDEIIbq+0eqPdd+vt2AlrsyCOH7HU8HpDwbr/zmsnjrb9gDgW5CydP7uxzP/7i7e5W7H7fu/pUtc8OrlpSWjj9x/q4+uQHSp7x34KWYlEcngMjPjImBp+GLPS9CimDTKdwV3dN7dTpRS6qmBoCoWUTiBiCqlkJkjNCbi3anN2JMtEMb0uk9AtMVRaHg+qDqeHMgx35lDkUwY3F5usND1G8xa6qtD2QoQARJLBWIula6iKwmLNwTJ1zo3mcIPoobHkbS/k7ZUWipp8n4uTedwgJowFVdtDVRRMTaHhhdxYbHK/6hBEMXNVd935tptsavsRLT+ilDa5U7E5t6ZLJ4pFr9V4JJekwrs3VFVJuog2jnbfC4QQvaFg/cygazfJokCt7fMnN5Z5Y77Rdymcpat819NjXL08yfOnittmJvptqh3JmUwbadmhsg88qklZIpEcbQbmN7nlBjS9EBELml7Ym2iaNTVaXsBC3SEmCUalYorFmkvNDihkDC5O5PDDiL+6tdJbP15KG+RSRq/8UEjrBOWYQlonikWvvfU9Z4dQFAVdTcyHiqKQTxk8M/lOtuH0UJrRrMlqy+PiRI6LkzmaXkzb9VlseDh+wHzNYbJgkU8ZnB5K03JDnpnIc2OxSc0OmC6lmRlK03IDsqZOywvQAoW6k3yPM8Npgujh8+3MALo+aPcLCBtvqClD6+uReFTiWNBwAxpOSBj3HwoGiQdjtZUsAfzirTK1TSaPXprKc/XyJB99ZnxXAW2tx6Nm+1i6ypnhzEMdKtKrsDc8qklZIpEcbQZGfNxbdZit2Kgkq+HurTq8H4jjmLmqzd2VFqaqMJo1cIKIIIywFUFkw5feXsHQkoDbcpP9LnnLoJQxuDCapelFFFwdP4q5tdJkqpjcgvutbRdCUHfWzwgRQlDMGGha8vfnT5WoOiFfv7vKW8tNNFWhZge8cKrEUNakkE68J6au8uR4jjPDmZ65VAhBLmXw6oMaKT3kyYkcX7tbpdzyMDSVhhMylDVwg4g75TZZU+sIsIcnp54ZzvDEWDYRKpkMbhDx5zeXGc2ZXJ7OYwfxuoCw2Q11u0C81eeDzlCw1jZDwfww5i9vlfnf35rn1Qf1Tc2j3/vsBC9enuT8aHbX/w0pSjIgruEk3pLRvMVkMdW3NVZ6FfaGRzUpSySSo83AiA8niDotqDotP8Tp7PF4bb7BNx/Uk+FcXsBozkQJBYamMdYpJ9TbIflUkr1o2BF2EOIFEau2x3QpRSGlM5Y10BQFVVlfTum+eY6tCbBdT0jXe3Gn3F7nvbhfdbi36vDmQpOFusvl6SLLDR8vjLhdbjORN8mlDEZz1kMdLStNj/mamww5c0OW6x4XRrOcGU6TSxlYuoobRLwxX6ftR8RCkLcM8injoSA5Xkjx4afGaHlJd8lX3q4QxgJDU7l6ZXLdnpetbqjbBeJ+ny90Fvt1RdFmvL3c4nPXFvnTLcyj33F+mBcvT/KBC+vNo13/RtsP8EOBqStkTWNdR4yqKD3DaLJDJflZt7uJS6+CRCKRbM7AiI/JYorxgoWuqGRSGhMFi+WGy51yi2orMXsKoGoHxICiwv2aw1Da5OxoholCijfm66y0HaJYIQgFigJvLTU5N5rn7EiGtGUmUzirNvdW7aSNteERhIl3otb2MXWNkZzJ86dKvdZZN4gotzxqts9IzqRmB9wut7EMjYYb8tZSC1NXaHsRiqp0vCAxIzlr0wFbl6YLvZ+7O7ujG1C/dqfC7bJNKW1wp9xmppTqlYHWBsm1t877qzZhLHhmssDNxQYrTW/d8251Q90uEO92KFjTDfjT68u8dG2Rt5b7Tx6dKaUT8+hzE4zm+n+vrn+j3g5YaDhMFlKUsibPTRU4NZwhZ/XvUNnJTVx6FSQSiWRzBuYd8fmZIjeXmizXPcaLFjOlNK8+qNN2IrwwouEFKEDe0kjrOmpKAQHPzeSZLKQTE+FUgVLGpG4H1J1EsFw+VURFxfZDIgFztcRP0fZDvvkg4Fv3qqR0jXLLYyRn9YaAdUeEu0HEg1UbQ1XxwpCUoVFuurS9kMm8yYXRDNOFFKdHsuRTGi0vwgmida2ta+kGvYWamzzfVGGLdL+CpWtomrppkOyWRLwwwg9jbizUMXVtV7X37QJx1tRwg4i/fKuM23kNRjv+mG52ouUF3F5p85e3ynzxVrnv2vqUrvJdz4zx4uVJnp/Z3jza9W/kUhpxTTBRtEjpOsWM8djeAulVkEgkks0ZGPGhKAo50yDICHKmge2FVJoeupbU8aeKFmEElbZDuR0RRRFnRrOcHc5RSJt4YcgLZ4bILDSYr3s8MZ7FCWL8IMaPYgrppGwxkjUZzhjcLrep2z5NN2RqPMVKy8PUYbbcJmWoOGEyCKvS8jFUlUvTBa7PN5it2Ghq4p+IgEuTedp+xL1Vm6GMwbefG8INRUcUJC2+3fLNZlNEN3oqTg+luTCape2FPDdd4NnpPGlT7xsk15ZETo+kGc6YPDWR5+Jkfsev/WaBeO1QsLSp4UWJqFpuugxlTEZyJjcXW/x/XnnAK7NVas7W5tGPPTNOdhcZhlLGoNpO/lsYyVnEMaRNbU+yFNKrIJFIJJszMOKjO2SslDYpt31mV11uLjW5vdKkavuoSjLn4VQpy6khBZRkA2rN8XlqMs9iXVC3A/Jpk7QTcno4w1AmmengBYKLU3kW6km2wQ0i5qouC41k4uV83SVn6WQsA8+P0BWVlYbHhdEcmgrllssrs8n01OGc2fNSpA2NB6ttvnm/RiljcWulnWQRNJWFukOlGTBdSpE2Nd5zdohLU4W+QW/jxNHL03menS70tr5enMyjqv27Na4vNKi0PC5NF1AVlacmcrvuYtl4pmQomL9uKJilqwxlTMbzKeZqNn92Y4m/vlvllU0mj5bSBt/zCObRlNGZMGpp6KrCVDHddwbKXiI7XyQSiWQ9AyM+ukPGwjCi6gQULRVDVxjOmlTaPvcqNnYQUUobZCwNU9dQUWh6EV+9s0re0hjOWjx/oUQhZZAyVaaKKdwg4rW5Bss3bdKmTimtUXMCLB3ee6bETDHNZMHkzEiOIIxYbvhYhspX71T40zeWMHUFVVVImyppMwmICzU32ZcSCSptn6odMpZP0fYFD1ZtQhSiOOZOpU0kIjKm0evE6JZY1ga8SssjjGNmSpmeobXuhOu2vm4szSw3XL50q8xC3WG1HSAQjOZSj5UVcPyIuhP0HQpWs31ul1v8n9fmee1Bo2cIXosCvPtMiR961wzvvzC85eTRd763T6Xtk9JVnp7IJ3ts1gT+8UJq37tQZOeLRCKRrGdgxEfG1HD9iDdWarT9CF2BSAhuLjWYr/soIiZWoGH7PDNZQFMS/8ez0yVqdrLITFUVFuoumqawavvcr9o8WE12qQSdaZ/LLZe6HWJqKiutgMmCxbvODHNxMpnJ8cZ8gzsVm5WmQ7UdkEvpWLrGs9NFLE1FUxUsQ0UhmcdxZabEzaUWCzUnmeUxnGZ21SGMkjHwi3X49vNZdFVZZ+RcG/CaboCivDNxFNi2E+Peqs3bK22KqeQcaUPj+VPFXWcFthsKNlux+f9+Y46/uVtlodF/odt43uK7nh7j45cmeHI8u23WQO0sbWt7IXcrbe6Uk4mm5Zbf2xJ8kAxa54vM9Egkku0YGPHR9qLOLThps52v2YxkU2iKgq4I3BCqbZ9CKpmBEQsFiHhzucWF0SwvnC6hKEmAv7Xc5O3lFi034K2lFufHcmRNnTuVpEOlbvs8PVGg3HKJhGCuZrPa9rlbbrPU8FhquChK0lmTNlQW2j5/cWORy6eGURRQVYVYJN6UKI45PZQhY6qcHs7y7FQB2495c6nJZCmNoSbG2JGcuS4rsTbgzVVFsuuks5+m36yRzVAUlYypkTbemQUymjMpt/wtg0scJ3tm6k7w0FCwWAhema3y0rVFvrSJeVRVEtHx3HSBH3h+iiunSpsGsOTnCYgFjOUsTg+nUVWV2ystbD+imDJoeSGz5TazI5kdnX8vGbTOF5npkUgk23Gy3wXXUHN8lpsufhSjqVCzI8YKCpPFNJBsYo2EIG2oLDZcdFXlfU+MoCGSOR55C1VVGQcqLQ8niKm6AV4Uc3ulRdbSQEkyLE1Xoe0FlLIWV2ZKLNZdFmp13FCAmrwhu36M3dlNIhSFCJVK26PphhTTJuWWx3vOlJgspni3MsTFqTw3F1pU2gEzpTSqAmdHcyzUkm2txbRBHMcs1ZOpqW4QoXayHbqm9uaBdG+kU0WLthf2Oko2Lq87M5zpmVLHchaNjtDS1GR3zXzN7Rtcws5QsGafoWCLDZeXry3y8rVFlje06nYZy5lkzaSbZtX2CaOYhbrHzFDQW1XfLdV4YcRoziJtqKy2A6JYULMD0qbGeCEpEeUsndsrNZabPuN5i3urNllL3/T8+8Ggdb4MWqZHIpHsnoERH6zZyBrFUMzofOTpMVYaHq/N1RjOWoRRCKj4UYwbxrz2oMaF8RxNL6Tc8nsB6sxwhomCieMHPH+qQLnpM5QxcPwYRMzFqQJPj+do+0lbbBgLcpZJLg1vzLt4QUjT8UkbKpoqyKYM3nduiFUnGaNuBxEtN6Tc8nhupkgYw82FFverNoIkYOZSBq4f4QYx1baDFwjultuoqkLOMtDUh/errL2RtrwAIZJb+WzF5uxIZt3AsvFCio88PdbzjFTaPlPFFDcWmpRbLipqz2Tb8kKKYeLnaHsRcRz3lq/pqsobC3Vefn1pU/NoMW3wbWdKTBYsFuout1ZazNeS9uMnJ/JkTK23ql5XVRw/ZKXpJSIucCh2RsZvDHZjeYsPPTmKrircr9pcni7idvb7HGRwHLTOl0HL9Egkkt0zMO8KsegEbUsjFvChJ0f4vitTlFs+z58uIYRgse7w/3t1gWYjROusmb84nieMBNcXGkAS0MYLKb7rmXEK6RpV26OUtnh6Is837tfIpXSmi2ne1SnT3Fu1URWoqQFV28P1Q2IBQlEopA0MLVnD3vIjJgspHD+m0vK4OJlnOGNh6SrPnypyfaFBLGImiilmV1pMldKUMgZ3yi3maw6LDYe6EzCSNfnI0+O4YUzK0Dg/mmWl6XGn3E6Mp1HMzFCGV+45IGAsn+LVuTptL1met3ZUezdg5iyduhNyY6HJ/apNIWPQsBN/RjaVCIO5qtN7rVfbAZ+/uczfzFZ59UG9r3lUVeB950c6k0eHWai7fP3uKmEskmm0gKkppHUVy9CYKqaYLiVi6vZKNwOTiAfoP9pdURQmimk+8MQo2Qd1vFCgqwqaqrDSdKnbAUNZ48gHx+PmoRi0TI9EItk9R/tddw+pO0HiP4hidE1N5nJoWi+bcW/VpmoHCEBRBKt2iO2HvDZfZzibpPuDSPRS9N05F28tNam0fGbLTRpOwKmhNFEsaPsR+ZTR6yppB2GSFUCQNpOOGBXB2aEspazFeN7i/RdGuDTl8417NUxNo5TR8cIYxQsZy1ss1V2+cGMFL4yIEAjSyQj1lseDVZsgFoznLSLg+ZkSOUvvm+2YrzlkTY2mG/L1uxXafkAhneUb92q8PldjLJ9kPZ6dLna6aEymSynKLZdCxuDbz5Z49X4dXYPxnIXjRVRaPlEs+Ma9Kv/Xtxa4t9p/bf2poc7k0WcnGM6arLYDFuouXhiTNnSCCExdZTRv8eRojnedKfHkeH5dwN14sz4znOn5cfoFu7XBMGmDtpNuojhmZih95IPjcfNQDFqmRyKR7J6BER/lpofjx2iqguPHlDueg5WmxxffKnO73Ga+6tB0AgxVJ458IlXl7kqLsdwIF6fy3Fho8sZ8nXLLo+n4vD7fRFEEi02X5ZpDuRXyda/KzFCKmaE0D6oOlZbHRMFipenj+SFNN8ILYsIoRlEUnCjmVEojjEFV1d6sjuWGS9ML+Zu7q5i6xnDOAAX8KPE5zFZsKi2fuu1TtX1MXSWnKUzkU+RNnTPDmd7emJ7xtCYYySbGU8cPeWO+QdsL0TyVm4tNHlQdMqbGfN0DRWEsn7Shlls+8zUXhMJK3eXLb6+STxucHs4SC8FL1xb5+t1Vri82iTZZW/+xZ8a5enmSyzOFnoiotPw16+mTabKXpnK4YcxI1uTsSJbxQuqhW35XDHXnlKz14/RjbTC8vdIiEsnY+buVNu1tdsccBaSHQiKRnDQGRnyINf8XRO//a3khLTdAQ0FFEMURi3WXphsyk84lWQsv5PM3lnl7pcVkIYVlqISxYLbiMF1I8dZKExVBytJIGQphJLhTbmJpBg+qNnfLbequzzMTecptj3LL48xwhjgWGGpiem25AbOVNnEcc32hwULd4fZKm4ypcW40GeqVMlRKGZN8yqDc8kjlVN53YYRV22Op7uHHUHMCrpwqcnYkaUldmyXQVbUX0G+vtCikTZ6ZLHBjoUnd8SikdFRFJWVCFMW9IFd3gl6JotgyGc6a5FI6//tbc3zutUUqbb/vaz6et3jPmRL/z28/w6mRTM8oavshGVNPPCEKlPIpFuqJcfa954a37GpZaXrMVtrMVmxyHeNovzklG7+m5YVkTQ3HD7mz0mKx4ZExk4Fj3dfkqCI9FBKJ5KQxMO9io1kTVQEviDB1FVNTuL3SwvFDanbAX9+t0PZD4ijGCWLCGB5U24xmTEQsuLXcpOEkI9kVVeGJ0SxxJGh6QWfGh6BScyllLDRNwfUFz5/P8aBqs9KwUVSVt5YaDGVMsqaOqiikTI17lRafv7GIisLdSpuLkzmuL7Zw/YiFustT4znKTY+0ofHEWJbVts9q22eqaCUekSDiVCnDeC7Jtqy2k2mtd8sthBCb1t97O2DqLsM5k+dm8ui6yltLLQxNZbKYxtSSUed1J8AJI2pVn7vVNi+/schrc/W+r3PG1Hj36RKFlM77L4wkJt+OllhtB7y13ERXVTJmyPnRTFJSmKsBSelrKyHQLT/M1WyWGh7vOz+C44e9PTn9/BAb552AIBKCIIy4eGYIS1ePfCZBeigkEslJY2DER8tPTIyaphJGglvLbc4ttWi6AX4ckbU0VAWcIETXVDKKStsLaQUhFdsjpetoWYWv3K5ALKi2fSYLKeJYRwMqTjKiu5jRmClmSJsaNxdbrLY9hKICggdVl/G8RcrQaLshD6o283WHuh2QMXVmVx3ulluAgqKq1NoetpfmqfEc7zpdRAiB40dYusp43mKikOL1+QaGphHFCqqqEsRwu2yz1PQ5P+Lw3ExhXcdLNzBvDGijOZPRnMW9aRs3iCimDbwwWbq30nT54psrfPFWGdvvbx59/lSRD1wYZbJo0XRCFhsuLTeimDUopAwKaQMviBjJWp0SkE3bCzuGW7XXibKVEOiWH86NZFlqeNwtt8haOm0/pNKZ27Gxa2dtyeKVWQcUuDJTwvZjarbPzFDmyGcSpIdCIpGcNI72u+4eEkYxuqJiGipNL8APo15ASuvJnIzZio0QgjhOJoCWsiZjeQs/FKhGzFzNwQ9jRrIGyw2HlK4yXkjKIF4skj0vfkzKUDg7kuHGQgM3iPHDKNnFYvu4YUwQge35eCH4UUzDDQljQcpQWW37jORSmCoMZU1KGZ3zYznaXsjf3KtSd0LGCxagUrUDml4iFCrtNitNhzAUpHQFTVFYqjv4UcxozuoZFcfyFssNt2cI7XpDANKWzmQxTdCZ1fH735jjpWuLvL3S7vuarjWPjuSsXlml7Qc8GxcopnXGCylODSWG0DgWzNXcxLfgBizUHNp+yGo7YKnh9YagbaRbOqm0PFpeQCySLNCZ4QwAlbZP2tCTrh0/pO6EPVPm2pJF1tJRFHCCiCfGspwqpQli0fPx9NtxI5FIJJK9Z2DEh1AU2kFIzUnMjWpnjXzW0qk7PnNVlyiOKaZ0cpZG3fZQVQXXC1EUNemoQNByQ1RFIYxDIgH3Vl2cIMT1IyxDI5PSGS+kWWq4PKi5LDUdoliQs3SWmy5LNRcviomiCE3TMXUFEAgBKUNnOGMyVUwRA5dmimRNnbsd0+hiw2UobbLS9LB0BUXRqNk+rh/R8gIsTWGh6RJGSUfNRN4CNREJThD1JpR2DbYA50czvHBqCMtQ8cOYV+5Veem1Rf7y7U3W1hsqH336YfMogKlrPDlukbE0LF176GvXZltuLTV5e6VNKW0SxR4pQ+2Jo42tpUIIXptrEHZG2I/mrHVD0+pO2MkYwbmRLG4Qr5v10X3OrJmcqe1HnU4gl5evLRFEcW9PTHepn0QikUj2j4ERH0MpnTNDmU57p88z4zmemsiRMVT+8PUAQ4OJQoaK7UIMp4bzVNoeQ2mTJ8dzTBZTTJVStL2Ym0t1FDUZVOZ4EYaiUiya2F7IE6MZCmmdaw8aqCRzNBqOj6ooqIrSGyCWNlRs10dRFNKmwVhO5/xIlotTBaZLaSq2z1DGJIwEpqYxMZxiqeHgBTGWngwhWao7hGHMvbqDbmjMjKQRCkwULNp+hCLA7izGe2IsS9bUErNmuQ2CZPjWqo2Cwrce1PjD15c2nTz63HSBT16e5LueGSNjvvOfjWVoZE2NjKlj6uoa4eA8VOpZWz6otDwUBVodz0za0HqP3biFd+0QsRsLzXWln664KKZ1cqs2ThChq+q6WR+blSyuLzQIophnJgvcXGywssnPLpFIJJK9ZWDEx8xwloylUW575CydZ2dKXBjLsdxwiUl8Cy0/IKOr6JrGs1MF3lxuMV1M4YUxq22fsbzFqZEM96ptghgerDqM5kxOD6eZKWVZaXmcG8lTt5OFZm8vt0jpKilLQ1WS0e0tL0QBhEj+ZAwN01CZLqb59vMjFNIGuZSOqqqcHcmQtXTmqg6OHzKaS+GFEYaqcWOhgSKgmDGpOwGKgOtzDdKmzlguxXBHXEwW09wttzg9lKbc8vjy22XuVNpUOyWgctNndpOZHEMZg088N8mLz01yZiTT+3jK0MhaOllTQ9+wWXanMynODGcYy1m8udTC1DXqTsBK02O8kHqotRSSIWLdIWcCsW7mynghxVg+yYbsxpQ5lrcwNJWbiw0MTd2xkfO4Df2SSCSSo8bAiI+hjMFw1kRBYShrMJQxEEJwt5wsiHt6ssjdcotSNslgzNVdEIIYQcP1CcIAXVOYKaYYzaWYKiabbadLKZ6dKqBrGmdGMpwZTnN7pY2pKXhhRBDHuGHSYYNCMjysaLDUcAnjZOS7oqgIoOmFZC2dtKlza6mJ7YdcnMgxXUqRMjRGcsnOl7oT0nB8LEOnvGojOj/fYt1FU6GU1simTO6W2yzWHdKGynzN5fpCnesLTe5VbR5Uk+ffiKrA+y+McPXyJO87P9wTF5ahkTN1stbDgmMtO51JMV5IcXmmiKoonBvNYXtBr2vFDSI0lYeGiF1faCAQXJousFBz133vfhmO7URCd1Bcd15I9+/bcdyGfkkkEslRY2DER7nlY6galyZTVNoB5ZbPStPj2lydr9xexfEiRvMm33a6wHIz5P5qi9JQimrbp+FFGKrK7KrLWN5MfBZBTCmjM5y2uDxTZDSfwg0iHqzafOt+jbeW2zhBiO1HGLpO2gBT09EVge2HhJEgFIlZMmtqmFqW2ytNZistNEWl0k5KMm+vtHj+VIkPPzVGztJ57UGdVx9Uma+5uEHUK3fcXvEI4mR5W70dcGEiR97UqbkhigJvLrV45X6NtvdwtwokpZofemGa731ukuGsiRCCthchgoixnMVU8eFhX/3IWTqqAtfnG/hRxOnh9ENL6yARC2dHstSdRGy0/Qh71Wa1HaAqD++l6X59EAkWai6qAm4QcXultWn2Ybnh8sW3yrQ7ou7DT40yUUz3Pq+q6iN5POTQL4lEInk8BkZ8tPyI2dU2by0LTF2h5ScGzOWmlxhAhWCl5fHmso0XRlTskFJaoe4mQkE1FKpNH0NNzJxLTRcvilDVJu8+P8zZkSx/fnOFW8tNZqttaraLomgEUYgfBQShxpkRnZGswYOqg6qA0ekAUVW4W26TTRsoQDalM55NIVBIaSoLdYfrCw1GcyZemIxoF0DVSUaaq4qCpkAQCxpOSM32uFNpMzWc5fZKm6WGS58kB7qq8OR4jlOlFM9NF/jQU+NMldLkTJ22F3T2wfhcixu863SR0ZzVM2tuVmoYy1vMDKVZbnoYHVPvxiFg3YxE0w2YLqWw9KTLp9L2ewE9ZWhcGMs99L3XjkmfrzlEMZtmH+6t2twuJ6bWpWabsyOZdeLjUZFDvyQSieTxGJh3zayedGqkDAEoZPUkiEQi8Q+M5pP5FA3Hp5Sx8COXuZpDywuo2SGhEBgKCJGMaDd1jZypEYYxby81sTSVt5ZbeKHA1FTyaYsojrB9haxpoKjQdnxsBdpeQCTAjwSWQZINCULGi2liIAxibD8EVWHVgUDAfNXmz64vstL08aOYasvF9iJcPxnTPpzVCUOIEPihYKnpcnvV7ftaDGcMnpsu8MR4FjcQTOQthjMWpYzBTCkJzpW2R6XlJxt9mx4Nx2csnyKfMrYsNSiKQsrQGM1Zm2YG+mUkuntw+gX0jeWT86PZzth4ts8+CEHLDai2kzH0/bIwu0UO/XoY6YORSCS7YWDEx3zD537VIQgjDF1jru7x3CnBU2M55lZtQhGTtVQ0VaXcdhEiThbKiRjPC8hnLBw/YrnpE0YxdhDRdFSemcwTxYLZSpsgivCCiLYXMZY3SRsaQSjQVBUnCGj4EWEkaAcCtbM1FwGqquEGMTcWmxiawlPjOc6NZDg/lsPxQ+brLt+4t8qX3l7FDwIEKqoSE8egq2BoKi03pumFNDcpq6QNlclCiqfGs5wbzfKdT4zghYJr83WylkbGUqjZPssNl7F8Mm8jjAXljh/CCULaXsgzk4VNg/3aeRxNN2CuKtA19aHMQL+MxHvPDW8a0Pt5LHZS3jkznGGsYPHWUgtTV2k4Yc/U+jjIoV8PI30wEolkNwyM+Gg5yf6RXMqg5UXcXmryWilDPmXwzGSBO+Umrojxg5CaE7La9Ci3fMIwIhBqz4ugqQpDGYuFmgOKoNx0mV1tc2WmhK4qRCJmKGNQypiMZQ0Kpo5QBN+8X6PWDkBJulxQIGuq6EryNWlLx1RVdE3h3HCGc6N5zo9luTZX563lFq/P16l2JqE6XshkwSQSES1f4Ll+37KKQrJAbSht8MR4hiiC6WKaJ8fyjOQsri80MTWVuhOgKQq3lpp8fbbKE2NZnp8p9qaqmppGMa0DW5caugEojGMgEVjFdDKno3/G4Z1DbxXQ+3kszo9mNy3vrL2FzxTTqALOjeVx/FD6M/aJQfTByGyPRPLoDIz40DQVN4iStlTgTsVmZLHB5ekiThBSa4dkUjqrdtJ1EQlw/Ih82iCjC0xD5+JEnrurDgu1Nqqmkrd0QiFYqrt84IJGKWOSMTQ0XeX+qkPdCVht+9hewGo7wAsFWmf2VhRDGAsiBEMpEx2VfFrn9FAWOxAs1G1KWZ2m66OrKiqJr8P1wqQM0wz6DgGDpPPl0mQeVUlKLC0vwtRUsmmDyVIaUHh7pc2dik0pbTJfTwahWYbGzcUm9yo2TTfkQ0+O8r3PTfYd0NWv1NANQDOlDG/M11lueggU6k6D5zviApKMxBNjSVvsE7l3JpVuRj+PxVblnbW38JYXkk0ZuEHUNwsj2RsG0Qcjsz0SyaNz8t8hOuQsLTFlRhGKUHlQcygtNVmuu9yv2Sy3fZS2R932CWLBUMak6YbYboCS0snrGlOlNFHnFl9tezS9CMtQqbQDPn9zBU1Lbj2u3+lCMVRuLrrUbB87iPEFaCGogK5D1tQIo5hCSsdQFXKmThiFVG0PgWC17bHU8Li/2sYOIvwIunoj3iA8FMDSFdKmypmhFIoS86DqsdLyyJoG50czTA1lKTeTaaKlbFc8CAw92dJbq7vkUzpjOYuWF9L2Iy6M5XZ8g10bgMI4yZj0uwmPF1J8+KmxdaJmq66V7ZbjbQx4LS8kjGLSps5i3WaqmOaJ8Sz5lCH9GfvEIPpgBjHbI5HsFQMjPvwo8UboioodRFRaPoqi4ked0eaawkLTx/NjFGC17RPHMWbK5MxQGkPXuF+1qTkBQRTiBBG2FxDHOkEYU217xEKh6QaEsaCQMihmNFw/xPVjohgMBUwdcqaOH0Y4foShKjS8gFTH9/GgFhMJaLgRbhBStQNqTrhpliOlq6R1hVzaQEUgUDBUhXLDo+76DKsmURyhIMhbOmlD491nSgxnDJpuUoa4Ml1kopDi2lydpYZPGMfkLH1Xt9duaaVbZsmYKnfLba7PNxjKGuu+19oSy8Zppt3bY7+U9sZb5VaipOWFvNrZvJtLGeRThryV7iOD6IMZxGyPRLJXDMxvy2QhRSGVBNyMpZHteCeG0xaQbIv1ghARxxiGhqlDSjfRFMimDBwv5O3lFrGAphfR8kNQVYI4xgsFy81khLofx+QMFS8MURWdlKkTOyERSdZCF0lHSiAEiqoSxIJ6O8DV48QrISASgsW6R9DPyAGYmoKhJWUYRFIu6QZcXVNwQ0HNjQhDwartE0WCrGXx3nND627/3exDd6vt0xP5vgvn1gqBjJHMICm3/N5gLlVVWWl6vDbXWLe63tQ1gjhmupSIibXZDUiEx1duV7i/anN5poTb2T+zsXSyWUp7s4CXTDvN0PZDzo1ke3ttdhIYZR1fslMGMdsjkewVAyM+nj9V5MmJPOWmSySUXonCsnQadkS57dN2k3X1dSfE0FRGsip+DHfLNiCo2T5+lLSy+qFAU0E1FFRVQVMVbD/EDWKKqTRVx8cN2vhhjKUqGJogCKGYNtBUUIRC1jKoOT5tL8L2QxKb5ubkLY3xvImpCubrAYoiiBVI6wpPjWfJGhpjhRTLDYev3vZRhI4TRmiaQtPx8MKYC2uC6cbAPVFM952DsVYIzFVtHtQcTE3F0BRWO7M5Ki2PMIqZGcr0Vte/58ww8zUH2496wqQrJAC+dKvMq3M1lhs+y02PcyNZRnImOUun4fistnwKaZ3VVkDTDXacuVg/wCxet+tlO2QdX7JTBjHbI5HsFQMjPoQQaIogbapomsZQKln3fq/cRlGS9fVNJyCMY2IBfhjT8iKGcybDORMvCHEjHdf2CaJk/LeuAkIhFskI9XzKIGMKarZLww6xdQ1dVVA1BSUGK6Vj6Rq6qlJMw0ozER6h2Fp0aMB4wex10ay2fLzQRZB8XZRSGcqYvPfcMHUnxI8EU0NZHD+k6vhMFFL4QuEb92oPDfza7LXq3v67y+jmqjbnRnNU2x62H3Ll/CivzK7y9TsVLk2XaHkBQtDbFNz0Al65t0rGSDYE3686PDmew9TV3nbdlhcyXcxQsAzaXoAXRVTaPnUnJGWo3K/aBOVk4+zlU4Vd/Xs/6q1U1vElEolk/xkY8fGlW6tcm2/S8gW27+KHJpauEaHgRzFNN0BVQVNVYhEnu1bcEMvQyJsRVcdnteXiheBGHdOoItDUCF1X8IKYMAwYy5m4fjLEwwsjQgXiGDQ1CeqOH4CiUHUUVu1w0/OqQLzm/6+3fVRVSWaM+BG6Boam4YURuiK4W27TdHxUVSdjqgznDDK6xVJDxzR0hrNJCWknwXTt7b/pBjS9gJWmz1LTw9JUMqbOzcUGADnLZLqUZq4mGMmajOQsHD/k9blkS+zt5RY128cJY26ttPmOc0N829lhIDHc3l5JskPDWYOhtMlMKZMYVqOYU0NpihmDuh1g6Zvvk+nHo95KZR1fIpFI9p+BeWettpObfBTFhGFMw/F55d4quqIQxjG6ppCxDPwgJBaJWIiipIPEDnxqdoDtJ9NGIREGoQBVgCKSTa/VtocfJuZQL4SQ5AVWVYgjcMOktLKJlQNDhbGciROEeKHACZIx6kJNvlfD9rDyacJYkDZ1bC9CoODFgns1Fz8U6HqIisJoPsmELDY9FmoudSfgzHBmR8F07e3/lVkHFXjf+WHuVto8M54liGGuk+EwO4FaV1XOjmQZL6S4vdICFFKmloxR9wK+4/woc1Wb4azZy0Jcmiqw0vKIYkHG0LD9kFfurZKzdE4N5QljiGLBSM4inzIe9z+BHSHr+BKJRLL/DIz4yFgarh/RcEJUBTJW0t4qBLTdEFVRyFsqkW6wavsEEWga2F5A21WJomRo1tr6SAyggOsLWr5HDDhhIhi6FsWw98D+KCSiYyJvMl3KMl0weW2+TqXt4YeJwInj5CwxCrYXEouYnGUiOiompatoKuRSKl4I06UUFzoljmJK5+yFYaq2z9mRzKbBVAjBcsPl3qpN1fZpuiFzVUHW0lEUcIOYmVKGQsZivuYylLHQ1GS8+doFcJBkD/woYqXpMT2U5nY5Zq5qM5ZP8dREvuc5SZs6F0bzTJfSvD5fY7Xlk1VVhICRrMlYPnXgIkDW8SUSiWT/GRjxkdZVihmDKI5x/BglFgxlTW4tNWn5MQqCWIAiYrwoGQJmqMkW1VAEtHzoN7g8iNZ/XGz4334oJHNH8pZGLqXjBskQMCcIuVuNyKdMhKIShA6hEEQd8WJqCqqi4IQxvh2gq2oifkLBdN7E0HUMXaGUNdFVhUrL517NJlhq8uRYjoypcXulhRfGWPo7Jsy2H+EGEdce1Hh9oYkfRkyVUrz//AjvPlPqPSZn6TTdYJ0nYrMFcO85O4SiKIlAKabQNZXJYorhjEEcx5RbPpWWR8sLmKslP+NoLsWl6WR8ux3EXBjLSBEgkUgkJ5CBER81N9mEiqKgaoCqsdRwWWp4+EEIqElnhJ4ID0uHMAQniBEiyTyoAkTcyWZ06L9JpT+aAuM5k5mhNK4fsWr7VFo+KUNFNxVGcxblpkPDTXaQBHHyNSlTBSHQVRUhYnKmQSgEiqIylNYZypp81zNjeH7EqhOgK1BueRRTOkNpk8W6S7nl8ZW3KzhBRKUVMD2UIoxigjimmDKwg4h628f2IsI45s5Km7PDWc6N5h5qN93OE6EoCpemCox2hpW5QcRc1SEWcG2+yVTb58Zik6abmFRPD6U5M5xhrursaLHcXrW/yrZaiUQiORwGRnxoiKR8oUBK1xjL6ozmUlxfbOHHEESJyVQJk6xFEL6TvdB0hTgUGBqkLY2WHxHGSUlkJyhA3lI4PZTmyswQC/U25VZAGEUIoSDimIyhsep43Kt5tN2QSHRKNypkdY0nxrKMZC0WGzZeEOGG0PIDUobG6aEMxbTJfcem3g7ImyZLDYeVhodlaEwUUiw1HBbrHqM5izvlFi0/pNJ0qTo+lyYLNL2QKIyouhGmCnYY8+W3y8zXHT7y1BjPTiftsd1BYpCIhjiO+dqdCpDMBhkvpFAUZV354vZKMh+lmy25tdzi7ZU2pbRBzQkeEis7WSy32SAyRVF2LCr6bdft12p8EpHCSyKRHCb7Lj7+1b/6V3zmM5/hp3/6p/mt3/qt/X66TQmFgqIqiEhBILBMg7SukTVVwkglimIikqyGgN7MDdPQaDpRYvwUkDESw+h2wmOtPUQADU9QcxNDZdOLWG37aKpCxtQwNZ1YKMyuNGl7ggDIGgphJBjOGpwaypI2NFAE50ayrNrJXIwo1iikdKaKKYYyBvdXFZwg4nalSbXlk0/p2PWQhYZOFCXj2UtpAyeImK200RSFcjvgxmITN4iZKFogBLaf7ERZaflUnRARw1g+ac999UGdajvAjyK8MGax7nC7nAwmuzCa5SNPjz3UyruxgySlq7S9kCgSuGHUWzq3m8Vy3UFk37pf653nPWeHEhPrDmd19NuuOyjiQ84zkUgkh8m+io+vfe1r/Pt//+95/vnn9/NpdsRozuTscBpVVai1fJ4ZzzIznOHGcoNKO+g9risYuh5R14uISNpdgxhW7J0VWvppk1o7oKEEKEIQRRBGAiEiTK3blquQSavUnQg3EKQMldPDGZ4az3Kv6tByQxpxjKooZHQNXVHIGDpBFOMGMaeGUuiawqv3qzhBzFBGpenGNFyXS9NFluouAsGlqTw120dVVFbbLn4YoSgKOVNjOGNRSOu8Pl/HD5IZGw03GfKlKArVdrf11qPc8tA1hVLaABTaXv+tsRs7SJYbDpqiUHd8MqZO1tK3vIlvtcNl7XkURellT3Y3q2OHKawThJxnIpFIDpN9Ex+tVosf+7Ef4z/8h//Ar/7qr+7X0+yYpycLXJwqUm65pHSNQiZFzQk5M5xmvuqgqhG2J3qDu7qZC7ejQrZoWNkxdiDQVdDVJPuiqqAiODeSYyJvMF+1ieKYlAqWoXBqKM3FyRxhJFBQEMB83cMJIrxQEMUx8zWXmudj6skSt+limrYb8vZKCy+O0dRkr42uwJmRDO86M8ST4znemG/w9kqLcjuNIgSFjImhqWTMZIFete0zX7MJnWRr70Ld5anxXK+LZTRvIWJBKOJkcZ4fM1EwcYN3MhmbkTI0npnM92Z4pAxty5v4Vjtc1p5HV5XeY3Yyq2O323VPEnKeiUQiOUz27R3nn/7Tf8r3fd/38fGPf/xIiI+Lk3k+9swYX3pzmbIaoCoxD2ouhbSJaWi4YUxKT+ZzROzPXVgAKUPFC2IUBYopHV1TCcKQm8s+kUj2vxiawjOTBb7j/CiOH+JHUS9QmJqKZaq03YgoVnH9ZIdLue1xcSrPcDYpnSw3E8Fgd7pU2l7E82fyvP/CSM+X4QYRlqax0nQZzacYyeoM51PkTI1SSuftZQs3EsRRMsTsqfFcr4tFVxWGs0ZnwJjD2ytthtIG8zXnoSmqG4XFdCnFSM5aN8Njq5v4Vjtc1p6nO5p9p7M61m7XHbSZHnKeiUQiOUz2RXz8j//xP3jllVf42te+tu1jPc/D87ze3xuNxn4ciUo7YKnhUXUj7lZs3lpq4YUREwWLjKERhQHVcHfdKztBV96Z+WHqgEi8I5qazOfwY8H9mkvTC4hFMvVT1ZTET9FwCKLOsrlYkDVVTF2j5Qa9QWVDWZNC2iSKkyCd7DOJKGUSYTBftbl8qoSlqTw7XWQsb7HS9Fhpeli6zt+6VOLmYouJosVY3mK+5uBHoGkqxYyJ4oaMDlsYmkrbj7g4mQcSQdFdLJc2dYRQNk3hbxQWlq72DXy7vYlvZlTd6ayOQZ7pMcg/u0QiOXz2XHzcv3+fn/7pn+aP//iPSaW2N7B99rOf5Vd+5Vf2+hgP0fJCmo5PGMWEYYQXRBi6QiwEFTtgtR3vebZDAdKGStbUCUUMIqbuJGIijMAJIwytM8CMZEOuG8VcGMoyWUgjYoGuKuiail1zO2ZIgakpWGaM5wcMZ1MMZ3Umixa2F+KHcHY0x1LLJ2OqnB3NU0pbDOdMzo5kKbd8Xn1Qp9LyeFB1ABjOmVyaKnREAr1x6U+O51hp+UmWI2fgBhF/M1vl3qpN1tKZr7mM5qwtU/hCCNwgotzyqNk+Izmzt95+beDb7U18o0fk/Gh2z7s1HqUjRHaRSCQSyfYoQog9jbl/8Ad/wN/+238bTdN6H4uixNCoqiqe5637XL/Mx+nTp6nX6xQKu1smthXLDZf/9pVZPndtgZWmixfGKEpym98vFJLMh66CqatkLT3pclFU/DhGBXKWihsmy+wURSFraJwZSaNrGpCIo6ypUXVDzg6n8fwY01CIhYJKzGQpw1NjeVQVwlhwf9VGU1RsL+DpyTzPThdIGRp+JLB0ldW2T6Wzifb6fIPJYopLU4VeRuTVB3XCOKbthZweSpNLGVi6ihtEvDHf4F7FpuEFfOyZcbxQ8NREjvOj2U0D7nLD7duR8rgBebnh7nu3xmbPsZXAOIhzSSQSyVGk0WhQLBZ3FL/3PPPx3d/93bz22mvrPvbjP/7jXLx4kZ/7uZ9bJzwALMvCsva/3jyaM2k4PosNj6YbdbIc+yc8IPF4GFq3zKIwmjOoOwF2EKPRaeftzPNQFYWUoZJLJf4MQwddVXHDZIlcydJYWLVxIzA1gRcpXJzIkTZ1IhEjYpVLUwXm6y53VlqU0ib3qw6XT5UopM11i+IUBRZqLiM5i0tThYeMnbOVNi03ZLUd0HAjnj9VZLXtc6dioysKy02fa/N1Lk4WyVl6L4U/1gnKd8rtXlBuecmunO7k0pSh7Ukm4HG7NXaSodiqxXczgSG7SCQSiWR79lx85PN5Ll++vO5j2WyWkZGRhz5+UPhhzA/+v77EjcVm38+ryubL3h4XL0wyH6CwVPcIOrtfQhLRUXMTIaKoAkOLafsx1ZaLZVnkLZXxfIqxnMVK28cJIgKhMJ6zWLUDlhsOc3WX6UKKQkan7gbU2h6GqnJhLMs37lX50lsrvOt0iTBOdrPMVQUjuWT7bMZQWWm6XF9o9Pwb44Vkn8pqO2CqmOLGQpPrCw28ThdLNqUznrc4PZTh+VPFbYeBPWpXxVpxkDUTwdod8T6W37rUsxN2MudiqxbfzQSG7CKRSCSS7RmId0ZTV5NAukZ8KAqcHUozU0px7UGVur8/zx0BSpwsZnNFUl7ReKejptfWG4MTCMI4QFUUam0PP9QYz1tMFS2EIiikdG4utJit2qQMnSASNNwAQ4VVW8ULI0azaeaqdf7o+hJ+EJPSdXQ12WszX3PQNZUzwxkUReEb91b5ws0VhBBkTIP/x7fN8NxMqRdAbyw0uV+1EQi0zsZdO4iYLFo8Of7w2PV+Qfn8aPaRuirWioNutiZnGT2h8LjdGjvJUGzV4ruZwJBdJBKJRLI9ByI+vvCFLxzE02zJ3/22U3z+5gqFlM50McWpUporMwW+fq+KZRoofrBvo6YU9f/f3r3GRnqehf//PudnzuOZsb32eu095LTJJts2J9K0hR/tryiKKvpHKgUFKSW83IiECEQLQgGhNi0SCNRWoQWUvoCoVEBaqFRKaCH55y9C06RpkzbNaZM92Ls+j+f8HO//i8ee2rverHd37Nm1r4/kF+t4Pfezu5n78nVf93WBpqlu59T1eoZoJEcwfqQwSAIm09BpBjELrYCOH7PQ9MmnTQwN2n7EbD3CXr52G6gAXVfcOlHCDyOmai0GMykGcw6GlvS0KC8Xhyql+OGJKv/10xl+eKLGNcMZFlshb8w0uGF3sbuBvnKqljQlWz4yyacsppc6eIHihWOL3dsm79QM7GJvVawODl441gYNrhnO/yxQyLvn/L4bOVLZSIbina74SoAhhBAXb0dkPgDGBlL8P+8aYbLapuNHlLI2accgDGOCsPc3XVYL4yT4MEk6pa5kO1aCkIgkG5K2dUxdx48idJI5NKau0/ZCLF0niCNM3SRtQdY1mat7eH4EpqKcd4lijeeOLXLVYI7hostszeN0zWM4b5NxTPaW08w1kqFux+ZahJGiHUUcW2xRSttJC3d+tulCMtX3VLWTZE9SJtVmiB8FTFY76Mera3p6vNOmfL5jlHcKDjKOiaax4aOMjRypXEoA8U4BlbQtF0KI89sxwUe1FZBxTEaLKV49VWO+0aGStTF0DbXJNyEVEEWgG0mvD7U8GyaOwQZiDdKWRs41SdkG9Y5GJ1BEscILQ6ptDdMwcE0Tx9RJWQbtICJl6Zi6hqXrDGZt9gykMQ2N3UWXYtrmuN1iruExkLaZqibXaqeqHeYbHm/PJ8Perh7MEMYxB0fy3Lg7z/RSm+MLyayWZBBevhskKKV49XT9rI6i52sGBms35YaXTLPNudaGgoP1gpV3spEjlc3qcyEFp0IIcX47JvgYzDloaMzWPUoZl73lHK6lE8YKYnBN6ISb9/oRoMdgmRq+Ut20h6aDpSWb4UAm6cURxorp5Z/4XVNnsppcDR4ppsnYOqAxXfeoeyEjOZdSzqKUsRktpjF0jYYfgRbiR4pyxu0em8zWPcI4Zjjv8FbKxLV1rhvJkbYN3jNRQtd1/t/X5zg61wTgwGCG9189yP7BLJBkL9brKLoRa45RjrdBwbW78hsODlZnToB37J/Rz6JPKTgVQlzOLpdeRDvmnfG6XTl+6dAu/vuVaZa8EMsE0JKrnzpo0dpJtJshUKCHyd1aTQfi5EjGMnXKWYdrhnIM5Rx+OFmjEyoMU2EZGpap044Us/UOumaTsi0GUjYtPyKIIjJWilsnSly9K898w+v28Vhsecw12jz1WjLI7dDuAo1OwI/mWmhojORT7CmlGcjYlDM2DS+k6YUUUzaQTLY9M7NxcCRPOWN3syNKqfPOcoG1m3KSRdn4MQpc2HFGP2sypB5ECHE5u1yOhndM8KHrOndeVeHqoSzHF1ostnxeP12n6JrkXYuOn9RZbG7nD/BWrriQ1H+kzCQbkrYNsq6JZRg4JkxUMnQ8n6xroGkO4GHoGteP5JlrBNQ6AbFKqkdcy2T3QIq0pfP92Savz9Y5sdCimEpuxEzXOkmAk7EBDQO4aleeth8y2/BRaCy1a4wWXTKOyXR9OfORzZwVGGia1m3jHsWKpXaNm1bViKx2Zp3HyhHOhR6jwIUdZ/Szdbi0LRdCXM4ul6PhHRN8QLIxDBdSDBdSHJ1tcGy2xVzLp9ryCKLeTK7dqJXrtmgaWdfk0GiBMFa8MVOn4yssPcY0LPwwZqraJogUBdcin7LoBEm30qGcy/sOVNhdStHyI7718mn+960FOkHEfN3n5/aXKaQsUrYFKGYaSQATKcUPjlexdMVQIYVjuhxbaJF3Dd53VZmJcjLddbyUZjDnnJWmq3eCDf3jXS/CXjnCuVBynCGEEJfucnkv3bHv4FnHJIgj5hsekdLQDIXapLTHSk9XS0uai60c7RQdg8GcTSWXohUELC5FtPyIKI5p+klB6VAumbo7WrQJQsWppQ4TpRwHRwu8cqpOJWdTySZTaheaPmnbZKSQou2HOJaOrlvdGo6cYzCQtsk6Fv97dI6BlM3UYouTiy0yjkXGNtlbyXLrvnI34HhrrkkniJhcbCc9Span0m7kH+9KhL26WRm8c73GuchxhhBCXLrL5b10xwYfgzmHq4ZzVLIuS+2QxZa/JjDoJR1I2ckUW8swSDsGQQS7Cg6FlE055+CaBoNZjROLTRabAUN5F9cyuGY4w3R9hoWWz56BDKWMg2vr5FMmE+UUxbTNaNGllE6KTt+YbdL0Q3YXUlw1lKWSdbqZjLRt8Mqp2vL8FihlHbwowtJ0btlXpu3/rMZjddZirpF0TV0pXD3XVNozrdesLIjURZ0xynGGEEJcusvlvXTHBB/rVfgeHity4tpBDF3j9Zk6s3UffxPOXgwd0o5J3rUZyNhkXQPbSIpM867NRDnF6VqHU9UOjmWyp2QyNpAhXi7k3FV0aXoRrm0wNpDiht1Fml5I04twTIOpaodyxuauQ7vYXUzR8kPKWQfH1NE0jVv2ltA0DaUULT/i9FKHwZxDx48opi0qWYfppQ5+FDEepFEq6Sq60PDJp0xaXohr6d1Mx+qptO9UOb1es7JT1Y5cPxVCiB1uxwQf56rwvevQLmKlWGp7LDQ2p8d6J4bFVogXJHmVtq9zeGyAkYJLyjFRaFSbEQNph/GSwY1jRXblHabrPicWmhwaLXL1UJZjCy32VrIcHMnz1lyThWbAaDHF5GKL4wstylmHd40PoJTipckab862MPR291k1TWOinGGpHTDf8Aljxbv2FAB48cQSlpEEGJWsgxfGnFhsEczFWIbGwdEyo8XUWZmOd6qcXq9ZmdRrCCGE2DG7wLoVvnmXN+daPP3aHCcX2nQ28aqLF0GsIsKlNoW0w55SGscyQINi2saxdA7vKaJpGrsH0mQdk2MLHQzNoNb2mK557C6mmShn0DRtTdFQvRMwVW3R9mN0HfZXMiiS73NmQWiSjSiuyVS8NdekknXW/Nk4ps7YQIpC2mKplQyZW69Y9FJmpAghhNiZdkzwkbEN6p2AF44lzbtWrnv+9FSNU7UOhq6jNvGi7crslhiNThDy8lSVw2MD3c3dMnRq7ZDScuOulU392pEs1bZHre0zkLaI4xilFIM5hxt35zm+0GK61uanp+rEJMFAvRNQybpM1zprnhXWP+9bCWQmqy2aXsh8wyPjmJSyFguNgDBWeGHyusCaY5aMbVz0jBQhhBA7044JPpRS1L2kjiFWScOuph+haxooqHWCTXttjeTGi64nRadpy2ChGTDX6DCUT4a97R5IMZyzCWKotX1O1zxm6x2OLTR5c7bBUjPkx1N1TlZb3H3jKMOFFADHF1q8PZdMui1nHLKuSaQUXhSRNpKZKOezkpk4Nt+k0QmZb/hUWwEp2ySIPGzDYHIxOY4B1hyz3Lg7f96sxuXSUa+XzvVM2/FZhRCi13ZM8HFisc1s3aeYsnh7oUnbjzgwlCPrmOwpp5istjbldR0dbANs06SYMhjIpii4JinbZKrq0fAWObS7QDnrEMQ/m71yYrFF0bWZWWpxfK5FK4gwjWQs3Y27iwwXUhxfaPHmbJOMbWPqGn4UMWg7FFybgbTNSCHF2/NNji+0ujUf61nJTDS8sFtHMlVtE8WKwZy75kgFWHPM0vQj9g9m3zGrsbouRNdg90AK1zIuaXPu9yZ/rlqXy6V7oBBCXM52TPDxMxpBGBOrZAONoogB16KQstC9gGaPa04tA3Ipi8Gsy3glQyltstQOqbZCimmTVhiRT5lEserOXlEk11vHBlKYDYNWENIOFVoY0Q7ss14j65ocGMxy1WCGg6OF7pXa/31rAYCM3WKinDnnJriykc83PBpewGRVYeo6gzmHqWrnrCOVC21Q0/BCwjgmZRm8NFnljdk6+ypZTF3f0Oa8XqDR703+XLUul0v3QCGEuJztmOBjvJRmfyVD0wu5ejnjMbnY4vWZBscX2zS8iHaPA4+UDoWUTdY1qeQc2n5IZiDFnoEML5yo0gpCYi/i5GKbfZUsgzmHV07VeOVUjaV2yCun6mRsnfFShroX0vJDDgxmGS/9rAPpyjPdNFbk/VdXGC6kuldqm17E3kp2Tf+O9axs5GEUoxSUlwfcVbI2layzZtNXSjFaTH7CH8w5VLJnB0NnyjomTS/kRyeXWGz62KbG9SMFOkG8oc15vUCj35v8uboEXi7dA4UQ4nK2Y94Zh/IuH7hmcM2I9uMLLeqdgKYXAKqn7dUdHQayNq5toGsa842AXMqgE4SYhkPaNtCUzmInJI4iRosu1w5naXohjXbAe8ZLLDY9Rgou+wazTC8l11QP7U42Xq2W9OpYeabV9RY/u1Ib0lk+rkmGua1/VLGyka/cjilnnW4W4cxC0dm6x1S1QxQrpqodKqu+9lwGcw7jpTSNTsi1wzlePV3j7fkmu4vpDWdOzgw0+r3Jn+sGj9zsEUKI89sxwcdqmqYxmHNo+hHD+RRoGn6oejLVVlv+yDoGhg4Z28Q2dGYbPk0fdhVSvD3fZr4Z4FoG842AyarHD45XgSSbsTK0bayU4cbd+W6A0PaTbMjR2SYZx+xmOtb7iX+9TfBcRxUXspFfTMZhdTAURjH7B7NMlJNrwxvZnNdbX783+XPd4JGbPUIIcX47JvhYb+PN2AaoGFNPbrxcSuBhkPx+AzDN5FbLrnwquX0Swa6ChmsaXDWY4ehsgyCMyNgarqWhaXBioYVSiv97/fBZm6qmaQwqxZM/Ps0LxxYoL1+jnSinu7dezrSyCQ6umtEy3/AIo/is/h8XspGfGQhkbIOZWue8hZ/rvcZGC0TP9XtlkxdCiCvTjgk+1vuJPWMbVDshrmlQylosNgOiOBn+dqEikoyHroFlaBQyNsN5C0NPrrv6oYVtKF6erDHT8PCCpKdIyjKIlvt22IbRvT2yOmhYOTJ5c7bBfCvAjyFj6xta1+qgq+EFKMVZGY4L2cjPDASUUhsq/LyUYEECDSGE2F52TPCxXuq+4YWkLZNi2qbWDun4EV4UE4dcVP2HAjwFttJYaoccne9gahpoOkM5k9GBLKeqHTK2QccP8ZZnqziGhoqhmDa7AcGZmZpCyqSUcTi4K8fpWoddBbdbeLpmDWfUddQ7QTfoOrkYY2gajqVvuFj0TGcGAkdnG3K7QwghxAXZMcHHuY4WhvIucRQz1/QIY4UCTA38SziDSdtJP475WtJK3Y9DBtImKI2MbbLY8plvBgwv3yTZM5Ah5RiMldLddXXH0RddfjK5xFQ1ptEJyNgmN4zkuXlvicGcw0ytQ70T4IUxjqnjhfFyj47kSuxo0e0GXU0vIumppnWH0a3Uk1xsr4x+F34KIYS48uyYnWK91H1yW6TC88fmME5pFNM2Cy0fpV1aAUjHj3FsCGNoh0kOZSBjM1RwsU2NU0stXFPHMjWUgolKmoG0g2sZ3c1/ZVN/ZarGa9NJdgFNsavgcvPeCgdH8t3syELD58Rii7GBFEEUYxk6148WmKq2cUy9G3TNNzzmm343S3FsvsnxhTZNL1xTwHoh+l34KYQQ4sqzY4KP9WialtwWybo4poFr6UQNhXcJd241wLU0XNuk3vJZanaoZFOMFVwGUhauqRMpxbUjipmah2UktRsNL2C+4XU38NXj6OfqHqaho2ngWHo3SFnJjuRTJsFcTCFtUWuFBHG8nIkAL4zRVs1hWWqH3SxFtRVwdK5JMWUzXW++YwHrO/0ZSj2GEEKIC7Gjgw9IaiQqOZsYODHfuqjAY+V6rQ4MZC1GCy7HF9qEaFimST5l4VoGDS/EtXRswyCfsdhbzrK3ksE2NI4vtJlv+Cy1w27R5krh5mzd4+hcE4AD2cxZDa0WGslguqVWQCljd9uXd4KIycU2sWLdOSxvzzVW/hQu9Y9RCCGE2LAdH3zM1j1q7YCMpTN7EaNBdCBtQjFjY+ka+ZRFO4jwwhDLNBguOMSa4tXpOqWcw/+5Zghd09lVcDk4kmcw53B0tsHbc20AFho+9U7QDTwGcw7vv7rCRPlnXU3PbGhV7wQcGsvjmDo51+rWbhydbRArzjmHRSnFgcGkSPRANrNuAet6+j1XRQghxJVtxwcfDS+k4cWkbBNjg/unTnIbRie5WqvrGq5lMJx3STsGC3WfrGvR9CNmah7FtE3GsZip+bw8tcR1uwocHMl3AwwvjDmx2CKYS+o1Do3lu6+1cjS03nFI98jjHB1Gz1cMOpR3ef/VZ3dIPZ9+z1URQghxZdvxwUfWMSmkLWxTp5S2mGkEBOscvTgGWDr4Ed3/rgMo0DRFa3l+ymDO4WinSRgpMraJricNx0ppE8cy2DOQ5qaxwpqN3jF1xgZSFNIWS60Ax9xYD4/V1stGnK8Y9GLrNfo9V+VyI5kgIYS4MDs++Fg51piudbANsMwOUwsdVs+Yc3QwNFjuC4apJZkPDdB0UOjoukHLj5hv+lSyDo6h0w5ibFMj71poWlJz8XP7y2dlCXKuRTnrEMWKctYh51oX/BznykZsRjGoXK9dSzJBQghxYXb2rsHKnBeXG0YL2EYy46XW9qm1Y2LA1iHraJimRRQGBEqjnLZYaofJT7c6RJHCNXXKGZtSxibvmHz/+CILLR/X1Mm7FhPlLB+4ZnDdo41eXFfdymyEXK9dSzJBQghxYXZ88AHQ9CPyKZt9lSzPvDGLqeukHTCJqeRShLGi4Ue4joMWRliWQckwSdsGoVJ4QYhrGbi2yUDapuBa2IZGOeNQTJsMZByG8uef/qqUYq7hUe8EawpHNyJjG9Q7AS8ca5NZvlZ7IS7k6ECu164lmSAhhLgw8i5JsnnoGjz/9iIzNZ+2HxIBXgSq5RPHMZ1AYWoBGcfC0nUWOj6GBsMFF9twKGUcDo8XOVXtMFVtYxkGtgleqMi55jkDD6UUr5yq8cKxRdp+xFI7YLyUoZS1Lzh9ry3f+b2YcgM5Orh4kgkSQogLI8EHyeaxeyCFpilSjoEfxbS95NjFayWFHqYOKoZWEC0PhlNoeoRe9xgrpUk5Jg0/ZqHpE6MxUnCptyMqeYtfftdurtuVW/e1Z+sePzhe5eRiG12HejsknzKXB8GF3QFz58tINP2IrGNxzXC+e632QsjRwcWTTJAQQlwYCT6WNToBjp1cl52ve92rtCsXX1ScZBSiOKbRjglj6AQhnmty9XCWnG3S8QPKGYesa3Bioc3+QYtb95UZKbjMNfx1A4eGF2Isdy59a7aBqSfNwso5h4xtdLMitmEwkLE4vKfIUN4965gkYxuXlPqXowMhhBBbRXYYkuzDSyeXOLnYJghjUpaBF0WoVY0/bQNSto5jGTQ6EVqchCW6pnF0rokfKcZLaRabPoXAJOuajBZdXp+u8+ZMnaxr8b6rzp6dknVMTEOn2kqOdEaKLvsG0+ytZFFKdbMiqwfODXH2McmZ3UsvNPUvRwdCCCG2igQfQL0TMFv3QGlEMViGRtbWaHoKw4C0AWnXIu8apCyTMGqjoWOYGnnXwNI1IqXww5hjCy0Gsxa6pvP2XINaJ+TwniJH51qYusYdByprMiCDOYeJcpqmH7K3nKEdRFRyyRXZo7MNTF2jknOYrXs4pt7NSJx5THJm99ILJUcHQgghtooEHyQdRmfqHnMNj2o7IIyTNummEWGbBhoKXTcoZVwqWYe0Y1Ft+TS9iIGMy95KGpTG5FIHL1DU2xGzzTamBvPLTcNsy+DEYovMyaVuMefK0QkkGZB2EGHq+prZLeWsDUDKMnj3eLGbkdguxyTSoEsIIXaeK3PH6jHH1Ll2JEsrCKlP+cRKUfdCbN1IpsmicEydjGPSDmPGSylunhjgtdMNdhddRgopJpfazNTb5FydmXqHpU5IJWuDrjHX9Lh2OM+h0QJeqKh3AoDlkfYt0raBUlDO2EyUMwzmHJRSKKUopCwKKYvxUpqhvLsmY7Idjknklo0QQuw8Enyw3GE045BLWaRsg3onxNENlKbR8UM0NBpehGsZGLpOFMYsdUJcW+fweIm2H+K2OnhhzMnFFnEMYRyz2Ao4UE4zXs4wOpCiE8aYuo4Xxrx1conJxRbTdY/b95XQNZ1y1ulmRF45VeMHx6uYukY5a6Np2pqMwHY5JpFbNkIIsfNI8MFK3UWGRicgjhQdf57RgQzTSy2iGAopi5OLLeptD9M0GcraLDR8iimb548tEEYxS+2AxWZAy1egYoYLLjoag7kUVw9l2TeYxTF1NE2j0QkIo5i9lSzTdY+355vsLqa7RyezdY8Xji1ycrFN5YxC0+1muxwfCSGE2Dh5pyfJIkyUMxybb6HrGrmURcsPcSyDpU7IYquDbhgU0jZhrNH0Q1K2ybW78pxYbNH2Q47Pt1jyAkoZk4VWiGPq7C5lKKZNXMvi9JKHrkPWsWh4Qfcmzf5KholyunvcAkmgYRsGg8uFpinL6G7K261GYrscHwkhhNg4CT6Wrdw6aXQC9lUy/GSyRr3jJzNcDBOdENc2Gc6lqLY8YqWYqbeBmAOVLEpB/VQAWjL75cBgngNDGUoZh4OjeV44tgAaXDOcZ7KqKGdsylln3QAi65gMZJLhco6p8+7xIpWszUyt060TyTgmpq5f8TUS2+X4SAghxMbt+OBjdSYh45iMldJMLrYZr6SptU1OLnaoZG0ans5gxmZfJc3xeUUYa7SDCA2N6UYHLwwZK2UopUzuuKrC7ftKBDFMLraZqibzVjQNpqptTF1nopw5Z9AwmHM4vKe4JhuwUpi5uk6kE8Tb9jhGCCHE9rXjg4/Vty10DXYPpCikLOIpxWzNwzI1Zho+A2mT0YE0+yoZbCO5BaOUotr2abYDHMvi0O4ssVJcuyvPVcN5ppfanFxs0QkirtuVpZJ1aAXxeY8X1ssGrBRmnqtORAghhLhS7Pid68zbFq5lcHAkD4CmIO+avHhikV2FNLV2wGzDoxWETM60sUydnGOSTpn4NY9qy8dYDkpm6x7/35vzvDnbBCCIFAdHNFp+xHzDQym15urs+awUZrb9sFsnMl5Ko5Ti6GxjW9R/CCGE2Bl2fPBxrtsWmeW255apk3MtvChkZqbDG9MNXFvDNExGCjYjxRSVjM3r000Wmj67CikyjknDC2l4IcWUBWicXmozW+9Q95KBb/srGd5/dSW5/bKB4tH1CjOlR4YQQogr0Y4PPs61qU9V21iGTj5lMpR3ObXUIQZOLrYoZxwKGZ0g1Gh0QuYbHgXX5PrRArmUibt8OyXrmEzXksxHzjWJ4rgbjDS9kOMLLZba4YaCh3c6ipEeGUIIIa4kOz74WNnUV0bXvzXXZL7hEUaK60cLTFat5eyIznyjw3TNxLVNau2ArKMzqlLM1Dwsw2Cx5VPK2uRci8Gcw/uuqjBeSgOQXp5Qe3SuBSSZD+CSggfpkSGEEOJKJLvVstVHGCt9OFZuptw8USLjWLx2uka15SfHMYZiOO8yUnTohBH7KxnmGz6mrqGWm3gMF1IMF1IopZipdRgvpcm7FsW0xUQ5CT6W2rWLDh6kR4YQQogrkQQfy+qdIDk+SVsEUcz+SoZKziXrmFSyNoM5l7GCg9JgbqlD04/R0fjp6TqGrlPvhMw3fZSmCN9QvO+qCsOFFJAENi9N1gij5GrsQCZpl17J2pcUPEiPDCGEEFciCT6WJXNZ2hydbRJEMaW0zd5KtlsEOpR3OTbfxNQNBgspGnNNxkoZHENjpJii6QW8PlvHasNsw2PPQKobfKzUZqRskx9NLtH0Q5baITfuzsvtFCGEEDuO3u8FXC4cU2fPQJp9gxkipZhaavOjk0vdkfer2bpOGClOV9ukHZPdAynqnZDZus9M3Wem5lNtBd2vX6nNeHuuAcDecoYoVhxfaPGjk0u8Pt0452sJIYQQ241kPpblXItS1mZyMWldvrecYbrm8ZOpJeYaHrah0QkisrbOqaUOrqVjmzr1TsArp2rUOiEayRXdfEqjmLa633ulNqOQMskutGgHEaausdj0OVXrsLecoR1E1DtJwLJd5rYIIYQQ65HgY9mZAcLpWofXTjd4a66BHypGii5L7QADjWrLJ2WZVHIOLT/CMHQO7S4w2/DIOxYTlUy3oBRW3ajJOYyX0hxfaLHY9Dmx0GKu6TNd8zgwmMELY96Svh1CCCG2OQk+zlDK2GQck9dO14iUIo41JpfalDIWYaQYzNsstByyrsGx+RaupZNxTdpBxE1jRcZLayfUrqZpGpqmsdQOOVXrMN/0uW5XnmrLZ7yUxjF16dshhBBi25PgY9mZ3ULTjpl0OdU0dA1m6h5KwXxTp5Ay0TUdRchg3iXnWFSyTjfoeKejku6MluVjnWrLZ/dAupspkb4dQgghtjvZ3Zad2S10IG1xYDBDreVTzli0Oj6FtEspY1DOZjhVbRNEFtcMZemEMeWss6Ejku6MliDiwGDmrEyJ9O0QQgix3UnwsWwlKJhcbCW9ONImB0fynFho8tJUjXonRjdDFtom7aDN6ZrHTD0ZMnfTWHHDWYr1GoOtzpRI3w4hhBDbXc+v2j7yyCPceuut5HI5hoaG+OhHP8qrr77a65fpuUrWZrTo4oURdS9gvukzVe3QCWLSlkHGMXh9usbR2TphFDFacLl6MEvesRgvpTecpVgpPt0/mL2gqbZCCCHEdtHz4OOpp57iyJEjPPvsszz55JMEQcCHP/xhms1mr1+qp+YaPpOLbU4utHn1VJ3Zhs/kYhMviAmimDdmGzQ6IdVmQM2LWGoHhEp1b7ZIECGEEEJsTM+PXf793/99za+/8pWvMDQ0xPPPP88HPvCBXr9czzS8kMVmQBDHnFpq89Z8i5GCzY2jBfYMuJystnHzDn4YE0cR79pbYiBtX1DWQwghhBBbUPOxtLQEQKlUWve/e56H5/2ss2etVtvsJa2hlqfZzjc85psdWn7EYM7h+EKLrGMx3woopWxsQ2euHpBPmWQdh6uGcuwfzG7pWoUQQojtYFPbq8dxzIMPPsidd97JoUOH1v2aRx55hEKh0P3Ys2fPZi7pLCtXbOcaHkEUo1CkbINy1mEg7QAa5azF4T1Frh/JMZxzKWetS74GuzLp9uhsg5lapzsJVwghhNjuNjXzceTIEV5++WWeeeaZc37Npz71KR566KHur2u12pYGIN2hb5bBXNPDREMHhnMOjqmxq5Di6uEckQLT0DA0jX2D2W4r9Atpgb6SZWl4IZ0gYnKxTayQbqZCCCF2lE0LPu6//36++c1v8vTTTzM2NnbOr3McB8fpX81ExjZoeAHfO7rEyYU2E6U0xxfaDOcddE3j4EiecsYGGuQciyhWTNc6tPz4goOG1Y3M5hoelq5zcDQv3UyFEELsKD0/dlFKcf/99/PEE0/w3e9+l3379vX6JXpOKVAoFBrVdkC1HWIaBg0/ouVHtIKYnGvxnokShq7R9CNGiymiWNHwwg2/zupGZqau4UeRdDPdYnLcJYQQ/dfzHe/IkSM8/vjjfOMb3yCXy3H69GkACoUCqVSq1y93yZp+RM61+MA1Q0SvzlJvexTTFgMpi2j5a1YakE1V22QcE03jooKG1d+nnLUZKbi0/ORVlFIopeTK7iY7s42+HHcJIcTW63nw8eijjwLwC7/wC2s+/9hjj/GJT3yi1y93yVYCgk4Yc9NYgaxjMLnYpumHmIZO2jaoZO1uV9KMbQBJ0LK6Bfrqeo71OpfC2d1NlVK8NFkjihVL7Ro3LTcgE5vnzDb6ctwlhBBbr+fBx5WWxj4zIKhkbX56us4LxxaxDYOpaofBnHvetucb+Yl6pbvpyvc5OtuQjXCLrc4+yXGXEEL0x45/5z0zIABwLYPBnHtBQcHF/EQtG+HWW2+2jhBCiK0lu906LiYouJjfIxvh1lsv2BRCCLG1JPhYx8UEBRfze2QjFEIIsRNJ8LGOiwkKJJAQQgghNmZT26sLIYQQQpxJgg8hhBBCbCkJPoQQQgixpST4EEIIIcSWkuBDCCGEEFtKgg8hhBBCbCkJPoQQQgixpST4EEIIIcSWkuBDCCGEEFtKgg8hhBBCbCkJPoQQQgixpST4EEIIIcSWuuwGyymlAKjVan1eiRBCCCE2amXfXtnH38llF3zU63UA9uzZ0+eVCCGEEOJC1et1CoXCO36NpjYSomyhOI6Zmpoil8uhaVpPv3etVmPPnj2cOHGCfD7f0+99OZDnu7LJ8135tvszyvNd2Tb7+ZRS1Ot1RkdH0fV3ruq47DIfuq4zNja2qa+Rz+e35T+sFfJ8VzZ5vivfdn9Geb4r22Y+3/kyHiuk4FQIIYQQW0qCDyGEEEJsqR0VfDiOw8MPP4zjOP1eyqaQ57uyyfNd+bb7M8rzXdkup+e77ApOhRBCCLG97ajMhxBCCCH6T4IPIYQQQmwpCT6EEEIIsaUk+BBCCCHEltoxwccXv/hF9u7di+u63H777Xzve9/r95J65umnn+YjH/kIo6OjaJrG17/+9X4vqaceeeQRbr31VnK5HENDQ3z0ox/l1Vdf7feyeubRRx/lpptu6jb+ueOOO/jWt77V72Vtms9+9rNomsaDDz7Y76X0xB//8R+jadqaj+uuu67fy+qpyclJfuM3foNyuUwqleLGG2/k+9//fr+X1TN79+496+9Q0zSOHDnS76VdsiiK+KM/+iP27dtHKpXiwIED/Omf/umG5q9sph0RfPzjP/4jDz30EA8//DAvvPAChw8f5pd+6ZeYmZnp99J6otlscvjwYb74xS/2eymb4qmnnuLIkSM8++yzPPnkkwRBwIc//GGazWa/l9YTY2NjfPazn+X555/n+9//Pr/4i7/IL//yL/PjH/+430vrueeee44vfelL3HTTTf1eSk/dcMMNnDp1qvvxzDPP9HtJPbO4uMidd96JZVl861vf4ic/+Ql//ud/zsDAQL+X1jPPPffcmr+/J598EoCPfexjfV7Zpfvc5z7Ho48+yhe+8AVeeeUVPve5z/Fnf/ZnfP7zn+/vwtQOcNttt6kjR450fx1FkRodHVWPPPJIH1e1OQD1xBNP9HsZm2pmZkYB6qmnnur3UjbNwMCA+tu//dt+L6On6vW6uvrqq9WTTz6pfv7nf1498MAD/V5STzz88MPq8OHD/V7Gpvn93/999b73va/fy9hSDzzwgDpw4ICK47jfS7lkd999t7rvvvvWfO5XfuVX1D333NOnFSW2febD932ef/55PvShD3U/p+s6H/rQh/if//mfPq5MXKylpSUASqVSn1fSe1EU8dWvfpVms8kdd9zR7+X01JEjR7j77rvX/L+4Xbz++uuMjo6yf/9+7rnnHo4fP97vJfXMv/7rv3LLLbfwsY99jKGhId797nfzN3/zN/1e1qbxfZ+///u/57777uv5cNN+eO9738t3vvMdXnvtNQB++MMf8swzz3DXXXf1dV2X3WC5XpubmyOKIoaHh9d8fnh4mJ/+9Kd9WpW4WHEc8+CDD3LnnXdy6NChfi+nZ1566SXuuOMOOp0O2WyWJ554guuvv77fy+qZr371q7zwwgs899xz/V5Kz91+++185Stf4dprr+XUqVP8yZ/8Ce9///t5+eWXyeVy/V7eJTt69CiPPvooDz30EH/wB3/Ac889x2//9m9j2zb33ntvv5fXc1//+tepVqt84hOf6PdSeuKTn/wktVqN6667DsMwiKKIT3/609xzzz19Xde2Dz7E9nLkyBFefvnlbXWmDnDttdfy4osvsrS0xD/90z9x77338tRTT22LAOTEiRM88MADPPnkk7iu2+/l9NzqnyBvuukmbr/9diYmJvja177Gb/3Wb/VxZb0RxzG33HILn/nMZwB497vfzcsvv8xf//Vfb8vg4+/+7u+46667GB0d7fdSeuJrX/sa//AP/8Djjz/ODTfcwIsvvsiDDz7I6OhoX//+tn3wUalUMAyD6enpNZ+fnp5m165dfVqVuBj3338/3/zmN3n66acZGxvr93J6yrZtrrrqKgBuvvlmnnvuOf7qr/6KL33pS31e2aV7/vnnmZmZ4T3veU/3c1EU8fTTT/OFL3wBz/MwDKOPK+ytYrHINddcwxtvvNHvpfTEyMjIWUHwwYMH+ed//uc+rWjzHDt2jP/8z//kX/7lX/q9lJ75vd/7PT75yU/ya7/2awDceOONHDt2jEceeaSvwce2r/mwbZubb76Z73znO93PxXHMd77znW13pr5dKaW4//77eeKJJ/jud7/Lvn37+r2kTRfHMZ7n9XsZPfHBD36Ql156iRdffLH7ccstt3DPPffw4osvbqvAA6DRaPDmm28yMjLS76X0xJ133nnW1fbXXnuNiYmJPq1o8zz22GMMDQ1x991393spPdNqtdD1tVu9YRjEcdynFSW2feYD4KGHHuLee+/llltu4bbbbuMv//IvaTab/OZv/ma/l9YTjUZjzU9Zb731Fi+++CKlUonx8fE+rqw3jhw5wuOPP843vvENcrkcp0+fBqBQKJBKpfq8ukv3qU99irvuuovx8XHq9TqPP/44//3f/823v/3tfi+tJ3K53Fn1OZlMhnK5vC3qdn73d3+Xj3zkI0xMTDA1NcXDDz+MYRj8+q//er+X1hO/8zu/w3vf+14+85nP8Ku/+qt873vf48tf/jJf/vKX+720norjmMcee4x7770X09w+W+NHPvIRPv3pTzM+Ps4NN9zAD37wA/7iL/6C++67r78L6+tdmy30+c9/Xo2PjyvbttVtt92mnn322X4vqWf+67/+SwFnfdx77739XlpPrPdsgHrsscf6vbSeuO+++9TExISybVsNDg6qD37wg+o//uM/+r2sTbWdrtp+/OMfVyMjI8q2bbV792718Y9/XL3xxhv9XlZP/du//Zs6dOiQchxHXXfdderLX/5yv5fUc9/+9rcVoF599dV+L6WnarWaeuCBB9T4+LhyXVft379f/eEf/qHyPK+v69KU6nObMyGEEELsKNu+5kMIIYQQlxcJPoQQQgixpST4EEIIIcSWkuBDCCGEEFtKgg8hhBBCbCkJPoQQQgixpST4EEIIIcSWkuBDCCGEEFtKgg8hhBBCbCkJPoQQQgixpST4EEIIIcSWkuBDCCGEEFvq/wfPqAyP2kEskwAAAABJRU5ErkJggg==", - "text/plain": [ - "
    " - ] - }, - "metadata": {}, - "output_type": "display_data" + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "9GIt_orUtNvA" + }, + "outputs": [], + "source": [ + "# Copyright 2023 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h7AT6h2ItNvD" + }, + "source": [ + "## Use BigQuery DataFrames to visualize COVID-19 data\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \"Colab Run in Colab\n", + " \n", + " \n", + " \n", + " \"GitHub\n", + " View on GitHub\n", + " \n", + " \n", + " \n", + " \"BQ\n", + " Open in BQ Studio\n", + " \n", + "
    " + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "n-MFJQxLtNvE" + }, + "source": [ + "## Overview\n", + "\n", + "The goal of this notebook is to demonstrate creating line graphs from a ~20 million-row BigQuery dataset using BigQuery DataFrames. We will first create a plain line graph using matplotlip, then we will downsample and download our data to create a graph with a line of best fit using seaborn.\n", + "\n", + "If you're like me, during 2020 (and/or later years) you often found yourself looking at charts like [these](https://health.google.com/covid-19/open-data/explorer/statistics) visualizing COVID-19 cases over time. For our first graph, we're going to recreate one of those charts by filtering, summing, and then graphing COVID-19 data from the United States. BigQuery DataFrame's default integration with matplotlib will get us a satisfying result for this first graph.\n", + "\n", + "For our second graph, though, we want to use a scatterplot with a line of best fit, something that matplotlib will not do for us automatically. So, we'll demonstrate how to downsample our data and use seaborn to make our plot. Our second graph will be of symptom-related search trends against new cases of COVID-19, so we'll see if searches for things like \"cough\" and \"fever\" are more common in the places and times where more new cases of COVID-19 occur." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "ffqBzbNztNvF" + }, + "source": [ + "### Dataset\n", + "\n", + "This notebook uses the [BigQuery COVID-19 Open Data](https://pantheon.corp.google.com/marketplace/product/bigquery-public-datasets/covid19-open-data). In this dataset, each row represents a new observation of the COVID-19 situation in a particular time and place. We will use the \"new_confirmed\" column, which contains the number of new COVID-19 cases at each observation, along with the \"search_trends_cough\", \"search_trends_fever\", and \"search_trends_bruise\" columns, which are [Google Trends](https://trends.google.com/trends/) data for searches related to cough, fever, and bruises. In the first section of the notebook, we will also use the \"country_code\" and \"date\" columns to compile one data point per day for a particular country." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Nf__tMR-tNvF" + }, + "source": [ + "### Costs\n", + "\n", + "This tutorial uses billable components of Google Cloud:\n", + "\n", + "* BigQuery (compute)\n", + "\n", + "Learn about [BigQuery compute pricing](https://cloud.google.com/bigquery/pricing#analysis_pricing_models),\n", + "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n", + "to generate a cost estimate based on your projected usage." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7_rsbkCktNvG" + }, + "source": [ + "## Before you begin\n", + "\n", + "### Set up your Google Cloud project\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n", + "\n", + "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", + "\n", + "3. [Enable the BigQuery API](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com).\n", + "\n", + "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XZKC6iMFxmMG" + }, + "source": [ + "#### Set your project ID\n", + "\n", + "**If you don't know your project ID**, try the following:\n", + "* Run `gcloud config list`.\n", + "* Run `gcloud projects list`.\n", + "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "4aooKMmnxrWF" + }, + "outputs": [], + "source": [ + "PROJECT_ID = \"\" # @param {type:\"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pv5A8Tm-yC1U" + }, + "source": [ + "#### Set the region\n", + "\n", + "You can also change the `REGION` variable used by BigQuery. Learn more about [BigQuery regions](https://cloud.google.com/bigquery/docs/locations#supported_locations)." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "bk03Rt_HyGx-" + }, + "outputs": [], + "source": [ + "REGION = \"US\" # @param {type: \"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "B9RWxD1btNvK" + }, + "source": [ + "Now we are ready to use BigQuery DataFrames!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wJ0gXezj2w1t" + }, + "source": [ + "## Visualization #1: Cases over time in the US" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "xckgWno6ouHY" + }, + "source": [ + "### Set up project and filter data" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "-uiY0hh4tNvK" + }, + "source": [ + "First, let's do project setup. We use options to tell BigQuery DataFrames what project and what region to use for our cloud computing." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "R7STCS8xB5d2" + }, + "outputs": [], + "source": [ + "import bigframes.pandas as bpd\n", + "\n", + "# Note: The project option is not required in all environments.\n", + "# On BigQuery Studio, the project ID is automatically detected.\n", + "bpd.options.bigquery.project = PROJECT_ID\n", + "\n", + "# Note: The location option is not required.\n", + "# It defaults to the location of the first table or query\n", + "# passed to read_gbq(). For APIs where a location can't be\n", + "# auto-detected, the location defaults to the \"US\" location.\n", + "bpd.options.bigquery.location = REGION\n", + "# Improves performance by avoiding generating total row ordering\n", + "bpd.options.bigquery.ordering_mode = \"partial\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "v6FGschEowht" + }, + "source": [ + "Next, we read the data from a publicly available BigQuery dataset. This will take ~1 minute." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "zDSwoBo1CU3G" + }, + "outputs": [], + "source": [ + "all_data = bpd.read_gbq(\"bigquery-public-data.covid19_open_data.covid19_open_data\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9qV2y3iHp13y" + }, + "source": [ + "Using pandas syntax, we will select from our all_data input dataframe only those rows where the country_code is US. This is called row filtering." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "UjMT_qhjf8Fu" + }, + "outputs": [], + "source": [ + "usa_data = all_data[all_data[\"country_code\"] == \"US\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IYCUayWkwq8c" + }, + "source": [ + "We're only concerned with the date and the total number of confirmed cases for now, so select just those two columns as well." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "IaoUf57ZwrJ8" + }, + "outputs": [], + "source": [ + "usa_data = usa_data[[\"date\", \"new_confirmed\"]]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "94oqNRnDvGkr" + }, + "source": [ + "### Sum data" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "TNCQWZW83U0b" + }, + "source": [ + "`usa_data.groupby(\"date\")` will give us a groupby object that lets us perform operations on groups of rows with the same date. We call sum on that object to get the sum for each day. This process might be familiar to pandas users." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "tYDoaKgJChiq" + }, + "outputs": [], + "source": [ + "# numeric_only = True because we don't want to sum dates\n", + "new_cases_usa = usa_data.groupby(\"date\").sum(numeric_only = True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3jcwFPgK5BLh" + }, + "source": [ + "### Line graph" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8GvJAgnH5Nzi" + }, + "source": [ + "BigQuery DataFrames implements some plotting methods with the matplotlib backend. Use `DataFrame.plot.line()` to draw a simple line graph." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "gFbCgfFC2gHw" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHkCAYAAADCag6yAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAfvpJREFUeJzt3Xd8U1X/B/BP0r0HUAq07L3LbgEBZYoKD4/ggwMcoD6KgjhBxcdZFBFQ+KGigqCIojJERRApyKbMsoplldVBoXsn5/dHaXpvmqRJm/Qml8/79eqL9OYmPYemud98z/ecoxFCCBARERGphFbpBhARERHZE4MbIiIiUhUGN0RERKQqDG6IiIhIVRjcEBERkaowuCEiIiJVYXBDREREqsLghoiIiFSFwQ0RERGpCoMbIiIiUpVbOrjZvn077r77bjRs2BAajQZr1661+TmEEPjwww/RunVreHl5oVGjRnj33Xft31giIiKyirvSDVBSXl4eunTpgkcffRRjxoyp1nNMnToVmzZtwocffohOnTrh+vXruH79up1bSkRERNbScOPMMhqNBmvWrMHo0aMNx4qKivDqq6/iu+++Q2ZmJjp27Ij3338fAwcOBACcPHkSnTt3xrFjx9CmTRtlGk5EREQyt/SwVFWmTJmC3bt3Y9WqVTh69CjGjh2L4cOH459//gEA/PLLL2jevDk2bNiAZs2aoWnTppg0aRIzN0RERApicGNGcnIyli5ditWrV6N///5o0aIFXnjhBfTr1w9Lly4FAJw9exYXLlzA6tWrsXz5cixbtgwHDhzAvffeq3DriYiIbl23dM2NJQkJCdDpdGjdurXseFFREerUqQMA0Ov1KCoqwvLlyw3nffnll+jevTsSExM5VEVERKQABjdm5Obmws3NDQcOHICbm5vsPn9/fwBAgwYN4O7uLguA2rVrB6As88PghoiIqPYxuDEjKioKOp0OaWlp6N+/v8lz+vbti9LSUpw5cwYtWrQAAJw+fRoA0KRJk1prKxEREVW4pWdL5ebmIikpCUBZMPPRRx9h0KBBCA0NRePGjfHggw9i586dmDt3LqKiopCeno4tW7agc+fOGDlyJPR6PXr27Al/f3/Mnz8fer0eTz/9NAIDA7Fp0yaFe0dERHRruqWDm7i4OAwaNKjS8YkTJ2LZsmUoKSnBO++8g+XLl+Py5cuoW7cu+vTpgzfffBOdOnUCAFy5cgXPPPMMNm3aBD8/P4wYMQJz585FaGhobXeHiIiIcIsHN0RERKQ+nApOREREqnLLFRTr9XpcuXIFAQEB0Gg0SjeHiIiIrCCEQE5ODho2bAit1nJu5pYLbq5cuYLIyEilm0FERETVcPHiRURERFg855YLbgICAgCU/ecEBgYq3BoiIiKyRnZ2NiIjIw3XcUtuueCmfCgqMDCQwQ0REZGLsaakhAXFREREpCoMboiIiEhVGNwQERGRqtxyNTfW0ul0KCkpUboZpFIeHh6VNmQlIiL7YHBjRAiBlJQUZGZmKt0UUrng4GCEh4dzvSUiIjtjcGOkPLAJCwuDr68vLzxkd0II5OfnIy0tDQDQoEEDhVtERKQuDG4kdDqdIbCpU6eO0s0hFfPx8QEApKWlISwsjENURER2xIJiifIaG19fX4VbQreC8tcZa7uIiOyLwY0JHIqi2sDXGRGRYzC4ISIiIlVxmuBm9uzZ0Gg0mDZtmsXzVq9ejbZt28Lb2xudOnXCb7/9VjsNJCIiIpfgFMHN/v378dlnn6Fz584Wz9u1axfGjx+Pxx57DIcOHcLo0aMxevRoHDt2rJZaSs5i586d6NSpEzw8PDB69GjExcVBo9E41RT+pk2bYv78+Uo3g4jolqN4cJObm4sHHngAS5YsQUhIiMVzFyxYgOHDh+PFF19Eu3bt8Pbbb6Nbt25YuHBhLbWWnMX06dPRtWtXnDt3DsuWLUNMTAyuXr2KoKAgpZtGREQKUzy4efrppzFy5EgMHjy4ynN3795d6bxhw4Zh9+7dZh9TVFSE7Oxs2Re5vjNnzuD2229HREQEgoOD4enpaXFBPJ1OB71eX8utJCJr6fUCM9ckYOXeZKWbQiqgaHCzatUqHDx4ELGxsVadn5KSgvr168uO1a9fHykpKWYfExsbi6CgIMNXZGSkTW0UQiC/uFSRLyGE1e0cOHAgnn32Wbz00ksIDQ1FeHg4/ve//xnuz8zMxKRJk1CvXj0EBgbi9ttvx5EjRwAAWVlZcHNzQ3x8PABAr9cjNDQUffr0MTz+m2++sfr/7tKlSxg/fjxCQ0Ph5+eHHj16YO/evYb7Fy9ejBYtWsDT0xNt2rTBihUrZI/XaDT44osv8K9//Qu+vr5o1aoV1q9fDwA4f/48NBoNMjIy8Oijj0Kj0WDZsmWVhqWWLVuG4OBgrF+/Hu3bt4eXlxeSk5PRtGlTvPPOO5gwYQL8/f3RpEkTrF+/Hunp6Rg1ahT8/f3RuXNnw/9FuR07dqB///7w8fFBZGQknn32WeTl5RnuT0tLw9133w0fHx80a9YM3377rVX/V0RUZtvpdKzcm4yZaxKUbgqpgGKL+F28eBFTp07F5s2b4e3t7bCfM2PGDEyfPt3wfXZ2tk0BTkGJDu1n/eGIplXpxFvD4Otp/a/o66+/xvTp07F3717s3r0bDz/8MPr27YshQ4Zg7Nix8PHxwe+//46goCB89tlnuOOOO3D69GmEhoaia9euiIuLQ48ePZCQkACNRoNDhw4hNzcX/v7+2LZtGwYMGFBlG3JzczFgwAA0atQI69evR3h4OA4ePGjImqxZswZTp07F/PnzMXjwYGzYsAGPPPIIIiIiMGjQIMPzvPnmm/jggw8wZ84cfPLJJ3jggQdw4cIFREZG4urVq2jTpg3eeust3HfffQgKCpIFT+Xy8/Px/vvv44svvkCdOnUQFhYGAJg3bx7ee+89vP7665g3bx4eeughxMTE4NFHH8WcOXPw8ssvY8KECTh+/Dg0Gg3OnDmD4cOH45133sFXX32F9PR0TJkyBVOmTMHSpUsBAA8//DCuXLmCrVu3wsPDA88++6xhBWIiqlpWAdd7IvtRLLg5cOAA0tLS0K1bN8MxnU6H7du3Y+HChSgqKqq0amt4eDhSU1Nlx1JTUxEeHm7253h5ecHLy8u+jXdSnTt3xhtvvAEAaNWqFRYuXIgtW7bAx8cH+/btQ1pamuH/4sMPP8TatWvx448/4vHHH8fAgQMRFxeHF154AXFxcRgyZAhOnTqFHTt2YPjw4YiLi8NLL71UZRtWrlyJ9PR07N+/H6GhoQCAli1bGu7/8MMP8fDDD+Opp54CUFY7s2fPHnz44Yey4Obhhx/G+PHjAQDvvfcePv74Y+zbtw/Dhw83DD8FBQVZ/N2XlJTg//7v/9ClSxfZ8TvvvBNPPPEEAGDWrFlYvHgxevbsibFjxwIAXn75ZURHRxteW7GxsXjggQcMM/latWqFjz/+GAMGDMDixYuRnJyM33//Hfv27UPPnj0BAF9++SXatWtX5f8XEZXhsk9kT4oFN3fccQcSEuTpx0ceeQRt27bFyy+/bHI5+ujoaGzZskU2XXzz5s2Ijo52WDt9PNxw4q1hDnv+qn62LYxnmzVo0ABpaWk4cuQIcnNzK20pUVBQgDNnzgAABgwYgC+//BI6nQ7btm3D0KFDER4ejri4OHTu3BlJSUkYOHBglW04fPgwoqKiDIGNsZMnT+Lxxx+XHevbty8WLFhgti9+fn4IDAy0ORPi6elpcgae9Fj5MGenTp0qHUtLS0N4eDiOHDmCo0ePyoaahBDQ6/U4d+4cTp8+DXd3d3Tv3t1wf9u2bREcHGxTe4mIyD4UC24CAgLQsWNH2TE/Pz/UqVPHcHzChAlo1KiRoSZn6tSpGDBgAObOnYuRI0di1apViI+Px+eff+6wdmo0GpuGhpTk4eEh+16j0UCv1yM3NxcNGjRAXFxcpceUX4Bvu+025OTk4ODBg9i+fTvee+89hIeHY/bs2ejSpQsaNmyIVq1aVdmG8j2TaspcX2zh4+NjssBY+tzl95s6Vv7zcnNz8cQTT+DZZ5+t9FyNGzfG6dOnbWoXERE5llNftZOTk6HVVtQ8x8TEYOXKlXjttdcwc+ZMtGrVCmvXrq0UJJFct27dkJKSAnd3dzRt2tTkOcHBwejcuTMWLlwIDw8PtG3bFmFhYbjvvvuwYcMGq+ptgLKsyBdffIHr16+bzN60a9cOO3fuxMSJEw3Hdu7cifbt21erb7WhW7duOHHihGx4Tapt27YoLS3FgQMHDMNSiYmJTrXmDhHRrcSpghvjzIKpTMPYsWMNtRFkncGDByM6OhqjR4/GBx98gNatW+PKlSv49ddf8a9//Qs9evQAUDbj6pNPPsG9994LAAgNDUW7du3w/fffY9GiRVb9rPHjx+O9997D6NGjERsbiwYNGuDQoUNo2LAhoqOj8eKLL2LcuHGIiorC4MGD8csvv+Dnn3/Gn3/+6bD+19TLL7+MPn36YMqUKZg0aRL8/Pxw4sQJbN68GQsXLkSbNm0wfPhwPPHEE1i8eDHc3d0xbdo0u2WxiIjINoqvc0OOp9Fo8Ntvv+G2227DI488gtatW+M///kPLly4IJtaP2DAAOh0OlltzcCBAysds8TT0xObNm1CWFgY7rzzTnTq1AmzZ8821FCNHj0aCxYswIcffogOHTrgs88+w9KlS61+fiV07twZ27Ztw+nTp9G/f39ERUVh1qxZaNiwoeGcpUuXomHDhhgwYADGjBmDxx9/3DA7i4iIapdG2LKYigpkZ2cjKCgIWVlZCAwMlN1XWFiIc+fOoVmzZg6dnk4E8PVGJLX+yBU8+90hAMD52SMVbg05I0vXb2PM3BAREZGqMLghm7z33nvw9/c3+TVixAilm0dERORcBcXk/J588kmMGzfO5H0soCWi6uIafmRPDG7IJqGhoWYX6CMiInIGHJYy4RarsSaF8HVGVIHbL5A9MbiRKF+lNj8/X+GW0K2g/HVmvBozERHVDIelJNzc3BAcHGzYw8jX19fk8v1ENSGEQH5+PtLS0hAcHGxyHzUiIqo+BjdGyneZtnWTRiJbBQcHW9zVnIiIqofBjRGNRoMGDRogLCwMJSUlSjeHVMrDw4MZGyIJDedLkR0xuDHDzc2NFx8iIiIXxIJiIiIiUhUGN0RERKQqDG6IiEhxnJhK9sTghoiIiFSFwQ0RESmOiRuyJwY3REREpCoMboiIiEhVGNwQEZHiWFBM9sTghoiIiFSFwQ0RETkBpm7IfhjcEBERkaowuCEiIiJVYXBDRERORQihdBPIxTG4ISIixUlnSzG2oZpicENERE6FsQ3VFIMbIiIiUhUGN0RE5FRYc0M1xeCGiIgUJ13lhqEN1RSDGyIicipM3FBNKRrcLF68GJ07d0ZgYCACAwMRHR2N33//3ez5y5Ytg0ajkX15e3vXYouJiMgRNJLpUoK5G6ohdyV/eEREBGbPno1WrVpBCIGvv/4ao0aNwqFDh9ChQweTjwkMDERiYqLhew13WyMiIiIJRYObu+++W/b9u+++i8WLF2PPnj1mgxuNRoPw8PDaaB4RESmAw1JUU05Tc6PT6bBq1Srk5eUhOjra7Hm5ublo0qQJIiMjMWrUKBw/ftzi8xYVFSE7O1v2RUREzoU5eLInxYObhIQE+Pv7w8vLC08++STWrFmD9u3bmzy3TZs2+Oqrr7Bu3Tp888030Ov1iImJwaVLl8w+f2xsLIKCggxfkZGRjuoKERHZATM3VFMaofCCAsXFxUhOTkZWVhZ+/PFHfPHFF9i2bZvZAEeqpKQE7dq1w/jx4/H222+bPKeoqAhFRUWG77OzsxEZGYmsrCwEBgbarR9ERFR9W06m4rGv4wEAJ94aBl9PRasmyAllZ2cjKCjIquu34q8eT09PtGzZEgDQvXt37N+/HwsWLMBnn31W5WM9PDwQFRWFpKQks+d4eXnBy8vLbu0lIiIi56b4sJQxvV4vy7RYotPpkJCQgAYNGji4VUREVFs4LEU1pWjmZsaMGRgxYgQaN26MnJwcrFy5EnFxcfjjjz8AABMmTECjRo0QGxsLAHjrrbfQp08ftGzZEpmZmZgzZw4uXLiASZMmKdkNIiKyI8Y2VFOKBjdpaWmYMGECrl69iqCgIHTu3Bl//PEHhgwZAgBITk6GVluRXLpx4wYmT56MlJQUhISEoHv37ti1a5dV9TlEROS8pEuWcW8pqinFC4prmy0FSUREVDv+OpWKR5eVFRQf/d9QBHp7KNwicja2XL+druaGiIiIqCYY3BARkVO5tcYTyBEY3BARkeI00jWKGdxQDTG4ISIi5cliG0Y3VDMMboiIyKlwWIpqisENERERqQqDGyIicipM3FBNMbghIiLFSUpuuIgf1RiDGyIicioMbaimGNwQEZHipAENEzdUUwxuiIiISFUY3BARkfKE9CZTN1QzDG6IiMi5MLahGmJwQ0REipNmaxjbUE0xuCEiIqfCgmKqKQY3RETkVFhzQzXF4IaIiBTHbA3ZE4MbIiJyKgx0qKYY3BARkeKEbCo4Uc0wuCEiIqfCvaWophjcEBGR4rj9AtkTgxsiIiJSFQY3RESkOA5FkT0xuCEiIqfCOIdqisENEREpTlZzw/lSVEMMboiIyKkMmBOH5bvPK90McmEMboiISHHGQ1Gz1h1XpiGkCgxuiIiISFUY3BARkRNgnQ3ZD4MbIiJySlNWHsTp1Bylm0EuiMENERE5pQ1Hr2LcZ7uVbga5IEWDm8WLF6Nz584IDAxEYGAgoqOj8fvvv1t8zOrVq9G2bVt4e3ujU6dO+O2332qptURE5Cjm1rbJzC+p3YaQKiga3ERERGD27Nk4cOAA4uPjcfvtt2PUqFE4ftx0lfyuXbswfvx4PPbYYzh06BBGjx6N0aNH49ixY7XcciIiInJWGuFka16HhoZizpw5eOyxxyrdd9999yEvLw8bNmwwHOvTpw+6du2KTz/91OTzFRUVoaioyPB9dnY2IiMjkZWVhcDAQPt3gIiIbPZbwlU89e1Bk/ednz2ylltDzig7OxtBQUFWXb+dpuZGp9Nh1apVyMvLQ3R0tMlzdu/ejcGDB8uODRs2DLt3mx+TjY2NRVBQkOErMjLSru0mIiLH2vHPNbyz4QSKS/VKN4VchLvSDUhISEB0dDQKCwvh7++PNWvWoH379ibPTUlJQf369WXH6tevj5SUFLPPP2PGDEyfPt3wfXnmhoiInIelMYQHv9wLAAgP8sak/s1rqUXkyhQPbtq0aYPDhw8jKysLP/74IyZOnIht27aZDXBs5eXlBS8vL7s8FxERKefSjQKlm0AuQvHgxtPTEy1btgQAdO/eHfv378eCBQvw2WefVTo3PDwcqampsmOpqakIDw+vlbYSEZFjcLNMsienqbkpp9frZQXAUtHR0diyZYvs2ObNm83W6BARkfNKySrE2E93Yf2RK6ynIbtSNHMzY8YMjBgxAo0bN0ZOTg5WrlyJuLg4/PHHHwCACRMmoFGjRoiNjQUATJ06FQMGDMDcuXMxcuRIrFq1CvHx8fj888+V7AYREVXD27+ewP7zN7D//A2rzneyyb3kxBQNbtLS0jBhwgRcvXoVQUFB6Ny5M/744w8MGTIEAJCcnAyttiK5FBMTg5UrV+K1117DzJkz0apVK6xduxYdO3ZUqgtERFRN2QVcoI8cQ9Hg5ssvv7R4f1xcXKVjY8eOxdixYx3UIiIiqi1ajcam8zU2nk+3LqeruSEioluDrbEKh6XIWgxuiIhIEbZmboisxeCGiIgUYWtow7wNWYvBDRERKYI1NOQoDG6IiEgRWsY25CAMboiISBG2FxQ7ph2kPgxuiIhIESwoJkdhcENERIpgbEOOwuCGiIgUwYJichQGN0REpAgOS5GjMLghIiJF2L7ODSuKyToMboiISBGcCk6OwuCGiIgUYeuwFKeCk7UY3BARkTKYuSEHYXBDRESKYEExOQqDGyIiUgQ3ziRHYXBDRESKYOaGHIXBDRERKcLW2IahEFmLwQ0RESnC1hWKOSxF1mJwQ0REiuA6N+QoDG6IiEgRtg5LcZ0bshaDGyIiUgQLislRGNwQEZEiGNyQozC4ISIiF8FxKbIOgxsiIlIEMzfkKAxuiIhIEZwtRY7C4IaIiBTBxA05CoMbIiJShK2L+BFZi8ENEREpguvckKMwuCEiIkWwoJgcRdHgJjY2Fj179kRAQADCwsIwevRoJCYmWnzMsmXLoNFoZF/e3t611GIiIrIXW0MbZm7IWooGN9u2bcPTTz+NPXv2YPPmzSgpKcHQoUORl5dn8XGBgYG4evWq4evChQu11GIiIrIXZm7IUdyV/OEbN26Ufb9s2TKEhYXhwIEDuO2228w+TqPRIDw83NHNIyIiB+JUcHIUp6q5ycrKAgCEhoZaPC83NxdNmjRBZGQkRo0ahePHj5s9t6ioCNnZ2bIvIiJyAjZmbgRXKCYrOU1wo9frMW3aNPTt2xcdO3Y0e16bNm3w1VdfYd26dfjmm2+g1+sRExODS5cumTw/NjYWQUFBhq/IyEhHdYGIiGzAzA05itMEN08//TSOHTuGVatWWTwvOjoaEyZMQNeuXTFgwAD8/PPPqFevHj777DOT58+YMQNZWVmGr4sXLzqi+UREZCPW3JCjKFpzU27KlCnYsGEDtm/fjoiICJse6+HhgaioKCQlJZm838vLC15eXvZoJhER2RFDG3IURTM3QghMmTIFa9aswV9//YVmzZrZ/Bw6nQ4JCQlo0KCBA1pIRESOwsQNOYqimZunn34aK1euxLp16xAQEICUlBQAQFBQEHx8fAAAEyZMQKNGjRAbGwsAeOutt9CnTx+0bNkSmZmZmDNnDi5cuIBJkyYp1g8iIrKdrdsvcJ0bspaiwc3ixYsBAAMHDpQdX7p0KR5++GEAQHJyMrTaigTTjRs3MHnyZKSkpCAkJATdu3fHrl270L59+9pqNhER2YFgtEIOomhwY80LOy4uTvb9vHnzMG/ePAe1iIiIagtjG3IUu9TcZGZm2uNpiIjoFmJrbMNYiKxlc3Dz/vvv4/vvvzd8P27cONSpUweNGjXCkSNH7No4IiIiIlvZHNx8+umnhoXwNm/ejM2bN+P333/HiBEj8OKLL9q9gUREpE4cliJHsbnmJiUlxRDcbNiwAePGjcPQoUPRtGlT9O7d2+4NJCIideJ2CuQoNmduQkJCDKv8bty4EYMHDwZQVhys0+ns2zoiIiIiG9mcuRkzZgzuv/9+tGrVChkZGRgxYgQA4NChQ2jZsqXdG0hEROpk67AUh7HIWjYHN/PmzUPTpk1x8eJFfPDBB/D39wcAXL16FU899ZTdG0hEROrEWIUcxebgxsPDAy+88EKl488995xdGkRERGQKa3TIWtVa52bFihXo168fGjZsiAsXLgAA5s+fj3Xr1tm1cUREpGIcZyIHsTm4Wbx4MaZPn44RI0YgMzPTUEQcHByM+fPn27t9RESkUgxtyFFsDm4++eQTLFmyBK+++irc3NwMx3v06IGEhAS7No6IiIjIVjYHN+fOnUNUVFSl415eXsjLy7NLo4iISP04KkWOYnNw06xZMxw+fLjS8Y0bN6Jdu3b2aBMREd0CbC4QZjBEVrJ5ttT06dPx9NNPo7CwEEII7Nu3D9999x1iY2PxxRdfOKKNRESkQszckKPYHNxMmjQJPj4+eO2115Cfn4/7778fDRs2xIIFC/Cf//zHEW0kIiIisprNwQ0APPDAA3jggQeQn5+P3NxchIWF2btdRESkcqYSN10jg3H4YqbV5xOZYnPNTUFBAfLz8wEAvr6+KCgowPz587Fp0ya7N46IiNTL1LCURlP77SD1sTm4GTVqFJYvXw4AyMzMRK9evTB37lyMGjUKixcvtnsDiYjo1mEpthEs0iEr2RzcHDx4EP379wcA/PjjjwgPD8eFCxewfPlyfPzxx3ZvIBERqZOp2VIapm7IDmwObvLz8xEQEAAA2LRpE8aMGQOtVos+ffoYtmIgIiKqkolEjJaxDdmBzcFNy5YtsXbtWly8eBF//PEHhg4dCgBIS0tDYGCg3RtIRES3Do3FgSki69gc3MyaNQsvvPACmjZtit69eyM6OhpAWRbH1MrFREREppisoGFsQ3Zg81Twe++9F/369cPVq1fRpUsXw/E77rgD//rXv+zaOCIiUi9TBcKMbcgeqrXOTXh4OMLDw2XHevXqZZcGERHRrUtroaCYc6XIWtUKbuLj4/HDDz8gOTkZxcXFsvt+/vlnuzSMiIjUjevckKPYXHOzatUqxMTE4OTJk1izZg1KSkpw/Phx/PXXXwgKCnJEG4mISIXKY5umdXwNxywFN1zmhqxlc3Dz3nvvYd68efjll1/g6emJBQsW4NSpUxg3bhwaN27siDYSEZEKlQcr0rVtOFuK7MHm4ObMmTMYOXIkAMDT0xN5eXnQaDR47rnn8Pnnn9u9gUREpG7SbA2HpcgebA5uQkJCkJOTAwBo1KgRjh07BqBsK4byPaeIiIiqUr5CsTSesbRC8fojV7gFA1nF5uDmtttuw+bNmwEAY8eOxdSpUzF58mSMHz8ed9xxh90bSERE6lQep2hlw1KW7TqT4bgGkWrYPFtq4cKFKCwsBAC8+uqr8PDwwK5du/Dvf/8br732mt0bSERE6iYLbqqIbi5e5wgBVc3mzE1oaCgaNmxY9mCtFq+88grWr1+PuXPnIiQkxKbnio2NRc+ePREQEICwsDCMHj0aiYmJVT5u9erVaNu2Lby9vdGpUyf89ttvtnaDiIichKzmpopzOShF1rA6uLly5QpeeOEFZGdnV7ovKysLL774IlJTU2364du2bcPTTz+NPXv2YPPmzSgpKcHQoUORl5dn9jG7du3C+PHj8dhjj+HQoUMYPXo0Ro8ebaj9ISIi12ByheIqUjcsuSFrWB3cfPTRR8jOzja5OWZQUBBycnLw0Ucf2fTDN27ciIcffhgdOnRAly5dsGzZMiQnJ+PAgQNmH7NgwQIMHz4cL774Itq1a4e3334b3bp1w8KFC2362URE5BzM1dyY2iFcMHdDVrA6uNm4cSMmTJhg9v4JEyZgw4YNNWpMVlYWgLKhL3N2796NwYMHy44NGzYMu3fvNnl+UVERsrOzZV9ERKS88jBFK7kSyda84bxwqiarg5tz585ZXKQvIiIC58+fr3ZD9Ho9pk2bhr59+6Jjx45mz0tJSUH9+vVlx+rXr4+UlBST58fGxiIoKMjwFRkZWe02EhGR/RgW8YPpgmJToQ2HpcgaVgc3Pj4+FoOX8+fPw8fHp9oNefrpp3Hs2DGsWrWq2s9hyowZM5CVlWX4unjxol2fn4iIasZcQKPRcFE/qh6rg5vevXtjxYoVZu9fvnx5tXcGnzJlCjZs2ICtW7ciIiLC4rnh4eGVCpdTU1Mr7VJezsvLC4GBgbIvIiJSnmERPzNTwTUmNmNg4oasYXVw88ILL2Dp0qV44YUXZMFFamoqnn/+eSxbtgwvvPCCTT9cCIEpU6ZgzZo1+Ouvv9CsWbMqHxMdHY0tW7bIjm3evBnR0dE2/WwiIlJWxbBUBW1VqRqOS5EVrF7Eb9CgQVi0aBGmTp2KefPmITAwEBqNBllZWfDw8MAnn3yC22+/3aYf/vTTT2PlypVYt24dAgICDHUzQUFBhiGuCRMmoFGjRoiNjQUATJ06FQMGDMDcuXMxcuRIrFq1CvHx8dzXiojIxRgKim3YW4qhDVnDphWKn3jiCdx111344YcfkJSUBCEEWrdujXvvvbfK4SRTFi9eDAAYOHCg7PjSpUvx8MMPAwCSk5OhlZTSx8TEYOXKlXjttdcwc+ZMtGrVCmvXrrVYhExERM5Ly13Byc5s3n6hUaNGeO655+zyw63ZAC0uLq7SsbFjx2Ls2LF2aQMRESnDMCwlqyI2c9voMUSW2Lz9AhERkX2U7wpufuNM47VuuCs4WYPBDRERKUoav1RVUMzQhqzB4IaIiBRhaliKk6XIHhjcEBGRIsoDFXN7S5lcodihLSK1sDm4mTVrFrZu3YrCwkJHtIeIiG4xWjP7SXF1Yqoum4Ob3bt34+6770ZwcDD69++P1157DX/++ScKCgoc0T4iIlKpihWKK44ZBzTG8c3plBzMXJOAlCx+wCbzbA5uNm/ejMzMTGzZsgV33nkn4uPjMWbMGAQHB6Nfv36OaCMREalQRc2N9evcfB9/ESv3JuPZ7w45smnk4mxe5wYA3N3d0bdvX9SrVw+hoaEICAjA2rVrcerUKXu3j4iIVM54s8yK4xqYq7I5cTXboW0i12Zz5ubzzz/H/fffj0aNGiEmJgYbN25Ev379EB8fj/T0dEe0kYiIVMjk9guKtITUxubMzZNPPol69erh+eefx1NPPQV/f39HtIuIiFTO1LBUlRtnGh7LeVNkns2Zm59//hkPPPAAVq1ahXr16iEmJgYzZ87Epk2bkJ+f74g2EhGRCpUXFFvaOJMzpqg6bM7cjB49GqNHjwYAZGVl4e+//8bq1atx1113QavVcoo4ERHZyPT0bwY2VF3VKijOyMjAtm3bEBcXh7i4OBw/fhwhISHo37+/vdtHRERqZWrjTCurbjgoRZbYHNx06tQJJ0+eREhICG677TZMnjwZAwYMQOfOnR3RPiIiUimTBcXM1pAdVKugeMCAAejYsaMj2kNERLcYc9svWMJ6YrLE5uDm6aefBgAUFxfj3LlzaNGiBdzdqzW6RUREt7DyGU/m6mw0sLzWDZE5Ns+WKigowGOPPQZfX1906NABycnJAIBnnnkGs2fPtnsDiYhIfVKzC5FXrANQvangRJbYHNy88sorOHLkCOLi4uDt7W04PnjwYHz//fd2bRwREalPSlYher+3BZtPpAIwvxO4xkKgI5jNIQtsHk9au3Ytvv/+e/Tp00f2wuvQoQPOnDlj18YREZH67D2XIfteY2ZXcEtYc0OW2Jy5SU9PR1hYWKXjeXl5Vr8oiYjo1uXj4Sb7Xmvm0mHpisLYhiyxObjp0aMHfv31V8P35QHNF198gejoaPu1jIiIVMm7UnBjoeaGn5mpGmwelnrvvfcwYsQInDhxAqWlpViwYAFOnDiBXbt2Ydu2bY5oIxERqZi5XcGJqsvmzE2/fv1w+PBhlJaWolOnTti0aRPCwsKwe/dudO/e3RFtJCIiFSnR6WXfa6qxzg3HpciSai1Q06JFCyxZssTebSEioltA5eDG9G0OSVF12Zy5ISIiqokSnTztYq6g2BJOBSdLrM7caLXaKmdDaTQalJaW1rhRRESkXpUyNzA/FZzJG6oOq4ObNWvWmL1v9+7d+Pjjj6HX682eQ0REBFQObrQcQyA7szq4GTVqVKVjiYmJeOWVV/DLL7/ggQcewFtvvWXXxhERkfoUGw1LmSsotrjODUelyIJqxctXrlzB5MmT0alTJ5SWluLw4cP4+uuv0aRJE3u3j4iIVKak1HhYisi+bApusrKy8PLLL6Nly5Y4fvw4tmzZgl9++QUdO3Z0VPuIiEhlLM2WshYTN2SJ1cNSH3zwAd5//32Eh4fju+++MzlMRUREVJVKNTeyueCSmxoNF/WjarE6uHnllVfg4+ODli1b4uuvv8bXX39t8ryff/7Z6h++fft2zJkzBwcOHMDVq1exZs0ajB492uz5cXFxGDRoUKXjV69eRXh4uNU/l4iIlFOp5sbMeQxsqLqsDm4mTJhg940x8/Ly0KVLFzz66KMYM2aM1Y9LTExEYGCg4XtTG3kSEZFzsrxCsbW7gnNgisyzOrhZtmyZ3X/4iBEjMGLECJsfFxYWhuDgYLu3h4iIHK+oxMKwFJEduOTqAl27dkWDBg0wZMgQ7Ny50+K5RUVFyM7Oln0REZEyNh5LwVc7z8mOmd1+wQLmbcgSlwpuGjRogE8//RQ//fQTfvrpJ0RGRmLgwIE4ePCg2cfExsYiKCjI8BUZGVmLLSYiIqknvzlQ6ZjZmhtYP0xFJFWtjTOV0qZNG7Rp08bwfUxMDM6cOYN58+ZhxYoVJh8zY8YMTJ8+3fB9dnY2AxwiIiei1VZjV3AiC1wquDGlV69e2LFjh9n7vby84OXlVYstIiIiW1QnoGE9MVniUsNSphw+fBgNGjRQuhlERFRNstlSTN2QHSiaucnNzUVSUpLh+3PnzuHw4cMIDQ1F48aNMWPGDFy+fBnLly8HAMyfPx/NmjVDhw4dUFhYiC+++AJ//fUXNm3apFQXiIiohrRmAhp7Lz9Ctw5Fg5v4+HjZonzltTETJ07EsmXLcPXqVSQnJxvuLy4uxvPPP4/Lly/D19cXnTt3xp9//mlyYT8iInIN8gWKGdBQzSka3AwcONDiQkzGa+u89NJLeOmllxzcKiIiqk2WAhomb6g6XL7mhoiIXJvZYSmj7+sHcnIIWYfBDRER1Rp3E5GMuYJijUYe4AT5eDiwZaQmDG6IiKjWuJkMbsyfLy1c4DYNZC0GN0REVGtMZW60Gi7iR/bF4IaIiGqNycyN2bPlpcacGk7WYnBDRES1psphKQsBDEMbshaDGyIiqjVu2sqXHUsZGel9Jh5KZBJfKkREVGtsrbmRroXGBf7IWgxuiIio1tgyW8r4OEtuyFoMboiIqNZUVVBcOaDhTCqyHYMbIiKqNaaCG2vXr+FsKbIWgxsiIqo1psITSxtnWjmRikiGwQ0REdUaU1slm8vIaIzOZ2xD1mJwQ0REtUYvKoc35jbONMZhKbIWgxsiIqo1On3l4MZiQbHktrVBEBGDGyIiqjVN6vhWOubmVnEpkmZ2jLM8XOeGrMXghoiIao2fp3ulY16S4KZUVxHQFJfqjdI6jmwZqQmDGyIiqjXloUuAV0WQ4+lecSkq0ekNt0uNhrA4LEXWYnBDRES1pnw7Ba0kUvFwkwY3QnK7ItABOCxF1mNwQ0REtaa8jEa6mJ80IyMNaKSBDsB1bsh6DG6IiKjWlBcJyzbLlAQtpXq98UNMnkdkCYMbIiKqNeW5GHn9TMU3xaWmlvkrfwyjG7IOgxsiIqo1poalpIwzNwxnqDoY3BARUa2paljKuIhYiisUk7UY3BARUa2TZm6kIYtxEbEUQxuyFoMbIiKqNeWZG3PDUpYyN8YPOZOea7d2kbowuCEiolpTXnMjDVSkw02lljI3RsNSr605Zte2kXowuCEiolpjKnMjDVmKjRfu05g+DwAKSnT2bh6pBIMbIiKqNRWZGzOzpXR6eLiZvs/4IcYbaxKVY3BDRES1pmKdG3OL+AnZdgxSxsNSloaw6NbG4IaIiGqNMDUsJYlZikv1cDdTbGx8lJkbMkfR4Gb79u24++670bBhQ2g0Gqxdu7bKx8TFxaFbt27w8vJCy5YtsWzZMoe3k4iI7MMwLGV2ET8h2yVcynhYSqdncEOmKRrc5OXloUuXLli0aJFV5587dw4jR47EoEGDcPjwYUybNg2TJk3CH3/84eCWEhGRPRgKiqWzpSQ5mRKdHgPbhAEAwgK8ZAGN8a7gDG7IHHclf/iIESMwYsQIq8//9NNP0axZM8ydOxcA0K5dO+zYsQPz5s3DsGHDHNVMIiKyE1M1N9KYpVQn8L97OqBteACGdwzHXZ/sMNynNfo4ruOwFJnhUjU3u3fvxuDBg2XHhg0bht27d5t9TFFREbKzs2VfRESkDFPDUhoAnjeLiDs1CoK/lzsm9W+OiBBf2WONMzcXMvJRXGp+0T+6dblUcJOSkoL69evLjtWvXx/Z2dkoKCgw+ZjY2FgEBQUZviIjI2ujqUREZIKhoNiogOb3af3x34Et8N6YTuYfbKJM58cDl+zZPFIJlwpuqmPGjBnIysoyfF28eFHpJhER3bLKB5Lks6U0aFHPHy8Pb4tQP0+zjzVVgpyRW2TfBpIqKFpzY6vw8HCkpqbKjqWmpiIwMBA+Pj4mH+Pl5QUvL6/aaB4REVXBsCu4mRWKjUnvM7XwX1gg39+pMpfK3ERHR2PLli2yY5s3b0Z0dLRCLSIiIluY2lvKWqYWNX75pwT8/U96zRpFqqNocJObm4vDhw/j8OHDAMqmeh8+fBjJyckAyoaUJkyYYDj/ySefxNmzZ/HSSy/h1KlT+L//+z/88MMPeO6555RoPhER2ah89rabmRWKLTF32kNf7qtZo0h1FA1u4uPjERUVhaioKADA9OnTERUVhVmzZgEArl69agh0AKBZs2b49ddfsXnzZnTp0gVz587FF198wWngREQuQpgcljIf3ci3aahGuoduSYrW3AwcONDwQjfF1OrDAwcOxKFDhxzYKiIicjRrMzfubrZneIhcquaGiIhcm97E3lLm9pIqu6/iMmUpw0MkxeCG6BZy6UY++r3/F5ZsP6t0U+gWVZ6sl2ZhpNkZYx7M3FA1MLghuoW8vzERl24U4N3fTirdFLpFmc7cmL8UebhJMzdE1mFwQ3QLuZ7HBc9IWYZF/CRpGEuZG3dJcGNqnRsiUxjcEN1C8ot1SjeBbnUm9paynLnhsBTZjsEN0S0kv4jBDdW+dYcv477PduNablHFsJS1mRstgxuynUttv0BENZNfUqp0E+gWNHXVYQDA7N9PGYalpMkai7OlpDU3jG7ISszcEN1CCiTDUjq9+TWmiBxhV9I1FJaUvQa1ssyN+UuRp5mC4pZh/nZvH6kHMzdEt4Bjl7Pg7qaR1dwUlOjg78W3AKo9V7IKDbetXufGTM2NpccQ8Z2NSOVyi0px1yc7Kh3PLyplcEOKkYYm1VnEjzOnyBIOSxGp3I28YpPH8zhzihQkHRS1draUNAZyY+aGLGBwQ+TClu48h9XxFy2eozVzEcgrYnExKUcv2VfQ2nVupAXF5l7XRACHpYhc1pXMArz5ywkAwJhuEWY/yerNFA5zzRtSknTPZEtZGA8z9zG2IUuYuSFyUTmFFZmXvGLzWZhind7kcUuPIXI0acztYWG2lLmCYjfW3JAFDG6IXFSJJGj562Qa+ry3Bfcu3oWiUnlGplRnJnNTpMPp1BxczSpwaDuJTBGS1I2lLIyHme0XjIelvt+fbL/GkctjcEPkonIlNTOvrT2GlOxCxF+4gVNXc2TnlZjJ3JxKycbQedsxfP7fDm0nkSnSYSlLi/OZ2zjTOHPz8k8J9moaqQCDGyIXJS0IlgY60lqaEp0e3+83XXD888HLAICsghLZp2hSnl4vVP870VvZP3PbL3C2FFnC4IbIhZTq9Pjwj0TsOnNNFtBIFUi2WPh+/0Ws2HNBdn/5xeJyZsVwFIuLnUdxqR7D5m/HpK/jlW6KQ1m7QDZnS1F1MLghciHf7b+IhVuTcP+SvWaDmw1Hr2LEgr9x4ko2dp/NqHR//UDvSsekxclU+7IKSvDQl3vx04FLOHDhBv5Jy8WWU2kAyrbMyMgtUriF9idgXXTTv1Vdw215QbG9W0RqwuCGyIUcuZhpuP3qmmMmz/n54GWcvJqNKd8dRD1/r0r3hweZCm5K7NZGst0nW/7B3/9cw/Orj1Sqker93p/o/s6fuG5mMUZXZe2oW9+WdbFyUm/snnG7bIViDkuRJQxuiFxIrg0Zlms5RcgqqBy0hJvI3GQzc6MoaeBSqq8Ibkp0esPvRhrYqoG1NTcAENOyLhoE+cgyN9x+gSxhcEPkQnKKbMuwZOZX/rTfKMSn8vMyc6MoneRCXyKZut/mtd8Nt//77QG8se4YsvJLMHNNAuLPX6/VNtpbdTal5/YLZC0GN0Qu5ExantXnajQak5mb6OZ1Kh1jzY2ySiVXeum6RNIAoLBEj693X8Dsjaewcm8y7v10d2020e6qMxtMugcVC4rJEgY3RC7iWm4RUrILZccs7aYMAJkmgps+kuCmS0QQAAY3Sjh5NRtjP92FvWczZFtkmFuXqNyxy1mOblqtqM5Md+kmmlyhmCxhcEPkIi5kVM7amJr5JJWVXzm48fF0w2/P9scPT0SjRZg/ACCbw1K1buJX+7D//A3c9/keWeamquAmQSXBTXW4STI3HJYiSxjcEDkhIUSl6b/pOZWnA5ua+SRlnLn5d7cIAED7hoHo1SwUgd4eAFhzo4Q0ye9Tmrl58cejSjSn1k2+rTnq+nvh8duaW/0YaebGVEHx6dScSsfo1sTghsgJzd10Gt3f+RNbE9MMx6oKbhoaBTq5RaXQGVVtzh3XRfZ9gLc7AA5LKU2nwtWI03OKkJSWa/b+sAAv7Jt5B2be2c7q55QOw5raa/PJFQdsaiOpF4MbIie0cGsSAODtDSfw1LcHMP/P07h0o/IGl9Jp3ff3biy7zziwMaU8uFkdfwm93v0Ti+PO1KTZVE2ZJoYPXV3Pd//E4I+24XJmgckhJI3G9qJgNzfpsFTly9fZa3l4/ocjtjeWVIfBDZGTkda/nE3Pw28JKZj/5z/4bPvZSuc2kGRrujcJtflnBdwclioo0SEtpwjvbzyFSzfyq9FqqonD1VzD5uL1fKfag2rtocvYfjpddizhUiZ8PdwAAHUli0pWZ52aqjI3APDTwUs2Py+pD4MbIichhECJTo+L1y0HF9K6gzr+nobbXSODbf6Z5ZkbqdRs9S317yw2HL2Cuz/ZgeQM+wSQ/T/Yis9NBL1KuJCRh2nfH8aEr/bJjpfohGHYTfrarU45sDQDxNlSZIlTBDeLFi1C06ZN4e3tjd69e2Pfvn1mz122bBk0Go3sy9vbclElkSt4+aej6PHOn0i4ZP1smK6RIYbbPp5u+PTB7nhtpOkahvG9Glc6Vl5QLKXGfYycxZSVh5BwOQuvrk2w23PG/n7Kbs9VE9dyKxaMlGaTnvnukGFjVg9puqUasYk0c8N1bsiSyh/batn333+P6dOn49NPP0Xv3r0xf/58DBs2DImJiQgLCzP5mMDAQCQmJhq+1zCCJxX4Ib4snT7vz9NWP6ZZXT/89N9oQ7p/eMdwFJbo8M6vJw3nDGpTD1Nub4XON9e0kTKVuVHbHkbOyNymp9XhLNd4aVal1Ey9l3sVs51s+RnM3JAlimduPvroI0yePBmPPPII2rdvj08//RS+vr746quvzD5Go9EgPDzc8FW/fv1abDGR/UmLf6saFnr29lYAgKHty1733ZuEokkdP8P9nkbFCCG+nujeJET+qfmmAFOZGwY3DmfPC7PPzXoWJej1AuuPXEFyRr6sT+bW6vGQFAFX53+AKxSTtRQNboqLi3HgwAEMHjzYcEyr1WLw4MHYvdv80uK5ublo0qQJIiMjMWrUKBw/ftzsuUVFRcjOzpZ9ETkbW7IlwzqG4++XBmHRA91M3m/8pi/9tGws0ETm5lpuEYQQsrVXyL7suQBdXrEOf55Itdvz2WLt4ct49rtDuG3OVlmfikpMBzfS12J1Mu7Sn8GNM8kSRYOba9euQafTVcq81K9fHykpKSYf06ZNG3z11VdYt24dvvnmG+j1esTExODSJdMV8rGxsQgKCjJ8RUZG2r0fRDVlag0bcwK9PRAZ6msyE2OKpYuAqcxNVkEJ3vn1JDq/uanK4maqHnuvrjtpebxdn89a+8/fMHm8sFRn8rj0pWgp6DbHzYrZUkSAEwxL2So6OhoTJkxA165dMWDAAPz888+oV68ePvvsM5Pnz5gxA1lZWYavixcv1nKLiaqWmiPfMyrEt3LQUS7Qx7ZSOUvpe2+Pym8BBy/cwJc7ziG3qBS/H7tq088i6zhq64Cz6bk4ebX2stPS2U96SRGxuUykdNa68fCpNdytrLmxZo0nUjdFg5u6devCzc0NqanylGpqairCw8Oteg4PDw9ERUUhKSnJ5P1eXl4IDAyUfRE5m7Pp8n2jejY1vWaNu1Zjc42FpeuoqaGB85Jpyu4mFkqjmnNEcJOVX4Lb527DiAV/m9xTzBGk2cNxn1WUEoz8eIdNj7WWm5nZUtL1c4Cq9+dSwtn0XJP7w5FjKPrO5enpie7du2PLli2GY3q9Hlu2bEF0dLRVz6HT6ZCQkIAGDRo4qplEDpeUJt8Tp1k9P5PnBft62FyrMLS9dR8UTDG1qzjV3NXMwqpPuql5XdOvBWNd3tpkuH05s/Jq1vZy7HIW7lm4A7uSrsmGlsqne1sizdxUJ8Azl7mpK1nvCXC+4Ca/uBS3z92GAXPiUOpkbVMrxT+WTZ8+HUuWLMHXX3+NkydP4r///S/y8vLwyCOPAAAmTJiAGTNmGM5/6623sGnTJpw9exYHDx7Egw8+iAsXLmDSpElKdYGoxo5fkQ8lhAWYXrvJx9P2mTG3ta5n1Xlt6gdUOpaZz5lTjpBowwaPvl62/86LHXgBnfR1PI5eysL9X+yVzX6yRk0Hi7RmMjfGWaBSnXMNS2VI1gAqKKk6CKSaU3ydm/vuuw/p6emYNWsWUlJS0LVrV2zcuNFQZJycnAyt5A/oxo0bmDx5MlJSUhASEoLu3btj165daN++vVJdIKqRzPxiJFyWL9xX198TXSODKy3LX2DFp2Mpa1Yt/t/d7bH6wCW8emc73P/FXqO2MXNTU0IIFJXq4V3NKdvVqU1xZObiuiTgrU5RcE3It1+ouG3cDmfL3EgVluhh5rML2ZHimRsAmDJlCi5cuICioiLs3bsXvXv3NtwXFxeHZcuWGb6fN2+e4dyUlBT8+uuviIqKUqDVRPaRmJIDIeS7enu4afHNpN5YOam37FzpKrDWsCbz/3DfZvj12f5oKhn+aBtelsW5wcxNjU1eHo8ub26yabp/qzB/w21Pd9Nv05aGq0p0esQlpuGtX07Y5UL/44FLGDhnK5LScmQBRnXqZmrC3CJ+xhmkDzclwplIfweFzNzUCqcIbohuVT/sv4i5m8tWJI4M9TUcbxseAH8vd/RsJi8sLl+4z1q2rAUSLJmh9Z+eZUsmZLHmpsb+PJmGolI9fjlyxerHSLM85gIIc0EPACzdeR4PL92Pr3aewzd7LljfWDNeWH0E5zPy8eKPR+UZExvrZmq6yae5Rfw83OXtKF/t21kUldovuEnLLsSkr/cjLjGtps1SNcWHpYhuVVkFJXjpp6OG78ODvPHn9NuQllOE5vXKPrlLLx4dGwXivTGdbPoZttQe+3q649MHuwMA6gWUFWgm39x1+lpuMeoFeFl6OFXB0vTktuEBOJVSUYcjjRm8zAQx5o4DwGbJon7nr9Vsho60ADansFQWbLnXcuZGmqCRjkQ5+6y+YklwU9Oam3d/O4k/T6bhz5NpOD97pOy+K5kFWLk3GQ9FN0H9QPNjX3P+OIXreSV4718dVbt9EYMbIoWcSc+VfR8e6I2WYQFoGVZR2Ct94/nvgJaVprxWRWPjIvfDO4bL2paZX4JmM34DACz4T1eM6trIpuejCnoLWYs6RrN9pMxlaLzcravhqUlxsU4vMPijbYbvS3V6WeamppkYW0mDGDfZ8JhzX6ClmRtb6+aMZVgYmh6/ZA8uZOTjn7QcPHN7Kxy5lIn7ezWWvY8UluiwaOsZAMATtzVHkzq+KNbprX49uQrnDneJVOxMmjy4sfRJq7raNqg8A8oawT6VFxHcmXStps255UizHtLNTAGgSZ2KYUg3C5kHcwXFloalpL7bd7Fai9rp9QIXr+fL1j0q0QlZNtFSwGZKTWMhc+vcuFLmprDUfLB5NasAr689hrNGH3ykLBWmX7j5uzqUnIm7PtmBV9ccQ8tXf8fesxm4Z+EOvLY2ASlZFcsQFJXq8eraY+j21mZcuqGu1cid+xVBpGJnjBbuMxfclH/oimocbPVzr326Lx7t2wwvDmtTrbYFmQhu8mr4ifNWZOlC5i35pFypdkVTddGupWEpY499vd+q8y5nFhgufhOX7sPAD+Mq3X9VcnHcmZRhdRsAQFfD6MZXshSCtJ7M1KytjU60unaRZDsK48xNblEp9p27Dr1eYPzne7BizwW8uuaY2efyNVoOIruwBAeTb8j2gkuTbOei0wvc9/keHL2UhW/2JOO/3x403JdTWIKVe5ORV6zDku1nq90/Z8RhKSKFVBqWCjI95HT49aHIKihBw2Afq5+7a2SwVdPAzTFVS2HL/ldUxtIQhJdk6wtLhd/mMjTST/CNgn0sLtwXl5iOtJxCs+snAWXDFX1n/wUASHp3BP7+p+pM3bbT6VWeI1XTzVgbBvtgcv9m8PFwk2W0TAWAT35zEM/c3hITY5raPJxrL5czC/DQl3tRX/L/nm1UpP/wV/sQf+EGYsd0MmTJTltYB8nPaN2jexfvwunUXHwy3rpZw9LtOXIKSw23i51sbaCaYuaGSCHWDksF+XqgsWQIo7YF3Nw5/FougxtbWZoZYylzI/1OeuGWBjrS26b2CDNWbCGLBMjXNMorsk+Wro6fJ+r4VdQT1TRzAwCvjmyP6UPbyIqLzdXcfPJXEqauOlTjn1ld7/56AmfT87D7bEWGq3wSQXm9UvyFss1Hv99fse9hRKgvsgtLMHl5fKVZdj4eFTmJ/4tLwunUsveRZ76zvZ/ZhRW/c2deG6g6GNwQKSA9pwjnjPaZsfSpWgkrJ/fGy8PbYs1TMQCYuakO4+BGmkGQZm7cLBTESoMYLzO3rVkgcGtiOqb/cFh2QZOSxlfmdvU2R7rfWfsGFfv3abUaWW2MPTe0lA9Lmb+U2Tp0Zi9CCPyWkGLyvpd+PIK+s/+S/S7cZf9Pesz9IxGbT6Time8OQacXeGJFPN799YQskP1gY83W81m5N9lwu7hUj6tZBViy/azZ14gr4bAUkQJ2nbkGIcqmAIf6eaJRsI/VBaK1JaZFXcS0qGvYhDGnsBR//5OO06m5mBjdpNanAbsi42m/Pp7S4MRCzY2E9D4vdzfkoGwoQfp6kQYXWg1gKoZ4fW1ZHUfr+gGYEN0Evp7yt/9SyYOs2SdKqmldP8Nwh7RdQgDSl4k9J1dJg5vqrOLsaH+eNL8OTfk6PI8uraiFkhZnl+oEjlyqWLV877kM/HG8bHr/kwNa2K2Ne89dN9wuLtXj4a/2IzE1B/+k5eCDe7tI2qPH7N9PoXfzOhhi41pbSnG+VwSRyv1y5AqmrjoMAOgSEYyVk/tgztgulh+koEAfd8PF46Ev9+HtDSewbNd5ZRvlIoxrbqRDUdJP4JY2kZQHN1VnbhqH+lpc32j276fQftYf+HjLP7Lj0v2Y8otLjR9mkbQvXrLgRsiCEHtmbmqymGBtsFQ3U658SAoADiZnGm6fSsmRbb2y92xFEFJg4+/GWueu5Rn2PPvdKOO07vAVfLHjHCYvj3fIz3YEBjdEtWj76XTZ2HiDYOcaijJFo9FUWsBvk2SROKos4VIWjl/JqpS5kQYh0syNm1E0Is1+SId15EGENFCSPJdWY9WGlh9tPo3iUj0eW7Yfnd74A3skdSG7z9g2lCPNnHhJ2iIgz7DYo+amnJ9XReapqiziL0eu4PW1x+waXFWlunuJmbJAEog6atVw2WaumrIFAZ9YEY89ZzMqFavnFpUir8gxQZa9MLghqkU7jNaKkRZbOrO6RovMXb5hfmbOrS6nsAR3L9yBkR/vkM1GAeTDR9JAxXgqs5+n6SGrQMkUfdmwlOR8jUYDa9duXL77PLacSkNOUalstWzjNXmq4iUL2irapRdClmGp6WwpKX9JcCMtKG4p2Zer3DPfHcKKPRew9tBlu/38qjhqYcHa2BIlp7AUr609hj+Op+I/n++R3VdUqkNM7BYM+jCuVoNFWzG4IapFxqnqIF/XCG5CjYKwK1kFsrU7qIJ0jRHjGSzSImLpJ3vjJfB9JRduaYYiRPJ6kQ1LGQ0FSZ+tqYWZdrYGMeZIf740i6PXy4Mbe07JlgY30kX8/DzNZ0zSHFwUL4TA+Wt50OmFzXVL1rqRXzvFvn+dqqgZkr6ejl/JRnZhKdJyipCR57yTDBjcEDnY1awC3L9kD/44noIDkjH2+oFeGHFzuwNn104yA8bbQwshmL0xp6jE/JRaHzMZDuPP+L6S8+pIAoKGkmFMczU3Qsj3FDMuHK6J5vVM70QuzdxIh9EE5G1Z9EAUejUNxbdGu91Xh7+3dFhKUpdkYTjI1hWVrVWq0+PSjXysO3wFAz+MQ4uZv8mG+exJWotTFXuVIkkDquNXKtbJSc7Ix+Tl8Vhvw6awtYWzpYgcbOFfSdh1JgO7JHUM3z/eB72ahbrMpnVPDmyBhMtZ6NU0FD8fuoxz1/KQLtng05ysghKTqx1bUqLTY84fibiSWYB593U1u0Kvs8qzUPDp7WG6TkajKQtWyvcgkq5CW1eSNWtdv2I7DQ9JcGP8fyTdU0y+qq/pmVTWMl4dt5w0cyMdjRECCPCu+P23DAvAD09GV78BEtLMjawtFoIbRw2jPPnNQfx5Ul6HFpcoX+DQXauRzUirDb6e7si9WRvjptVUu/9f7TxnuH1MMotrxZ4L2HwiFZtPpGJgm3oI9Lbtb92RXOtdg255Or3AjJ8T8L/1xyGEwLbT6Vi0NcmuY/n2diFDvmdL/1Z10bt5HZcJbAAg0NsDKx7rjWfuaIV6NzMJaTlFWPDnP9h0PAVbE9Pw78W7DL8XoGxn6i5vbqo0K6dcblEpfoi/iNOpOUhKy8Gkr+Nx8mo2Pt9+Fp9vP4sNR6/itg+2oukrv6L/B3+5zN43xivQSuuVpHU20otzbmGpYbFEQD4sJc3ctJQEk9JaHOnsprIi3oqfLx0KC7Qx0DTmYyZwkK3Zo5UPkUU3r1Ojn2mOdPhJujGlt4UlFRwV3BgHNqZ4uGktzopzBGmgZy4wtdWGoxVZmut5FZt4HrucZep0xTBzQy6hRKeHVqPBR5sT8d2+soWnmtTxxZu/nABQtt1Am/AAZOYXo0U9f8UDh1KdHkcvZ6FLRDCOX5H/0fdx0Jt9bakbUHaxXrbrvGGYzdNNi2KdHgcu3MDDMU3RtK4fZvycAKBsVs6zd7SSPceNvGJEvb0ZQNmbbr0AL1zIyMeWU6mytVDK9zG6eL0Am46n4tF+zRzdvRozLiL283LHtZs7OfuYydxculEAf8l50gu3dMfwMMkq1tILuvRnCiFkr39pDUyAt7thJWJPd22VqxYbM5cVkc38ksQWegE8NagF/jqVavfXvXSGlLT+y1LmxlHDUtZwd9NALyqyJ+V/M44kDWh8Pd1krxM/TzfDfnG2ZJWke8xdkgxNO9sinwxuyOmlZBViyLxt6BIRLBvHLg9sACAuMQ1PrjiAnKJSLJnQQ/GFpl5dcwzfx1/E04NaVCoAHB3VSKFW2Ud55kZaPyR9k160NQnFOr3F7RqOSj7l5RfrDNktS9eeq1kFKNHp4aaRr3rrbHKMVnf1k9S8SC+80jqRSzcKMLxjOJbtOg8fDzdZEBPi6wk/TzcUlOgQEVKxv5h0i4QcybRcvZDX8EhnVZUNGxTcvF0RTPl6ullVAGsucyPNSEmHxAQEAr09sOm5AVU+d01IM7eWtqL45K8kPDe4tSKvHw+3slq18qDUz8sNxfllt2s6XGguOJEGN2Wvw4q/SR9Pd0Og4uvphuxC26d2n7tWscp6ek4Rtp9Ox9e7zuOdf3VEgyDr98JzBA5LkVMTQmDIR9uQU1iKHUnXzH66WPL3OcMb/O4zGSjR6bHu8OVamTZprKhUh+/jy/aJWbT1TKX7G9mwAaYzCjOzB1a51QcuYd1heYHh6EU7cSY9F2nZhdh2Oh3JRltPWFJ+Qf3lyFV0eXMTer77Z6VsmDMxfs1JNzqUrhfkrtWgT/NQAMDIzg3w8vC2eGVEW/z6bD/8K6oRRnZugLdGdYCbVoPdM+/AwdeHyIKjLpFBhtvSgEpAHt14GGVuKm5XDFFJh8jahlfU9RjzMTO0Ic3cSLMjjh4tfqxfM7QND8BdnRsajlW1vsy+89ct3m8ra3cfd9NqZAGttCDaXP1QgJnjxqTPJSX9fRn/7qQF6fYoOk/PKcKEr/Zhy6k0vLn+RNUPcDBmbsgpFRTrsGLPeTQM9pF9KgWAEF8Pi9Mh/0nLwfu/n8IXO87hni4NMXdcF+w7dx0tw/zNbk5pD7vPZGD7P+mymUXl6vp74lpuMe7tHuGwn19bWplYRwQAwgO9kZJdaPK+wxcz8b/1x3Hgwg2bp8h2jgjC3nPXDc+dX6zDsp3n8eaoDjidmot2DQJkF1cl6PUCi7YmoXvTEGw/LV/LSDqVW7oGy/W8Ynw+oQe2JaZjcLv68PF0ky2tv+j+bobb0kLNHS8Pwpn0PMS0qGs4Jh1uyCksNZu5MRfQ+Hu7G6ZJ+xldUKWFqNZkbqQL9QkHDwO9fld7AECSZBPaqoKbr3acw4yfE/D5Q93Rqr75QM6S/OJSfLAxEXd3aYgnvzlo9rwW9fxwJr0ikJdPWZcHN6YyJ/7e7pXe/0zx83SXbXxaTpq58TKqRZKuwyPP8LjJhp6slSr523eGTXaZuSGnI4TAxKX78N5vpzBlpXydEDetBise641ezco+8b49qkOlx//9zzV8saOsun/9kSto9erveOCLvXh46X6Lb7aFJTpcsCKjkJiSgytGK3amZhdi/JI9WBx3Bs+a2J3312f7Y9XjffDevzpV+fzOThq8dWscbLjd+2YWwpy//7lmMbAZ16Mi8Hv29paG210igyudezD5BgbP3YbRi3bipR+PVrq/tq09fBlzN5/G/Uv2VsoMSIMF6SfkvCIdAr09cHeXhmYzIqZEhPhiQOt6AMqK0wHgPz0jDfdn5pfIhl2Ma27KBfpIsjhmFsQD5L8Xc+2U96viYlxbi7xJs2NV7TO16UQqzl3Lw/Orj1T75725vmwLkn8v3mXxPOOMjHQ0zM/MWkbmHm8pi2Mu8yPdQdx47zppzZL09yptiy373Un3wrLl9ewozNyQ01l7+DL2nZNfIF4c1gb3dGmIwhIdWtUPwJcTe2D/+esY2DoMOr1A7O+nMGNEW8zdfLpSQWe5k1ezcexyNjpFBJm8f+aaBPx88DJWPd7HbPHjxev5GDZ/OwK93XHkjaF477eTSErLRYKFmQL3dGmI+oHeDs0a1aaIEB/c3jYMuUWleHd0R4z8ZAcCvNxxV+eGlYajqqLRVNTZtJDMBIpqHGK43Vny+yrPDkk/Dcefr6j9qU2L487gYPINLLq/GxJTzO8jJC0O9nDT4M17OuCng5fwQJ/GNW7D5w/1wLErWejWOES2IF/ZHlYVhcPlpBmgAK+K236y4EZ+QZNmQsxlbqQzt6QzaGprEqN0kckSK4t0j16yfWhz4V//IMjXExuPm97t25hx0CL97zDOnJniLxtGNJ/FkQZ3Ur6y155RcKM1nbnx96rI4gV4uSOjtOz3WdVUcmn9jT23nqguBjfkNLILS7DlZCrWHqp8gezeJASRoRUrrQZ4e+D2tmVFww/3bYaHopvCTavBX4np2H66bH0J6boh5d785TgSU3LQq1koXhreFmsOXYaXuxbP3tEKPx8sW5r94y3/oFvjEGw/nY7bWtdDUlouHl8Rj84RQTh3Lf9mW0ux5WQalvx9DlVx9j1YbKXRaPDVwz0N3x+ZNRRaLVBYooe3hxaFJXq8PbqjYRfqMd0aGf5vGwZ540pWRfq6Z5NQQ6Yj2LfiYttaUvfRrG7FwnFtGwRUGvq6klWAg8k3sPVUGro1CcGgNmF27K157288BQD4LeFqpdeZlHHgMDGmKSbGNLVLG3w83dCzaVnGTPp/Wz/Qy/D/JM1kBBpdLMvJV/s1Wi3Zs+rgRpopysgrlgWttUE6LGlpnSFj/1t/HGN7ROBGXgl6NQuFp7vWMNusVKfH9B+OoFvj4LL3mC/34u9/yoYcLWU0GgX7GPZikma0yv4/Kv5TZMGNFZkbb083eLhpUHJzg9PyvzXAfObH8rCUNHMjXYZAnsXJuBmsBnq7G8oBqiqANvc6qU0MbshpTFl5yBCYGOsSEWzxseXrRwxoXc/wHK+ObIdZ644DKEvbr9p/0bAL75ZTadgiWV7873/SZc/1f3FJmP/nP7i/d2PEnUrDlaxC2bRHAJhkYYfcMVGN0LSuH+b9eRrPGE2DVpvyFLSXuxvWT+kHHw83+Hi6GYKbYR3CDcFN54hgXMmq+NTbrUmIIbiRXjDq+Hnimdtb4mpWIdpLhsHqmVi+XwhgzP+VDQ/4ebrh6P+GOXw9EensnJTswkpbUYQFeBk+/Qb7VGQVjPeQsqcvH+6J6T8cwfNDWt8saC/LTEgvfNJ1bqT1N9Lbbkabbkov0OaGG6QBUUZuMRoF+1T6e6kt5jK3pizbdd6ww/1DfZrg14SruJ5XjNfvao/mdf2w/sgVrD9yBf/uHmEIbABYnEIf5ONhCG68jGZuSQM+aXZDWn8jnSJuHHR6umlRoiuf4eSOwpLiSufJAyDzmRvpn4ivbD0c00FXoE9FraOvpzuKSnWGn2OMwQ3RTUcuZlYKbN7/dye8/FMCnh/S2uox3InRTZBbWIq+Leugc0Qw9p27jobBPmhRzw+r9l80+7iDyZmG25duFBjeyFbuTbapH2+P7ogD56/jrdEd4XezQNSWcWtXV76CrvTiL63hMF5n5Nk7WqK4VI+RncNxObMiI+PlrsXzQ9sYvh/Uph62JqZjQnRTrD5wyXC8eV0/nJWkw/OKdYg/fx0r9yUjxNcTb9zd3q5rHh2+mAkvdy0aSqa55haWVrqQN63jZwhu2jaQrCrswNWW2zUIxO9T+wMoq0nafHPndun0ceneVAFmsjjywRP5hcrcIoBuWo1h2CIixAeP9G2GF1YfwVAFlmSQzlazZf2WFXsuGG6/veGEYSYbAOyvYoaVNGsmzUDKsyVC9vqXvi/I6rIkU8SlwYUGGni6aw3FvtLfi/Tx3h5uKNGVBXjSzI2lndONh6UMt828RrzctdAAhp9jzBne8xjc3IJ0eoHcolIEeLnj233JqOvniRGdGsjOKSzRYfWBS8gvKsXjtzVHblEpPNy0DhlLLS7V4+0N8qmDfZqHYlyPSAxqEyabPlsVdzctpg6uyJQsvDnjJKewBAu3JiE1qwgPRTfBlzcLjqWfsMtJx45N6d+qriH46deyrmGn77dGdcBDfZrgoT5NDOd6ujvveiyOpNVqMK5HBI5czEJMi7qY3L8Zlu06j2dub4WCEh3+/ucaYlrUga+nO2bdXTbjxc+rYs8a44Dks4d6ID23qPI0etkeSmVrtdwn2cX43u4R6NjIdI2VrbLySzB60U4AwJbnK9ZtWbrzXKXZJdJgvJPk59dWge1Tg1riyKVMDO/YAE3qVAzrSYvBpRcrac2GcRulfZHOlHvzng54Y31ZZtRNq8EvU/ph0dYkPD+0NZrV9UPb8ACTO3Q7WvO6/oatD2qy3cGesxUBzaPLzGdpASDU39MQ3Jjb3BSQh43S+/y95Fmc8plP0uCibIuOiloq6e9MGpD4eFQs1me89YY55gqKA2S3K4I2rVYDLw8tzK3bZ23dkyMxuLnF6PQCD36xF7uNNnX74N7OyC4owY6ka5h6Ryss2PKP4Q0ixNcT7288hVA/T/w+tb/sE0BiSg5C/DwQFlC9YtlZ645h+e6yT0xuWg02PXcbMvOL0bp+ADQaTZVrqlgrwNsDG6b0R15xKTzdtfgh/iJCfD3xzWO98fzqw9hfRVHqzDvb4r3fTuGJAc3xUJ8mGPzRNnRqFITPHuqOez/dDQ83DR7s3cTic9xqPri3i+H2zDvb4bkhreHr6Y6PxnXF6gMXMbZ7pOz8tuGB+PTB7rLNIct5umsNgc1rI9vhnV9P4uPxUVj4V8XWDoPahuHXo/I1RxJTchDg7Y4Ab49KO5vbKi2nIrN0WlJAbGrarHSYJizAC72bheJyZgGaWNih2578vdzx7aQ+AMouNB0bBSLQ2wOt6lcEG9I6IekFzTgekF4gpXtbSfvi5+WO9g0DseiBiunr9goqrfXbs/3xy9Er+O/AFrK9kKRa1/fH6dRck/fVRB2/ig9gQbLMjfzDoLmMprk1b6TDVVqNRpYRqRfgZeiLLLiRrW0jzfwY0ZgrKJbcNjO7TgPA08LyC7aufO0IDG5uMZ9uO1MpsAEgm05rvOHbSz+V3ZeRV4w5mxKx4chVtAkPwIvD2uDuT3YgxM8Tf780CAeTbyArvwTDO4ZDo9HIloE/lHwDoX6eaFLHD4cvZuJsei7ahgcaAhugbAuFFlVsxFgTQb4ehjeePTPugLeHG9y0GvzwRDQKSnRIyy7Cm78cx9ab/R/TrRG0Gg2eHNACLcP8cXvb+ogM9YGXuxsOvDbE8Phfn+kHAE69aq7SNBqNYSy/XoAXnhrY0uR5w63YJf2xfs0wOqoR6vp7oZ6/F+7/Yg+eGdRSlu3x8Shb0bd8uq+nuxbrnu4LoCzAr86FVzrccayKRQSDfaV1NlqserwPSvVCkU1APdy0+GVK2WtUo9Hgrs4NsO/cdQzvGI45fySWHZdc+rILSwz/f0Dl4Y//9IzEmfRc9GtZFz88EY1DyTfQV7LmjlLaNwxE+4aV15iSMld4W1N1JIGzdKNYaTBSqheymhvpa0HarshQXySmlgXP0qUTjEqh0Lp+AHYmZVR6vI+Z2W2VsliSxkiDMGnNjZ+s6Fk+JGlcTyRVxMwN1aa//0k3vJmV+2hcFyzY8k+lzR0B03vPfLbtLADgcmYB/rpZkJueU4S2r280nPPc4NZoFOKD/60/ju5NQnB/78b47zcHEOjjgY//E4XJy+MrzS7xdNPiucGt7dJPa0j/aMsvvE3rumNI+3BDcDPtjtZoLPl0Kk2xSx/PoKZ2aTQa1L1ZWBzdog6OvDEUAV7u+GrnecM5zw9tLZsaXVyqx4gFfwMoS8///FRfdDWxfo4l0inO3xvVbzUI8jbsgwUA7RrIF4fTaDSV1o+pTdLAb+H93aDTC9kwxeXMir//U1dzMLZHhOGDR58WZcsitL6Z9Zn9786Gc3s1CzWsOeVMyrN75VnoctIMR3TzOoYPeuWLbFbFeFG+ctL9v6SBjvT9MzO/RFboHiYZbpe2S5oRS5VkC3MKS2UrUUuzaNJsS5/mdXDqZmZROrpraV8taaG7dOmCAFktjzyYsbRwJjM3VCvSc4rw2Nf7Des6DGlfHzEt6uDyjQKM6toILer5491fT6Jfq7ro07wO1h2+jJZh/hjeMRxj/m8XMvNLMP8/XfHOrydw8XrVMyDm/XnacHvb6XRsu1konJlfgglf7ZOd2zDIGysn94Gfl7tNtTWOcl/PSOj0erRvGCgLbMh5la/dMrJTA8zbfBp9mofKApcODQNx/EpFPY9eAHP+OIXCEj2a1/XD/+7pgHs/3Y2TV7Px4rA2eHqQPKv0T2oOEi5nyd6wjS+EIb6esuBmTLcIzP/zH9lML2diPJusuFSPTo2CkHA5Cz2ahmDmne3g4abFsA7hCPT2wIm3himSdaqux/o1w+1tw9C0jh8+3XbG8GFKOvzSIKhi+LN5XX9cy7VcNOzhppEVVI/qWrGuk3QNK+nt06ny9Y9ua1UXWxPTEeTjgZiWFdkuaXAjbVd6dkVRy4WMfNlwZ2SIfGmMch0aBmLh/VH46cAl3NmxAcKDvBH720k80rcZJvdvjglf7cMTtzXH5pMVs0XNZZGk7ZIGMxqN0X5iRlP/GdxQrfhg4ylDYOPj4YaXhrWRLTveJTIYPzwZbfhe+kls6wsDkVdUijr+XujQMBC7zmTg7s4N8fiKeENRbfkLu0eTEDSt64cfb85m8XTXop6/F67lFskyNeGB3ijVC2TkFeGNezqgqWQdE6W5aTV4KLqp0s2gaggP8sbuGbfD02h9o6cHtcRT38qXyC9P5x+4cAO7zmQYpu7O+SMRd7QLw0s/HkWPJqEY3ysSQ+Ztr/JnS2fIAGWLyu179Q7ZcvvOaO7YLvhyxzlMGdQKvl5uWL77Av7TMxLeHm6GrQ0A++w9VJs0Gg2a3xzibhTsY5hRJ60TkWZbIkJ8sO982e3yZSMAoG/LOobXivGwknRjyOgWFYt+5haVYvqQ1pj352k81q8Zjl7KQu7Nta6m3N4KDYJ98MRtzdGkjh9mjGgLrUYjG46vF+CFSf2a4Ysd5/D80Nb482Qqlvx9Do/2bYajlzIRf+EG2tQPQJ/moejfqi4aBvmge5OKRS/Dg7zRv1U9w35bg9qEydZ+OjxrKNy0GsNECEC+FpKfmdlSxsNQ0oLoIB8P2fYPDG5uWrRoEebMmYOUlBR06dIFn3zyCXr16mX2/NWrV+P111/H+fPn0apVK7z//vu48847a7HFrmPR1iT8eLAs2HhxWBs8OaCFTWuAeHu4GWZIRYT4YlyPsk8LXz/SCynZhWgQ5I3sglKk5RQaAqb7ezfGwQs3cF/PSAR4e0CIsjeFb/ZewLXcYtzTpSGa1fVDUanO5d40ybmVf4L1cnfDkgk9oNMLDG1f3zCT6r1/dcLMNQmyx1w22kpj+Pyy4aujl7LMFqYaK9HpsWRCDzyxIh7je5WtPOwKr+1/d4/AvyX7nU0fUntDw7WloSS4kQ7f1JWsmRQhWSBUOiNPOvNJCPkwj3RYqUGQD8b3aoxfj17BgNb10CDIGxNjmiLIxwPz7uuKycvj8eKwNujeJEQWiDxxcy8xaaFxyzB/3NOlIZ65oxWCfDzKFqZsG4ZujUNQUKzD3M2JGNI+HO5uWqx4rLfhcbe1rofTKTmy1b1NKX//f3l4W/zr/3ahe5MQdJO0SZo5kmaEpFmj3MJSWaDYMMhHFtxwthSA77//HtOnT8enn36K3r17Y/78+Rg2bBgSExMRFlZ5pdFdu3Zh/PjxiI2NxV133YWVK1di9OjROHjwIDp27KhAD5zTgQs38OYvxw0Zm4djmlZKt9eEVqtBw5tvAtJCXQDo1jgE3SR/YBqNBhoNMMEoI+IKb/7kuoZI1lj5Y9ptuJCRj97NQ/H6umN2mZIt3Si0fqA3hrSvjz0z70Cob81mZZF9PTmgBXYkXUP/VnXRMqwiY91Isv5PG0kmW/peZrytxkN9muBQciZahvnjX1GNkHw9H83q+iHUzxOxYzrh7VEdDLNJywuLh7SvjyOzhsqe15hWq8Gf0wcg+XoeOjQMkj3ew01r2CTV28MN74w2vT/d0od7QqupvIyCOVGNQ7Bnxh0I8HaHn5c7PhkfhYJiHQa1CcOwDvWx5+x1xLSog4Ft6iEuMR2t6wdgXI8I/BB/CWN7RCIzv2JotmfTEJy4WjH0W+wEwY1GOHrb1ir07t0bPXv2xMKFCwEAer0ekZGReOaZZ/DKK69UOv++++5DXl4eNmzYYDjWp08fdO3aFZ9++mmVPy87OxtBQUHIyspCYKBjxsPLMxWi/LbhOCBQkdo0/Avz50PI74fknPLnK7+jsESPcxl52HDkimyhM1N1BES3qm/2XMBrN1dPLvfisDYID/TG86uPoFndsuGCx1ccAAD8MqUf7l64AwDw9aO98Niy/SjVC4zu2hCTb2uOT7YkYergViZ3gyfncCY99+aqyfkY/FHZMOPJt4aj3ayyiRAHXhuMd387iXWHr+CPaf2xaOsZrDl0GW+P6oDk6/lY8vc5DGpTD0sf6YUd/1xDeJCXLFBSI51ewE2rgV4vcDmzAJGhvtDrBfaczUDHiCCUlOox4+cExLSog46NgjD9hyNoGx6ATSdS0TY8ABun3Wb3Ntly/VY0uCkuLoavry9+/PFHjB492nB84sSJyMzMxLp16yo9pnHjxpg+fTqmTZtmOPbGG29g7dq1OHKk8i6vRUVFKCqqKMrKzs5GZGSk3YObAxduVLlDbG37d7cIPDGguayqnojKNlF95aejhp2Mz8XeCY1Gg/3nr6NJHV/U8/fCsl3nEezrgX9FReDAhRu4nFmAe7o0xK6ka9iamIanBrZESA3XzqHa9/c/ZQW9nSOCcfF6PrIKStCxURCEEMgv1sHPyx06vcDB5BvoGhkMN40GvyZcRY+mIbI6G6ps37nrGPfZbgBl+wH+9N8Yuz6/LcGNouMC165dg06nQ/368iW669evj1OnTpl8TEpKisnzU1JM79IaGxuLN9980z4NdkIaTcXiTJ7uWjQM9kHHhkGYGNME3Zs43xRNImfQrkGgbOZLeSq/fBNKAHikbzPDbWmtREzLurKZLuRa+reqZ7gdGeqL8qUkNRqNoZjWTauRvRbu7tKwNpvoslqF+SPEt2wPKoUHhZSvuXG0GTNmYPr06YbvyzM39tapURD2vzrYUHCmwc1aE5QHIBVRSHlAUul+VBSslR+TnlvxvNaPqxKRac4wo4NITUL8PLFn5h3ILihVdF0nQOHgpm7dunBzc0NqaqrseGpqKsLDTa9UGh4ebtP5Xl5e8PJy/Popnu5ap1inhYisc3/vxth77jp6O+EidESuysvdDfUClN8VXNFFGDw9PdG9e3ds2bLFcEyv12PLli2Ijo42+Zjo6GjZ+QCwefNms+cTEZlyT5eGWPd0Xyx9pKfSTSEiO1N8WGr69OmYOHEievTogV69emH+/PnIy8vDI488AgCYMGECGjVqhNjYWADA1KlTMWDAAMydOxcjR47EqlWrEB8fj88//1zJbhCRi9FoNOhi4xYMROQaFA9u7rvvPqSnp2PWrFlISUlB165dsXHjRkPRcHJyMrSSVT5jYmKwcuVKvPbaa5g5cyZatWqFtWvXco0bIiIiAuAE69zUttpY54aIiIjsy5brt3NvfEJERERkIwY3REREpCoMboiIiEhVGNwQERGRqjC4ISIiIlVhcENERESqwuCGiIiIVIXBDREREakKgxsiIiJSFQY3REREpCoMboiIiEhVFN84s7aVb6WVnZ2tcEuIiIjIWuXXbWu2xLzlgpucnBwAQGRkpMItISIiIlvl5OQgKCjI4jm33K7ger0eV65cQUBAADQajV2fOzs7G5GRkbh48aLqdhxXc98A9s9VqbVf5dg/16TWfpVTqn9CCOTk5KBhw4bQai1X1dxymRutVouIiAiH/ozAwEBVvqABdfcNYP9clVr7VY79c01q7Vc5JfpXVcamHAuKiYiISFUY3BAREZGqMLixIy8vL7zxxhvw8vJSuil2p+a+Aeyfq1Jrv8qxf65Jrf0q5wr9u+UKiomIiEjdmLkhIiIiVWFwQ0RERKrC4IaIiIhUhcENERERqQqDGyIiIlIVBjdERESkKgxunIRer1e6CQ6RmpqKK1euKN0MqgG1rhZx8eJFnD59WulmUDXxPZMsYXCjsKysLABle16p7Y/10KFD6NWrF06dOqV0Uxzi/PnzWLJkCT7++GP8/vvvSjfH7q5fvw4A0Gg0qgtwDh06hB49eiAhIUHppjhEUlIS5syZg5dffhkrVqzAtWvXlG6S3fA903XV6numIMUcP35cBAUFiXfffddwTKfTKdgi+zl8+LDw8/MTU6dOVbopDnH06FERFhYmBg0aJAYOHCi0Wq146KGHxN69e5Vuml0cP35cuLu7y35/er1euQbZUflr87nnnlO6KQ6RkJAg6tSpI0aMGCHGjBkjPD09xe233y7Wr1+vdNNqjO+Zrqu23zMZ3Cjk4sWLIioqSrRu3VqEhoaK2NhYw32u/sd67NgxERAQIF555RUhhBClpaXi0KFDYufOneLYsWMKt67mrl27Jrp06SJeffVVw7HffvtNaLVacffdd4u//vpLwdbV3OXLl0WvXr1Et27dhJ+fn5g2bZrhPlcPcE6ePCl8fX3FzJkzhRBClJSUiG3btom1a9eKnTt3Kty6mrtx44aIiYkx9E+IsmDHzc1NdO/eXSxfvlzB1tUM3zNdlxLvmQxuFKDT6cT8+fPFmDFjxF9//SVmz54tAgMDVfHHWlhYKKKiokSDBg3E1atXhRBCjB49WkRFRYnQ0FDh5+cnPvjgA4VbWTNJSUmie/fu4vjx40Kv14uioiJx5coV0aFDBxEeHi7GjBkjrl+/rnQzq0Wv14tvvvlGjB07VuzcuVOsXLlSeHl5ybIcrhrgFBUViVGjRomwsDCxb98+IYQQd999t+jSpYsICwsTHh4e4tlnnxXp6ekKt7T60tLSRFRUlIiLixM6nU7k5eWJkpIS0b9/f9G1a1cxZMgQcfz4caWbaTO+Z/I901YMbhRy+vRpsXLlSiGEENevXxexsbGq+WPdunWraNOmjfjPf/4junXrJoYOHSr+/vtvsX//fvHxxx8LjUYjFi9erHQzq+3QoUNCo9GILVu2GI4lJSWJ4cOHi2+//VZoNBrx+eefK9jCmrlw4YJYt26d4ftvv/1WeHl5qSKDs3//fjF06FAxfPhw0bZtWzF8+HBx4MABcf78ebF+/Xrh4eEhXnvtNaWbWW1nzpwR3t7e4ocffjAcO3/+vOjdu7f49ttvRXBwsHjrrbcUbGH18T2T75m2YHCjIOkFIj09vdKnkdLSUrF+/XqX+SQp7c/WrVtFeHi4GDBggLhy5YrsvOeff1506tRJZGRkuORFsqSkRDz00EOiZcuWYuHCheK7774TISEh4qmnnhJCCDFt2jTxn//8R5SUlLhk/4SQ/y5LS0srZXBKSkrEN998IxISEpRqYrXt379fxMTEiCFDhohz587J7luwYIGoV6+euHz5ssv+7p577jnh5eUl3njjDfHxxx+LoKAg8cQTTwghhJgzZ47o27evyMvLc8n+8T2T75nWcndsuTKVu3LlCi5fvoyMjAwMHjwYWq0WWq0WpaWlcHd3R926dfHoo48CAN577z0IIZCRkYEFCxYgOTlZ4dZbJu3bHXfcAQAYOHAgNmzYgBMnTqBevXqy8729veHr64uQkBBoNBolmmwTaf+GDBkCd3d3vPzyy1i0aBHeeOMNhIeH46mnnsI777wDoGw2x40bN+Du7hp/XhcvXsTJkyeRnp6OIUOGIDg4GJ6enobXppubG8aOHQsAeOSRRwAAOp0OixcvRlJSkpJNr5K0b4MHD0ZQUBB69OiBzz77DImJiYiIiABQNt1do9FAo9GgQYMGqFOnjku8No1/d6GhoXjrrbcQGBiI5cuXo379+pg+fTpmzZoFoGIGnK+vr5LNtgrfMyvwPbMa7BIikUVHjhwRkZGRon379sLd3V1ERUWJxYsXi5ycHCFE2aeNcunp6SI2NlZoNBoREhIi9u/fr1SzrWKqb4sWLRJZWVlCCCGKi4srPebJJ58Ujz76qCgqKnL6TyHG/evatav4/PPPRX5+vhBCiEuXLsk+Zen1ejFhwgTx8ssvC71e7xL9q1+/vujWrZvw9PQUHTp0EC+++KK4ceOGEEL+2iwtLRUrVqxwqdemcd+ef/55kZGRIYQw/dqcOnWquPfee0VeXl5tN9dmxv1r166dePnllw2/u/T0dMPtco8//riYNGmSKC4udurXJt8z5fieaTsGNw6Wnp5ueNM5d+6cSEtLE+PHjxe9e/cW06ZNE9nZ2UII+VjxQw89JAIDA52+8M/avpW7cuWKeP3110VISIjT900I8/3r2bOnmDZtmsjMzJSdf+bMGTFz5kwRHBwsTpw4oVCrrZeZmSm6detmuOAXFBSIGTNmiJiYGDFq1ChDEFB+IdHpdOKxxx4TgYGBTt8/a/tW7uzZs+L1118XwcHBLjE7xVz/oqOjxT333COuXbsmhKgY9vjnn3/ESy+9JAIDA52+f3zPrMD3zOpjcONgCQkJomnTpuLIkSOGY0VFRWLWrFmiV69e4tVXXxUFBQVCiLI3ohUrVoj69euLAwcOKNVkq9nSt3379omxY8eKiIgIcejQIYVabBtb+peeni6efPJJ0aZNG3Hw4EGlmmyTc+fOiebNm4u4uDjDsaKiIvHVV1+J6Oho8cADDxjebPV6vfjtt99Es2bNnP6TsRC29S0hIUHcc889omnTpi7z2rTUvz59+oj777/f0L+MjAzx2muviR49erjEa5PvmXzPtAcGNw6WmJgomjVrJn755RchRFlhVfm/L774oujatavYvn274fyzZ8+K8+fPK9JWW9nSt4sXL4rVq1eLpKQkxdprK1t/d2fOnBGXLl1SpK3VkZ6eLjp27Cg++eQTIUTFp3ydTicWLVokunXrJlsXJSUlxTBV1dnZ0rf8/HyxZcsWcfbsWcXaaytbf3eXL18WqampirTVVnzP5HumPTC4cbDCwkLRo0cPcddddxnS++W/cL1eLzp16iQmTJhg+N6VWNO3hx56SMkm1ogtvztXVFxcLP7973+LmJgYkxeHoUOHipEjRyrQspqzpm933nmnAi2zDzX/7vieyfdMe+DeUg6k1+vh5eWFpUuXYvv27fjvf/8LAHB3dzfMzrjnnnuQlpYGAC5RBV/O2r6lp6cr3NLqsfV352qEEPDw8MD//d//4cyZM3j22WeRlpYm20Pq7rvvxrVr11BYWKhgS21nbd8yMjJcrm+Aun93fM/ke6a9MLhxIK1WC51Oh44dO+Lrr7/Gd999hwkTJiA1NdVwzrlz5xASEgKdTqdgS22n5r4B6u+fRqNBcXExwsLCsHHjRuzduxcPPvgg4uPjDf05fPgw6tSpA63Wtd4m1Nw3QN39U/PfnZr7Bjhf/zRCqGy7XydSvh5Dbm4uioqKcPjwYdx///1o0qQJQkNDUadOHaxbtw67d+9Gp06dlG6uTdTcN0D9/dPpdHBzc0NGRgaKi4tRUFCAESNGwN/fH6WlpWjevDm2bNmCHTt2oHPnzko31yZq7hug7v6p+e9OzX0DnK9/rhXWOynj+FAIYfhFnz9/Hq1bt8b+/ftxxx134Pjx47jzzjvRqFEjhIWFYd++fU79QlZz3wD198+U8ovj+fPn0blzZ2zZsgXNmzfH/v37MW3aNAwZMgQ9e/bE/v37Xe7iqOa+Aerun5r/7tTcN8A5+8fMTQ0lJibi22+/RXJyMvr164d+/fqhbdu2AIDk5GR069YNo0ePxpIlS6DX6+Hm5mYYf9Tr9U6dNlZz3wD19y81NRVZWVlo3bp1pfsuXbqETp06YezYsfjss88ghHD6/kipuW+Auvt37tw5/PHHHzh9+jRGjBiBqKgo1K1bF0DZisvdunXDqFGjXPLvTs19A1ysf7VQtKxax48fF0FBQYZZC7179xYRERFi8+bNQoiyfWqmTZtWqaK//HtnrvRXc9+EUH//Tpw4IRo3bizGjRtnctG2NWvWiOeff97p+2GKmvsmhLr7d/ToUdGwYUMxYsQI0apVK9GmTRvx/vvvi9LSUlFcXCwWLlwonnvuOZf8u1Nz34Rwvf4xuKmm0tJS8eCDD4oHHnjAcOzQoUNi0qRJws3NTWzatMlwnqtRc9+EUH//Ll++LGJiYkSXLl1Er169xGOPPVZpg0tTS7y7AjX3TQh19+/8+fOiVatWYubMmYY+vPLKK6Jly5aGhd2MV7B1FWrumxCu2T/nzoE5Mb1ej4sXLyIyMtJwrGvXrnjvvfcwefJkjBo1Cnv27IGbm5uCraweNfcNUH//Tp06hYCAAHz99dd46qmncOjQIcyfPx/Hjh0znOPh4aFgC6tPzX0D1Ns/nU6HdevWISoqCs8884xheGLatGkoLi7G6dOnAQBBQUFKNrNa1Nw3wHX7x+Cmmjw8PNCxY0ds27YNN27cMByvV68eZs6ciTvvvBNvv/02srOzFWxl9ai5b4D6+xcTE4M33ngDXbp0wcSJEzFlyhTDRTIhIcFwnrhZbqfX65Vqqs3U3DdAvf1zc3NDUFAQ+vbti/DwcMMHB41Gg+zsbMNu5VLCRcpB1dw3wIX7p2TayNV9//33IioqSsydO7fShmfLli0TDRs2FMnJyQq1rmbU3Dch1N8/4/HtZcuWiW7dusmGOd58803ZHjCuQs19E0L9/ROioo8FBQWibdu2Yu/evYb71q1bp4q/PTX2TQjX6Z+70sGVq7hy5QoOHjyI4uJiNG7cGD169MC4ceMQFxeHJUuWwMfHB/fddx9CQ0MBAD179oSvry9ycnIUbnnV1Nw34NbqX5MmTdC9e3doNBqIspo6aLVaTJw4EQDw8ccfY8GCBcjOzsaPP/6Ie++9V+HWW6bmvgHq7p+pvzugYjo7ULbwm1arNaw0PHPmTCxduhR79+5VrN3WUHPfAJX0T8nIylUcPXpUNG/eXPTq1UvUrVtX9OjRQ3z33XeG+x9++GHRqVMnMW3aNJGUlCTS09PFSy+9JFq3bi2uXbumYMurpua+CXFr9m/16tWyc3Q6neH2l19+KTw8PERQUJDT7zSs5r4Joe7+WdM3IYS4ceOGqFevnti5c6d4++23hbe3t9PvOq/mvgmhnv4xuKlCUlKSiIiIEC+99JLIzMwU8fHxYuLEieLRRx8VhYWFhvPefPNN0b9/f6HRaET37t1FeHi4Q7Zxtyc1902IW7t/paWlsuENvV4vSktLxbPPPitCQkJMTjF2JmrumxDq7p8tfcvJyRFRUVFi4MCBwtvbW8THxyvY8qqpuW9CqKt/DG4sKCoqEtOnTxfjxo0TRUVFhuNffvmlqFOnTqVP9teuXRO///672LFjh7h48WJtN9cmau6bEOyfqazTvn37hEajcapPV6aouW9CqLt/tvYtMzNTNGnSRISGhorDhw/XdnNtoua+CaG+/rHmxgK9Xo+IiAi0a9cOnp6ehpUWY2Ji4O/vj5KSEsN5Wq0WderUwfDhwxVutXXU3DeA/Svvn1TPnj1x/fp1BAcH136DbaDmvgHq7p+tfQsKCsLkyZPx73//27A6uLNSc98AFfZPsbDKRZw9e9Zwuzwld/XqVdGyZUtZVbgrDGMYU3PfhGD/ykn75+yroJZTc9+EUHf/rO2bs2ehTFFz34RQV/+4zo2Rq1evYt++fdi4cSP0ej2aNWsGoKxKvLwqPCsrS7Y+yqxZs3DHHXcgIyPDOeb3m6HmvgHsH1B1/8rPczZq7hug7v5Vt29Dhw51+r87NfcNUHn/FAurnNCRI0dEkyZNROvWrUVQUJBo27atWLlypcjIyBBCVESyiYmJol69euL69evi7bffFj4+Pk5XTGVMzX0Tgv1z5f6puW9CqLt/7Jtr9k0I9fePwc1NaWlpom3btmLmzJnizJkz4vLly+K+++4T7dq1E2+88YZIS0sznJuamiqioqLEfffdJzw9PZ3+F63mvgnB/rly/9TcNyHU3T/2rYyr9U0I9fdPCAY3BsePHxdNmzat9It7+eWXRadOncQHH3wg8vLyhBBlu/ZqNBrh4+Pj9OtNCKHuvgnB/rly/9TcNyHU3T/2zTX7JoT6+ycEa24MSkpKUFpaivz8fABAQUEBAGD27NkYNGgQFi9ejKSkJABASEgInnrqKRw8eBBdu3ZVqslWU3PfAPbPlfun5r4B6u4f++aafQPU3z8A0AjhzBVBtatXr17w9/fHX3/9BQAoKiqCl5cXgLKpmC1btsR3330HACgsLIS3t7dibbWVmvsGsH+u3D819w1Qd//YN9fsG6D+/t2ymZu8vDzk5OTIdn7+7LPPcPz4cdx///0AAC8vL5SWlgIAbrvtNuTl5RnOdeZftJr7BrB/gOv2T819A9TdP/bNNfsGqL9/ptySwc2JEycwZswYDBgwAO3atcO3334LAGjXrh0WLFiAzZs3Y+zYsSgpKYFWW/ZflJaWBj8/P5SWljr19Dc19w1g/1y5f2ruG6Du/rFvrtk3QP39M0uhWh/FHD9+XNSpU0c899xz4ttvvxXTp08XHh4ehsWy8vLyxPr160VERIRo27atGD16tBg3bpzw8/MTCQkJCrfeMjX3TQj2z5X7p+a+CaHu/rFvrtk3IdTfP0tuqZqb69evY/z48Wjbti0WLFhgOD5o0CB06tQJH3/8seFYTk4O3nnnHVy/fh3e3t7473//i/bt2yvRbKuouW8A++fK/VNz3wB19499K+NqfQPU37+q3FJ7S5WUlCAzMxP33nsvgIp9hZo1a4br168DAETZ9HgEBATg/fffl53nzNTcN4D9A1y3f2ruG6Du/rFvrtk3QP39q4rr98AG9evXxzfffIP+/fsDKFtiGgAaNWpk+GVqNBpotVpZ4ZWzLnsupea+Aewf4Lr9U3PfAHX3j31zzb4B6u9fVW6p4AYAWrVqBaAsOvXw8ABQFr2mpaUZzomNjcUXX3xhqBx3lV+2mvsGsH+A6/ZPzX0D1N0/9s01+waov3+W3FLDUlJarVa2GV15JDtr1iy88847OHToENzdXfO/R819A9g/V+6fmvsGqLt/7Jtr9g1Qf/9MueUyN1LltdTu7u6IjIzEhx9+iA8++ADx8fHo0qWLwq2rGTX3DWD/XJma+waou3/sm+tSe/+MqStUs1F59Orh4YElS5YgMDAQO3bsQLdu3RRuWc2puW8A++fK1Nw3QN39Y99cl9r7V4kDppe7nP379wuNRiOOHz+udFPsTs19E4L9c2Vq7psQ6u4f++a61N6/crfUOjeW5OXlwc/PT+lmOISa+wawf65MzX0D1N0/9s11qb1/ADfOJCIiIpW5pQuKiYiISH0Y3BAREZGqMLghIiIiVWFwQ0RERKrC4IaIiIhUhcENERERqQqDGyJyGQMHDsS0adOUbgYROTkGN0SkSnFxcdBoNMjMzFS6KURUyxjcEBERkaowuCEip5SXl4cJEybA398fDRo0wNy5c2X3r1ixAj169EBAQADCw8Nx//33Iy0tDQBw/vx5DBo0CAAQEhICjUaDhx9+GACg1+sRGxuLZs2awcfHB126dMGPP/5Yq30jIsdicENETunFF1/Etm3bsG7dOmzatAlxcXE4ePCg4f6SkhK8/fbbOHLkCNauXYvz588bApjIyEj89NNPAIDExERcvXoVCxYsAADExsZi+fLl+PTTT3H8+HE899xzePDBB7Ft27Za7yMROQb3liIip5Obm4s6dergm2++wdixYwEA169fR0REBB5//HHMnz+/0mPi4+PRs2dP5OTkwN/fH3FxcRg0aBBu3LiB4OBgAEBRURFCQ0Px559/Ijo62vDYSZMmIT8/HytXrqyN7hGRg7kr3QAiImNnzpxBcXExevfubTgWGhqKNm3aGL4/cOAA/ve//+HIkSO4ceMG9Ho9ACA5ORnt27c3+bxJSUnIz8/HkCFDZMeLi4sRFRXlgJ4QkRIY3BCRy8nLy8OwYcMwbNgwfPvtt6hXrx6Sk5MxbNgwFBcXm31cbm4uAODXX39Fo0aNZPd5eXk5tM1EVHsY3BCR02nRogU8PDywd+9eNG7cGABw48YNnD59GgMGDMCpU6eQkZGB2bNnIzIyEkDZsJSUp6cnAECn0xmOtW/fHl5eXkhOTsaAAQNqqTdEVNsY3BCR0/H398djjz2GF198EXXq1EFYWBheffVVaLVlcyAaN24MT09PfPLJJ3jyySdx7NgxvP3227LnaNKkCTQaDTZs2IA777wTPj4+CAgIwAsvvIDnnnsOer0e/fr1Q1ZWFnbu3InAwEBMnDhRie4SkZ1xthQROaU5c+agf//+uPvuuzF48GD069cP3bt3BwDUq1cPy5Ytw+rVq9G+fXvMnj0bH374oezxjRo1wptvvolXXnkF9evXx5QpUwAAb7/9Nl5//XXExsaiXbt2GD58OH799Vc0a9as1vtIRI7B2VJERESkKszcEBERkaowuCEiIiJVYXBDREREqsLghoiIiFSFwQ0RERGpCoMbIiIiUhUGN0RERKQqDG6IiIhIVRjcEBERkaowuCEiIiJVYXBDREREqvL/fw7LsBqgXnMAAAAASUVORK5CYII=", + "text/plain": [ + "
    " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "new_cases_usa.plot.line(\n", + " rot=45,\n", + " ylabel=\"New Cases\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sM5-HFDx70RG" + }, + "source": [ + "## Visualization #2: Symptom-related searches compared to new cases" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "se1b6Vf4XB9_" + }, + "source": [ + "### Filter data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Wl2o-NYMoygb" + }, + "source": [ + "We're curious if searches for symptoms like \"cough\" and \"fever\" went up in the same times and places that new COVID-19 cases occured, compared to non-symptoms like \"bruise.\" Let's plot searches vs. new cases to see if it looks like there's a correlation." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "olfnCzyg8jYi" + }, + "source": [ + "First, we select the new cases column and the search trends we're interested in." + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "id": "LqqHzjty8jk0" + }, + "outputs": [], + "source": [ + "regional_data = all_data[all_data[\"aggregation_level\"] == 1] # get only region level data,\n", + "symptom_data = regional_data[[\"location_key\", \"new_confirmed\", \"search_trends_cough\", \"search_trends_fever\", \"search_trends_bruise\", \"population\", \"date\"]]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "b3DlJX-k9SPk" + }, + "source": [ + "Not all rows have data for all of these columns, so let's select only the rows that do. Finally, lets add a new column capturing new confirmed cases as a percentage of area population." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "g4MeM8Oe9Q6X" + }, + "outputs": [], + "source": [ + "symptom_data = symptom_data.dropna()\n", + "symptom_data = symptom_data[symptom_data[\"new_confirmed\"] > 0]\n", + "symptom_data[\"new_cases_percent_of_pop\"] = (symptom_data[\"new_confirmed\"] / symptom_data[\"population\"]) * 100\n", + "\n", + "\n", + "# remove impossible data points\n", + "symptom_data = symptom_data[(symptom_data[\"new_cases_percent_of_pop\"] >= 0)]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# group data up by week\n", + "weekly_data = symptom_data.groupby([symptom_data.location_key, symptom_data.date.dt.isocalendar().week]).agg({\"new_cases_percent_of_pop\": \"sum\", \"search_trends_cough\": \"mean\", \"search_trends_fever\": \"mean\", \"search_trends_bruise\": \"mean\"})" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IlXt__om9QYI" + }, + "source": [ + "We want to use a line of best fit to make the correlation stand out. Matplotlib does not include a feature for lines of best fit, but seaborn, which is built on matplotlib, does.\n", + "\n", + "BigQuery DataFrames does not currently integrate with seaborn by default. So we will demonstrate how to downsample and download a DataFrame, and use seaborn on the downloaded data." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "T9Hub_EAXWvY" + }, + "source": [ + "### Graph with lines of best fit" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "hoQ9TPgUPJnN" + }, + "source": [ + "We will now use seaborn to make the plots with the lines of best fit for cough, fever, and bruise. Note that since we're working with a local pandas dataframe, you could use any other Python library or technique you're familiar with, but we'll stick to seaborn for this notebook.\n", + "\n", + "Seaborn will take a few minutes to calculate the lines. Since cough and fever are symptoms of COVID-19, but bruising isn't, we expect the slope of the line of best fit to be positive in the first two graphs, but not the third, indicating that there is a correlation between new COVID-19 cases and cough- and fever-related searches." + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": { + "id": "EG7qM3R18bOb" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAsz5JREFUeJzs/XeQZPd5341+Tu7ck/PMJgCLBbBYLLEgGCVSlEVRDKIYAPnVK8tSlZOqLMv0lSW6JNlyWWLJrlLR9vWVS657JbkcXoCkSNGiKVqiKAYxYReZ2AV2sWly6BxOPuf+caYbE3pmZ2Z7Znpmfp8qFLlzZrp/3bvTz/c84ftIYRiGCAQCgUAgEOwR8n4fQCAQCAQCwdFCiA+BQCAQCAR7ihAfAoFAIBAI9hQhPgQCgUAgEOwpQnwIBAKBQCDYU4T4EAgEAoFAsKcI8SEQCAQCgWBPEeJDIBAIBALBnqLu9wHWEgQBMzMzpNNpJEna7+MIBAKBQCDYAmEYUqlUGBkZQZY3z210nPiYmZlhfHx8v48hEAgEAoFgB0xOTjI2Nrbp93Sc+Ein00B0+Ewms8+nEQgEAoFAsBXK5TLj4+PNOL4ZHSc+GqWWTCYjxIdAIBAIBAeMrbRMiIZTgUAgEAgEe4oQHwKBQCAQCPYUIT4EAoFAIBDsKUJ8CAQCgUAg2FOE+BAIBAKBQLCnCPEhEAgEAoFgTxHiQyAQCAQCwZ4ixIdAIBAIBII9RYgPgUAgEAgEe4oQHwKBQCAQCPYUIT4EAoFAIBDsKR2320UgaBCGIYsVm6rtkTJU+tPGlnYGCAQCgaCzEeJD0LEsVmxenCrhByGKLPHwWJaBTGy/jyUQCASCu0SUXQQdS9X28IOQka44fhBStb39PpJAIBAI2oAQH4KOJWWoKLLETNFEkSVShkjUCQQCwWFAfJoLOpb+tMHDY9lVPR8CgUAgOPgI8SHoWCRJYiATY2C/DyIQCASCtiLKLgKBQCAQCPYUIT4EAoFAIBDsKaLsIugohLeHQCAQHH6E+BB0FMLbQyAQCA4/ouwi6CiEt4dAIBAcfoT4EHQUwttDIBAIDj/ik13QUQhvD4FAIDj8CPEh6CiEt4dAIBAcfoT4OKCIqRCBQCAQHFSE+DigiKkQgUAgEBxURMPpAUVMhQgEAoHgoCLExwFFTIUIBAKB4KAiItYBRUyFCAQCgeCgIsTHAeWoToWIRluBQCA4+AjxIThQiEZbgUAgOPiIng/BgUI02goEAsHBR4gPwYFCNNoKBALBwUd8cgsOFKLRViAQCA4+QnwIDhRHtdFWIBAIDhOi7CIQCAQCgWBPEZkPQUchRmkFAoHg8LPtzMc3vvENPvjBDzIyMoIkSXzhC1/Y8Hv/4T/8h0iSxKc//em7OKLgKNEYpb06X+XFqRKLFXu/jyQQCASCNrNt8VGr1Th37hz/6T/9p02/7/Of/zzf/e53GRkZ2fHhBEcPMUorEAgEh59tl13e97738b73vW/T75menuYf/+N/zFe+8hXe//737/hwgqOHGKUVCASCw0/bP9mDIOBnf/Zn+ZVf+RUefPDBO36/bdvY9hup9XK53O4jCQ4QYpRWIBAIDj9tn3b53d/9XVRV5Zd+6Ze29P2f+tSnyGazzf/Gx8fbfSTBAaIxSnuyP8VAJiaaTQUCgeAQ0lbxcenSJf79v//3/NEf/dGWg8YnP/lJSqVS87/Jycl2HkkgEAgEAkGH0Vbx8c1vfpOFhQUmJiZQVRVVVbl16xb/7J/9M44fP97yZwzDIJPJrPrvMBKGIQtli+uLVRbKFmEY7veR2sphf30CgUAgaB9t7fn42Z/9WX70R3901dfe+9738rM/+7P8/M//fDuf6sDRSdtY7+SlsROvjU56fQKBQCDobLYtPqrVKteuXWv++caNGzz//PP09PQwMTFBb2/vqu/XNI2hoSFOnz5996c9wKwcIZ0pmlRtb98swu8kFHYiJDrp9QkEAoGgs9l22eXixYucP3+e8+fPA/CJT3yC8+fP85u/+ZttP9xhopNGSO/kpbETr41Oen0CgUAg6Gy2HSHe9a53bauef/Pmze0+xaGkk0ZI7yQUdiIkOun1CQQCgaA1puNTtb19/4wWt6d7RCdtY72TUNiJkOik1ycQCASC1dieT77mYDo+urr/O2WF+DiC3EkoCCEhEAgEhwPXDyjUnI5bVSHEh0AgEAgEhww/CCnUHSqW15HWB0J8CAQCgUBwSAiCkJLpUjJdgg4UHQ2E+BAIBAKB4IAThiFl06NoOvhB54qOBkJ8CAQCgUBwgKlYLsW6i+sH+32ULSPEh0AgEAgEB5C645GvOTjewREdDYT4EAgEAoHgAGG50dis5fr7fZQdI8SHQCAQCAQHAMcLKNQdah02NrsThPgQCAQCgaCD8fyAQt2lYrn7fZS2IcSHQCAQCAQdiL9ibLYTvTruBiE+BAKBQCDoIMLwDdFxEMZmd4IQH3tAGIYsVuxVu1IkSdrvYwkEAoGgwyhbLsWaixccvAmW7SDExx6wWLF5caqEH4QossTDY1kGMrH9PpZAIBAIOoSaHY3NHiSvjrtBiI89oGp7+EHIcFeMyzNlLs+WAUQGRCAQCI44luuTqznYB3hsdicI8bEHpAwVRZa4PFNmqmAiIeH6JZEBEQgEgiOK7fkUai515+CPze4EIT72gP60wcNjWS7PlpGQuH84zWzJomp7Ym29QCAQHCFcP/LqqFpHU3Q0kPf7AEcBSZLoTxv0pw3cIODybBlFjjIiAoFAIDj8+EFIrmozVTCPvPAAkfnYMxYrNtMFE02WcXyfka44/Wljv48lEAgEgl3koKy432uE+NgjqrZHEMKZkQwzRZOYphzIZlMxNiwQCAR3JgxDypZHsX4wVtzvNUJ87BGNptOZookiSwe25CLGhgUCgWBzqrZH4QiNze6EgxkBDyCNptOVGYODSGNseKQrzkzRFE2zAoFAsIzp+OTrR29sdicI8bFHSJLEQCZ24AP1YcngCAQCQbuwXJ9C3cF0hOjYKiJyCLZFJ2dwRD+KQCDYS1w/oFBzqB6CFfd7jRAfgm3RyRkc0Y8iEAj2As8PKJouFcs7dNtm9wrh8yE4NKzsR/GDUNyNCASCthIEIfmaw1TBpHwI19zvJSLzITg0iH4UgUCwG4RhSNn0KJpibLZdiE9nwaGhk/tRBALBwaRiuRTrrhibbTNCfAgODZ3cjyIQCA4WdSdace94QnTsBkJ8CAQCgUCwjOX65GsOlvDq2FWE+BAIBALBkcfxom2zNdGovicI8SEQCASCI4vnBxTqLhXL3e+j7All0+VLL81yY6nOf/k7j+6bF5IQHx2MMM0SCASC3cEPQop1h/IR8eqYK1l89tIU//vlWSw36mP5/o08j5/s3ZfzCPHRwdzJNEuIE4FAINgeYfjGivujMDb72nyFp56Z5OuvLbL25f7//uaGEB+C9dxpiZtw9BQIBIKtU7ZcijUXLzjcEyxhGPL9m3mevjjFc7eL665n4xp/923H+TtvPbb3h1tGiI8O5k6mWWLDrEAgENyZmh2NzR52rw7XD/jalQWeujjFjaXauuvD2Rgff3SMDz0ywj0D6X044RsI8dHB3Mk0Szh6CgQCwcZYrk+udvhX3Fdtjz97cZY/eXaKpaqz7vrpoTRPXhjnnff2ocgSurr/m1VEtOpA1vZynOhLtuzlEI6eAoFAsB7b8ynUXOrO4R6bXazYfO7ZKb704iw1Z73AesvJHp58bJyHR7Md1w8oxEcHstVejt129BQNrQKB4CDh+pFXR9U63KLj+mKVpy9O8dUrC+uaZjVF4kfPDPLxC2Mc703u0wnvjBAfHUin9HKIhlaBQHAQ8IOQQt051CvuwzDkuckiTz8zyfdvFtZdTxoKHzo3wkfOj9Kb6vws+LYLP9/4xjf44Ac/yMjICJIk8YUvfKF5zXVdfvVXf5WzZ8+STCYZGRnh7/ydv8PMzEw7z3zo6ZReDrGiXiAQdDJBEFKoOUzm64d2xb0fhPzVlQX+4X97lv/XZ15cJzwG0gb/6F2neOrvv4W/986TB0J4wA4yH7VajXPnzvELv/ALfOQjH1l1rV6v8+yzz/Ibv/EbnDt3jkKhwD/5J/+ED33oQ1y8eLFthz7sdEovR6eIIIFAIFhJGIaULY9i/fCuuDddny+/NMtnL00zV7bWXb+nP8WTj43xw/f1oyr730C6XaTwLqSiJEl8/vOf58Mf/vCG3/PMM8/w5je/mVu3bjExMXHHxyyXy2SzWUqlEplMZqdHaxtHue/hbl/7UX7vBALB7lC1PQqHeGw2X3P4/HPTfPGFGSotelcuHOvmycfGedNE144/T3VVZqw7cbdHXcd24veu38qWSiUkSaKrq6vlddu2sW27+edyubzbR9oWR7nv4W4bWo/yeycQCNqL6fjkavahXXF/O1fn6UuT/MUr87j+6pyAIku8+3Q/T14Y59RAap9O2F52VXxYlsWv/uqv8rf/9t/eUAV96lOf4rd+67d28xh3Rac0fx5ExHsnEAjuFsv1KdQdzBajpAedMAx5ebrMUxcn+fbruXXXE7rC+88O89E3jR66G7ddEx+u6/LEE08QhiG///u/v+H3ffKTn+QTn/hE88/lcpnx8fHdOta2EX0PO0e8dwKBYKc4XkCx7hzKRnc/CPmb15d4+plJXpmtrLvem9L56PlRPvDwCKnY4fzc3JVX1RAet27d4q/+6q82rf0YhoFhdG53bjubP49aD0SnNM4KBIKDQ2PFfdU+fGOztuvzlVfm+eylKaYK5rrrx3oTPHFhnPfcP9ARLqS7SdvFR0N4XL16la997Wv09u7Pxrx20arvYaci4qj1QOy2CZpAIDg8BEFI0XQpmy7BIRMdpbrLn74wzReem6FouuuunxvL8uRj47z5RA/yIb4hXcm2xUe1WuXatWvNP9+4cYPnn3+enp4ehoeH+djHPsazzz7Ln/3Zn+H7PnNzcwD09PSg63r7Tr6P7FREiB4IgUAgWE0YhpRNj6J5+MZmZ4omn7k0xZ+/PIe9plFWluCH7u3nicfGuH9o/yc795pti4+LFy/y7ne/u/nnRr/Gz/3cz/Gv/tW/4otf/CIAjzzyyKqf+9rXvsa73vWunZ+0g9ipiNitHoijVs4RCASHg4rlUqy7h25s9vJs1ET6ratLrNVTMVXmxx8a4mOPjjHSFd+fA3YA245+73rXuzatwx22Gl0rdioidqsH4qiVcwQCwcGm7kQr7g/T2GwQhnzvep6nLk7y4lRp3fWuuMZPnR/lQ4+MkI1r+3DCzuJwttHuMjsVEbvVAyHKOQKB4CBguT75moN1iFbcO17AVy/P8/TFKW7l6+uuj3XHeeLCGH/rzCCGpuzDCTsTIT6W2U7pYqciYqflkTv9nBhpFQgEnYzjRdtma4dobLZiufyvF2b5k+emydecddcfGM7w5GPjvO1UL4osyuBrEVFqmb0oXez0Oe70c1vNxIjeEIFAsJc0xmYr1voJj4PKfNnis5em+N8vzWGuyeBIwNtO9fLkY+M8NJrdnwMeEIT4WGYvShc7fY47/dxWMzGiN0QgEOwFfhBSrDuUD9GK+2sLVZ56ZpKvvbqwrolUUyTe+2DURDrR0/6dKYcRIT6W2YvSxU6fo11nE70hAoFgNwnDkJIZTbAcBq+OMAy5eKvA089Mcul2cd31dEzlQ+dG+Knzo/QkD4eVxF4hxMcye+HGudFz3Kkc0q6zid4QgUCwW5Qtl2LNxQsO/gSL5wf89WuLPPXMJK8v1tZdH8rE+NijY7zv7BBx0US6I0T0WWYv3Dg3eo47lUPadTZhdy4QCNpNzY7GZg+DV0fd8fjSi7N87tlpFir2uuv3DqR48rFxfvi+ftFEepcI8dEB7EU5RDSbCgSCdmK5Prmag30IxmaXqjZ/8uw0/+vFGWr2+tfz5hM9PHlhjEfGu8TnZpsQ4qMD2ItyiGg2FQgE7cD2fAo1l7pz8MdmbyzVePriJF+9vIC3potUlSXec2aAJy6Mc6IvuU8nPLwI8dEB7EU5RDSbCgSCu8H1I6+OqnWwRUcYhrw4VeKpi5N893p+3fWkrvCBh4f5yJvGRGl6FxHiowPYi34T0WwqEAh2gh+EFOoOlQM+NusHId+8ushTF6d4da6y7npfSuejbxrjAw8PkxSfj7uOeIePCJtlV0Q/iEAgWEsQRGOzpQO+4t50ff785Tk+e2mK2ZK17vrJviRPPDbOu0/3oynyPpzwaCLExxFhs+yK6AcRCAQNwjCkbHkU6wd7xX2h7vCF56b50+dnKLcoFb1poosnHxvnwrFucbO1DwjxIRD9IAKBAIg+CwoHfGx2qlDnMxen+Mor8+u25soSvOv0AE9cGOO+wfQ+nVAAQnwIiPpBZAkuz5RxfJ/xnjhhGIq7AYHgiGA6PrmafaBX3P9gpsRTz0zxN9eWWJuviWkyP3F2mI+9aYyhrMjqdgJCfOwSB6mPoj9tMNodZ6FioykyM0WTvpQhSi8CwSHHcn0KdQfTOZheHUEY8u1rOZ66OMkPZsrrrvckdT5yfpQPnhsmHdP24YSdRWPYoBPeCyE+domD1EchSRIxTaEvZXRM6eUgiTeB4KDheAHFukP1gK64d7yA//PKHE9fnGKqYK67PtGT4IkLY/zomUF09Wg3kUqSREJXSBkqCV3pmM9RIT52iYPWR7GTUdzdFAgHSbwJBAeFxor7qn0wx2bLpsufvjDDF56bplB3110/O5rlycfGeMvJXuQOCbL7habIZGIaqZjakVbwQnzsEp3iq7FVgbATo7OFssW3ri01f+Yd9/QxmI235dwHTbwJBJ1MEIQUl8dmD6LomC2ZfPbSNF9+aRZrTV+KBLzz3j6efGycM8OZ/TlghyBLEklDJR1TiXX4wjshPnaJTlnittUMwk6Mzm7n67y+WKMrrjFfrjHRk2ib+OgU8SYQHGTCMKRsehTNgzk2++pchacvTvL11xZZe3xdlfnxB4f4+KNjjHa353PnoBJfLqukDLVjyip3Qnyi7xJ74Vq6FfYmg9D+f+ydIt4EgoNKxXIpHMAV92EY8r0beZ6+OMnzk6V117NxjQ8/MsJPPjJCV0LfhxN2Bqosk46ppGLqgTRHE+KjBYep2XE3MwgTPQlO9iWp2R4n+5JM9CTa9tidIt4EgoNG3YlW3B+0sVnXD/jq5QWevjjJzVx93fWRrhgff3Sc9z442PElhd1CkiSSukI6phHXD/Z7IMRHCw5Ts+NuZhAGMjF+6L5+kZ0QCDoAy/XJ1xysA7bivmp7/NkLM3zuuWlyVWfd9TPDaZ68MM7b7+nryMbJvcDQorJK2lCRD8l7IMRHCw5Ts+NuZhBEdkIg2H8cL9o2WztgY7MLZYvPPTvNl16apd7CZ+StJ3t58rExzo5mD2zm+W5oZKpTMRVDPdhZjlYcGfGxnVLKVkoVe1WaOUwlIIFA0D48PyB/AFfcv75Y5emLU/zVlYV1TbCaIvG3zgzy8QtjHOtN7tMJ9w9JkohrCulYZ3ly7AZHRnxsp5SylVLFXpVm7vQ8QpwIBEcLPwgp1h3KB2jFfRiGPHu7yFPPTHLxVmHd9ZSh8qFzw/zU+VF6U0evfKspy82jhop6AJtHd8KRER/bKaVspZywV6WZOz3PYepPEQgEGxOG0Yr7Yv3grLj3g5C/fnWRpy5Ocm2huu76QNrgY4+O8RNnh0joRyYcAQfLk2M3ODJ/2+2e+tgrH4o7Pc9h6k8RCAStKVsuxQM0Nms6Pl96aZbPXppioWKvu35Pf4onHxvjh+/rPzJ3+g1iy2WVg+TJsRscGfHRrqmPRpmjYrmMdMUwVJl0TNu1SY87nVuYcQkEh5eaHY3NHpQV9/maw+efm+aLL8xQadGLcuFYN08+Ns6bJrqOVOBVZZlULMpyHERPjt3gyESqdk1m7HWZ407n7gQzLtF3IhC0F9Pxydcd7AMyNnsrV+MzF6f4i8vzuP7qkpAiS/zI/QM88egYpwZS+3TCvecweXLsBkdGfGyHzYJpp5U5OmHctZUg608bQpAIBNvE9nwKNZe60/kTLGEY8tJ0iaeemeI713Prrid0hfefHeajbxo9Un1oDU+ORlZa0BohPlqwWXajU8sc+5l9aCXIANEIKxBsEdePvDoOwtisH4T8zbUlnro4yeXZyrrrvSmdj54f5QPnRjrm83G3OeyeHLvB0fiXsU02y250QpmjFfs59dJKkHVahkgg6ET8IKRQd6gcgLFZy/X5yg/m+eylKaaL5rrrx3sTPHFhnPecGTgyfQ0JXT0Snhy7gRAfK2hkD3LVqKF0uhCiKvIq9d4JZY5W7Gew30iQyRJcninj+D7jPXHCMBS/oAIB0Yr70vKK+04fmy3WHb7w/Ax/+vwMJdNdd/2R8SxPXBjn8RM9R+L3+yh6cuwGQnysoJE98IIASYrSh8d6kx2T3diM/SwHtRJk/WmD0e44CxUbTZGZKZr0pQxRehEcacIwpGx5FOudv+J+umDymUtT/PkP5tYtqZMl+OH7+nniwjinh9L7dMK946h7cuwGQnysoJE9GO1KMFM06T1AwbLTykGSJBHTFPpShii9CAREny+FAzA2e3m2zFPPTPLNq0uslUcxVeZ9Z4f52KOjDGfj+3K+vaThyZHUD89Ct05BiI8VdGoz6VboxHLQZu+nGM8VHBUOwor7IAz57vUcTz0zxUvTpXXXuxMaHz4/yofOjZCNa/twwr1DeHLsDQcnuu4B7cgeiKD6Bpu9n8IWXnDYsVyfQt3BbLGxtVNwvIC/vDzP0xenuJ2vr7s+1h3niQtj/NgDQ+jq4Q3EDU+OVEw9cjbv+4V4l1fQjuzBToPqYRQtm72fYhpGcFhxvIBi3WmOnHciFcvlf70wy588N02+5qy7/tBIhicujPO2e3qRD/jn0Gboyw7VwpNj7xHio83sNKgetUzAQS5xCQSt8PyAQt2lanfu2Oxc2eJzl6b40kuzWO7qMpAEvP2ePp58bIwHR7L7c8A9QJHfaB4Vnhz7h/jEbzM7Dap3kwk4iFmTTmuQFQh2ShCEFJfHZjtVdFydr/D0xSm+9uoCa4dsNEXivQ8O8fFHxxjvSezPAfeAhB6ZgCWFJ0dHsG3x8Y1vfIN/9+/+HZcuXWJ2dpbPf/7zfPjDH25eD8OQf/kv/yX/5b/8F4rFIm9/+9v5/d//fe699952nrtj2WlQTeoKVdvl2dsmKSP6BdkqBzFr0okNsgLBdgjDkLLpUTQ7c2w2DEMu3irw1DOTPHu7uO56Jqbyk4+M8OHzo3Qn9L0/4B4gPDk6l22Lj1qtxrlz5/iFX/gFPvKRj6y7/m//7b/lP/yH/8Af//Efc+LECX7jN36D9773vbzyyivEYp0dENvB3QTVMATC5f/dBqJ/Yv85iNknwc6pWC6FDl1x7/oBX7uywNMXp7i+VFt3fTgb42OPjvHjDw0RP4SeFbIkkTAUMjFNeHJ0MNsWH+973/t43/ve1/JaGIZ8+tOf5td//df5yZ/8SQD+63/9rwwODvKFL3yBn/7pn7670x5AWgUlYN3Xao5POqZxeijDTNGkto0OedE/sf8cxOyTYPt08thszfb4sxdn+ZNnp1ms2uuunx5M8+RjY7zz3v5D2VwpPDkOFm2NUjdu3GBubo4f/dEfbX4tm83y+OOP853vfKel+LBtG9t+4xelXC6380j7TqugBOuXrt2NgBD9E/uPyD4dbizXJ19zsDpwxf1ixeZPnp3iz16cbXnT8viJHp58bJxzY9lDl41reHKkDPVQjwIfRtoqPubm5gAYHBxc9fXBwcHmtbV86lOf4rd+67faeYyOYqONr2u/dqIvuWMBIfon9h+RfTqcOF60bbbWgWOzN5ZqPH1xkq9eXsBb03OiyhLvOTPAExfGOdGX3KcT7g6SJJHQleWFbuL37KCy739zn/zkJ/nEJz7R/HO5XGZ8fHwfT9ReNgpKa78mBMTBRmSfDheeH5DvwBX3YRjy/GSRpy5O8f0b+XXXk4bCBx8e4SNvGqUvdbj+DeqqTNrQSMWEJ8dhoK3iY2hoCID5+XmGh4ebX5+fn+eRRx5p+TOGYWAYh+uXZCUbBaW1X9tJw6JocuwchHg8HPhBSLHuUO6wFfd+EPL11xZ5+uIkr81X113vTxl87NFRfuLsMMlDlHUTnhyHl7b+Kz1x4gRDQ0N89atfbYqNcrnM9773Pf7RP/pH7XyqtrHbAXyjoLT2awtla9sNi1E/SZFc1cELQs5PdHFmOLPt8wsRIzjqhOEbK+47aWzWdH2+/NIsn700zVzZWnf9ZH+SJy+M8+7T/YdqlFR4chx+ti0+qtUq165da/75xo0bPP/88/T09DAxMcEv//Iv82/+zb/h3nvvbY7ajoyMrPIC6SQ6ZUphJw2LVdsjV3Wo2B5LFZswDHe0tr5T3gOBYD8oWy7FDhubzdccvvD8NF98foZyi9LPoxNdPPHYOBeOdR+a4Cw8OY4W2xYfFy9e5N3vfnfzz41+jZ/7uZ/jj/7oj/jn//yfU6vV+Pt//+9TLBZ5xzvewZ//+Z93rMfHXk8pbJRl2EnDYspQ8YKQpYpNf9pAV5QdnV9MagiOInUnEu+dtOJ+Ml/nM5em+MoP5nD91RkYWYJ3nx7giQtj3DuY3qcTthfhyXF0kcJOKmwSlWmy2SylUolMJrPrz7eTcsduPN92Sx9hGLJQtnhhqsjrC1V6EgY9KZ1z413bPv9evwcCwX7SiWOzL0+XeOriJN++lmPtB3JMk3n/2WE++ugYQ4fk9zKmRRtkU8KT41Cxnfh9eDqTdsheTynsNMuwVpyEYchL02WCMOofmehJcKw3uaPzi0kNwVGg08ZmgzDk29dyPHVxkh/MrPc36knqfOT8KB88N0w6pu3DCduL8OQQrOTIi4+9nlLYqLxyp76LtdezcRU/CBntSjBTNOndQa9HAzGpITjMNLbNVix3v48CgO36/J9X5vnMpSmmCua668d6EjxxYYz3nBk88EFaeHIINkL8a9hjNsoybJYRCcOQW7ka04U6x/tSmE505yZMrQSCjem0bbMl0+WLz8/w+eemKZrrhdDDY1mevDDO4yd7kA94E6nw5BDcCRGx9piNsgybNZwuVmxu5+vMV2zmKzYn+5Kcn+hCkiRRKhEI1tBp22ZnSyafuTjFn788h7VmJ4wswTvu7ePJC+OcGd79HrfdRJakZllFNI8K7oQQHx3CZn0XVdsjaag8fqKHm7kax3oTq0osDcv2RpOq8O0QHFUqlkux7nbEBMurcxWeemaSb1xdZK0GMlSZH39wiI89OsZod3x/Dtgm4rpCOqYJTw7BthDio0PYrO8iZaiosozlBox2RY2lkiRtOKWyU9+O/RAtQigJ2kGnbJsNwpDv38jz9MVJnp8srbuejWv81PkRfvLcKNnEwW0i1RSZ1LLzqPDkEOwEIT4OANvtE9nORM3K4G+5PtMFkyBkQ9HSbrEgDM4Ed4PtRWOzZottrnuJ4wV89coCT1+c5Fauvu76aFecj18Y470PDGIc0JKEJEkkDYW0oRHXD+ZrEHQOQnzskL28Y99un8h2DMtWBv+lqo0my5wZyWwoWtotFoTBmWAnuH5AoeY0S477RdX2+LMXZvjcc9Pkqs666w8Mp3nisXHefqrvwDZeGlo0rSI8OQTtRIiPHdKuIHw3ImajjMh2fDtWBv9i3cHx/U1FS7vFglhFL9gOfhBSqDtU9nnx20LZ4nPPTvOll2apt8i6vO1UL09eGOeh0e3vWuoEGr+L6Zh24Md9BZ2J+KTfIe0KwncjYjbKiGzHt2Nl8O9N6Yx0xSP3wQ1ES7vFgjA4E2yFIHhj8Vuwj6Lj9YUqT12c5GuvLq6bpNEUib/1wCBPPDrORG9in064cxqeHClDJSGaRwW7jBAfO6RdQXi/yw6tgv9mHzrtFgvC4EywGWEYUrY8SvX9W/wWhiGXbhV46uIUl24V1l1Px1Q+dG6Enzo/Sk9S34cT3h2aIpOJCU8Owd4ixMcO2SwIb6eUsp9lh52UfIRYEOwVVdujUNu/xW+eH/D11xZ56pkpri1W110fzBh8/NEx3vfQ8IFrwJQlieTytIrw5BDsB0J87JDNgvB2Sin7WXY4zJMmYoT34GK5Prmag71Pi9/qjseXXprjc5emWKjY667fM5Dipx8b54fv6z9wmYL4clklZaji90GwrwjxsUW2E8y2U0q5UyZhN4PobpV8OiHwH2ZhdVixPZ9CzaXu7M8ES65q8yfPTfO/XphtOUXz5uPdPPHYOOfHuw5U4G54cqRiKprw5BB0CEdGfGw1IDZW1d/OR7P6Ez2JbRt3tbOUcrdBdLPXvVsln04I/PvdSyPYOp4fkK87VK39ER23cjWevjjFX16ex/VXN5EqssSP3D/AkxfGONmf2pfz7QRJkkguO48etJKQ4GhwZMTHVgPiYsXmW9eWeH2xBsDJviQ/dF//toJZO0spdxtEN3rdYRgShiHZuEoYhiQNtbn1825t2jsh8IsR3s7HD0KKdYfyPozNhmHIi9Mlnnpmku9ez6+7ntAVPvDwMB9909iBmsASnhyCg8KR+UTeakCs2h5V26MrrgESteU/byeYrSyl3G0J4m6D6Eave7Fi89J0GT8IqVgukgQpQ9u2TXur19cJgV+M8HYuYRiNzRbrez826wch37q2xFPPTHJlrrLuem9K56NvGuMDDw8fGMEqPDkEB5GD8dvVBrYaEBvNWPPlNzIfjeC1k2DWKoD3p40tC5K7DaIbve6VouTZWyZIcN/gG86m/WHIrVyN6UKd430pTMfbsuNpOwP/VsTbRt8jpnI6j7LlUqzt/dis5fr8+ctzfObSFLMla931E31Jnrgwxo/cP3Ag+iKEJ4fgoHNkxMdWA2J/2uAd9/Qx0ROZBE30JO4qmLXKPABb7om42yC60eteKUqShooksUqgLFZsbufrzFds5it2U4Rt5fUNZGJtC/xbyb50Qo+JYHP2a/Fbse7whedm+MLz05Rb9JQ8Mt7Fk4+N8ebjPQcigAtPDsFh4ciIj60GcUmSGMzGGcy2Z811q8zDXvRErM0GnOhLrvpwXSlKkssNaTXHbwqUG0s1kobK4yd6uJmrcaw30dLLJFe1qdou08UQVZbbnqreynvVCT0mgtZYbrT4zdrjsdnpgsnTlyb5yg/m1wkeWYIfvq+fJx8b577B9J6eaycITw7BYeTIiI/9YqPMw273RNwpG3AnMZYyVFRZxnIDRrsSHOtdLV4aj+/5AWEIvUmdY73JtvdWbKVc1gk9JoLVOF5Aoe5Q2+PFb6/MlHnq4iTfurrE2m6SmCrzE2eH+dijYwxlOz8zJjw5BIcZ8SndZlr1H6wN8lspAW3W67CVPoi7zQbc6YyNxx/tTizvhTF2pdSxlfdqP5tLO8HTpJPYj7HZIAz5zus5nr44yUvT5XXXuxMaP3V+lA+dGyET1/bsXDtBeHIIjgpCfLSZrfQfbKUEtNnjbOU5tpMN2EnD5lYevx2BeSvv1Va+Z7dEgug3idiPsVnHC/iLV+b5zKWppi/PSsa743z8wjg/9sBgR0+BCE8OwVFEiI8dsFkgu5uMw8rHzVVtvCAqeax9nDs9x0oPD3ijaXYjdhJAt5Jt2I3AvFMRsVsi4aj3m+zHttmK5fLFF2b4k2enKdTdddfPjmZ44sI4bz3Vi9zBWShjeXt02hCeHIKjhxAfO2CzQHY3/QcrH7fhvdHqce70HCs9PBRZQpKkTQP0TgLoVrINuxGYdyoidkskHNV+k8a22WLdWbdafreYK1l89tIU//vlWSx3dROpBLzj3j6evDDOAyOZPTnPTmj8G0nFVAxVZDkER5ej8UnZZjYLZBtlBLbbpzFdCOlN6fSmjHWZhb6UzkhXZALWnzboS+kbPs5WAu1uBdB2PO7a961iuTsSEbv1Gg+zmdlG/2YrVmQQtlfbZl+br/DUM5N8/bVF1uocXZV574ODfPzRMca6E3tynu0iSRLxZedR4ckhEEQI8bGGrYiEzQLZRhmB7fZpqIrMsd5ky7v6parDTNHCD0JmihZ9a5o97xRo177GvpS+KwG0HYF5sWLzwmSRQs3F8X2O9yVR5NYZod0+SysOs5nZ2n+z9w6kUBRpT7w6wjDk4q0CTz0zybO3i+uuZ2IqH35klJ88P0J3Ql//AB2ArsqkDeHJIRC0QogPVgdjy/WZKZr4ARuKhJ0Esq1kI7b6uHd6rFaPs/Y1ThdMgnD1a2x3AG1HYK7aHoWaS8V2WVxeb/6mY93EluvlWxURh1kk7BaNf2d9KYPX5itoisR4z+5mF1w/4GtXFnj64hTXl2rrrg9nY3z80TF+/KGhjvS8UOQ3PDlEWUUg2BghPlh9h7dYsdAUmQdGshuKhJ0Esq2k/bf6uHd6rFaPs1C2mq9xqWqjyTJnRjId3ySZMlQc32exYtOXNtAUmZimHKgNowcVQ5WpWC7zZQtZlkjou/dxUbM9/uzFWT737BRLVWfd9fuH0jz52DjvuKev47IIoqwiEGwfIT5YnUko1V3cIOjo3oC7zbwU6w6O77f1Ne7WKGt/2uBNx7qRJAlVluhN6UemqXO/aHh1WK7Psd4kdccjoav0JNvvkbFYsfncs1N86cVZas56F9S3nOzhycfGeXg023FBXZRVBIKdIz7FWZ1J6E5qjHTFqNkexbrLzaUqYRgykInd1YdfO9P+d5t56U3pjHTFm6WLvpTOfMlseiVM9CS2/Xp3a5RVkiTODGfoSxkblpGEuVd7WOvVIUmR2Oul/T0VN5ZqPH1xkq9eXsBb00WqKRI/emaQj18Y43hvsu3PfTeIsopA0B6E+GB9JiEMQ67MVXh9sbHZ1uSH7uvfcjDtxMDYKlvSONNC2eKbV5eaNfZT/UneeW//trbv7qbfxZ3KSEfZ3KsdBEEYbZvd5RX3YRjy3GSRp5+Z5Ps3C+uuJw2FD50b4SPnR+lNdc7UkNggKxC0HyE+WB/cri9WqdoeXXENkKjZrdfJb0Qnul5uli2p2h4126MrrgMh1eXXC1vfvrvXfhdH3dyrHTS8Okr13V1x7wchX39tkaeemeTqQnXd9YG0wUcfHeP9Z4d2ta9ku4iyikCwe3TOb/ous51sRGOZ03y5kflovU5+I+42MLYrc7LR46z9elJXSBoq85XlzEcque3tu3vtd3FUzb3aRTRF5OyqV4fp+Hz55Vk+e2maubK17vo9/SmefGyMH76vH7VD9pgIE7CDTydmngXrOTKf2BtlI4Ig4MpcpWnYdf9Qmv60wTvu6WNieazwTvbka7nbwNiuzMlGj7P669H44kRPnExMpSuhrdpOu9XXsdejrIfZ3Gs3qTse+Zqzq14d+ZrD55+b5osvzFBpsWDuwrFunnxsnDdNdHVEUBBllcNFJ2aeBes5MuKjYrnkqw6ZuEq+6lKxXAYyMa7MVfjyS3O4ftDcIvnASJbBbJzBbHzLj79SbSd1hbOjGWqOv6PA2K6SwkaPs/Lrr8yUmCtZ9KdjKLLM8b5U8xe1kwO88O3YHpbrU6g7mC0mStrF7Vydz1ya4v+8Mofrr+4dUWSJd5/u58kL45wa6IwxaV2VSce05s2C4HAgSrIHgyMjPmwvYLJQx12KRMZDY9H+h8WKjesHnB7K8OpcuWlktV1aqe2VXhTbLfu0o6Sw0eOs/LoXhOiK0vIXtVMDfCemVTvxTBBtfi3UHWr27qy4D8OQl6fLPHVxkm+/nlt3Pa4pfODhYT76ptGOuPsUZZXDjyjJHgyOzN+KocqMdcfJJjRKdRdjecV2/7Jx1atzZTRF3vHd/Z3U9nZSge3IOGy22Xbl44/3xJkumHvyi9quAN14L70goGZ7TPQkmqWi/Qr4nZbqbXh1VFuUPdqBH4T8zetLPP3MJK/MVtZd703q/NT5UT50boRUbH8/ZhpllXRMJa6Jssphp5MztoI3ODLiIx3T6E0Z+EFIb8ogHYsMk+4fSgOs6vnYCXdS29tJBbYj47DZZtuVjx+G4ToPjd2iXQG68V7GNYUXp0pULY+S6e1rwO+UVO9ar452Y7s+X3llns9emmKqYK67fqw3wRMXxnnP/QPo6v42kYqyytGkUzO2gtUcGfGxkRqW5chKfbcev8Fm4uROGYEwDFkoW9syAdtKMGy1YG43SwftCtCN9/JmLprOOd6XwnL9fa3t7lWqd7MJppK5e14dpbrLn74wzReem6FouuuunxvL8uRj47z5RA/yPmYWRFlFIDgYtP0T0vd9/tW/+lf8t//235ibm2NkZIS/+3f/Lr/+679+qNOdd1Lbm4mTO2UEFis237q2tML0LNnS9Gzt8jhZ2nz769rnHemKNbfl7kbpoF0BuvFeZuMqSb2O6Xioiryvtd29SvWu/Ts7O5ohpqu75tUxUzT5zKUp/vzlOew1EzKyBO+8t58nHxvj/qFM2597q4iyikBw8Gj7p/Xv/u7v8vu///v88R//MQ8++CAXL17k53/+58lms/zSL/1Su59uy2wU4BsBu2K52F6AsZyqbfdd/51MvhoZgelCnVu52qog1jD9upPp2doR2tHu+KbbX9dmIhYr9q6WDtoVoBvvZX/a4FhvsiNqu3uV6l35d3Z9scq1xSrD25jK2ipX5so89cwU37y6yBr3cwxV5scfGuLjj44x0tX+594qxvK/bVFWEQgOHm0XH9/+9rf5yZ/8Sd7//vcDcPz4cf7n//yffP/732/3U22LjVL+jYCdq9pMFUzGuxP0pPQ97R9YmRGo2h41xyNfc5siaSumZ2EYcitXY7pY53hvEtP1N9z+2hBcuapN1XaZLoaoctRsO1O0dq100O4Avdnjder0yd2SMlQ8P+ClqSIBtDX4B2HI967neeriJC9OldZd74przSbSbKL9S+a2girLJA2FdEzb954SgUCwc9ouPt72trfxB3/wB7z22mvcd999vPDCC3zrW9/i937v91p+v23b2PYb463lcrndRwI2Tvk3REk2oXFjqUYmruIH4Z72D6zMCOSqNrmas0oknehLNk3PwjAkaahULLf5s5IksVixuZWrM1+2mS/bnOrf2JW1OS3iB4RhNJlwrDdJX0rfs+bT3WY3p0/2S9hYro8XBAxkYqRiats2zTpewFcvz/P0pSlu5errro91x/n4o2P82AODGNre91FIkkRSV5qvWSAQHHza/pv8a7/2a5TLZe6//34URcH3fX77t3+bn/mZn2n5/Z/61Kf4rd/6rXYfYx19KZ2RrlhzqqUvFW3qbIiSXNVBlSWm8iYxXSJpKIRhuGEJpp0BaOUdfMpQKZneKpEkSVLT9GyjhWqNczx+opebS9VNXVkbgmu0O7G85dZoBuatZiY2e/07fW/a+Z7u5vTJXo/VrvXqaNem2arl8cUXZvj8c9Pkas666w8MZ3jysXHedqp3X8oaoqwiEBxe2i4+nn76af77f//v/I//8T948MEHef755/nlX/5lRkZG+Lmf+7l13//JT36ST3ziE80/l8tlxsfH230sFis2l2fLVG2PpapNb1JnMBtvZh0qlstod5ybSzVML+C713NMdCc3LMHsVgBa2xfRl9JZKFvNP1cst2VQTRkqqiJjuT6j3ZHvxVZNzJK6suo5thL0N3v9d+qv2eh52vme7ub0yV6N1Xp+QKHuNrNc7WK+bPG5Z6f40otzmO56x9O3n+rlycfGeWj07qfAtosqy6RikeAQZRWB4PDSdvHxK7/yK/zar/0aP/3TPw3A2bNnuXXrFp/61Kdaig/DMDCM3U/v387XeX2xRldcY75cY6Insco+XZIkDFWmL20QhiG3l2qYKZdcNWxasa9kJ6OsWwnqa/sY1mY6RrpiLYPqdpo5135vEAR88+oSNdsjaai8896+O1rLb/b679Rfs5G4aGdQ383pk90eq/WDaGy2ZLpt9eq4tlDl6YuT/NWVhXVNpJoi8WMPDPHxC2PNnUZbJQxD8jWXuuM1S0HbyViJsopAcPRo+296vV5HllffsSiKQrCLK7u3QhiG1G0f3w+xvYAgCFgoW9zK1biVq5PUFWZLFrbvY7sB+ZrDtQXoSuicHVt/B9gIQNPFOrXlXo21AqMdd/JrA7Khyi2D6lrR0vAGaSV81n7vMzdyXF+qkY1pXF8qoSkSbz3V19JvZOUoryK3HuW9U3/NRuKinUF9N6dPdkvYhGFI2fQomg7+WnVwF4956VaBpy5OcelWYd31TEzlQ4+M8OFHRulJ7qyUk6+5vDpfIQhCZFni9GCa3tSdH8vQovHYlK4ii7KKQHCkaLv4+OAHP8hv//ZvMzExwYMPPshzzz3H7/3e7/ELv/AL7X6qbZE0VGQJSqZDQldx/JAXp4pcmSszUzC5fzjLYsUiGVMhDLmnP8X9w2kqlt+0Yl9JIwDdytWoWh65qrPOZXNlsJ0q1Hj+dgFDU+hL6fQmdepusO09L+mYtqWgupnwWZuRCZZtyit1h6mSRV9KI2loq8olC2WL5ycLvDhVIq4qDGRiPDiaIa6r6wLwRsH5TuLioNgi74awqVguhVr7vDo8P+CvX1vkqWcmm/4wKxnKxPjYo2O87+wQ8btsIq07HkEQMpCOsVCxqDvehj0poqwiEAhgF8THf/yP/5Hf+I3f4Bd/8RdZWFhgZGSEf/AP/gG/+Zu/2e6n2hYxTeH0ULq528UPQnJVB8fzuZGrc3WuzGB3go+eH2Wh4uD6AZIsoSjRivB02VqXPehPG9zK1ajZHv3pGKaz2n9jZbCdK1lM5k10VcbxA8a64ox2J3Ztz0vV9vCCgLimcDNXIxOLGmhrjo/peFyerTTLLANpHUWWWKw4SFLIeHdi1cRPw+TsW1eXuJmrM9odY6nmcqI/yYOjXeuee6PgfKfXspOgvpXSVrsaWXdjyqXdK+7rjseXXprjc5emWGixJPG+wRRPXhjnh+7rb1sTZ2I5c7FQsZBlaV3ppFFWScc04rpwHRUIBLsgPtLpNJ/+9Kf59Kc/3e6HvivW7nYZyMSYLlpMFWw8P6DmBkzl6zx3u8TZsQyj3YnIzGuDrAZEQfl2vs58xWa+Yq/z31gZbC3XY65kc3oow/duLFGoOTx2onfT3oaVwS6pR+LhxlJtS4EvZajUbK/p1xAEIbfzJumYxvXFCnNlm9GuBPOVGqoMpwfT3DeY4vJcmZLpkorpq8olVdsjaSjENAnHDbC97a9m342MwVZKW+1qZG1nQ6zt+eRr7Vtxv1S1+ZNnp/lfL85Qs9c/5ptP9PDkhTEeGe9q+1hwT1Lj9GB6Vc8HRII/JcoqAoGgBUemu6s/bXB2NNPcj9KT0HhkPMurMyWCICQb13D8gGLdZrQ7wZnhDDeWauRr7oY9CtXlzMHjJ3q4matxrHf1eOvKYGu5PtcWarw6Vyahq3Qn9Tv2NqwMdlXbJQwjEdWw1ZYkacO78P60wURPgqrlcbwvxY2lKNNxeijDtfkKrh8AUV9BwlBJxWS8IODh0a5VW2KBVeOOcU0laajcN5RqNibup6HX2j6SxmTIWofYdjSytuNxXD+gUHOotmnF/c1cjaefmeIvL8/jrekTUWWJ95wZ4IkL45zoS7bl+VohSVJz/FeUVQQCwVY4MuKjsdW1ZHrL0wQeZ0czPHaql8sLFUqmR09KpzthEFveD3GnHoWUoaLKMpYbMNq1+Xjryu25rXo+VtII5pdny+SqNmdGMjx324QQTg9lmCma3MrVmCyYzSD7jnv61k3vHOtNUjI9TNcjBEzX5wfTRWKaTHdCw/F9TvYleHg0iyzLmwqZd9zTx3h3nGLdpSuhcaw3ecfR2r1g7d+R7QXcWHOWVn+POxFMd9MQ285ts2EY8uJUiacuTvLd6/l115O6wgfPjfBT50f3pG9GlFUEAsF2OTLio2k/XqhzvC+F6XjUHJ8zQ2neeqKPhaqF64X0prQtj69upx9jO9tzG8E8X3WYLNQp2x6e7xNTFaaLdVRZplh3eX2hiirLXJmtkI6p/K01m25XNsVWTI+EGpKv2xiazERPEtcPODO8eQYFWGVy1or9XCe/9u+gbDrkqw6ZuEq+GnlknOxPrft72olg2kn/TTu3zfpByDevLvHUxUlenausP1/K4KOPjvL+s8Mk92DJniirCASCnXJkxMdG/RlhGDLRm0BXJRRF5tHjPeuMvU70JZtryxfK1roldMd7EyxWbC7ejO5CV6683+4d9sodLRPdcaYKEpO5KuPdSZK60rRCv7lUpe4E1BybiuXx+mKNRyr2qgDaKPtUba9ZPnr2Vh4keGAky0zRpO74vDRd3rYh2Er2ep18qyWAjde9VLWZLNRxlwI0ReahsUzLXpOdCKbt9qy0a4LFdH3+/OU5PntpitmSte76yb4kT1wY4933D6Apu1vqaJRV0jF1159LIBAcXo6M+FjbnzHREycMQ24u1ZgpmbhuQLcR+ZH85SvzXFuo0psy6EnqnBvvYiATW5eRGOuO05syGOmKcXm23HLl/ULZ2paB18odLdcXa1QttzlNAHKzWTYMQwYzOrdyHqeH0nTHtQ0D6EpxkDRUJOkNfw5gR4ZgK9nrdfKbLQE0VJmx7nhzqqnVmPTa96TdgqldEyyFusMXnpvmT5+foWyt7xF5dKKLJx4b58Kx7l3tsZGkaN1A2hBlFYFA0B6OjPhI6go122O+bJEyoqbJl6bLXJktcWW2zD39aW7lTHJVh0LdIVdzOTOUQUJqBuTG3XImruIuBWQTGn7wRoZg5cr7RuPjd6/neHm6zHA2xnwlakrdTHys3NFy8UaOhKbQm9ZZrNgYqtwMkgOZGD98eoDnbhdR5ajhb6MAulIcJJeDR83xm5mfklnetiHYSvZ6nfxmSwDXTjWlY60Xr+2GYFo7wbJT58/JfJ3PXpriK6/MrxMwsgTvOj3AkxfGuHcwfddn3gxRVhEIBLvFkREfAGEIhNH/ThXqzJVtdFUmCMH2AhwvwJRCepMGjg9zJZO+FUG9cbecr7poikyp7tKbMuhPGyxV7VUr7xuNj5P5OgsVk0xsa2/1yh0tx/qSQIgfQFxTOT/RtcrR9MxwZktbaO+0ev7hNT0fK1/r2ibNhbLVnBhaWV7aC1YuAdQUmfJyk/BG4807fU+2y0YTLNt1/nx5usRTz0zy7ddzrO0OiWkyP3F2mI89OsbQLjbzakokcFOirCIQCHaRIyM+ao5POqZxeijDKzMlri/WqNg+VcuhK6GRiakMpA1qjstcyUKWQo71JnnTse5mAFu5hO6hsUwzExGGIePdcdIxla54NAnSuEt/aKyLxapDSMip/uQd92ZslqVY23fRjgC6HUOwxYrNN68ucX0pElmn+pO8897+OzZqbtarsR3hsvL9PzuWXfU4d3o9u4HnRzb8t/ORxf7a7MZWnD+DMOTb13I8fXGSl2fK656jK6HxsTeN8cFzwxtmce4WWZJIiLKKQCDYQ46M+Fh5J+8FIT0JgwdG4txcrDCUjdGd1CnUHabyISPZGLIs80P39TenQVY2YKZjGieXA+dC2VrRsClzvC8VZQPKFoosYTk+Z0ezHOtd7Z2xEXsZPFfSqsG0cY5GxuO713O8MlMkqWuklntMtrJQLwxDXpour+uV2e5IbvO92aMx3o1YOTa7VLE3zG5s5PwZhiFzJZu/uDzHV34w37KJdCgT4/ETPXzw3DAn+1O78jpiy7tVkqKsIhAI9pgjIz5W3smP98SZLphYrs9Id4K4rvDafJVC3Wax4nBmOEPN8lms2CxW7OZd/wuTRQo1F8f3edOxbs4MZzbsjWiVOdiN8kS7sgprG0xXmphZrs8rMyVemi5zO28iUWe8N8HDo10t+0zWPlZ2uTdjba/MXo7ktoMgCCmaLmXzjbHZzbIbrZw/y6bL//PM5IZOpGeG0jx6rJt7BlKoikw2vrNlbxshyioCgaATODLiYyW9ycjkq+b4WK7PpZt5XpuvYjoetwp1JgtVFEkhCAO8gKaIKNRcKrbLYsVGkiT6UsaGUxN7lcHYygTISjYaoV0rom7n601DtqWqTaFuM5KNkVm2bT833sVbTva2zOSsfSygZa9MuyZMdtthteHVUTLdddtmN9trstL5c65k8f/+2ut8+aVZrDVNpBLwznv7ePKxce4fSq9rUr1bxLSKQCDoNI6M+FgoW3zr2tIqR9CT/SmuL1ax/BDHD7iVMylaLt3xOBXLj6YXqg4VyyUdixxBFys2fWkDVY4C9om+5I6Mp9oVLLcyAbKSjUZo14ooeGMEt1h3UCSJoulSd3wG0wb3DqY3bDZd+1gTPQkkSVrVK7O2V+Nu3qvdclgNw5Cy5VGqb+zVsdFekwavzVd46plJvv7aImt0C6os8aZj3fzfj0/w0OgbBnQNwXK3iJX1AoGgUzky4uN2vs7rizW64hrz5RoTPdHIa8pQiasyuizTm9LwggBFUajaNt+5kWM4U2e4y+Dt90TNpwCmF+D6AZYbpc23k+EIw5DLs2WevVVAVxS6k1rTR2Tt921FoKyeAJGYzNdJGCrjyz4ma3+mIVaGu2JcnilzeTZqcuxbzpas7NNojOD2pDRGumJcX6xyY6mGHwa8MlOmN6m3HBveqOS0W8vcdsNhNcp02cyV7E1HZVdmNxqEYcj3buT579+7zQ9aNJFmYirvfXCIH7qvj6FMvC3ZjQaN7Fs6pondKgKBoGM5MuIjIqRiuRTrNoWaQxiG9KV0jvclubFYxQtCNAVyFQs/CPG8kNmSyYtTRU4PZTgznCEMQ77x2iJF1+OVmdKGAbj5jC2aL5+7XWSqYDbv/FsFy416TNYGv5UTIKPdcW4sVtFkmemCSV/KWBeoG2Ll8kyZqYKJhITrl5pBvXGOlSO4luszXTCpWB7zFSfajLu0sWdJO0tOWxEW7TQMMx2ffN3Bdn1yVWdbo7KuH/BXVxZ4+uIUN5YnglbSm9T5qfOj/NT5EeJ6+371JEkivpzlSOjKno0+CwQCwU45MuJjvDtOTJN5ba5CXFcpmVHvBkQbZ3VNwfYCTvanmSrUCG03KsZLMpXGVEcmRt3xqdg+XXGN60t1jvXWGczGN8xUtGq+VGWJvuUm1pXGYSvZqMek0fy6bipluQRSs/1Vgbp/zbkaGY7Ls2UkJO4fTjNbstYF9ZUC4vpilSCEvnSMYKaM4wco8s7uqle+TytHiTcaK96KsGiHYZjt+RSWey0abGVUFqK/qz97cZY/eXaKpaqz7npfUmeiN8FbT/UwnE1QdwLa0UeqKTKZmEbSUFBF82hHsJ8bngWCg8SRER+SJKHJMilDYzAba/ZFAPgBHOtN8PpiFVWRMTSZ7rhBKqbh+AGZmEpSV1goW0wX6ixWLDwvwPaD5obSrU7DAM2757imrDIOW0nKUFv2mAAbliFaBerN7N1dv8RsybpjtqDxuBIwmo38TIaz8Tt6lrRipRirWC6SBEldZaZoYvs+PQmD3pTOw2NRKWorwuJuMi2uH1CoO1Rb2Jdv1kzaeC2fe3aKP3txlrqzfnLlnv4kx/uSGApoqspEd2Q+t5GI2QqyJJE0ot0qMU00j3Ya+7nhWSA4SBwZ8VFzfHqTMXRVYbFi44c0yyBV22WpYhNXFaqmQ0xV8P2QmCZxsi/JWFec77y+RKEeCYtCzcH2fPqSseb20K1OwzSaL+90Z9SfNnjT8s6Olfbpm5UhWgXqizfzXF+q0RXXV9m7bydb0J82ODua4VauRndCoyuhNT1LVi7g28pd3srzP3vLBAn6UjGuLlQJCdEUpfl9A9ydsNjsLnQrK+43aia9vljl6YtTfPXKwrrpF02R+FtnBvnYo2OkDZWZkknZdKPyleejyPI6EbMVGp4cKUMVd9IdzH5ueBYIDhJHRnykDJXu5eBhqDLnJ7roS+lcni2zULYIw5BsXKVQt7H9kKLpoSoyARLPT5aoOz4ly+P8WIbhbJxTAynimkJMUwjDEMv1mSnVWara9KUMCnWbW7kajx7rXhXk+1J6y9T8WjazT9+oDLF5oF4dJLcT1CVJQpIkypZPSPS/kiSxVHW2fZfXasndzaUquirTndAiEagpzUzT3aSvW92F9qeNLa+4X9lM2ujVefriJN+/WVj3vQld4b0PDPG33zxGX/qN96A3bbTc8bIVGhtkU4YqmkcPCHu14VkgOOgcmd+MvpTOaHccTZFQFRldkbgyV+G528VmILqdr7NQsSiaLnFNpS9lMF0wCUM4M5IlP11krmyTTagUajbTro8EmI7HTNEipWtMOnWmiyb9KYNbuTrHepOrnEK3MunSoJVA2G5/w0RPglP9kd37qdSd7d0brM0aVCx33R0dtN6Iuxmt7ONv5+skDQU/iMog5ye6gI3LS1utq6+9C50rW1husK0V934Q8tevLvLUxUmuLVTXXe9L6bz5eA/nJ7qI6yqStF4ktJqI2QhJkkjojebRI/PreWjYqw3PAsFB58h8ui1WbC7PlpktmeRrLqcH07h+gOkFxHSFSzfz1B2PuKbi+QG6olA2PVJxBUL4wXSJnoTO4yd6cIKQr70yT950mcyb3MrXONaT4s0ne7B8D9sJuHCiF9NZbT++WLG3NOmyks1sz7fCQCbGO+/t3/aH4dqswUhXrOUd3Xbv8loJqoFMrLkPpyFIrsxVyFVtzoxkmC1a697HrWRcGneh1xermK5PT1LHM7YmPEzH53+/PMtnL00xX7bXXb9nIMWTF8Y52Z9gumDdsSn1TuiqTNrQSMXUps+K4OCxX+sRBIKDxpERHw2fD98PmC6a3DeYQl/uL7CkaBV7V0JjpmAhSyxbW8vcP5Tm5ECa64s1zo5l+dEzg3zz6hK6pnAiGaXwTcfD8X1mSxbD2ThhGE3QKLKE6Xg8cyMHREJCkbnjpMtK7raBbacfhmuzBoYqt7yja8dd3sozLpQtXpwqka86TBUaDbqr3VArlku+6pCJq+SrLhXLXfWeNATbUsVCUySyCZURfWt+Gvmaw588O8UXX5hdt6UW4LHj3Tx5YZzzE11IUuSvMivbGzalboZoHhUIBEeVIyM+GuiqgixJLFYshjJxBtIGuiIxmTeZr5hU7Kjkoqug6xpVO8DxQs6Nd3N2NMNS1cFyPUzPY7pQR9NkHhzu403Huokt9yoATev2V2bKzS2wfcsmVZbrkU2oG066rGS7DWztGvVbW7tOx7SWIqbdd3mN13v/cBqAwazBmeHMqvfJ9gImC3XcpQBNkXloLLPqMWaKJt95PUfd8bfkzwFwO1fn6UuT/MUr87j+6l4QRZb4kfsHeOLCGKfWLHm7k8NpK+K6Eu1XEc2jAoHgiHJkxMd4d5z+lB6l8odS3DuQouYE+GHIzaU6i1Ub0w0IQjAUiVRcRw6jlemlus2DI2kWK1bUfGp5GIrCaHecnqTB4yd7WhqAXV+sUrM9uuI6EFJzPFQ5Sq8njainZO3PBEHAlblKc6FdT0LbVmljq5mSO4mU/apdJ3WFiuUyV4oaUu8fSq87v6HKjHXHySY0SnUXY7kZ0/MDCnWXawtVao5HTFWYKtZJG0pLd9Jo226Jp56Z4jvXc+vOktAV3nP/AO8+3c94T7KlsNhqP4cqy9G0iljoJhAIBEdHfACEIUhIpAyNnoSOJPnoqsQrsyWmCiaaIpM0FGpuQDFXIwhBlsAnRFEkcjWXqXwdWZKQkXjsZDeOH2K6rfsIUoZK0lCZr0SZj3RMoTcR48xIhpmiSa2FN8SVuQpffmkO14/u6n/8ocFtiYCtZkruJFJ2q3a9VlzdP5RGXmNYJkmAtPy/LUjHNHpTBn4Q0psySBpqJBJNlzAMSegqpuPz6lwFgIRmMdKVaGY//CDkb64t8dQzk1xe/p6V9KZ0Pnp+lLed6mOqaFK1fV6dr2wpg7L6dUgkdYV0TCx0EwgEgpUcGfFxO1/n5rKgmCzUSRkKPSmD713PsVSz0VSJsuUyEU9wfDjOUs1hoezgeD4V0+W1uSoly6VseZSX77YVTaI/FSOpvzHVspL+tME77+3jWG80YZLQFWaK1qZZjMWKjesHnB7K8OpcmaWqw4OjXatszxsjqI0ST9X2sL0AQ5WxvQBZouVzrMx25Ko2nh8w2p1gpmhSsdzmY+2mM+NacQXwwMgbS9Uih1ON+wY3FmgrLeUb/TUrTb56khrD2Rg122OsO7F83SPlKvz5D+b57KUpppcN31Yy3hPn/3rzBD9y/wCaIjOZr2/J4XQt+vLivEbpSiAQCASrOTLio2i6TBXqmE6A7fmMdMd4aKyL3qROXzKyJ7+5VOOxEz2896EhvvTCDLOlJQIkcnWHnqRKTFWQ41HZJKZJDKWMllMtDSRJYjAbbzqKhmFIfzq2aRajP22gKTKvzpXRFJn+ZZ+IhmiwXJ+Zookf0HQI9f1IUI11x+lN6Yx0xZrBOAzD5oK5ldmOqh0F7oZIsb2AG3vgzNgQV/cNpnnudoEXp4pN2/hGpqBquzx724wyRy0yBpIkEdNkpgseZctdt/RNkiRGuhJUbB/bC7C8gC+9OMtXXpmnZLrrHu/0YJrHjnfzo2cGmOhNNr9+J4fTlSjyG82jhiqyHAKBQLAZR0Z8dMU1sjEdVfbpVQ0SWjRh8LZ7+pgt2SxWTVIxBV2RCMOQs2MZZssWsiQThAFvPdlDzQm4ulBFU2SO9cTJxHUs10dV7jy1AlsrZdw/FDVarixLrBQNixULTZF5YCTbdAgdTMdwlwKyCQ0/IDJEM6PyS8ks8/Dy864syUwXQ3qTenOSZKWPx3Sxzq1cbVeyIA1x9dztAoW6Q8X2eXGqtMbHAwiX/3cNdccjX3OYLVqbLn3rSWpkYipfeH6ab13L4XirS2OyBG852cu5sS6GszFkWSJprO7p2EozaXy5rJIUC90EAoFgyxwZ8XGsN8nZsSzXFipoqsxQNkbKUDnem+BHzrh86cUZ8jWXF6fL5E2Pd5/u5+339Dd3orzjnl4kSeJ2vg7AWFeMfN1lqerQnzbou0MvwFanUGRZXlWGgNV9HKW6ixsEqxxCy6aHpsiU6i69qSib0qrvo9HM+eytOkEIPQmtpXNqzfaoWh75mtv2LEhDXL04VaRi+7z5eDdzJbu5BO9WrsZcyaQ/HcPzA6q2xyBRaSVfc7DcKKOzdulbzXGhGn19umDy5R/M8a2rS6zVLzFV5n1nh/nYo6MMZWLkay41x8XxwuZjNLIoGzWTastic7PmUbFgTCAQCDbmyIiPvpTOvYMp/CAgG9d5+6neZtA1VBkFiUxcZzBtUHeiYP9D9/WvCx6NEspC2WK2VMUPQmaKVsv19StpZC+8IKBme0z0JDjWm2zarW8WpFaOvXYnNUa746vGequ2x0NjGYzlXoMwjDIerS3YoWJ75KtRuaJs+U3b8UZja65qk6s6u7KfoiGu+lIGL06VmCvZUclCV7g8W+avX1vghckSsgSj3QnuH04zX7aorfHcWFsSsdyAv3xlmm9eXeLWskBcSXdC48PnR/nQuRGy8TcyGL0pHaowVWhkUayWjaXbbR4VC8YEAoFgY46M+LgyV+Frry4up9BtHhzNMNwtsVC2uJ2vYwcB82UL0/E40ZdEVeRmU2cYhlxfrDabOtMxjYrl4gUBcU3hZq5GNr753W3FcslVbUJCLs+VqVouJdMlpincytVRZFBliWO9yebSNkmSmj0b2Xj0VzXRk2AgE2teW7nEriGmFsrWqu9vfL3RzHnPgMrzVpFsXGtu9x3IxJoloZShUjK9tu2naJUFWDvKG4Yhz94qMFUwsb2AuCZTrDncytXJtNg/3yiJlEyHi7cK/H++do2ZkrXu+8a64zxxYZwfe2AQXZUJw5Bc1VlVSlmbRVnZWGpokSdH2ojEzlYRC8YEAoFgY46M+Li2UGW6aDKSjTNdNLm2UOXB0S7KpkOuajPeFaNu+4xmdc6Od2E6Llfn/WZjZqHmcDtfZ6IvyYneBCNdcWq2x4tTJYBVEy+NiZRGiWaiJ5q4mCqYLFas5QV1XdzI1ZlcqlJzApK6RMH0ONZTozdl8OBIhuN9qWUvivLyHTTkas6yiFDXXIvuroHm12QJksYb35/UFRRZYqlq4/gB1xYrDGfj65o6d+rxsVGpYaMswMr+l+uLVXRFIRvXeH2xFi3t26DBMwxDbudNvvTSDH95eYFivXUT6c88PsHb7ulFXiEI8zV3Xa/I2ixKOqaSjUdW5zttHhULxgQCgWBjjswnYlyLnE1LpossScSX7axnSxbfv5GnVHex/YDBlM53Xs8RhCFvOdVHZXkdes3xmS/bpGIqGUPlRF+S8e44c0WLvrSB74dNm+/Fis23ri3x+mLk73GyL8lET5zx7gSj3XEuz5aZLNSZL9vkay5ly6ViunhBSNLQeH2pTs32KJnRuvfZksXxvhSzxTpzJau56VZTJGw3cgOdLVnrlr1dnimzUIm27CqyxNnRDA+PZbm5pFC3fWSpdVPn2sbYleO9m/UvbFRaarWUbm0WIKkr6KpESlcZyURTOxM9CUaWy1wN5soW/+07t/jLyws4/uomUgk4P9HFR86P8dZTPS3P2CrLMdYd5/RgmiAMGUgbHOtNrPMe2S5iwZhAIBBszJERHw+PZZkqmBRqDt1JnYfHss1yStl08cOQXNXihekitgdBGLBQcXhkogtNkalaNt1JjZrl4QUh6ZjWHOO8sVRDU2TOelHmoWpHo7ddcQ2QqNkekiTRk9Lx/ICzo1k0RSKuaqR0k8uzHn0pHdP1CYKAIAzpS8co1FxydYuK5TNfsUnHVHoTBnFd5cXpEgldxnYj9dCT0tcte3N8H02Rm0G/5vic7E9RtT1Guz2GszGuzFa4MldBkqQ7ioo79S80Sg1xTeHFqRJVKxJQGy2la2RK5ssWrhcw2hWnK6lxfqKbuuM1zxKGIdcWqjx1cYq/fnWBYI1g0hSJ9z44xMcfHWP8Dlt7oywHXFuo4Ich4z1xepI6x3qjUlu72I8FY53S5Nop5xAIBJ3LkREfA5kYbznV2xxhbWQoFio2ZdulUHMxHZ+aVaU7ZXBvfxJVkZjojnPvYJpnbxWw3ADHD+hP6YRhiK5ILW2+k7pCEIbczNVQFYnjPQmCICSmyXgyTPRm6I6rfPNqjtv5AEOTGO+OEyAR0xRShoYEOL5Pd0LngeE4N3M1hrMxJCRuLFao2R7j3Smqls9ARueBkey6ZW/jPZHoWBv0U4aKLMH3r+e5kavQX4oxma9zfqKLvpTRLNM0gsZW+xcapYabuSjjc7wvheX6Gy6lu5Wr8d3reRwvaJZAJnqS5KoOC1UH3w949naRZ28XeWm6tO754prC4yd7+L8fP8bJ/uS6663oSWoMZWJUTY/umI7nh7h+2FbhsV90SpNrp5xDIBB0LkdGfCxVHWaK1qrplKrtMd6d4ERvkppVIpHUKdYdKqbDrYLMPf0p+tKx5cVmMW4uVfnBTJnZkknZ8jgznF5l852OvTFFkdI1RrKRwFmq2rwwVWSuZEfBr+pw/1CKmuthez5hCMW6zfnjfTx+rAs3lJpupTNFE8sNGO1KcHY002w0vV0wuZkz0ZeNyABuLNWawb3Re9J4nSuDfn/aYLQ7zg9mSlh+VNbJVx3KpstgNkbK0FBkGOmKpmqiDb2tXVNX0ig1ZOMqSb2O6XioirxuKZ3p+ORqNi9MFpku1hnrSmB5frPRs2w5PHurwPdu5Fs6kfandN5zZpB3n+6jJxnb0jI3iMSK5fqoskRP0lhVrurkZtCtZhI6pcm1U84hEAg6lyMjPkp1m5cmiwRhgCzJHOuJkU0Y9KYM7h3KsFSxcfyQ7qRGQlfx/YDxngSm67FUdRjIxLiVq7FYdcjGNK4vlVBluG/ojRHXlVMlmXj05y88N81i1SZZcVioWGhKlrpbQ1MkZCnaMzNVNKm7PldmyxzvTTLSFTWBJvWwOWK6csrl1ECKQt1tZlxqtsdsaf2d5kap/8ghVGEkm4gaT+dr3DOQJAijyZf7BjO8MlNirmTRn44tj73Gl7MyG/cvNJ6vP21wrDe5TvTYXuTVYTo+uWUxmKs65KoOEz0JJCQ+c3GSpy9Okas56x7/VH+SC8d7uH8wjabK9CRjd9y1oinLC92W97+8vlgjV7WZKkSiZmW5qlPZaiahU5pcO+UcAoGgczkynwqX5yr89WsLWK5PTFM4NZjkg+cynBvv4nhvnIG0wQu3CpieT0pXMHSVt57qw3KDpgFWoe5QrDtYjs9CxWKqqJOK6ZwdjVa6NzIPjamSl6eLlC2XuCZzbaGKH4a4no/phhTrDiXL5dp8iWLN5Z7BNAtlm7++Ms+9AxnydRtDVRjpiqMqctP0CtYvVpMkacM7zY3umlOGSndSo2TqDKRdehIG2UQ09TFTNHH9AMsJCMOQQt3jZH+Sk2vWyW/EWtHj+gGFmt1siAWWR10Vzo11cWWuzDM38/zHr12jZq/f5fLmEz38xENDxFSZparD0HJGaaNdK7K00upcZrFic7tWj/bZBAFnRqK/r8GssZzVekNMdWK/wlYzCZ3S5Nop5xAIBJ3LkREfsyUTPwgZ7k6wVLaYLZnNJsvFikXZdOlJ6yiSxPHeJLIsYbo+qhy5WS5WbEp1D11RmCuZqAroisyVuRJ+4C/bsNOcKjk7mmGmUCMTU4hpKprqkDUkcnUbCYmZYp3Zko2hq8iWz0LZQlUkbuXruEEkdFK6yqmBNJbrrwo4az/cgyDgVq7Os7ci19OVo7Mb3TX3pw3OjXdxsj/Z9C9p3KHWHJ+kofDd6zmWJm00Reahscy233PPDyiaLhUrmtpZSUJXWao5fPPqIi9OldY1kaqyxI+eGeTjF8bIxDRena+Qq7rMlSMvj66kvmrXShiGzX02A8uOs5IU+bg0Xn9jF85s0aI3FQmPtRmETuxX2GomYT+aXDv5HAKBoHM5MuKjO64jyxKFio0sS3QvG1ctVmy+8VoUAFMxlYSuEBKl62UJHhyOvDauzEXeEO863c93ry9xK1fj29dy+GHIUsVmNJtgvDdBvhqN5qZjGglDI5PQmSnW6UlqvPVUH7eWquTrDjUnYLJg8cBQKrpT1xVODab59rUlrs5X6UsZeEHIzaUqo92JpshotY5+vmRuuIZ+o7vmZoBY7g1Zebd/oi9JGIaMdyfWNdNuBd8PeH2xynzZJqYpq5a+hWHIC1Mlnnpmku/dyK/72bgm82MPDvF/vXmc/nQU9BvbZU8ORE2lfWmdk/0pepJa0+rcdDxuLNXxg5D5st16n00hjOzSl/fZtLoj78R+BZFJEAgEh40jIz7ODKc50Z9sjtqeGY52jFRtjyCAdEwlCGChbFOpLWIHEqbjcu1ED31Jg4VK5MkBMNoVp+Z4LJYja/C5YtSAmqs7zSyBtBwoHp3oplx3kYC5okkQgu0F1CyTuuNStl3ScY2TfUkSetSHEdejLMpIV4wHRjJNx9PLs+WW6+g3W0O/lbvmVnf7a0s7K5tpNyIMQ8qmx+uLFV6ZXW3k1ZXQ+MZrizx1cZLX5qvrfja7/B68+Xg3471JZOkNsdMwAVus2GSTGqcGUkz0Jkgbb1idF+pOS9Gw8vWritw0gmucd61/SSf2K4hMgkAgOGzs/yfrHhHXFE70JhnvTqDKkclYsLygrVCzsJyAhCHTk9SYLVoU6g6FmsdscYaTgymGUjGuLlSQCPnh0/2ULZeFikPc0PC9yJ78kfGuZpYgWg3v8cpMEQh5ZKKbiuVRd3xMJ6BQcwiCkJLposkyY91xBjMx4ppKyXSp2S5nx7p49Fh30/Bq5Tr651eso2/0mLQKmFu5a251t3+iL7mtu+2K5VKsu7h+QMV6w8hrqlDnT5+f5qtXFphtYX/enzJ4/EQ3E70JinWXk/0pbC9Y1c/RsFL3goDBdIxjvQmUNaOxG4mGzV5/K9ElsgwCgUCw++yK+JienuZXf/VX+fKXv0y9Xueee+7hD//wD7lw4cJuPN2WmC3bvDxTwnR84rrCo8e6cQL43vUclheiqxKn+hMslBxuLJapOyFj3QZLNY+XJvNc0zQsL/L5GOmKMdGToGi6qJJMfzpFJq4jITWzBGEYUrHdZQdTn9cXI5+OvpRONq4zV6ozX7HoThp4Ychkvs5jx3tIGirfuLqIqsrMlUwWyhayHO2ZkSWwXZ+/fm2euu0z1hPnxalS07m0ETD7Uvq6O/rN+hYi34+Q71xbZLFqUzYdEprMYDZ+x36Hmu1RqDurVtYndJW66/M/n7nF928UMN31TaTHehJ86JERehMa3clIZMwUrWisV5Gb/RyRkNAY70mib1L62Ug0bJY1aFliWWP7LhAIBIL203bxUSgUePvb38673/1uvvzlL9Pf38/Vq1fp7u5u91Nti1zVxvNDBtMx8vVon0sQguuHPHq8l2dv5bk8U+FW3sQNJOqOy818iO36aIpE2fQZ70nQmzSYLproqsJAKkbN8bh3MM29AynqbtAMfNcXq9Rsj8G0QW/SIK7LnOpPUbY8Xl+sEkoSYQhT+Tq9SZ0buTq383UkSaJiechI/M21HPNli8FMjFRMo2I6dCc1XD/A0BTuHUjh+GHTubQRMFc2WW6labI/HbmmXpmvUKg5TBVNqo7H+8+OLDfk2quEzWLF5upClYrl0pc06Flu7oSoP+Ppi5P8n1fmcf3VXaSyBA+NZjk3miUEYoqCqiqMdCXoSWqMdCWiKRhNIabJmI5PX0qnJ6lvOHHSql8F2NLESieWWAQCgeAo0PZP29/93d9lfHycP/zDP2x+7cSJE+1+mm0T16PdLlXbR5YkYpqMIkfW58/eyhMSYrk+ITDeHcf2AkzbRVVluhM6JdOlbgdUbI+B0MBcduW03GjS5PRQhpP9kbV3Yx/LjcUauZpDXFd47FgP58a7AMjE1EiABCGvzJZJx2CpYnNlrhK5b1oeuZrNTDHKQoz1JviR04PMlwOycZ1z4z1870aO2wWT0a7EuqBZtT08PyCuq9xcqjY37sL6oNz42lShjucHHOtLUjE98lWnORq7Usj0p3Qu3S5wbSHq2xjvTnDheA+zJZOnLk7y7Ws51q6LiWsK7394iLed7KVq+/SnDV5fqK5qHJUkiaFsjHRMpe54vDJTwQ+i97HRPNqKVqWTxpk9PxqTPtabWLUpuIEosQgEAsH+0Hbx8cUvfpH3vve9fPzjH+frX/86o6Oj/OIv/iJ/7+/9vZbfb9s2tm03/1wul9t9JABGsjHShkqhZtOdNIhrCnXb41hPkorl0J+N8bzjkZ+r4vk+EiGj3Uls36fu+KRiCkNZg9GuGN0JDc/zKVs+fSkDywm4PBuduz9tRJmHyQJly8VQJXqTGg+OpJvGX/c4PiES3QmdpaqDJkPJCpgpmMS1yLBsqWIz2h1jIGlgeT43c7XlTbZgOh4n+5Jk4irZeLTdNgzDZmBNGSpV2+PFZUvyVD7auAtsGKgrdvQ6K0s14rrSNN9qlCb60wbX5qvkqjaFutMsLb00XeJzz003xchKepI6Hzk/ygfPDZOOaeSqDq/OV1ioRGPFMS2aKErHNDJxjdjysr98rXXzaCtalU4gWq4X7cApMlc2eW2+yvmJLs4MZ5rvk2jkFAgEgv2h7eLj+vXr/P7v/z6f+MQn+Bf/4l/wzDPP8Eu/9Evous7P/dzPrfv+T33qU/zWb/1Wu4+xDtMN6ErqDHXFsVyfQt0lpqvcO5TixakCs4U6EiE9SY3+VBLTDQj8gLIjo8kyfSmdEAlVlsnXXR4czpCJh1hOQNF0mC2aLFZsJnri3Fyq8+JUkYWyje1FHiCpmNa0Rrdcn6VqZJI1lDVYqtrEDZmYruAFIcd6E9iuR9ny0VWJsZ4Uw9kYXXGNpKES0xRsL2C6YJKvuZTMcjM70BAimiKR0GUeGsliecG6jbdrA/Wbj3dHXhxhyPG+JA+PRs2XXhBQsVyuzlUoWQ5dCR3X9XlussjluSoVy1v3Xh/rSfDEhTHec2YQTZHI11wm83USmsLpgRQzJRPHC/D9kNcXqwRhyLHeJIYqNw3QVpZDkrqy4VbdjUoniixxc6lKzfHQFZnJfL1pN7/fvh0CgUBw1Gm7+AiCgAsXLvA7v/M7AJw/f56XX36Z//yf/3NL8fHJT36ST3ziE80/l8tlxsfH232sN1g2u4ovT6O8OFVipmCyWLWIaRKuHxIEcOFYN0PZGK/NVZgumthewNWFGgEhCV3hZF9k9X11voLlecR0hVfnKswW6zw/WeTGYhUngIyhEgQhU/k6cV3Fcn2mC3VUWcLxAsa646iShOWHvL5YwfUDjvcmuWcwzWS+zmAmxom+ZNPHIl+zOdWfoiuh4Ycho10Jpot1buVqVG0Py/Wb+2BsN2Sh7LTceLs2UM+VbE72pZr9IX4Qkqs5WG5AJh6ZfE0XLb57I8+1hVrLJtKHR7M8+dg4j5/sQQLyNZfpQi3KiixnOH74vn7uT2QwtBq263PxRp6FikXJdHl4rKtpgLayHBKG4YY9LBuVTho7ZuquR7HmMZAx0BVlS+6vAoFAINhd2i4+hoeHeeCBB1Z97cyZM3zuc59r+f2GYWAYu19rTy7fIZcsl4SuMtoVp+74zBZNhjIG04U6rhfi+j75uoPtBzw4kiVfc7m+VGe+ZON6PrmKRU1XmS7USRoaFdul7vh8/Upk3Z6IqUwWTPwwpGJ6KDIsVm3++rUFCqZLfrnRdbwnScWK9rK8PFOkbPmoShT4RrtiJA2NfM0lrincWKyyVHcpVm1+MFvmG68uMNaT4P6hNIRQczyqlke+5rK4XNIYysRYrFoYWuS4unbj7dpA3fhab1KnUHMomS7BslArmy7fu57jldkK3horUgl4YDjDO+/r4z33DzZ3rTRKLDcWK1xfqnN6IE2xHrmdHutN8rJd4uLNAvm6y0A6Rm65x2SVAdryc1xfrG5YhmlVOmm4qfYkdc6OdnFzqYquRHbyK/tjOtHNVCAQCI4CbRcfb3/723n11VdXfe21117j2LFj7X6qbWGoMsNdcTRZwvUD6rZH0fTI110cz0eWJXI1B9OJTMe+9/oSYRC5fE70JMhVbdJxDdv1kaUAP4S5kkk6pnJ6OMP1pVpUGljylqdcYhTrkSiwHQ8vCPB8sNyApKHw+kKVuutSdwIm8yZuAClD4dXZCp4fEtMUana02n6hZDFXsZgrmyxUXHRFomi6JA2Nh8e76UUnV3UY6YpTrDvcztd4caqEpsgMZeLNu/mN7vIHMjH6lw3CpoqRDT3Aq3MVnnpmkq9fXWwkjJroqsw77unl4dEu7h/KsFCxmt4cQRhybaHC7aUqcU1Fk6BqR2KmUHN49Fg3Ez0JZosmg5kYpuPjBeGG0ybbnUpZKSokQo73pZp9K30rFtF1opvpQUVkkQQCwXZou/j4p//0n/K2t72N3/md3+GJJ57g+9//Pn/wB3/AH/zBH7T7qbaF7QXMFk3qjocEKJKEIoEfBDw4kiGpK1ydrzJVNCnUbeqOzPdvFjFUlfHuJLmqRcX2sN0ASZZZKJkYqkzd9bg8W6ZmOSQMFU2RqNoe8xUTQ5UYysZxPB93ub8haSjcP9zLS9NFbi7VmCs52J5PQFT+8EOXhbJFfyZOX1pnMl9HkcFQFTJxnbmSjaKqIEnYrkd3QsP2Ai4uVbk2H5Vt8jWbuh1wrC+B6/ncytWW/6uTNBTqjs9EzxsTIFXbo1Bz8YJokdz3buR5+uIkz0+W1r2PcU3hPff3896HBjFUlfmyxULFQpYlMjGNbFxjKl/n6nyV6ZKJ60Ur7Ku2TzahUbZclqoOx3qTFOsuhZqL4/ucn+jacNqkL6Uz0hVr2sr33WGT7UpRcXmmzGLVoS9lMFO0VvV8iFHb9iGySAKBYDu0/dP2scce4/Of/zyf/OQn+df/+l9z4sQJPv3pT/MzP/Mz7X6qbVFbDkjZmMZ82aZmuzw83sN81cHxQ4aycQo1lxtLFSwnZKBXR5UjcXLPYJKBjMa3ri7x6lwFRQXPD3H9gKLpcStXp267hBJ0xzV0RcHzo7HdpWq0SyauhRgJDc8PeWmqwGLFpeYEmJ6PCvghOJ7HcDbFyb4UXhBiOR4xTeGewSRThTq6EqOUcajYPoocLXKZK9vMlSzmyjalukvZclFlCV2VmCqYkaOq61NzPGaKNmeG08wUTWaLJi9OFTnZn2KiJ4EXhPzl5Xn+n+9PMrm8bn4lKUPhVF+SR8a7GO6KU6h5yLLHUDZGNq4xkDKI6wol0+VmroaqSLz1ZB9XZov0pwx0TaY/HcMPIjfUk/0pzo13belOeanqMFO08INwnYBoxUpR4fg+miK3zG6IUdv2IbJIAoFgO+zKrd4HPvABPvCBD+zGQ7cBCV2VURS5ObJ6rDdBEATkqzY9yRi5epWlqkMYQt32eO5WkWsLJRYqDgEhMVkiHlPRVYWYEpCNafQmdVzPR5KjPSUJ3WC6aAEhCU1FkaM+CAgpWRK5qo3lhVE5Q4a0pjCYNehK6tRcD5DoSxm4fsBi2cZQVSa6de4fSUd9Ktk4hCGvTBeZK1tkDB3PC7iZr5HUFfK1gO6EymAmxq28ia5I5OsO37+RR1dlbNdnpmhx6VYeJJnvXc+Tqznr3q2TfUl6kxqaLNOV1HC8gJrjcc9AmrLpcqwnQVdCb2ZWUoZKrurgBSFzJZP+dJz7hlK8Nlfl5lIdTZE5O5bd1pjrdgPbSlEx3hP9TKvshhi1bR8iiyQQCLbDkfmEiKkSs8U6+Wo0/XFmsJ9UXKc3pTPRk2CxYi1nClxSukJaV+hKqORrFi/MlJhdnngZ705guT6LZRtVkZkumJTqLklD4dxENwMpnb94ZZ7rizUsL6AvpRPXleXyRtTbEPgeM26A6QTIEshIDHXFeOLCONcXa1EvSdxgrCtOoR5lMi6c6MV0PHpTOif60uSqNq/OVVis2ixWoqyAtGy/3p3QsN2A4a4Ej5/o4U+fm8b2fEa74jhegO1Edu9X5qssLTfAruX0YJqPPTrKPQNJvne9QMl0qDs+3QmVTExluhgJDdcPeHGqxHSxznzZ5vETvQxnY4x2x7DcgLimkImp1LpidCX1DTfkbtYzsN3AtlJUNMZrRXZjdxFZJIFAsB2OjPi4MldlrmwhSRJzZYvXF+sc71fw/MihtGa7UU+HFzVe1l0fFJlCzaVs+yQNjbJpslS1uXcgTUJXSWoKKV0httz7UTEdzo+mOTeeRZZgvmItB0oJSWK5idTDCcJoWZwUoMmRv0dSV3hhsojth6TiBqoSlU2Gu2P0p2JYro+qyIx3x0kaLktVC9f30ZSoD2OyUCcmS2iqzHAmsnRPxRReni7h+gFBEJKvOXTFdV7JVXjudnGdE6kqS5wb7+JtJ3sZzMY4PRht/j3Zn8R0YiiyxLHeBHNli/mSFZWy/ADT9elK6MyVLG4uVRntTjDSFWuWSqZLFqoir9p9s5aFssU3ry5Rsz2Shso77+1jMBsH7i6wbTQNI5oj24vIIgkEgu1wZMRHwXQIfOjL6CyVbRaqFuO9qWUXzBJ10+ZWrkax7uCFAXgSvh8QShKe51MLojX2hiKTjMkYikQ6plF3q5RNj6SuMlO0+N7NIklDpSdlkKvZVG2XdEzjvoEUZ4bTTBdtnrmZo2756LKE7QfoCvSlDXJVB0WRONWXoGj69CQ1HhrJNqdbXC/gz16YYa5skTA0anY0rVO1oqmaVCaGD/hhyFhXfNmIrI4qQzoe41vXllr2c+iKxHvODPILbz8OSNQcFz8AWQpRZYnjvQkkSWKiJ0HV9pgpWsR0lVtLkYeHqkioUpS9OTOc5nhfiorlNksl08WQ3mS0o8X2ItMyYFXQv52vc32pRldcZ75S41hvoik+dhLYGgKjYrnYXoChRs6xjV01ojlSIBAI9o8jIz6Gs3EURWK2aKIoEjFFpWq7zBZrUTbCDymYDpYXRmUIWSIIJdKGjKpoLFUckrqM7ftMLtUJJZnbOZOi6WC6Hgk9gefLvDRVZLQ7juf5KEh4AZRNj2uLVe4byhDXVSZ6U+QqDpoqo8oSkixRMV2ySYNS3WWh6hDXVFRF4up8hbiucHW+xmLF5vXFCn4YMpKNM5Q1GMoavFKxCMOQIAwIwhDX8ZkrmxTrkTh5db5CyVzvRNoV13jseDcffHiYB0azKLJMQlewHJ+rC1WuLdSYKpiMdyeay+PSMQ0vCFmq2Mt7WHzimort+qiSzLHeZDOQN0olqiw37d1vtFhhv1ixmSma1ByXbHx9VmQnNARGrmqveg2NDIofhAxnY1yZrayyxhcZEIFAINh9joz4ODOU4rETPRSqNiXbIxtXCENI6BoyJpdnSzhuSDauUazZkZdH4JOMxRiMKSzWPGquj+kEuH6IH4YkVIWYriLLMktVB9sLCAEvBN8Po+93o4bU2bLF119doCcZIwxCVEUicAK6EzFkKZpcMZabYC9Pl6gt93fIwGA2wWLVRldlZEnCD2G2ZJLUFU71p5mM1zG9gLmSRdzQuFW0mC/bLC6faS2DGYM3H+/hTRPdaKrMeG+SvpRBylBRFZnri9Vo7JaQxarFaHcML4gs2k/0JTk/0RXZxDsBJdMhCELuHUyTNjRqTuR82qpUcmOp9kY2pFBfNQLs+QEKMq7vc6o/yURP4o5/p5uVTxoCI5vQuLFUIxNX8YOw+b2KLHFltsJkoU5IiOuHIgMiEAgEe8SRER+OD0EAphfgB9CbjhPXVQxVYqakEYQSfhAFU0NTuW8wSUxXcL2Qct1DCkLCQMIPQ6xlUyzXC1BVhfHuBJIUUqi5GJpK3fYICIjpKhXbxvJ8Yq5MxfIwfZNrc2VMN8APQ1JuiKFKGKrEbMlkumRStz38QKLmRN8zU3KWd54oOH50/v60jipLvL5Qxnaj7ENd8XD9kBemyuucSAGO9yb48COjeEG0b+ZkX5KpgknZdHG8ACX+RoNnzY78S0qmx+XZCg+PZUkZKpIkcWY4Q1/KoGK53F9Kt3QQbVUqWdk4WrW9yJnV9pgv27z5eA+yJDOYNTgznNlSX8dm5ZPGc+WqDpoiUza9ps18Qxhdni0TEnJmJMNs0Vo3RbORuBE9IwKBQHB3HBnx0eiLUBVwTJ/buRoPjHY1g5Uqw0A6hul6GJrCQCbGWHeCxYpDTIXXF6vU3Mj91PYDejMxug0VLwzRVehKxKhaPhXLJQwjvw9JAl2OzMx0VSZfd6haHqbrM5iKkatH9uphqKArEoYqYzkhXgCaIlO1XTRJYrg3hapI9KdUMgmDmYKJF4Rcni1h++D6AWXLo2r765pIJeDBkQz39CfpzxgMZAyCIMDxAl6aKZKvOtRdj6mC2dz62p82ov4O0+VNEz0UajYTPQn6UvqqBW8n+1Oc7E9x32B6S82gK7MhuapNrhaZf82XbW7lqqRiGgld2frf6SYjuI3nqlguZ8eyq3o+GsIIwPVDZotWyymajcSN6BkRCASCu+PIiI9i3Wa6VIcQHD8gG1d5eCxLX0onV7W5dDNPyfIZ605E22+zMXrTMTRFJl+zUBSJlK6gyDKaDCe64wykYxRMj3RMiTbUZmIYmoTthZiOR75qY6gKigxhKJPUFYIgxPZ8luo2VcslHdM5N5amWPcp1m16kjpFcznToUeZBNsP8AKJgWyC0a44QRhyY6HKbNnGdAPc9ZUVZAmO9cQ53pukO6GTiasktGiqpicZZ7ZoUqi5VO1IlOWqzqqtr8d6k5TMKLiP9SQ51ptkqeq0DLorx1o3ywiszIakDJWSGQmxU/1J0oZKefkcJdPbUkDfbAS3+VybPMadpmg2EjfCUEsgEAjujiMjPkqmR9ly8fwQP4hq/I3geO9A1A/y0lQJRZW5MJ7l9HAWPwhRlTQvT+bpS8WwHJ+a4zLWneAtp/oAmC/bxFSZ1xaq9CZVKrZHvmLjBlFJBknCcn1kOSRf96nbHoaqUrEd0jEDVQZJkulORqO3QSiRjav0pXQuHO+CUMIJYKFkkavYXLyR4/pSnaoTtCytyBKkdIWkIfOW4z1oioSqyjw81sVrC1WKpkvVjizP7xlMcyNX5cZSlRN9KYp1h1u5Gv1pY8OeDc8PiOsqN5eqZOOrBcZ2MgJrH79iuVxbqG0roG9lBHczQXSnKZqNxI0w1BIIBIK748h8auqqTNrQCAipmj5zJZPFis1AJkbdDbhvMMO58R5uLlWI62q0SbbqULMdTCdACn0UBRKawkA6FmUo6i7S8kRL3fGoWB6W6+F5IYau4AVQsxxScY1MTGOxYmGoMgohrhcy0qVHS+RUmftHstiuzzevLtKXjvHosW4eGM4wW7LIVR1uLFb4+tUchbrb0hTMUCWSeuRb4ocBVSvg2ckCJ/pSZOMal2fLlC2f00MZTCcqe1QtF0kiesylKmNdcW7n682JlVY9G1Xb48XpIjXHo+5GnhxnhjNIkkTFcslVbbIJjVzVoWK5G4qPVoF/uwF9KyO4d1Mi2UjcCEMtgUAguDuOjPi4ZyBFNqExuVQjk9SJa2ozODamPCzXJ6Gr/GCmxGtzFZbqDrbjI0mQiatMdMcpWS5Vy+XybIWuuErSkCnWXRK6Sq5qkU3oVEwH23PRpKjPYLQrRtX26YprdCV1nrtZIGe6FCaLpAyVt5zsRpUlXpiroioKg+kYhbrL1fkKC2Wbr7wyx4vTZVx/vepQZehJqPQldUw3YKnqEPghqiJTNB38wKfmyCgK6KrKq3NlTvYluWcgxWvzFYYycWp2Fcv1uH8oHTXJWi5hGHI7XwdgoidBf9ogDEM0RSIMoSumU6x5PHur0CzV2F7AVMHkxlKtaaMOWzP12m5A38zHY+Vj302JZCNxIwy19g7R3CsQHE6OjPjoTer0JjWuL4SUaw5zZQvLjcZCV25NvbVk8/J0icl8jYrlk4lHO1ykUGK+bJOrWQQhXJkvM5KN8fZ7+hnOxkgbKhctD9fxcfyQuKLghxKu73M7b2K6Hv3JGEsVC8v10GTw/Kj/ZDJXI2lohASMdcVZqlhcum1yK1dnumi2zHRIy/8RgucFpOM6tm8jKxBTFBRJwg8kbB8sz2O0O8ZbT/Xz8lQRVZaI6wpF0+XaQpWkrmK6HtMFi9PDGWwv4PnJJV5frAHRfpczw2muzFWYLZkslC0SusrxviS6ojQDuqHKjHcnyMRVyqaHocqEYcjl2TLP3Y6etzel8/BY17rsw9qAHobhqubWtUFnMx+PlY8tSiQHG9HcKxAcTo7MJ/Fkvs5Uvo7l+dhIzFdNqst3+Jdny/z1qwtULI+XJgtM5ev4AbhBiOeHQOTKaXk+S7XInTMMwPVBvZEnrsk4fkiuahHXZUzHx5GgavnEdBnbDfCCAAUIkZAVCSkEQ5NJqjLzFQdlvoLlhcwWba7MVZiv2C1fhy5HoiMMIZSiHg/LC3D9gPsGkqSKCkEIZdMlZqiMd8dx/Wib71zJjBxRHQ/rdoHFqsNSxSLbn2K0O85YT7w5IVK1PbriGiBRsz2uLVR5fbFGNqYiK1Lk8Gpoq8Zr0zGNnpSOH4T0pHTSMY3Fis2ztwpMFUz6lrMZW8k+3CnobObjsfKxRYnkYCOaewWCw8mRER83cya3ChZlK3L6zFVVinWXH0wX+f9+6zovTZWRgULdpur4GArIIXh+QDqmoWtg+9JysAcFsF2P1+YqQEhvysDxAzQ/8pZwvBBJhoodeYyEgB3YGLJETFMiE68wJBlTycYUSnWPF2fKLTfLSoCuRLtXErqK5fo4fkAQRAJEV2R6Ehq9KQPTCyiZHif6kxiagqHK3DOQ5MKxLp69XWS2UEdVJGZKNroaiaHpQp1TgwM8fqIHgHwtMg4r1KOpm5N9SeJaNAIrSTL9KYNHxrq4ZzC9rhfi7Ghm2abe5eZSFQBNlptOpvHliZtGViO5PFpbc/xVGY47BZ3NfDxWvXd7XCIRZYL2IjJXAsHh5Mj8JuuqRNZQ8Xwf1w+JKVAyHb7+2iIXbxUo1aNg5wcBQQhVP3IqDb2AuOsThBKOF9Iw0vCBihMi46PKoJsObkDTgExXJLwwZGWbhuMFGLrCyb4krh9QsX08P+Q7N0tUbb/1ueVoekVRJRw3pGq7JA0VRQbLCQhDSMZUZoomCxWHpKGiSjJnR7spmjZLVZuy5aEuW8vPVRwUKSRXiRpDHx7roma7dMV1bufrTBZM4lpULhlK68R1jfEuI1p4Zyg4XuRyOtodX3dWSZKQJInbeZPrS1HJpj+tk9I10oaGocqcn+gCaGY1qnbki5KOaasyHHcKOpv5eGyV3RAKokzQXkTmSiA4nBwZ8XGqP0lXQmWubKKpCt1JnVt5k9mSiR9Ekxau7xMEIEmR8AAIfKjaPr2KhhZNzq4iANwASnWfAJpiI1hWKbICuiYREgW7VEzB9kJuFUzyNa/1uCwQ0yQ8PyQTV9BkhbgmgaFQrNsogKrJWE6AIoEmSeSqHql4iCrLhIS8vlhlvmxRMl1C4Op8hVP9SUa6YziOj67IJAwZxwtQJYmi6fCD6RI38yYjXQavz9eigC9blCyXkunSFdOJxWSGs3FmilHviyJLnB19Y9rl9YUqt5aqqJJM0lCQgeN9CXqX7dvX2qw/e9uEEE4PZVZlOO4UdLbi43EndkMoiDJBexHNvQLB4eTIiA9JkkgYGn0pg2xcZziTIK7JDGUMfjBTxPHeyDzYK0y7PMD1fVKxGFU7JAyjksvKPEUIWOEbTaDB8vWYstwUSkha14jpErKscGmy1LKJNGMoxDWZiuUiEU2s9CcN+jMxbNcjCCUGMjrTxTrFmotPJIbyposiS8iShh8EpGIadcdjqlCn6vgMpA0qtkfJ9KISUkzjLSd7GMjEeH2xhu0FWI5PV9xgoWJzO1+lbPoc70lQdX16EhqeHzLWE0eSJPwgWr7XCLC383VKpke+6nBlvrzcMxI978Nj2VXL5uCNVPp0oU4QhJiOz+WZ8h3t2bfKVjMauyEURJlAIBAI7syR+WRcWt4Ue/9wlsWKjaHJDGZizJVNDEXBU6OSSbgsIlZqg4QelSHmK3a0I2aD5wjX/JwiR5kTWQI7CFkouOvszyEqqwxldABqtosEZAyNkCgDMJiOkTMdpnJ1ZAk8PyQkpCuuYtoeCnCsN0kYhnQndMa64wRByK1cnbrrU3c8srGoJ2SiN4EiS7zlVB8xTUFXVWKazPdu5FksmwykdeJanFfnKsQNhboXULP9VX0V/WmDmaLVDLAAfhCSiatoisyjE90s1WzGuxO85WTvuqxFI6txK1ejYrl4QchMqc5Idw99KX3Lf6cbiYxGRsMLAmq2x0RPgmO9yXUiZDeEwsrJqf60sa3XIxAIBEeFIyM+FFmiZDqU6i6qIvPweJZTA2mevVXA9EIqlo8URhmLhkBQpeVsxnJAszxaioeNMN1loeJAlEN5AwnQFIgvi6D+lIYmK5RtF1m2SMdUdFUlbqjEdIUhxaBU93EcB9MN8Hyo45PQZU72Zbh/NMOV2TLZuEbZ8pgtWrhBQGy5wfP0cJrzE108ONLFldkKS1WH/rSBIoPp+pzsSxAEITdzNcqWS0yNfu5Eb5KzoxlScb3ZV9GX0ulLGc2gH4YhJbNMvuqiKzKSJHH/UHbDMkYjq1G1Pa4v1pAkCcsNuLlU477BNAOZGEEQcGWu0gzi9w+lkWV51eNsVDapWC75qkNAwOXZChXTbWnZvhv9BEtVh5mihR+EzBStpgeKQCAQCN7gyIgPXZHoTur0pgyCMGQwEyOuqyhKtFzMbeWlsSw+6o6Pu03h0Si/rEWRQFMkDFkiHVfxAuhNGvSkdCzXJ4WGkwio2z66EuL5QRQ8LZe67UaTLq6Pqig4no9mqMR0iclcDVmS6E5o3MxFUyZjXQaKotKfUHnbPf10p/RVa+Rt18P2Q2p25FRaM10Wqw5zJYuupEpXQuet9/Q1HUyhdbYB4OHlno+HxjJbbv5MGSpeELK0LDBWeoZcmavw5ZfmcP0ATYlExwMj2VU/v1HZxPYCJgt1FqsWJdPjTRPdLcdwd6OfQPR8CAQCwZ05MuJDlmX60zG64hpF00WW5cgu3PLx/NaTJkEA3QmNmuPS+js2Zq1QUSRQlWgsViLKxPQkNEAmFVNIaTIKkTApWxKW6+EDsiQRLPuNpAwFy3GRJAlVliIvEdvjVt6kJ6GiKApLNZeqHVBY7gPxfI/uZAZZlhjJxqlZleaY78szFfI1i5ShU1sWEz0JAz+IplQShkpMU1YJj40Mw7bT/LnSnfRYb4IgCDBUdVXPx2LFxvUDTg9leHWuzGIL35ONyiaGKjPWHWe0O8bl2TLFms1oT7ItZZU79ZMclJ4PMRIsEAj2k878ZNwFJnoSnOpPUrU9TqWSzRXxg5kYcU3Bdr01hZGoBON4HrIkIxMQsL3sB0STK3EtGiWtmi6qLCFLEpoqcc9AGtsPWKxELp1126U7YbBUNglQUKWQqUKd2SIkDJ3RnhiD2Th+uLyPJYCaH6KYDgOZGLoqEYQho90xVEVmIG0wW7Ii87GYRs32uJ2v8/J0icuzZTQZMnGDR49lePZWHtf3cYOoDGN5PkldwXJ9ri9Wm+WVnRiGrWV1uQQePd5DTFPWeYZoisyrc2U0RaYvpa9zPN2obJJe7m/xgoCHx7pW9XzcLXeakDkoo6FiJFggEOwnR0Z89KcNzgxnWKzY9KV0wjDk4s08FdNBksJ1wqOB6URtpAGRkNiqANFkSOoKtucThETZFilqzIzHFDRZYqpoYnkB+apF3Qmomj6SJFH3QhzPxfVlTCdAV8G1bIJ8wOnBFA+ODPLXry4wV3ZY9kylZrk8cKKXwWyMuuNTs8tYbkA2rhFXNVQ52kEzW7JIGyp1x0eRJEJCrsyW6UpovPlED4YafV9XQiNpqEwXzOZIbTauoivKKsOwrd7Zr7zTXqpYLFUtuhI6uarLib4kJ/tTq77//qE0QLPnoyehtQyWrcomrQRAu+7q71RWOSijoaI8JBAI9pMjIz5WNgJenq0gSTBTqPPCZAmzVcPHMu6K/7+V0ktXTGE4a1AyfSzXIWkoxFSFkukRhpGJWc3yiGsqxZpDwXSpOR6EEh4wW7KIqRKqIuP7Ufur7UZOpkEQ9ac8eqyb2YqN4xVwAgldCrlvMMOP3D9ATFPI12w0WcLxAnRN4eHxLCf6U9xcqqKrMo4fMF+xGO+Oc7w3ygrcO5he1dTZEGczJZPjvUnM5T043UkNoGkY1ioj0SrQr7zTni7UuLpQJQQSuspDo5l13y/L8qoej+uL1S0Hy60KgJ2UHg5KWeVOHJbXIRAIDiZH5hOnse49JOS1uTI9SQNDUyhaLqaz0fDs1tFkGMnqPH6il5Sh8txkiWJdwg2i8VnL/f+3d+cxkl3l4fe/d69ba3dX79PTPYs9M8bjcWy8xGbTC7xE/lkkefOKkMiRDM5f0ZCYoEQsUWRQAoZIiYgAESCR+SNYBCUYEiSHGAJ2/BLD2GbAxvuMPXvvXdutuvt5/6jununZe6Z6amb6+Ugtu2uqu57ylO957jnPeU6KZWqkabvTqR+HzHtgmhBHkGoKywBD0zB0nSSFdj2ITsWLsCwD29RJleK5w1VqXkjGtjDihJ1jvfw/N28giBWtKGa2ETFUdHnTaImjlRYDizMESik2lXMcmW9SzJiM92UZLGYY7XHJWMbyDhiAF4/VePrAApNVn6maz9aBPDdt7GGirK0YrM93+v7EO+1Xpqq0wpShooMft7fDnstaDJYXsvRwpSyrnMvV8j6EEFemdZN8+FHCzw/Os2/GI05grDfDaE8G2zBWXcdxIkcH19YZ6ckSLZ6H4scpcw2fRGnUWgFx0i44DSK1PHuiL/YLSSMouCb1IMYyQUejlLGwzfYJcgXHJE0VBdfCjxIWmhGvTDdIgbde28+cF/DO7YOMlDI8/vIclgHzXkR/wT5loB4sZti5oUQzSIjSlFaYsNAMOTDXZN6LlgdggGcPLFDxQnqyFpauLScqmqatmFE43+n7E5MH09ApZS3K+QyVVnheSyJrMVheyNLDlbKsci5Xy/sQQlyZ1k3y0fBjpmoBC15AkipMXbF9KM94b4b5RkikFM1o9TMgarEjWd0PMXWD2XrEwYUqlWa8XB9i0O4ZcuKyTUr7P75pgB/GZG2d3oxFmEDGNujNO5SzFj1ZC03XiGPwzZgdw0V0YN+sx7wXMt6XY9twkal6yN7DC4RxewblmuERrh3Kk3fMFUsjOcdk23CeBS8mTNpdSE/sVtpYnIWwdB3XNpis+kz0twt0T0wSlpYs5hoBjSDiSKXd2n2pMPXk5YwTk4ex3gwvHK3RDBO2LP7uc1mLwVKWHoQQojvWzdW26kdUWgHzzZAgVvhRwlStSU/WQdMh8i9s6SVJYKDXxtB0bEun0gxoRfGKwtQUCNVSq/U2Rbvt2FIjs1TBnBeQcyx2bSzR49qM9rgM5i0ylslco4UXKtJUUQtjhksuAwWHX99SZsdwgSdemaE3a7Oh1+WVqTrHFpoMFzNkLX15e6xtGJRcg1zGwjaN5ULO54/WTxmAdV2j0ozRNI2MtbK5FxxfsoiT9uF25Zy9vKPkTMsZS8mDUoqBQqbrU/6y9CCEEN2xbpKPUsbC0AyCKEWhESt4baZFFMcEcXLG3S7nkgBHqz6OaWAa4PkJYXI88dAAx2wnKUod73NqLvVwT6GZghani4fPRRyd96BPo+BY7J/1mKy2yNkmEFHKmEz0Z9nYm6XSinBMnTRNOVxpcWi+wZFKC6VSXp3ROFIN2smEpogTGCw41FoRrhPRn2+3SC/n7NMOwJv6szSjeLnY1AtXltsuLVls6M1ytNKifEInzytlR8iFxiE9MoQQ4uKsm+QjnzHJ2u2lBJ32ttljNZ+5RkjrQjOPRV7Urimxjfb36oQikqVll56cSZykNCNFELcPZtNon4i79DwWvz9cCYjRsXSdvYcX8PwE2zKwDI3hokvesXj2YIW6HzFdC3h5ssZP9s1QC1JqLZ+JvhyDeZvJetDufKrD5v4807UA19IouFlGe1yOLDQ5ON9cceLs0iA6Uc5RbcX4UYqpayv6fQwUnLMuWXR6OeNyG+ylR4YQQlycdZN8ZCyDGzf20IoSDs77LLQigkZ07h88TwnQOmFyYKkniAFYls5EOUcQp7wx56FrijO9tKmDHydMVnxUCkGUkrUNTEOnN9c+U8XQNWqtiDiF12YavDZdY9aL6c1a1P2IVhRztBqw0AyxDI04Vsw2ArYNF7hhQw9+lCzPSHhhvKLYdGkQPXFJwo+SFf0+do2Vzrpkcbo/u5gEotOD/cUmM9IjQwghLs66ST7iJOXFyQa/PFIniE+t79AXl0EuftNtW0p7ZsM2IIkV816IaxvkbBMvTLDihFgdn/HQAV2HrGMykHNoRCleGJN1DHpdi1akcC2drG0yVHBwbZMgTsnYBq0wptZqMlcPyTo6m8pZdowUOTjXxA9TygUHy9C4ZaKPN0/0MtsIaQQxc42AOS887SB64pLE/pnGKUWpZ2rwdfLPwvG27M8eWMA2DHpzFjdu7DnvBOJcg/1qk4mLTWakUFUIIS7OurhqekHM//sPTy3v5FiiAWO9LjeM5tnz+jwLzaRjyQe06zpSBa1IMesF2L5OX9YiThIaamWnVMcA19IY783i2jqanpAmioxlUs7ZJEoxWMxQdE029LpcO5jjuaNVkhjGemyKTh91P0KhsWO4yG9cP8KcF644h2WinEPX9eXEIO+YVFvxOQfRix1sZ+oBPz9Y4fBCa3mGZDWzBed6/dUmExc7c3G2WZ/LbYlICCEuR+si+cg5Jndu7eO/XpgG2ksHt27q5f/sHObVqRo/emmG2WZyUf0+TlawwDI1aq126/a6n+JYECUhcaJQtGc7DK2dhGzsy6I0jb68hWOY1P0mqQaVZkgriunLZdjUb6GUjmub3L6ljB8lHKv61PyEwUKGGzf2EKeKmyd6l2cm+vPOGXdzLA2idT8iiFPqfrT8+IkD5smD7fl2NV3SCGJMXaN/cSeMY+qrSmDOtStltcnExSZTZytUlXoQIYQ4t3WRfAC8/Zoy//3SDBmr3cTrprECw8UMB2bqTNeCjiYesLiVNgbLgDiBWIGVKkKlyJgGTlaj1kowF/8GojjFNA2GSxlGenK8Ntug3oqwDYNEpeh6yC8OLeCYGnmnHwA/Usx6IV4QE8aK6zeU+LXxXvrz9oq77839udP26Fj687xj8vps7YwD5smD7XTNX9UAm3dMynkbANcyuGm8Z1XbWs+1K2W1ycRabrGVehAhhDi3dZN8HK60KDjt4+yrfsxkLWChGfHyVI2w05kH7R0w2uKBdEviVGGaOq04xjZ1MrZOOWcTJoqsbTDa6+JaJi8cqWItDqIpUPdj4lThWMby7pggTnl9ts7RBZ/BogO6hmMZDBYzy8lBnLZbl594qqumaUzXfP7n1Vm8xaZj430uSaoY6cnw4tEaLx6roRa37HhhcsrsxmoH2PZg37NmSxGrTSbWcquv1IMIIcS5rZsro9eKsQyDrGMwU/MXT3J1mPM6t+PlRCcPrRbtnSw5WydRBrbR3m6botB1jQ19LtsGCwRxiqFrjPZkqfkhKOhzbcbKWTaUXEqZdsGqY+psLhfwgoSKF1LMtJdD4Hhy4FoGvzxc4VilxStTDW4a7+G6kSIH55vsn/XocW2m6h7FjImh67x4tMbhhRYaGjP1AE2DvGOtmN1Qqt2gbabuU21G9Oascw6wnRzsz1RTcTn0DQFpXLaeSH2PEBdu3SQft27p46dvzDPfjLAMA13TOFbxKWctDM7vxNrzpXF8t8uSiPYyjB4kGIZOELd7g6DFOKbBvpkm/TmHzQMFhnqyvDHbYKI/y7WDBWrNiIMLTWa9kL68Tc420DSNsT6XybqPa0dM9Gcp59rJx9Ld9xtzHl6QYBk6h+abpGl72uRopYUXRJQy7RNqe7IWm/rzvHC0Sr0VU8gY7J/xyDkG/XmHN+Y8Su7xg+SOVlpYhk6Upoz2tBOSE3uArOUF+MTOqo0gZqK8clan2y6nREisLanvEeLCrZvk466dw+ybbvKTfdPkbAvHhHrQ7m8xXLSotGKakepI7YeinXic/LsU4EUKc7EhWcbUSNHIOSaOobGhN8um/iwLXvsMl5snetk+lOcn++aYavjknOOFmgMFh80DOVpxsqIL6VS1xYE5jyRNcE0dS4e5esCm/hxBpJZ3vxi6TpQmbB3IMVHOMVjMMNsIeOZAhdnDAWGckmDx09fnAcjZTSbKucVZFZZPzG2GCc8dOXO9CHT2DnF5Vsc2+eWRKl4YU23Fq77wy12ruFhS3yPEhVs3ycecF6Hr4FgG042AomOyc6wHTUsZ7snx+nSVfbMtvCAh6MB+2zMlMTpg6KCZEIYK3dRQKmWsL8vODUVc2yRV7b6oDT/ipck6b8x66Og4ps5U3efgfJPBYuakLqQ6QZzy84Oz7J/18IIIQ9PIuxZ+HIDScCwNU9e4bqSIhsZQyeG6keLy0oBj6oz1upSyFhUvJGPpVFsxm/rztMJ4eaA+saYBOOMFeGmAPzDncXC+Sc4xMXX9ou4Ql2d1ZhsAbCrn8KN01Rd+uWsVF0vqe4S4cGv+f8tnP/tZPv7xj3P//ffz+c9/fq1f7ox+cbjCnjfmWfBiGn5MIWMyUHRIkoQgDlDoNDuUeJyNqYNpaJRdi9hpz5CM9rhMlHNM10MSFbD3YIUgTii6FkMFhyBS+HHCy1N1RksOB4rN5aWGE+sL6n6EF8T0uDZJklL1I27d3Eet5TJcyjBQcDhaaXGs6tOXt7lupLhiwC1kLMp5hyRV9BcyjPZkOFrx8aME09CXZwhOfE2lFNVW7bQX4KUB/shCk6l6wO2b+2iFCQfmvAuecVh6/ZJrkp9v0oqS5dN0V0PuWsXFkvoeIS7cmiYfe/bs4Stf+Qq7du1ay5c5L1M1nzkvRCkwjPZ228G8w4uTNX51tMbrsx4XeLDtWS2dB6sBjgVZu721NWebZC2DYtZmy0CONFUcnGtScE0OznvkHRvDSJmsBiQqpeEnpGnCjtESecc8Y5fRnGMyVffw44SsbVJvJZTzx2c4zqfvx4n9PE5+/um6l+7StNP+zqUBflN/nql6wBtz3mKH19O3dD8fS68/UHCWl4Eu5MIvd63iYkl9jxAXbs2uuI1Gg3vuuYevfe1r/PVf//Vavcx5GyxkcAyNyZpPkkIYJcx5AUcWWiSpIko6m3notM91MYz2PwdLLpAwUsqybTDP04cWqIcxfpLiWjoF1yJWcKjiESaKKE6YrSdsLbv0ZDOMlDQOzOukyfFZiJMNFBzedm0/E+UsSilyjknGMihkrPManE93MT3XxfVsF+ClAb4VxmzpzzFRzgKcsaX7alzshV/uWoUQonvWLPnYvXs3d999N+9+97vPmnwEQUAQBMvf12q1NYlnrNelN+8w70W4WQPT1Nk3XWey4nNsoUkUd67Zhw5kLY1EKWxDR9c1FpoBPVmbZpjw/NEatVbCSNHBdSw292cZ7c3iWgZP7Z+j6BjkHINUQT5jU29FmIZOwbEY7ckuH+x2Mk3TGCq5DJXc08Z1puZga1V8eboBfqYenFdL97V2scmLFKwKIcSFW5Mr/ze/+U2effZZ9uzZc87nPvjgg3zqU59aizBWcG2T7YMFbMNYLJRUGIaOY+v4seI0Z81dsJR2Q7E4gSRJMRbXXrwgwmtFWKZGM1JkbIOsY1FyHSqtmJcmG6Bp9OczXDOU50ilRZy2iynH+7I4tsmWgdwFF0aeqc5hrYovTzfAXy0zDlKwKoQQF04/91NW59ChQ9x///184xvfIJM598X44x//ONVqdfnr0KFDnQ4JaBdTXjNUYLiUoSdrsWOkwEgpQ8GxKbkmeodvWv2kvePFNAGt3V696ifMtxIqrRilKVpBhB+n+FFMmiiCOGFjXxbd0Jistton2JZc6kHC4YUWtWZEmLRnaJRSTNd89s80mK75yx1Jz+ZMdQ4nJiVJqk45gO9sVhvHUkKyZSDPYDFzxc4WXMx/MyGEWO86PvPxzDPPMD09zc0337z8WJIkPPHEE3zxi18kCAIMw1j+M8dxcJy1v/sdKDi89ZoyxYxJK2r3twDQNZ05z2e6EUCHx48EaC42UNWXvjSIErBNHV03sHQNxzLZNlxkphFyeL6FYxq4tkGva3F4wSOMEgYH8gwXbQ7NeczUg3YtRRSTphqGrnHDhiIAB+ebAIz3ZU8Z3M8067Ca4suTlxuUUufs83E1koJVIYS4cB2/Yr7rXe/iueeeW/HYBz/4QXbs2MFHP/rRFYnHpaRpGrquo+s6GUtjshZyw4Yiv3PzGBlTY7LW4nAlvPDfz9l7eyw9J+fouJaJYWpsH8ox1pul4YdMVloMFixSZXHDWC/NMKLeinlxsk6SKp4/WmWyZmPoGkXXxvMjJgby/PrmMkcrLQ7ONzk432TfjAfAlv4cb982cNYD4pasZink5OWGkmuuyy2rV8vykRBCdEPHk49CocDOnTtXPJbL5SiXy6c8fikppTgw53FkwaOUtTk838QLIm7f3Edv1iaKLq7o42yLDQrImOCaOn35xYZetoFlGiilkc86xEqxa2MvfpTgRwmWYeBYKf2FDJsH8vx0/xwLno9umGwfLrIviPH8aPHOGxa8kDdmPUyt3THVC+LzTgRWU3x5ct0IsC5nAGSbpRBCXLj1MVLAYqfNJvtnmxycmyVKUl6davCrY1X2TTXwwuSssxfnQ6O9rVbXIUqP/y4dcC2dG8d6iFNFzY/J2gamplHO29y2qZeXjtWJk5TRHhfH1ClkLGbqPq9NexxeaJLPWOwcLfL80RovTdbozzncsqmP0R4XP0r41ZEqNT9mpu4zWHDZuaG4JonAycsN431ZtDP0+egU2VkihBCdcblcTy9J8vHjH//4UrzMWS39h37TSJFD8x6mDmGS8uwbFfw4Rjc0rFQRJReegCjaO11srd1CPU4XvzfaBa9DxQxhotC0iKl6C8swcC2dH7w4xUvHqowUXHaO9fCO7e3lkv68jaZpvDpVZ64RMlSwcS2Dct7m2qECO4YL6Lq+fKjbTeM9/PLQApv6s7z1mvI5E4EL+RCebrlB07Q1nQGQnSVCCNEZl8v1dN3MfOQdE3Nxz2tv1mbOC0mUwjQ0htwM8/UQTcVYeooXXfjrpItf/QWHWivCT1J0XccydOIUFpohx6rtotKCa6EUvHikyv65JofnWxyr+UyUswyVXDRNoz/v4Jjtc1scU+fWxYZhJyYJecfECxP2z3pkbIucY6Lr+jkTiQv5EHZjuUFaoQshRGdcLtfTdZN8LN2x1/2IkZLDU/vnmfcCMoaOacDWwRzzXsB0PSCKEyJ14TMgcQILzQBd03AtgzSFUsZGociYBsWMRW/OppyzUECoFGkKC0FCmLQPYbt96/knB+1W41m8MF4+4fZ8PlCXy4fwXGRniRBCdMblcj1dN1fxE88E8aOEQsYkY2nMNUL8KKG/4HB4oUUQp2Qcjci/8OoPXQM0yGdMerM2NT+mN2fS8GN6shYberNM1Vs4lsFwKYOtayRpSilrUHAsLKM9Y9EIYuIkxbVN3phtUHKPL3OcvGQy3pddccLt0gfqbEsrl8uH8FxkZ4kQQnTG5XI9vTxHmzU0Uw/Ye6hKtRWTsXSaQcpk3efArEctSNrdTi+i7gMgUpCGUCwZjPZkcL0YTdOotCLQoBmlWLrBUN7FNnTesX0QDQ1N0xjtyXDtUAFg+QC5Xx6ptr+fb59mO1jMnDIrcsOG4mk/UGebPblcPoTnIjtLhBCiMy6X6+m6Sz4aQYypa/QXHF6bqhOmKT2uzWtJgyhOiZN2zcbFULT7lR2rh4yUXHaMFDB1jfmjNWyjfdBaT8FlQ2+Gaivh1zf38eaJPmbqAQMFhx3DBZRSKKWwDI2srbNztIQfp8tLIycvmXhh0u4aepr3e6allcvlQyiEEGJ9WXfJR94xKedtAMbLWaZrPnONgJ6cRaV1cU3GNFYmLi0/Zb4ZkXMjGn7MQjOi5FqEiWK2GbL3UIUoUZSyBhv7coz1uhQyFpqmMVMPeO5IDT9KCSLFdC2kL28vL42c75LJ2Z53uWy5EkIIsb6su+SjvdTQQyOIaYUxP90/x1TVJ4wSLFMjitWqZz4MIGdBkEBwwg/HQJwkOIZOoZih4kfkHIM8GkXHJO8YvDLV4Kn9c/x0/wLbhwuU88eXQpJUcd1ou236UMnhupHi8tLI+S6ZnO15F7LbRRIWIYQQF2vdJR8nLjXsn2mQsXTiVOFHCVp6Yce7WCagaSSpQmfl7Ee1GbPQDOnPO5RzNqauM1DMEEUpr043OFr1cSyDehCwbbiwfEjZ0ozFsYpPOd9OPM6nVfrZ3u/JLmS3y+WyR1wIIcSVa90lHyfKOyYvT9Z5eapOI1Q0QrXqLqcmoFJItPbP6os/rwFZS0M3dRxToy9ns22ogB8njPW4BLFitmFR92Mypo4XaszWffrzzvKMwloXg17IbpcrZXvupSSzQUIIsTrrOvkYKDiUXBvb1Mk5Og3/7AfEnU7e0QmSFIVGuviTOav9e3pyFhPlHP2FTLt9uxdhGTr9hQxTtQCUxlAxw2DRZutgnutHi2zqzx/vGrrGxaAXkuBcKdtzLyWZDRJCiNVZ1yOHpmncurmPpw8scGDew7E0wujMNR/LZ7cYx/9dI8UyNKJEUc5boBTlvE0YK1zbBAVJCkNFh+3DBWqtGNvQ0TQouCbb3QLXj6xMOi7l+19tgnOlbM+9lGQ2SAghVmddJx8Ad24tU2lF/OBXxzi40KLeDFloRVSaCclJz9UAxwLL0HEsHVM3cC2NMIZaK8I0NPpzGXYM52kEitGSzWszTSqeT5oqhksZ+vMZdF0j71hsGypytNKiv5C5Yu6UZXvuqWQ2SAghVmfdXyU1TWNzOcu2oQK6rlHNmGgLLeLYpxquXICxNXBNA9PQ2NKfpy9n0woTXpys4zomlqGjUMRKQ9cVlVZCLYjJ2jbTjZAwStg1VkIpRbVVk8HqKiGzQUIIsTrretRTSvGTfXP829OHODDfpBlEpArSVJHNWDTCcMXsh6/AjBUqTnEtk768w4vH6qQKbFMnbxuUcxl2bSgRpYoDM3VU2q4HUWlK0bUYLGZQSrHrNMfQS+HilUlmg4QQYnXWdfIxUw94+vU59s96VFsRUZrQClNMHaJYnbLsAhDEKamC549W0XUouQb0Zak1Q0CjN2fi2iYFQ2O6ZhEreGPWo7/gUM63k4wzDVZXSuGiJElCCCEuxrpOPtqDp41tanhhQtbSiZOIhq9I1el3vkQKXAOSJGG2EWEZoNAo5RzKeYt37hjiupEi+2c8LB2uHymQKujL24yUzp5IXCmFi51MkiSREUKI9WddJx95x2S87LJrQy9R0j5LxQsigjghSY8nHgYsz4LoQJxCrDRKGZNKK2S0J8vbtw+glGKomGGhGXF4ocV0I2S6HjJccrhmIE/Rtc8ZT7cKF1eTBHQySbpSZnuEEEJ0zrpOPgYKDr823sumssvmgSyPvzzJkXltRbOwdPGfFmCYkLNNwjghjBMOV1qYps5g0aEna3N0ocnjr8zghwk1P6aUsYhiRdExlw+L2z/TOOPgvlS4WPcjgjil7kfLj6/1bMBqkoBOJklXymyPEEKIzlnXycdS7cVsI2D/TJOqn6K09lZa0wRdQV/OJEwUGduk2gwxdY1C3mHOC4mShIGCzXUjBfqyFk/t93h1ysO1DOa9kJ6sxa6xXkZKGVpRynNHamcd3JfiAXj9Es8GrCYJ6OTuDtmmKoQQ649c6Wnf9TfDmM3lPK0wptqMcSydkmuxfTBPlEIpazBZCzk832SmEWAbBo5tUS5k2NyfB2CqGtIMEmp+hEoUGhYLzZANPe3E4XwH927MBqwmCejk7g7ZpiqEEOuPJB+0B0DXMnl9roGh6QwUHPKOQStK0HTYNphjrDcHKuVHL88QRgn9eYMwSig4BuN9WX5xuELNb9eLVFohG3tc3rF9EKUUm/pzjPdlz7u3RzdmA7qVBMg2VSGEWH8k+QC2D+W5eaJEPQhxDI3ZesCcF+EFCXknoC/n8lyzypGFFq/PeSz4EUkzwrV0DP14LYZj6hQyFkGU4NomkzWfrQN5Jsq59uB+mt4ep9ONRECSACGEEJeKJB/AnBdR8xPKuQyaBvtnPUCj4JrU/ZijCx4J0AwTVJpiGzqxpujLZYgSxaGFFr1Zm/G+9rJNxjK4Y2sfUaIwdQ2l2vtmzndwl0RACCHE1UySD9o1Fqau4do6M5MBug5BlJBFZ2PZZcdwiYOVJkGUUPUTKo0Aw9CJ3IR0cT/uRDnHzg1FJqstco6JpmkEcYq/WGi664RiUmhvbZ2u+RycbwIw3pdlsJg5466Wpa2wSzthHFNfXo7xwkR6ZAghhLhiSPJBu8ainLeZafj05CxGSg6zXkSPa/Gbv7aBawZy/H/75vlFOo+hgWWZmLpGolIGC85y4vD2bQPLycF0zWeqFnDdaJFjFf+UotGZesD/vDrLc0eqREnKtUN5/s/OEYZK7mljXNoKO98IObTQZKzXxdA1NA3yjiU9MoQQQlwxJPlgqcaih5JrYWgalWbE1qESBcdkQ2+W4Z4sb99mMFdv9/XI2gapSsnbJjdu7FmesRgsHj+dtj/vEKdVjlX80xaNNoKYyWoLL4xRKbwy2WDnaPOMycfSDpiiaxLNppSyFlNVHzSWT8eVHhlCCCGuBJJ8cLzGYqDgkHNMfn6wgqlrlPM2OdtYXh45UvXxgpg4UcRJylCPy41jPadd6ji5aLQ/bzNd85dnRhp+RCtKqbciiq6FYxpnjXFpB8x8I8IydKrNaHF5B+mRIYQQ4ooio9UJNE3jupEi/XlnOWlI05RHfzXJq1MNDkzXMHSNgmPQ8BNMUqZrLdI05XDFB1bWbpxYNDpd8/nl4SpzjYDDCy3Gel368zaQJ2uZDBYzjPdlzxjbid1Pd44Vz1jzIYQQQlzuJPk4yclJw57X53hlskEYp2DoxFHCTBCRJBr7Zlv881OH2NCXoRWmNMOE4aLD27cN0J93ViQFjSAmTlMUipm6z4Zel+FShp0bSpTzzjmTh5OXdYQQQogrlSQfZ7C0u+RopUUcp/hxTNWL0A0DC9AUGJrOdKNFnCS4Trub6UIzYL4ZMlpqJxfNMGFjr0uYKPZPN3hxqkbFi0nUPLdvLvPmiZwkFEIIIdYVST4WnXyqq1KK547U8MMUXQcvaLdcz6U6mmZS89uJhm1pWHrEwfkms15IOW9TbYQcWWhxx9Z+jlVbHKu08KOUaiuk4oWM9WRRQNGVpRIhhBDrjyQfi04+1bXkmiSp4rrRIq/PNvCCmC3lPC9N1tA1yGVMSBL6cjZRFBGnUG9F+GFMT9ZC90IWvIANvTlumejljbkmQ0WHmUZI0bUxDI3erC19OYQQQqw7knwsOvEwtyOVJgvNkJl6QLUZUcxa9McuQ6UMsUrJWSYzXoAXJJi6xoFaxHTNR9M0/ChhupZSzjs0g5h5L+T12QZRApZrMlpyKWZMhkpnLzAVQgghrlaSfCzKOya6Bi8erTHn+dimTqJgthGwdSBHf86hFSVs7s/TDGJqfsTRVotqKwTaBaEZ26DSTLA0cGyz3ZMjToiShLG+HNePlCi41vIZMCcvuZy89CMdS4UQQlyNJPlYNFBw2NDrMl0PSJTiwFyT3pyNHyYcXmiydbDA5myONE355eHachfTFLB0jZJrYRkaqbLYsNgorNaK6M05FF2HvG0zUMywZSB/xhhOXvqRjqVCCCGuRpJ8nMBb3A67sS/HkQWfqarPcCnDZDUkUXV6XJtS1qLSCllohpiGzq2by8w3WqQJBKmiN0oZLTnYloGhafTmHFphQpgk52wCduLSj3QsFUIIcbWS5GPRTD3gwFyTqVrAsYUmhg6tIObQXJNGGNIKYyYtn3LWpifn8Otb+vnf1+cI45QtA0U29mZpRjG9WZtKM2Sk5KJpMO9FxKnipvGeFcssp1tiWepiKh1LhRBCXM1kdFu0lATcvrnM/+6bZs4LSFLFZLVJmqZMGhETfVn68g5Zy6AvZ/E2s5++rM21QwX6shbPH62TpIoNvRY3bCiiadoZ6zdm6gG/OFRhwYsIk4SbJ3rZMVxY0ZJdtuEKIYS4Gq375GNpBmKuEeCFMWgQJ4pWmNKXtQmj9hbZjG3iRwmkivE+F8fU6c8fP9EWQNf1U5KNMy2bNIKYBS+iHkTM1AM0TaM/76zorio6T4p6hRCi+9Z98jFd8/mfV2dp+BE1P2K8L8tw0eVIpUUrTrEsA8cySJXCCyKaQcKxShNdNyhkLKqtGrtOaH1+volD3jEJk4SZekB/wcHUNanxuASkqFcIIbpv3ScfB+eb7J/10FTKs4cqvHisSn/Oote10ICxTX0M5E2e3DdPGMNU3SdIEnpcm1s2l2mFMY0gZmCVd9QDBYebJ3rRNG35BF2p8Vh7UtQrhBDd1/HR7sEHH+Tb3/42L730Eq7rcuedd/K5z32O7du3d/qlOupYLeBwxafq6bxwNMbSdbYMFrh9MI+pa+i6Tilrsn/Woz9noQ/o/PT1Obb058g75nndUZ885b9juLDiBF2p8Vh7UtQrhBDd1/Er7+OPP87u3bu59dZbieOYT3ziE7znPe/hhRdeIJfLdfrlLtp4X5atAzkmKx6OoYGmMeeFoGmkaDiWzo7hAq5toAFZy2C8nOX/2j7AgfkmE+UsAwWH12e9c95RnylBkTvvS2eg4EhRrxBCdFnHk4///M//XPH917/+dQYHB3nmmWd4+9vf3umXu2iDxQxvu3aAvGOQKo3nDlfQ0ShlbRRQ92O29udwLZMFL2T7UIHRngxBrNjQk2WinEPTtPO6o5Yp/+7Tlupzuh2IEEKsY2s+51ytVgHo6+s77Z8HQUAQBMvf12q1tQ5phaXB6P9+0zBjvVm+98sjPPnKHEGcYOo624by/Np4Lzcv7mTJ2QYAXpisuHM+nztqmfIXQgghQFNKqbX65Wma8pu/+ZtUKhWefPLJ0z7nk5/8JJ/61KdOebxarVIsFtcqtDNKkoSf7JvjxWM1erM2b72mzHBPtiPbMWWbpxBCiKtVrVajVCqd1/i9psnHH/3RH/Hoo4/y5JNPMjY2dtrnnG7mY+PGjV1LPoQQQgixeqtJPtZs3v9DH/oQ3/ve93jiiSfOmHgAOI6D40jRnxBCCLFedDz5UErxx3/8xzzyyCP8+Mc/ZvPmzZ1+CSGEEEJcwTqefOzevZuHH36Y7373uxQKBSYnJwEolUq4rtvplxNCCCHEFabjNR9nKqB86KGH+MAHPnDOn1/NmpEQQgghLg9drflYw/pVIYQQQlwF9G4HIIQQQoj1RZIPIYQQQlxSknwIIYQQ4pKS5EMIIYQQl5QkH0IIIYS4pCT5EEIIIcQlJcmHEEIIIS6py+5M96U+IbVarcuRCCGEEOJ8LY3b59Pv67JLPur1OgAbN27sciRCCCGEWK16vU6pVDrrczreXv1ipWnK0aNHKRQKZ2zVfqFqtRobN27k0KFDV2Xrdnl/VzZ5f1e+q/09yvu7sq31+1NKUa/XGR0dRdfPXtVx2c186LrO2NjYmr5GsVi8Kj9YS+T9Xdnk/V35rvb3KO/vyraW7+9cMx5LpOBUCCGEEJeUJB9CCCGEuKTWVfLhOA4PPPAAjuN0O5Q1Ie/vyibv78p3tb9HeX9Xtsvp/V12BadCCCGEuLqtq5kPIYQQQnSfJB9CCCGEuKQk+RBCCCHEJSXJhxBCCCEuqXWTfHzpS19i06ZNZDIZbr/9dn72s591O6SOeeKJJ3jve9/L6Ogomqbxne98p9shddSDDz7IrbfeSqFQYHBwkN/+7d/m5Zdf7nZYHfPlL3+ZXbt2LTf+ueOOO3j00Ue7Hdaa+exnP4umaXz4wx/udigd8clPfhJN01Z87dixo9thddSRI0f4gz/4A8rlMq7rcsMNN/D00093O6yO2bRp0yl/h5qmsXv37m6HdtGSJOEv//Iv2bx5M67rsnXrVv7qr/7qvM5fWUvrIvn4l3/5Fz7ykY/wwAMP8Oyzz3LjjTfyG7/xG0xPT3c7tI7wPI8bb7yRL33pS90OZU08/vjj7N69m6eeeorHHnuMKIp4z3veg+d53Q6tI8bGxvjsZz/LM888w9NPP8073/lOfuu3fotf/epX3Q6t4/bs2cNXvvIVdu3a1e1QOur666/n2LFjy19PPvlkt0PqmIWFBd7ylrdgWRaPPvooL7zwAn/7t39Lb29vt0PrmD179qz4+3vssccAeN/73tflyC7e5z73Ob785S/zxS9+kRdffJHPfe5z/M3f/A1f+MIXuhuYWgduu+02tXv37uXvkyRRo6Oj6sEHH+xiVGsDUI888ki3w1hT09PTClCPP/54t0NZM729veof//Efux1GR9XrdXXttdeqxx57TL3jHe9Q999/f7dD6ogHHnhA3Xjjjd0OY8189KMfVW9961u7HcYldf/996utW7eqNE27HcpFu/vuu9V999234rHf+Z3fUffcc0+XImq76mc+wjDkmWee4d3vfvfyY7qu8+53v5v//d//7WJk4kJVq1UA+vr6uhxJ5yVJwje/+U08z+OOO+7odjgdtXv3bu6+++4V/y9eLV599VVGR0fZsmUL99xzDwcPHux2SB3z7//+79xyyy28733vY3BwkJtuuomvfe1r3Q5rzYRhyD//8z9z3333dfxw02648847+eEPf8grr7wCwC9+8QuefPJJ7rrrrq7GddkdLNdps7OzJEnC0NDQiseHhoZ46aWXuhSVuFBpmvLhD3+Yt7zlLezcubPb4XTMc889xx133IHv++TzeR555BHe9KY3dTusjvnmN7/Js88+y549e7odSsfdfvvtfP3rX2f79u0cO3aMT33qU7ztbW/j+eefp1AodDu8i7Z//36+/OUv85GPfIRPfOIT7Nmzhz/5kz/Btm3uvffebofXcd/5zneoVCp84AMf6HYoHfGxj32MWq3Gjh07MAyDJEn49Kc/zT333NPVuK765ENcXXbv3s3zzz9/Va2pA2zfvp29e/dSrVb513/9V+69914ef/zxqyIBOXToEPfffz+PPfYYmUym2+F03Il3kLt27eL2229nYmKCb33rW/zhH/5hFyPrjDRNueWWW/jMZz4DwE033cTzzz/PP/zDP1yVycc//dM/cddddzE6OtrtUDriW9/6Ft/4xjd4+OGHuf7669m7dy8f/vCHGR0d7erf31WffPT392MYBlNTUysen5qaYnh4uEtRiQvxoQ99iO9973s88cQTjI2NdTucjrJtm2uuuQaAN7/5zezZs4e///u/5ytf+UqXI7t4zzzzDNPT09x8883LjyVJwhNPPMEXv/hFgiDAMIwuRthZPT09bNu2jddee63boXTEyMjIKUnwddddx7/92791KaK1c+DAAX7wgx/w7W9/u9uhdMyf//mf87GPfYzf+73fA+CGG27gwIEDPPjgg11NPq76mg/btnnzm9/MD3/4w+XH0jTlhz/84VW3pn61UkrxoQ99iEceeYT//u//ZvPmzd0Oac2laUoQBN0OoyPe9a538dxzz7F3797lr1tuuYV77rmHvXv3XlWJB0Cj0WDfvn2MjIx0O5SOeMtb3nLK1vZXXnmFiYmJLkW0dh566CEGBwe5++67ux1KxzSbTXR95VBvGAZpmnYporarfuYD4CMf+Qj33nsvt9xyC7fddhuf//zn8TyPD37wg90OrSMajcaKu6zXX3+dvXv30tfXx/j4eBcj64zdu3fz8MMP893vfpdCocDk5CQApVIJ13W7HN3F+/jHP85dd93F+Pg49Xqdhx9+mB//+Md8//vf73ZoHVEoFE6pz8nlcpTL5auibufP/uzPeO9738vExARHjx7lgQcewDAMfv/3f7/boXXEn/7pn3LnnXfymc98ht/93d/lZz/7GV/96lf56le/2u3QOipNUx566CHuvfdeTPPqGRrf+9738ulPf5rx8XGuv/56fv7zn/N3f/d33Hfffd0NrKt7bS6hL3zhC2p8fFzZtq1uu+029dRTT3U7pI750Y9+pIBTvu69995uh9YRp3tvgHrooYe6HVpH3HfffWpiYkLZtq0GBgbUu971LvVf//Vf3Q5rTV1NW23f//73q5GREWXbttqwYYN6//vfr1577bVuh9VR//Ef/6F27typHMdRO3bsUF/96le7HVLHff/731eAevnll7sdSkfVajV1//33q/HxcZXJZNSWLVvUX/zFX6ggCLoal6ZUl9ucCSGEEGJdueprPoQQQghxeZHkQwghhBCXlCQfQgghhLikJPkQQgghxCUlyYcQQgghLilJPoQQQghxSUnyIYQQQohLSpIPIYQQQlxSknwIIYQQ4pKS5EMIIYQQl5QkH0IIIYS4pCT5EEIIIcQl9f8DvBq4eqmKlScAAAAASUVORK5CYII=", + "text/plain": [ + "
    " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import seaborn as sns\n", + "\n", + "# draw the graph. This might take ~30 seconds.\n", + "sns.regplot(x=\"new_cases_percent_of_pop\", y=\"search_trends_cough\", data=weekly_data, scatter_kws={'alpha': 0.2, \"s\" :5})" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": { + "id": "5nVy61rEGaM4" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGeCAYAAAA0WWMxAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAArzVJREFUeJzs/XmMnOl1349+3r32qt437rORM+SMJMvWYkmWYtkaymtyE8O5RqDYQBLAAWxHQGwrsA07sK04fxhGcgM7zgWcBNkQ3Fz7l5ufZrxKtmRLsuSRNMMZkjMckk2y96qu/d3f97l/vFU13c3qjey9ng9Ay+yu7nq6OF3n+5zzPecoQgiBRCKRSCQSyQGhHvYBJBKJRCKRDBZSfEgkEolEIjlQpPiQSCQSiURyoEjxIZFIJBKJ5ECR4kMikUgkEsmBIsWHRCKRSCSSA0WKD4lEIpFIJAeKFB8SiUQikUgOFCk+JBKJRCKRHCj6YR9gI3EcMz8/Tz6fR1GUwz6ORCKRSCSSHSCEoNlsMj09japuk9sQu+TP//zPxfd///eLqakpAYjf//3f733O933xsz/7s+Ly5csik8mIqakp8Q/+wT8Qc3NzO/7+9+/fF4D8I//IP/KP/CP/yD/H8M/9+/e3jfW7zny0221eeOEFfuInfoK/83f+zrrP2bbNK6+8wi/+4i/ywgsvUK1W+emf/ml+8Ad/kK9//es7+v75fB6A+/fvUygUdns8iUQikUgkh0Cj0eD06dO9OL4VyuMsllMUhd///d/nh3/4hzd9zNe+9jW+4zu+g9nZWc6cObPt92w0GhSLRer1uhQfEolEIpEcE3YTv/fd81Gv11EUhVKp1PfznufheV7v741GY7+PJJFIJBKJ5BDZ124X13X5uZ/7Of7+3//7m6qgz372sxSLxd6f06dP7+eRJBKJRCKRHDL7Jj6CIOBHfuRHEELw27/925s+7jOf+Qz1er335/79+/t1JIlEIpFIJEeAfSm7dIXH7Owsf/Znf7Zl7ceyLCzL2o9jSCQSiUQiOYLsufjoCo+33nqLz3/+84yMjOz1U0gkEolEIjnG7Fp8tFotbt261fv7nTt3+OY3v8nw8DBTU1P83b/7d3nllVf4P//n/xBFEYuLiwAMDw9jmubenVwikUgkEsmxZNettl/4whf42Mc+9tDHP/WpT/HLv/zLnD9/vu/Xff7zn+ejH/3ott9fttpKJBKJRHL82NdW249+9KNspVceY2yIRCKRSCSSAUAulpNIJBKJRHKgSPEhkUgkEonkQJHiQyKRSCQSyYGy7+PVJZKtEEKw0vRoeSE5S2csb6EoymEfSyKRSCT7iBQfkkNlpenx6oM6USzQVIXnTxUZL6QO+1gSiUQi2Udk2UVyqLS8kCgWTJfSRLGg5YWHfSSJRCKR7DNSfEgOlZylo6kK8zUHTVXIWTIZJ5FIJCcd+U4vOVTG8hbPnyqu83xIJBKJ5GQjxYfkUFEUhfFCivHDPohEIpFIDgxZdpFIJBKJRHKgSPEhkUgkEonkQJHiQyKRSCQSyYEixYdEIpFIJJIDRYoPiUQikUgkB4oUHxKJRCKRSA4UKT4kEolEIpEcKHLOh2RfkAvjJBKJRLIZUnxI9gW5ME4ikUgkmyHLLpJ9QS6Mk0gkEslmSPEh2RfkwjiJRCKRbIaMCJJ9QS6Mk0gkEslmSPEh2RfkwjiJRCKRbIYsu0gkEolEIjlQpPiQSCQSiURyoEjxIZFIJBKJ5ECRno8TiBzwJZFIJJKjjBQfJxA54EsikUgkRxlZdjmByAFfEolEIjnKSPFxApEDviQSiURylJFR6QQiB3xJJBKJ5CgjxccJRA74kkgkEslRRpZdJBKJRCKRHCgy8yGR7AGyvVkikUh2jhQfEskeINubJRKJZOfIsovkSCOEYLnhcnulxXLDRQhx2Efqi2xvlkgkkp0jMx+SI81xySjI9maJRCLZOfIdUnKkWZtRmK85tLzwSHbxyPZmiUQi2TlSfEiONMcloyDbmyUSiWTnHM13comkg8woSCQSyclDig/JkUZmFCQSieTkIbtdJBKJRCKRHChSfEgkEolEIjlQBr7sIidTSiQSiURysAy8+DgucyQkEolEIjkpDHzZRU6mlEgkEonkYBl48XFc5khIJBKJRHJSGPhIK+dISCQSiURysAy8+JBzJCQSiUQiOVgGvuwikUgkEonkYJHiQyKRSCQSyYEixYdEIpFIJJIDRYoPiUQikUgkB4oUHxKJRCKRSA4UKT4kEolEIpEcKFJ8SCQSiUQiOVCk+JBIJBKJRHKgSPEhkUgkEonkQNm1+PiLv/gLfuAHfoDp6WkUReEP/uAP1n1eCMEv/dIvMTU1RTqd5uMf/zhvvfXWXp332CKEYLnhcnulxXLDRQhx2EeSSCQSieRQ2LX4aLfbvPDCC/y7f/fv+n7+X//rf82/+Tf/ht/5nd/hq1/9Ktlslk984hO4rvvYhz3OrDQ9Xn1Q562lFq8+qLPS9A77SBKJRCKRHAq73u1y9epVrl692vdzQgh+67d+i1/4hV/gh37ohwD4z//5PzMxMcEf/MEf8KM/+qOPd9pjTMsLiWLBdCnNfM2h5YVyn4xEIpFIBpI99XzcuXOHxcVFPv7xj/c+ViwWed/73seXv/zlvl/jeR6NRmPdn5OGEAI3iCi3PN6Yr6OpkLMGfqefRCKRSAaUPRUfi4uLAExMTKz7+MTERO9zG/nsZz9LsVjs/Tl9+vReHulIsNL0mKs6GKpKEMVMl9KM5a3DPtaJRnpsJBKJ5Ohy6N0un/nMZ6jX670/9+/fP+wj7TktLyQWcGm6wFg+RcrQUBTlsI91LHhUESE9NhKJRHJ02VPxMTk5CcDS0tK6jy8tLfU+txHLsigUCuv+nDRylo6mKszXHDRVkSWXXfCoImKtxyaKBS0v3OeTSiQSiWSn7Kn4OH/+PJOTk/zpn/5p72ONRoOvfvWrfOADH9jLpzpWjOUtnj9V5KmJHM+fKsqSyy54VBEhBZ9EIpEcXXb9jtxqtbh161bv73fu3OGb3/wmw8PDnDlzhp/5mZ/hV3/1V3nqqac4f/48v/iLv8j09DQ//MM/vJfnPlYoisJ4ISW7Wx6BRxURXcHX8kJyli4Fn0QikRwhdi0+vv71r/Oxj32s9/dPf/rTAHzqU5/iP/7H/8jP/uzP0m63+cf/+B9Tq9X40Ic+xMsvv0wqldq7U0sGhkcVEVLwSSQSydFFEUesDaDRaFAsFqnX6yfS/yGRSCQSyWEgRFK6doKI8fzeJwR2E79lIVwikUgkkhNMHAsabkDDCQnjGFM/9EZXKT4kEolEIjmJhFFM3QlouiHx0SpySPEhkUgkEslJwgsj6k5A24uO7IBFKT5I6mArTW+dqVEOAZNIJBLJccLxI2qOj+NHh32UbZHig3cGWUWxQFMVnj9VZLwgu3MGHSlKJRLJUadrIq07AX4YH/ZxdowUH8iNs5L+SFEqkUiOKnEsaLqJ6Ajj4yM6ukjxgZyGKemPFKUSieSoEUYxDTek4QRHzkS6G2SURU7DlPRHilKJRHJU8MOYmuMfaRPpbpDvpshpmJL+SFEqkUgOG8dPOlds/2Qtx5TiQyLZBClKJRLJYdE1kXrB0e9ceRSk+JBIJBKJ5AgQx4Kml/g5guj4mUh3gxQfEolEIpEcIlEsOpNIA6L4+Ps5doIUHxKJRCKRHAJ+mIw/b3nhiTCR7gYpPiQSiUQiOUDcoDv+/GSZSHeDFB+PwHGffHkUzn8UziCRSCQHSdsLqZ1gE+lukOLjETjuky+PwvmPwhkkEolkvxFC9IaCnXQT6W5QD/sAh40QguWGy+2VFssNd0d1t7WTL6M4mat/nDgK5z8KZ5BIJJL9IooF1bbPvVWbSsuTwmMDA5/5WGl6fOt+jWo7wI8i3nN2iEtThS1LAEdh8uXjlC2Owvl3cgZZmpFIJMeNIIo7nSuDZyLdDQMvPlpeSLUd0PQCVpoeiqIwmrO2LAEchcmXj1O2OArn38kZZGlGIpEcF6SJdHcMvPjIWTp+FLHS9BjNW+iqsu0CsaMw+fJxlp4dhfPv5AxysZtEIjnqtDuTSF1pIt0VAy8+xvIW7zk7hKIo6KrCSM48FgvEsqZGywt45Z5DztLJmtphH2nPOQrlIYlEItmIEMkk0rotTaSPysC/myuKwqWpAqM569gtEBMCEJ3/PYEchfKQRCKRdIliQdMNqDuDM4l0vxh48QFHowyxW9p+RD5l8MxkgfmaQ9s/eSm/4/jvIpFITh5dE2nLDYlP6m3vgJHi45giSxISiUSyv7hBRKMz/lyyt8iIdUyRJQmJRCLZH2w/MZE6JzCjfFSQ4uOYIksSEolEsncIkQw7rEkT6YEgxYdEIpFIBpY4FjTcgIYTEsaDIzqEEIc6tFGKD4lEIpEMHOGaSaSDZCK9U27zR68v8vpCg//rn34ITT0cASLFh0QikUgGBi/sTiKNBmb8ecsL+fyNZV66tsiNxWbv4198a4WPPnM4xXspPiQSiURy4nH8iJrjD4yJVAjBqw/qfO7aIn/x5gpe+HBJ6X+9MifFh2TvOazFbHIhnEQiOQp0TaR1J8DvE3xPIitNjz98fZGXX19kvub2fczFyTw/9v6z/OAL0wd8uncYePHRL1ACJyJ4HtZiNrkQTiKRHCZxLGi6iegYBBOpH8Z8+XaFl64t8vW7q/QbvlpI6Xz82QmuXp7k0lSBU0OZgz/oGgZefPQLlMCxCZ5bZRkOazGbXAgnkUgOgzCKabghDScYCBPp7ZUWL11b5I/fWKLhPjwITQG+/dwQL16e4oNPjGDq6sEfchMGXnz0C5TAsQmeW2UZDmsK6mFOX5UlH4lk8BgkE2nLDfmzm4l59OYa8+hapoopXrw8ySeenTiyF+eBFx+bBcrjMrp8qyzDYU1B3fi8ozmT5YZ7IIJAlnwkksHB8RPRYfsne/x5LATful/jpWuL/MVb5b7+FVNX+chTo1y9PMkLp0uoR/zSdXSj6gGxWYDu97GjeKveKstwWFNQNz7vcsPtCQJVgZmhNClD25fXUJZ8JJKTT9dE6gUnu3Nlpenx8uuLvHxtkYV6f/PoMxN5Xrw8yXdfHCeXOj4h/ficdJ/YLED3+9hh3aq3Ej3HYcfLWkFwfb7BctNjNGfty2soF+5JJCeTOBY0vcTPcZLHn/fMo68t8PXZ6qbm0e95doIXL0/yxFju4A+5B8h35g1sZ+AMo5i0qXO33KKYPpjsx1ai5zCyG7vNAK0VBH4UYWjqvmUmjoMYk0gkO6drIm26AVG/SHxCuL3S4nPXFvmTY2gefRSk+NjAdgbOlhfy6lw9+fuqzdmR7L5nP45aKWG3GaC1guD0cPIz7FdmQi7ck0hOBn6YjD9veeGJNZH2zKOvLXJzaXPz6NXLk3zvETaPPgpSfGxgOwPn2ZEMbT/k3EgWJ4gORAgcVClhpxmN3YqhtYJACMFozpKZCYlE0hc3iKjZJ9dEuhPzqKWrfOTpMV58buJYmEcfBSk+NrCdgfPsSJa6E+IGMbqqHoinYK9LCZuJjJ1mNB5HDMnMhEQi6UfbC6mdYBPpcsPlD19f4uXXNzePXpxMzKN/6+L4iferneyfbgdsDMSjOXPLQH8YnoK9DtibiYydZjSkr0IikewFQojeULCTaCL1w5i/ervcmTxapV/xqGsevXp5kgvH1Dz6KAy8+NgsEG8W6E/Czb3pBlRaHsWMQaXl03QDxgupHWc0TsJrIJFIDo8oFjScgMYJNZG+vdLipdcW+ZPr/c2jqgLvPTfMJy9P8oEnRjC0420efRQGXnwcNTPnQeCFMQ+qDnfKbQxN5UpnpLzMaEgkkv0kiGJq9sk0kbbckD+9scxL1xZ4c6nV9zHTpa55dHLg318HXnx0b/tzNZu2F1JpeUdmgNh+Yekqp4cyFNI6DSfE6rRsyYyGRCLZD9ygO/78ZJlIYyH45v0aL+/APPrJy5NcOVU8kebRR2HgxUf3tj9badNyQyotn7oTcmWmgKIoR2qa6V6RTxkM50yiWDCcM8mnjMM+kkQiOYG0O5NI3RNmIl1quPzRDsyjn7wyyUefOfnm0Udh4F+R7m2/5YWstoNe+eXeqk3dCbft/NjNwK2jMp5dllckEsl+IUQyibRunywT6U7Mo8W0wfd2Jo+eH80e+BmPEwMvPrpsNFvCzjbb7mbg1lFZenbSyitHRdRJJIPMSTWRvr2cTB790y3Mo99xfpgXL0/ygQuDaR59FKT46LAxGyCEoO40tu382I1hdRDNrQfBURF1EskgEkTJJNKme3JMpE034M9uLPO51xZ5a7m/eXSmlObq5Um+59kJmT1+BAZWfPS7LY8XUoyt+fh0KYWlq+RTxqb/ce1m4JZcerY/SFEnkRw8bhDR6Iw/PwnEQvDNezU+d22RL761QhA9LKRSHfPo1SuTPD9TlBnWx2Bgo99mt+XH2Vuy1j/RT9xIr8X+IEWdRHJw2H5IzT45JtLFhssfXlvk5dcXWWp4fR9zaSrP1ctTfOyZMbLy/WVPGNhXsXtbniqmuLHQ5PpCA6C3OXG7W/RGcXF+NLtOBW86vOwEeS2OClLUSST7ixCClpeIjpNgIvXDmL+8VeZz1xZ5Zba/ebSUNvj4s+NcvTwlzaP7wMCKj+5t+cZCk/tVG4EgiATTpdSObtHbZUhkKeDgOGkGWonkqBDHgoYb0HBCwvj4i463lpq8dG2RP72xTFOaRw+VgRUf3dvy9YUGAsGl6QILNRdLV3d0i95OXOxVKUB2ckgkkoMmXGMijY+5ibThBJ3Jo4vc2sY8+r3PTTCak5nTg2BgxUf3tgwQRIKFmoumKuRTxo5u0duJi70qBchODolEclB4YUTdDmj70bHuXImF4JXZKi9dW+RLt8qbmke/65kxXrwszaOHwcCKjy79RMJOsg3biYu9KgXI8o1EItlvbD+ZROr4x9tEuthwefnaIi9fW2S52d88+mzHPPpRaR49VAb+le8nEpYb7rbZhoPyGchODolEsh90TaR1J+i7k+S44IcxX3yrzMvXFnjlXm1T8+j3PDvB1SuTnBuR5tGjwJ5HsiiK+OVf/mX+y3/5LywuLjI9Pc0//If/kF/4hV84Nmmto5RtOKqdHNKLIpEcT+JY0HQT0XGcTaRvLTU7k0eX+84aURV43/kRrl6e5P0XhtGlebTHUXiv3nPx8Ru/8Rv89m//Nv/pP/0nnnvuOb7+9a/z4z/+4xSLRX7qp35qr59uz1gbTN0gQlXYVbZhv4LxUe3kkF4UieR4cRJMpA0n4E+uL/PytUVurfQ3j54aSvPic9I8uhFVUchYGhlTJ2Noh32cvRcff/VXf8UP/dAP8X3f930AnDt3jv/+3/87f/3Xf73XT7WnrA+mMDOUJmVoO842DFowPkrZIYlEsjle2F1nfzxNpLsxj169PMkVaR7toasqGUsja+qkDPVIvS57Lj4++MEP8ru/+7u8+eabPP3003zrW9/iS1/6Er/5m7/Z9/Ge5+F57xiDGo3GXh9pR2wMpilD48JY7pG//qQH48SLAm/M1wljwenhNEKII/Uft0QyyDh+Ijps/3iOP1+su7z8+tbm0eemC1y9PMlHnxkjY0o/HIChqWQtnYypkToCGY7N2PN/rZ//+Z+n0Whw8eJFNE0jiiJ+7dd+jR/7sR/r+/jPfvaz/Mqv/MpeH2PXPK6xc9CMoWN5i+lSmsW6i6lpzFUdRnPWic72SCRHHSEEbT+iZvvH0kS6E/PoUOadtfVnpXkUgJSRZDcylnZsBqPteYT8n//zf/Jf/+t/5b/9t//Gc889xze/+U1+5md+hunpaT71qU899PjPfOYzfPrTn+79vdFocPr06b0+1pYIIRBCUEwnL8eZ4cyWO1r63e6PqjF0v1AUhZShMZZPPVa2Z+PrO5ozKbd8aWSVSHZB10TacI/f+HMhBG8tt5LJo9uYRz95ZZL3nZfmUUVRSBtar6SiqcfvPXLPxcc//+f/nJ//+Z/nR3/0RwG4cuUKs7OzfPazn+0rPizLwrION1CvND1em2v0/BqKovQC3kn3cjyOUXYvsj0bX9/pUor5mnskX2/Z4SM5aoRRTMMNezupjhN1J+BPry/z0rUF3l5p933MqaHO5NFnJxgZcPOopiqkzURspA0N9RgKjrXsufiwbRtVXa9KNU0jPqItXUIIZitt5mo250ayOEG07ga/Uy/HcRUpj3Puvcj2bHx9V5rekfXOHNd/Y8nJww+TzpWWFx4rE2kUC165V+Wl1xb5y7c3MY8aKh99epxPXpnkuenCQAv8o2wYfVz2XHz8wA/8AL/2a7/GmTNneO655/jGN77Bb/7mb/ITP/ETe/1Ue8JK02O2YrPU8FhqeDwxll13g9/p7f64Gk4f59x70Qa88fUdy1vM19wj6Z05rv/GkpODG0TU7ONnIl2oO/zhtSVefn1r8+gnL0/yXQNuHj0uhtHHZc//hf/tv/23/OIv/iI/+ZM/yfLyMtPT0/yTf/JP+KVf+qW9fqo9oXtrf9/5Ee6WW+v8HrDz2/1uShBHKX2/2bkP6owbX9/RnMlozjqS3plBMxVLjg7dSaRecHzGn3tBxJc6a+u/ca/W9zFd8+jVy1OcGckc7AGPEJahkTWTGRymPhh+FkUcsZxdo9GgWCxSr9cpFAr7/nw7GaW+E3YTrPfqOfeCzc692RmPknA6aAb5Z5ccPEIIGm5Iwzk+JtKeefS1ZG39ZubR919IJo8Osnk03REbWVM7Ma/BbuL3wF/d9qpLZTcliL1K3+9FMNzs3JudcZB9D0d12qzkZBHFgoYT0DhGJtLEPLrES9cWNzWPnu6aR5+bZDhrHvAJD59uh0q2M2X0OHao7CUDIz42C9SHEVD2Kn2/n0JgszPut+9BZhckg8pxM5F2zaOfe22Rv9rCPPqxZ8a5enkwzaOqopAxNTJWMtL8uHeo7CUDIz6O0o19r7It+ykENjvjfvsejtK/k0RyELhBd/z58TCRztccXn59kT+8tsRKq7959PJ0gatXpvjo02OkzZNrmuyHrqqkzWQ1x0nrUNlLBkZ8NN2A1ZZPIa2z2gpousGhBbW9yrbspxDY7Iz7PUxNdpRIBoV2x0TqHgMTqRdEfPFWmc+9tsg379f6PmYoY/CJ5yZ58fIkZ4YHyzxqaCoZUyNr6Se6Q2UvGRjx4YUx96s2QTnG0FQun0rMMMc5zX8YU1X3u0y1naDa6t/rOP9bSgYDIQRNL6RuH30TqRCCN5dafO7aAn92Y5m297BIUhX4wIURXhxA86ipq72R5pYuBcduGRjxYekqp4bSFDMGdTvA6rQzHec0/0k0QG4nqLb69zrO/5aSk81xMpHW7YA/ubHES68tcrvc3zx6ZjjDi53Jo4NkHj2OO1SOKgMjPvIpg5GcRRQLRnIW+ZQBrE/zz1VtZittmm6AF8ZYuko+ZezJDXq7W3m/zwNH/ibfPfdevWbbCaqtyjKyZCM5agRRYiJtukfbRBrFgr+ZrfK5awv81a0KYR+BlDY0PnYxWVv/7NRgmEdPwg6Vo8rAiI+dGChbXkjbD7m90uZB1eH0UIbhnLknN+jtbuX9Pg8c+Zt899yVlrfmNTOYLqVJGdqei6atyjJyCJjkqOAGEY1O58pRZifm0SszBV68PDjmUdmhcjAMzLvzTgyUlZZHpe0jhKA+5zOaNVhtsSfm1O1u5f0+DxzKTX433onuuYsZgzvlNoW0TqXls1h3Gcun9lw0bVWWGbTNwpKjh+2H1OyjbSJ1g4gvvlXmpWubm0eHs2Zvbf0gmEc1VUkGflkaaUMbiKzOYTMw4mMz1oqSnKVTd0LuVNqs2j5iBUoZs2dO3Q0bA3jW1La8la+/tSdvEG0vpOUFzNUEuqoe2E1+N96J7rkrLR9DU2k4IWEsMDVtX0TTVmWZk+iBkRx9joOJVAjBzaUmL11b5M+uL9P2HxZHmqrw/gvDncmjIye+xCA7VA6XgRcfa+nenHUViAWnhtM0nLBnTt0NGwP4lZnClrfytbd2N4iYqzpEsUAIGMmanB3JHthNfjfeie65m27AlVNFLF3FC2Pmqo4sfzwCsmPn+BDHgoYbdAT30RQddTvgj68v8fK1rc2jVy9P8j0DYB6VHSpHBxkV1tC9OQMEkaDaTm4yXhgjhNhVENgYwNt+xIWx3KZBfO2t/fZKi1jAzFCG+ZrDSM7a930za9mNd6J37jXnE0Ic2eVwRx3ZsXP0CdeYSOMjaCKNYsHXZ1d56dri1ubRZ8a4euXkm0dlh8rRRIoPHg7SozmTmaE0y00PQ1OZrzmM7lIAPI75cS/Hr3/rfo1qO8CPIt5zdohLO3ijeVzvxH6XP05ydkB27BxdvDCibge0/ehIdq7M1RxevrbIH76+SLnl933MlZkiVztr69MntNQgO1SOB1J80P+2mTI0RnPWuiAwtoug9zgBfC/Hr1fbAU0vYKXpoSjKjkTUUfdOnOTsgOzYOXrYfjKJ1Onjkzhs3CDiL94q8/K1Bb55v973MSNZk+95doKrlyc5fULNo7JD5fgh39nof9vsFwR2E/QeJ4Dv5fh1P4pYaXqM5i10VTkRN+mTnB2QHTtHAyEErc74cz88Wn4OIQQ3FhPz6OdvbG4e/cCFET55ZZJvPzd8Im//skPleCPFB93bJrw+V6PqBCiK4PmZIldmCrT9qBcE7pTbxyrojeUt3nN2CEVR0FWFkZx5Im7SJzk7cNSzTiedOBY03UR0HDUTac32+ePry7z02gJ3K3bfx5wdznD1SmIeHcqcPPOo7FA5OZycd+3HYCyflFfeWmqy1PBpdsxkH35qjAtjud7j9iroHZRnQVEULk0VtjV/HjcPhcwOSPaao2oijWLB1+4m5tEvv93fPJoxNT76zBifvDzFpan8kf7dfRRkh8rJRIoPkiCdMjQyps50SQOSlOvGzMZeBb2D9Czs5CZ93DwUMjsg2Su8sLvO/miZSLvm0ZdfX6SyiXn0+VOJefQjT58886jsUDn5SPHRIWfpZC2dpWbSC/9ELvtQZmOvgt5R8yzs13mOW0ZFMjg4fiI6bP/ojD93g4i/eHOFl64t8q0Hm5tHP/FcMnn01NDJMY+u7VDJGNpAbccdVKT4IAmSQgjODKcppHSK6USI3C23mK20OTOcYbyQOpD9JIfBfp3nuGVUJCeflhdSs/0jYyJdax79sxvL2ANkHpUdKoPNwIiPrbbGzlba3Fu1yVo6mqIQxPDFW2UW6y5ZU+fCWI6PPD3GWN56rJv82g2w06XUug2w+81WWYj98lActQyPZDCJ42T8ecM5OuPPa7bPH7+xxEvXFjc3j45k+OTlST5+gsyjskNF0mVgxMdWW2PnqjZLTY/3nR9mqe7x+nydxYZLGEMhpXZ2rIS9xz/qTf5RMgF7VbrY6rn3y0Nx1DI8Jx1Z5lpPFIuOiTQg6mPUPIzzdM2jf/V2pe+ZMqbG37o4ztXLk1ycPBnmUV1VyVqyQ0WynoGJBlttjT03mmOp6XG30kZTFLKmzmQhxfXFJqaa3EBylv7YN/lH+fq9Kl0cRhZCdqUcLLLMleCHSedKywuPhIn0QdVOJo++sbSpefSFNebRkxCgDU0layUZDtmhIunHwIiPzW7hCoKbC3VqLRcRx5wbzWCmNfJpHUtXeWIsxwunS73A+Tg3+UfJBOyVaHicLMSj3qhlV8rBMuhlLjfodq4cvonUWWMefXUz82jO5MXnJnnxuUlmhtIHfMK9xzI0smbSNWg+wjJOyWAxMOJjs1t4xtK5udRivubgL7cpN30uTRe4cqrImWcSN3nb70wJzZmPdZN/lEzATkTDTsTB42Qh5I36eDCoZa52ZxKpGxzu+HMhBNcXOpNHb25uHv3gEyNcvXz8zaPJiIIkwyE7VCS7ZTDendj8Fh7FAkNTmComt0VNS94gRnJJAO8XdB/1NvkomYCtRENXdKw1zOqq2lccPE4W4iBv1NK38OgMUplLiMREWrcP30RaXWMend3EPHpuJMPVK1N8z6VxSsfYPKoqCmlTS6aMmrrsUJE8MgMjPjZjNJe8EczV2jhBRBSnyVr6nng89oKtREM3I7HWMOsG8Z6f8yBv1DLL8ugMQpkrigVNN6DuHK6JNIoFf32nM3n0dn/zaLZjHn3xmJtHNTURHFlTJ2PKDhXJ3jDw4mMka/L0RI60odIOIp6dzHNpKt8TJUc5jd0VR2sNszOlzJ6f8yBv1EdB8EmOHkFn/HnrkMef31+1efn1Rf7o9SUq7S3Mo1em+MhTo8fWPKqram8lfdo8nj+D5GhztKLpIWAHMTNDWS6M5fi/X1vg1lKbWCgMZwxUVaWYTl6iM8OZI5fG7mYkHD/kwmiWM8NpcimDphsA7FnJYrsb9V6WSgbVtyDpjxtENDqdK4eF40f8ecc8+tpcf/PoaM7kE8fcPNrtUMmY2rEVTZLjw8C/s3eD3Vdvr3JruU0pYzBbtYljwbmxLFGcZD8URTly6caNGQkhBK/NNXZUslgrGLKdm83aDb7AjgXFXpZKBsm3INkc2w+p2YdnIhVC8MZCo7O2fgWnzzl0VeGDTybm0feePZ7mUdmhIjksBl58dIPdnZUmKUNFQdDwQm4uN8inDZ6dLm6a/j9sc6SiKL3g3PJCKi2PMIqZGcpsW7JYKxhaXoAQkE8ZDw1g24mg2MtSySD4FiT9EUJ0xp8fnol0tZ2YR1++tsjsan/z6PnRLFcvT/I9lyYoZowDPuHj0e1QyZg6WVN2qEgOj4EXH91g98EnR7m+2GSp4XJ2OMNUMUMYi176P2tqLDfcdUJjJzf+/RYo/UTETkoWawXDK/ccEPDMZOGhAWw7ERSyVCJ5HOJY0HADGk5IGB+86IhiwVfvVHjptUW+cmd1c/PopXE+eXmKpydyRy4LuhVKd4dKJ8NxHDM0kpOHjBIdLk0V+DvvOcXX7lQQisJoVufsSIbJvMli0+fLb5dZbQdMFVMYutYrDWwXoPe7e2PtGeZqgpGsyUjO2rRk0RVDlZZHywuYq4lOyeZh0bJTQbEfpZLDzipJ9p+wYyJtHpKJ9N5qMnn0j95YYnUT8+i7The5enmKDx8z86jsUJEcdQZGfGwXzFRV5TufHGU4a/KNezV0VcENIhabPl+9vcpKy6XuhLz43CSqKnrfZ7sA3U+gjO25QRPemK8TxoIzwxnOj2b7fr9kCFKDb9yroSmgayojWZN3ny4BD3s+dioo9qNUIltuTy5eGFG3A9p+dODjzx0/4gtvrvDytQVem2v0fUzPPHp5kpnS8TGPru1QSRmqFBySI83AiI/lhsuXbpV7wfRDT44yUVz/xpLUQzVGc1ZPLDyo2gRRzMXJAl++XeHWUpMXzgz1AvJ2AbqfQNlrg+Z0Kc1i3cXUNOaqDqM5q+/3W2l6vDJb5UHVYTRvkbeSYWobX4cuh+m9kC23Jw/bTyaROn0mf+4nuzGPfvLyFN92dujYlCZkh4rkuDIw4uPeqs3bK21KaYOlRpszw5l1QbdfOUJXVU4NZZiruizUXWZKaa6cKvL8qWIvW7FdgB7NmUyXUqw0PcbyFqM5k7sV+51SSdVmttJ+5CxIVzCN5VPbBuqWF2JqWs+vkja0I+vPkD6Sk0HXRFp3AvzwYP0cq22fP+qYR++dIPOo7FCRnAQG8B1963bRMIoRIhk+dnYky0jWYDhr9sTDxck8qrrzX/hyy2e+5hJGMW/MN2h7IVlLR1XoCYW2H7LaDh45C7LTQJ2zdIayyRuspau8+0zpsfwZu/Vl7ObxsuX2eHNYJtIoFnzldoWXO5NH+w1BzVoa331xgquXJ4+FeVR2qEhOIgMjPs4MZ7gwmqXtdQdyZdZ9vpvm77apjqwpXTw7XXzk5+1+37Sp8+pcnbYfMlNKMzOUJmVoVFoelbb/WOWFnQbqsbzFC6dLj+01WbtTZrZik7N0dK3/Tpm17KbctJ8tt9LMun8clol0Z+bREp+8MsmHnjz65lHZoSI56QyM+BgvpPjI02ObBuisqdF0A16Zdchaem/w1mbsNIB1sxJ3yy0Azo1kcYOYlKFxYSxHztKpOyFzNZt2Z1bHbgPiTgN193Fdw+udcvuRgm9vp0zNZqnh8b7zI7hB9JBw2vgaNd3gSCyok2bWvccLu+vsD85E6vgRX7i5zEvXFrk23988OpazePHyBJ94bpLpAzCPCiFYbQfYfkjG1BnOGjv+3ZIdKpJBYmDEx04CdPK7Lmh6AbOVdm+IV783gZ0GsG5WopjWya3aOEGErqq90kj387OVNi03pNLyqTvhvgbExw2+vZ0yI1mWGh53yy1mhh7eKbPxeaZLqSOxoO4gzKyDkl1x/Iia4x+YiVQIwevzjd7aejd4uKRjaArf+cQoV69M8p4zB2seXW0H3FxqEscCVVV4ZiLPSG7zLbayQ0UyqAyM+NiOpM3UYDRn8dU7q1ynScONekHrUW/xvWxD3uLsSPahzEv38y0v8X0cRFbgcYNvb6dMEPHEWFLCOjuSfSibtPF5LF09EgvqDsLMehKyK5sJqMMwke7EPHphLDGPfvzSBMX04ZhHbT8kjgXj+RTLTRfbDxlhvfiQHSoSiRQfPXrlkUobgHOjuXWlhMe9xW+XeTnI7o7Hfa6NHpPRnEm55T9Uxtn4PPmUcWDtu1v9jAdhZj0JrcIb/5u/PFMgbejUneBATKRhFPPVztr6r2xhHv34xQmuXpnkqfHDN49mTB1VVVhuuqiqQsZM/rsz9STbKTtUJJIEKT46rC2PZE0bxw/RNbU3Vv36QoPVls/FqTwLdXfdLT5ragghuL3SeuQU+0F2d+z2ufrdgNfulCm3POaqDrFg3S3/MDtW+gmkjePx9zMTcRJahbsCaqKQ4tZyk7eWWgeysfVexealawv80RtLVO2g72Pec6bEi5cn+fCTo1hHKHswnDV4ZiKP7YcMZ01OD2fIWjqG7FCRSNZx/N4RH4ONQbR7Y98YVNeWR4QQvPqgTqXl8aDqADCcM8mnjF4w3W3XRz8OcqHabp+rXwkB3lk8V255GKrKpenCulv+fvxMO/VSbHzu5YZ7oGWQk9AqbGoqLS9k6UENVVX2tURg+yFfuLnC515b5I2F/ubR8bzFi89N8onLE0xtMhjvMEk6VHTGCimyskNFItmSgRIf/Uon8zWXKBaoCswMpbF0FS+MsXQVIQSzlTZzNZuzwxkEgomixaWpwrrFctt1fewFh2lg7FdCgHcWz9VsHz+KDt1IutufYT+F3nHezuv4SeeKF0acGc6s69zYS4QQXJtLzKNfePPomUd3gtptibV0MoaGesTOJ5EcVQZKfGwMQCtNr/f36/MNlpsemgpvLrUYzhpkLZ04ElTsgKWGxxNjWS5NFR7qmtiu62Mv2E8DoxCC5YbbM/KdGc4wXkj1xM1mJYTux0ZyJtOlZG7JYRpJt+IklEH2EyEEbT9KhGTHRKooCiM58yHD5ONSaXn80RtLvHRtsZdN3MiFsSyfvDzJdx+ieXQzdFVNWmItjbQhW2IlkkdhoN6BNwagsbzFfM1lvubgRxGGpiIQzNUc/CBpIZwppfnAk2PMlpOR7GsD6067PvaC/by5LzdcPndtgTcXW1i6xnPTBb7rmbFedqfh+KQMlSCMMHSVhuOTTxlcmSmsW0Z3EG/C6/8NwQ2iHXltTkIZZD+IY0HTDWm4AUG0fybSrnn0c68t8tU7x8M8uhZDU8mYGllLlx0qEskeMFDio58JcTRn0fJCTg8nQf3GYoMgjKm6AbYXsdL0Waq7zAwlwmLtG2K/gPaob5jblVX2+ua+9vluLTV5c7GJHwrCOO4ZM4F1fpdiyqDuBpwaSjOSS372C2O5Xf8sj8Pa19wNor5G134c5zLIfhBGMQ037LWM7xezlTYvXVvkj7cwj777TIlPXk4mjx4l86ipq8nAL0vD0o/OuSSSk8DAiI/NAmL3BixEklUoWBqOH1Nuujw5miNn6kwWUz2fx1r2MqBtV1bZ65v72ue7XW4RxgJFhaYboqqJ2OlmW4oZgzvlNoYGQRRTzBhEsdg0+7KfJaK1r/ntlRaxgKliihsLTa53jIondaDXXuCHyfjzlhfu2yTS42weTRlaT3DIDhWJZP8YGPGxWUBc+3FVgelSihdOF3l7WWM4YzGcM9f5PLo8yu2+39d0z3Z9oUGl5XFpusBCzX0osO/1zX1tGadqe5wfgVgINE3lI0+N9s6mqQqVlo+hqQRRkn6u2wEjOWvT7Mt+mzvXbiBuuj73Km1mV9uc9XIEUczzp0rHbqDXfuMGETU7Gfu9H3TNo5+7tsCf31zB7TN8zNAUPvTkKC9ePjrmUUVRSBtab8roUTiTRDIIDIz46BcQx7rdLFWbc6M5FmsOy02PkZzJeCHFmeEMZ4YzfWd4PMrtfquW1dWW3zPfbRXY+/EoQmhtGWcka3JqKEMUi97m3m5W6PlTRZpuwJVTRUxNwY8Elq6uazXe6nvvh7lz7QbihhOy1HRQUFAQVDqt0/tZXjlOo9O7k0i9YH/Gn1daHn/4+hL/92sLLNTdvo95YizL1ctTfPzSOIUjYB6VHSonm+P0+znIDIz46BcQV5oe91ZtlpoeS02PvKUxlDVJ6Sq3lh10NXmTmq+5D/kKHuV2v1XL6sWpPMC6Vt6dsp0Q2mxI2FrvxHzNIYwF1xcatL2wZ5wdL6R2nUXYb3Pn2g3ESw2XUsYkbST/nmlD3/dOlqM+Oj2OBU0vpOHsj4k0jGK+fHuVl64t8Nd3VvuaR3OWzndfGueTlyd5aiK/52fYLVpn2qjsUDn5HPXfT0nCwIiPfgHxTrlN1tL5jnNDXJuvkzU1HD/i8zeXWW76rDQ95usOI5kUF6fzXJ9v9HwFWVNDVeD6fAM/ijg9nEYIseWb2lYtqwt1d9MSz3ZsJ4Q2+2Vc652IYkgbGq8+qNNyw8dabrff5s61r2PWSgJKHCtYusq7z5T2vZPlqI5Oj2LRWWe/PybSu5U2L722yJ9c728eVYDzo1k+8dwEP/SumUMfIy47VAaTo/r7KVnPwIiPfgExZ+noqspSw8MLBFZOZ7XtoSnwZGfdfRBH+FHEG3N13lxqUW56rDQ9PvTkCDNDaZabHoamMl9zGM1tPbJ7s4zA42YJsqZG0w14ZbYTjM31b7Tb/TJut9fmqLH2dez+rAfZ8nvUZobsp4m07SXm0ZeuLfDGQrPvY0ZzJldmirzrdImRnMUzE/lDEx6yQ0Vy1H4/Jf0Z6H+VbhC7vtBAQeHiVJ4bC00EsNTwKLc8nhrP8u4zJW4tt3qll2tzDQxNYaqYQlOhmDGotHyabtATH5vVHftlBPYiS6AogNL53w1s98u42V6bo/pLe9hts/1E5GHUmd0gmUTa9vbWRCqE4LW5Oi9dW9zWPPrJK1O863SRmh3u2yTU7ZAdKpK1yJk+x4OjGV0OiG4QAwiiOgt1l6GswVTJQlGSMkUhbTCas7D9iJtLLWw/YrnpcH81ERwPak6nEyRmKGP0jJgHWXdMbv0GT08ku1Xa/npz4Xa/jN3XYeNem8P4pX3UIH6Qwb+f+DnI3TFtL6S2DybScsvjj15PJo/O1fpPHn1yPMfVy5N898X15tH9mIS6GbJDRbIVh305keyMgRUf3WDVdAPcIKKQSkxoZ4YzNN2A+ZpLMWNQt5N09pnhDE+MZblbbjOWT3H5VIm7K8kY9tGcxfWFJvM1B1V9Z9vt2lJH0w0QQmw6wvxxfg43iFhputTtgKGs8VDGYqtfxn5B+zDNWY8q2g7bZHYQ7cUNd+9NpEEU85VtzKP5lM53Xxzn6iGaR2WHikRystgX8TE3N8fP/dzP8dJLL2HbNk8++SS/93u/x3vf+979eLpHohusutM7Tw9lGM6ZKErSTvqg6nQGa6lcOVXkwliKDz81xpnhDLMVG9ePyKUM8mkj8R5YOufH8j2vxMZShxfGfONemdvlxFfxxFiWDz819kgBcq1gcIOIuZqdZF/imJmh9CN0ytSotHzCWPDuMyUuTRV6n9ssk/Coc0622iEDjx7ED9tktl915igWNJyAxh6bSLvm0T9+Y4ma0988+m1nh7h6eZLvfHL0UDwcskNFIjm57Ln4qFarfOd3ficf+9jHeOmllxgbG+Ott95iaGhor5/qsegGq0JapzbnM5o1WG1B0w2wdJXTQxkKaZ2GE2Lpai97MJozyXbadE8PpxnJmtyvOg95JTaWOppuUpsvpU0gmQ76qAFy7S1/peliaCrPTheZrzmkdvAmvVY4VFoe5aZHy49Yabg0HJ92R0zN1xyiOAkCV2YKKIrS+3kSX0Bj13NOvnSrzNsriQC7MJrlI0+vF2CPGsQP22S213Xm/TCRtr2Qz3fMo9c3MY9OFlK8eHmC731ukslDyIDJDhWJZDDY83fo3/iN3+D06dP83u/9Xu9j58+f3+uneWy6wWp2xWah7rDUdMlbOlMli6cn8gznTKJYMJwzyafeqW2XWz7zNZcoFizUPcbyKd57bnidV2I0Z/adZJq1dJaancxHLrsuQO4mk7D2ll+3A4I43lXQXSteWl7Aqu2zUHNRFbizEpAxNTRVXSdq7q3a1J2wJzaKaf2R5py0vJBS2gAU2n0E2KMG8cM2me1VnXmvTaRCCF6dq/PyNubRjzw1xtXLk7zrTAn1gDMMskNFIhk89lx8/O///b/5xCc+wd/7e3+PP//zP2dmZoaf/Mmf5B/9o3/U9/Ge5+F5Xu/vjUb/XRB7TS9YuT5pU8ULBJW2z6sPajw9kd80kLW8kDCKSZs6d8stiul3fBLdwNPPfDiWt/jwU6OcHckA9DbkdkXHbKXNvVWbbKf9d6tMwtpb/lDWYGZod+vs14qXuZpgNGcxX3Oo2QF+LChlLTw/XidqgHViA9h1piFnJQPAlhrvZD5240/ZisMyme2V0XWnJlIhBKvtYF1nSb/nW2l6/PEbW5tHn57I8eJzk3z3pfF1AvsgkB0qEslgs+fi4/bt2/z2b/82n/70p/kX/+Jf8LWvfY2f+qmfwjRNPvWpTz30+M9+9rP8yq/8yl4f4yE2M1bODGXIpwwMLdnt0nRDbiw2uTRV4Pxo9qE39u7CtVfn6snfV23OjmTXCYXNBMpEMc3EhiVaXaHyYLXNnYrNpak8Kuq6tt2NjOZMpkvJXpq149BXmh53yu2H9sZsDIxrxYuuqpwbydDN7L+x0KDW9pguZdaJGiEEdafRExtnhjPryjA7ET1jeYsPPTnKmeH1Auw48zhGVyGSSaR1e+cm0tV2wM2lJnEsUFWFZybyjOSSLpMgivny7QovvbbI1+72N48WUjofvzTB1cuTPDH+8Ebi/WJth0rG0NCl4JBIBpo9Fx9xHPPe976XX//1Xwfg3e9+N9euXeN3fud3+oqPz3zmM3z605/u/b3RaHD69Om9PtamQeLMcIYnx3K8+qCOHURoCizWXIJI9A0kSTtqhrYfcm4ki9NnGNdOBEqXbhZiKGvx1btVvDBiLJfiuZk8S3WnrzlzbelnvuYymksC+GZ7Yzb+zBvFy3DGoOFGhFHMlZkiZ0cyvfHqXfElhOB5RaHpBnhhTMsLyaeMvgJtMxRF6SvAHoWdZhz2uwX3UYyuj2Mitf2QOBaM51MsN11sP6RRDnj52vbm0U9emeSDTxyceVR2qEgkks3Yc/ExNTXFs88+u+5jly5d4n/9r//V9/GWZWFZ+3/73SxIjBdSvO/CCF4UU254hDGMFyxWmj5vzNcptzxMLelWaXth5wankjU17laSLMPGiaI7EShdulmIWttjPJ+MV1c6fojrC82+3TFb7YiZLqWZq9rMVtrYfsRqy+fiVJ6FukvTTQLTbKXNbMUmZ+nM11xGsua6MtNozqTc8tdlUdbORLnzGDf9vRICO8047HcL7m6MrkEUd8afP7qJNGPqqKrCvdU2ry80+M9fmeXWcqvvY7vm0U88N8nEAZlHZYeKRCLZCXsuPr7zO7+TmzdvrvvYm2++ydmzZ/f6qXbFVkHC9iPSusbZ0SxvLDT46u0KXhhzu6LhBzFThRQLDZeImKxpMJIxUFUFVVHoF0O680JmKzZ3O/tjugJlYwAezZm96aK5tNHzfCiKsml3zFY7YrozRRbqDm0vZLWdzBcZyVt4YcydBzVuLCblk/edHwElGVJ2YSy3pWelG7Afp6V1s7beRwlQ252j+zpfX2isE2B73YK7E6OrG0Q0Op0rj4MQgvurbf74jSW+/HYFv0+p5jDMo7qqkrVkh4pEItk5ey4+/tk/+2d88IMf5Nd//df5kR/5Ef76r/+a3/3d3+V3f/d39/qpdsVmQaK72fZOxWa50/HS8AKCSGDoKvN1h+GMTtsLMHSVKBLMVR3OjGZ4z9nhvhNFu/Qbeb52HXzLC3sljm87O7SuY0YIwWzF7tsds9l4724ppe7AStOlmLGIhE/KTAysTTdIAn8kqLQDvnJ7lfeeG3rott50A1ZbPoW0zmoroOH4AL25IqrCI7W0Jq29Pk0vpNz0EEJsuw9nM7bLOGyc4wIwnDP3vAV3K6Or7YfU7GSI3eOw0vT4w9cXefn1ReZr/dfWPzWe45NXJvlbFw/GPGpoam+pn+xQkUgku2XPxce3f/u38/u///t85jOf4V/+y3/J+fPn+a3f+i1+7Md+bK+faldsFiS6A8IuTeXxwoh3nS5RbnrM1VwsPdlc2/ZiFEVhvu5gaD5DaQMhkgCsKsnN9vZKa10pYbOR590be9rUeXWuTttfv0G2ez4hRN/umM1+lpWm1/OBlJse7SCiRNLeO11KM15IJZ0SnbbaM0MZCimtr+nTC2PuV22CcoyhqUwPpbhbcTqZEHbdXdMlZ+mEnfON5S1MTXvkTMR2GYfu63xpOhmYNlG0uDRV6D1uv7wgj2Ii7UcQxXz57Qqfu7bI14+IedQyNHKmTtrUDn1jrUQiOd7syySm7//+7+f7v//79+Nb7zk5S0dTFJpOiB9GvDHXIJdSmSxYFFI6xlSBU8UUq22DtKEwVcqQT+k8MZZjNJ/CDaJ1w7i6ImKzm3lvg2w5qdOfG8niBvFDQXi35sy1ZYia7aOoYBkqT+Syve4SgJShoqoKQSSYLCZZF0VR1gXjlhswM5SilDGp2wFhFK8rcaQMjQtjuw92Y3mLd58pIYTA1LS+o+B3ynattd3XeaHmMpJLhMfaDMteloAg8ds03YCGExLGjy467pTbfO61Bf7k+jL1Tcyj7z2XTB7db/OooiikjCTDITtUJBLJXjIwu1363XS7H49FzP3VFvcqNg3X5/RQliszBTRNJWcm49bn6x6xolBzQoazFudGc4wXUtxeaRHFPOQ92OxmvnaDbG7VxgkidPXxN8iuFTvDWZMrp4qdWQpJSvz2SotKy2OykOLCaI67lTbnRjOM5kyWGy53yy1en2+gKhDFUEjrKCiMdMoi8zWXuZpNuzMV9VGyBYqicGmqwGjO2vdhYDvJjOxFCahrIm25IfEjmkhbXsjnbyzzuWuL3FzsP3l0qpjixecm+d7nJvbVPKoqCmlTS6aMdsytEolEstcMjPjY2PVwZaZApe3zjXs17lfaXJtvstz0iEXMg6qDqas8KRQiAWesDLqmMJXL0HADCunEKAqbew82u5k/ygbZnZQINgbbbsfK2uFlrc7NXFMVspbOmeEM5ZbPqw/q3Fio8/pCkyfHskSx4NRQmicncmRNDSEEbS+kavsIAZWWv65UtBsOahjYTjIjj1MC2omJdKuBYEIIXn1Q53PXFvmLN1fw+kweNXWVjzw1youXJ3nX6f0zj2pqIjiypk7GlB0qEolk/xkY8bGxO+Leqs3NxSYPqg52EOH4EQpgKEkQqtsBY4UUi3WHctMljAUPajZZy6DhhJRbfk9E7Gas90YhsZM5GTtpF90YbLsdK3NVm6Wmx/vOD1OzfbwwImPpPRNs93UZzaeI5xv4UYymqgxlTS6M5VhuuL0dLkt1h1U7oJQxCWLBuZH0I7et7vf8je141BJQ2wupOzszkfYbCBYLsa159JmJPC921tbnUvvzK6qram8lfdqUhlGJRHKwDIz4yJoaTTfglVmHrKUzlNExNY2xvMWdlYCpksVSTWAHMTrJTXB+1Wa8aDFTTLPYcDtGzTRRJGg4PkKIdUPAdhJAdzp3Ym1wLjddyi2XUsZMSgVbTD/t0hUV50ZzLDU97lba6KrKSDbFpel3TLBJ5gYajk/O1FEVhQujmZ5PZK1oe2OuzqtzNQxNw9QVLk3meXKi/5nXZl/6CYz9nr+xHTspAXV/noYbJMJUUwl3MRSsOxBsOGPyxVsr/M+v3efafH1z8+izHfPoI/hpdkK3QyVjarIlViKRHCoDIz6SLoSkhTRGkDFzDGWTlsSnJ3I8M5njb2ar3Fu1iYWglE5S5NPFFE0vZLHu8dZyi5VWsgsmk9Jw/Zg7lYeHgG183rUBudmZarndnIy1wXm+ZnN/NSkFGZrKlc700n4/Y/e53CBCU8HxQy6MZjk7kiFr6cxVnXUlorG8xXQpzULN4dJkActQeHb6HSGwtqwEMcPZZPHeg6pNuKGbY6OgmC6leh04ezkvZK+yJtuVZhbqDl+9vUrbi0Bh3SjznbDS9Hj59UW+eb+G3acd+x3z6BQffGJkX8yj3R0qskNFIpEcJQZGfNyvOqw0fUppg5Wmj+1HvHC6RMsLcfywV3c3NJWWG3BrpY0XxYzkTVbbPlEcUXd8IAZS3FpsYOr6Q0PAxoRgueH2MiIZU2O+5hILegF5JxMx1wbnhbrNcNbgyfE8DSfE6hNEhBBcX2jwymwVU9MoZXRODWceaondeNNPOho0xgvpdd0s3WC+tqyUtTTi2xWqtk8pYz4ktDYKipWmt6nA6OeV2amo2C5r8rjipLvO/tZyi6YbrhtlPsJ68bHR12FqCp9/c4WXtjGPXr08yfc+O7Hn2Z61O1Sypt5bCiiRSCRHiYERH0IIbC8iikTP3NcNyK89qHG73MYLImYrbYSAfNqg4QS8vdRE11WCUFBt+wQRjORAoKKqUOsM4OoOAVtpenzpVpm3V5KMSD6lM5KxeqUOS1cfMoYuN9wtl7/lUwY5K8nEDOfMvkOkVpoe37hX40HV6f1cT0680xK7VUDeamDX2uzAuZEMw1lz3UK7tWz8PmN5i/ma2/f79vPK7LQUs13W5FFLOo6frLO3/cREmjaSbo/lpovaGRu+kdV2wPXFBrdX2nzjXpU3FhoE0cN1la559JNXprgyU6Bmh9h+0nGz2WbanSJ3qEgkkuPGwIiPtKFSbXtU2x5DWYu0ofaC1P2qTauz90TXVHRVwQsjFBSCWGApClXXp5Q1KaQMohgsI1kJrygKpYzRW8R2p9ym5YWU0gag4AUBlbbLK7PJMLOcpfc1hm4MlOsyDh1DYNej0c/U2vJCdFVhtBPELX19++76gJy0BnezIt0R79uZZlVV5dnp/iUf6N9xs5mnol/JY6elmO2mm+62pNPqmEg3rrMfzho8M5Ff162yluWGy//8+gM+f3OZqv3wTA5IynEfenKUjz0zzunhNIqiUGn5m26m3Slyh4pEIjnODIz4mK+7NL2QlGnQ9ELm6y7ZlEkUCy5PF7mx0KDS8jp7WFRW2xGWnqSwz49kGc1bPDmRp9zySBkqacNAoKCpam/mBySBMWfpLDXaCCFI6WoS3NyAQkqn3PJ622mTEept5qptSlmLWtujmF6/yG3txNNu5gJ4qJSQs3RGciYCgR9qmJrK3XILIcRDy+jemK+zWHcZy6d6bcc7DV47MZWuzTJs/Bn6ZXnW/gz9RMVm+3A2E0s7WfYWx4KmG9JwN59EqigKIzlzXanFD2P+6u0KL19b4Gt3q/Szn3bNox+8MIIbxsSx4EEtMTqP5My+m2k3lnP6YWhqMn9D7lCRSCTHnIERH44fEccCXYeaHbJYd3jX6SFUBd5cauKHMaWMiaGrtNyQrCUQioKhqVwYz+GFMZWWz1g+xVjOJBYwM5R56GY9lrf4zidGyKd0Fusuiw2Hhh1S9wIWUFBQGM1ZTBTTrDQ9Zis2t8s2K3dWGc+nyVpJFmVjmWC7UkKSdSgl22y9iLuVNndXbZ4Yc/jwU2PrAnIYJ+2la9uO6064ozLFbkyl231t/5+h//6dfl+3WTZjNGf29tyM5a3eTBaAcM1m2d0MBXt7pcVL1xb5kzeWaLgPz/ZQgBdOl/jBF6Z6k0fvr9rMVuyHREZ3M+1W5ZwuskNFIpGcRAZGfIzkLCIhuFNuo+sqNTsJIDNDaV6fr1NImahpaPkRKTNmJJflzEiW4ZzFVDFFIW2uW/r22lxjU4+EqqooKDScgPmqQyjg3qrNVD7FYiNZZDdRTPe+36WpPLW2x0hGY7Xl8Ve3VpgZStpdu1mSbuZiqpTi+nyD6wsN4OEMiO0nM0uKaRNFoWeEPT+a7QX208Np5qoOc1W70xkT4gYxl6YLLNS23vy6G1Ppdl/bb6T8Zvt3ul83V7OZrbS3NJOWW35PEM3XXEZzFsWMQd0JaHvRjtfZt9yQP72xzEvXFnhzqf/a+q3Mo5uJjO3KOVZnMm3G1GWHikQiOZEMjPiYKqa4PF1kte2Ts3SKaYO2H5EyNKaKaXKWzmzFJhIxGSNNxlRpuiEzpTSFdNLZ0e1kma20iUXM0BqvR5duKeXmUoO6G+AEMbYfkdZ1Tg1lMI13gknO0tE1laYXEQm4u+pSd3yGcyajuTYXRrN85Omxdbtirs83eFB1Ej9KVO9lAdZuca20fWIBGUvrGWHXBvbuKPHZSpu2nwiP7ubXkZy1ZefJ2gyKqiTeg5WmS90Oth3UtZNyyHZf1/ZCWm7IajvYNNOyVqzcKbe4U27veIx7LATfvF/j5WuL/MVbZfw+k0ctXeXDOzCPbiYyNpZzujtUMqZO1pQ7VCQSyclnYMRHPmUwXrSoOQGRgKz1TqDseiXyKY2a47NUbxEJheGMzvsuDPfS9itNjy++VeZ2+Z3ZHudGc+tu3t1SylzVZbXtc2Y4g6YqqIrCeN7qpdBvr7TImhpXZgroKhALDE3hlftVhtIGpbSZBNoNu2KuLzRQUHhmKseNhWYvA9KdH3JpuoAQgrSZlFX6ba3tCpGWlwTxqWIKBWXd5tfNSh1rSyNuEDFXszE0lSCOmRlKM5a3NhUuu50G22Xt11VaHpWWv2WmJWtqeGHEqw9qCGCquL2fZanh8kevL/Hy64ss1DeZPDqZ55OXJ/nYxfHefztbmUf7eUbW/htkOjtUMrIlViKRDBgDIz6EEIgY0kYyvfTSVK4X/LpeifurDqstr2cSDCKdt5ea3BjLcWmqQMsLaXvhQ7M9NnZs5Cyd918Y4Su3y+iqynQpxfmxHFPFFF4YP7QF99JUgXLLZ7HmoAiFxYaDGwouTxce2hUjhKDc8vjiWyustpIOiyASTBUtmm7A4qxNLODCWLaXldnMTNqd+rpYT8yQFyfz2w4BW5tBub3SIo6ToWRr54Ns1sHzqHtd1n5dztKpO2Hf7EkUCxpOgBNETBXTFNMhGUMDIbi/aj+0XyUxj5b53GuL/M1sf/NoMW3w8UvjXL082XeT727Mo3KHyu457DH8EolkfxgY8XG/6lBu+0wWs9QcHyeIex0n0N3Z4REKQdsN8EJBxtS5X3X5m7urjHbKEVlLZ6nZyXx0Shpr6ZZSAJ4az+MGIV4guLPSYjhrYunqQ1twM4aaTF9t+6AIJgspdE2jkE7KH0KIdW+4QoAXxERxjKWrzFVtojgCkg6OB1WHlhtwb9Xhw0+NMlFMb/q6KAqgwMb38+7Y9Tfm64Sx4PRwuneObkCotDxaXsBcTazbzPs400u3o1/2pDsUrOWFvX/PbsahX2aiZvuJefR6f/OoqsC3nxvm6uVJPvDECMYWZZDtzKNrd6ikDFUGzl1y2GP4JRLJ/jAw4iPZzBoQRTFu+I7psPvmdqfS5m7ZZrHu0e4EpCCK8MOIuZrDbKXNt50d4sNPjXJ2JNl7cmb4nZX0TTfAC2NMLekAsXSVkZzJ7ZXEHDlXc1Hv1XjX6SItL+CVe04iZkyNe6s2K00fQ1dxQ8FoPk3VDlise7ymNni+c/OHzqyPlM6TE3k+f32JL9xcZqqUxvZDhrMWo3mL1+YbFNMmt8ttznRmS3TPZ+kq+ZTBWN7qzA0xeHrinV0vXcbyidH2zcUmsRC8PldnJGv2unRefVAnjGKEgJGsyZnhDEIIbq+0eqPdd+vt2AlrsyCOH7HU8HpDwbr/zmsnjrb9gDgW5CydP7uxzP/7i7e5W7H7fu/pUtc8OrlpSWjj9x/q4+uQHSp7x34KWYlEcngMjPjImBp+GLPS9CimDTKdwV3dN7dTpRS6qmBoCoWUTiBiCqlkJkjNCbi3anN2JMtEMb0uk9AtMVRaHg+qDqeHMgx35lDkUwY3F5usND1G8xa6qtD2QoQARJLBWIula6iKwmLNwTJ1zo3mcIPoobHkbS/k7ZUWipp8n4uTedwgJowFVdtDVRRMTaHhhdxYbHK/6hBEMXNVd935tptsavsRLT+ilDa5U7E5t6ZLJ4pFr9V4JJekwrs3VFVJuog2jnbfC4QQvaFg/cygazfJokCt7fMnN5Z5Y77Rdymcpat819NjXL08yfOnittmJvptqh3JmUwbadmhsg88qklZIpEcbQbmN7nlBjS9EBELml7Ym2iaNTVaXsBC3SEmCUalYorFmkvNDihkDC5O5PDDiL+6tdJbP15KG+RSRq/8UEjrBOWYQlonikWvvfU9Z4dQFAVdTcyHiqKQTxk8M/lOtuH0UJrRrMlqy+PiRI6LkzmaXkzb9VlseDh+wHzNYbJgkU8ZnB5K03JDnpnIc2OxSc0OmC6lmRlK03IDsqZOywvQAoW6k3yPM8Npgujh8+3MALo+aPcLCBtvqClD6+uReFTiWNBwAxpOSBj3HwoGiQdjtZUsAfzirTK1TSaPXprKc/XyJB99ZnxXAW2tx6Nm+1i6ypnhzEMdKtKrsDc8qklZIpEcbQZGfNxbdZit2Kgkq+HurTq8H4jjmLmqzd2VFqaqMJo1cIKIIIywFUFkw5feXsHQkoDbcpP9LnnLoJQxuDCapelFFFwdP4q5tdJkqpjcgvutbRdCUHfWzwgRQlDMGGha8vfnT5WoOiFfv7vKW8tNNFWhZge8cKrEUNakkE68J6au8uR4jjPDmZ65VAhBLmXw6oMaKT3kyYkcX7tbpdzyMDSVhhMylDVwg4g75TZZU+sIsIcnp54ZzvDEWDYRKpkMbhDx5zeXGc2ZXJ7OYwfxuoCw2Q11u0C81eeDzlCw1jZDwfww5i9vlfnf35rn1Qf1Tc2j3/vsBC9enuT8aHbX/w0pSjIgruEk3pLRvMVkMdW3NVZ6FfaGRzUpSySSo83AiA8niDotqDotP8Tp7PF4bb7BNx/Uk+FcXsBozkQJBYamMdYpJ9TbIflUkr1o2BF2EOIFEau2x3QpRSGlM5Y10BQFVVlfTum+eY6tCbBdT0jXe3Gn3F7nvbhfdbi36vDmQpOFusvl6SLLDR8vjLhdbjORN8mlDEZz1kMdLStNj/mamww5c0OW6x4XRrOcGU6TSxlYuoobRLwxX6ftR8RCkLcM8injoSA5Xkjx4afGaHlJd8lX3q4QxgJDU7l6ZXLdnpetbqjbBeJ+ny90Fvt1RdFmvL3c4nPXFvnTLcyj33F+mBcvT/KBC+vNo13/RtsP8EOBqStkTWNdR4yqKD3DaLJDJflZt7uJS6+CRCKRbM7AiI/JYorxgoWuqGRSGhMFi+WGy51yi2orMXsKoGoHxICiwv2aw1Da5OxoholCijfm66y0HaJYIQgFigJvLTU5N5rn7EiGtGUmUzirNvdW7aSNteERhIl3otb2MXWNkZzJ86dKvdZZN4gotzxqts9IzqRmB9wut7EMjYYb8tZSC1NXaHsRiqp0vCAxIzlr0wFbl6YLvZ+7O7ujG1C/dqfC7bJNKW1wp9xmppTqlYHWBsm1t877qzZhLHhmssDNxQYrTW/d8251Q90uEO92KFjTDfjT68u8dG2Rt5b7Tx6dKaUT8+hzE4zm+n+vrn+j3g5YaDhMFlKUsibPTRU4NZwhZ/XvUNnJTVx6FSQSiWRzBuYd8fmZIjeXmizXPcaLFjOlNK8+qNN2IrwwouEFKEDe0kjrOmpKAQHPzeSZLKQTE+FUgVLGpG4H1J1EsFw+VURFxfZDIgFztcRP0fZDvvkg4Fv3qqR0jXLLYyRn9YaAdUeEu0HEg1UbQ1XxwpCUoVFuurS9kMm8yYXRDNOFFKdHsuRTGi0vwgmida2ta+kGvYWamzzfVGGLdL+CpWtomrppkOyWRLwwwg9jbizUMXVtV7X37QJx1tRwg4i/fKuM23kNRjv+mG52ouUF3F5p85e3ynzxVrnv2vqUrvJdz4zx4uVJnp/Z3jza9W/kUhpxTTBRtEjpOsWM8djeAulVkEgkks0ZGPGhKAo50yDICHKmge2FVJoeupbU8aeKFmEElbZDuR0RRRFnRrOcHc5RSJt4YcgLZ4bILDSYr3s8MZ7FCWL8IMaPYgrppGwxkjUZzhjcLrep2z5NN2RqPMVKy8PUYbbcJmWoOGEyCKvS8jFUlUvTBa7PN5it2Ghq4p+IgEuTedp+xL1Vm6GMwbefG8INRUcUJC2+3fLNZlNEN3oqTg+luTCape2FPDdd4NnpPGlT7xsk15ZETo+kGc6YPDWR5+Jkfsev/WaBeO1QsLSp4UWJqFpuugxlTEZyJjcXW/x/XnnAK7NVas7W5tGPPTNOdhcZhlLGoNpO/lsYyVnEMaRNbU+yFNKrIJFIJJszMOKjO2SslDYpt31mV11uLjW5vdKkavuoSjLn4VQpy6khBZRkA2rN8XlqMs9iXVC3A/Jpk7QTcno4w1AmmengBYKLU3kW6km2wQ0i5qouC41k4uV83SVn6WQsA8+P0BWVlYbHhdEcmgrllssrs8n01OGc2fNSpA2NB6ttvnm/RiljcWulnWQRNJWFukOlGTBdSpE2Nd5zdohLU4W+QW/jxNHL03menS70tr5enMyjqv27Na4vNKi0PC5NF1AVlacmcrvuYtl4pmQomL9uKJilqwxlTMbzKeZqNn92Y4m/vlvllU0mj5bSBt/zCObRlNGZMGpp6KrCVDHddwbKXiI7XyQSiWQ9AyM+ukPGwjCi6gQULRVDVxjOmlTaPvcqNnYQUUobZCwNU9dQUWh6EV+9s0re0hjOWjx/oUQhZZAyVaaKKdwg4rW5Bss3bdKmTimtUXMCLB3ee6bETDHNZMHkzEiOIIxYbvhYhspX71T40zeWMHUFVVVImyppMwmICzU32ZcSCSptn6odMpZP0fYFD1ZtQhSiOOZOpU0kIjKm0evE6JZY1ga8SssjjGNmSpmeobXuhOu2vm4szSw3XL50q8xC3WG1HSAQjOZSj5UVcPyIuhP0HQpWs31ul1v8n9fmee1Bo2cIXosCvPtMiR961wzvvzC85eTRd763T6Xtk9JVnp7IJ3ts1gT+8UJq37tQZOeLRCKRrGdgxEfG1HD9iDdWarT9CF2BSAhuLjWYr/soIiZWoGH7PDNZQFMS/8ez0yVqdrLITFUVFuoumqawavvcr9o8WE12qQSdaZ/LLZe6HWJqKiutgMmCxbvODHNxMpnJ8cZ8gzsVm5WmQ7UdkEvpWLrGs9NFLE1FUxUsQ0UhmcdxZabEzaUWCzUnmeUxnGZ21SGMkjHwi3X49vNZdFVZZ+RcG/CaboCivDNxFNi2E+Peqs3bK22KqeQcaUPj+VPFXWcFthsKNlux+f9+Y46/uVtlodF/odt43uK7nh7j45cmeHI8u23WQO0sbWt7IXcrbe6Uk4mm5Zbf2xJ8kAxa54vM9Egkku0YGPHR9qLOLThps52v2YxkU2iKgq4I3BCqbZ9CKpmBEQsFiHhzucWF0SwvnC6hKEmAv7Xc5O3lFi034K2lFufHcmRNnTuVpEOlbvs8PVGg3HKJhGCuZrPa9rlbbrPU8FhquChK0lmTNlQW2j5/cWORy6eGURRQVYVYJN6UKI45PZQhY6qcHs7y7FQB2495c6nJZCmNoSbG2JGcuS4rsTbgzVVFsuuks5+m36yRzVAUlYypkTbemQUymjMpt/wtg0scJ3tm6k7w0FCwWAhema3y0rVFvrSJeVRVEtHx3HSBH3h+iiunSpsGsOTnCYgFjOUsTg+nUVWV2ystbD+imDJoeSGz5TazI5kdnX8vGbTOF5npkUgk23Gy3wXXUHN8lpsufhSjqVCzI8YKCpPFNJBsYo2EIG2oLDZcdFXlfU+MoCGSOR55C1VVGQcqLQ8niKm6AV4Uc3ulRdbSQEkyLE1Xoe0FlLIWV2ZKLNZdFmp13FCAmrwhu36M3dlNIhSFCJVK26PphhTTJuWWx3vOlJgspni3MsTFqTw3F1pU2gEzpTSqAmdHcyzUkm2txbRBHMcs1ZOpqW4QoXayHbqm9uaBdG+kU0WLthf2Oko2Lq87M5zpmVLHchaNjtDS1GR3zXzN7Rtcws5QsGafoWCLDZeXry3y8rVFlje06nYZy5lkzaSbZtX2CaOYhbrHzFDQW1XfLdV4YcRoziJtqKy2A6JYULMD0qbGeCEpEeUsndsrNZabPuN5i3urNllL3/T8+8Ggdb4MWqZHIpHsnoERH6zZyBrFUMzofOTpMVYaHq/N1RjOWoRRCKj4UYwbxrz2oMaF8RxNL6Tc8nsB6sxwhomCieMHPH+qQLnpM5QxcPwYRMzFqQJPj+do+0lbbBgLcpZJLg1vzLt4QUjT8UkbKpoqyKYM3nduiFUnGaNuBxEtN6Tc8nhupkgYw82FFverNoIkYOZSBq4f4QYx1baDFwjultuoqkLOMtDUh/errL2RtrwAIZJb+WzF5uxIZt3AsvFCio88PdbzjFTaPlPFFDcWmpRbLipqz2Tb8kKKYeLnaHsRcRz3lq/pqsobC3Vefn1pU/NoMW3wbWdKTBYsFuout1ZazNeS9uMnJ/JkTK23ql5XVRw/ZKXpJSIucCh2RsZvDHZjeYsPPTmKrircr9pcni7idvb7HGRwHLTOl0HL9Egkkt0zMO8KsegEbUsjFvChJ0f4vitTlFs+z58uIYRgse7w/3t1gWYjROusmb84nieMBNcXGkAS0MYLKb7rmXEK6RpV26OUtnh6Is837tfIpXSmi2ne1SnT3Fu1URWoqQFV28P1Q2IBQlEopA0MLVnD3vIjJgspHD+m0vK4OJlnOGNh6SrPnypyfaFBLGImiilmV1pMldKUMgZ3yi3maw6LDYe6EzCSNfnI0+O4YUzK0Dg/mmWl6XGn3E6Mp1HMzFCGV+45IGAsn+LVuTptL1met3ZUezdg5iyduhNyY6HJ/apNIWPQsBN/RjaVCIO5qtN7rVfbAZ+/uczfzFZ59UG9r3lUVeB950c6k0eHWai7fP3uKmEskmm0gKkppHUVy9CYKqaYLiVi6vZKNwOTiAfoP9pdURQmimk+8MQo2Qd1vFCgqwqaqrDSdKnbAUNZ48gHx+PmoRi0TI9EItk9R/tddw+pO0HiP4hidE1N5nJoWi+bcW/VpmoHCEBRBKt2iO2HvDZfZzibpPuDSPRS9N05F28tNam0fGbLTRpOwKmhNFEsaPsR+ZTR6yppB2GSFUCQNpOOGBXB2aEspazFeN7i/RdGuDTl8417NUxNo5TR8cIYxQsZy1ss1V2+cGMFL4yIEAjSyQj1lseDVZsgFoznLSLg+ZkSOUvvm+2YrzlkTY2mG/L1uxXafkAhneUb92q8PldjLJ9kPZ6dLna6aEymSynKLZdCxuDbz5Z49X4dXYPxnIXjRVRaPlEs+Ma9Kv/Xtxa4t9p/bf2poc7k0WcnGM6arLYDFuouXhiTNnSCCExdZTRv8eRojnedKfHkeH5dwN14sz4znOn5cfoFu7XBMGmDtpNuojhmZih95IPjcfNQDFqmRyKR7J6BER/lpofjx2iqguPHlDueg5WmxxffKnO73Ga+6tB0AgxVJ458IlXl7kqLsdwIF6fy3Fho8sZ8nXLLo+n4vD7fRFEEi02X5ZpDuRXyda/KzFCKmaE0D6oOlZbHRMFipenj+SFNN8ILYsIoRlEUnCjmVEojjEFV1d6sjuWGS9ML+Zu7q5i6xnDOAAX8KPE5zFZsKi2fuu1TtX1MXSWnKUzkU+RNnTPDmd7emJ7xtCYYySbGU8cPeWO+QdsL0TyVm4tNHlQdMqbGfN0DRWEsn7Shlls+8zUXhMJK3eXLb6+STxucHs4SC8FL1xb5+t1Vri82iTZZW/+xZ8a5enmSyzOFnoiotPw16+mTabKXpnK4YcxI1uTsSJbxQuqhW35XDHXnlKz14/RjbTC8vdIiEsnY+buVNu1tdsccBaSHQiKRnDQGRnyINf8XRO//a3khLTdAQ0FFEMURi3WXphsyk84lWQsv5PM3lnl7pcVkIYVlqISxYLbiMF1I8dZKExVBytJIGQphJLhTbmJpBg+qNnfLbequzzMTecptj3LL48xwhjgWGGpiem25AbOVNnEcc32hwULd4fZKm4ypcW40GeqVMlRKGZN8yqDc8kjlVN53YYRV22Op7uHHUHMCrpwqcnYkaUldmyXQVbUX0G+vtCikTZ6ZLHBjoUnd8SikdFRFJWVCFMW9IFd3gl6JotgyGc6a5FI6//tbc3zutUUqbb/vaz6et3jPmRL/z28/w6mRTM8oavshGVNPPCEKlPIpFuqJcfa954a37GpZaXrMVtrMVmxyHeNovzklG7+m5YVkTQ3HD7mz0mKx4ZExk4Fj3dfkqCI9FBKJ5KQxMO9io1kTVQEviDB1FVNTuL3SwvFDanbAX9+t0PZD4ijGCWLCGB5U24xmTEQsuLXcpOEkI9kVVeGJ0SxxJGh6QWfGh6BScyllLDRNwfUFz5/P8aBqs9KwUVSVt5YaDGVMsqaOqiikTI17lRafv7GIisLdSpuLkzmuL7Zw/YiFustT4znKTY+0ofHEWJbVts9q22eqaCUekSDiVCnDeC7Jtqy2k2mtd8sthBCb1t97O2DqLsM5k+dm8ui6yltLLQxNZbKYxtSSUed1J8AJI2pVn7vVNi+/schrc/W+r3PG1Hj36RKFlM77L4wkJt+OllhtB7y13ERXVTJmyPnRTFJSmKsBSelrKyHQLT/M1WyWGh7vOz+C44e9PTn9/BAb552AIBKCIIy4eGYIS1ePfCZBeigkEslJY2DER8tPTIyaphJGglvLbc4ttWi6AX4ckbU0VAWcIETXVDKKStsLaQUhFdsjpetoWYWv3K5ALKi2fSYLKeJYRwMqTjKiu5jRmClmSJsaNxdbrLY9hKICggdVl/G8RcrQaLshD6o283WHuh2QMXVmVx3ulluAgqKq1NoetpfmqfEc7zpdRAiB40dYusp43mKikOL1+QaGphHFCqqqEsRwu2yz1PQ5P+Lw3ExhXcdLNzBvDGijOZPRnMW9aRs3iCimDbwwWbq30nT54psrfPFWGdvvbx59/lSRD1wYZbJo0XRCFhsuLTeimDUopAwKaQMviBjJWp0SkE3bCzuGW7XXibKVEOiWH86NZFlqeNwtt8haOm0/pNKZ27Gxa2dtyeKVWQcUuDJTwvZjarbPzFDmyGcSpIdCIpGcNI72u+4eEkYxuqJiGipNL8APo15ASuvJnIzZio0QgjhOJoCWsiZjeQs/FKhGzFzNwQ9jRrIGyw2HlK4yXkjKIF4skj0vfkzKUDg7kuHGQgM3iPHDKNnFYvu4YUwQge35eCH4UUzDDQljQcpQWW37jORSmCoMZU1KGZ3zYznaXsjf3KtSd0LGCxagUrUDml4iFCrtNitNhzAUpHQFTVFYqjv4UcxozuoZFcfyFssNt2cI7XpDANKWzmQxTdCZ1fH735jjpWuLvL3S7vuarjWPjuSsXlml7Qc8GxcopnXGCylODSWG0DgWzNXcxLfgBizUHNp+yGo7YKnh9YagbaRbOqm0PFpeQCySLNCZ4QwAlbZP2tCTrh0/pO6EPVPm2pJF1tJRFHCCiCfGspwqpQli0fPx9NtxI5FIJJK9Z2DEh1AU2kFIzUnMjWpnjXzW0qk7PnNVlyiOKaZ0cpZG3fZQVQXXC1EUNemoQNByQ1RFIYxDIgH3Vl2cIMT1IyxDI5PSGS+kWWq4PKi5LDUdoliQs3SWmy5LNRcviomiCE3TMXUFEAgBKUNnOGMyVUwRA5dmimRNnbsd0+hiw2UobbLS9LB0BUXRqNk+rh/R8gIsTWGh6RJGSUfNRN4CNREJThD1JpR2DbYA50czvHBqCMtQ8cOYV+5Veem1Rf7y7U3W1hsqH336YfMogKlrPDlukbE0LF176GvXZltuLTV5e6VNKW0SxR4pQ+2Jo42tpUIIXptrEHZG2I/mrHVD0+pO2MkYwbmRLG4Qr5v10X3OrJmcqe1HnU4gl5evLRFEcW9PTHepn0QikUj2j4ERH0MpnTNDmU57p88z4zmemsiRMVT+8PUAQ4OJQoaK7UIMp4bzVNoeQ2mTJ8dzTBZTTJVStL2Ym0t1FDUZVOZ4EYaiUiya2F7IE6MZCmmdaw8aqCRzNBqOj6ooqIrSGyCWNlRs10dRFNKmwVhO5/xIlotTBaZLaSq2z1DGJIwEpqYxMZxiqeHgBTGWngwhWao7hGHMvbqDbmjMjKQRCkwULNp+hCLA7izGe2IsS9bUErNmuQ2CZPjWqo2Cwrce1PjD15c2nTz63HSBT16e5LueGSNjvvOfjWVoZE2NjKlj6uoa4eA8VOpZWz6otDwUBVodz0za0HqP3biFd+0QsRsLzXWln664KKZ1cqs2ThChq+q6WR+blSyuLzQIophnJgvcXGywssnPLpFIJJK9ZWDEx8xwloylUW575CydZ2dKXBjLsdxwiUl8Cy0/IKOr6JrGs1MF3lxuMV1M4YUxq22fsbzFqZEM96ptghgerDqM5kxOD6eZKWVZaXmcG8lTt5OFZm8vt0jpKilLQ1WS0e0tL0QBhEj+ZAwN01CZLqb59vMjFNIGuZSOqqqcHcmQtXTmqg6OHzKaS+GFEYaqcWOhgSKgmDGpOwGKgOtzDdKmzlguxXBHXEwW09wttzg9lKbc8vjy22XuVNpUOyWgctNndpOZHEMZg088N8mLz01yZiTT+3jK0MhaOllTQ9+wWXanMynODGcYy1m8udTC1DXqTsBK02O8kHqotRSSIWLdIWcCsW7mynghxVg+yYbsxpQ5lrcwNJWbiw0MTd2xkfO4Df2SSCSSo8bAiI+hjMFw1kRBYShrMJQxEEJwt5wsiHt6ssjdcotSNslgzNVdEIIYQcP1CcIAXVOYKaYYzaWYKiabbadLKZ6dKqBrGmdGMpwZTnN7pY2pKXhhRBDHuGHSYYNCMjysaLDUcAnjZOS7oqgIoOmFZC2dtKlza6mJ7YdcnMgxXUqRMjRGcsnOl7oT0nB8LEOnvGojOj/fYt1FU6GU1simTO6W2yzWHdKGynzN5fpCnesLTe5VbR5Uk+ffiKrA+y+McPXyJO87P9wTF5ahkTN1stbDgmMtO51JMV5IcXmmiKoonBvNYXtBr2vFDSI0lYeGiF1faCAQXJousFBz133vfhmO7URCd1Bcd15I9+/bcdyGfkkkEslRY2DER7nlY6galyZTVNoB5ZbPStPj2lydr9xexfEiRvMm33a6wHIz5P5qi9JQimrbp+FFGKrK7KrLWN5MfBZBTCmjM5y2uDxTZDSfwg0iHqzafOt+jbeW2zhBiO1HGLpO2gBT09EVge2HhJEgFIlZMmtqmFqW2ytNZistNEWl0k5KMm+vtHj+VIkPPzVGztJ57UGdVx9Uma+5uEHUK3fcXvEI4mR5W70dcGEiR97UqbkhigJvLrV45X6NtvdwtwokpZofemGa731ukuGsiRCCthchgoixnMVU8eFhX/3IWTqqAtfnG/hRxOnh9ENL6yARC2dHstSdRGy0/Qh71Wa1HaAqD++l6X59EAkWai6qAm4QcXultWn2Ybnh8sW3yrQ7ou7DT40yUUz3Pq+q6iN5POTQL4lEInk8BkZ8tPyI2dU2by0LTF2h5ScGzOWmlxhAhWCl5fHmso0XRlTskFJaoe4mQkE1FKpNH0NNzJxLTRcvilDVJu8+P8zZkSx/fnOFW8tNZqttaraLomgEUYgfBQShxpkRnZGswYOqg6qA0ekAUVW4W26TTRsoQDalM55NIVBIaSoLdYfrCw1GcyZemIxoF0DVSUaaq4qCpkAQCxpOSM32uFNpMzWc5fZKm6WGS58kB7qq8OR4jlOlFM9NF/jQU+NMldLkTJ22F3T2wfhcixu863SR0ZzVM2tuVmoYy1vMDKVZbnoYHVPvxiFg3YxE0w2YLqWw9KTLp9L2ewE9ZWhcGMs99L3XjkmfrzlEMZtmH+6t2twuJ6bWpWabsyOZdeLjUZFDvyQSieTxGJh3zayedGqkDAEoZPUkiEQi8Q+M5pP5FA3Hp5Sx8COXuZpDywuo2SGhEBgKCJGMaDd1jZypEYYxby81sTSVt5ZbeKHA1FTyaYsojrB9haxpoKjQdnxsBdpeQCTAjwSWQZINCULGi2liIAxibD8EVWHVgUDAfNXmz64vstL08aOYasvF9iJcPxnTPpzVCUOIEPihYKnpcnvV7ftaDGcMnpsu8MR4FjcQTOQthjMWpYzBTCkJzpW2R6XlJxt9mx4Nx2csnyKfMrYsNSiKQsrQGM1Zm2YG+mUkuntw+gX0jeWT86PZzth4ts8+CEHLDai2kzH0/bIwu0UO/XoY6YORSCS7YWDEx3zD537VIQgjDF1jru7x3CnBU2M55lZtQhGTtVQ0VaXcdhEiThbKiRjPC8hnLBw/YrnpE0YxdhDRdFSemcwTxYLZSpsgivCCiLYXMZY3SRsaQSjQVBUnCGj4EWEkaAcCtbM1FwGqquEGMTcWmxiawlPjOc6NZDg/lsPxQ+brLt+4t8qX3l7FDwIEKqoSE8egq2BoKi03pumFNDcpq6QNlclCiqfGs5wbzfKdT4zghYJr83WylkbGUqjZPssNl7F8Mm8jjAXljh/CCULaXsgzk4VNg/3aeRxNN2CuKtA19aHMQL+MxHvPDW8a0Pt5LHZS3jkznGGsYPHWUgtTV2k4Yc/U+jjIoV8PI30wEolkNwyM+Gg5yf6RXMqg5UXcXmryWilDPmXwzGSBO+Umrojxg5CaE7La9Ci3fMIwIhBqz4ugqQpDGYuFmgOKoNx0mV1tc2WmhK4qRCJmKGNQypiMZQ0Kpo5QBN+8X6PWDkBJulxQIGuq6EryNWlLx1RVdE3h3HCGc6N5zo9luTZX563lFq/P16l2JqE6XshkwSQSES1f4Ll+37KKQrJAbSht8MR4hiiC6WKaJ8fyjOQsri80MTWVuhOgKQq3lpp8fbbKE2NZnp8p9qaqmppGMa0DW5caugEojGMgEVjFdDKno3/G4Z1DbxXQ+3kszo9mNy3vrL2FzxTTqALOjeVx/FD6M/aJQfTByGyPRPLoDIz40DQVN4iStlTgTsVmZLHB5ekiThBSa4dkUjqrdtJ1EQlw/Ih82iCjC0xD5+JEnrurDgu1Nqqmkrd0QiFYqrt84IJGKWOSMTQ0XeX+qkPdCVht+9hewGo7wAsFWmf2VhRDGAsiBEMpEx2VfFrn9FAWOxAs1G1KWZ2m66OrKiqJr8P1wqQM0wz6DgGDpPPl0mQeVUlKLC0vwtRUsmmDyVIaUHh7pc2dik0pbTJfTwahWYbGzcUm9yo2TTfkQ0+O8r3PTfYd0NWv1NANQDOlDG/M11lueggU6k6D5zviApKMxBNjSVvsE7l3JpVuRj+PxVblnbW38JYXkk0ZuEHUNwsj2RsG0Qcjsz0SyaNz8t8hOuQsLTFlRhGKUHlQcygtNVmuu9yv2Sy3fZS2R932CWLBUMak6YbYboCS0snrGlOlNFHnFl9tezS9CMtQqbQDPn9zBU1Lbj2u3+lCMVRuLrrUbB87iPEFaCGogK5D1tQIo5hCSsdQFXKmThiFVG0PgWC17bHU8Li/2sYOIvwIunoj3iA8FMDSFdKmypmhFIoS86DqsdLyyJoG50czTA1lKTeTaaKlbFc8CAw92dJbq7vkUzpjOYuWF9L2Iy6M5XZ8g10bgMI4yZj0uwmPF1J8+KmxdaJmq66V7ZbjbQx4LS8kjGLSps5i3WaqmOaJ8Sz5lCH9GfvEIPpgBjHbI5HsFQMjPvwo8UboioodRFRaPoqi4ked0eaawkLTx/NjFGC17RPHMWbK5MxQGkPXuF+1qTkBQRTiBBG2FxDHOkEYU217xEKh6QaEsaCQMihmNFw/xPVjohgMBUwdcqaOH0Y4foShKjS8gFTH9/GgFhMJaLgRbhBStQNqTrhpliOlq6R1hVzaQEUgUDBUhXLDo+76DKsmURyhIMhbOmlD491nSgxnDJpuUoa4Ml1kopDi2lydpYZPGMfkLH1Xt9duaaVbZsmYKnfLba7PNxjKGuu+19oSy8Zppt3bY7+U9sZb5VaipOWFvNrZvJtLGeRThryV7iOD6IMZxGyPRLJXDMxvy2QhRSGVBNyMpZHteCeG0xaQbIv1ghARxxiGhqlDSjfRFMimDBwv5O3lFrGAphfR8kNQVYI4xgsFy81khLofx+QMFS8MURWdlKkTOyERSdZCF0lHSiAEiqoSxIJ6O8DV48QrISASgsW6R9DPyAGYmoKhJWUYRFIu6QZcXVNwQ0HNjQhDwartE0WCrGXx3nND627/3exDd6vt0xP5vgvn1gqBjJHMICm3/N5gLlVVWWl6vDbXWLe63tQ1gjhmupSIibXZDUiEx1duV7i/anN5poTb2T+zsXSyWUp7s4CXTDvN0PZDzo1ke3ttdhIYZR1fslMGMdsjkewVAyM+nj9V5MmJPOWmSySUXonCsnQadkS57dN2k3X1dSfE0FRGsip+DHfLNiCo2T5+lLSy+qFAU0E1FFRVQVMVbD/EDWKKqTRVx8cN2vhhjKUqGJogCKGYNtBUUIRC1jKoOT5tL8L2QxKb5ubkLY3xvImpCubrAYoiiBVI6wpPjWfJGhpjhRTLDYev3vZRhI4TRmiaQtPx8MKYC2uC6cbAPVFM952DsVYIzFVtHtQcTE3F0BRWO7M5Ki2PMIqZGcr0Vte/58ww8zUH2496wqQrJAC+dKvMq3M1lhs+y02PcyNZRnImOUun4fistnwKaZ3VVkDTDXacuVg/wCxet+tlO2QdX7JTBjHbI5HsFQMjPoQQaIogbapomsZQKln3fq/cRlGS9fVNJyCMY2IBfhjT8iKGcybDORMvCHEjHdf2CaJk/LeuAkIhFskI9XzKIGMKarZLww6xdQ1dVVA1BSUGK6Vj6Rq6qlJMw0ozER6h2Fp0aMB4wex10ay2fLzQRZB8XZRSGcqYvPfcMHUnxI8EU0NZHD+k6vhMFFL4QuEb92oPDfza7LXq3v67y+jmqjbnRnNU2x62H3Ll/CivzK7y9TsVLk2XaHkBQtDbFNz0Al65t0rGSDYE3686PDmew9TV3nbdlhcyXcxQsAzaXoAXRVTaPnUnJGWo3K/aBOVk4+zlU4Vd/Xs/6q1U1vElEolk/xkY8fGlW6tcm2/S8gW27+KHJpauEaHgRzFNN0BVQVNVYhEnu1bcEMvQyJsRVcdnteXiheBGHdOoItDUCF1X8IKYMAwYy5m4fjLEwwsjQgXiGDQ1CeqOH4CiUHUUVu1w0/OqQLzm/6+3fVRVSWaM+BG6Boam4YURuiK4W27TdHxUVSdjqgznDDK6xVJDxzR0hrNJCWknwXTt7b/pBjS9gJWmz1LTw9JUMqbOzcUGADnLZLqUZq4mGMmajOQsHD/k9blkS+zt5RY128cJY26ttPmOc0N829lhIDHc3l5JskPDWYOhtMlMKZMYVqOYU0NpihmDuh1g6Zvvk+nHo95KZR1fIpFI9p+BeWettpObfBTFhGFMw/F55d4quqIQxjG6ppCxDPwgJBaJWIiipIPEDnxqdoDtJ9NGIREGoQBVgCKSTa/VtocfJuZQL4SQ5AVWVYgjcMOktLKJlQNDhbGciROEeKHACZIx6kJNvlfD9rDyacJYkDZ1bC9CoODFgns1Fz8U6HqIisJoPsmELDY9FmoudSfgzHBmR8F07e3/lVkHFXjf+WHuVto8M54liGGuk+EwO4FaV1XOjmQZL6S4vdICFFKmloxR9wK+4/woc1Wb4azZy0Jcmiqw0vKIYkHG0LD9kFfurZKzdE4N5QljiGLBSM4inzIe9z+BHSHr+BKJRLL/DIz4yFgarh/RcEJUBTJW0t4qBLTdEFVRyFsqkW6wavsEEWga2F5A21WJomRo1tr6SAyggOsLWr5HDDhhIhi6FsWw98D+KCSiYyJvMl3KMl0weW2+TqXt4YeJwInj5CwxCrYXEouYnGUiOiompatoKuRSKl4I06UUFzoljmJK5+yFYaq2z9mRzKbBVAjBcsPl3qpN1fZpuiFzVUHW0lEUcIOYmVKGQsZivuYylLHQ1GS8+doFcJBkD/woYqXpMT2U5nY5Zq5qM5ZP8dREvuc5SZs6F0bzTJfSvD5fY7Xlk1VVhICRrMlYPnXgIkDW8SUSiWT/GRjxkdZVihmDKI5x/BglFgxlTW4tNWn5MQqCWIAiYrwoGQJmqMkW1VAEtHzoN7g8iNZ/XGz4334oJHNH8pZGLqXjBskQMCcIuVuNyKdMhKIShA6hEEQd8WJqCqqi4IQxvh2gq2oifkLBdN7E0HUMXaGUNdFVhUrL517NJlhq8uRYjoypcXulhRfGWPo7Jsy2H+EGEdce1Hh9oYkfRkyVUrz//AjvPlPqPSZn6TTdYJ0nYrMFcO85O4SiKIlAKabQNZXJYorhjEEcx5RbPpWWR8sLmKslP+NoLsWl6WR8ux3EXBjLSBEgkUgkJ5CBER81N9mEiqKgaoCqsdRwWWp4+EEIqElnhJ4ID0uHMAQniBEiyTyoAkTcyWZ06L9JpT+aAuM5k5mhNK4fsWr7VFo+KUNFNxVGcxblpkPDTXaQBHHyNSlTBSHQVRUhYnKmQSgEiqIylNYZypp81zNjeH7EqhOgK1BueRRTOkNpk8W6S7nl8ZW3KzhBRKUVMD2UIoxigjimmDKwg4h628f2IsI45s5Km7PDWc6N5h5qN93OE6EoCpemCox2hpW5QcRc1SEWcG2+yVTb58Zik6abmFRPD6U5M5xhrursaLHcXrW/yrZaiUQiORwGRnxoiKR8oUBK1xjL6ozmUlxfbOHHEESJyVQJk6xFEL6TvdB0hTgUGBqkLY2WHxHGSUlkJyhA3lI4PZTmyswQC/U25VZAGEUIoSDimIyhsep43Kt5tN2QSHRKNypkdY0nxrKMZC0WGzZeEOGG0PIDUobG6aEMxbTJfcem3g7ImyZLDYeVhodlaEwUUiw1HBbrHqM5izvlFi0/pNJ0qTo+lyYLNL2QKIyouhGmCnYY8+W3y8zXHT7y1BjPTiftsd1BYpCIhjiO+dqdCpDMBhkvpFAUZV354vZKMh+lmy25tdzi7ZU2pbRBzQkeEis7WSy32SAyRVF2LCr6bdft12p8EpHCSyKRHCb7Lj7+1b/6V3zmM5/hp3/6p/mt3/qt/X66TQmFgqIqiEhBILBMg7SukTVVwkglimIikqyGgN7MDdPQaDpRYvwUkDESw+h2wmOtPUQADU9QcxNDZdOLWG37aKpCxtQwNZ1YKMyuNGl7ggDIGgphJBjOGpwaypI2NFAE50ayrNrJXIwo1iikdKaKKYYyBvdXFZwg4nalSbXlk0/p2PWQhYZOFCXj2UtpAyeImK200RSFcjvgxmITN4iZKFogBLaf7ERZaflUnRARw1g+ac999UGdajvAjyK8MGax7nC7nAwmuzCa5SNPjz3UyruxgySlq7S9kCgSuGHUWzq3m8Vy3UFk37pf653nPWeHEhPrDmd19NuuOyjiQ84zkUgkh8m+io+vfe1r/Pt//+95/vnn9/NpdsRozuTscBpVVai1fJ4ZzzIznOHGcoNKO+g9risYuh5R14uISNpdgxhW7J0VWvppk1o7oKEEKEIQRRBGAiEiTK3blquQSavUnQg3EKQMldPDGZ4az3Kv6tByQxpxjKooZHQNXVHIGDpBFOMGMaeGUuiawqv3qzhBzFBGpenGNFyXS9NFluouAsGlqTw120dVVFbbLn4YoSgKOVNjOGNRSOu8Pl/HD5IZGw03GfKlKArVdrf11qPc8tA1hVLaABTaXv+tsRs7SJYbDpqiUHd8MqZO1tK3vIlvtcNl7XkURellT3Y3q2OHKawThJxnIpFIDpN9Ex+tVosf+7Ef4z/8h//Ar/7qr+7X0+yYpycLXJwqUm65pHSNQiZFzQk5M5xmvuqgqhG2J3qDu7qZC7ejQrZoWNkxdiDQVdDVJPuiqqAiODeSYyJvMF+1ieKYlAqWoXBqKM3FyRxhJFBQEMB83cMJIrxQEMUx8zWXmudj6skSt+limrYb8vZKCy+O0dRkr42uwJmRDO86M8ST4znemG/w9kqLcjuNIgSFjImhqWTMZIFete0zX7MJnWRr70Ld5anxXK+LZTRvIWJBKOJkcZ4fM1EwcYN3MhmbkTI0npnM92Z4pAxty5v4Vjtc1p5HV5XeY3Yyq2O323VPEnKeiUQiOUz27R3nn/7Tf8r3fd/38fGPf/xIiI+Lk3k+9swYX3pzmbIaoCoxD2ouhbSJaWi4YUxKT+ZzROzPXVgAKUPFC2IUBYopHV1TCcKQm8s+kUj2vxiawjOTBb7j/CiOH+JHUS9QmJqKZaq03YgoVnH9ZIdLue1xcSrPcDYpnSw3E8Fgd7pU2l7E82fyvP/CSM+X4QYRlqax0nQZzacYyeoM51PkTI1SSuftZQs3EsRRMsTsqfFcr4tFVxWGs0ZnwJjD2ytthtIG8zXnoSmqG4XFdCnFSM5aN8Njq5v4Vjtc1p6nO5p9p7M61m7XHbSZHnKeiUQiOUz2RXz8j//xP3jllVf42te+tu1jPc/D87ze3xuNxn4ciUo7YKnhUXUj7lZs3lpq4YUREwWLjKERhQHVcHfdKztBV96Z+WHqgEi8I5qazOfwY8H9mkvTC4hFMvVT1ZTET9FwCKLOsrlYkDVVTF2j5Qa9QWVDWZNC2iSKkyCd7DOJKGUSYTBftbl8qoSlqTw7XWQsb7HS9Fhpeli6zt+6VOLmYouJosVY3mK+5uBHoGkqxYyJ4oaMDlsYmkrbj7g4mQcSQdFdLJc2dYRQNk3hbxQWlq72DXy7vYlvZlTd6ayOQZ7pMcg/u0QiOXz2XHzcv3+fn/7pn+aP//iPSaW2N7B99rOf5Vd+5Vf2+hgP0fJCmo5PGMWEYYQXRBi6QiwEFTtgtR3vebZDAdKGStbUCUUMIqbuJGIijMAJIwytM8CMZEOuG8VcGMoyWUgjYoGuKuiail1zO2ZIgakpWGaM5wcMZ1MMZ3Umixa2F+KHcHY0x1LLJ2OqnB3NU0pbDOdMzo5kKbd8Xn1Qp9LyeFB1ABjOmVyaKnREAr1x6U+O51hp+UmWI2fgBhF/M1vl3qpN1tKZr7mM5qwtU/hCCNwgotzyqNk+Izmzt95+beDb7U18o0fk/Gh2z7s1HqUjRHaRSCQSyfYoQog9jbl/8Ad/wN/+238bTdN6H4uixNCoqiqe5637XL/Mx+nTp6nX6xQKu1smthXLDZf/9pVZPndtgZWmixfGKEpym98vFJLMh66CqatkLT3pclFU/DhGBXKWihsmy+wURSFraJwZSaNrGpCIo6ypUXVDzg6n8fwY01CIhYJKzGQpw1NjeVQVwlhwf9VGU1RsL+DpyTzPThdIGRp+JLB0ldW2T6Wzifb6fIPJYopLU4VeRuTVB3XCOKbthZweSpNLGVi6ihtEvDHf4F7FpuEFfOyZcbxQ8NREjvOj2U0D7nLD7duR8rgBebnh7nu3xmbPsZXAOIhzSSQSyVGk0WhQLBZ3FL/3PPPx3d/93bz22mvrPvbjP/7jXLx4kZ/7uZ9bJzwALMvCsva/3jyaM2k4PosNj6YbdbIc+yc8IPF4GFq3zKIwmjOoOwF2EKPRaeftzPNQFYWUoZJLJf4MQwddVXHDZIlcydJYWLVxIzA1gRcpXJzIkTZ1IhEjYpVLUwXm6y53VlqU0ib3qw6XT5UopM11i+IUBRZqLiM5i0tThYeMnbOVNi03ZLUd0HAjnj9VZLXtc6dioysKy02fa/N1Lk4WyVl6L4U/1gnKd8rtXlBuecmunO7k0pSh7Ukm4HG7NXaSodiqxXczgSG7SCQSiWR79lx85PN5Ll++vO5j2WyWkZGRhz5+UPhhzA/+v77EjcVm38+ryubL3h4XL0wyH6CwVPcIOrtfQhLRUXMTIaKoAkOLafsx1ZaLZVnkLZXxfIqxnMVK28cJIgKhMJ6zWLUDlhsOc3WX6UKKQkan7gbU2h6GqnJhLMs37lX50lsrvOt0iTBOdrPMVQUjuWT7bMZQWWm6XF9o9Pwb44Vkn8pqO2CqmOLGQpPrCw28ThdLNqUznrc4PZTh+VPFbYeBPWpXxVpxkDUTwdod8T6W37rUsxN2MudiqxbfzQSG7CKRSCSS7RmId0ZTV5NAukZ8KAqcHUozU0px7UGVur8/zx0BSpwsZnNFUl7ReKejptfWG4MTCMI4QFUUam0PP9QYz1tMFS2EIiikdG4utJit2qQMnSASNNwAQ4VVW8ULI0azaeaqdf7o+hJ+EJPSdXQ12WszX3PQNZUzwxkUReEb91b5ws0VhBBkTIP/x7fN8NxMqRdAbyw0uV+1EQi0zsZdO4iYLFo8Of7w2PV+Qfn8aPaRuirWioNutiZnGT2h8LjdGjvJUGzV4ruZwJBdJBKJRLI9ByI+vvCFLxzE02zJ3/22U3z+5gqFlM50McWpUporMwW+fq+KZRoofrBvo6YU9f/f3r3GRnqehf//PudnzuOZsb32eu095LTJJts2J9K0hR/tryiKKvpHKgUFKSW83IiECEQLQgGhNi0SCNRWoQWUvoCoVEBaqFRKaCH55y9C06RpkzbNaZM92Ls+j+f8HO//i8ee2rverHd37Nm1r4/kF+t4Pfezu5n78nVf93WBpqlu59T1eoZoJEcwfqQwSAIm09BpBjELrYCOH7PQ9MmnTQwN2n7EbD3CXr52G6gAXVfcOlHCDyOmai0GMykGcw6GlvS0KC8Xhyql+OGJKv/10xl+eKLGNcMZFlshb8w0uGF3sbuBvnKqljQlWz4yyacsppc6eIHihWOL3dsm79QM7GJvVawODl441gYNrhnO/yxQyLvn/L4bOVLZSIbina74SoAhhBAXb0dkPgDGBlL8P+8aYbLapuNHlLI2accgDGOCsPc3XVYL4yT4MEk6pa5kO1aCkIgkG5K2dUxdx48idJI5NKau0/ZCLF0niCNM3SRtQdY1mat7eH4EpqKcd4lijeeOLXLVYI7hostszeN0zWM4b5NxTPaW08w1kqFux+ZahJGiHUUcW2xRSttJC3d+tulCMtX3VLWTZE9SJtVmiB8FTFY76Mera3p6vNOmfL5jlHcKDjKOiaax4aOMjRypXEoA8U4BlbQtF0KI89sxwUe1FZBxTEaLKV49VWO+0aGStTF0DbXJNyEVEEWgG0mvD7U8GyaOwQZiDdKWRs41SdkG9Y5GJ1BEscILQ6ptDdMwcE0Tx9RJWQbtICJl6Zi6hqXrDGZt9gykMQ2N3UWXYtrmuN1iruExkLaZqibXaqeqHeYbHm/PJ8Perh7MEMYxB0fy3Lg7z/RSm+MLyayWZBBevhskKKV49XT9rI6i52sGBms35YaXTLPNudaGgoP1gpV3spEjlc3qcyEFp0IIcX47JvgYzDloaMzWPUoZl73lHK6lE8YKYnBN6ISb9/oRoMdgmRq+Ut20h6aDpSWb4UAm6cURxorp5Z/4XVNnsppcDR4ppsnYOqAxXfeoeyEjOZdSzqKUsRktpjF0jYYfgRbiR4pyxu0em8zWPcI4Zjjv8FbKxLV1rhvJkbYN3jNRQtd1/t/X5zg61wTgwGCG9189yP7BLJBkL9brKLoRa45RjrdBwbW78hsODlZnToB37J/Rz6JPKTgVQlzOLpdeRDvmnfG6XTl+6dAu/vuVaZa8EMsE0JKrnzpo0dpJtJshUKCHyd1aTQfi5EjGMnXKWYdrhnIM5Rx+OFmjEyoMU2EZGpap044Us/UOumaTsi0GUjYtPyKIIjJWilsnSly9K898w+v28Vhsecw12jz1WjLI7dDuAo1OwI/mWmhojORT7CmlGcjYlDM2DS+k6YUUUzaQTLY9M7NxcCRPOWN3syNKqfPOcoG1m3KSRdn4MQpc2HFGP2sypB5ECHE5u1yOhndM8KHrOndeVeHqoSzHF1ostnxeP12n6JrkXYuOn9RZbG7nD/BWrriQ1H+kzCQbkrYNsq6JZRg4JkxUMnQ8n6xroGkO4GHoGteP5JlrBNQ6AbFKqkdcy2T3QIq0pfP92Savz9Y5sdCimEpuxEzXOkmAk7EBDQO4aleeth8y2/BRaCy1a4wWXTKOyXR9OfORzZwVGGia1m3jHsWKpXaNm1bViKx2Zp3HyhHOhR6jwIUdZ/Szdbi0LRdCXM4ul6PhHRN8QLIxDBdSDBdSHJ1tcGy2xVzLp9ryCKLeTK7dqJXrtmgaWdfk0GiBMFa8MVOn4yssPcY0LPwwZqraJogUBdcin7LoBEm30qGcy/sOVNhdStHyI7718mn+960FOkHEfN3n5/aXKaQsUrYFKGYaSQATKcUPjlexdMVQIYVjuhxbaJF3Dd53VZmJcjLddbyUZjDnnJWmq3eCDf3jXS/CXjnCuVBynCGEEJfucnkv3bHv4FnHJIgj5hsekdLQDIXapLTHSk9XS0uai60c7RQdg8GcTSWXohUELC5FtPyIKI5p+klB6VAumbo7WrQJQsWppQ4TpRwHRwu8cqpOJWdTySZTaheaPmnbZKSQou2HOJaOrlvdGo6cYzCQtsk6Fv97dI6BlM3UYouTiy0yjkXGNtlbyXLrvnI34HhrrkkniJhcbCc9Span0m7kH+9KhL26WRm8c73GuchxhhBCXLrL5b10xwYfgzmHq4ZzVLIuS+2QxZa/JjDoJR1I2ckUW8swSDsGQQS7Cg6FlE055+CaBoNZjROLTRabAUN5F9cyuGY4w3R9hoWWz56BDKWMg2vr5FMmE+UUxbTNaNGllE6KTt+YbdL0Q3YXUlw1lKWSdbqZjLRt8Mqp2vL8FihlHbwowtJ0btlXpu3/rMZjddZirpF0TV0pXD3XVNozrdesLIjURZ0xynGGEEJcusvlvXTHBB/rVfgeHity4tpBDF3j9Zk6s3UffxPOXgwd0o5J3rUZyNhkXQPbSIpM867NRDnF6VqHU9UOjmWyp2QyNpAhXi7k3FV0aXoRrm0wNpDiht1Fml5I04twTIOpaodyxuauQ7vYXUzR8kPKWQfH1NE0jVv2ltA0DaUULT/i9FKHwZxDx48opi0qWYfppQ5+FDEepFEq6Sq60PDJp0xaXohr6d1Mx+qptO9UOb1es7JT1Y5cPxVCiB1uxwQf56rwvevQLmKlWGp7LDQ2p8d6J4bFVogXJHmVtq9zeGyAkYJLyjFRaFSbEQNph/GSwY1jRXblHabrPicWmhwaLXL1UJZjCy32VrIcHMnz1lyThWbAaDHF5GKL4wstylmHd40PoJTipckab862MPR291k1TWOinGGpHTDf8Aljxbv2FAB48cQSlpEEGJWsgxfGnFhsEczFWIbGwdEyo8XUWZmOd6qcXq9ZmdRrCCGE2DG7wLoVvnmXN+daPP3aHCcX2nQ28aqLF0GsIsKlNoW0w55SGscyQINi2saxdA7vKaJpGrsH0mQdk2MLHQzNoNb2mK557C6mmShn0DRtTdFQvRMwVW3R9mN0HfZXMiiS73NmQWiSjSiuyVS8NdekknXW/Nk4ps7YQIpC2mKplQyZW69Y9FJmpAghhNiZdkzwkbEN6p2AF44lzbtWrnv+9FSNU7UOhq6jNvGi7crslhiNThDy8lSVw2MD3c3dMnRq7ZDScuOulU392pEs1bZHre0zkLaI4xilFIM5hxt35zm+0GK61uanp+rEJMFAvRNQybpM1zprnhXWP+9bCWQmqy2aXsh8wyPjmJSyFguNgDBWeGHyusCaY5aMbVz0jBQhhBA7044JPpRS1L2kjiFWScOuph+haxooqHWCTXttjeTGi64nRadpy2ChGTDX6DCUT4a97R5IMZyzCWKotX1O1zxm6x2OLTR5c7bBUjPkx1N1TlZb3H3jKMOFFADHF1q8PZdMui1nHLKuSaQUXhSRNpKZKOezkpk4Nt+k0QmZb/hUWwEp2ySIPGzDYHIxOY4B1hyz3Lg7f96sxuXSUa+XzvVM2/FZhRCi13ZM8HFisc1s3aeYsnh7oUnbjzgwlCPrmOwpp5istjbldR0dbANs06SYMhjIpii4JinbZKrq0fAWObS7QDnrEMQ/m71yYrFF0bWZWWpxfK5FK4gwjWQs3Y27iwwXUhxfaPHmbJOMbWPqGn4UMWg7FFybgbTNSCHF2/NNji+0ujUf61nJTDS8sFtHMlVtE8WKwZy75kgFWHPM0vQj9g9m3zGrsbouRNdg90AK1zIuaXPu9yZ/rlqXy6V7oBBCXM52TPDxMxpBGBOrZAONoogB16KQstC9gGaPa04tA3Ipi8Gsy3glQyltstQOqbZCimmTVhiRT5lEserOXlEk11vHBlKYDYNWENIOFVoY0Q7ss14j65ocGMxy1WCGg6OF7pXa/31rAYCM3WKinDnnJriykc83PBpewGRVYeo6gzmHqWrnrCOVC21Q0/BCwjgmZRm8NFnljdk6+ypZTF3f0Oa8XqDR703+XLUul0v3QCGEuJztmOBjvJRmfyVD0wu5ejnjMbnY4vWZBscX2zS8iHaPA4+UDoWUTdY1qeQc2n5IZiDFnoEML5yo0gpCYi/i5GKbfZUsgzmHV07VeOVUjaV2yCun6mRsnfFShroX0vJDDgxmGS/9rAPpyjPdNFbk/VdXGC6kuldqm17E3kp2Tf+O9axs5GEUoxSUlwfcVbI2layzZtNXSjFaTH7CH8w5VLJnB0NnyjomTS/kRyeXWGz62KbG9SMFOkG8oc15vUCj35v8uboEXi7dA4UQ4nK2Y94Zh/IuH7hmcM2I9uMLLeqdgKYXAKqn7dUdHQayNq5toGsa842AXMqgE4SYhkPaNtCUzmInJI4iRosu1w5naXohjXbAe8ZLLDY9Rgou+wazTC8l11QP7U42Xq2W9OpYeabV9RY/u1Ib0lk+rkmGua1/VLGyka/cjilnnW4W4cxC0dm6x1S1QxQrpqodKqu+9lwGcw7jpTSNTsi1wzlePV3j7fkmu4vpDWdOzgw0+r3Jn+sGj9zsEUKI89sxwcdqmqYxmHNo+hHD+RRoGn6oejLVVlv+yDoGhg4Z28Q2dGYbPk0fdhVSvD3fZr4Z4FoG842AyarHD45XgSSbsTK0bayU4cbd+W6A0PaTbMjR2SYZx+xmOtb7iX+9TfBcRxUXspFfTMZhdTAURjH7B7NMlJNrwxvZnNdbX783+XPd4JGbPUIIcX47JvhYb+PN2AaoGFNPbrxcSuBhkPx+AzDN5FbLrnwquX0Swa6ChmsaXDWY4ehsgyCMyNgarqWhaXBioYVSiv97/fBZm6qmaQwqxZM/Ps0LxxYoL1+jnSinu7dezrSyCQ6umtEy3/AIo/is/h8XspGfGQhkbIOZWue8hZ/rvcZGC0TP9XtlkxdCiCvTjgk+1vuJPWMbVDshrmlQylosNgOiOBn+dqEikoyHroFlaBQyNsN5C0NPrrv6oYVtKF6erDHT8PCCpKdIyjKIlvt22IbRvT2yOmhYOTJ5c7bBfCvAjyFj6xta1+qgq+EFKMVZGY4L2cjPDASUUhsq/LyUYEECDSGE2F52TPCxXuq+4YWkLZNi2qbWDun4EV4UE4dcVP2HAjwFttJYaoccne9gahpoOkM5k9GBLKeqHTK2QccP8ZZnqziGhoqhmDa7AcGZmZpCyqSUcTi4K8fpWoddBbdbeLpmDWfUddQ7QTfoOrkYY2gajqVvuFj0TGcGAkdnG3K7QwghxAXZMcHHuY4WhvIucRQz1/QIY4UCTA38SziDSdtJP475WtJK3Y9DBtImKI2MbbLY8plvBgwv3yTZM5Ah5RiMldLddXXH0RddfjK5xFQ1ptEJyNgmN4zkuXlvicGcw0ytQ70T4IUxjqnjhfFyj47kSuxo0e0GXU0vIumppnWH0a3Uk1xsr4x+F34KIYS48uyYnWK91H1yW6TC88fmME5pFNM2Cy0fpV1aAUjHj3FsCGNoh0kOZSBjM1RwsU2NU0stXFPHMjWUgolKmoG0g2sZ3c1/ZVN/ZarGa9NJdgFNsavgcvPeCgdH8t3syELD58Rii7GBFEEUYxk6148WmKq2cUy9G3TNNzzmm343S3FsvsnxhTZNL1xTwHoh+l34KYQQ4sqzY4KP9WialtwWybo4poFr6UQNhXcJd241wLU0XNuk3vJZanaoZFOMFVwGUhauqRMpxbUjipmah2UktRsNL2C+4XU38NXj6OfqHqaho2ngWHo3SFnJjuRTJsFcTCFtUWuFBHG8nIkAL4zRVs1hWWqH3SxFtRVwdK5JMWUzXW++YwHrO/0ZSj2GEEKIC7Gjgw9IaiQqOZsYODHfuqjAY+V6rQ4MZC1GCy7HF9qEaFimST5l4VoGDS/EtXRswyCfsdhbzrK3ksE2NI4vtJlv+Cy1w27R5krh5mzd4+hcE4AD2cxZDa0WGslguqVWQCljd9uXd4KIycU2sWLdOSxvzzVW/hQu9Y9RCCGE2LAdH3zM1j1q7YCMpTN7EaNBdCBtQjFjY+ka+ZRFO4jwwhDLNBguOMSa4tXpOqWcw/+5Zghd09lVcDk4kmcw53B0tsHbc20AFho+9U7QDTwGcw7vv7rCRPlnXU3PbGhV7wQcGsvjmDo51+rWbhydbRArzjmHRSnFgcGkSPRANrNuAet6+j1XRQghxJVtxwcfDS+k4cWkbBNjg/unTnIbRie5WqvrGq5lMJx3STsGC3WfrGvR9CNmah7FtE3GsZip+bw8tcR1uwocHMl3AwwvjDmx2CKYS+o1Do3lu6+1cjS03nFI98jjHB1Gz1cMOpR3ef/VZ3dIPZ9+z1URQghxZdvxwUfWMSmkLWxTp5S2mGkEBOscvTgGWDr4Ed3/rgMo0DRFa3l+ymDO4WinSRgpMraJricNx0ppE8cy2DOQ5qaxwpqN3jF1xgZSFNIWS60Ax9xYD4/V1stGnK8Y9GLrNfo9V+VyI5kgIYS4MDs++Fg51piudbANsMwOUwsdVs+Yc3QwNFjuC4apJZkPDdB0UOjoukHLj5hv+lSyDo6h0w5ibFMj71poWlJz8XP7y2dlCXKuRTnrEMWKctYh51oX/BznykZsRjGoXK9dSzJBQghxYXb2rsHKnBeXG0YL2EYy46XW9qm1Y2LA1iHraJimRRQGBEqjnLZYaofJT7c6RJHCNXXKGZtSxibvmHz/+CILLR/X1Mm7FhPlLB+4ZnDdo41eXFfdymyEXK9dSzJBQghxYXZ88AHQ9CPyKZt9lSzPvDGLqeukHTCJqeRShLGi4Ue4joMWRliWQckwSdsGoVJ4QYhrGbi2yUDapuBa2IZGOeNQTJsMZByG8uef/qqUYq7hUe8EawpHNyJjG9Q7AS8ca5NZvlZ7IS7k6ECu164lmSAhhLgw8i5JsnnoGjz/9iIzNZ+2HxIBXgSq5RPHMZ1AYWoBGcfC0nUWOj6GBsMFF9twKGUcDo8XOVXtMFVtYxkGtgleqMi55jkDD6UUr5yq8cKxRdp+xFI7YLyUoZS1Lzh9ry3f+b2YcgM5Orh4kgkSQogLI8EHyeaxeyCFpilSjoEfxbS95NjFayWFHqYOKoZWEC0PhlNoeoRe9xgrpUk5Jg0/ZqHpE6MxUnCptyMqeYtfftdurtuVW/e1Z+sePzhe5eRiG12HejsknzKXB8GF3QFz58tINP2IrGNxzXC+e632QsjRwcWTTJAQQlwYCT6WNToBjp1cl52ve92rtCsXX1ScZBSiOKbRjglj6AQhnmty9XCWnG3S8QPKGYesa3Bioc3+QYtb95UZKbjMNfx1A4eGF2Isdy59a7aBqSfNwso5h4xtdLMitmEwkLE4vKfIUN4965gkYxuXlPqXowMhhBBbRXYYkuzDSyeXOLnYJghjUpaBF0WoVY0/bQNSto5jGTQ6EVqchCW6pnF0rokfKcZLaRabPoXAJOuajBZdXp+u8+ZMnaxr8b6rzp6dknVMTEOn2kqOdEaKLvsG0+ytZFFKdbMiqwfODXH2McmZ3UsvNPUvRwdCCCG2igQfQL0TMFv3QGlEMViGRtbWaHoKw4C0AWnXIu8apCyTMGqjoWOYGnnXwNI1IqXww5hjCy0Gsxa6pvP2XINaJ+TwniJH51qYusYdByprMiCDOYeJcpqmH7K3nKEdRFRyyRXZo7MNTF2jknOYrXs4pt7NSJx5THJm99ILJUcHQgghtooEHyQdRmfqHnMNj2o7IIyTNummEWGbBhoKXTcoZVwqWYe0Y1Ft+TS9iIGMy95KGpTG5FIHL1DU2xGzzTamBvPLTcNsy+DEYovMyaVuMefK0QkkGZB2EGHq+prZLeWsDUDKMnj3eLGbkdguxyTSoEsIIXaeK3PH6jHH1Ll2JEsrCKlP+cRKUfdCbN1IpsmicEydjGPSDmPGSylunhjgtdMNdhddRgopJpfazNTb5FydmXqHpU5IJWuDrjHX9Lh2OM+h0QJeqKh3AoDlkfYt0raBUlDO2EyUMwzmHJRSKKUopCwKKYvxUpqhvLsmY7Idjknklo0QQuw8Enyw3GE045BLWaRsg3onxNENlKbR8UM0NBpehGsZGLpOFMYsdUJcW+fweIm2H+K2OnhhzMnFFnEMYRyz2Ao4UE4zXs4wOpCiE8aYuo4Xxrx1conJxRbTdY/b95XQNZ1y1ulmRF45VeMHx6uYukY5a6Np2pqMwHY5JpFbNkIIsfNI8MFK3UWGRicgjhQdf57RgQzTSy2iGAopi5OLLeptD9M0GcraLDR8iimb548tEEYxS+2AxWZAy1egYoYLLjoag7kUVw9l2TeYxTF1NE2j0QkIo5i9lSzTdY+355vsLqa7RyezdY8Xji1ycrFN5YxC0+1muxwfCSGE2Dh5pyfJIkyUMxybb6HrGrmURcsPcSyDpU7IYquDbhgU0jZhrNH0Q1K2ybW78pxYbNH2Q47Pt1jyAkoZk4VWiGPq7C5lKKZNXMvi9JKHrkPWsWh4Qfcmzf5KholyunvcAkmgYRsGg8uFpinL6G7K261GYrscHwkhhNg4CT6Wrdw6aXQC9lUy/GSyRr3jJzNcDBOdENc2Gc6lqLY8YqWYqbeBmAOVLEpB/VQAWjL75cBgngNDGUoZh4OjeV44tgAaXDOcZ7KqKGdsylln3QAi65gMZJLhco6p8+7xIpWszUyt060TyTgmpq5f8TUS2+X4SAghxMbt+OBjdSYh45iMldJMLrYZr6SptU1OLnaoZG0ans5gxmZfJc3xeUUYa7SDCA2N6UYHLwwZK2UopUzuuKrC7ftKBDFMLraZqibzVjQNpqptTF1nopw5Z9AwmHM4vKe4JhuwUpi5uk6kE8Tb9jhGCCHE9rXjg4/Vty10DXYPpCikLOIpxWzNwzI1Zho+A2mT0YE0+yoZbCO5BaOUotr2abYDHMvi0O4ssVJcuyvPVcN5ppfanFxs0QkirtuVpZJ1aAXxeY8X1ssGrBRmnqtORAghhLhS7Pid68zbFq5lcHAkD4CmIO+avHhikV2FNLV2wGzDoxWETM60sUydnGOSTpn4NY9qy8dYDkpm6x7/35vzvDnbBCCIFAdHNFp+xHzDQym15urs+awUZrb9sFsnMl5Ko5Ti6GxjW9R/CCGE2Bl2fPBxrtsWmeW255apk3MtvChkZqbDG9MNXFvDNExGCjYjxRSVjM3r000Wmj67CikyjknDC2l4IcWUBWicXmozW+9Q95KBb/srGd5/dSW5/bKB4tH1CjOlR4YQQogr0Y4PPs61qU9V21iGTj5lMpR3ObXUIQZOLrYoZxwKGZ0g1Gh0QuYbHgXX5PrRArmUibt8OyXrmEzXksxHzjWJ4rgbjDS9kOMLLZba4YaCh3c6ipEeGUIIIa4kOz74WNnUV0bXvzXXZL7hEUaK60cLTFat5eyIznyjw3TNxLVNau2ArKMzqlLM1Dwsw2Cx5VPK2uRci8Gcw/uuqjBeSgOQXp5Qe3SuBSSZD+CSggfpkSGEEOJKJLvVstVHGCt9OFZuptw8USLjWLx2uka15SfHMYZiOO8yUnTohBH7KxnmGz6mrqGWm3gMF1IMF1IopZipdRgvpcm7FsW0xUQ5CT6W2rWLDh6kR4YQQogrkQQfy+qdIDk+SVsEUcz+SoZKziXrmFSyNoM5l7GCg9JgbqlD04/R0fjp6TqGrlPvhMw3fZSmCN9QvO+qCsOFFJAENi9N1gij5GrsQCZpl17J2pcUPEiPDCGEEFciCT6WJXNZ2hydbRJEMaW0zd5KtlsEOpR3OTbfxNQNBgspGnNNxkoZHENjpJii6QW8PlvHasNsw2PPQKobfKzUZqRskx9NLtH0Q5baITfuzsvtFCGEEDuO3u8FXC4cU2fPQJp9gxkipZhaavOjk0vdkfer2bpOGClOV9ukHZPdAynqnZDZus9M3Wem5lNtBd2vX6nNeHuuAcDecoYoVhxfaPGjk0u8Pt0452sJIYQQ241kPpblXItS1mZyMWldvrecYbrm8ZOpJeYaHrah0QkisrbOqaUOrqVjmzr1TsArp2rUOiEayRXdfEqjmLa633ulNqOQMskutGgHEaausdj0OVXrsLecoR1E1DtJwLJd5rYIIYQQ65HgY9mZAcLpWofXTjd4a66BHypGii5L7QADjWrLJ2WZVHIOLT/CMHQO7S4w2/DIOxYTlUy3oBRW3ajJOYyX0hxfaLHY9Dmx0GKu6TNd8zgwmMELY96Svh1CCCG2OQk+zlDK2GQck9dO14iUIo41JpfalDIWYaQYzNsstByyrsGx+RaupZNxTdpBxE1jRcZLayfUrqZpGpqmsdQOOVXrMN/0uW5XnmrLZ7yUxjF16dshhBBi25PgY9mZ3ULTjpl0OdU0dA1m6h5KwXxTp5Ay0TUdRchg3iXnWFSyTjfoeKejku6MluVjnWrLZ/dAupspkb4dQgghtjvZ3Zad2S10IG1xYDBDreVTzli0Oj6FtEspY1DOZjhVbRNEFtcMZemEMeWss6Ejku6MliDiwGDmrEyJ9O0QQgix3UnwsWwlKJhcbCW9ONImB0fynFho8tJUjXonRjdDFtom7aDN6ZrHTD0ZMnfTWHHDWYr1GoOtzpRI3w4hhBDbXc+v2j7yyCPceuut5HI5hoaG+OhHP8qrr77a65fpuUrWZrTo4oURdS9gvukzVe3QCWLSlkHGMXh9usbR2TphFDFacLl6MEvesRgvpTecpVgpPt0/mL2gqbZCCCHEdtHz4OOpp57iyJEjPPvsszz55JMEQcCHP/xhms1mr1+qp+YaPpOLbU4utHn1VJ3Zhs/kYhMviAmimDdmGzQ6IdVmQM2LWGoHhEp1b7ZIECGEEEJsTM+PXf793/99za+/8pWvMDQ0xPPPP88HPvCBXr9czzS8kMVmQBDHnFpq89Z8i5GCzY2jBfYMuJystnHzDn4YE0cR79pbYiBtX1DWQwghhBBbUPOxtLQEQKlUWve/e56H5/2ss2etVtvsJa2hlqfZzjc85psdWn7EYM7h+EKLrGMx3woopWxsQ2euHpBPmWQdh6uGcuwfzG7pWoUQQojtYFPbq8dxzIMPPsidd97JoUOH1v2aRx55hEKh0P3Ys2fPZi7pLCtXbOcaHkEUo1CkbINy1mEg7QAa5azF4T1Frh/JMZxzKWetS74GuzLp9uhsg5lapzsJVwghhNjuNjXzceTIEV5++WWeeeaZc37Npz71KR566KHur2u12pYGIN2hb5bBXNPDREMHhnMOjqmxq5Di6uEckQLT0DA0jX2D2W4r9Atpgb6SZWl4IZ0gYnKxTayQbqZCCCF2lE0LPu6//36++c1v8vTTTzM2NnbOr3McB8fpX81ExjZoeAHfO7rEyYU2E6U0xxfaDOcddE3j4EiecsYGGuQciyhWTNc6tPz4goOG1Y3M5hoelq5zcDQv3UyFEELsKD0/dlFKcf/99/PEE0/w3e9+l3379vX6JXpOKVAoFBrVdkC1HWIaBg0/ouVHtIKYnGvxnokShq7R9CNGiymiWNHwwg2/zupGZqau4UeRdDPdYnLcJYQQ/dfzHe/IkSM8/vjjfOMb3yCXy3H69GkACoUCqVSq1y93yZp+RM61+MA1Q0SvzlJvexTTFgMpi2j5a1YakE1V22QcE03jooKG1d+nnLUZKbi0/ORVlFIopeTK7iY7s42+HHcJIcTW63nw8eijjwLwC7/wC2s+/9hjj/GJT3yi1y93yVYCgk4Yc9NYgaxjMLnYpumHmIZO2jaoZO1uV9KMbQBJ0LK6Bfrqeo71OpfC2d1NlVK8NFkjihVL7Ro3LTcgE5vnzDb6ctwlhBBbr+fBx5WWxj4zIKhkbX56us4LxxaxDYOpaofBnHvetucb+Yl6pbvpyvc5OtuQjXCLrc4+yXGXEEL0x45/5z0zIABwLYPBnHtBQcHF/EQtG+HWW2+2jhBCiK0lu906LiYouJjfIxvh1lsv2BRCCLG1JPhYx8UEBRfze2QjFEIIsRNJ8LGOiwkKJJAQQgghNmZT26sLIYQQQpxJgg8hhBBCbCkJPoQQQgixpST4EEIIIcSWkuBDCCGEEFtKgg8hhBBCbCkJPoQQQgixpST4EEIIIcSWkuBDCCGEEFtKgg8hhBBCbCkJPoQQQgixpST4EEIIIcSWuuwGyymlAKjVan1eiRBCCCE2amXfXtnH38llF3zU63UA9uzZ0+eVCCGEEOJC1et1CoXCO36NpjYSomyhOI6Zmpoil8uhaVpPv3etVmPPnj2cOHGCfD7f0+99OZDnu7LJ8135tvszyvNd2Tb7+ZRS1Ot1RkdH0fV3ruq47DIfuq4zNja2qa+Rz+e35T+sFfJ8VzZ5vivfdn9Geb4r22Y+3/kyHiuk4FQIIYQQW0qCDyGEEEJsqR0VfDiOw8MPP4zjOP1eyqaQ57uyyfNd+bb7M8rzXdkup+e77ApOhRBCCLG97ajMhxBCCCH6T4IPIYQQQmwpCT6EEEIIsaUk+BBCCCHEltoxwccXv/hF9u7di+u63H777Xzve9/r95J65umnn+YjH/kIo6OjaJrG17/+9X4vqaceeeQRbr31VnK5HENDQ3z0ox/l1Vdf7feyeubRRx/lpptu6jb+ueOOO/jWt77V72Vtms9+9rNomsaDDz7Y76X0xB//8R+jadqaj+uuu67fy+qpyclJfuM3foNyuUwqleLGG2/k+9//fr+X1TN79+496+9Q0zSOHDnS76VdsiiK+KM/+iP27dtHKpXiwIED/Omf/umG5q9sph0RfPzjP/4jDz30EA8//DAvvPAChw8f5pd+6ZeYmZnp99J6otlscvjwYb74xS/2eymb4qmnnuLIkSM8++yzPPnkkwRBwIc//GGazWa/l9YTY2NjfPazn+X555/n+9//Pr/4i7/IL//yL/PjH/+430vrueeee44vfelL3HTTTf1eSk/dcMMNnDp1qvvxzDPP9HtJPbO4uMidd96JZVl861vf4ic/+Ql//ud/zsDAQL+X1jPPPffcmr+/J598EoCPfexjfV7Zpfvc5z7Ho48+yhe+8AVeeeUVPve5z/Fnf/ZnfP7zn+/vwtQOcNttt6kjR450fx1FkRodHVWPPPJIH1e1OQD1xBNP9HsZm2pmZkYB6qmnnur3UjbNwMCA+tu//dt+L6On6vW6uvrqq9WTTz6pfv7nf1498MAD/V5STzz88MPq8OHD/V7Gpvn93/999b73va/fy9hSDzzwgDpw4ICK47jfS7lkd999t7rvvvvWfO5XfuVX1D333NOnFSW2febD932ef/55PvShD3U/p+s6H/rQh/if//mfPq5MXKylpSUASqVSn1fSe1EU8dWvfpVms8kdd9zR7+X01JEjR7j77rvX/L+4Xbz++uuMjo6yf/9+7rnnHo4fP97vJfXMv/7rv3LLLbfwsY99jKGhId797nfzN3/zN/1e1qbxfZ+///u/57777uv5cNN+eO9738t3vvMdXnvtNQB++MMf8swzz3DXXXf1dV2X3WC5XpubmyOKIoaHh9d8fnh4mJ/+9Kd9WpW4WHEc8+CDD3LnnXdy6NChfi+nZ1566SXuuOMOOp0O2WyWJ554guuvv77fy+qZr371q7zwwgs899xz/V5Kz91+++185Stf4dprr+XUqVP8yZ/8Ce9///t5+eWXyeVy/V7eJTt69CiPPvooDz30EH/wB3/Ac889x2//9m9j2zb33ntvv5fXc1//+tepVqt84hOf6PdSeuKTn/wktVqN6667DsMwiKKIT3/609xzzz19Xde2Dz7E9nLkyBFefvnlbXWmDnDttdfy4osvsrS0xD/90z9x77338tRTT22LAOTEiRM88MADPPnkk7iu2+/l9NzqnyBvuukmbr/9diYmJvja177Gb/3Wb/VxZb0RxzG33HILn/nMZwB497vfzcsvv8xf//Vfb8vg4+/+7u+46667GB0d7fdSeuJrX/sa//AP/8Djjz/ODTfcwIsvvsiDDz7I6OhoX//+tn3wUalUMAyD6enpNZ+fnp5m165dfVqVuBj3338/3/zmN3n66acZGxvr93J6yrZtrrrqKgBuvvlmnnvuOf7qr/6KL33pS31e2aV7/vnnmZmZ4T3veU/3c1EU8fTTT/OFL3wBz/MwDKOPK+ytYrHINddcwxtvvNHvpfTEyMjIWUHwwYMH+ed//uc+rWjzHDt2jP/8z//kX/7lX/q9lJ75vd/7PT75yU/ya7/2awDceOONHDt2jEceeaSvwce2r/mwbZubb76Z73znO93PxXHMd77znW13pr5dKaW4//77eeKJJ/jud7/Lvn37+r2kTRfHMZ7n9XsZPfHBD36Ql156iRdffLH7ccstt3DPPffw4osvbqvAA6DRaPDmm28yMjLS76X0xJ133nnW1fbXXnuNiYmJPq1o8zz22GMMDQ1x991393spPdNqtdD1tVu9YRjEcdynFSW2feYD4KGHHuLee+/llltu4bbbbuMv//IvaTab/OZv/ma/l9YTjUZjzU9Zb731Fi+++CKlUonx8fE+rqw3jhw5wuOPP843vvENcrkcp0+fBqBQKJBKpfq8ukv3qU99irvuuovx8XHq9TqPP/44//3f/823v/3tfi+tJ3K53Fn1OZlMhnK5vC3qdn73d3+Xj3zkI0xMTDA1NcXDDz+MYRj8+q//er+X1hO/8zu/w3vf+14+85nP8Ku/+qt873vf48tf/jJf/vKX+720norjmMcee4x7770X09w+W+NHPvIRPv3pTzM+Ps4NN9zAD37wA/7iL/6C++67r78L6+tdmy30+c9/Xo2PjyvbttVtt92mnn322X4vqWf+67/+SwFnfdx77739XlpPrPdsgHrsscf6vbSeuO+++9TExISybVsNDg6qD37wg+o//uM/+r2sTbWdrtp+/OMfVyMjI8q2bbV792718Y9/XL3xxhv9XlZP/du//Zs6dOiQchxHXXfdderLX/5yv5fUc9/+9rcVoF599dV+L6WnarWaeuCBB9T4+LhyXVft379f/eEf/qHyPK+v69KU6nObMyGEEELsKNu+5kMIIYQQlxcJPoQQQgixpST4EEIIIcSWkuBDCCGEEFtKgg8hhBBCbCkJPoQQQgixpST4EEIIIcSWkuBDCCGEEFtKgg8hhBBCbCkJPoQQQgixpST4EEIIIcSWkuBDCCGEEFvq/wfPqAyP2kEskwAAAABJRU5ErkJggg==", + "text/plain": [ + "
    " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# similarly, for fever\n", + "sns.regplot(x=\"new_cases_percent_of_pop\", y=\"search_trends_fever\", data=weekly_data, scatter_kws={'alpha': 0.2, \"s\" :5})" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": { + "id": "-S1A9E3WGaYH" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAqohJREFUeJzs/XlspPl534t+3v2tvbh3k71M9+yjmdZmWRrZkpVcL3F8z7VwLozA/8hGHAM5UIIYOUgA5Qa4cYJkAjiBE+AAsgMjUXIAHd2bc46cC8NLFOfItqDN2mdGM6OZ6Z07WXvVu7+/+8fLqi6yi+xik2ySzecDcIaset+3flVsvs/396yaUkohCIIgCIJwTOjHvQBBEARBEM42IkYEQRAEQThWRIwIgiAIgnCsiBgRBEEQBOFYETEiCIIgCMKxImJEEARBEIRjRcSIIAiCIAjHiogRQRAEQRCOFfO4FzAOaZqytLREqVRC07TjXo4gCIIgCGOglKLdbjM/P4+u7+7/OBViZGlpiYsXLx73MgRBEARBeAju3LnDhQsXdn3+VIiRUqkEZG+mXC4f82oEQRAEQRiHVqvFxYsXB3Z8N06FGOmHZsrlsogRQRAEQThlPCjFQhJYBUEQBEE4VkSMCIIgCIJwrIgYEQRBEAThWBExIgiCIAjCsSJiRBAEQRCEY0XEiCAIgiAIx4qIEUEQBEEQjhURI4IgCIIgHCsiRgRBEARBOFZEjAiCIAiCcKyIGBEEQRAE4Vg5FbNpjgKlFOvtgE4QU3RMZkrOA3vnC4IgCIJw+JxZMbLeDvjB3SZJqjB0jWsXKsyW3eNeliAIgiCcOc5smKYTxCSpYr6aI0kVnSA+7iUJgiAIwpnkzIqRomNi6BpLDQ9D1yg6Z9ZJJAiCIAjHyr7EyGc/+1muXbtGuVymXC7z8ssv80d/9Ee7Hv+5z30OTdO2fbnuyQiFzJQcrl2o8PRckWsXKsyUnONekiAIgiCcSfblDrhw4QL/8l/+S55++mmUUvzH//gf+cVf/EW++93v8p73vGfkOeVymbfeemvw80lJEtU0jdmyy+xxL0QQBEEQzjj7EiP/w//wP2z7+Z//83/OZz/7Wb7+9a/vKkY0TePcuXMPv0JBEARBEB5rHjpnJEkSvvCFL9Dtdnn55Zd3Pa7T6XD58mUuXrzIL/7iL/L6668/7EsKgiAIgvAYsu+szVdffZWXX34Z3/cpFot88Ytf5IUXXhh57LPPPsu///f/nmvXrtFsNvlX/+pf8dGPfpTXX3+dCxcu7PoaQRAQBMHg51artd9lCoIgCIJwStCUUmo/J4RhyO3bt2k2m/zv//v/zu/93u/xZ3/2Z7sKkmGiKOL555/nl3/5l/ln/+yf7XrcP/kn/4Tf/M3fvO/xZrNJuVzez3IFQRAEQTgmWq0WlUrlgfZ732JkJz/90z/Nk08+ye/+7u+Odfwv/dIvYZom/9v/9r/teswoz8jFixdFjAiCIAjCKWJcMXLgPiNpmm4TDnuRJAmvvvoq58+f3/M4x3EG5cP9L0EQBEEQHk/2lTPymc98hp//+Z/n0qVLtNttPv/5z/PlL3+ZP/mTPwHgU5/6FAsLC7zyyisA/NN/+k/5yEc+wlNPPUWj0eC3fuu3uHXrFn/rb/2tw38ngiAIgiCcSvYlRtbW1vjUpz7F8vIylUqFa9eu8Sd/8if8zM/8DAC3b99G1+85W+r1Or/+67/OysoKExMTfPCDH+SrX/3qWPklgiAIgiCcDQ6cM/IoGDfmtB9kaq8gCIIgHC3j2u8zO5BFpvYKgiAIwsngzA7Kk6m9giAIgnAyOLNiRKb2CoIgCMLJ4Mxa4P7U3uGcEUEQBEEQHj1nVozI1F5BEARBOBmc2TCNIAiCIAgnAxEjgiAIgiAcKyJGBEEQBEE4VkSMCIIgCIJwrIgYEQRBEAThWBExIgiCIAjCsSJiRBAEQRCEY0XEiCAIgiAIx4qIEUEQBEEQjhURI4IgCIIgHCsiRgRBEARBOFZEjAiCIAiCcKyIGBEEQRAE4VgRMSIIgiAIwrEiYkQQBEEQhGNFxIggCIIgCMeKiBFBEARBEI4VESOCIAiCIBwrIkYEQRAEQThWRIwIgiAIgnCsiBgRBEEQBOFYETEiCIIgCMKxImJEEARBEIRjRcSIIAiCIAjHiogRQRAEQRCOFREjgiAIgiAcKyJGBEEQBEE4VszjXsBxoZRivR3QCWKKjslMyUHTtONeliAIgiCcOc6sGFlr+fzF2xt0g5iCY/Kxp6eZq+SOe1mCIAiCcOY4s2Ga27Ue1ze6BLHi+kaX27XecS9JEARBEM4kZ1aM3EMd9wIEQRAE4UxzZsXIpck8T84UcCydJ2cKXJrMH/eSBEEQBOFMcmZzRmbLLh97emZbAqsgCIIgCI+eMytGNE1jtuwye9wLEQRBEIQzzpkN0wiCIAiCcDIQMSIIgiAIwrEiYkQQBEEQhGPlzOaMPE5IN1lBEAThNHNmxUiapry50ma9HTBTcnjuXAldP52OovV2wA/uNklShaFrXLtQYbbsHveyBEEQBGEszqwYeXOlzR+9ukKUpFhGJkJemK8c86oejk4Qk6SK+WqOpYZHJ4ilSkgQBEE4NZxOV8AhsNbyafZCJvI2zV7IWss/7iU9NEXHxNA1lhoehq5RdM6sxhQEQRBOIWfWapmGTq0XstIKsE0N0zi9umym5HDtQkUauAmCIAinkjMrRs6VHd57oYptaoSx4lz59BpwaeAmCIIgnGbOrBgp52yuzBQHSZ/lnH3cSxIEQRCEM8mZFSMS2hAEQRCEk8GZFSMS2hAEQRCEk8G+sjY/+9nPcu3aNcrlMuVymZdffpk/+qM/2vOc//yf/zPPPfccruvy0ksv8Yd/+IcHWrAgCIIgCI8X+xIjFy5c4F/+y3/Jt7/9bb71rW/xV//qX+UXf/EXef3110ce/9WvfpVf/uVf5td+7df47ne/yyc/+Uk++clP8tprrx3K4gVBEARBOP1oSil1kAtMTk7yW7/1W/zar/3afc/9jb/xN+h2u/zBH/zB4LGPfOQjvO997+N3fud3xn6NVqtFpVKh2WxSLpcPstwB0kJdEARBEI6Wce33QzfXSJKEL3zhC3S7XV5++eWRx3zta1/jp3/6p7c99nM/93N87Wtf2/PaQRDQarW2fR02/Rbqb692+MHdJuvt4NBfQxAEQRCEB7NvMfLqq69SLBZxHIe//bf/Nl/84hd54YUXRh67srLC3Nzctsfm5uZYWVnZ8zVeeeUVKpXK4OvixYv7XeYDGW6hnqSKThAf+msIgiAIgvBg9i1Gnn32Wb73ve/xjW98g//pf/qf+JVf+RV++MMfHuqiPvOZz9BsNgdfd+7cOdTrg7RQFwRBEISTwr4tsG3bPPXUUwB88IMf5C//8i/5t//23/K7v/u79x177tw5VldXtz22urrKuXPn9nwNx3FwnKPt+3GS+4xIPosgCIJwljjwQJY0TQmC0fkWL7/8Mn/6p3+67bEvfelLu+aYPEr6fUauzhSZLbsnythLPosgCIJwltiXZ+Qzn/kMP//zP8+lS5dot9t8/vOf58tf/jJ/8id/AsCnPvUpFhYWeOWVVwD4e3/v7/FTP/VT/Ot//a/5hV/4Bb7whS/wrW99i3/37/7d4b+Tx4jhfJalhkcniM90czbxFAmCIDze7EuMrK2t8alPfYrl5WUqlQrXrl3jT/7kT/iZn/kZAG7fvo2u33O2fPSjH+Xzn/88//gf/2P+0T/6Rzz99NP8/u//Pi+++OLhvovHDMln2U7fU9SfI3TtQoXZsnvcyxIEQRAOiQP3GXkUHEWfkZOMeAK2c329w9urnYGn6Om5Ildnise9LEEQBOEBjGu/z/aW+4Qic3O2I54iQRCExxu5qwsnnpNc+SQIgiAcHBEjwoE56rCSeIoEQRAeb0SM7MFpzd141OuWBFNBEAThIJxZMTKOwT6tRvZRr1tKkQVBEISDcOCmZ6eVtZbPX7y9Pvhaa/n3HXNa59c86nVLgqkgCIJwEM6s1bhd6/HuepdqzmK11eXSZJ65Sm7bMafVyB71und6laaLtiSYCoIgCA/N6bCuR8ruuRT9Ko62HxHEKW0/Gjx+knNHjrr6ZLcwkIRmBEEQhIfhzIqRixM5pgs29W7IdMHm4kTuvmP6VRwAN05R7shRV59IjoggCIJwmJzZnBFN06jkLabLDpW8taen47TmjhwVpzV8JQiCIJxMzqwV6QQxcaKYK7s0exGdIGZul2PF+G7PEynYBi8tlOmGieSICIIgCAfm7FnVLfwo4a3VNr0wJm+bvLiwe8/8nTkY00WbtZZ/6vqPHIRReSIyH0YQBEE4DM6sGOluhV4qroUfp3T3CL3szMFYa/n3GeaZknMqG6SNi+SJCIIgCEfFmRUjmqZRcEyqOZuGF+5LOIwyzMCpbJAG4zWAO8pQ1WntdHsWkd+VIAhHwZkVI5cm81ydLtANYq5OF7g0mR/73FGG+TR7Dsbp2HqU5cKntdPtWUR+V4IgHAVnVozMll0+/szMQxnXUf1HgjhF1ziVSa7jCKkHlQsfZMd8moXcWUN+V4IgHAWnx2KeIEb3H4GFiRyuZZy6CpPDCMEcZMcs1UqnB/ldCYJwFJzZO8lhuJt37hJdyziVFSaHEYI5yI75qDvGCoeH/K4EQTgKzqwYOQx38+OySzyMjq0H+SyOumOscHjI70oQhKPgdFrPQ+AwhMRh7hJPe5WC7JgFQRCEh+XMipHpos181WW9HTBTcpgu2vu+xmHuEk97lYLsmAVBEISH5czOptnohCw1fPwoZanhs9EJRx6nlGKt5XN9vcNay0cpte/XGucap3n+zWF8RoIgCMLZ5cx6Rtp+RK0TUs6Z1DoRbT8a6Yk4DI/FONc4zfknp92rI5wuTntIUxCE+zk9Fu+QCeKUO/Ue0UaKZei8eGH0bJrDSHQd5xqnOf+k//7OV1zeXG7zxnILQIyEcCSI+BWEx48zK0YcU+fCRI5K3qLZi3DM0RGrw/BYjHON05x/0n9/by63uVPvoVBEiRIjIRwJ0nhNEB4/zqwYKbkWU0WHJFVMFR1KrjXyuMPwWDzqSpNHfbOeKTm8tFDm69c3cSyNubKDH6diJIQj4TSHNAVBGM2Z/SvuG9DbtR6QhTaUUveFFQ7DY3FYXo9xwy/j3qwfJpyz2zmaphElil6Y8s2bdZ6cKYiREI4EKSMXhMePM2st+ga06cXEScqtzR6Xp/Jcniqc2FyHccMv496sx73esADxo4SlhkeSsu2c/mt9+MoUNzc6XJrMH9hInMZExdO45tOGlJELwuPHmRUjcC+ckbNNfrDYpBvGNL34xOY6HHb4ZdzrDYuW9baPZei8MF/Zdk7RMTENHT9KWJjIRN1BjfBpTFQ8jWsWBEE4bs60GOmHM25udAB4YqqAFyXc2uyeyJ3tuOGXcQ3iuNcbFi3NXkSUpvedcxSu84OKr+PwUkhypSAIwv45s2KknyNSyZmkqUvBMfCihG4Q0/Fjat3oWHe2owzpuAZ/XIM47vWGRctEwRo5nfgoXOcHTVQ8Di+FJFcKgiDsnzN7p1xvB7y62BoYqhfmK7iWwWYnYLMTPpKd7V47990M6YMMvlIKP0pYb/s0exETBWtXgziugBglWsbxMBzUM3FQb8txeCkkuVIQBGH/nFkx0vJCbmx0CKKEbpBQftbg+fPTFB2Tphc/kp3tXjv3hzWk6+2ApYaHZehEacrCRO7ABvFhvR4H9Uwc1NtyHF4KSa4UBEHYP2dWjKy0Ar5xY5P1ToihabiWzhMzpZE726PKPdirc+nDGtLsmgwSTF3LOLacl+POnxAvhSAIwungzIqROEkp2CblKQsvTon6TbpGhELWWv6R5B7s1bn0YQ3pScpZOO61iJdCEAThdHBmxchs2WWq6LDY8NA1jcmiM1Y1ycPu8PdKSH1juYVC8fx8meWGv6soGoeT5A04SWsRBEEQTi5nVow8M1vg/Rcr2DpMF13+2ntmx6omedgd/qj8if7r5W0D08iub+r6gTwIj7rb66NYiyAIgvB4c2bFyI/WuvxotQuaTtOPafgJ87sY28PY4Y/yrgD84G6TOElRCqYK9qAD7HEjzbsEQRCER8XoUbVngLWWT6MXMpG3afRC1lr+rsf2d/hXZ4rMlt2HSggd5V3pC5SFifxgcN/DXv+wGRZPSaoG4kkpxVrL5/p6h7WWj1LqmFcqCIIgnHbOrGfENHTqvZDbtR66Bu0gHjko77DYzbtyFAmehxFi2S00JR4TQRAE4bA5s2LkXNnhyZkCtzZ69KKE2xvZTn+ukjuS1xuVP3FUCZ6HIRh2W9txl+uOiwysEwRBOD2cWTFSci3iVLHZC6nkLNY6EbdrvYcSIw9r+I4qwfMwBMNuaxsnmfcgQuCwRIR4cARBEE4PZ1aMKKXoBCHNXkicpLim8dD5DyfN8B1lf49xvDkH+TwO67M8LR4cQRAE4QwnsN6u9djsxqQK1jsh3Sii8JBGe7dkz+OiLxienituKyE+KON6LQ7yeRzWZzksyHQN/CiRpFtBEIQTypn1jDR6EY1uiGUamIbOTMHFtYyHutZxdBrdSxgcRvinXzVzu9YD4NJkHmDbcMHdvBYH+TwO67Mc9uD4UcJSwyNJeaSeK8lbEQRBGI8zK0aqeYvz1Rzr7YBemFDMGQc2fG0/IohT2n40ePyojM9Rh4bW2wF/8fYG1ze6ADw5U+DSZH6s0MdeoZwHGejDSuodFmTX1zskKY88ZHPSwneCIAgnlTMrRi5N5rkwkaPeCZjM2UzmHj6U0Td8ADcekfE56pyIThDTDWKqORu4Fy4Zx2uxl2fmQQb6KJJ6j2tGjuStCIIgjMeZFSOapmGbBtMll3MVl5Jr0Q2TA13zURqfozawRcek4Jistrc8I8XMM6Jp2qF3oj1qA31cM3KOe1CgIAjCaeHM3h07QYyhgW1p3NjoYhsaBXv/OSPDYQc/StA1Rhqfw84fOGoDO1Ny+NjT01yeynJFLk3mB91hDyIejsNAH9eMHBkUKAiCMB5nVoz0qyuur3fRNbgyVXio62wPO8DCRA7XMu4zPoedP3DUBlbTNOYquYduAreb+DpLBloGBQqCIIzHvkp7X3nlFT70oQ9RKpWYnZ3lk5/8JG+99dae53zuc59D07RtX657/El83SAmiFOqeZupooMfJby50t536ef2UlRwLWPkDJuTVv571PTF19urHX5wt8l6OwAOZ87PwyAzdQRBEE4u+xIjf/Znf8anP/1pvv71r/OlL32JKIr42Z/9Wbrd7p7nlctllpeXB1+3bt060KIPA03TKOdsynmLXpiw1g5YafrbDOc4jBt2OIzwxGkyqCdNfO0mjgRBEITjZ18W8Y//+I+3/fy5z32O2dlZvv3tb/Pxj3981/M0TePcuXMPt8Ij4tJknhfny7y71gGlOF/O8dz5EivNYF9JleOGHQ4jPHGaSkVPWvKmVLYIgiCcXA5kIZrNJgCTk5N7HtfpdLh8+TJpmvKBD3yAf/Ev/gXvec97dj0+CAKC4N7OtdVqHWSZI5ktu7xnoUKYKKbKLs1eyFvLHSaL9r4M57h5AXsd9zCdTQ9iUA+aTDvO+SctN+SkiSNBEAThHg99R07TlN/4jd/gJ37iJ3jxxRd3Pe7ZZ5/l3//7f8+1a9doNpv8q3/1r/joRz/K66+/zoULF0ae88orr/Cbv/mbD7u0sdA0DdcymC46nK+6vLHUYq7i8Pz58iM3nON6PIqOia7BG0stwiTh4mQOpdS+8y4O6mEZ5/yTlrx50sSRIAiCcI+Hnk3z6U9/mtdee40vfOELex738ssv86lPfYr3ve99/NRP/RT/5//5fzIzM8Pv/u7v7nrOZz7zGZrN5uDrzp07D7vMPenvlpcbPlPFTIg8yqTKPuPmV8yUHBYmckRpimXoLDW8feU+9HNO3lhuUeuEnK+4D5XPcdLyQcbhuBJnBUEQhAfzUJ6Rv/N3/g5/8Ad/wJ//+Z/v6t3YDcuyeP/7388777yz6zGO4+A4R79zPazd8s6wxXTRZqMTjh0GGTeEMOzNeZhQTd+jsdkJuFv3AHYNS+0VipGQhyAIgnCY7MuKKKX4u3/37/LFL36RL3/5y1y5cmXfL5gkCa+++ip//a//9X2fe5ikacobyy3eWeuQswyuXag89LV2hi3mqy5LDf++MMZh9N44iBDoezSeny8D7BmW2isUM7zegm2glOL6ekeGwQmADAgUBGH/7EuMfPrTn+bzn/88/+W//BdKpRIrKysAVCoVcrmsOdanPvUpFhYWeOWVVwD4p//0n/KRj3yEp556ikajwW/91m9x69Yt/tbf+luH/Fb2x5srbf6P7yyy2PDQNY27dY//+3vnH6o6ZWdi6Xo7GJloupuB309+xUG8ObuFpcZ5T/3hf8OvO1t2WWv5p6bCR3g0nKaqL0EQTgb7EiOf/exnAfjEJz6x7fH/8B/+A7/6q78KwO3bt9H1e6ko9XqdX//1X2dlZYWJiQk++MEP8tWvfpUXXnjhYCs/INm03pj5So6mF1Hvhg9dnbLTWzFTclhq+Pd5Lw6jGuYgiaH78WjsfE9BnI4cAvgoSmZlp326kDJqQRD2y77DNA/iy1/+8raff/u3f5vf/u3f3teiHgXTRRsNeHu1jW3qvOd8+aFzH3Z6K6aLNtNF5z7vxVHnWjzIaA8LmQd5NHa+p7YfjTQwjyJ/RHbapwvJKRIEYb+c2bvEVMHm6dkSOUsnZ1t8+OoE00WbtZa/7x34KG/FKO/FUZeX7sdo77V7HSVqgJEGZj/v6WE9HMNrXWz0uLXZFS/JCUbKqAVB2C9nVox0w4SiY/HjV6ZpeTE522SjEx7pDrwvWma2jPKNje6hGtT9uMf32r2OEjW7GZj9hI0e1sMxvNZuENPxY2rdSLwkJ5ST1mNGEISTz5kVI0GccrveYflWQBgl5Byd58+VHkms+6jCDvtxj48KLfW9QpudgChJyNsmNze7VHL3ElbH+Tx284A8bC7B8Fo32j7X17soFJudkLYfiRgRBEE45ZxZMWIbGn6Ustb0iBLF197ZYCJvH0mse6dx3i3/4qD0jXbbjwjidFABM8rzsnP3OpxD0vYj2kHEejsEoGD3uDxVGNvo7ya2HjaXYHitfpSw2PC5udnDMnRe2kdJtiTCCoIgnEzOrBgJE8Vqy6flJcyWbYJYESfpkTRBU0rx6mJrWx+SvYzywxrNvtEGRla+7MW2vIy6wjI0dDSemC7ihfG+BNNuHpDDyCVwTJ2LE3nKOZOWF+OY4zcRlkRYQRCEk8mZFSOOqXNlpkgUKxpehGNEmIY+CEcchJ1Gr5Iztxlnx9T3NMo7z39poTwIc4zT4fVhwiHDXgvT0Lk0lWep4eNHCaah78tLtNMDUrCNbYnBV6YLD+2RKLkWk0WbJFVMFm1KrjX2uVJyKgiCcDI5s2Kk5Fq8tFBGB95d73BlpogXRryx3MK1jLE8EuPmRsD2SpSSa+2Zf7Hz/Nu1Hk0vJk5SOkE88AoUHRPT0O/r8LrZCWh5EY1uSJSmYw3UG7c8eRx2XksptatHYr9eoMNo+iYlp4IgCCeLM3s3nik5vPfCBLZhMF/J89z5Em8st1ht1ZkpuWO58cfNjbg0md/m2XiQAd15PkCcpARxyndu1chZBo5l8OGr0/hRcl+H1zhN6YQRfpQymbdZbPQA9hRZ45Ynj8POa11f7+zqkdhv6OSwmr5JyakgCMLJ4cyKkT5520DXYanhEacqEydjuPGVUtza7LJY792XVzHK6GmaNrYBnS7azFdd1tsBMyWHybzFrc0e375Vp9aLuDRp0g1jbm50WJjI39fhdaGaZ7XpkSaKSs7i5nqPlYbPbDk3dq7EsMeiYBtAvxx6/4mfe3kkHhQ6GcdzMq53RUpOBUEQTiZnVowMexE0DaaKNpem8izWvfuM5ihjt94OuF3rsdoOWG0HXJ0uDI4fx+jtZUA3OiFLDZ84SfnhUotLkznKrsl81WGu7NALE85VXF6YL3N5qjCyw2sKbHZD2ncadIOYy5OFfeVKDHsssqocRZJCnCref6nK8+fLYwuSvTwSDwqdjOM5kcRUQRCE082ZFSMtL+T6epsgTuiGCReqLs+dK43Mkxhl7DpBTMEx+fCVSW5udrk8ld+X238vA9r3FuRskx8sNumG2XrOlfMoBWGS8IHLE/cJgmGjP5E3yZkG1YLNnbqHY9xv8PuCqF8K7Jg6JddipuTQ9iNqnZByzmS16aNQuLbJRjtAKcV0cfxE373E2YNCJ+MknUpiqiAIwunmzIqRxXqPP3p1mfV2gGsb2JrOlZnSSKM5ytgVHRNT1/GjlIVqnstT+6sQGXXNmaEE1LYfsdzo0Q0iHDNHlKRcnS4wXXLHyvsoOiYtPyFJFVem8sxXc9tyRuCeINrsBLy10mayaHG+kuMnn5omiFPu1HtEGylhnJJzdDpBwkzJwTaMQzP4D/IijZN0KompgiAIp5sze9d+9W6TpYZPmCo6QcJf3trgJ5+ZHhj54TCKHyUYW3klfWM3bjLkbuGYUQZ0Z+gob5sYus6NjR62oXPtQpWrM8U939ewt2O+6m7zduwUL31BpIDFpodpwHo7xNQ1zldcFqou1YJNoxsyUbBYb4fYhsFEwXpkBn+cz1kSUwVBEE43Z1aM1HsBYZoSxVmVyq3NHq8tNgcejlubXW5t9gaiYWEid181yjjJkLuFY2ZKDi8tlLldyypdlFLbElCzfiQaoO2rwdd+8if6722j7aNrGnGiWGsH3Kn3iFOFaeigwDR0JvI2FycL28TNo2Ccz1kSUwVBEE43Z1aMPHOuTMleZzMKsQ2dyYJNN4wHPT0WGz1WWwEfvjKFHyW4lvFAr8Qodstn0DQNTdNoetnzTa+1ozNrJgLCJOF2PSRJUt5d71B0TGbL7jYvx7D3ZbMTEKfpQNDsFU7pC6IkSfCihF4YUXQMpvI2tW7ITNEmqyxW1HoRLT+R5FBBEATh0DmzYuTjT8/w7VsNvne7jmnoTOYtTMNAKcVmJ8AxdbphxM2NNvPVPH6UcH1LDOyntHU/Za3DnVn9KOFurYcXJry53MLUddp+TNuP+djTM9sEwXB4Z7nh4ccJzV7EVNG+r/vp8Nr7gsgwdC5NFrhd6+KFIa8vtWh4Ee+7MIFrJ1iGPpa4OS720zjtsOfTHPe8m+N+fUEQhMPgzIqRc9U8v/LRJ7g8VaDjx5Rcg48/PQ3A3bpHGKcY6JyruORsg+/cqmEbJtW8yYXJ/NhdWnfLZ1BK4UcJG52ARi9kaqu1eT/ccH29g0Lj4lSed9c7VHI21bxNJ7jXz6RviN5YblHrhMyWbdbaAQXbIEpS5qs5gD3DNpkgghfmK3hRTM4yKLkm76x1WZhwafsJUZqe6OTQ/YSmDrsMeD/XOwrhIGXNgiA8Dpw8y/KI0DSNF+YrzJTcbcbh+npn2yC2ibzN22td7tZ9ZkoOLS9ivRMyXXQO1DF0vR2wWPew9CwUM1/Njey/sdkJydsmQZzS8CKeLN7rZzJcDXO37rHW8dA0jZcuTAxCS90w2bPsddhzU3Itio5FkiqqeZu2nzBRsLbly0wX7V09LcfFXpVJD2rVf1BPz36udxTCQcqaBUF4HDizYmTnLnW6aLPeDqh1Q3Q9e17X4c2VFktNH8fSWWsH5EyNUi7/wJv/g3bBnSAmVfD8fJmlhodrGSN7hrT9iBcXynSDGE3TuDiRzZm5vt7J8kOSlOfnywC4lk6UKLww3jbcbq+y12HPTb/TaieIeelCZWQlzlrLP3E78b0qkx7Uqv+gnp6CbdD2I75zy6PgmIPPcBRHIRykrFkQhMeBM3vn2mms5qtu1vV0q6zW0DXaQcSN9S7rnYCiY3J1psBLC1X8KHngzf9Bu+AHGZGBR2WHoR8WA50gQilYbvhMFZ37pvvOlByUUoPW8lMFi7WWxxvLLWZKDs+dK6Hr+n2em7k9PrfDMKiHHa4YFQq7sdEduc6jKAPWsqInHvQWjkI4SFmzIAiPA2dWjOw0quvtYHtZraWjaxoL1TyVnI1SKU9OFzlXdggT9cAS1weFDgq2wUsL5W3zXva77sWGYqpgM1V0dp2Bs94OWGr4JKnimzfqvL3WRpH1MPl/fmCB9yxU9/W5PYxB3Sk+lFK8utg6NO/KqFDYbus87DLg7Pdn8cxc5uHqhsmuxx6NEJKyZkEQTj9nVowUHRNdU3z93Q26YcyTs0VcUx8Yr5mSw0YnYLXVBWC64NAOYt5d741lQMcNHYxTLjzcyGy56bPe9gdJr5enCnuuY1i8fOP6OndqHk/PlVhseLyz1tm3GHkYg7rzfVdy5tjelYf1ojwqj8F+xJkIB0EQhNGcWTEyXbTxo4Rv3txER6feDfnpF+YGU3CnizZTBZtLk3kA0jTlxmYPhWKzE9L2oz1FwH5CBw9iOFH1Tr1H1bXB5r6k11EMG8ucbWGbOk0vQtc0ctbu+Q278TAGdaeXCPbOYxnmYZM+H5XhlzCJIAjCwTmzYmSjE/L9u82sMqZgc6vmUeuGfOyZe+ZrrpJjrpKVx/5wqclivcbNjR6WofPShcqe199P6OBB9I15JW9xY0NxYTKHpmn3Jb2OYthYLlQdJvM2jV7IRMHm2gPew34Zt/X9pcn8fbktD3rv41TKHAfi7RAEQTg4Z1aMdIIYU9fIWwb1boSuQRCnKKVGGjbH1LeV/PZbs+8njLBXz5G9rjFc5msZOi0vZrJojyVmho2lUorZcm7frz8ue7W+3/m++7kt4773cSplBEEQhNPJmRUjBdtgpuhgmzrdIObiRA6NzKCOMmwl12KyaJOkismtBmWwvzDCXj1H9rrGcJnvzpLbUexm4B/29cflYSptxn3vhxHuEgRBEE4mZ1aMAFTyFk9OF6jnbT7+zAyuZexq2HbzahxGqeuDrrFbme9u7FdcHFb/i93CUMPt6rtBzKXJPJenCsyUnPHf+xivIwiCIJxOzuxdvBsmlFybjz87yzdu1Gh6EUXX2tWw7eZV2GkY95oF02en56JgG/syruM0VNuPuDgs4/4gwZazDH5wt0nHj2l6MdcuVB7qtSVpVBAE4fHizIqRgm3QCSJWmhEzJZvnz5d4Yro4MGxpmvLmSpv1drCtQdhOdhpGpdQDvRI7PRcvLZT3ZVwP2lDtQe/hYY37gwTbzc2sTPqJ6SJ+lNAJYq5MF/b92pI0KgiC8HhxZsUIgFIAGiXHvK9fx5srbf7o1RWiJMUyMhHywvz91Sc7DeP19c4DvRI7PRfdMOHqTHEs46qU4tZml8V6jyemi3hhfN9r7Fdc7Ne4P8gzM6rV/rULFSo5k4Ld29auXoSFIAiCcGbFSNuPqHUDgijh3bUOhqb46FMzzJZdNE1jvR0QJSnPnivz5kqLt1fbY03qHccrcZCwyHo74Hatx2o7YLUdcHW6cN/5+zHwD1NJ8yDPzF5VNZenChJeEQRBELZxZsXISivgmzdq3K536fgJb6+3qfdifuHaeeYqWTMxy9B5a6VFlKRsdkLeXu08MCF0HK/EqGPGFQWdIKbgmHz4yiQ3N7tcnsofyKg/TCXNg3JSdnv+JHtBDntejiAIgjA+Z1aMxEmKrmuYmk4YRyw3PP7i7XUmCxYffWqGZ+eKwDnW2wF+FFPvRKRpyp1Nn60WI9sM1k5jdmW6gKZpKKVGJrTOlt1B867r6x2Wmz43N7pYhs5U0ebahepIUVB0TExdx49SFqpZVcpBjObDVNI8yLNzGqtdpHeJcBSIyBWE8Tj5VuKImC7axEnKRjsgjFNCQ2OxmU20TRRcmsxzaTLPVMHm+3ca/Gi9QxinNHohSoc4ZZvBWmv5fOWdjcFN5yefmmaukttm5HQNFiZyg3BPmqZ85Z1NVpoe19e7FGyDyzNFFFleyKgb2GFXkhxFNctprHY5rPJmQRhGRK4gjMeZFSMAJcek6BjESlF1TSo5m8mizbvrXTp+zK3NHpoGHS8iSVPmyjYasFBx2OwEvLHcAjLje7vW4931LtWcxWqry6XJPHOV3MDIna+4fPN6jdeXmsxX8kwULJRSXN/o0PIiFhs9Lk4VtnJVTG5t9tjshvf15Rg31DHujuxhhMNua9jNO3QaOI3eHOHkIyJXEMbjzN5xN7sR56p5npor8Rc/2mC6YDNXdQnjFIDLUwVeW2wQxClPzZbItwMALEPnB4stwigFBVGidsx42W58+0buzeU2N2sdNDSKrkXTCwHFRjugG8QEiaLtRTw5XeDJ6SKpYmRfjr12VcNiwAtj3lhu093KMfnY09ODOTvDHGYex2neBZ5Gb45w8hGRKwjjcWb/MmZKDrah0wkSXrxQ4cNXJrg4WaDjR9yueay2fGrdiISUt5abuJZB2bVQaIRRgq9gruISxCmdLe/F1ekCHS/EMTQW6z0Kjsmzc0WuXajwxnKLy36BbhDzg7t1TE3nufOlLcMNH3pigrJj8mNPTHJpMs+ri62RfTn2Eg3DYuD6epuVVsBCNc9qO0t0navk9l2W2zfK4ybX7ncXeFJi6ic5uVY4vYjIFYTxOLNi5Nm5IrXuJHdqXYquxdXpApW8w7NzRYquxffv1Cm5BufKBW7XPCwjM5B+lJX7vrXS5tZml4WJ/KCXxgvzZb51Y5Pllk83THhnrUvtySnmq1l1zlrL5269R5qCbmnMFB1qxZAkTXlqusRk0R6EY14CUpXSC2JWGt5Yg/GGxcA7q22iJEWh6IYRSw2PtZaPUopXF1sDETRfzW0rWR7l3QDG8ngUHRNdgzeWWoRJwsXJ3K6DB/ucZm+KIDwIEbmCMB5nVoxsdEJWmv5WyW6Xnp8wVXKYr7osNXw6fsK7613q3QgFTBUcnpgusdrepNENuDqdp+xaVHLmYHe/1PBZ74Q0ehHPnavw7lqbb92s8fz5CroGlZzF1ekiH7hk8e1bdf7yZo1qwWaumufqbGFbXoimaeiaxmTBIU4VCxO5B+6qhl3Cs2WHBMVa0yOMUrwo4ft3GiilWG76PDFdZLnRY6XpM1NyB0JglHcDGMvjMVNyWJjIsdYOsAydpYbHdNHZ11ycth8NHpfqA0EQhLPBmRUjt2u9QfLorc0e5youlbzFejsgSRUXJnPcqvWYKTt0g5gwiekFEVenC1yeylNwTBbrHrVuRNNrUcmZJKniqbkS7653eHO5ialrFFyL8xWXN5fbOJZGwTGxTZ3zFZdEKV5aqNILM4PfN/z3BshlXV+XGh6uZTzQKA+7hL0whq2QUiuIs86tGz2iNKEdJKy2A0quyVTe2SYydotxjxP31jQN1zKYLjoPORcHlps+X79ew9S1PUucBUEQhMeHMytGALphTMOL6AQJP1xqMlmwuTSVZ6nh0+jE2IbORiek4BjkHJOpos1l18IxdWrdkEQpFqp5lhoekBls29D40JVJJvM2U0UHL4p5c7nNnXqPhQkXy9CZLmadSBe3PBO1XsBK0+d8NYep6w89QG7YJXx9vUM5Z/H0XJl3N1b4/t0mpqZxaSrPs3NF3lnrMFWwqebMba+xW4x73Lj3Qebi+FHCt2/WWGz4TA8N2RMXtyAIwuPNmRUjlybzzBVdlus+c2WLomtwccLdanYGpp7VxXSCkKmiS6OXhV+aXkyqsnbymsbA6F6azKNpGp0g5oOXJ7clfr6x3EKheH6+zHLDZ6rocGW6AMBqs06SKNa8gKszRfwofegBcsP0RUGjGzBbsnn+fJm2FxOnKW+tdgDQNbgwmb+vzf2oGPe4ce+DzMW5vt7BMe/lruS21iUIgiA83pzZO/1s2eXiVJ5v3thEKY26FhGlWcnvUsPPEioNDdAHxrsXJUzlXZ6fL7NYV0wVM+/HNkM+4nUgKwFebvgYukbBNlhvB6y3A2zD4MWFKt+8WePmZpeFav5QBsj1RUElZ1LMWRQck6mCgyLLGbk8mWel5bPeDnj+fPnQcjN2dpe9sdEdO/ej6JhMFCwAHFPn/ZeqUn0gCIJwBjizYkTTsmm9FycLXJjMc7fWI05S2n7EZiegkreIkpTJgoVhaDwxVWCp0WOz6/OdW1nvjvdfyvIZHmR0d3oLlFL84G6TzU7A3bqHQg1yUfpJrA/LzlLZD16e2DacTimFrrVYbQUs1n10dKKkeehVLA9TJTNTcnjvxaokrwqCIJwxzqwYgcxrUc3b1DpZ9UeYpCw3s/LbGxsKy9D58NVJim6KH6VYhoFrp6BB30bu1gZ+mGEvh1KKb92ssdjocXkyj0JxruIe2DvRFyG3Nrvc2uxlM2wMfSACZoeOu6ZpvLHcQkPjufMllpv+oedmPEzPkYN4gx5Fv5KT0hNFEAThceNMi5HnzpUAeHu1Ta0XkqSKG+sdSq5JOWex0QmwdHh6oUw3TNjsBGx2w0HSav+xUW3gdzNcmWDosdoKWG0FPDlT4Pnz5QN7JfqeiMVGdu0PX5ka2Sitb/ABoqTJctM/ks6Qj7rz5KPoVyI9UQRBEI6GMy1GdF3nhfkKrmXw9mqH+WqOlhez0vK5sdHDMQ1u13yuzJS4Ml3AC2O+davDO2ttzpVdCrbB5uBq23fIuxmuth+RpIpLk3k22j4Xh/qH7CZgxtmR9z0RT0wVWG0F3NzoDBqyjeKoO0M+6s6TD/LEHIZXQ+aMCIIgHA1nVoykacqbK23WWj7tIKbRDWn0QvQt+6SUopo3SVM16P/xw6UWK81sym/Bzj66ixM5pgs29W7IdMHm4kQWotnNcPlRwlsrbXphTN42KWwlq8LuAmacHXnfE+FFCU/OFLYN1xvFXiGRwzDcR9F5cq91PcgTcxheDZkzIgiCcDSc2bvpG8st/o/vLLLR8en6Me+ZrzJbdpgtO1xUeSYKDuutrAfIZif76gYxC9U8oNB1jW6YkLf0LH9kKI8Edjdc3SAmIaWSt/DjhO6W0IHdBcw4O/JRnoiHzWc4qeGIvdb1IE/MYXg1ZM6IIAjC0XBmxcg7ax0WGx4F22CjF2GZGjMll4m8ha5paIREBRvX1Hl3vUO9FxJGKX6comkaTxYLFB2Tmxsdbm70SJTibs1jvuoyV8lCLy8tlLld6wHZrr4/p6VgW1RzNg0v3CYYdnYj9aOE6+sd/CjB0NlzR36YnoiTGo7YbV3jeHIOw6shc0YEQRCOBn0/B7/yyit86EMfolQqMTs7yyc/+UneeuutB573n//zf+a5557DdV1eeukl/vAP//ChF3xYuKZOGKdstP0sFONHg+Zl1y5U+dCVSX7s8gQ526ATJDR6MZah8f6LVf7KszP85FPTzJQcGl7EnXqXd9c63Kp1+cFik/V2MJgv0/Riat2IVxdbrLeDQVin7W0P68C9nffTc0XmqzkW6x5vr3ZYrHvMV3M8vTUB+Kh35Cc1HLHbuvoek7dXO/zgbvb572T4s30Un6EgCIIwPvsSI3/2Z3/Gpz/9ab7+9a/zpS99iSiK+Nmf/Vm63e6u53z1q1/ll3/5l/m1X/s1vvvd7/LJT36ST37yk7z22msHXvxBWJjIMVt0MA2dmaLNlak85yvOID/kynQ2uC5OFLc3uhga2IbJk7NFPnRlirlKDk3TqOYsyq5FybWYr+bImcbgGsM7+WQr90TTNCp5i+myQyVvbdvBa5rGTClrorbeDqh3I85XXFIFrmVwdabIbNkdJLWutXyur3cG03j77PXcOJxUw73bukZ9zjvpezWGP8PTwkF/n4JwlpG/n9PBvra8f/zHf7zt58997nPMzs7y7W9/m49//OMjz/m3//bf8tf+2l/jH/yDfwDAP/tn/4wvfelL/C//y//C7/zO7zzksg9OzjZ56lyJqa5D24uoexFvrrQpOtYgH2Gm5PDEdIE3l1u0/Rhd0wjidNt1Lk8VuHahyjtrbUxDp+CYbHYCio5J3tJp+xHfueVRcEwKtkE3TCg6Fs/MlQflwcNhBj9KWGp4bHZC7tazmTeTRXvQsGz4uMW6R6q4L39inN4nw4wKc5zEcMRuYZKT6sk5LE5qDo8gnAbk7+d0cKC7drPZBGBycnLXY772ta/x9//+39/22M/93M/x+7//+7ueEwQBQXDP1d5qtQ6yzJEUHZM4Sah1A86VXZIUOn7EdNHh5maXSi4zyucrLi9dqFLOmdxt+Ky1fKYKNkopbtd61Hsh81WHhYnsH3fbT9jshDS9mPMV577k1lGGc/iPZb3tYxk6z8+XAZirOIOGaMPHZT1QsuN25nXcrvVG9j7ZjYP8sZ6ERmCPe2LpSc3hEYTTgPz9nA4eWoykacpv/MZv8BM/8RO8+OKLux63srLC3Nzctsfm5uZYWVnZ9ZxXXnmF3/zN33zYpY2NYxhoaNR7EZcnc6TAN27UQCmSJHPlFbam9W52QhqdgO/2Ir7y9jq6ruGFCRvdiNlSNur+0mQeiAb/6Dc64cALstjocbvWY7JgM191cUydkmsxU3K4sdEd/LE0exFRmg4G6g03RBv+o2r0QsIkeYA3YDxRcJA/1pOw6zjMxNKTIK528rh7fgThKJG/n9PBQ/9WPv3pT/Paa6/xla985TDXA8BnPvOZbd6UVqvFxYsXD/U1OkE2X+a9F6tstH3eM19G0zTeoE01b/PmSosfLrc4X3bJOyYKRRClrPsBi02PME65NJEnZxm4Q3kihq6xWO/RCWKavZRbNY+3VhqAhqlrTBdzTBQs3nuxOjDaw38sEwWLhYnctkm6fYaPmyrazFdHH3dpMs/V6QLdIObqdGFLJO3OQf5YH7ddx0kQVzt53D0/gnCUyN/P6eChxMjf+Tt/hz/4gz/gz//8z7lw4cKex547d47V1dVtj62urnLu3Lldz3EcB8c52n8wO5uPFV2LmZJLy09YrPfQtGw43mozYL0TYuoaqx2PlpdwaSrPrY0u9V4Amk6appyrulycyHF5SufWZpfllseNtS53Gx6WoWPoMFmwcazsI2/7EbAlimyDF+dL3NnKEZkq2COTLHeWCw8f10/S6l/vY09Pb+WnbP/jG7XzP8gf6+O26ziJ4kpKigXh4ZG/n9PBviyHUoq/+3f/Ll/84hf58pe/zJUrVx54zssvv8yf/umf8hu/8RuDx770pS/x8ssv73uxh0nHj2gHEbqm0Qoi7tS6uJbBfNWl7BoUXRMvSgiTBNPQmCs7VF2bZq9Lx4uYKTqcqzj4UYprG3hBwnrbR9d13llrc2OtixfF5Cydgm3Q9BPiJOWNpRbzFYeco9PshdiGOfCGNL3MEDa9FteGZsj0GS4X3nncqB391Znife97t53/w/6xPm67jsdNXAmCIJwG9nWn/fSnP83nP/95/st/+S+USqVB3kelUiGXyxIkP/WpT7GwsMArr7wCwN/7e3+Pn/qpn+Jf/+t/zS/8wi/whS98gW9961v8u3/37w75reyPhhexueXx6IYxX31nk9VWSMEx+cmnpnhiukgniLk0mef1xSb/11trtPysSVmSgmZAwbbw44icZdLyY/74h6v0/ISVls+PllsYpoZjmcyVbECj5Sf0ggjL0rj+2jKppnG56nK3brLR8dG1LCF1ubH7FN2DdGnd6/w++82ZeNx2HY+buBIEQTgN7EuMfPaznwXgE5/4xLbH/8N/+A/86q/+KgC3b99G1++1L/noRz/K5z//ef7xP/7H/KN/9I94+umn+f3f//09k14fGRqAoulFXN/oUMnbrLQ8yq7JU3Mlio7JE1N57tS6NHsRXqTY7ISUpyxafkwnjFlteKy3fJ45X6LZC1ls+PhRTDtMqOgmJdvg4oSLbZoUHZMbmz1QsNkJCRNFrR3iGBpeVMKPUlpBzJWp3Qfc7bZzH3dH/6Dj1lo+f/H2Bt2tnJqPPb13WfB+OInJoTt53MSVIAjCaWDfYZoH8eUvf/m+x37pl36JX/qlX9rPSx05E3mbixN5kjRlvR3hxynLrQBNU7y70UGhDcIYmqYRJilBlNANYxpewGY3Jk1TUgUFy+CdtQ5xnLLS9Kh1A+JUYRgaXpzS6MWU8wZ+FIGCMFZUchbrnQDXAMPQBgP6ul7I+cokSimur3fuM9q77dzH3dE/6LjbtR7XN7pUczar7S6Xp/YuC94Po0JE/ZLlkyxQTiunQfwJgiDAGZ5Nc2kyzxNTed5d63BhwmW25GQJn67JRD6rVFls9Li12aXrRxiaTsk1KLsWYZxQyVvkLINeFKMb8M5qC1PXiGLFZNHB9mNKjkmiIFGKas4iiGNmy3k0NGrdED9KyTsmXT9ioxfx/FwJlMY7ax3eWG5TdExMQx+7okMpNRjqp5QamQQ7/s7/8LsUjgoRASeueuVx4SRWBgmCIIzizIoRTdMouRbnKi6moVGwTUquyZNzJRxT442lFptdn6W6QZQkeFFM3jayRmY6NDoR37/bJggjUnSiNKWac0hVSkWDSs6i6BrkHYsr00Xq3ZBUKZ6YKmIaOvPVHOfKec5XbL57u0mYJLS9mFil+ElML0z58JUp/CjZVnmzW+fV9XbAV97Z4N31rDX/1ekCH39mZt/G59JknidnCnSCmKuFPHnbGOmhSdOUN1farLcDZkoOz50rbQvPjWJUiOiwc1iEe5zEyiBBEB4NSaqI05Q0zTbESapIU0WcKtKtn5Oh7xcmcjimcWzrPbNipBPEpCk8OVskTBWaUpyv5nFNnZxtstYKafoRzV6PK9MFzpVdnporcbfW5RvXN7jd8Ol6CaYJcZK1iN/sBliGhmkYaFqMGxs4VvZcNW8B2VyatpcwP5mj6NrUOiGuYzCXz9H1I6qOzUzJ4RvXa7x2t8Ez50oEccqNPTqvzijFrc0uNze6mJpGwTHp+BG3Nrv7NuKzZZePPT2zrTV9kt7fcv7NlTZ/9OoKUZJiGZkIeWG+sue1dwsR7ZXDIrv7h0cqgx49Ip6Fo6IvHpJUbRMXibonMobFxWnjzN6dgjjlTr1HtJHSDWIuTxZ4Yb7C3VqX6+sdbm508OOEpbrPestnqugSJelWImuKY+hEZkqcKFIFpg5Rkt2MojgBx8APE2xdp94LeeZcmeWGx/duNwnjBEipFmxsEy5Uczw/X+ab12ustX3u1ntYpoauQ842WGv5bHYCnp8vj+y8ut4OuF3r0Qoi1lohsyWHy5N5btd61LrRvoz4cBjn+nqHJGXkznq9HRDGKeerOd5cbvL2apvnz5f3XXnzoBwW2d0/PFIZ9OgR8SyMy07PRLLV+bsvLhKltuzL6RQX++XMihHH1LkwkaOSt7hT93CMbAe51PB4dbHJ7VqXphczkbeYzDu4FtS7IUGcEKUKlaakKHQdXD0rzJksWpCAYWYGuRcmVPI2SmlstHyqBYeSY/CdW3W+e6fBRMFhpmRTcixeX2zw6t0GjV6Ibmj8P67NEyVwa7OHpeuDoXmjOq/e2OhScEz+yjOzvLbU5OJEnvMVl81uiGsZ3NzoDGbt7GeXttfOeqbkECUpX7u+ga5lOTDr7WDfN94H5bBka4AfLjWJU8XFyRxKKdltjoFUBj16RDyfXcYRF8PeDWE7Z1aMZMmhGqstn4mcyfPny+Rsk9WWhxclTBcdbtd6BFGW1LrWDglqPZSCOE3JOSa6rjFZsMlZBnfqHkmiyFsGkwWLqZJLL0hwbZ2JgsVmN6QbJQRuNhV4puhSdE104PJUnm9c3+BH6x00pehGKd+5U+e5cxUsQ+e58yVg+9C8YWNcdExMXSeIFc+dq3DtQhYuuV1b59XFbMhgsdbj8lRhX2Jhr531c+dKfOTqJK8uNnlqtoht6kdy450pOcxXc6w0fWzDYLHuMV10DrTbFFe6cFRIaOzx4b78iqFwSLotVJLlZIxTbSrszpn+S1GKraIRjemiw1wlxztrbQxdoxPEGLqOaWgsNXxUmhKlbCWzpuQsHdc2cQyDjW5EGKckChSKOStHyTG4PFmk3gu4sdElTlKKroVt6Jwru/hxSsMLBwa+3smqa6YKNp3ARwM+cHmCpYbHctNnsmhvG5o3zG6i4fJUnm4Y88RUAS9KRoqFvQzzXjtrXdd536UJdF0fuKSP4saraRquZTBTcg9ttymudOGokNDYyea+pM4tz0X/seHnRFw8Ws6sGOmGCSXX4tlzWSJoN0wAuLZQYbHu8a2bNaYKNpW8haFptHohkR8TJIogSgjibKZNJ4hJ4gTXNvHCBJVCz4+Jyw4/dmWC62vZnJqJvEPRNXFNbZBbUe+FNHsxm52QWClypk6cpkwULF5cqDBVsOlulb9emszvemPbTTRcnirQ9GL8KMXU9ZFi4SCG+VHdeA97tymudOGokNDYo2dYYAz/fzgs0n9MOLmcWTGym4Gbq+T48NUpemGCrmmstX1c26DgWvSiFDtO6ClQKeiaRhjG2KaJH6Z4UQqmTt2LcOsebyy1uTCRI+8arLdDml6IXcw8D5enCkzkLb59M8sTSZKUy9M58pbJE9MFXjhf5tXF1kAk9OfS9Bkn1DCOWDiIYX5UN97DFj3iSheEk4tS2ytGRlWRSO7F48eZvQvvZeA6fkSapMwUHerdgJmiRZKa6LoijCw6YQsvgno3wjZgumiy3g7Jm1kJb5SkKDTeWG6ioXjufIn5SuZtaHkxG+2AW5s9lFJ8906dlaZHy49ZqOZ4arbEJ56bxTF1kmY4EAnDvUaKjolSaptYGeXRGEcsnAbDfNiiZ+fvfrpoDyYeSw6JIBw+ewoMSe4UOMNiZJSBU0rxw6Umf/jqCj+426DtR9iWwazKhulZhkkQBhRsC02LCGOFH8FKK8AxDSYKNkmaousmlqFxu96j3g1o+gkvLpS5PJVHoRFECd+5XcM1dfw4ppyzyTsW56s5Co45qJQZFgnDvUYMXaOSMw8l1PCwXoejSgJ9FMmlO3/3ay1fckgEYZ/0BUY/ybOf2Dmc4HmWSlOFg3Fmxcgo1tsBf/bWOj9abVLvBrSDGNc0eO1ug0rOJG9b9CJFN4zpBopk67xOkBImKXGqqOZMJooOdS/C0DRc28QPI1aaHiXX4MZGl7eWW3hRVqZqGwahSgGFHyc0vJCvvL3OXNlhoZojZ5uUXIu2H5GkivMVlzeWWyzVu3SjlHo3YLrkPLRH42G9DkeVBHocyaWSQyIIGfeVoO6oIImHvBepJHgKh4iIkSE6QUyqFKau0/ZjWkGMpyWEysqERK1HGERESToQIgCRgjQCTU9wTJuLEy63NlN6YcKdzQ71bsBq26fRDUgU+FHKxck8OjBdtMjbJhpgGzpvr3V4Y6mFrmt86Mokv/DSPLNlF6UUbT/i7dUWd+setqmhawYoxUsXKrsO1jvKz+ooDPhxCIPTEKoShIdhN3Fx7/vtVSVSQSIcF3LXHaLomMwUbbwwIUpSyjmLkm0QJglr7YC1Vohl6ETx/eemQBAq7tZ7GIZO3jaJkpgwUfTClOvrXe7UelycyNMJIvK2jm0apApcK6HoWjS8GD9OOV/N0/Iiap1wmzHWtKxzbBAnzJQKlFwLx4TFusf37jQxdY2pos21C9WHnoY7bpjkqAz4cQgDKccUThM7Bca28MhQuES8F8JpQsTIENNFm7xjohs6xZxFEKZEaYqpGTT9iDiFJN3uFemjgFhBL8raqM9W8lh6JjjCVBHGCV4npunFmAb0ogRD0/DDhNlKjvderOLHCb0gYqMTopGSszX+4kdrvLPappIzKTgmP/bEFC0/ZqMTkCjFTNHmnbUOLT9mesuIPsw03L4IyWbc9EhTRZSmfODyxMg270dlwI9DGEg5pnDc7GywtZvAiFPxXgiPJyJGhlhvB3zvdoMkTlio5Njs+BQcg412QNtPRoqQYRTZjJpEQa3jYes6UQpoYOtZPXw7iHBNg9UkwDYMYqVor3cxdY1yzkLXNPwwouSa/Gi1w92aT84xeW6uyIXJPCh4aaFCOWcykbdRSnFjo4djGay3A3KWQcE2uLXZZbHe44npIl4Y31eNs9Pj0c/VWGz0uLHeo5o38aMUTdNGdjw9KgMuwkB4XNgpJkZVkIjAEIQMESND3K71WOuENLyYhpeV7eZskyge7Q0ZRZhkc2rSVNEjIWfqKE3RjRVRApoOUZKSphoaGgXLQGmZx2WjE6Cjsd7JZuC0g4Qnp4vkTIM4Sbk0mWeq6GwTE2stn6YXo6HhmDrvv1QdvJfVdsBqO+DqdOG+apydnpJ+rsYTUwXeXG6z0ox5aq6EudWNVsSBIGyfP7KbwJD8C0HYPyJGdpC3dMquRS+MyFkmK02Pund/5z69/38t84QYgG1CmoJpaXQChQY4aFiGRqISHAsMTSdKU1zTIElT/AQKtomORt0LUUC9F+IFGikaNze7aLq2VRq8fbaM2rrhVfMW1bzFpck8s2WX6+sdoiTl0mSOjU7ApQmXjh9xt95lIm/T6IX3Dc7r52p4UcLTcwU2OxF+GFPNWRRs48g/d0F41Az3vhgkcg6Vpcr8EUF4dIgYGeLSZJ7n5sts9kKiNMU2oO3HGHpCskOPpIClQ96EBA2UwjQ0/EjhBdkNSyPLDcmhc76ap+tFoEGcGFyZKdDxIhxbR2k6mgYdPwunRHGKbehMFmwuTuX4xLMz/OwLc/flT/RDK/VuhB9FrLR8So5JO4i5s9kjUWAZOmEKK02PGxs9vtGuM1u2KTjmtp4m00V7kKtxcSLHG8stumGC9P4STgvbxMMOEZGkktwpCCcZESNDzJQcfvyJSXSleG2pTaPrcWvTY7d7VpJmSaupUgQxaJHC1DNviVJgaoAG58sOFyoudSsLtyhgruziRwlJqrAtjXYQo28NhUs1DR0wDI3nz1f4ufecY66SA7ZXu2x2AjY7ASstnx8utWj7EecqObpBRCln8WOXJ9A1PRvS52STiYOozvPnysSp4ju36syU3G1hm1myBNySa/Psudy2uT2C8CgZdyS7eC0E4fQjYoTtlSS3az0c26TkmvixSc7WiZLM4xGM8I50M2cHGlnoRtfAMjTQNCwdJgo2f/W5OVY7PgqdXhDRiRLeXm0Rp6CSBIXOxQmXhhfhxQkqVcRKwzV1pvL2ttccbgrWCSJu13pc3+jSC2M2uxF522Sl5ZPrRhQdayAylho+OhozJRcNjThV2IYxsp/HYZfXPoquqsLJRg3lWOzmtdgpPgRBODuIGGGokqTeY7XlM5G32Wj7bHQi4iTFNEysNMLQwE8yETKM2vrSAMvUeXG+QjeMIVWEqeK7d2pMFF0uTLjUugaqE9DohTiGTqXkEsQppq5RckwSpdH1I2xL4+m5CkXX3uaZ6Ceanq+6/HApwNQ1cpZONZen3g1ZbflU8haXJguUHTMLPZ0rMV10aPsRL14o45g6QZyyWO/xw6UmcZp1g1VKoWnaoZfXrrcDvn+nQb0bESbJruXCIlpOD6O8Fjuback4dkEQxkXECEOVJNNFbmz0WG42uVP3uL3ZpRclWQgGsI37hcgwOQvytkHB1VHKYK0VEClFqqDopvhRymY3IIoVlYKF56cUHJP5CYtzJScr8av3CCKN85UcjqkRbYVY+vS9Fm8stVisZzkitqGDUlydLWIAUZqiqZSJgs2lyTy6rmchmB3JrwCrrTq2YbBY9wYlvIddXtsJYurdiHYQsd4Odi0XPo5W8EJGnKT3JXLeJzjEayEIwhEhYoShSpIwZq5sM1XMqlveXu0QxPcEiPeA1IkwBT9MWGv6GJpOyw8xDBPXSKjmLZ6YypGkmSDp+gmpSrOpvnNFnpotstYO0DSdgmOStw1ytsnlqTxJkvDN6xs0ehEV18A2oNkLiJOU6WKOt9fAi2Kmig6TeYswUeRsg3Iu+/Xu5nFwLYOZkrvrZODD8kwUHZMwSVhvZ3N0disXlhkxh8fOJlo7Z4zIEDNBEE4SIkbY3vXz0lSepYbHWivAMDTSaPzrhAkYmuL6Rhel9K0JvgrbtPGihF6UEiQptW5ErRtiGnBjs0s3jLmx0UGplPecL2/lg8RUciZvr7b52rubrHcCWl6EZer0/JgUhYbGctMDFFemixRdi7WmR94xeWmhihcldMNkV4/DgyYDH5ZnYqbk8IHLE2iaNmhZPyoP5aC5Ko9zmGfnhNR+zkWcphIWEQRhLFKliOKUKFVESUqcqGzIa5LS8iPKrsXVmeKxrE3ECNu7fiqlmMxbvLHYYCJn0vT3V0kSxQoP0EkxAEtLKTsGOVPn1Tt1rtc8ap0AgErOou2HbHZCrm/0mCnaFB0LTVMUXZtyzubN5RarLZ8oViQoom7CZickZxmU8zZlpagWLDY72TA+TWUVPt+4UePqdB4/SrhT61HrhDx3vsRy0x94HHbmhvQnA+/XM/EgEaBpGs+fLzNddPbMQzlorsppC/Pc1yxrRwvwnaESQRBOPumW17Fv5KMkM/zbjX/22PD3UbolFIa+j9Ps/DDOpsL3rxMlinjr/P730bbXGnrN9J4AeZAX9KeemeE//s0ff0Sf1HZEjOxA0zRqvYjNbojxELvq/gy9dOtLJZmRbAQx3V6EF6ckKmuOttGO0A2YKtgYOli6RtuPSNMUP0z58zdX0Q0dXdNY7Xh4fkTOtXBMHT9J8ds+Boqia+CaBn6suDyd45m5Mq8vNekFMT9cahEnKXcbHi0vwrX1bcmqO3ND9uuZUErxxnKL79zKck8mClkFj6Zp94mTB+WhHDRX5UFhnkfhOcm8F+k2T0WcptsERyYuEO+FIDwkfYMf3WeE7xn8UcZ/27FjGPydxj8TEOp+kbF1rTBJOc1RzzDeKyvyaBExMoL1dkDLiwn38XvRyLqw7hzoaxrgRwnNICZV98p/bQssXacXprR7IaZpoKFA6diWRkqCbRkUdIVSMJGzKNkmeUfD0AwaXkjONjF0nds1j2fmynhxQNOLWWsHBLGiFcSstEM+9MQEKy2f2/UOFycKLNZ72xJI+0a67UfMV10cU6fkWmN5JtbbAd+93eBu3Rscf7vWo+nFj9xD8aAwzziek52CZapgk8LAO5GqHR4NaQEuPKYopQYGd2DU05Qo7hvzre/Te4Z/u7G+36BvP25YJGw9l6aEsbpn4He5juQ5HS6WoeGYBq6lP/jgI0LEyA6UygzVrVqHza4//nlsFyKGBqkCTUGiNOIkExUp2XyavGNRcU1q3RDH1FGApenEmoYXJnikvHC+RCXn8MZKg7afUHR0pgoOFyby1HohGjqGBu+sd1lseFl5sQaupXNhIsdc2eGbN+u8vthksxMCGrquUe/G27wGBwlvdIIYU9eYLjmstwMcM/vHfByJqA8K83SCmChJmSu7LNY91tsBrm1sa6S12vJ5falFnCg0DZ6ZKzFVtHd5RUE4GPs1+FGced7CkTv18Qz+duO/ddyWVyAeOicWg39o9Ns+WLqGZeiYRvZ/e+h7y+g/t/W9rmfnbH0/6jjb0LaOv/e9qWvYpr792K3Xtcyt19S1wXoMXUPTNBYmcjjm8Y3+EDGyg/V2gBfFFF2LNM08GeM4SPpNzzQt+940sg6sidJIVea6U4BjZOEYXaX0ooSya26FWMC2NYgV1ZybdWO1DVAJSarRi2PqvYRGL+LSZImnZsr4UYwfKgq2T5oqnj9fZr6aY76ao+lttYd3DQq2iW3qdMOYt9faXJ7Mb5s3c5AqlqJjDox1zjJ4/6UqUwWbptc6tKZpe7Gz5NS1DWxTJ0kV651gW7ik1g3Z7IastwN0XaMXJmy0g23Xq3VDwjhltuSy1vbphTFTiBg5zfQbrj3UTn3L2Pe/j5ItYRAPf98XCPe+33n94bDBTjEgHB7WwGBnxtse+t4ytsTAkCgYPs4cMt47Db61y/PWDuPfN/K7GXxhd86sGNktfyAzzHCu4mLqEA3lrxps5YGMuJ6xNTCv76VPY8AEHUXONgnizG9iGRoqVbTDBPwEpYFrQt61iL1MvCxUHJ6eLWFbOvVeRDeIcHUNx7UwdI2KqxOnKZsdn16UcnEyTxAlFB1zqxW9wrUMojhlKu+iaZmhbvQidLhP/R6kiiXzRlTv80Zc25EzMs7vo59LMagW2TnA7ID9LibyFs/OleiFMXnbZLJg3XdM3jbRdY21to+ua+TtM/snsi+2Gfzhnfx+d+pb34f3Gf+9Df7OnIFBnH8rH0A4PIYN8U7jv83gG9s9ATsNvrnjOqa+ZfiHDfuO19j5vbnlXbAMMfinnTN7p92r3LXjRyzWPDRNw9DVYEjeXnU18Y77XQKDtqyJUjhbE32V0ggShaFls2eCSBHG4MfRVojFouFHtIOENExwDR2FhhemGKaOUorXFtu0guyYXhjz3JZHZKZkkyiyBNxOiGOaPD9fZrHewzI0ur6DYxn0gphbm91Bg7ODVLHslnQ6W3aZ2hIOfpRuKzsd/v5RDi3TtKyseC9Px2ThwYLluOgb/J2Z+iOz9tMsIW9gkOMtd35/V39fad9oYRDuEBPRNoEgBv+o2OZeN/XMZT/SEN9z04/2Cmw9v1MgmDrm1jVt857hH/5+ZzhBDL5wlJxZMbJbaGKm5FB2LfKOSd7SMw/GQ+Il2VTfNFUUHRMvjElVimNmXhQ/UoPpv2GchXYsA3KWyWKzR72TJam2PB/bMLk4kSNvm6SkGIbGubLDj1ZD3l1r4+gaFyou6+2Q6ZJD14/I2QZLDQ/T0Jl2XX5wt8VSs4WuQSFn8sR0cV+Jpfd6XaSsNgNafoRj6qRK0fZjXMugmrdR6nT3uijnsqZzYZJS64YjjfxwrD3aYeRHufZHGfy9svOHDf5wiEE4PHY1+EPfjzL4pq5jmdqOY4YM/667+R1Gvn+dHceZYvCFM8iZFSN7hSZSlQVjLMtgb3/I3uhAwTYJEkUnSLBNg6m8zVTeYrXj0+zFWROaRIG2leyaKOpeBKmiGyYst32SFGw9wmr5fOyZGXK2yd3NHmstD7U1/C5V8J3bde7WPTQNFqo5fuHaPAsTeYqOScsLmSxYmEaWTLvRCnhnrU3ZNVnb8hJFSUoniJivZHknEwV7UD0yHBbZ7IS8tdomTRXdIEbT7oU3nh0j4bMvarYZ36Ea+1EGf2fMP052y9ofw+APGflRIkE4PEa52u+57Pd4bofB3ykQrG0iYYRBH3psZJKgGHxBOFGcWTGyW2hivR3wo9UOt2sem+3wQK9h6qDrGgXToNEN6AYJlbzCtjTec76MqcFbq13WOwE528DUMxFRci3afoJpxLT9rMFZJW9l+RKx4sXLRZ6cyvHVdzcIkxQtVby71iZKFb0wJW/prGg+HT9C16Dei1hp+nhRyo2NLvVuQNG1aHgxP1pps9LyudvwcAyd5aZPzjYoOBZTBRvL1O8z+I1eRL0XYuk6m72AJFXkLZNelGAZ2U1+VLOf4ZwBMfmHx7gGP3PL3zPyIxP8RsTxs4S8ISM/fB1DH+zws5j/9muJwRcEYRzOrBjp5zrkg5gkUbT8LMF0uemx1vJBZaVYXvLwTWCSrVk1fhKRKogU3Kn5NHshE3mH6ZKF0jQSBZ0wwdIN/ATCXky9F9H0IsIkmztT8zJD/9/eWuPP392k60d0woQwTrOSYXWvD0afL7+9edCPSdjC0LX7yuTsHYl2tnnP4KdK4UUJhqZhGtlgwJJr7pmpbw+HALaM/LBgGDb41uC1xOALgnD6ObNi5PWlJv+fv7xDoxfiR+lg197sRdyudekGCQf12CdAc0fntBRo+CkN3+NGzdtxRkr97m7DcLKwxu37znl8MHRte6Ldzhr5XbLpHxSfHzb+LS+i3g2ZKji0g4gLE1lIai8PgWlo6Ps0+HdqPW5t9gYlwpen8lyczB/RJycIgnC6ObNi5G7d4z997dZxL+ORY2gaug761tC6JM1CJv0dfNExMQ0NDY1yziRJs4TOSs4aZOL3eyPkbIOya2KZxr26/a26+r5I2O62370070EGXylFrRtlVS5WVprci5JBxcu43oHhfJdzujvIcRm+vmNqVHLjX3MUUiIsCIIwPmf2Dmmbj67tbd+kqaHvLUNjIm/jWvqgvXGqsq6fhq4TJwll16KSs2j5MV6QTep9z0KFas6mkrMI45TNXsDdWo87NY+cbeCFCVdn8vy1F+dxDI2bmz3iOCVIUybzFkXHpuGFOJaOBtzY6OKYJlES0+jFXJrMU81bFByTgmMNklInCxa1bsRSozfIKzF0fayE1cOg1o0GIqIXxigFBWf8pNk+u5XuDl9/v9fcz+sIgiAI93NmxcjlyTy//rErhEkKikH8/0crLb55s0YvSO7rHfIwmEDR0VDoREkCmk6aplyYzPHxp6f5xLOz9IKIL/9og41OyFurTbpBRNGxeWI6z/mKS9dPWO8E+FGMrmnMlhx+4dp5nj9fZqMT8v/9y1v8wQ+W0YCibfCTT03zxFSBt1Zb3NjsUbAN6t2Q1bbPpYk8aZo1ALsyXcDQNJabHm0/zbwhrkUQpzx3Ls/V2eKgw+p6O2C15bPS9FlvB3zw8gR+lGIZGpMFeyhvBRRq8LPa6jybbnlT1NDzivGHxfXCmDRVzJZcXl9ugIIr08V9d0ndrdfI8PUPo/PqOD1NBEEQhIwzK0auzhT5f/3CCySpylq565nP4ge3N/l///8CXltsHcrrKCCIFYnKSoRtQ2HZBqSK6+s92sESjqlzt9bl1mYva3ZGSi+IuLnRJYpTyq6JYxt4UcJK0ydv67yx3Gam5DJbdvmpZ2Z4e71LvRsyUbB57nyZVMF00SGIYtIkJVWKtVZAGKVcmCxgGQZTRYeFiQTT0DEMjzhWVPI2TT9iruLywnxl8D5WWwGmoXGu4nKr1uX6RofnzmXN1qr5BxvcvSbmqq3mZ2tbz+dtY8sr0X8+CyO1/ZggTjhXzkqZvSimkrMGa1AqCzmlW0qn//1ugmh4cq6EVQRBEI6PM3/HNfTteQF+rKi6FkXXoOUlB+gykpGQNT8btJJXipyhaPkRNze73Kl10VCUXZtumBDEWY8Tpae0vJiCEzKRt1naaBPEiqmizWTOYaXp8cZyJphytslHr05TyVs0exFTBYeWn6Bt9SBZbXmkKRQci4WJPI6hk7OzMJVSGk/OFgmSlJ4fUe8FWdM320ApNRAMRcekG8S8s9bBMgx0tK2ur+N1a91rGJ+maWx2A15fau06rO/SVJ6cbdAJ4sFcnW6Y3CdsHgalFJcn8yxMuHT8mMJg3o52n3BRgEp3eH/Y4fHZec6IxwRBEIR7nHkxshNN04hJsxJKM8GLH3zOuBha9qUpiJKEtp9N2A3irHeHYZhUrRQvSNH0LHS02QlxdI+5kotl6DS9mKWmh9IYhEzKbpZ0CjBVdLg8VUDTNNp+RDVv8vZqB8c0uFPvMVOymSy4vP9SFaUUd+o9GoshtV7I+ZJLJ4iZKrksN/2B5wWyviyXJvN0/Jgnpot4YdZxdZQIGOUFedAwvgc9v1vb+cNA0zQMQ2O+erTVLsOfS8E2mC46oGlD3pvRwiVVwI7ybbXl+klHCB/Y7hES8SMIwklHxMgOLk3muTpd5J3VNslB3SJD9GfVKAUqTLOeFFqKSk2Kjo4OuLaRhSSICLYskKUbFFyDF+ZLdIIEXfOJkpRmNxyEds5VHSquhWObTBds1tt+ZujIvCHVvI1paLyUr1DJWUwUbKYKNm0/K22dKtpoax0uTOZYbgaUHIPFhkclZw28DpqmcXmqQNOL8aMstLPbQL1RXpAHDeM7yLC+k8ZuIandvEMGR98npC9q1JAnpz8PaFgIDXt62PbzdhG0Vwhs+BxBEIRxOL13/CNipuTwgUsT/F9vrqG0ePSI3gOQALaeDclLEkUvDck7DtPFLA+i5BokFZe3Vzq0/AhTT0hSl8m8gx95TBZt3lzu8M5ah+/caWKbGn6cYBk63TDh4kSeH9xtcXEqR94yuVPrUclZREnKdMmh6WUN2JpezHzVZarosNkJqORt4gTCOOE7t+pZ2W+iuDSZZ66SG3w24wzUG+XluDJdGJxb2AoBXV/vDK5zkGF9sHdOyqNmN9HxIO/PUaJpGpnz7NF+Jml6v5gZ9v7cF+oaFjgP8BhJ+EsQHh9EjOxgreXz7ds1vDBBPXzzVSC77Y+6PUZp9p9SzkKprMx3vurQ8mOuXahQ64S0vZC8b9ALY2rdgDv1HmGieGetS5Sk2JaOUpCzNTphwkROB5W1k//hcpMwSXhxoUIYZ2/ibr3HRscnSTVemC+jofHEVI5rFyq0/YiXLlSwDY3v3Grw3Tt1posu622f799p8NRQbsY4oZK+l2Ox0aMbxGx2gm3nr7X8bcb6pYUymqZtEyo3Nrr7EhWZAGiw2QmJU8X7L1V5/nz5kQmSYTG02QmI05SFan6b6HicvD/jog9ysh7d7yEdquTqe38G4a2hvJ++CBqIniHBtFPw9ENkgiAcDY//3XCf/OBuk1fvtuiGMQdNFxl169K3ntB1OFe2afkJvTDlrZU2fpiy2Q2xjaz3SNuPiBPFZhpza7PHX3lulju1Hp0gJmcZBHGKbRi4RpaMGkQJ319sEkYxrm3yxkqLIExZbwf04oTpos1ywyOIE2ZKLi9eKGfiYihRtN6LuFnrUXRNFhsh37/bYLnlU3RMfvKp6YGXZC/6Xo5bm106fsxmJ6Tpxbt6CG7XejS97LG2H6FpUHSskYmsu5GJgJB2ELPRDlBKMV10xjr3MLwqw96Q/nvYKToO6v05TRyXp6rvATqq0NdwuGunp2ebuEnvz/vZWfKeKvHuCEIfESM76AYRm52A3gGVSP+DVWxV0Wz9nJIJkjiGtVaAY5mUXJMoSWj6EZ0wq+bI2wa6DrZuYGrQ9GOWmz7zFZckTVlrB2hAwdJJSYkSjamSDWnKfDXHBy9P0Ohlg/KaXkjLT7hb6+GYGu+7WEXTNJwRjd8uTeZ5cqaQGRHXpOPFVHIpq60ulybzzJbdBxqZfrJpJ4ipdaP7whI7PQTAQJx855YHGjwzV95XKKPomMSpYqMdMF108IKEr727wXw1N1j3bsZwr0qfcRkWWIv1rOppquhsEx1HmYR70jiMz/Qk8ijCXelWA8TtXpsR4kap+0RQP6l58Fh6zzM07CkShJOGiJEdGHoWLjgopgZKgzgF18xuJkkCEaBpmSjJ2wbnqjnWWz6dIMaPU1AKxzLQUEy4NkrTMHSNvJW1WS/YJtFW9Y2GhmEEOKbBZF6jlLPoBglxqvjRWpcnZwo8f77M64stNrstXMvIrq1pTBUdSu79XUFnyy4fe3qGThDzzmqb795poBR0g5ilRlZOvNTwSFIeaGSGRYeugR8lvLvWxo8Sym5WnltwTDp+1tl1pdkbtJ/fbyhjpuQMKoT8MGWp6XG36fHWaocnZwp87OmZXdd5GLkcw+/VNHQuTxUeC+P7sBxnfsxpR9c19CMUO7t5d7YlJPcrs0aVtu/8fsf5IN4eYf+IGBkim08S7nso2ih8BYYC28zKeRMYhH0ile2rco7JBy9V+fr1GqlSNLcmB9e7IbbhcGW6wGY3RinF+Yk8QZLy1mqbpZaPuTX/pRskoDTeXu9wu9ZlruTy8tVJQOPSZJ7nzpXoBjG9KOby5DQrLZ+5cha+aPvZUL5h78bw7r1gG7T8mJWmh6FpeFGW3GoZOi/MVx5oZIbDEn6UsNTw2OyE3Kn3qLo2YZKQsw1ylsGdmsdk0eJc2eX582XcrTDUqDWOQtM0nj9fZrro8MZyCz+OsczMWd8J4j3XeRi5HP332vajfa37ceUs5secFo4jmXk4MXmnCIL7S9H7eT07uzXvVdG1M6lZSttPF/u+Q/z5n/85v/Vbv8W3v/1tlpeX+eIXv8gnP/nJXY//8pe/zF/5K3/lvseXl5c5d+7cfl/+SFlr+dzc7I31J6qTeTf2QgFenDU829pkDDA1aHQCvvyjDeJUZc3QTIOSa9INEwwNnjtXYrXt40cK19B4bbFBy4uIk5QgVthKUdhqRhZECU7OIkqh5Uc8f77K5akCuq5zcSLHa0tNvn2rzmTRZrpos1j3qHcjgjjmykyR8xV3YDCGm4l97Olpvn59E+hxrpJjtekTp2osIzMsbK6vd0hSqOQtXl+MSNMt4afDtQsT2KbOU7MlNDRytknRMbkxws2/Vy5C//UgCxNc3+gC8GSxsOc6DyOXY/i1R637rHGW8mOEB5O1Bxj89Mhff1Rp+6jKrlFeofsqvnb0+JHy9sNh32Kk2+3y3ve+l7/5N/8m/+P/+D+Ofd5bb71FuVwe/Dw7e/KctrdrPZJUUXZN1h+QNDJOoU3/mFHtSiwDTNNgsxNg6lByLYJEoekaE1tD1b5+o44GtIOYME7pBgmmkTVDq+RMyo6BrmvUOgGu4+AaoGvQ9WNcS2dq6zqb3ZA7mx69MKYbJMyXXRq9mOWWx431Lt+70+RDVyawjKxCp+TeSx7VtGxKby9K+eaNGlem8rz/UhXXMkaW6AIjxUJ/p7zZCUlRtP2YcxWXuhey0fazhm69aJBnsZubf1QuwkzJ2faa00Wbjz09zeWprInZpcn8nsbwMHM5JDyRcZbyY4STz3GVto/K69m1tH2PpOedAmfUY6edfYuRn//5n+fnf/7n9/1Cs7OzVKvVfZ/3qCm62ayT1XZAGCnCI3iNvqdksxOgaTqJUmx0QkqujWMqTN1gsmCTKMV606ftxxg6BFFKGKUUXBtL18g5Jn6osC2TjW5ITwfQWGz2+O9vrDGRt3jPQpWNTohl6jw3VebNlRarLZ9uGPPmSptUKcKtkEInyPqqPHOuxA+Xmnzt3ezxKEn58ScmuFXr8cR0YVAyu9r0+Iu3N+gGWdLtx56eRtO0kWJBKZUJKNdgvupya7OHaWhcmMgSTIuuhWPqlFxrIBx25ptcX+9kZbNJysLEvbJZ4L7XnKvkxqr8GeYwKkAkPCEIQp++R+goc4BgdH+e4bL2UaJnp8fHOOZw8iO7U77vfe8jCAJefPFF/sk/+Sf8xE/8xK7HBkFAEASDn1utwxla9yAuTuSYylvEaZZEaugJYXD4ilMDwjgL9VhmOvS4Yr7qAjqOodMKY0zTIFIRXT8hVRoTBQcNMA2dat5mOfTRyCpjbB2iVJGzbRabHu+sdXhhvoKha6y3fd5ayZJYo0QxXXSZLPi4psFyy2OjHVBwTfwo5Rs3aizVPNKtwJKha2hozFdyFBxz0APk1maX6xtdqjmb1XaXy1N5porOfZ4BgFcXW9v6ijx7rryn0R/OwVhu+nznVg3bMNF1BWw39ofljTiMCpAHhSdOUnM2QRAeD4bDYI+io/NRcORi5Pz58/zO7/wOP/ZjP0YQBPze7/0en/jEJ/jGN77BBz7wgZHnvPLKK/zmb/7mUS/tPjRNoxcl9MKIKE3pHVCIDOeVaIClZQmtALECS9fwt7JZDV2j6cWsNAOuTBeo5Cw6YUySxNiGxlQhhx8nXKy6xKnGpakc9W5IrRuQKo1qzqKSt6l1QurdENPQyVkGay2f5aaHbeiYmsbLV6eYLTlZC3gNlpseObvA7JZRzJuw3PJwLA1D02j6EUGscEyN+aq7rZImTfvv7t7nNMozMEosjKrk2fm76AuBr1+vcbfuM1PKQjhXZzLR0w8TbXYCOkHEYkNh6ru3qX8QhyFqHhSeGCfMJAJFEISzxpGLkWeffZZnn3128PNHP/pR3n33XX77t3+b//V//V9HnvOZz3yGv//3//7g51arxcWLF496qXTDhChW2IZBmqgDNz2r5jSCWFGyDWbLLlemi3z3boONTohSCsvQMQxFEityjkHHj9GA5ZbPRjskQZF3THTDZLZoESWK3Nao+zhJWGt5xCn4UYTSIGclXJrOM192mSjYVHMm37/T4Pp6l5mSQ9OPafsRCxN5Lk8VKDgm6+2AvKWz3vaxLJNLk3laYUwYJ/xwpUOjFzGRt6h7Eb0wIUnhfNXljaUWjqkxU7LRgSdnCoPcjFGegWGBEsTpA5NT+5N531xp0wtipoo26+0Ax8zKZmdKWdXMd283MLTMUzRVsAfPjeJBXomjCrHc1511jDDTWUx6FQTh7HIsAe0f//Ef5ytf+cquzzuOg+M8+uz7gm1gmRp36h6d6OBekYaXXcM2UizDIGcbnK/kCKOUbhSja3C+nMOLFY1eiFJQ60WobkjOtgjjLBHVsUwsQ6NoW6y1PSoll9VWQKw0XFOjHSgqBlTyJlem8pwruyg0NnsRNze79MKE+WqO2bLNxck8Ly1kicTvrHVYbYWcr7i8u9qlEyVcX+uQtw2enilQ64aUXIPpgou29XEYusYbSy3u1j0uTOQoORaXp/IDETDKM7BToLT9iCRVnK+4vLnc5o3lLAynlBqEc1peSCeI6QYxtW7IubLLxcksebbvSfjOrTp36h45W8fUNS5P5e8TGMNCwI8SFhs9ap1oZMv4o6oAGfaGdIIIpTiSMJMgCMJp5VjEyPe+9z3Onz9/HC+9K5nR8umFWbKoSRZiGadqxuD+ipnh8+q+4m6jR6VgEUYJmgbTRYc4VpyvOKQKvt320XVoeTG2CUGcoNDIWyZBrPDjlCiJ8WK4knfYiBWoCNsyydspUaRYaQa0ejF5x8S1TT54aQJT05gr2biWzrWFKh+5OjVIMr1T67HW9ii7JmGakCYplZxNEKcYhk7etmh4MSstj4tTWaKppmn8cKlJ24spuyZtP2Ein4Vcbmx0Bx6N4fLgUQLF0DXeWG7x1mqbtY7FRifg4kRuYJTfXm2x1PS4PFUkUTBXcfnI1anB62x2AixDJ2fpvLHcZipvcavcu6/ZWF8IxEnK9fUO7SDGMXW8ML2vZXx/nTNbAma/83F2Y1t31oZiqnB/d1ZJehUE4Syz77tep9PhnXfeGfx848YNvve97zE5OcmlS5f4zGc+w+LiIv/pP/0nAP7Nv/k3XLlyhfe85z34vs/v/d7v8d//+3/nv/7X/3p47+IQWG8H/MU7m9zc9Jgs2Ky2s+m2D8ICCg40gr2P2+jEXF/v4oUJLS9C92I0TXG7puNaJroGtmngRwnRlrJRKFJSUHr2XKhwbYO1pkfRNXhytkijG4LK+o50/Kz7ajOISVPo+hFPzZb4v70wx3w1NzB+Nza6JKnixQtV1jshCsXCZJ6On4VDGl6IFyVMFEzOVaoEccJ7zpeZKTlsdELCOGWp5bHeDbANnfkJl5ub3tizZfoeiK+9u0GSpli6wbvrXUquiaHrWxU0GpaZ5aAXHJP5am5bpU7bjzLRaGhM5m0+fHUKx9Rp+xFKKW7XetlnqBRxkpKzTVbbAZudgChVPH+ujG0YI70Qh93KfFt3Vv3+7qzSk0MQhLPOvsXIt771rW1NzPq5Hb/yK7/C5z73OZaXl7l9+/bg+TAM+Z//5/+ZxcVF8vk8165d47/9t/82shHacdL2I5pehALiJEXXQUtGD7sbRtcgTjVM9s4xSYG1tr9VTgVxqtA0WG37WLpOkChIU2wTXMOgnDMJU8W5Sg7LMJjMWzS1iMKWz6Zomzw5U2C9E1LvRdyu9bhd6+HFMbZhcGEyt5WUCrahcWW6MNjd942jHya8tFDh8lSevG3wxnKLbpgwYzjUuxFrbR/T0AfnvrnSZrHusdzwSNKUZ+aKaGjESbqv2TJ9D8R8Ncdbq53Buqo5iyemi3SCmAsT7mA9TxazfJS2n80NquQt4iTl6kyBy1MFbpV7OKaOaegEccp3b28MGp7NlGxKjsVK00MpxdWZIrc3uxiaYqJgjfRCHHbY5EFiQ3pyCIJw1tm3GPnEJz6xZ4OVz33uc9t+/of/8B/yD//hP9z3wh41QZzihzHtXkS9F6EpMHWIHhCnCRToicLUsgqZPY+NsmumChIFOVMjToB0q2mNBpah4ToGBddmztF538Uq6BqtbkCYKOpdH4VGnGrMT+SZK+dA01hueswUbaIkpZq3mSrY9OKUolK8vtRC07RBXsduxnGm5A5m0qy2fKaLLrdrXTY6AZudkB+tdrB0nSdmSqy2AzY6ARN5B9PQiZKs3XvBMUdOrIV7+Rv9lulpmjJdsOkGESXXpLC1ln4ya389/TVuLme5Kjc2uliGzrWLWc7H5anCtnyUbhBTzdmAQgcuT+WpdQPeWm2z2vTI2QbPnCvx3ovVkV6Iw05kFbEhCIKwNxKc3sI2NMqOxWzJZrNj0fRjojHLabwxEksUmXckK4vNxEiSKAxDR+kKx9CZyDlYpsZM0WIib2PoGq6VVdnc3PRY6wQopWHoKV6UcH2jw4cuT/LMbDZ/Jm+ZdIKID17O2qvfrXssVF2+e6dJrRNyebrHx56eZq6SG2kc+4bZixK8KGUib5OzTXK2wfxEjjsNj67vk6qU6aJDECdYuo4XxixM5AddWWF7zkiffvhjsxMMEmA1fasSJu+w1PCZKbmDCbvDa1RK0fEjHFNjYaKABjimPtLQ522D6xstwjjz3lyazKOUwjZ1XNPAjxMm8vauoRcJmwiCIDxaRIxsESaKzW7ISjtEKQ3X0omTNJtBwPYmwuPU2eycR6O2rmGZ4FomXhiTt3Wmizb1XowXxvhRTDnnoOsG3UjhhwmLjU0MXePGRpdumGLqmREu2BampnNlpshTMwUMQxsYz598qt8JtcG3btW4udHl6dkS19c7XJ7K79qZdL0d8P27da6vd1hq9EhVypPTOQzD4M9/tEatE3K+kiNOFRcmc6QKFqpZiaprGVydKe75mfTDH5W8xY2NLpWchR8n5G2T5+d3D+v013an7tGLUm7XelydLozsVTJTcnhhvsxGNyBJFSU3+yeuaRoFx6Kay3JiHjR0TzwZgiAIjw4RI1s4ps5U0eFWrUMvTomSlFTdq4rZb6FvQjYMT6lsDk2UZA3PkgR6KivrDSJFrZslneqaTppCwTVpdkOCRDFVtAmSlMBPiBKFa2pomsZk3uIjVyaYLuaYK9kAlBwTU9d4cqaQeRGCOOu2GiXoukbDC9GDLPSw1vK3VYj0wydvLLd4fbHFcssnjBXNXsTsE5PZOjshiYKn54oEsaKaM7lT9/jO7RoF28AL420zajRNu6+vR8E2BvNpLEOn5WWP7yx1HUVnq+X8h69McnOjQzln7jp1OGebXJ0uDXI+umHCxYkcM0WbWjdkpmhzcWJ/reKFgyGdZwVB2AsRI1tkM1FsyjkH1/SJktED7vpo7C1Q+rdZU4d4KxE2ikHTIU3BsjSiRBHECaapYWg6mq6x3gwpuiZTeZMkUbiGjq3rBLEiZ+loCiZLDg0vJkp9VloB37/bGiRsZvNuNLphTNOLKNgW771Q5eZmF5MsBPODu81tnT9vbXa5tZkNCXx3rUPLjzPREaX0woSJvMOPXZniGzc2uVXrsVDNU9gSESpVLNYzgTNdzDFRsHjvxSqzZfe+qpSXFsqDFu8vLpTpbjX8KjgmrmVsm0uzk6JjYuo6fpRSdC1aXsw7a92R1S6jcj6UUpRcC13TtvJaxBA+Sg67QkkQhMcLESNbzJQcPvjEJBvtgOVmD63h7Xm84v7+ItqO5zQtEyOOpRFGiiAFY8vVkqTZgKJEKYxUJ0ySrYTZhErOZbrgUs2bPDFbotHx+d7dJmGUYFvZ5IE4hVrX5921Dh0/JkkUtqHxo9U2620fTdNpehFTeYtn5sqYWuY1UKlio+PT9rOJtj+422Sx0WO1FfDjT0xydbrIjc0OQZRSyVlcmMjjRyleGHN1ujBocNb2I4qOiWXofOtmHd2Aa6aJQnFrs3uv22iaDkI53TDh6kxx0D317bUupq4xVbS5dqE6SFxdbXqD0txLk/ms98dQHsfmVkLtbtUuo3I+bmx0KbkWz54rD9ay7fcpO/cjRRq7CYKwFyJGttA0jefPl1lvefzxa0sEYySv7vScKMDeCslAlqhqGVnCZLLlRtGNLFSjUpgsWiRxim1qlHQTXTeYLlg8da5MxTGZLTl4UUInjJnIWQSWkXVeTRXdKMXWdX643KQdJPhhAlomgGq9CC/Myn/9OGWjFxCkCbfWu9za7PLkTJGXFipomkaSKp6YKrDayjwk71ko86GrkySpYqbk8Oxckc1uRNuP8KOEbhBza7NL3jZo+zHfuV2nE8aUcxa3N7ucq7iYhkZt65xRlTX97ql36x7TW56QvnFabwd85Z0N3l3PPD1Xpwt8/JmZLIdjK4+j6Jg0vXjPip2domLYWzI8Bbh/jOzcjxaZZiwIJ5OTshGTO8IQmqax0vJZ7UT7zhHpEyVZK/j++X6SEKdZ9YxGFrLRAdfWKTkWPS1GpYqcbZGzdGYqOZZqPVo5kx+ttrlb92l6EbFKmSnYtP0Yw9CJlWJhtoQfxcRxQiVnkaI4V3ao5W2+e7tBnCaECaSpouJa2HoISrHR9nl9sckT0wU6QUSqjMFsmctTBaaLNhudrB37ZjcahE6+d6exTSRU8iYLEy6zJYfNbsBEwWa65NDxI6aLDqkymC4693Ub7QRZL5S+CHBNfSAONjsBHT+i4lp0gphbG11uTeW3ralgG7y0UN5WsdP/g7q12eV2rUdhK6zTFxXD3hI/SrYN/Os/Ljv3o0MqlAThZHJSNmIiRnawWPNIk3TbxN0HoZEJjIR7uSTm1vdJmnkrNJX9P2dm/y/aFk0vwjIMSjmdSs5C1zXu1DzSVOGaOrapkXNMml5I24uIowTLMsjrgNIJkphUaTiWQb0XYZsaRcfCNU2u53uoNHvxXpjQ8iNafkTeMehFCV+/UcvKhA2N6aKzTYR8+1adW5u9LE/D0AdGpBPEVHMWoNENYi5P5Xl2rkx9S7A8MV1gueGx2g5Zbdd4cqsp2c5/2EXHZKKQVcI4ps4T0wUW6x6pyprPpcBSs8daO2S25AzExVLDJ0kVugYLEzlcyxhcs/8HtVjvsdoO+PCVSfwoHYiK4QqZ6+sdkpRtwkN27keLVCgJwsnkpGzE5I67gwuTeSaLDn7DI0qzDqvJLm4SnUxwpNwL2fQFTMxW3gjZNXKOTpKmmHpWYmoYGmmswFDEqSKIFK6lCKKEmZLLasvH1iFWEV4YY5sGQZwQpYrJnItr6+RMkyhN8UJF3jbIWwamrnFpMk+UKJJU4ZgGQRyhAx03Jk5T5is5XNuknDNp+Vm4A2C97bPc9FlseIMckpWmxx+/luVvNHshfpwVOl+dLnBxIkfBMbFNnZmSg21odPyYD1+Z4uZGZzDFt89w07OFiRxXZ7Ly3LYf8c5al/lqjrv1lJJrEMcpOdvgw09MEiTZef0/mDeWWqy1A6aLzn2ejSemi6y2A25udlmo5keKilHCY5yd+2G5M0+KW1QQBOGkbMREjAyhlOL58yU+9vQkP7jTZKXpo2tQ78X4I9wk5lZlzCitYgAl18SxDNpeRN42MfWs06vSsiqakmtyZTrPajvA0MGPU+JE0Q0jgjhhYSrPVMlho+3T8BI6XkTTD7nT8MjZJnPlbHhdwwtBaVyczIOCphfhRRGNXkw5b1JyLCp5h8mCw+vLLeIkxdI17tY9lps+CSlvrXYoOQaTBWeQQ/L6YpM79R5rHR9T15nIWfzYE5NcnsqqabIW9B7FLa/FfNXFNLImaIWh/JC+sd3LHWjoGov1HssNj41OSKoUQZSyOiQ61ts+zV5EEGfibKdnQ9dgpeFRcgzOV1xeWigPRMWwABgV5hln535Y7syT4hYVBEE4KSFUESNDrLeDwTAz19I5V3XpehG9KMEP7kkOE8jbUMzZ9IKIhr9djhhAwdVJFUzmbS5N5ImSFD9OqXUDvCilYJmUchYFx6QYZt6GjUZAmKR4zRjXspgp53jufBnb0PjmjRrrnRAvSfDCFC1UvHq3STVv8TMvnOPWZg8/TFls9uiGMevtAMfSmbdzXJ7Mc3Ozh6Hp5G0Dx9bJOwa2AbapoZSOoWUVPnGq8KKEJ2cKGBrcqXcxNJ1qzmKzG/LOejt7kxp0g5i1dsiHr0zhRwmOmYV0bm126YYxm92QphcPjO1u7sDpos181eXt1Ta3ax5LDY+ya6HpkLMy0bFY72EZOlGacmWmOMj7GPZsLEzkWGsHTBYcdC3rydL3OIwSAA9q0raTw3JnnhS3qCAIwkkJoYoY2UKprCT1K+9u8s3rmyw3A9Ag3QqD9DGAnJ2FQiaKDou1Hl7oE6RZSEYja3IWRFnTtIKj80sfvEDDi/jBnTorrsntjR5o0NsySjlTp9aLsA2DomuSpIq8bbLW8nEtnY9cnWK6aHOr1sUPE8I4xbEMYpUSxglPzxZ5Zq7E197d4J21LEHTtYwsjGJvhVHKDrV2yHTRoeRabHZCukGMF6Y0/Ahd0/jQ5Srvv1TFtQyKjsl62+cbN2psdELu1HskSUo3SHhjqc25ao6feHKKtXbIzY0OCxN5Sq41EB21bnSfse27AxcbPbpbJbr9HiBLDZ9GL6LphdimTgo4us58NcsNSZXGC/MVlhoe5ysupa0E12HPhmPqWLpOOWdS62TVPH2PwzgC4EHhk8NyZ54Ut6ggCMJJQe6CW2SVGD1ub3RZ6wRYhoauaTSCBAW4BvgJFGyYLLosVBwmSy7NrsdU0aLRiyjnLKI4m+sSp+DoGn6k+N7dJkopat2YpVrmubANgyTJkkt1w0DTInKOQcePiBKFZRrkbYM0hXdW21lYpuRQzdncbXgkacJCJUcpZ/GDLQ/J7brHaidgqelDqoiTlIuTWSfXkmOxUM2R3Fa8vdbGtkzW2x7z1TyfeGaGzU7Ae+bLTBXsQQ8Ox9R578UqV6eL/OXNTTp+zNPniqy3Q3pBRH2rm2k1bzFfdZkuZt1gtxvbe2W0/fDIrc0uS3WP170mtzZ7XJpw2ewEuJaBZegoleBaGk/PZnNlNE3bZrz7omenmAjilDv1HtFGimXovHihPHhupwAo2AZrLX+b8HhQ+ORh3JmjBM5JcYsKgiCcFESMbNHPJXhhocrbax2afoKupeQcnY6f0u+RlSpoeiE/XInR13okKkVDZ6aYVZO0/ZBWkNL1I1AQRAnfv9skDCP8RNHwItJUYdg6Jcek5GZD8WZLFrc3fVpeiK7rKJXSCROC2KMZxkzlLfKOxUxJZ6Jg0+iFVPMOXpjw9es1pgo2Sw2PuZJLmiqiJKXiWpyr5nhhocJyw+fJmQIoWG56WYKrysSQoek8d75C0bX4yjsbAyP53LkS00WXibzCNDXeWm6z2grQgGfOl5mv5mgHMY5lsNTwmS46I8to+5UyfQOvaRob3ZBqzub6Rpc0VdytewT///buNDau8zr8//cuc+/sM+RwF0Vq8SLZlhwvsaM4aX9tjPrvv2G0CJCkhQuoVfsigILaMbrELQo3KBInBdI2iAMnbgobRWOkRlu7SxqkrtPYPwNxvCq2Y1mOrIUS9232ufv9vRjOmKRIiZKGHok8H0AvRIrDMxybz5nznOc8no+qwJU9Ka7sTS1JBtayeJu6ymBHjEw8QqHqYupq83OLY0oYGjNlm0OnCkuGrp2renIh5czVEpxLoSwqhBCXCklGFiRNnYrj4/s+V/clGS9YKIBlubhuvXvVDaHmQqhAvuaB4tObjuIFIWmz3qxqRGLUvCooGkHo44VQrbkoYUix5gIKMUMjCBTsAEqWT3dK47rBLLqar38uDJks1ihaDpmowXTBYqpQoydpcuNQF9tycY7PVHhnooSpa5Qdj1zSYLrsMFOuN7d+qDcFYX0r6J2xEh2JCKlofVT70akyI3NVruhJkjB1ejMmu/vTnJgp8950hWwswmSxwtaOWHMBv34wzYeHO3hvukLM0Ni7JUPF8ZunYBYv3suP0Qbh0mO076tvfxm6wtaOOAEBh8cDEqaOqqqoqtrcJlm+eK9UcUhFI+SSJn4QklvYjmpYHNNU0eL1kXxz6FpjaixA2XYZzYfoqtqS7ZPFCc5ovtqcTiunaIQQ4n2SjCzoTpkM5+JMFGtc0ZvG0CIEYcBUyaIaKDiuD149uYgoUHFDdDXE8eq9Idm4Xl9gkiauF5KLe+iaRi4R4eh0hULNxfZCohGlfgxWV9jakeQjO3MoQCaqk4lHGC3UsFyfiKbSla5XOabLNt1Jg0Q0wrauBNu6krw1VmKi5FBzPXRFYbgzwbUDaSDFTNlGV1TylkPWNJitWmTj9d6M7pTJ/9nVw+sj+WZVYFdfCoDxgkXF9khHdSq2x3jBYltXku1dCRRFoS8b57rBjubPbKponXPrY6X+iIRRH7JWtj12JhNc2ZtivGAzmq/PE9nencJy/bM2dq5UcVjr9sfyoWuur3NytkrC0ChU3frx6N54c9vpYix+/hXbo2zV+2nkFI0QQrxPkpEFiqIwnEtwZKLE6fkaqXiEjKkThAEly8P1AsKFJlXH9+lK6HTG64uV7YfMVRzKdkDZ9omZGvt29HJ0usRk3sLzA6quT9RQ6klG0uSGoQ6SZgTHC5oL6mBHjLLtUq66hGqUbCzCkckSUV3jqp4MplGvFJRtD4WQbFQj9H1UTcX3XSDC1s4Y1wykmS7ZTBZtetMmL52Y493JElMlm21dCfrSJjcMZTF1lVQ0QhiGvHG6gOUEaIpCvuaiKQqWEzQv1Vtp0Vy++DceZ/HFeACZWP0/s8VzRz5+ZXfz67qSBt0ph0xMJ2FUqTkeunb2ysSKWypr3P5oDF0Lqc91SRj1Swljhs5MxUHTlCXbTg0XMh/kfO7UEUKIzUqSkUW6U/VFerxQY3S+hhtR6M3Ub5+drzooav0HpqkqHXGD7lS0PqsChdFChYgWkI7qJDSdfMXC83xqroem1CsrWzpiRFSVPYMZPrG7h7fHirw3U2Z0roap10+OTBbshUUyYLJokzQjdCWidCbq/R+Nhs6S7XFkqoLt+oRhvUckopXpScW4bkuavkx9++it0QLTRQsUhTdOF3htZI5btuXoSkWbScZ7UyXmyg7pmM5AJkYiqqIpGrv6U4wX6pWO7kUDy2wvaCYyjerBShfjjcxVKdS8ZnKy+Kjt8qSh0WsynEusqbFz8cmcsuVydLLEbNluXqy3PElYPmdk72CGkbkquqbg+gGn52vMlG0AtuUSS6a3NlzIfJDF20Nnu1NHCCE2M/ltuIii1Eejb+9KEoto1Nx6D0l04Z2z6wEKmGpI1akP5Ko6Hn4IlhcSEjBRtBjIxrD9et/BVNkmX3GouUF9amgiytaOOHEzsrAwWfxiukyp5mFEVCKaynBnnFwySs3x+KWruyGE/myM3f3vD/Ea6owzmDXRVY13J0u4vk/KjGDqKhOFGuWazak5i7fH88zVPFTAiGiULI2i5TJTsanYLh/ZUZ8Rcmq+upDQqOwaqI9SHy9YzUWzsRDPlm1Oz9fY2hGnc2E+yOh8jfmKy2zFImrUR7Trar15dK3zNIIg4J2JUnNI2rZc/Vbh5ds+jSSjUXE4OVthPF/jvekKiqIsuVhvsZUSiVzSZK7i0p+NoqAQjajNOSsr9Yxc7HwQOUUjhBArk2RkmYrjN6+af/XkLMemKhCGhAtvtHWtvsBqmkJHwsDxAopVp34cV1MxdY2UqdOViKApClXbb46B15X6MLWQkNmyzVTJYrZs0xEz8IOQqK6RiunUXJ+kGZCORZgp2fRlYuzqSy1ZYK/qS/PG6SLHZiooKlSdEFVxSccMslqE0/M2b40XOTVvYbkBnfEInUkdLwh59eQ8qqoyXXLwgpCtHTEGO2KkYzqn8xau5zOQjTWrH90pk2PTZebKDpbrMV91GMiaC1UJh+mSw3y1fn/OQEeMXMJgOJcgDEMKteKaKgHvTJT4wZsTzYQIoCtprlqJaFQcyraHqip0xA0ad+aslCSslEg0qivjeYtc0mTPlnRzG2ylZOFi54NcKsOFhBDiUiPJyDKNseKHx4pMl2ymKzZhqBCPaHh+gKFr6CromornhySjGtl4gqrjLTRGKkQjGiWnXm1IGDqKAio+PWmTdFTn7bFifYqqqpCO6syUHfwgBCUkFzcY6oqzrTNB2alv8azUlrCrL8VHdnSSMFS60t0UKjaZWISYoVNzPX5eqJKv2nSnTebLDqqqkIlF6ElFUQhJRQ0SplbvP1EUckmT2bJNseoyXXLxw6WLf2OGR77qMFGsH8PtTkXJVxxOzlUpWz4diQi6rpFb6LUIw5C9Z1ncF6s3kgZc3ZfmyET9Zx+NaOesRCTNeuPwZPH924Qv9D6axkWBq5HKhhBCrA9JRpZZPFY8FqlXOdRMyFTZxg0CDE3jQ1szDOXiZKIGFcej7PgUqw62G3LjcAcpUydqqKgo5BImJ+fKFCouCTOCE/i8N10hHTXQFbhpuIPD40UUFBKGytaOGHftGSAa0fjFVJlYROPEbIWRuWqzFyIMQ2bKDh0Jg+GuJAmzPjHV90PemSihqVB2PYIQapZPOqpzzUCG23f3sqUjxttjRY7PVilYHl0pk6HO+pbIi8dmieoqPWkTy/UpWS5QryqULZctHVGuGUjxxqkCmgof3p7j+HSJnrRJX0ahVPOI6e9vbyyuBJyr+bM7ZRLRVI5MFIlo6qoncVh4rKmixchclTAMubo3ydaOGIqinHE53+LHX55ILK9UTBWts/aESGVDCCHWhyQjyyhKvbKRSxokozqjeYua4xPVFYYHMozMVqg5Pn2ZGHde24eiKBw6lednp/MUqw5BEJBLGWzpiJNcGLu+rSuB7fq8O1ViumxzYqpKJl5hqDOGqWtEjQhDOQ1VhWTUIBrRsL2A49NlJoo2cUMjYegM5xL0pKPN/gcvCICQYtXh8ESJ0bkyZQf27ejE0DS25xKoGuzqS/Ppm7fSm47yzkSJiKawrTPOcC7W3E55Y7RQP3FTtDidr3JVb4qtnTGOz1Txg5CS5RLRVFRF5YreJGFYn6yaMCP0EFJ1fFKmzg1D2RWTgXM1fzaOFzd6Rnb1pVAUZcVKxHTJ5oWjM7w3/X415Jeu6m4e1T0+Uzkj4VlLInG53Bkjt/4KITYaSUZWkDTrczbemypj6hqZqIGVqp9YcXyo2j6n5mrMVV26U1HGixYnZqtYts98zaMjaXLdgE4YRuu9IprKxHyZEzMVxuarWF7IyZkiWzpihIAXhCgKlCwPdeFm37F8DT8McT2fXUMdmLraXBwbi+aWbJy3xwr8bLTA4fEyfuBTqHm8eGKWqK6zszuBqqgYmsbp+RqvjuT56XuzKIpCNh7husEMqqryf38xzSsn5hkv1DA0laihUnN8KosHds3Xx8rnkiaJhSbViuNTczwOj5fQlPpNvV3JlRfGcy30qqpyzUDmjK9bKYEoL/SFZGMRFveJABd1G+6F9oR80MmB3PorhNhoJBlZQffC1kXZ8tjWlWQsX+HIZJGfny4SjdQvnSvVvGZfw+nZKv5C/8jIXI3nj0xydKpEJqZjaDr5msPpuSpHJkq4fn3CaESNEAQhpqaRNDSMiMbOniS/dGUXpq7iB7BnS5aqE5CvOvVKy8LiuHjR9IKQIKgPUzM1Ez+AroRBR9yk5nqoispc1cYPA96dKFOseVzRlyRfdZvxl22P7qTJ3EIT6hU9WbqS0SV3wuia2qzMNIRhyCsn5ihbLh0Jk3zFXrKdtFgrL4dbrU+kXaddPujk4HKp4AghxFpJMrKCxgC0Qq1+t0p3KkouYeJ7cGSqzOl5i45Evafi1RNzHJ4ocGq+Rs32UFSVuKnz7uQUPWmTHT0pyjWPk7NlXD9A01Qc3yfEpzcV5Zot9epEYyR7Y6tBUxVqrs/O7gRDnXGGc4nm4rh40dzaGcNyPKZKDmXbozNhcN2WLJbrc2K2ihd4uH6IqWuYukbcDDk1W6M3bTb7MpKmzmTBImPqJCIanXGTjkSkfuuv6Ta3TpZPJJ0u1ZOP47NVfnpinp6UQSKqkzD15s2/jZjDMCQTqw9GS5h6sx/lQqoI3SmTj13R1ex1Wdwn0o7TLh90ciC3/gohNhr5LbaKlaaLThZrVN0AQ1eJGSqHx4u8M17i1JxFzQmouT6GBoT1iatl2+fwWBGVkGLNR1UVVAXius7Nw11s64ozXrDoTBrs7k83302v1my5klzC4P/f08fWzvjC/SoKt2zv5NCpeXZ0J+hKmhyeKGJ7Pr0ZE12JoSghN2/PNfsyGgt7I1GIRrTmZNaxvIUfhCtOJC3b9a2Z3f0pbM9nd38aLwh57eQ83alos0oA8OZoET8IKdsuYQipaOSCqwiKotCbidGbiZ31NfugTrvUkwN4e6xQPyrdGSMMw3XbqpFTPUKIjUaSkVUsf5cchiE3bcuhqhq6qjBXsXh3sowXhFQdF9uvJymhojBfsUku3MhLGKIpCpl4gKopWK7Ph7ZmOfCxYXRdX3FBOdc79JW2BX7tuv7maZCJok3CjJCMRkiYOnu3ZNnaESMZjSyZHdJYLFda2KF+yd3Z3vEnTR1dVVFR6U7WB4d5QYihaWdcjNd4nNdGahDC1X3pllcR2nXapTtlMpCNMVGwMDSN0fnaGYlbK8mpHiHERiPJyCpWakpcfOJDVWCsYDFdtrE8H9uBMAJxQ+GqvlS9Z8PxycQNijWHmKFxTcqgavv8f9f2omnaGYnIatNGl1ttW2DxO+Z4RGW24jBTdhjqjLOrL4W6MBV1rc85YWhn3Q5ofL+S5XLtlhQV2yNfdSnUXEbnq0vul2k8TsLQKFker43MNb/H5a5xAqs7FT3jNZGTL0IIcW6SjKxiqmjxwtGZ5iJy284cc1WXV0/MMl1yqdoOYRhiaAqdcRMrEpCJ1weJ/cpVPVzVn+G/3hxnrFBDVxSyCYP+TIz+TIzBzkRz26JxodxsxVlyk+7eweyq76xX6xlovGPuDkMOjxd5fSSPoWk4XrCmd+rLKy57tqTPuh3QfIeejjJVtBgvFAgAdeE5LO5zaTxOzfF4e6xI1fYoVF1OztbHuF/ui/Rqr4mcfBFCiHOTZGQVI3NV3puukI1FmCxWSJk602WHQ6eKTJYsVMCMqGzJxkmbBm+NFYioMJCN0pWO0RmPEIZgOwG5bIyetMnVfWl296cpWe6SysbJ2Qqvnpzn5GyV3kyUIAg4OVs564CwsyUJ0yWb10fynJ6vNT93tnfqUE++Xjw2y6n5KtcNZLC8gIrjs6M7uabtgMXHjcfyteYU1obGtsKx6TLpmEFPOsZPj89yeKJE0fIv+0V6tddETr4IIcS5STKygjAMma84zFcc9IWJp/XL0xQMXaVYddmai+N6AX4QkIppbM3F8P2QK3vSWK7Pm6NFqk5AIqpzOl8jlzKXNKkubngsVB0mijZ+GHJkooTnhxgRjbmKe0GTQMu2h64qdC2czDEXTUVd6Z06wAtHZ3jjdIGpks10yWbvYPa8Tmms9YRH49+dmCkDq9+Qe7lZ7TWRky9CCHFu8ptxBdMlm6LlEtHg1FyF/myMzoRRP+abNknM6syXHdLxCIZev9Pk2i0dHJ+ucM1AmiBUsFyPuKGSjsbQVZud3UuP5i5ueJwp1wjDkP50/d/2pg0Spn7B76aTpk5u4RhuLKItmYq60jv1xscHMlEyC6dolo9VX8s497Wc8Gj8u0xMJzlXXfWG3I1CTr4IIcS5bcwV4CKVbY9kNMJNw5389PgscVPD8nz6M1H8IKRSs5muOFzbn0FdaF5UqVdNClWXXNLkip4kXhBStj2Gu+JcP5hdMpp8ccPj22MhiqoQN3SGu+rNpuMF+6zvps+WHNQXwOyKn1vtnXp9iJgN1IeIDecSS5KNc/U+rPWER7OvJWUynEts+EVaTr4IIcS5STKygsaR1cmqRTZusmdLFssNGMtb/HysRL4WMJ63MdQKfVmTvmyMlKkz0BGlL22Sjhl0JQ26U9E1XUffmTDYM5hpDgqrf61z1oX6bMnB2RbA1d6przZErKHVvQ+ySAshhGiQZGQFq20lVGwPxw/oy5iMzFVIxVTS0QgjM5XmTI8re5LNpOBsi+25Bput16VuqyUBK80aWVx9sVy/fpxZeh82JDmCLIRoJ1lRFln8CzlhaGzteH9xHuqMM1O2+dnpAkcmyvghlO2AQs2lbPtEdJ3JUoXhXHzFAWLLXWxl4INojFxafYEtHbEzxryvt1YvkhfyeJthoZYjyEKIdpJkZJHFv5CXjy1XFIXd/Wk+sr1GXFeImRGqtkvc0PCDgJLlkq86zFeddR0F3vBBNEYur75EIxo7upMt/z5n0+pF8kIebzMs1HIEWQjRTmcfybnJLP6FXLY9KrZHfybKXNnh8HiRmbLD3sEMnckoo/NVyo5PLKIRN3VmyjYRVeXUbI1XTswxVbQIw/CM7xGGIVNFi2PT5VX/zVo0Kis7upMr3pLbCpfCsdTFr4m/0BD8QT9eq2O4FF0Kr7UQYvOS3ziLLP6FXL8cj/pFePNVQkJcP6Q/Y2J7PgHQkTCImzq5pEkmapCNGxyZKPL2eJFCzVvxHfTl9C77UjiW2upF8kIebzMs1JfCay2E2Lw23m/Vi7DS3S5vjhZIx3R60yYn56pUbJfOhImha0yXbPwAruxNMZa3GJ2vEgKZqM474wUqtstHduSWVC4up3L4pXDipdWL5IU83mZYqC+F11oIsXlJMrLI4rtdfj6a5/tvjDOWr1GouRyZKNGdMvH9kFRUIwhCohGV4Vycq3uTdCVNMjGdIAx5/VSeqZLDdMXG9QOuGXj/2G798rkP7rr5D8J6Nni2epG8kMeThVoIIdaXJCPLhAuXzD3x0givnpwnDMDxQ1w/YHtXgrmKTRAYmLpCJhan5vjMVtzmIC+AuYpDytRRFJW3x4pMlSxyiSiuH3DDUJb+TPQDu25+vYWLLuVbyyV/QgghxHKSjCwzXbJ57eQ8kwULxw0wIipxTcULQn56bJaYoaOpFYY643xkZ5KJfI3D40WA5lTR4VyVN0cLTJWqmDqUHA/HC7HcAEVRuKo3SXcqSn8myjvjpSVffyEVhXYePW38vE7P1+hadimfEEIIsRaSjCxTtj0MTWN7d5JTc1XKtkfK1IkZKumozlV9GV45PssvJktMFCyiEQ0FBdcP2TuYoTtl8vEru4hoCqfmqwxmY/z0+BzjhRp9mRjzFZv5ioECvHRsjhOzZYZrCRwv4PqtF1ZRaGdTbOPn1b1wKV9sYTtKCCGEWCtZNZZJGBphGGA5HumojrLwsYrtoyoq704WURSFvkyUubJLoeYyVapRsjy25WL0pKP0ZmLs29lF/FSeuYpDR9yg6njMVxySUZ2i5dKXiVF2XBRFQVEV5isuJcsFOO8KR6MptlWVlvORNHU6EhEATF1dcimfEEIIsRaSjKxgsmwxMm9RcXzKrk/M1Jkp2LheSDxSrwLEDR1S8ObpKj85NkdnwmDXQIqdPfUtk5LlEjM03GLAUC7OXNkmAPZsyVJz/YXkIUYyGmGmZBPVVSzX5/WRPBXbI2HqfPzKrjVNc20cPV1+DPmDqJB0p0yu37rypXxCCCHEWkgyskzF8XG9kK6kiabA3FiR2bIFqGiaSmJhrkgqpuMXAzrjJnsGs+SrDq7nL2nmdH2fiKaxuz/NS8fmKDsuEwWLXLJ+kd50ycJyPTJxnRuGslRsj2MzFbIx47xGyzeOnh4eLxISsnsgzXjeWrF3o9X9JXLSRAghxMWSZGSZpKnTEY/w1liRUs0lG9fxghDbC5kt2wxkouyMR/jQ1iz5mgvM4Xg+2bhBRNeWNHOGQYhiqrwzXiJfc8jEIrh+wEA2Rmc8AiikzPoFe11Jk6rjL0RxflNZGwkBgOuHjOetVYdzXU5D14QQQmwOkows050yuWV7JzNlm7mKg+V6qCjkLY+R2QoTRYtc3uCagQw7uhLEDR3X84noGq7nYzkBXUmDmZLNYEeMG4ayTJfsJRWLaESj6gakohGu7kszlq9RcXyGOuPs7E5Qtj12JhMMdcbPO/bGcK5670vIsenykgrIpT50baNcSrdRnocQQnwQzvtumueff567776bgYEBFEXh6aefPufX/PjHP+bGG2/ENE2uuOIKHn/88QsI9YOhKApxM8K2XIoretNous58zUNT4KreFEOdCSzP583TeY5OVbDcgN5MDMsNmCo55C0HQoXBjhg3Dnewuz/N7v40uaTJ2HyNkuUyW7axXB9NZcmI8Z50lI9f2d38c74Vi8X31SiKwpujRX4xWeaN0wWmSzZw6Y82b1Rulsd9qVt+59BU0bosn4cQQrTDea9ElUqF66+/ngMHDvDJT37ynP/++PHj3HXXXXz2s5/lu9/9Ls8++yy///u/T39/P3fccccFBb3eEobGbMXi8EQRQoW4oeH4AZqqUXU9ooHKeMFiIBtnsmhRthwsL4AQ/CCkKxVh386u5hj4RsXi5GyFiuMxW3HIV122dMSak1kb75xb1X+xWgXkUh9t3o7KTSuqGMu3vzIx/ZKuQAkhxKXkvJORO++8kzvvvHPN//5b3/oW27dv52tf+xoAu3fv5oUXXuBv/uZvLtlkBMDQNKq2R7Hms3drhu6kgaoo2H7AYEec10fmefHYLB0Jg4LlMDZvMV9zURWF7qTBbMWh4vjNxa0nHaVse8xV3OYCFY1o7OhOnndsa1k8V6uAXOoNpxdaubmYhKIVfTTLkyjgkq5ACSHEpWTdf0P+5Cc/4fbbb1/ysTvuuIP77rtvvb/1BWskEdu7krw9XmS2ZHN1b4prt2QYna8xV3GIaAoxQ+OWbR2cmCmTj2hkYgamrlJxPF47OU9kYXLrDUNZdven17zQnmthXcviealXQFZzoXFfTELRimrM8td2qDPe7NG5nH7+QgjRDuuejExMTNDb27vkY729vRSLRWq1GrHYmUdXbdvGtt/fYy8Wi+sd5hJJU8cNAlRV5cPbO9FVhW1dCXb1pQCYKtn0ZuIUqg5TRYdUzGCwU2Gm4uCFITFVo+YFWF7ATMkmDOtHhde60J5tYQ3DkJOzFUbzVbblEtRcf8XF81KvgKzmQuO+mISiFX00K722iqJcdj9/IYRoh0uydvzQQw/xxS9+sW3fvztlcuNwB4qiNC9/G84lUFWVaESjK2nSn41yeKxIb8ZkV1+KMAw5NV8vz8cNjddH8pyer9GdMjE0rb44pqNrWmjPtrBOl2xOzlaZLNpMFm12didImvqmP71xMQlFK6pIl2vyJ4QQl4J1T0b6+vqYnJxc8rHJyUnS6fSKVRGABx54gPvvv7/592KxyNatW9c1zuVyCYOreuv9HEOd8eYC1Vj0xvMWuaTJ7v50s2rRl60fxQ3DsJkIGJpGRyJyXotj0tRRFTg8VsTxfbZ2xpqP2Vgwb92e48RMuRnbZp8fcjEJhSQSQgjRXuuejOzbt4//+q//WvKxZ555hn379q36NaZpYprt22OfLtm8OVpsLuyKojSTi7UseoqisLs/TVfSvKDFsTtlsqUjxlTJJqKpjOVrdCXrTbBJU0fX6qPjt3TEGc4lVpwfcqH33FyuJKEQQojL13knI+VymaNHjzb/fvz4cQ4dOkRnZydDQ0M88MADjI6O8g//8A8AfPazn+Xhhx/mj//4jzlw4AA/+tGPePLJJ/n+97/fumfRYmfbJlm+6DXmSyxf9C9mcVQUpbkdtDy5KFkuA9kopq6SikbOqNg0tilsL+D4Jq6UCCGEuHycdzLyyiuv8Cu/8ivNvze2U/bv38/jjz/O+Pg4IyMjzc9v376d73//+3z+85/n61//OoODg3znO9+5pI/1rtR/sFpPxrm2R87Wy3G2z51vcrG8YlOyXJlzIYQQ4rKghGF4fhehtEGxWCSTyVAoFEin0+v+/VZKEhYnHapCc2DZbNlmtuywpSPOWL7Glb1JdnQnm49xcrbCyFyVhKmjq+qSJKIxpXO1UzOLYyhZLkenKgxkY4zmq+QSBrmkueoWzNkeWwghhPggrHX9viRP07TbSlssi7du3h4rcHS6RNzQCcKQpBE54xRHI3kZna8yWbK5dXsnlhssqVCcz3YQvD9Eq2J7lK36ALWNNmdECCHE5iPJyBot3jaZKVmcmK/SGTOwPJ+P7sxxZW9yyaLfSDS2dSWZLNmcmK2wJRtfcqrmfI6jLk4uGtWYs23BSEOnEEKIy4UkI2u0OBkoVB3eGi/i+1BzfRSU5lj3RkPrbNmmZLkEQcCOrgTDufrJl8UViq6kwUA2ynTJpitpEATBGbfsNixOLpKmTqHmyahxIYQQG4KsYmu0OBmYKVn0jJtEdQ3L88nG9OaJGsv1GZ2v4YchigJdKZObFpKQ5X0dM2WHsbyFH4S8M1EiDCEVjSzZelmpf0W2YIQQQmwkkoxcgOFcgr2DWUqWSxjCfM1l5N1pkqbObMUhoqrsHkgzlq+RW5gPspLFPSOvjdQghKv70ku2XlY7rSNbMEIIITYKtd0BXI66U/XJqx1xAxQYz1scm6kQM3R0VcHxfUbnq5Qsl9myzVTRYqVDS4t7RpKmTsLUGc1XKdvvf93iI7p+EFK2vTY8YyGEEGL9SGVkjRZvl1iuz1i+Rr7qMl1yuLovxVTZ5s3T82TjBtu6EhiaQsXxmK04FGreOU+8JAwNgJG5KmXLY7Zc/7qBbFSuohdCCLGhycq2Rou3S6ZLFrqqkI0bHJkocmpWpTtpYrk+hqZRc3yMmI7n16shjWbW5cnISideKo7PXMVtnpQxdZU9W9KMzFWBelLUuKdms1+OJ4QQYmOQZGSNFvd3FKous9X6cV03CMlXHXpSUfrSUQY7E82qyen5GsdnKkQ0lT2DmTV9n+XHfVPRCACFWv37F2pF9i4kMZv9cjwhhBAbgyQja7Q4SehIRMgmdN6dDIhGNGpuwGzVRl2URKSjOoMdMULqp2/KlrvkNt/VrHRS5vhMZcXhaGcbmiaEEEJcLiQZWaPlSUJ9nojN6fka3SmTpKkxnIs3R7SHYcjIXI1jMxUATs3X2NZlr+nemuVbN6sNRzufoWlCCCHEpUpWrzVaniQEQcC2rgQzZZswCMklDIZziSV3ywzn4lQcj225BDXXP6Ny0dhm8YKAiu0x1Pn+YLTFFZTV5orIvBEhhBAbgSQjF2i6ZDNRqKGrCvmaQxDGljSXKorCcC5BoeZhuQG6qp5RuWhss8QiGm+cLlC2vBVP3qw22l1GvgshhNgIJBm5QCNzVY7NVNEVheMzFWIRDU3Vms2lcO7KRWOb5cRsBQjJxiOM5qtkYnIyRgghxOYhycgaLO/t6EoazFcc8lUbVVEIQuhORZtDyc528+5ijWQlE9MJFkbCK4pCwqgu2fIRQgghNjJJRtZg+RHagWyUQs0lomkUqg7ZeIQwDM+7ibSRrDQqJm+PFsgmTOYrNidnK1IdEUIIsSlIMrIGy4/QTpdsUtEIv7qrl+PTJQayMXb2JElFIxfURNroLzk5W+XIZAmA1JxUR4QQQmwOkoyswfIjtN0pk7G8heX6DHYmWjJsrDtlnvP0jRBCCLERSTKyBssbUbuSBl1Js6VHatdy+kYIIYTYiGS1W4OVGlHX40itzA0RQgixGUkycgmRuSFCCCE2I7XdAQghhBBic5NkRAghhBBtJcmIEEIIIdpKkhEhhBBCtJUkI0IIIYRoK0lGhBBCCNFWkowIIYQQoq0kGRFCCCFEW0kyIoQQQoi2kmRECCGEEG0lyYgQQggh2kqSESGEEEK01WVxUV4YhgAUi8U2RyKEEEKItWqs2411fDWXRTJSKpUA2Lp1a5sjEUIIIcT5KpVKZDKZVT+vhOdKVy4BQRAwNjZGKpVCUZSWPW6xWGTr1q2cOnWKdDrdsse9VGz05wcb/znK87u8yfO7vMnzu3hhGFIqlRgYGEBVV+8MuSwqI6qqMjg4uG6Pn06nN+R/aA0b/fnBxn+O8vwub/L8Lm/y/C7O2SoiDdLAKoQQQoi2kmRECCGEEG21qZMR0zR58MEHMU2z3aGsi43+/GDjP0d5fpc3eX6XN3l+H5zLooFVCCGEEBvXpq6MCCGEEKL9JBkRQgghRFtJMiKEEEKItpJkRAghhBBttamTkW9+85ts27aNaDTKrbfeyksvvdTukFrm+eef5+6772ZgYABFUXj66afbHVLLPPTQQ3z4wx8mlUrR09PDb/zGb3DkyJF2h9UyjzzyCHv37m0OItq3bx8/+MEP2h3WuvnKV76Coijcd9997Q6lZf7iL/4CRVGW/Nm1a1e7w2qp0dFRfvu3f5tcLkcsFmPPnj288sor7Q6rJbZt23bG66coCgcPHmx3aC3h+z5//ud/zvbt24nFYuzcuZO//Mu/POf9Metp0yYj//RP/8T999/Pgw8+yGuvvcb111/PHXfcwdTUVLtDa4lKpcL111/PN7/5zXaH0nLPPfccBw8e5MUXX+SZZ57BdV1+7dd+jUql0u7QWmJwcJCvfOUrvPrqq7zyyiv86q/+Kr/+67/Oz3/+83aH1nIvv/wy3/72t9m7d2+7Q2m5a6+9lvHx8eafF154od0htcz8/Dy33XYbkUiEH/zgB7z99tt87Wtfo6Ojo92htcTLL7+85LV75plnAPjUpz7V5sha46tf/SqPPPIIDz/8MIcPH+arX/0qf/VXf8U3vvGN9gUVblK33HJLePDgwebffd8PBwYGwoceeqiNUa0PIHzqqafaHca6mZqaCoHwueeea3co66ajoyP8zne+0+4wWqpUKoVXXnll+Mwzz4S//Mu/HN57773tDqllHnzwwfD6669vdxjr5k/+5E/Cj33sY+0O4wNz7733hjt37gyDIGh3KC1x1113hQcOHFjysU9+8pPhPffc06aIwnBTVkYcx+HVV1/l9ttvb35MVVVuv/12fvKTn7QxMnEhCoUCAJ2dnW2OpPV83+d73/selUqFffv2tTucljp48CB33XXXkv8PN5Jf/OIXDAwMsGPHDu655x5GRkbaHVLL/Pu//zs333wzn/rUp+jp6eGGG27g7/7u79od1rpwHId//Md/5MCBAy29qLWdPvrRj/Lss8/y7rvvAvCzn/2MF154gTvvvLNtMV0WF+W12szMDL7v09vbu+Tjvb29vPPOO22KSlyIIAi47777uO2227juuuvaHU7LvPnmm+zbtw/Lskgmkzz11FNcc8017Q6rZb73ve/x2muv8fLLL7c7lHVx66238vjjj3P11VczPj7OF7/4RT7+8Y/z1ltvkUql2h3eRTt27BiPPPII999/P3/6p3/Kyy+/zB/8wR9gGAb79+9vd3gt9fTTT5PP5/md3/mddofSMl/4whcoFovs2rULTdPwfZ8vfelL3HPPPW2LaVMmI2LjOHjwIG+99daG2o8HuPrqqzl06BCFQoF//ud/Zv/+/Tz33HMbIiE5deoU9957L8888wzRaLTd4ayLxe8w9+7dy6233srw8DBPPvkkv/d7v9fGyFojCAJuvvlmvvzlLwNwww038NZbb/Gtb31rwyUjf//3f8+dd97JwMBAu0NpmSeffJLvfve7PPHEE1x77bUcOnSI++67j4GBgba9fpsyGenq6kLTNCYnJ5d8fHJykr6+vjZFJc7X5z73Of7zP/+T559/nsHBwXaH01KGYXDFFVcAcNNNN/Hyyy/z9a9/nW9/+9ttjuzivfrqq0xNTXHjjTc2P+b7Ps8//zwPP/wwtm2jaVobI2y9bDbLVVddxdGjR9sdSkv09/efkRjv3r2bf/mXf2lTROvj5MmT/M///A//+q//2u5QWuqP/uiP+MIXvsBv/uZvArBnzx5OnjzJQw891LZkZFP2jBiGwU033cSzzz7b/FgQBDz77LMbbl9+IwrDkM997nM89dRT/OhHP2L79u3tDmndBUGAbdvtDqMlPvGJT/Dmm29y6NCh5p+bb76Ze+65h0OHDm24RASgXC7z3nvv0d/f3+5QWuK222474zj9u+++y/DwcJsiWh+PPfYYPT093HXXXe0OpaWq1SqqunT51zSNIAjaFNEmrYwA3H///ezfv5+bb76ZW265hb/927+lUqnwu7/7u+0OrSXK5fKSd2HHjx/n0KFDdHZ2MjQ01MbILt7Bgwd54okn+Ld/+zdSqRQTExMAZDIZYrFYm6O7eA888AB33nknQ0NDlEolnnjiCX784x/zwx/+sN2htUQqlTqjvyeRSJDL5TZM388f/uEfcvfddzM8PMzY2BgPPvggmqbxW7/1W+0OrSU+//nP89GPfpQvf/nLfPrTn+all17i0Ucf5dFHH213aC0TBAGPPfYY+/fvR9c31lJ5991386UvfYmhoSGuvfZaXn/9df76r/+aAwcOtC+otp3juQR84xvfCIeGhkLDMMJbbrklfPHFF9sdUsv87//+bwic8Wf//v3tDu2irfS8gPCxxx5rd2gtceDAgXB4eDg0DCPs7u4OP/GJT4T//d//3e6w1tVGO9r7mc98Juzv7w8Nwwi3bNkSfuYznwmPHj3a7rBa6j/+4z/C6667LjRNM9y1a1f46KOPtjuklvrhD38YAuGRI0faHUrLFYvF8N577w2HhobCaDQa7tixI/yzP/uz0LbttsWkhGEbR64JIYQQYtPblD0jQgghhLh0SDIihBBCiLaSZEQIIYQQbSXJiBBCCCHaSpIRIYQQQrSVJCNCCCGEaCtJRoQQQgjRVpKMCCGEEKKtJBkRQgghRFtJMiKEEEKItpJkRAghhBBtJcmIEEIIIdrq/wHRnL6tjOlowAAAAABJRU5ErkJggg==", + "text/plain": [ + "
    " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# similarly, for bruise\n", + "sns.regplot(\n", + " x=\"new_cases_percent_of_pop\",\n", + " y=\"search_trends_bruise\",\n", + " data=weekly_data,\n", + " scatter_kws={'alpha': 0.2, \"s\" :5}\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Hd2A8707Uhz2" + }, + "source": [ + "We see that the slope of the line is positive in the graphs for cough and fever, but flat for bruise. That means that in places with increasing new cases of COVID-19, we saw increasing searches for cough and fever, but we didn't see increasing searches for unrelated symptoms like bruises. Interesting!" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Recap" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We used matplotlib to draw a line graph of COVID-19 cases over time in the USA. Then, we used downsampling to download only a portion of the available data, used seaborn to plot lines of best fit to observe corellation between COVID-19 cases and searches for related versus unrelated symptoms.\n", + "\n", + "Thank you for using BigQuery DataFrames!" + ] } - ], - "source": [ - "# similarly, for fever\n", - "sns.regplot(x=\"new_cases_percent_of_pop\", y=\"search_trends_fever\", data=weekly_data, scatter_kws={'alpha': 0.2, \"s\" :5})" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": { - "id": "-S1A9E3WGaYH" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 63, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAqohJREFUeJzs/XlspPl534t+3v2tvbh3k71M9+yjmdZmWRrZkpVcL3F8z7VwLozA/8hGHAM5UIIYOUgA5Qa4cYJkAjiBE+AAsgMjUXIAHd2bc46cC8NLFOfItqDN2mdGM6OZ6Z07WXvVu7+/+8fLqi6yi+xik2ySzecDcIaset+3flVsvs/396yaUkohCIIgCIJwTOjHvQBBEARBEM42IkYEQRAEQThWRIwIgiAIgnCsiBgRBEEQBOFYETEiCIIgCMKxImJEEARBEIRjRcSIIAiCIAjHiogRQRAEQRCOFfO4FzAOaZqytLREqVRC07TjXo4gCIIgCGOglKLdbjM/P4+u7+7/OBViZGlpiYsXLx73MgRBEARBeAju3LnDhQsXdn3+VIiRUqkEZG+mXC4f82oEQRAEQRiHVqvFxYsXB3Z8N06FGOmHZsrlsogRQRAEQThlPCjFQhJYBUEQBEE4VkSMCIIgCIJwrIgYEQRBEAThWBExIgiCIAjCsSJiRBAEQRCEY0XEiCAIgiAIx4qIEUEQBEEQjhURI4IgCIIgHCsiRgRBEARBOFZEjAiCIAiCcKyIGBEEQRAE4Vg5FbNpjgKlFOvtgE4QU3RMZkrOA3vnC4IgCIJw+JxZMbLeDvjB3SZJqjB0jWsXKsyW3eNeliAIgiCcOc5smKYTxCSpYr6aI0kVnSA+7iUJgiAIwpnkzIqRomNi6BpLDQ9D1yg6Z9ZJJAiCIAjHyr7EyGc/+1muXbtGuVymXC7z8ssv80d/9Ee7Hv+5z30OTdO2fbnuyQiFzJQcrl2o8PRckWsXKsyUnONekiAIgiCcSfblDrhw4QL/8l/+S55++mmUUvzH//gf+cVf/EW++93v8p73vGfkOeVymbfeemvw80lJEtU0jdmyy+xxL0QQBEEQzjj7EiP/w//wP2z7+Z//83/OZz/7Wb7+9a/vKkY0TePcuXMPv0JBEARBEB5rHjpnJEkSvvCFL9Dtdnn55Zd3Pa7T6XD58mUuXrzIL/7iL/L6668/7EsKgiAIgvAYsu+szVdffZWXX34Z3/cpFot88Ytf5IUXXhh57LPPPsu///f/nmvXrtFsNvlX/+pf8dGPfpTXX3+dCxcu7PoaQRAQBMHg51artd9lCoIgCIJwStCUUmo/J4RhyO3bt2k2m/zv//v/zu/93u/xZ3/2Z7sKkmGiKOL555/nl3/5l/ln/+yf7XrcP/kn/4Tf/M3fvO/xZrNJuVzez3IFQRAEQTgmWq0WlUrlgfZ732JkJz/90z/Nk08+ye/+7u+Odfwv/dIvYZom/9v/9r/teswoz8jFixdFjAiCIAjCKWJcMXLgPiNpmm4TDnuRJAmvvvoq58+f3/M4x3EG5cP9L0EQBEEQHk/2lTPymc98hp//+Z/n0qVLtNttPv/5z/PlL3+ZP/mTPwHgU5/6FAsLC7zyyisA/NN/+k/5yEc+wlNPPUWj0eC3fuu3uHXrFn/rb/2tw38ngiAIgiCcSvYlRtbW1vjUpz7F8vIylUqFa9eu8Sd/8if8zM/8DAC3b99G1+85W+r1Or/+67/OysoKExMTfPCDH+SrX/3qWPklgiAIgiCcDQ6cM/IoGDfmtB9kaq8gCIIgHC3j2u8zO5BFpvYKgiAIwsngzA7Kk6m9giAIgnAyOLNiRKb2CoIgCMLJ4Mxa4P7U3uGcEUEQBEEQHj1nVozI1F5BEARBOBmc2TCNIAiCIAgnAxEjgiAIgiAcKyJGBEEQBEE4VkSMCIIgCIJwrIgYEQRBEAThWBExIgiCIAjCsSJiRBAEQRCEY0XEiCAIgiAIx4qIEUEQBEEQjhURI4IgCIIgHCsiRgRBEARBOFZEjAiCIAiCcKyIGBEEQRAE4VgRMSIIgiAIwrEiYkQQBEEQhGNFxIggCIIgCMeKiBFBEARBEI4VESOCIAiCIBwrIkYEQRAEQThWRIwIgiAIgnCsiBgRBEEQBOFYETEiCIIgCMKxImJEEARBEIRjRcSIIAiCIAjHiogRQRAEQRCOFREjgiAIgiAcKyJGBEEQBEE4VszjXsBxoZRivR3QCWKKjslMyUHTtONeliAIgiCcOc6sGFlr+fzF2xt0g5iCY/Kxp6eZq+SOe1mCIAiCcOY4s2Ga27Ue1ze6BLHi+kaX27XecS9JEARBEM4kZ1aM3EMd9wIEQRAE4UxzZsXIpck8T84UcCydJ2cKXJrMH/eSBEEQBOFMcmZzRmbLLh97emZbAqsgCIIgCI+eMytGNE1jtuwye9wLEQRBEIQzzpkN0wiCIAiCcDIQMSIIgiAIwrEiYkQQBEEQhGPlzOaMPE5IN1lBEAThNHNmxUiapry50ma9HTBTcnjuXAldP52OovV2wA/uNklShaFrXLtQYbbsHveyBEEQBGEszqwYeXOlzR+9ukKUpFhGJkJemK8c86oejk4Qk6SK+WqOpYZHJ4ilSkgQBEE4NZxOV8AhsNbyafZCJvI2zV7IWss/7iU9NEXHxNA1lhoehq5RdM6sxhQEQRBOIWfWapmGTq0XstIKsE0N0zi9umym5HDtQkUauAmCIAinkjMrRs6VHd57oYptaoSx4lz59BpwaeAmCIIgnGbOrBgp52yuzBQHSZ/lnH3cSxIEQRCEM8mZFSMS2hAEQRCEk8GZFSMS2hAEQRCEk8G+sjY/+9nPcu3aNcrlMuVymZdffpk/+qM/2vOc//yf/zPPPfccruvy0ksv8Yd/+IcHWrAgCIIgCI8X+xIjFy5c4F/+y3/Jt7/9bb71rW/xV//qX+UXf/EXef3110ce/9WvfpVf/uVf5td+7df47ne/yyc/+Uk++clP8tprrx3K4gVBEARBOP1oSil1kAtMTk7yW7/1W/zar/3afc/9jb/xN+h2u/zBH/zB4LGPfOQjvO997+N3fud3xn6NVqtFpVKh2WxSLpcPstwB0kJdEARBEI6Wce33QzfXSJKEL3zhC3S7XV5++eWRx3zta1/jp3/6p7c99nM/93N87Wtf2/PaQRDQarW2fR02/Rbqb692+MHdJuvt4NBfQxAEQRCEB7NvMfLqq69SLBZxHIe//bf/Nl/84hd54YUXRh67srLC3Nzctsfm5uZYWVnZ8zVeeeUVKpXK4OvixYv7XeYDGW6hnqSKThAf+msIgiAIgvBg9i1Gnn32Wb73ve/xjW98g//pf/qf+JVf+RV++MMfHuqiPvOZz9BsNgdfd+7cOdTrg7RQFwRBEISTwr4tsG3bPPXUUwB88IMf5C//8i/5t//23/K7v/u79x177tw5VldXtz22urrKuXPn9nwNx3FwnKPt+3GS+4xIPosgCIJwljjwQJY0TQmC0fkWL7/8Mn/6p3+67bEvfelLu+aYPEr6fUauzhSZLbsnythLPosgCIJwltiXZ+Qzn/kMP//zP8+lS5dot9t8/vOf58tf/jJ/8id/AsCnPvUpFhYWeOWVVwD4e3/v7/FTP/VT/Ot//a/5hV/4Bb7whS/wrW99i3/37/7d4b+Tx4jhfJalhkcniM90czbxFAmCIDze7EuMrK2t8alPfYrl5WUqlQrXrl3jT/7kT/iZn/kZAG7fvo2u33O2fPSjH+Xzn/88//gf/2P+0T/6Rzz99NP8/u//Pi+++OLhvovHDMln2U7fU9SfI3TtQoXZsnvcyxIEQRAOiQP3GXkUHEWfkZOMeAK2c329w9urnYGn6Om5Ildnise9LEEQBOEBjGu/z/aW+4Qic3O2I54iQRCExxu5qwsnnpNc+SQIgiAcHBEjwoE56rCSeIoEQRAeb0SM7MFpzd141OuWBFNBEAThIJxZMTKOwT6tRvZRr1tKkQVBEISDcOCmZ6eVtZbPX7y9Pvhaa/n3HXNa59c86nVLgqkgCIJwEM6s1bhd6/HuepdqzmK11eXSZJ65Sm7bMafVyB71und6laaLtiSYCoIgCA/N6bCuR8ruuRT9Ko62HxHEKW0/Gjx+knNHjrr6ZLcwkIRmBEEQhIfhzIqRixM5pgs29W7IdMHm4kTuvmP6VRwAN05R7shRV59IjoggCIJwmJzZnBFN06jkLabLDpW8taen47TmjhwVpzV8JQiCIJxMzqwV6QQxcaKYK7s0exGdIGZul2PF+G7PEynYBi8tlOmGieSICIIgCAfm7FnVLfwo4a3VNr0wJm+bvLiwe8/8nTkY00WbtZZ/6vqPHIRReSIyH0YQBEE4DM6sGOluhV4qroUfp3T3CL3szMFYa/n3GeaZknMqG6SNi+SJCIIgCEfFmRUjmqZRcEyqOZuGF+5LOIwyzMCpbJAG4zWAO8pQ1WntdHsWkd+VIAhHwZkVI5cm81ydLtANYq5OF7g0mR/73FGG+TR7Dsbp2HqU5cKntdPtWUR+V4IgHAVnVozMll0+/szMQxnXUf1HgjhF1ziVSa7jCKkHlQsfZMd8moXcWUN+V4IgHAWnx2KeIEb3H4GFiRyuZZy6CpPDCMEcZMcs1UqnB/ldCYJwFJzZO8lhuJt37hJdyziVFSaHEYI5yI75qDvGCoeH/K4EQTgKzqwYOQx38+OySzyMjq0H+SyOumOscHjI70oQhKPgdFrPQ+AwhMRh7hJPe5WC7JgFQRCEh+XMipHpos181WW9HTBTcpgu2vu+xmHuEk97lYLsmAVBEISH5czOptnohCw1fPwoZanhs9EJRx6nlGKt5XN9vcNay0cpte/XGucap3n+zWF8RoIgCMLZ5cx6Rtp+RK0TUs6Z1DoRbT8a6Yk4DI/FONc4zfknp92rI5wuTntIUxCE+zk9Fu+QCeKUO/Ue0UaKZei8eGH0bJrDSHQd5xqnOf+k//7OV1zeXG7zxnILQIyEcCSI+BWEx48zK0YcU+fCRI5K3qLZi3DM0RGrw/BYjHON05x/0n9/by63uVPvoVBEiRIjIRwJ0nhNEB4/zqwYKbkWU0WHJFVMFR1KrjXyuMPwWDzqSpNHfbOeKTm8tFDm69c3cSyNubKDH6diJIQj4TSHNAVBGM2Z/SvuG9DbtR6QhTaUUveFFQ7DY3FYXo9xwy/j3qwfJpyz2zmaphElil6Y8s2bdZ6cKYiREI4EKSMXhMePM2st+ga06cXEScqtzR6Xp/Jcniqc2FyHccMv496sx73esADxo4SlhkeSsu2c/mt9+MoUNzc6XJrMH9hInMZExdO45tOGlJELwuPHmRUjcC+ckbNNfrDYpBvGNL34xOY6HHb4ZdzrDYuW9baPZei8MF/Zdk7RMTENHT9KWJjIRN1BjfBpTFQ8jWsWBEE4bs60GOmHM25udAB4YqqAFyXc2uyeyJ3tuOGXcQ3iuNcbFi3NXkSUpvedcxSu84OKr+PwUkhypSAIwv45s2KknyNSyZmkqUvBMfCihG4Q0/Fjat3oWHe2owzpuAZ/XIM47vWGRctEwRo5nfgoXOcHTVQ8Di+FJFcKgiDsnzN7p1xvB7y62BoYqhfmK7iWwWYnYLMTPpKd7V47990M6YMMvlIKP0pYb/s0exETBWtXgziugBglWsbxMBzUM3FQb8txeCkkuVIQBGH/nFkx0vJCbmx0CKKEbpBQftbg+fPTFB2Tphc/kp3tXjv3hzWk6+2ApYaHZehEacrCRO7ABvFhvR4H9Uwc1NtyHF4KSa4UBEHYP2dWjKy0Ar5xY5P1ToihabiWzhMzpZE726PKPdirc+nDGtLsmgwSTF3LOLacl+POnxAvhSAIwungzIqROEkp2CblKQsvTon6TbpGhELWWv6R5B7s1bn0YQ3pScpZOO61iJdCEAThdHBmxchs2WWq6LDY8NA1jcmiM1Y1ycPu8PdKSH1juYVC8fx8meWGv6soGoeT5A04SWsRBEEQTi5nVow8M1vg/Rcr2DpMF13+2ntmx6omedgd/qj8if7r5W0D08iub+r6gTwIj7rb66NYiyAIgvB4c2bFyI/WuvxotQuaTtOPafgJ87sY28PY4Y/yrgD84G6TOElRCqYK9qAD7HEjzbsEQRCER8XoUbVngLWWT6MXMpG3afRC1lr+rsf2d/hXZ4rMlt2HSggd5V3pC5SFifxgcN/DXv+wGRZPSaoG4kkpxVrL5/p6h7WWj1LqmFcqCIIgnHbOrGfENHTqvZDbtR66Bu0gHjko77DYzbtyFAmehxFi2S00JR4TQRAE4bA5s2LkXNnhyZkCtzZ69KKE2xvZTn+ukjuS1xuVP3FUCZ6HIRh2W9txl+uOiwysEwRBOD2cWTFSci3iVLHZC6nkLNY6EbdrvYcSIw9r+I4qwfMwBMNuaxsnmfcgQuCwRIR4cARBEE4PZ1aMKKXoBCHNXkicpLim8dD5DyfN8B1lf49xvDkH+TwO67M8LR4cQRAE4QwnsN6u9djsxqQK1jsh3Sii8JBGe7dkz+OiLxienituKyE+KON6LQ7yeRzWZzksyHQN/CiRpFtBEIQTypn1jDR6EY1uiGUamIbOTMHFtYyHutZxdBrdSxgcRvinXzVzu9YD4NJkHmDbcMHdvBYH+TwO67Mc9uD4UcJSwyNJeaSeK8lbEQRBGI8zK0aqeYvz1Rzr7YBemFDMGQc2fG0/IohT2n40ePyojM9Rh4bW2wF/8fYG1ze6ADw5U+DSZH6s0MdeoZwHGejDSuodFmTX1zskKY88ZHPSwneCIAgnlTMrRi5N5rkwkaPeCZjM2UzmHj6U0Td8ADcekfE56pyIThDTDWKqORu4Fy4Zx2uxl2fmQQb6KJJ6j2tGjuStCIIgjMeZFSOapmGbBtMll3MVl5Jr0Q2TA13zURqfozawRcek4Jistrc8I8XMM6Jp2qF3oj1qA31cM3KOe1CgIAjCaeHM3h07QYyhgW1p3NjoYhsaBXv/OSPDYQc/StA1Rhqfw84fOGoDO1Ny+NjT01yeynJFLk3mB91hDyIejsNAH9eMHBkUKAiCMB5nVoz0qyuur3fRNbgyVXio62wPO8DCRA7XMu4zPoedP3DUBlbTNOYquYduAreb+DpLBloGBQqCIIzHvkp7X3nlFT70oQ9RKpWYnZ3lk5/8JG+99dae53zuc59D07RtX657/El83SAmiFOqeZupooMfJby50t536ef2UlRwLWPkDJuTVv571PTF19urHX5wt8l6OwAOZ87PwyAzdQRBEE4u+xIjf/Znf8anP/1pvv71r/OlL32JKIr42Z/9Wbrd7p7nlctllpeXB1+3bt060KIPA03TKOdsynmLXpiw1g5YafrbDOc4jBt2OIzwxGkyqCdNfO0mjgRBEITjZ18W8Y//+I+3/fy5z32O2dlZvv3tb/Pxj3981/M0TePcuXMPt8Ij4tJknhfny7y71gGlOF/O8dz5EivNYF9JleOGHQ4jPHGaSkVPWvKmVLYIgiCcXA5kIZrNJgCTk5N7HtfpdLh8+TJpmvKBD3yAf/Ev/gXvec97dj0+CAKC4N7OtdVqHWSZI5ktu7xnoUKYKKbKLs1eyFvLHSaL9r4M57h5AXsd9zCdTQ9iUA+aTDvO+SctN+SkiSNBEAThHg99R07TlN/4jd/gJ37iJ3jxxRd3Pe7ZZ5/l3//7f8+1a9doNpv8q3/1r/joRz/K66+/zoULF0ae88orr/Cbv/mbD7u0sdA0DdcymC46nK+6vLHUYq7i8Pz58iM3nON6PIqOia7BG0stwiTh4mQOpdS+8y4O6mEZ5/yTlrx50sSRIAiCcI+Hnk3z6U9/mtdee40vfOELex738ssv86lPfYr3ve99/NRP/RT/5//5fzIzM8Pv/u7v7nrOZz7zGZrN5uDrzp07D7vMPenvlpcbPlPFTIg8yqTKPuPmV8yUHBYmckRpimXoLDW8feU+9HNO3lhuUeuEnK+4D5XPcdLyQcbhuBJnBUEQhAfzUJ6Rv/N3/g5/8Ad/wJ//+Z/v6t3YDcuyeP/7388777yz6zGO4+A4R79zPazd8s6wxXTRZqMTjh0GGTeEMOzNeZhQTd+jsdkJuFv3AHYNS+0VipGQhyAIgnCY7MuKKKX4u3/37/LFL36RL3/5y1y5cmXfL5gkCa+++ip//a//9X2fe5ikacobyy3eWeuQswyuXag89LV2hi3mqy5LDf++MMZh9N44iBDoezSeny8D7BmW2isUM7zegm2glOL6ekeGwQmADAgUBGH/7EuMfPrTn+bzn/88/+W//BdKpRIrKysAVCoVcrmsOdanPvUpFhYWeOWVVwD4p//0n/KRj3yEp556ikajwW/91m9x69Yt/tbf+luH/Fb2x5srbf6P7yyy2PDQNY27dY//+3vnH6o6ZWdi6Xo7GJloupuB309+xUG8ObuFpcZ5T/3hf8OvO1t2WWv5p6bCR3g0nKaqL0EQTgb7EiOf/exnAfjEJz6x7fH/8B/+A7/6q78KwO3bt9H1e6ko9XqdX//1X2dlZYWJiQk++MEP8tWvfpUXXnjhYCs/INm03pj5So6mF1Hvhg9dnbLTWzFTclhq+Pd5Lw6jGuYgiaH78WjsfE9BnI4cAvgoSmZlp326kDJqQRD2y77DNA/iy1/+8raff/u3f5vf/u3f3teiHgXTRRsNeHu1jW3qvOd8+aFzH3Z6K6aLNtNF5z7vxVHnWjzIaA8LmQd5NHa+p7YfjTQwjyJ/RHbapwvJKRIEYb+c2bvEVMHm6dkSOUsnZ1t8+OoE00WbtZa/7x34KG/FKO/FUZeX7sdo77V7HSVqgJEGZj/v6WE9HMNrXWz0uLXZFS/JCUbKqAVB2C9nVox0w4SiY/HjV6ZpeTE522SjEx7pDrwvWma2jPKNje6hGtT9uMf32r2OEjW7GZj9hI0e1sMxvNZuENPxY2rdSLwkJ5ST1mNGEISTz5kVI0GccrveYflWQBgl5Byd58+VHkms+6jCDvtxj48KLfW9QpudgChJyNsmNze7VHL3ElbH+Tx284A8bC7B8Fo32j7X17soFJudkLYfiRgRBEE45ZxZMWIbGn6Ustb0iBLF197ZYCJvH0mse6dx3i3/4qD0jXbbjwjidFABM8rzsnP3OpxD0vYj2kHEejsEoGD3uDxVGNvo7ya2HjaXYHitfpSw2PC5udnDMnRe2kdJtiTCCoIgnEzOrBgJE8Vqy6flJcyWbYJYESfpkTRBU0rx6mJrWx+SvYzywxrNvtEGRla+7MW2vIy6wjI0dDSemC7ihfG+BNNuHpDDyCVwTJ2LE3nKOZOWF+OY4zcRlkRYQRCEk8mZFSOOqXNlpkgUKxpehGNEmIY+CEcchJ1Gr5Iztxlnx9T3NMo7z39poTwIc4zT4fVhwiHDXgvT0Lk0lWep4eNHCaah78tLtNMDUrCNbYnBV6YLD+2RKLkWk0WbJFVMFm1KrjX2uVJyKgiCcDI5s2Kk5Fq8tFBGB95d73BlpogXRryx3MK1jLE8EuPmRsD2SpSSa+2Zf7Hz/Nu1Hk0vJk5SOkE88AoUHRPT0O/r8LrZCWh5EY1uSJSmYw3UG7c8eRx2XksptatHYr9eoMNo+iYlp4IgCCeLM3s3nik5vPfCBLZhMF/J89z5Em8st1ht1ZkpuWO58cfNjbg0md/m2XiQAd15PkCcpARxyndu1chZBo5l8OGr0/hRcl+H1zhN6YQRfpQymbdZbPQA9hRZ45Ynj8POa11f7+zqkdhv6OSwmr5JyakgCMLJ4cyKkT5520DXYanhEacqEydjuPGVUtza7LJY792XVzHK6GmaNrYBnS7azFdd1tsBMyWHybzFrc0e375Vp9aLuDRp0g1jbm50WJjI39fhdaGaZ7XpkSaKSs7i5nqPlYbPbDk3dq7EsMeiYBtAvxx6/4mfe3kkHhQ6GcdzMq53RUpOBUEQTiZnVowMexE0DaaKNpem8izWvfuM5ihjt94OuF3rsdoOWG0HXJ0uDI4fx+jtZUA3OiFLDZ84SfnhUotLkznKrsl81WGu7NALE85VXF6YL3N5qjCyw2sKbHZD2ncadIOYy5OFfeVKDHsssqocRZJCnCref6nK8+fLYwuSvTwSDwqdjOM5kcRUQRCE082ZFSMtL+T6epsgTuiGCReqLs+dK43Mkxhl7DpBTMEx+fCVSW5udrk8ld+X238vA9r3FuRskx8sNumG2XrOlfMoBWGS8IHLE/cJgmGjP5E3yZkG1YLNnbqHY9xv8PuCqF8K7Jg6JddipuTQ9iNqnZByzmS16aNQuLbJRjtAKcV0cfxE373E2YNCJ+MknUpiqiAIwunmzIqRxXqPP3p1mfV2gGsb2JrOlZnSSKM5ytgVHRNT1/GjlIVqnstT+6sQGXXNmaEE1LYfsdzo0Q0iHDNHlKRcnS4wXXLHyvsoOiYtPyFJFVem8sxXc9tyRuCeINrsBLy10mayaHG+kuMnn5omiFPu1HtEGylhnJJzdDpBwkzJwTaMQzP4D/IijZN0KompgiAIp5sze9d+9W6TpYZPmCo6QcJf3trgJ5+ZHhj54TCKHyUYW3klfWM3bjLkbuGYUQZ0Z+gob5sYus6NjR62oXPtQpWrM8U939ewt2O+6m7zduwUL31BpIDFpodpwHo7xNQ1zldcFqou1YJNoxsyUbBYb4fYhsFEwXpkBn+cz1kSUwVBEE43Z1aM1HsBYZoSxVmVyq3NHq8tNgcejlubXW5t9gaiYWEid181yjjJkLuFY2ZKDi8tlLldyypdlFLbElCzfiQaoO2rwdd+8if6722j7aNrGnGiWGsH3Kn3iFOFaeigwDR0JvI2FycL28TNo2Ccz1kSUwVBEE43Z1aMPHOuTMleZzMKsQ2dyYJNN4wHPT0WGz1WWwEfvjKFHyW4lvFAr8Qodstn0DQNTdNoetnzTa+1ozNrJgLCJOF2PSRJUt5d71B0TGbL7jYvx7D3ZbMTEKfpQNDsFU7pC6IkSfCihF4YUXQMpvI2tW7ITNEmqyxW1HoRLT+R5FBBEATh0DmzYuTjT8/w7VsNvne7jmnoTOYtTMNAKcVmJ8AxdbphxM2NNvPVPH6UcH1LDOyntHU/Za3DnVn9KOFurYcXJry53MLUddp+TNuP+djTM9sEwXB4Z7nh4ccJzV7EVNG+r/vp8Nr7gsgwdC5NFrhd6+KFIa8vtWh4Ee+7MIFrJ1iGPpa4OS720zjtsOfTHPe8m+N+fUEQhMPgzIqRc9U8v/LRJ7g8VaDjx5Rcg48/PQ3A3bpHGKcY6JyruORsg+/cqmEbJtW8yYXJ/NhdWnfLZ1BK4UcJG52ARi9kaqu1eT/ccH29g0Lj4lSed9c7VHI21bxNJ7jXz6RviN5YblHrhMyWbdbaAQXbIEpS5qs5gD3DNpkgghfmK3hRTM4yKLkm76x1WZhwafsJUZqe6OTQ/YSmDrsMeD/XOwrhIGXNgiA8Dpw8y/KI0DSNF+YrzJTcbcbh+npn2yC2ibzN22td7tZ9ZkoOLS9ivRMyXXQO1DF0vR2wWPew9CwUM1/Njey/sdkJydsmQZzS8CKeLN7rZzJcDXO37rHW8dA0jZcuTAxCS90w2bPsddhzU3Itio5FkiqqeZu2nzBRsLbly0wX7V09LcfFXpVJD2rVf1BPz36udxTCQcqaBUF4HDizYmTnLnW6aLPeDqh1Q3Q9e17X4c2VFktNH8fSWWsH5EyNUi7/wJv/g3bBnSAmVfD8fJmlhodrGSN7hrT9iBcXynSDGE3TuDiRzZm5vt7J8kOSlOfnywC4lk6UKLww3jbcbq+y12HPTb/TaieIeelCZWQlzlrLP3E78b0qkx7Uqv+gnp6CbdD2I75zy6PgmIPPcBRHIRykrFkQhMeBM3vn2mms5qtu1vV0q6zW0DXaQcSN9S7rnYCiY3J1psBLC1X8KHngzf9Bu+AHGZGBR2WHoR8WA50gQilYbvhMFZ37pvvOlByUUoPW8lMFi7WWxxvLLWZKDs+dK6Hr+n2em7k9PrfDMKiHHa4YFQq7sdEduc6jKAPWsqInHvQWjkI4SFmzIAiPA2dWjOw0quvtYHtZraWjaxoL1TyVnI1SKU9OFzlXdggT9cAS1weFDgq2wUsL5W3zXva77sWGYqpgM1V0dp2Bs94OWGr4JKnimzfqvL3WRpH1MPl/fmCB9yxU9/W5PYxB3Sk+lFK8utg6NO/KqFDYbus87DLg7Pdn8cxc5uHqhsmuxx6NEJKyZkEQTj9nVowUHRNdU3z93Q26YcyTs0VcUx8Yr5mSw0YnYLXVBWC64NAOYt5d741lQMcNHYxTLjzcyGy56bPe9gdJr5enCnuuY1i8fOP6OndqHk/PlVhseLyz1tm3GHkYg7rzfVdy5tjelYf1ojwqj8F+xJkIB0EQhNGcWTEyXbTxo4Rv3txER6feDfnpF+YGU3CnizZTBZtLk3kA0jTlxmYPhWKzE9L2oz1FwH5CBw9iOFH1Tr1H1bXB5r6k11EMG8ucbWGbOk0vQtc0ctbu+Q278TAGdaeXCPbOYxnmYZM+H5XhlzCJIAjCwTmzYmSjE/L9u82sMqZgc6vmUeuGfOyZe+ZrrpJjrpKVx/5wqclivcbNjR6WofPShcqe199P6OBB9I15JW9xY0NxYTKHpmn3Jb2OYthYLlQdJvM2jV7IRMHm2gPew34Zt/X9pcn8fbktD3rv41TKHAfi7RAEQTg4Z1aMdIIYU9fIWwb1boSuQRCnKKVGGjbH1LeV/PZbs+8njLBXz5G9rjFc5msZOi0vZrJojyVmho2lUorZcm7frz8ue7W+3/m++7kt4773cSplBEEQhNPJmRUjBdtgpuhgmzrdIObiRA6NzKCOMmwl12KyaJOkismtBmWwvzDCXj1H9rrGcJnvzpLbUexm4B/29cflYSptxn3vhxHuEgRBEE4mZ1aMAFTyFk9OF6jnbT7+zAyuZexq2HbzahxGqeuDrrFbme9u7FdcHFb/i93CUMPt6rtBzKXJPJenCsyUnPHf+xivIwiCIJxOzuxdvBsmlFybjz87yzdu1Gh6EUXX2tWw7eZV2GkY95oF02en56JgG/syruM0VNuPuDgs4/4gwZazDH5wt0nHj2l6MdcuVB7qtSVpVBAE4fHizIqRgm3QCSJWmhEzJZvnz5d4Yro4MGxpmvLmSpv1drCtQdhOdhpGpdQDvRI7PRcvLZT3ZVwP2lDtQe/hYY37gwTbzc2sTPqJ6SJ+lNAJYq5MF/b92pI0KgiC8HhxZsUIgFIAGiXHvK9fx5srbf7o1RWiJMUyMhHywvz91Sc7DeP19c4DvRI7PRfdMOHqTHEs46qU4tZml8V6jyemi3hhfN9r7Fdc7Ne4P8gzM6rV/rULFSo5k4Ld29auXoSFIAiCcGbFSNuPqHUDgijh3bUOhqb46FMzzJZdNE1jvR0QJSnPnivz5kqLt1fbY03qHccrcZCwyHo74Hatx2o7YLUdcHW6cN/5+zHwD1NJ8yDPzF5VNZenChJeEQRBELZxZsXISivgmzdq3K536fgJb6+3qfdifuHaeeYqWTMxy9B5a6VFlKRsdkLeXu08MCF0HK/EqGPGFQWdIKbgmHz4yiQ3N7tcnsofyKg/TCXNg3JSdnv+JHtBDntejiAIgjA+Z1aMxEmKrmuYmk4YRyw3PP7i7XUmCxYffWqGZ+eKwDnW2wF+FFPvRKRpyp1Nn60WI9sM1k5jdmW6gKZpKKVGJrTOlt1B867r6x2Wmz43N7pYhs5U0ebahepIUVB0TExdx49SFqpZVcpBjObDVNI8yLNzGqtdpHeJcBSIyBWE8Tj5VuKImC7axEnKRjsgjFNCQ2OxmU20TRRcmsxzaTLPVMHm+3ca/Gi9QxinNHohSoc4ZZvBWmv5fOWdjcFN5yefmmaukttm5HQNFiZyg3BPmqZ85Z1NVpoe19e7FGyDyzNFFFleyKgb2GFXkhxFNctprHY5rPJmQRhGRK4gjMeZFSMAJcek6BjESlF1TSo5m8mizbvrXTp+zK3NHpoGHS8iSVPmyjYasFBx2OwEvLHcAjLje7vW4931LtWcxWqry6XJPHOV3MDIna+4fPN6jdeXmsxX8kwULJRSXN/o0PIiFhs9Lk4VtnJVTG5t9tjshvf15Rg31DHujuxhhMNua9jNO3QaOI3eHOHkIyJXEMbjzN5xN7sR56p5npor8Rc/2mC6YDNXdQnjFIDLUwVeW2wQxClPzZbItwMALEPnB4stwigFBVGidsx42W58+0buzeU2N2sdNDSKrkXTCwHFRjugG8QEiaLtRTw5XeDJ6SKpYmRfjr12VcNiwAtj3lhu093KMfnY09ODOTvDHGYex2neBZ5Gb45w8hGRKwjjcWb/MmZKDrah0wkSXrxQ4cNXJrg4WaDjR9yueay2fGrdiISUt5abuJZB2bVQaIRRgq9gruISxCmdLe/F1ekCHS/EMTQW6z0Kjsmzc0WuXajwxnKLy36BbhDzg7t1TE3nufOlLcMNH3pigrJj8mNPTHJpMs+ri62RfTn2Eg3DYuD6epuVVsBCNc9qO0t0navk9l2W2zfK4ybX7ncXeFJi6ic5uVY4vYjIFYTxOLNi5Nm5IrXuJHdqXYquxdXpApW8w7NzRYquxffv1Cm5BufKBW7XPCwjM5B+lJX7vrXS5tZml4WJ/KCXxgvzZb51Y5Pllk83THhnrUvtySnmq1l1zlrL5269R5qCbmnMFB1qxZAkTXlqusRk0R6EY14CUpXSC2JWGt5Yg/GGxcA7q22iJEWh6IYRSw2PtZaPUopXF1sDETRfzW0rWR7l3QDG8ngUHRNdgzeWWoRJwsXJ3K6DB/ucZm+KIDwIEbmCMB5nVoxsdEJWmv5WyW6Xnp8wVXKYr7osNXw6fsK7613q3QgFTBUcnpgusdrepNENuDqdp+xaVHLmYHe/1PBZ74Q0ehHPnavw7lqbb92s8fz5CroGlZzF1ekiH7hk8e1bdf7yZo1qwWaumufqbGFbXoimaeiaxmTBIU4VCxO5B+6qhl3Cs2WHBMVa0yOMUrwo4ft3GiilWG76PDFdZLnRY6XpM1NyB0JglHcDGMvjMVNyWJjIsdYOsAydpYbHdNHZ11ycth8NHpfqA0EQhLPBmRUjt2u9QfLorc0e5youlbzFejsgSRUXJnPcqvWYKTt0g5gwiekFEVenC1yeylNwTBbrHrVuRNNrUcmZJKniqbkS7653eHO5ialrFFyL8xWXN5fbOJZGwTGxTZ3zFZdEKV5aqNILM4PfN/z3BshlXV+XGh6uZTzQKA+7hL0whq2QUiuIs86tGz2iNKEdJKy2A0quyVTe2SYydotxjxP31jQN1zKYLjoPORcHlps+X79ew9S1PUucBUEQhMeHMytGALphTMOL6AQJP1xqMlmwuTSVZ6nh0+jE2IbORiek4BjkHJOpos1l18IxdWrdkEQpFqp5lhoekBls29D40JVJJvM2U0UHL4p5c7nNnXqPhQkXy9CZLmadSBe3PBO1XsBK0+d8NYep6w89QG7YJXx9vUM5Z/H0XJl3N1b4/t0mpqZxaSrPs3NF3lnrMFWwqebMba+xW4x73Lj3Qebi+FHCt2/WWGz4TA8N2RMXtyAIwuPNmRUjlybzzBVdlus+c2WLomtwccLdanYGpp7VxXSCkKmiS6OXhV+aXkyqsnbymsbA6F6azKNpGp0g5oOXJ7clfr6x3EKheH6+zHLDZ6rocGW6AMBqs06SKNa8gKszRfwofegBcsP0RUGjGzBbsnn+fJm2FxOnKW+tdgDQNbgwmb+vzf2oGPe4ce+DzMW5vt7BMe/lruS21iUIgiA83pzZO/1s2eXiVJ5v3thEKY26FhGlWcnvUsPPEioNDdAHxrsXJUzlXZ6fL7NYV0wVM+/HNkM+4nUgKwFebvgYukbBNlhvB6y3A2zD4MWFKt+8WePmZpeFav5QBsj1RUElZ1LMWRQck6mCgyLLGbk8mWel5bPeDnj+fPnQcjN2dpe9sdEdO/ej6JhMFCwAHFPn/ZeqUn0gCIJwBjizYkTTsmm9FycLXJjMc7fWI05S2n7EZiegkreIkpTJgoVhaDwxVWCp0WOz6/OdW1nvjvdfyvIZHmR0d3oLlFL84G6TzU7A3bqHQg1yUfpJrA/LzlLZD16e2DacTimFrrVYbQUs1n10dKKkeehVLA9TJTNTcnjvxaokrwqCIJwxzqwYgcxrUc3b1DpZ9UeYpCw3s/LbGxsKy9D58NVJim6KH6VYhoFrp6BB30bu1gZ+mGEvh1KKb92ssdjocXkyj0JxruIe2DvRFyG3Nrvc2uxlM2wMfSACZoeOu6ZpvLHcQkPjufMllpv+oedmPEzPkYN4gx5Fv5KT0hNFEAThceNMi5HnzpUAeHu1Ta0XkqSKG+sdSq5JOWex0QmwdHh6oUw3TNjsBGx2w0HSav+xUW3gdzNcmWDosdoKWG0FPDlT4Pnz5QN7JfqeiMVGdu0PX5ka2Sitb/ABoqTJctM/ks6Qj7rz5KPoVyI9UQRBEI6GMy1GdF3nhfkKrmXw9mqH+WqOlhez0vK5sdHDMQ1u13yuzJS4Ml3AC2O+davDO2ttzpVdCrbB5uBq23fIuxmuth+RpIpLk3k22j4Xh/qH7CZgxtmR9z0RT0wVWG0F3NzoDBqyjeKoO0M+6s6TD/LEHIZXQ+aMCIIgHA1nVoykacqbK23WWj7tIKbRDWn0QvQt+6SUopo3SVM16P/xw6UWK81sym/Bzj66ixM5pgs29W7IdMHm4kQWotnNcPlRwlsrbXphTN42KWwlq8LuAmacHXnfE+FFCU/OFLYN1xvFXiGRwzDcR9F5cq91PcgTcxheDZkzIgiCcDSc2bvpG8st/o/vLLLR8en6Me+ZrzJbdpgtO1xUeSYKDuutrAfIZif76gYxC9U8oNB1jW6YkLf0LH9kKI8Edjdc3SAmIaWSt/DjhO6W0IHdBcw4O/JRnoiHzWc4qeGIvdb1IE/MYXg1ZM6IIAjC0XBmxcg7ax0WGx4F22CjF2GZGjMll4m8ha5paIREBRvX1Hl3vUO9FxJGKX6comkaTxYLFB2Tmxsdbm70SJTibs1jvuoyV8lCLy8tlLld6wHZrr4/p6VgW1RzNg0v3CYYdnYj9aOE6+sd/CjB0NlzR36YnoiTGo7YbV3jeHIOw6shc0YEQRCOBn0/B7/yyit86EMfolQqMTs7yyc/+UneeuutB573n//zf+a5557DdV1eeukl/vAP//ChF3xYuKZOGKdstP0sFONHg+Zl1y5U+dCVSX7s8gQ526ATJDR6MZah8f6LVf7KszP85FPTzJQcGl7EnXqXd9c63Kp1+cFik/V2MJgv0/Riat2IVxdbrLeDQVin7W0P68C9nffTc0XmqzkW6x5vr3ZYrHvMV3M8vTUB+Kh35Cc1HLHbuvoek7dXO/zgbvb572T4s30Un6EgCIIwPvsSI3/2Z3/Gpz/9ab7+9a/zpS99iSiK+Nmf/Vm63e6u53z1q1/ll3/5l/m1X/s1vvvd7/LJT36ST37yk7z22msHXvxBWJjIMVt0MA2dmaLNlak85yvOID/kynQ2uC5OFLc3uhga2IbJk7NFPnRlirlKDk3TqOYsyq5FybWYr+bImcbgGsM7+WQr90TTNCp5i+myQyVvbdvBa5rGTClrorbeDqh3I85XXFIFrmVwdabIbNkdJLWutXyur3cG03j77PXcOJxUw73bukZ9zjvpezWGP8PTwkF/n4JwlpG/n9PBvra8f/zHf7zt58997nPMzs7y7W9/m49//OMjz/m3//bf8tf+2l/jH/yDfwDAP/tn/4wvfelL/C//y//C7/zO7zzksg9OzjZ56lyJqa5D24uoexFvrrQpOtYgH2Gm5PDEdIE3l1u0/Rhd0wjidNt1Lk8VuHahyjtrbUxDp+CYbHYCio5J3tJp+xHfueVRcEwKtkE3TCg6Fs/MlQflwcNhBj9KWGp4bHZC7tazmTeTRXvQsGz4uMW6R6q4L39inN4nw4wKc5zEcMRuYZKT6sk5LE5qDo8gnAbk7+d0cKC7drPZBGBycnLXY772ta/x9//+39/22M/93M/x+7//+7ueEwQBQXDP1d5qtQ6yzJEUHZM4Sah1A86VXZIUOn7EdNHh5maXSi4zyucrLi9dqFLOmdxt+Ky1fKYKNkopbtd61Hsh81WHhYnsH3fbT9jshDS9mPMV577k1lGGc/iPZb3tYxk6z8+XAZirOIOGaMPHZT1QsuN25nXcrvVG9j7ZjYP8sZ6ERmCPe2LpSc3hEYTTgPz9nA4eWoykacpv/MZv8BM/8RO8+OKLux63srLC3Nzctsfm5uZYWVnZ9ZxXXnmF3/zN33zYpY2NYxhoaNR7EZcnc6TAN27UQCmSJHPlFbam9W52QhqdgO/2Ir7y9jq6ruGFCRvdiNlSNur+0mQeiAb/6Dc64cALstjocbvWY7JgM191cUydkmsxU3K4sdEd/LE0exFRmg4G6g03RBv+o2r0QsIkeYA3YDxRcJA/1pOw6zjMxNKTIK528rh7fgThKJG/n9PBQ/9WPv3pT/Paa6/xla985TDXA8BnPvOZbd6UVqvFxYsXD/U1OkE2X+a9F6tstH3eM19G0zTeoE01b/PmSosfLrc4X3bJOyYKRRClrPsBi02PME65NJEnZxm4Q3kihq6xWO/RCWKavZRbNY+3VhqAhqlrTBdzTBQs3nuxOjDaw38sEwWLhYnctkm6fYaPmyrazFdHH3dpMs/V6QLdIObqdGFLJO3OQf5YH7ddx0kQVzt53D0/gnCUyN/P6eChxMjf+Tt/hz/4gz/gz//8z7lw4cKex547d47V1dVtj62urnLu3Lldz3EcB8c52n8wO5uPFV2LmZJLy09YrPfQtGw43mozYL0TYuoaqx2PlpdwaSrPrY0u9V4Amk6appyrulycyHF5SufWZpfllseNtS53Gx6WoWPoMFmwcazsI2/7EbAlimyDF+dL3NnKEZkq2COTLHeWCw8f10/S6l/vY09Pb+WnbP/jG7XzP8gf6+O26ziJ4kpKigXh4ZG/n9PBviyHUoq/+3f/Ll/84hf58pe/zJUrVx54zssvv8yf/umf8hu/8RuDx770pS/x8ssv73uxh0nHj2gHEbqm0Qoi7tS6uJbBfNWl7BoUXRMvSgiTBNPQmCs7VF2bZq9Lx4uYKTqcqzj4UYprG3hBwnrbR9d13llrc2OtixfF5Cydgm3Q9BPiJOWNpRbzFYeco9PshdiGOfCGNL3MEDa9FteGZsj0GS4X3nncqB391Znife97t53/w/6xPm67jsdNXAmCIJwG9nWn/fSnP83nP/95/st/+S+USqVB3kelUiGXyxIkP/WpT7GwsMArr7wCwN/7e3+Pn/qpn+Jf/+t/zS/8wi/whS98gW9961v8u3/37w75reyPhhexueXx6IYxX31nk9VWSMEx+cmnpnhiukgniLk0mef1xSb/11trtPysSVmSgmZAwbbw44icZdLyY/74h6v0/ISVls+PllsYpoZjmcyVbECj5Sf0ggjL0rj+2jKppnG56nK3brLR8dG1LCF1ubH7FN2DdGnd6/w++82ZeNx2HY+buBIEQTgN7EuMfPaznwXgE5/4xLbH/8N/+A/86q/+KgC3b99G1++1L/noRz/K5z//ef7xP/7H/KN/9I94+umn+f3f//09k14fGRqAoulFXN/oUMnbrLQ8yq7JU3Mlio7JE1N57tS6NHsRXqTY7ISUpyxafkwnjFlteKy3fJ45X6LZC1ls+PhRTDtMqOgmJdvg4oSLbZoUHZMbmz1QsNkJCRNFrR3iGBpeVMKPUlpBzJWp3Qfc7bZzH3dH/6Dj1lo+f/H2Bt2tnJqPPb13WfB+OInJoTt53MSVIAjCaWDfYZoH8eUvf/m+x37pl36JX/qlX9rPSx05E3mbixN5kjRlvR3hxynLrQBNU7y70UGhDcIYmqYRJilBlNANYxpewGY3Jk1TUgUFy+CdtQ5xnLLS9Kh1A+JUYRgaXpzS6MWU8wZ+FIGCMFZUchbrnQDXAMPQBgP6ul7I+cokSimur3fuM9q77dzH3dE/6LjbtR7XN7pUczar7S6Xp/YuC94Po0JE/ZLlkyxQTiunQfwJgiDAGZ5Nc2kyzxNTed5d63BhwmW25GQJn67JRD6rVFls9Li12aXrRxiaTsk1KLsWYZxQyVvkLINeFKMb8M5qC1PXiGLFZNHB9mNKjkmiIFGKas4iiGNmy3k0NGrdED9KyTsmXT9ioxfx/FwJlMY7ax3eWG5TdExMQx+7okMpNRjqp5QamQQ7/s7/8LsUjgoRASeueuVx4SRWBgmCIIzizIoRTdMouRbnKi6moVGwTUquyZNzJRxT442lFptdn6W6QZQkeFFM3jayRmY6NDoR37/bJggjUnSiNKWac0hVSkWDSs6i6BrkHYsr00Xq3ZBUKZ6YKmIaOvPVHOfKec5XbL57u0mYJLS9mFil+ElML0z58JUp/CjZVnmzW+fV9XbAV97Z4N31rDX/1ekCH39mZt/G59JknidnCnSCmKuFPHnbGOmhSdOUN1farLcDZkoOz50rbQvPjWJUiOiwc1iEe5zEyiBBEB4NSaqI05Q0zTbESapIU0WcKtKtn5Oh7xcmcjimcWzrPbNipBPEpCk8OVskTBWaUpyv5nFNnZxtstYKafoRzV6PK9MFzpVdnporcbfW5RvXN7jd8Ol6CaYJcZK1iN/sBliGhmkYaFqMGxs4VvZcNW8B2VyatpcwP5mj6NrUOiGuYzCXz9H1I6qOzUzJ4RvXa7x2t8Ez50oEccqNPTqvzijFrc0uNze6mJpGwTHp+BG3Nrv7NuKzZZePPT2zrTV9kt7fcv7NlTZ/9OoKUZJiGZkIeWG+sue1dwsR7ZXDIrv7h0cqgx49Ip6Fo6IvHpJUbRMXibonMobFxWnjzN6dgjjlTr1HtJHSDWIuTxZ4Yb7C3VqX6+sdbm508OOEpbrPestnqugSJelWImuKY+hEZkqcKFIFpg5Rkt2MojgBx8APE2xdp94LeeZcmeWGx/duNwnjBEipFmxsEy5Uczw/X+ab12ustX3u1ntYpoauQ842WGv5bHYCnp8vj+y8ut4OuF3r0Qoi1lohsyWHy5N5btd61LrRvoz4cBjn+nqHJGXkznq9HRDGKeerOd5cbvL2apvnz5f3XXnzoBwW2d0/PFIZ9OgR8SyMy07PRLLV+bsvLhKltuzL6RQX++XMihHH1LkwkaOSt7hT93CMbAe51PB4dbHJ7VqXphczkbeYzDu4FtS7IUGcEKUKlaakKHQdXD0rzJksWpCAYWYGuRcmVPI2SmlstHyqBYeSY/CdW3W+e6fBRMFhpmRTcixeX2zw6t0GjV6Ibmj8P67NEyVwa7OHpeuDoXmjOq/e2OhScEz+yjOzvLbU5OJEnvMVl81uiGsZ3NzoDGbt7GeXttfOeqbkECUpX7u+ga5lOTDr7WDfN94H5bBka4AfLjWJU8XFyRxKKdltjoFUBj16RDyfXcYRF8PeDWE7Z1aMZMmhGqstn4mcyfPny+Rsk9WWhxclTBcdbtd6BFGW1LrWDglqPZSCOE3JOSa6rjFZsMlZBnfqHkmiyFsGkwWLqZJLL0hwbZ2JgsVmN6QbJQRuNhV4puhSdE104PJUnm9c3+BH6x00pehGKd+5U+e5cxUsQ+e58yVg+9C8YWNcdExMXSeIFc+dq3DtQhYuuV1b59XFbMhgsdbj8lRhX2Jhr531c+dKfOTqJK8uNnlqtoht6kdy450pOcxXc6w0fWzDYLHuMV10DrTbFFe6cFRIaOzx4b78iqFwSLotVJLlZIxTbSrszpn+S1GKraIRjemiw1wlxztrbQxdoxPEGLqOaWgsNXxUmhKlbCWzpuQsHdc2cQyDjW5EGKckChSKOStHyTG4PFmk3gu4sdElTlKKroVt6Jwru/hxSsMLBwa+3smqa6YKNp3ARwM+cHmCpYbHctNnsmhvG5o3zG6i4fJUnm4Y88RUAS9KRoqFvQzzXjtrXdd536UJdF0fuKSP4saraRquZTBTcg9ttymudOGokNDYyea+pM4tz0X/seHnRFw8Ws6sGOmGCSXX4tlzWSJoN0wAuLZQYbHu8a2bNaYKNpW8haFptHohkR8TJIogSgjibKZNJ4hJ4gTXNvHCBJVCz4+Jyw4/dmWC62vZnJqJvEPRNXFNbZBbUe+FNHsxm52QWClypk6cpkwULF5cqDBVsOlulb9emszvemPbTTRcnirQ9GL8KMXU9ZFi4SCG+VHdeA97tymudOGokNDYo2dYYAz/fzgs0n9MOLmcWTGym4Gbq+T48NUpemGCrmmstX1c26DgWvSiFDtO6ClQKeiaRhjG2KaJH6Z4UQqmTt2LcOsebyy1uTCRI+8arLdDml6IXcw8D5enCkzkLb59M8sTSZKUy9M58pbJE9MFXjhf5tXF1kAk9OfS9Bkn1DCOWDiIYX5UN97DFj3iSheEk4tS2ytGRlWRSO7F48eZvQvvZeA6fkSapMwUHerdgJmiRZKa6LoijCw6YQsvgno3wjZgumiy3g7Jm1kJb5SkKDTeWG6ioXjufIn5SuZtaHkxG+2AW5s9lFJ8906dlaZHy49ZqOZ4arbEJ56bxTF1kmY4EAnDvUaKjolSaptYGeXRGEcsnAbDfNiiZ+fvfrpoDyYeSw6JIBw+ewoMSe4UOMNiZJSBU0rxw6Umf/jqCj+426DtR9iWwazKhulZhkkQBhRsC02LCGOFH8FKK8AxDSYKNkmaousmlqFxu96j3g1o+gkvLpS5PJVHoRFECd+5XcM1dfw4ppyzyTsW56s5Co45qJQZFgnDvUYMXaOSMw8l1PCwXoejSgJ9FMmlO3/3ay1fckgEYZ/0BUY/ybOf2Dmc4HmWSlOFg3Fmxcgo1tsBf/bWOj9abVLvBrSDGNc0eO1ug0rOJG9b9CJFN4zpBopk67xOkBImKXGqqOZMJooOdS/C0DRc28QPI1aaHiXX4MZGl7eWW3hRVqZqGwahSgGFHyc0vJCvvL3OXNlhoZojZ5uUXIu2H5GkivMVlzeWWyzVu3SjlHo3YLrkPLRH42G9DkeVBHocyaWSQyIIGfeVoO6oIImHvBepJHgKh4iIkSE6QUyqFKau0/ZjWkGMpyWEysqERK1HGERESToQIgCRgjQCTU9wTJuLEy63NlN6YcKdzQ71bsBq26fRDUgU+FHKxck8OjBdtMjbJhpgGzpvr3V4Y6mFrmt86Mokv/DSPLNlF6UUbT/i7dUWd+setqmhawYoxUsXKrsO1jvKz+ooDPhxCIPTEKoShIdhN3Fx7/vtVSVSQSIcF3LXHaLomMwUbbwwIUpSyjmLkm0QJglr7YC1Vohl6ETx/eemQBAq7tZ7GIZO3jaJkpgwUfTClOvrXe7UelycyNMJIvK2jm0apApcK6HoWjS8GD9OOV/N0/Iiap1wmzHWtKxzbBAnzJQKlFwLx4TFusf37jQxdY2pos21C9WHnoY7bpjkqAz4cQgDKccUThM7Bca28MhQuES8F8JpQsTIENNFm7xjohs6xZxFEKZEaYqpGTT9iDiFJN3uFemjgFhBL8raqM9W8lh6JjjCVBHGCV4npunFmAb0ogRD0/DDhNlKjvderOLHCb0gYqMTopGSszX+4kdrvLPappIzKTgmP/bEFC0/ZqMTkCjFTNHmnbUOLT9mesuIPsw03L4IyWbc9EhTRZSmfODyxMg270dlwI9DGEg5pnDc7GywtZvAiFPxXgiPJyJGhlhvB3zvdoMkTlio5Njs+BQcg412QNtPRoqQYRTZjJpEQa3jYes6UQpoYOtZPXw7iHBNg9UkwDYMYqVor3cxdY1yzkLXNPwwouSa/Gi1w92aT84xeW6uyIXJPCh4aaFCOWcykbdRSnFjo4djGay3A3KWQcE2uLXZZbHe44npIl4Y31eNs9Pj0c/VWGz0uLHeo5o38aMUTdNGdjw9KgMuwkB4XNgpJkZVkIjAEIQMESND3K71WOuENLyYhpeV7eZskyge7Q0ZRZhkc2rSVNEjIWfqKE3RjRVRApoOUZKSphoaGgXLQGmZx2WjE6Cjsd7JZuC0g4Qnp4vkTIM4Sbk0mWeq6GwTE2stn6YXo6HhmDrvv1QdvJfVdsBqO+DqdOG+apydnpJ+rsYTUwXeXG6z0ox5aq6EudWNVsSBIGyfP7KbwJD8C0HYPyJGdpC3dMquRS+MyFkmK02Pund/5z69/38t84QYgG1CmoJpaXQChQY4aFiGRqISHAsMTSdKU1zTIElT/AQKtomORt0LUUC9F+IFGikaNze7aLq2VRq8fbaM2rrhVfMW1bzFpck8s2WX6+sdoiTl0mSOjU7ApQmXjh9xt95lIm/T6IX3Dc7r52p4UcLTcwU2OxF+GFPNWRRs48g/d0F41Az3vhgkcg6Vpcr8EUF4dIgYGeLSZJ7n5sts9kKiNMU2oO3HGHpCskOPpIClQ96EBA2UwjQ0/EjhBdkNSyPLDcmhc76ap+tFoEGcGFyZKdDxIhxbR2k6mgYdPwunRHGKbehMFmwuTuX4xLMz/OwLc/flT/RDK/VuhB9FrLR8So5JO4i5s9kjUWAZOmEKK02PGxs9vtGuM1u2KTjmtp4m00V7kKtxcSLHG8stumGC9P4STgvbxMMOEZGkktwpCCcZESNDzJQcfvyJSXSleG2pTaPrcWvTY7d7VpJmSaupUgQxaJHC1DNviVJgaoAG58sOFyoudSsLtyhgruziRwlJqrAtjXYQo28NhUs1DR0wDI3nz1f4ufecY66SA7ZXu2x2AjY7ASstnx8utWj7EecqObpBRCln8WOXJ9A1PRvS52STiYOozvPnysSp4ju36syU3G1hm1myBNySa/Psudy2uT2C8CgZdyS7eC0E4fQjYoTtlSS3az0c26TkmvixSc7WiZLM4xGM8I50M2cHGlnoRtfAMjTQNCwdJgo2f/W5OVY7PgqdXhDRiRLeXm0Rp6CSBIXOxQmXhhfhxQkqVcRKwzV1pvL2ttccbgrWCSJu13pc3+jSC2M2uxF522Sl5ZPrRhQdayAylho+OhozJRcNjThV2IYxsp/HYZfXPoquqsLJRg3lWOzmtdgpPgRBODuIGGGokqTeY7XlM5G32Wj7bHQi4iTFNEysNMLQwE8yETKM2vrSAMvUeXG+QjeMIVWEqeK7d2pMFF0uTLjUugaqE9DohTiGTqXkEsQppq5RckwSpdH1I2xL4+m5CkXX3uaZ6Ceanq+6/HApwNQ1cpZONZen3g1ZbflU8haXJguUHTMLPZ0rMV10aPsRL14o45g6QZyyWO/xw6UmcZp1g1VKoWnaoZfXrrcDvn+nQb0bESbJruXCIlpOD6O8Fjuback4dkEQxkXECEOVJNNFbmz0WG42uVP3uL3ZpRclWQgGsI37hcgwOQvytkHB1VHKYK0VEClFqqDopvhRymY3IIoVlYKF56cUHJP5CYtzJScr8av3CCKN85UcjqkRbYVY+vS9Fm8stVisZzkitqGDUlydLWIAUZqiqZSJgs2lyTy6rmchmB3JrwCrrTq2YbBY9wYlvIddXtsJYurdiHYQsd4Odi0XPo5W8EJGnKT3JXLeJzjEayEIwhEhYoShSpIwZq5sM1XMqlveXu0QxPcEiPeA1IkwBT9MWGv6GJpOyw8xDBPXSKjmLZ6YypGkmSDp+gmpSrOpvnNFnpotstYO0DSdgmOStw1ytsnlqTxJkvDN6xs0ehEV18A2oNkLiJOU6WKOt9fAi2Kmig6TeYswUeRsg3Iu+/Xu5nFwLYOZkrvrZODD8kwUHZMwSVhvZ3N0disXlhkxh8fOJlo7Z4zIEDNBEE4SIkbY3vXz0lSepYbHWivAMDTSaPzrhAkYmuL6Rhel9K0JvgrbtPGihF6UEiQptW5ErRtiGnBjs0s3jLmx0UGplPecL2/lg8RUciZvr7b52rubrHcCWl6EZer0/JgUhYbGctMDFFemixRdi7WmR94xeWmhihcldMNkV4/DgyYDH5ZnYqbk8IHLE2iaNmhZPyoP5aC5Ko9zmGfnhNR+zkWcphIWEQRhLFKliOKUKFVESUqcqGzIa5LS8iPKrsXVmeKxrE3ECNu7fiqlmMxbvLHYYCJn0vT3V0kSxQoP0EkxAEtLKTsGOVPn1Tt1rtc8ap0AgErOou2HbHZCrm/0mCnaFB0LTVMUXZtyzubN5RarLZ8oViQoom7CZickZxmU8zZlpagWLDY72TA+TWUVPt+4UePqdB4/SrhT61HrhDx3vsRy0x94HHbmhvQnA+/XM/EgEaBpGs+fLzNddPbMQzlorsppC/Pc1yxrRwvwnaESQRBOPumW17Fv5KMkM/zbjX/22PD3UbolFIa+j9Ps/DDOpsL3rxMlinjr/P730bbXGnrN9J4AeZAX9KeemeE//s0ff0Sf1HZEjOxA0zRqvYjNbojxELvq/gy9dOtLJZmRbAQx3V6EF6ckKmuOttGO0A2YKtgYOli6RtuPSNMUP0z58zdX0Q0dXdNY7Xh4fkTOtXBMHT9J8ds+Boqia+CaBn6suDyd45m5Mq8vNekFMT9cahEnKXcbHi0vwrX1bcmqO3ND9uuZUErxxnKL79zKck8mClkFj6Zp94mTB+WhHDRX5UFhnkfhOcm8F+k2T0WcptsERyYuEO+FIDwkfYMf3WeE7xn8UcZ/27FjGPydxj8TEOp+kbF1rTBJOc1RzzDeKyvyaBExMoL1dkDLiwn38XvRyLqw7hzoaxrgRwnNICZV98p/bQssXacXprR7IaZpoKFA6diWRkqCbRkUdIVSMJGzKNkmeUfD0AwaXkjONjF0nds1j2fmynhxQNOLWWsHBLGiFcSstEM+9MQEKy2f2/UOFycKLNZ72xJI+0a67UfMV10cU6fkWmN5JtbbAd+93eBu3Rscf7vWo+nFj9xD8aAwzziek52CZapgk8LAO5GqHR4NaQEuPKYopQYGd2DU05Qo7hvzre/Te4Z/u7G+36BvP25YJGw9l6aEsbpn4He5juQ5HS6WoeGYBq6lP/jgI0LEyA6UygzVrVqHza4//nlsFyKGBqkCTUGiNOIkExUp2XyavGNRcU1q3RDH1FGApenEmoYXJnikvHC+RCXn8MZKg7afUHR0pgoOFyby1HohGjqGBu+sd1lseFl5sQaupXNhIsdc2eGbN+u8vthksxMCGrquUe/G27wGBwlvdIIYU9eYLjmstwMcM/vHfByJqA8K83SCmChJmSu7LNY91tsBrm1sa6S12vJ5falFnCg0DZ6ZKzFVtHd5RUE4GPs1+FGced7CkTv18Qz+duO/ddyWVyAeOicWg39o9Ns+WLqGZeiYRvZ/e+h7y+g/t/W9rmfnbH0/6jjb0LaOv/e9qWvYpr792K3Xtcyt19S1wXoMXUPTNBYmcjjm8Y3+EDGyg/V2gBfFFF2LNM08GeM4SPpNzzQt+940sg6sidJIVea6U4BjZOEYXaX0ooSya26FWMC2NYgV1ZybdWO1DVAJSarRi2PqvYRGL+LSZImnZsr4UYwfKgq2T5oqnj9fZr6aY76ao+lttYd3DQq2iW3qdMOYt9faXJ7Mb5s3c5AqlqJjDox1zjJ4/6UqUwWbptc6tKZpe7Gz5NS1DWxTJ0kV651gW7ik1g3Z7IastwN0XaMXJmy0g23Xq3VDwjhltuSy1vbphTFTiBg5zfQbrj3UTn3L2Pe/j5ItYRAPf98XCPe+33n94bDBTjEgHB7WwGBnxtse+t4ytsTAkCgYPs4cMt47Db61y/PWDuPfN/K7GXxhd86sGNktfyAzzHCu4mLqEA3lrxps5YGMuJ6xNTCv76VPY8AEHUXONgnizG9iGRoqVbTDBPwEpYFrQt61iL1MvCxUHJ6eLWFbOvVeRDeIcHUNx7UwdI2KqxOnKZsdn16UcnEyTxAlFB1zqxW9wrUMojhlKu+iaZmhbvQidLhP/R6kiiXzRlTv80Zc25EzMs7vo59LMagW2TnA7ID9LibyFs/OleiFMXnbZLJg3XdM3jbRdY21to+ua+TtM/snsi+2Gfzhnfx+d+pb34f3Gf+9Df7OnIFBnH8rH0A4PIYN8U7jv83gG9s9ATsNvrnjOqa+ZfiHDfuO19j5vbnlXbAMMfinnTN7p92r3LXjRyzWPDRNw9DVYEjeXnU18Y77XQKDtqyJUjhbE32V0ggShaFls2eCSBHG4MfRVojFouFHtIOENExwDR2FhhemGKaOUorXFtu0guyYXhjz3JZHZKZkkyiyBNxOiGOaPD9fZrHewzI0ur6DYxn0gphbm91Bg7ODVLHslnQ6W3aZ2hIOfpRuKzsd/v5RDi3TtKyseC9Px2ThwYLluOgb/J2Z+iOz9tMsIW9gkOMtd35/V39fad9oYRDuEBPRNoEgBv+o2OZeN/XMZT/SEN9z04/2Cmw9v1MgmDrm1jVt857hH/5+ZzhBDL5wlJxZMbJbaGKm5FB2LfKOSd7SMw/GQ+Il2VTfNFUUHRMvjElVimNmXhQ/UoPpv2GchXYsA3KWyWKzR72TJam2PB/bMLk4kSNvm6SkGIbGubLDj1ZD3l1r4+gaFyou6+2Q6ZJD14/I2QZLDQ/T0Jl2XX5wt8VSs4WuQSFn8sR0cV+Jpfd6XaSsNgNafoRj6qRK0fZjXMugmrdR6nT3uijnsqZzYZJS64YjjfxwrD3aYeRHufZHGfy9svOHDf5wiEE4PHY1+EPfjzL4pq5jmdqOY4YM/667+R1Gvn+dHceZYvCFM8iZFSN7hSZSlQVjLMtgb3/I3uhAwTYJEkUnSLBNg6m8zVTeYrXj0+zFWROaRIG2leyaKOpeBKmiGyYst32SFGw9wmr5fOyZGXK2yd3NHmstD7U1/C5V8J3bde7WPTQNFqo5fuHaPAsTeYqOScsLmSxYmEaWTLvRCnhnrU3ZNVnb8hJFSUoniJivZHknEwV7UD0yHBbZ7IS8tdomTRXdIEbT7oU3nh0j4bMvarYZ36Ea+1EGf2fMP052y9ofw+APGflRIkE4PEa52u+57Pd4bofB3ykQrG0iYYRBH3psZJKgGHxBOFGcWTGyW2hivR3wo9UOt2sem+3wQK9h6qDrGgXToNEN6AYJlbzCtjTec76MqcFbq13WOwE528DUMxFRci3afoJpxLT9rMFZJW9l+RKx4sXLRZ6cyvHVdzcIkxQtVby71iZKFb0wJW/prGg+HT9C16Dei1hp+nhRyo2NLvVuQNG1aHgxP1pps9LyudvwcAyd5aZPzjYoOBZTBRvL1O8z+I1eRL0XYuk6m72AJFXkLZNelGAZ2U1+VLOf4ZwBMfmHx7gGP3PL3zPyIxP8RsTxs4S8ISM/fB1DH+zws5j/9muJwRcEYRzOrBjp5zrkg5gkUbT8LMF0uemx1vJBZaVYXvLwTWCSrVk1fhKRKogU3Kn5NHshE3mH6ZKF0jQSBZ0wwdIN/ATCXky9F9H0IsIkmztT8zJD/9/eWuPP392k60d0woQwTrOSYXWvD0afL7+9edCPSdjC0LX7yuTsHYl2tnnP4KdK4UUJhqZhGtlgwJJr7pmpbw+HALaM/LBgGDb41uC1xOALgnD6ObNi5PWlJv+fv7xDoxfiR+lg197sRdyudekGCQf12CdAc0fntBRo+CkN3+NGzdtxRkr97m7DcLKwxu37znl8MHRte6Ldzhr5XbLpHxSfHzb+LS+i3g2ZKji0g4gLE1lIai8PgWlo6Ps0+HdqPW5t9gYlwpen8lyczB/RJycIgnC6ObNi5G7d4z997dZxL+ORY2gaug761tC6JM1CJv0dfNExMQ0NDY1yziRJs4TOSs4aZOL3eyPkbIOya2KZxr26/a26+r5I2O62370070EGXylFrRtlVS5WVprci5JBxcu43oHhfJdzujvIcRm+vmNqVHLjX3MUUiIsCIIwPmf2Dmmbj67tbd+kqaHvLUNjIm/jWvqgvXGqsq6fhq4TJwll16KSs2j5MV6QTep9z0KFas6mkrMI45TNXsDdWo87NY+cbeCFCVdn8vy1F+dxDI2bmz3iOCVIUybzFkXHpuGFOJaOBtzY6OKYJlES0+jFXJrMU81bFByTgmMNklInCxa1bsRSozfIKzF0fayE1cOg1o0GIqIXxigFBWf8pNk+u5XuDl9/v9fcz+sIgiAI93NmxcjlyTy//rErhEkKikH8/0crLb55s0YvSO7rHfIwmEDR0VDoREkCmk6aplyYzPHxp6f5xLOz9IKIL/9og41OyFurTbpBRNGxeWI6z/mKS9dPWO8E+FGMrmnMlhx+4dp5nj9fZqMT8v/9y1v8wQ+W0YCibfCTT03zxFSBt1Zb3NjsUbAN6t2Q1bbPpYk8aZo1ALsyXcDQNJabHm0/zbwhrkUQpzx3Ls/V2eKgw+p6O2C15bPS9FlvB3zw8gR+lGIZGpMFeyhvBRRq8LPa6jybbnlT1NDzivGHxfXCmDRVzJZcXl9ugIIr08V9d0ndrdfI8PUPo/PqOD1NBEEQhIwzK0auzhT5f/3CCySpylq565nP4ge3N/l///8CXltsHcrrKCCIFYnKSoRtQ2HZBqSK6+s92sESjqlzt9bl1mYva3ZGSi+IuLnRJYpTyq6JYxt4UcJK0ydv67yx3Gam5DJbdvmpZ2Z4e71LvRsyUbB57nyZVMF00SGIYtIkJVWKtVZAGKVcmCxgGQZTRYeFiQTT0DEMjzhWVPI2TT9iruLywnxl8D5WWwGmoXGu4nKr1uX6RofnzmXN1qr5BxvcvSbmqq3mZ2tbz+dtY8sr0X8+CyO1/ZggTjhXzkqZvSimkrMGa1AqCzmlW0qn//1ugmh4cq6EVQRBEI6PM3/HNfTteQF+rKi6FkXXoOUlB+gykpGQNT8btJJXipyhaPkRNze73Kl10VCUXZtumBDEWY8Tpae0vJiCEzKRt1naaBPEiqmizWTOYaXp8cZyJphytslHr05TyVs0exFTBYeWn6Bt9SBZbXmkKRQci4WJPI6hk7OzMJVSGk/OFgmSlJ4fUe8FWdM320ApNRAMRcekG8S8s9bBMgx0tK2ur+N1a91rGJ+maWx2A15fau06rO/SVJ6cbdAJ4sFcnW6Y3CdsHgalFJcn8yxMuHT8mMJg3o52n3BRgEp3eH/Y4fHZec6IxwRBEIR7nHkxshNN04hJsxJKM8GLH3zOuBha9qUpiJKEtp9N2A3irHeHYZhUrRQvSNH0LHS02QlxdI+5kotl6DS9mKWmh9IYhEzKbpZ0CjBVdLg8VUDTNNp+RDVv8vZqB8c0uFPvMVOymSy4vP9SFaUUd+o9GoshtV7I+ZJLJ4iZKrksN/2B5wWyviyXJvN0/Jgnpot4YdZxdZQIGOUFedAwvgc9v1vb+cNA0zQMQ2O+erTVLsOfS8E2mC46oGlD3pvRwiVVwI7ybbXl+klHCB/Y7hES8SMIwklHxMgOLk3muTpd5J3VNslB3SJD9GfVKAUqTLOeFFqKSk2Kjo4OuLaRhSSICLYskKUbFFyDF+ZLdIIEXfOJkpRmNxyEds5VHSquhWObTBds1tt+ZujIvCHVvI1paLyUr1DJWUwUbKYKNm0/K22dKtpoax0uTOZYbgaUHIPFhkclZw28DpqmcXmqQNOL8aMstLPbQL1RXpAHDeM7yLC+k8ZuIandvEMGR98npC9q1JAnpz8PaFgIDXt62PbzdhG0Vwhs+BxBEIRxOL13/CNipuTwgUsT/F9vrqG0ePSI3gOQALaeDclLEkUvDck7DtPFLA+i5BokFZe3Vzq0/AhTT0hSl8m8gx95TBZt3lzu8M5ah+/caWKbGn6cYBk63TDh4kSeH9xtcXEqR94yuVPrUclZREnKdMmh6WUN2JpezHzVZarosNkJqORt4gTCOOE7t+pZ2W+iuDSZZ66SG3w24wzUG+XluDJdGJxb2AoBXV/vDK5zkGF9sHdOyqNmN9HxIO/PUaJpGpnz7NF+Jml6v5gZ9v7cF+oaFjgP8BhJ+EsQHh9EjOxgreXz7ds1vDBBPXzzVSC77Y+6PUZp9p9SzkKprMx3vurQ8mOuXahQ64S0vZC8b9ALY2rdgDv1HmGieGetS5Sk2JaOUpCzNTphwkROB5W1k//hcpMwSXhxoUIYZ2/ibr3HRscnSTVemC+jofHEVI5rFyq0/YiXLlSwDY3v3Grw3Tt1posu622f799p8NRQbsY4oZK+l2Ox0aMbxGx2gm3nr7X8bcb6pYUymqZtEyo3Nrr7EhWZAGiw2QmJU8X7L1V5/nz5kQmSYTG02QmI05SFan6b6HicvD/jog9ysh7d7yEdquTqe38G4a2hvJ++CBqIniHBtFPw9ENkgiAcDY//3XCf/OBuk1fvtuiGMQdNFxl169K3ntB1OFe2afkJvTDlrZU2fpiy2Q2xjaz3SNuPiBPFZhpza7PHX3lulju1Hp0gJmcZBHGKbRi4RpaMGkQJ319sEkYxrm3yxkqLIExZbwf04oTpos1ywyOIE2ZKLi9eKGfiYihRtN6LuFnrUXRNFhsh37/bYLnlU3RMfvKp6YGXZC/6Xo5bm106fsxmJ6Tpxbt6CG7XejS97LG2H6FpUHSskYmsu5GJgJB2ELPRDlBKMV10xjr3MLwqw96Q/nvYKToO6v05TRyXp6rvATqq0NdwuGunp2ebuEnvz/vZWfKeKvHuCEIfESM76AYRm52A3gGVSP+DVWxV0Wz9nJIJkjiGtVaAY5mUXJMoSWj6EZ0wq+bI2wa6DrZuYGrQ9GOWmz7zFZckTVlrB2hAwdJJSYkSjamSDWnKfDXHBy9P0Ohlg/KaXkjLT7hb6+GYGu+7WEXTNJwRjd8uTeZ5cqaQGRHXpOPFVHIpq60ulybzzJbdBxqZfrJpJ4ipdaP7whI7PQTAQJx855YHGjwzV95XKKPomMSpYqMdMF108IKEr727wXw1N1j3bsZwr0qfcRkWWIv1rOppquhsEx1HmYR70jiMz/Qk8ijCXelWA8TtXpsR4kap+0RQP6l58Fh6zzM07CkShJOGiJEdGHoWLjgopgZKgzgF18xuJkkCEaBpmSjJ2wbnqjnWWz6dIMaPU1AKxzLQUEy4NkrTMHSNvJW1WS/YJtFW9Y2GhmEEOKbBZF6jlLPoBglxqvjRWpcnZwo8f77M64stNrstXMvIrq1pTBUdSu79XUFnyy4fe3qGThDzzmqb795poBR0g5ilRlZOvNTwSFIeaGSGRYeugR8lvLvWxo8Sym5WnltwTDp+1tl1pdkbtJ/fbyhjpuQMKoT8MGWp6XG36fHWaocnZwp87OmZXdd5GLkcw+/VNHQuTxUeC+P7sBxnfsxpR9c19CMUO7t5d7YlJPcrs0aVtu/8fsf5IN4eYf+IGBkim08S7nso2ih8BYYC28zKeRMYhH0ile2rco7JBy9V+fr1GqlSNLcmB9e7IbbhcGW6wGY3RinF+Yk8QZLy1mqbpZaPuTX/pRskoDTeXu9wu9ZlruTy8tVJQOPSZJ7nzpXoBjG9KOby5DQrLZ+5cha+aPvZUL5h78bw7r1gG7T8mJWmh6FpeFGW3GoZOi/MVx5oZIbDEn6UsNTw2OyE3Kn3qLo2YZKQsw1ylsGdmsdk0eJc2eX582XcrTDUqDWOQtM0nj9fZrro8MZyCz+OsczMWd8J4j3XeRi5HP332vajfa37ceUs5secFo4jmXk4MXmnCIL7S9H7eT07uzXvVdG1M6lZSttPF/u+Q/z5n/85v/Vbv8W3v/1tlpeX+eIXv8gnP/nJXY//8pe/zF/5K3/lvseXl5c5d+7cfl/+SFlr+dzc7I31J6qTeTf2QgFenDU829pkDDA1aHQCvvyjDeJUZc3QTIOSa9INEwwNnjtXYrXt40cK19B4bbFBy4uIk5QgVthKUdhqRhZECU7OIkqh5Uc8f77K5akCuq5zcSLHa0tNvn2rzmTRZrpos1j3qHcjgjjmykyR8xV3YDCGm4l97Olpvn59E+hxrpJjtekTp2osIzMsbK6vd0hSqOQtXl+MSNMt4afDtQsT2KbOU7MlNDRytknRMbkxws2/Vy5C//UgCxNc3+gC8GSxsOc6DyOXY/i1R637rHGW8mOEB5O1Bxj89Mhff1Rp+6jKrlFeofsqvnb0+JHy9sNh32Kk2+3y3ve+l7/5N/8m/+P/+D+Ofd5bb71FuVwe/Dw7e/KctrdrPZJUUXZN1h+QNDJOoU3/mFHtSiwDTNNgsxNg6lByLYJEoekaE1tD1b5+o44GtIOYME7pBgmmkTVDq+RMyo6BrmvUOgGu4+AaoGvQ9WNcS2dq6zqb3ZA7mx69MKYbJMyXXRq9mOWWx431Lt+70+RDVyawjKxCp+TeSx7VtGxKby9K+eaNGlem8rz/UhXXMkaW6AIjxUJ/p7zZCUlRtP2YcxWXuhey0fazhm69aJBnsZubf1QuwkzJ2faa00Wbjz09zeWprInZpcn8nsbwMHM5JDyRcZbyY4STz3GVto/K69m1tH2PpOedAmfUY6edfYuRn//5n+fnf/7n9/1Cs7OzVKvVfZ/3qCm62ayT1XZAGCnCI3iNvqdksxOgaTqJUmx0QkqujWMqTN1gsmCTKMV606ftxxg6BFFKGKUUXBtL18g5Jn6osC2TjW5ITwfQWGz2+O9vrDGRt3jPQpWNTohl6jw3VebNlRarLZ9uGPPmSptUKcKtkEInyPqqPHOuxA+Xmnzt3ezxKEn58ScmuFXr8cR0YVAyu9r0+Iu3N+gGWdLtx56eRtO0kWJBKZUJKNdgvupya7OHaWhcmMgSTIuuhWPqlFxrIBx25ptcX+9kZbNJysLEvbJZ4L7XnKvkxqr8GeYwKkAkPCEIQp++R+goc4BgdH+e4bL2UaJnp8fHOOZw8iO7U77vfe8jCAJefPFF/sk/+Sf8xE/8xK7HBkFAEASDn1utwxla9yAuTuSYylvEaZZEaugJYXD4ilMDwjgL9VhmOvS4Yr7qAjqOodMKY0zTIFIRXT8hVRoTBQcNMA2dat5mOfTRyCpjbB2iVJGzbRabHu+sdXhhvoKha6y3fd5ayZJYo0QxXXSZLPi4psFyy2OjHVBwTfwo5Rs3aizVPNKtwJKha2hozFdyFBxz0APk1maX6xtdqjmb1XaXy1N5porOfZ4BgFcXW9v6ijx7rryn0R/OwVhu+nznVg3bMNF1BWw39ofljTiMCpAHhSdOUnM2QRAeD4bDYI+io/NRcORi5Pz58/zO7/wOP/ZjP0YQBPze7/0en/jEJ/jGN77BBz7wgZHnvPLKK/zmb/7mUS/tPjRNoxcl9MKIKE3pHVCIDOeVaIClZQmtALECS9fwt7JZDV2j6cWsNAOuTBeo5Cw6YUySxNiGxlQhhx8nXKy6xKnGpakc9W5IrRuQKo1qzqKSt6l1QurdENPQyVkGay2f5aaHbeiYmsbLV6eYLTlZC3gNlpseObvA7JZRzJuw3PJwLA1D02j6EUGscEyN+aq7rZImTfvv7t7nNMozMEosjKrk2fm76AuBr1+vcbfuM1PKQjhXZzLR0w8TbXYCOkHEYkNh6ru3qX8QhyFqHhSeGCfMJAJFEISzxpGLkWeffZZnn3128PNHP/pR3n33XX77t3+b//V//V9HnvOZz3yGv//3//7g51arxcWLF496qXTDhChW2IZBmqgDNz2r5jSCWFGyDWbLLlemi3z3boONTohSCsvQMQxFEityjkHHj9GA5ZbPRjskQZF3THTDZLZoESWK3Nao+zhJWGt5xCn4UYTSIGclXJrOM192mSjYVHMm37/T4Pp6l5mSQ9OPafsRCxN5Lk8VKDgm6+2AvKWz3vaxLJNLk3laYUwYJ/xwpUOjFzGRt6h7Eb0wIUnhfNXljaUWjqkxU7LRgSdnCoPcjFGegWGBEsTpA5NT+5N531xp0wtipoo26+0Ax8zKZmdKWdXMd283MLTMUzRVsAfPjeJBXomjCrHc1511jDDTWUx6FQTh7HIsAe0f//Ef5ytf+cquzzuOg+M8+uz7gm1gmRp36h6d6OBekYaXXcM2UizDIGcbnK/kCKOUbhSja3C+nMOLFY1eiFJQ60WobkjOtgjjLBHVsUwsQ6NoW6y1PSoll9VWQKw0XFOjHSgqBlTyJlem8pwruyg0NnsRNze79MKE+WqO2bLNxck8Ly1kicTvrHVYbYWcr7i8u9qlEyVcX+uQtw2enilQ64aUXIPpgou29XEYusYbSy3u1j0uTOQoORaXp/IDETDKM7BToLT9iCRVnK+4vLnc5o3lLAynlBqEc1peSCeI6QYxtW7IubLLxcksebbvSfjOrTp36h45W8fUNS5P5e8TGMNCwI8SFhs9ap1oZMv4o6oAGfaGdIIIpTiSMJMgCMJp5VjEyPe+9z3Onz9/HC+9K5nR8umFWbKoSRZiGadqxuD+ipnh8+q+4m6jR6VgEUYJmgbTRYc4VpyvOKQKvt320XVoeTG2CUGcoNDIWyZBrPDjlCiJ8WK4knfYiBWoCNsyydspUaRYaQa0ejF5x8S1TT54aQJT05gr2biWzrWFKh+5OjVIMr1T67HW9ii7JmGakCYplZxNEKcYhk7etmh4MSstj4tTWaKppmn8cKlJ24spuyZtP2Ein4Vcbmx0Bx6N4fLgUQLF0DXeWG7x1mqbtY7FRifg4kRuYJTfXm2x1PS4PFUkUTBXcfnI1anB62x2AixDJ2fpvLHcZipvcavcu6/ZWF8IxEnK9fUO7SDGMXW8ML2vZXx/nTNbAma/83F2Y1t31oZiqnB/d1ZJehUE4Syz77tep9PhnXfeGfx848YNvve97zE5OcmlS5f4zGc+w+LiIv/pP/0nAP7Nv/k3XLlyhfe85z34vs/v/d7v8d//+3/nv/7X/3p47+IQWG8H/MU7m9zc9Jgs2Ky2s+m2D8ICCg40gr2P2+jEXF/v4oUJLS9C92I0TXG7puNaJroGtmngRwnRlrJRKFJSUHr2XKhwbYO1pkfRNXhytkijG4LK+o50/Kz7ajOISVPo+hFPzZb4v70wx3w1NzB+Nza6JKnixQtV1jshCsXCZJ6On4VDGl6IFyVMFEzOVaoEccJ7zpeZKTlsdELCOGWp5bHeDbANnfkJl5ub3tizZfoeiK+9u0GSpli6wbvrXUquiaHrWxU0GpaZ5aAXHJP5am5bpU7bjzLRaGhM5m0+fHUKx9Rp+xFKKW7XetlnqBRxkpKzTVbbAZudgChVPH+ujG0YI70Qh93KfFt3Vv3+7qzSk0MQhLPOvsXIt771rW1NzPq5Hb/yK7/C5z73OZaXl7l9+/bg+TAM+Z//5/+ZxcVF8vk8165d47/9t/82shHacdL2I5pehALiJEXXQUtGD7sbRtcgTjVM9s4xSYG1tr9VTgVxqtA0WG37WLpOkChIU2wTXMOgnDMJU8W5Sg7LMJjMWzS1iMKWz6Zomzw5U2C9E1LvRdyu9bhd6+HFMbZhcGEyt5WUCrahcWW6MNjd942jHya8tFDh8lSevG3wxnKLbpgwYzjUuxFrbR/T0AfnvrnSZrHusdzwSNKUZ+aKaGjESbqv2TJ9D8R8Ncdbq53Buqo5iyemi3SCmAsT7mA9TxazfJS2n80NquQt4iTl6kyBy1MFbpV7OKaOaegEccp3b28MGp7NlGxKjsVK00MpxdWZIrc3uxiaYqJgjfRCHHbY5EFiQ3pyCIJw1tm3GPnEJz6xZ4OVz33uc9t+/of/8B/yD//hP9z3wh41QZzihzHtXkS9F6EpMHWIHhCnCRToicLUsgqZPY+NsmumChIFOVMjToB0q2mNBpah4ToGBddmztF538Uq6BqtbkCYKOpdH4VGnGrMT+SZK+dA01hueswUbaIkpZq3mSrY9OKUolK8vtRC07RBXsduxnGm5A5m0qy2fKaLLrdrXTY6AZudkB+tdrB0nSdmSqy2AzY6ARN5B9PQiZKs3XvBMUdOrIV7+Rv9lulpmjJdsOkGESXXpLC1ln4ya389/TVuLme5Kjc2uliGzrWLWc7H5anCtnyUbhBTzdmAQgcuT+WpdQPeWm2z2vTI2QbPnCvx3ovVkV6Iw05kFbEhCIKwNxKc3sI2NMqOxWzJZrNj0fRjojHLabwxEksUmXckK4vNxEiSKAxDR+kKx9CZyDlYpsZM0WIib2PoGq6VVdnc3PRY6wQopWHoKV6UcH2jw4cuT/LMbDZ/Jm+ZdIKID17O2qvfrXssVF2+e6dJrRNyebrHx56eZq6SG2kc+4bZixK8KGUib5OzTXK2wfxEjjsNj67vk6qU6aJDECdYuo4XxixM5AddWWF7zkiffvhjsxMMEmA1fasSJu+w1PCZKbmDCbvDa1RK0fEjHFNjYaKABjimPtLQ522D6xstwjjz3lyazKOUwjZ1XNPAjxMm8vauoRcJmwiCIDxaRIxsESaKzW7ISjtEKQ3X0omTNJtBwPYmwuPU2eycR6O2rmGZ4FomXhiTt3Wmizb1XowXxvhRTDnnoOsG3UjhhwmLjU0MXePGRpdumGLqmREu2BampnNlpshTMwUMQxsYz598qt8JtcG3btW4udHl6dkS19c7XJ7K79qZdL0d8P27da6vd1hq9EhVypPTOQzD4M9/tEatE3K+kiNOFRcmc6QKFqpZiaprGVydKe75mfTDH5W8xY2NLpWchR8n5G2T5+d3D+v013an7tGLUm7XelydLozsVTJTcnhhvsxGNyBJFSU3+yeuaRoFx6Kay3JiHjR0TzwZgiAIjw4RI1s4ps5U0eFWrUMvTomSlFTdq4rZb6FvQjYMT6lsDk2UZA3PkgR6KivrDSJFrZslneqaTppCwTVpdkOCRDFVtAmSlMBPiBKFa2pomsZk3uIjVyaYLuaYK9kAlBwTU9d4cqaQeRGCOOu2GiXoukbDC9GDLPSw1vK3VYj0wydvLLd4fbHFcssnjBXNXsTsE5PZOjshiYKn54oEsaKaM7lT9/jO7RoF28AL420zajRNu6+vR8E2BvNpLEOn5WWP7yx1HUVnq+X8h69McnOjQzln7jp1OGebXJ0uDXI+umHCxYkcM0WbWjdkpmhzcWJ/reKFgyGdZwVB2AsRI1tkM1FsyjkH1/SJktED7vpo7C1Q+rdZU4d4KxE2ikHTIU3BsjSiRBHECaapYWg6mq6x3gwpuiZTeZMkUbiGjq3rBLEiZ+loCiZLDg0vJkp9VloB37/bGiRsZvNuNLphTNOLKNgW771Q5eZmF5MsBPODu81tnT9vbXa5tZkNCXx3rUPLjzPREaX0woSJvMOPXZniGzc2uVXrsVDNU9gSESpVLNYzgTNdzDFRsHjvxSqzZfe+qpSXFsqDFu8vLpTpbjX8KjgmrmVsm0uzk6JjYuo6fpRSdC1aXsw7a92R1S6jcj6UUpRcC13TtvJaxBA+Sg67QkkQhMcLESNbzJQcPvjEJBvtgOVmD63h7Xm84v7+ItqO5zQtEyOOpRFGiiAFY8vVkqTZgKJEKYxUJ0ySrYTZhErOZbrgUs2bPDFbotHx+d7dJmGUYFvZ5IE4hVrX5921Dh0/JkkUtqHxo9U2620fTdNpehFTeYtn5sqYWuY1UKlio+PT9rOJtj+422Sx0WO1FfDjT0xydbrIjc0OQZRSyVlcmMjjRyleGHN1ujBocNb2I4qOiWXofOtmHd2Aa6aJQnFrs3uv22iaDkI53TDh6kxx0D317bUupq4xVbS5dqE6SFxdbXqD0txLk/ms98dQHsfmVkLtbtUuo3I+bmx0KbkWz54rD9ay7fcpO/cjRRq7CYKwFyJGttA0jefPl1lvefzxa0sEYySv7vScKMDeCslAlqhqGVnCZLLlRtGNLFSjUpgsWiRxim1qlHQTXTeYLlg8da5MxTGZLTl4UUInjJnIWQSWkXVeTRXdKMXWdX643KQdJPhhAlomgGq9CC/Myn/9OGWjFxCkCbfWu9za7PLkTJGXFipomkaSKp6YKrDayjwk71ko86GrkySpYqbk8Oxckc1uRNuP8KOEbhBza7NL3jZo+zHfuV2nE8aUcxa3N7ucq7iYhkZt65xRlTX97ql36x7TW56QvnFabwd85Z0N3l3PPD1Xpwt8/JmZLIdjK4+j6Jg0vXjPip2domLYWzI8Bbh/jOzcjxaZZiwIJ5OTshGTO8IQmqax0vJZ7UT7zhHpEyVZK/j++X6SEKdZ9YxGFrLRAdfWKTkWPS1GpYqcbZGzdGYqOZZqPVo5kx+ttrlb92l6EbFKmSnYtP0Yw9CJlWJhtoQfxcRxQiVnkaI4V3ao5W2+e7tBnCaECaSpouJa2HoISrHR9nl9sckT0wU6QUSqjMFsmctTBaaLNhudrB37ZjcahE6+d6exTSRU8iYLEy6zJYfNbsBEwWa65NDxI6aLDqkymC4693Ub7QRZL5S+CHBNfSAONjsBHT+i4lp0gphbG11uTeW3ralgG7y0UN5WsdP/g7q12eV2rUdhK6zTFxXD3hI/SrYN/Os/Ljv3o0MqlAThZHJSNmIiRnawWPNIk3TbxN0HoZEJjIR7uSTm1vdJmnkrNJX9P2dm/y/aFk0vwjIMSjmdSs5C1zXu1DzSVOGaOrapkXNMml5I24uIowTLMsjrgNIJkphUaTiWQb0XYZsaRcfCNU2u53uoNHvxXpjQ8iNafkTeMehFCV+/UcvKhA2N6aKzTYR8+1adW5u9LE/D0AdGpBPEVHMWoNENYi5P5Xl2rkx9S7A8MV1gueGx2g5Zbdd4cqsp2c5/2EXHZKKQVcI4ps4T0wUW6x6pyprPpcBSs8daO2S25AzExVLDJ0kVugYLEzlcyxhcs/8HtVjvsdoO+PCVSfwoHYiK4QqZ6+sdkpRtwkN27keLVCgJwsnkpGzE5I67gwuTeSaLDn7DI0qzDqvJLm4SnUxwpNwL2fQFTMxW3gjZNXKOTpKmmHpWYmoYGmmswFDEqSKIFK6lCKKEmZLLasvH1iFWEV4YY5sGQZwQpYrJnItr6+RMkyhN8UJF3jbIWwamrnFpMk+UKJJU4ZgGQRyhAx03Jk5T5is5XNuknDNp+Vm4A2C97bPc9FlseIMckpWmxx+/luVvNHshfpwVOl+dLnBxIkfBMbFNnZmSg21odPyYD1+Z4uZGZzDFt89w07OFiRxXZ7Ly3LYf8c5al/lqjrv1lJJrEMcpOdvgw09MEiTZef0/mDeWWqy1A6aLzn2ejSemi6y2A25udlmo5keKilHCY5yd+2G5M0+KW1QQBOGkbMREjAyhlOL58yU+9vQkP7jTZKXpo2tQ78X4I9wk5lZlzCitYgAl18SxDNpeRN42MfWs06vSsiqakmtyZTrPajvA0MGPU+JE0Q0jgjhhYSrPVMlho+3T8BI6XkTTD7nT8MjZJnPlbHhdwwtBaVyczIOCphfhRRGNXkw5b1JyLCp5h8mCw+vLLeIkxdI17tY9lps+CSlvrXYoOQaTBWeQQ/L6YpM79R5rHR9T15nIWfzYE5NcnsqqabIW9B7FLa/FfNXFNLImaIWh/JC+sd3LHWjoGov1HssNj41OSKoUQZSyOiQ61ts+zV5EEGfibKdnQ9dgpeFRcgzOV1xeWigPRMWwABgV5hln535Y7syT4hYVBEE4KSFUESNDrLeDwTAz19I5V3XpehG9KMEP7kkOE8jbUMzZ9IKIhr9djhhAwdVJFUzmbS5N5ImSFD9OqXUDvCilYJmUchYFx6QYZt6GjUZAmKR4zRjXspgp53jufBnb0PjmjRrrnRAvSfDCFC1UvHq3STVv8TMvnOPWZg8/TFls9uiGMevtAMfSmbdzXJ7Mc3Ozh6Hp5G0Dx9bJOwa2AbapoZSOoWUVPnGq8KKEJ2cKGBrcqXcxNJ1qzmKzG/LOejt7kxp0g5i1dsiHr0zhRwmOmYV0bm126YYxm92QphcPjO1u7sDpos181eXt1Ta3ax5LDY+ya6HpkLMy0bFY72EZOlGacmWmOMj7GPZsLEzkWGsHTBYcdC3rydL3OIwSAA9q0raTw3JnnhS3qCAIwkkJoYoY2UKprCT1K+9u8s3rmyw3A9Ag3QqD9DGAnJ2FQiaKDou1Hl7oE6RZSEYja3IWRFnTtIKj80sfvEDDi/jBnTorrsntjR5o0NsySjlTp9aLsA2DomuSpIq8bbLW8nEtnY9cnWK6aHOr1sUPE8I4xbEMYpUSxglPzxZ5Zq7E197d4J21LEHTtYwsjGJvhVHKDrV2yHTRoeRabHZCukGMF6Y0/Ahd0/jQ5Srvv1TFtQyKjsl62+cbN2psdELu1HskSUo3SHhjqc25ao6feHKKtXbIzY0OCxN5Sq41EB21bnSfse27AxcbPbpbJbr9HiBLDZ9GL6LphdimTgo4us58NcsNSZXGC/MVlhoe5ysupa0E12HPhmPqWLpOOWdS62TVPH2PwzgC4EHhk8NyZ54Ut6ggCMJJQe6CW2SVGD1ub3RZ6wRYhoauaTSCBAW4BvgJFGyYLLosVBwmSy7NrsdU0aLRiyjnLKI4m+sSp+DoGn6k+N7dJkopat2YpVrmubANgyTJkkt1w0DTInKOQcePiBKFZRrkbYM0hXdW21lYpuRQzdncbXgkacJCJUcpZ/GDLQ/J7brHaidgqelDqoiTlIuTWSfXkmOxUM2R3Fa8vdbGtkzW2x7z1TyfeGaGzU7Ae+bLTBXsQQ8Ox9R578UqV6eL/OXNTTp+zNPniqy3Q3pBRH2rm2k1bzFfdZkuZt1gtxvbe2W0/fDIrc0uS3WP170mtzZ7XJpw2ewEuJaBZegoleBaGk/PZnNlNE3bZrz7omenmAjilDv1HtFGimXovHihPHhupwAo2AZrLX+b8HhQ+ORh3JmjBM5JcYsKgiCcFESMbNHPJXhhocrbax2afoKupeQcnY6f0u+RlSpoeiE/XInR13okKkVDZ6aYVZO0/ZBWkNL1I1AQRAnfv9skDCP8RNHwItJUYdg6Jcek5GZD8WZLFrc3fVpeiK7rKJXSCROC2KMZxkzlLfKOxUxJZ6Jg0+iFVPMOXpjw9es1pgo2Sw2PuZJLmiqiJKXiWpyr5nhhocJyw+fJmQIoWG56WYKrysSQoek8d75C0bX4yjsbAyP53LkS00WXibzCNDXeWm6z2grQgGfOl5mv5mgHMY5lsNTwmS46I8to+5UyfQOvaRob3ZBqzub6Rpc0VdytewT///buNDau8zr8//cuc+/sM+RwF0Vq8SLZlhwvsaM4aX9tjPrvv2G0CJCkhQuoVfsigILaMbrELQo3KBInBdI2iAMnbgobRWOkRlu7SxqkrtPYPwNxvCq2Y1mOrIUS9232ufv9vRjOmKRIiZKGHok8H0AvRIrDMxybz5nznOc8no+qwJU9Ka7sTS1JBtayeJu6ymBHjEw8QqHqYupq83OLY0oYGjNlm0OnCkuGrp2renIh5czVEpxLoSwqhBCXCklGFiRNnYrj4/s+V/clGS9YKIBlubhuvXvVDaHmQqhAvuaB4tObjuIFIWmz3qxqRGLUvCooGkHo44VQrbkoYUix5gIKMUMjCBTsAEqWT3dK47rBLLqar38uDJks1ihaDpmowXTBYqpQoydpcuNQF9tycY7PVHhnooSpa5Qdj1zSYLrsMFOuN7d+qDcFYX0r6J2xEh2JCKlofVT70akyI3NVruhJkjB1ejMmu/vTnJgp8950hWwswmSxwtaOWHMBv34wzYeHO3hvukLM0Ni7JUPF8ZunYBYv3suP0Qbh0mO076tvfxm6wtaOOAEBh8cDEqaOqqqoqtrcJlm+eK9UcUhFI+SSJn4QklvYjmpYHNNU0eL1kXxz6FpjaixA2XYZzYfoqtqS7ZPFCc5ovtqcTiunaIQQ4n2SjCzoTpkM5+JMFGtc0ZvG0CIEYcBUyaIaKDiuD149uYgoUHFDdDXE8eq9Idm4Xl9gkiauF5KLe+iaRi4R4eh0hULNxfZCohGlfgxWV9jakeQjO3MoQCaqk4lHGC3UsFyfiKbSla5XOabLNt1Jg0Q0wrauBNu6krw1VmKi5FBzPXRFYbgzwbUDaSDFTNlGV1TylkPWNJitWmTj9d6M7pTJ/9nVw+sj+WZVYFdfCoDxgkXF9khHdSq2x3jBYltXku1dCRRFoS8b57rBjubPbKponXPrY6X+iIRRH7JWtj12JhNc2ZtivGAzmq/PE9nencJy/bM2dq5UcVjr9sfyoWuur3NytkrC0ChU3frx6N54c9vpYix+/hXbo2zV+2nkFI0QQrxPkpEFiqIwnEtwZKLE6fkaqXiEjKkThAEly8P1AsKFJlXH9+lK6HTG64uV7YfMVRzKdkDZ9omZGvt29HJ0usRk3sLzA6quT9RQ6klG0uSGoQ6SZgTHC5oL6mBHjLLtUq66hGqUbCzCkckSUV3jqp4MplGvFJRtD4WQbFQj9H1UTcX3XSDC1s4Y1wykmS7ZTBZtetMmL52Y493JElMlm21dCfrSJjcMZTF1lVQ0QhiGvHG6gOUEaIpCvuaiKQqWEzQv1Vtp0Vy++DceZ/HFeACZWP0/s8VzRz5+ZXfz67qSBt0ph0xMJ2FUqTkeunb2ysSKWypr3P5oDF0Lqc91SRj1Swljhs5MxUHTlCXbTg0XMh/kfO7UEUKIzUqSkUW6U/VFerxQY3S+hhtR6M3Ub5+drzooav0HpqkqHXGD7lS0PqsChdFChYgWkI7qJDSdfMXC83xqroem1CsrWzpiRFSVPYMZPrG7h7fHirw3U2Z0roap10+OTBbshUUyYLJokzQjdCWidCbq/R+Nhs6S7XFkqoLt+oRhvUckopXpScW4bkuavkx9++it0QLTRQsUhTdOF3htZI5btuXoSkWbScZ7UyXmyg7pmM5AJkYiqqIpGrv6U4wX6pWO7kUDy2wvaCYyjerBShfjjcxVKdS8ZnKy+Kjt8qSh0WsynEusqbFz8cmcsuVydLLEbNluXqy3PElYPmdk72CGkbkquqbg+gGn52vMlG0AtuUSS6a3NlzIfJDF20Nnu1NHCCE2M/ltuIii1Eejb+9KEoto1Nx6D0l04Z2z6wEKmGpI1akP5Ko6Hn4IlhcSEjBRtBjIxrD9et/BVNkmX3GouUF9amgiytaOOHEzsrAwWfxiukyp5mFEVCKaynBnnFwySs3x+KWruyGE/myM3f3vD/Ea6owzmDXRVY13J0u4vk/KjGDqKhOFGuWazak5i7fH88zVPFTAiGiULI2i5TJTsanYLh/ZUZ8Rcmq+upDQqOwaqI9SHy9YzUWzsRDPlm1Oz9fY2hGnc2E+yOh8jfmKy2zFImrUR7Trar15dK3zNIIg4J2JUnNI2rZc/Vbh5ds+jSSjUXE4OVthPF/jvekKiqIsuVhvsZUSiVzSZK7i0p+NoqAQjajNOSsr9Yxc7HwQOUUjhBArk2RkmYrjN6+af/XkLMemKhCGhAtvtHWtvsBqmkJHwsDxAopVp34cV1MxdY2UqdOViKApClXbb46B15X6MLWQkNmyzVTJYrZs0xEz8IOQqK6RiunUXJ+kGZCORZgp2fRlYuzqSy1ZYK/qS/PG6SLHZiooKlSdEFVxSccMslqE0/M2b40XOTVvYbkBnfEInUkdLwh59eQ8qqoyXXLwgpCtHTEGO2KkYzqn8xau5zOQjTWrH90pk2PTZebKDpbrMV91GMiaC1UJh+mSw3y1fn/OQEeMXMJgOJcgDEMKteKaKgHvTJT4wZsTzYQIoCtprlqJaFQcyraHqip0xA0ad+aslCSslEg0qivjeYtc0mTPlnRzG2ylZOFi54NcKsOFhBDiUiPJyDKNseKHx4pMl2ymKzZhqBCPaHh+gKFr6CromornhySjGtl4gqrjLTRGKkQjGiWnXm1IGDqKAio+PWmTdFTn7bFifYqqqpCO6syUHfwgBCUkFzcY6oqzrTNB2alv8azUlrCrL8VHdnSSMFS60t0UKjaZWISYoVNzPX5eqJKv2nSnTebLDqqqkIlF6ElFUQhJRQ0SplbvP1EUckmT2bJNseoyXXLxw6WLf2OGR77qMFGsH8PtTkXJVxxOzlUpWz4diQi6rpFb6LUIw5C9Z1ncF6s3kgZc3ZfmyET9Zx+NaOesRCTNeuPwZPH924Qv9D6axkWBq5HKhhBCrA9JRpZZPFY8FqlXOdRMyFTZxg0CDE3jQ1szDOXiZKIGFcej7PgUqw62G3LjcAcpUydqqKgo5BImJ+fKFCouCTOCE/i8N10hHTXQFbhpuIPD40UUFBKGytaOGHftGSAa0fjFVJlYROPEbIWRuWqzFyIMQ2bKDh0Jg+GuJAmzPjHV90PemSihqVB2PYIQapZPOqpzzUCG23f3sqUjxttjRY7PVilYHl0pk6HO+pbIi8dmieoqPWkTy/UpWS5QryqULZctHVGuGUjxxqkCmgof3p7j+HSJnrRJX0ahVPOI6e9vbyyuBJyr+bM7ZRLRVI5MFIlo6qoncVh4rKmixchclTAMubo3ydaOGIqinHE53+LHX55ILK9UTBWts/aESGVDCCHWhyQjyyhKvbKRSxokozqjeYua4xPVFYYHMozMVqg5Pn2ZGHde24eiKBw6lednp/MUqw5BEJBLGWzpiJNcGLu+rSuB7fq8O1ViumxzYqpKJl5hqDOGqWtEjQhDOQ1VhWTUIBrRsL2A49NlJoo2cUMjYegM5xL0pKPN/gcvCICQYtXh8ESJ0bkyZQf27ejE0DS25xKoGuzqS/Ppm7fSm47yzkSJiKawrTPOcC7W3E55Y7RQP3FTtDidr3JVb4qtnTGOz1Txg5CS5RLRVFRF5YreJGFYn6yaMCP0EFJ1fFKmzg1D2RWTgXM1fzaOFzd6Rnb1pVAUZcVKxHTJ5oWjM7w3/X415Jeu6m4e1T0+Uzkj4VlLInG53Bkjt/4KITYaSUZWkDTrczbemypj6hqZqIGVqp9YcXyo2j6n5mrMVV26U1HGixYnZqtYts98zaMjaXLdgE4YRuu9IprKxHyZEzMVxuarWF7IyZkiWzpihIAXhCgKlCwPdeFm37F8DT8McT2fXUMdmLraXBwbi+aWbJy3xwr8bLTA4fEyfuBTqHm8eGKWqK6zszuBqqgYmsbp+RqvjuT56XuzKIpCNh7husEMqqryf38xzSsn5hkv1DA0laihUnN8KosHds3Xx8rnkiaJhSbViuNTczwOj5fQlPpNvV3JlRfGcy30qqpyzUDmjK9bKYEoL/SFZGMRFveJABd1G+6F9oR80MmB3PorhNhoJBlZQffC1kXZ8tjWlWQsX+HIZJGfny4SjdQvnSvVvGZfw+nZKv5C/8jIXI3nj0xydKpEJqZjaDr5msPpuSpHJkq4fn3CaESNEAQhpqaRNDSMiMbOniS/dGUXpq7iB7BnS5aqE5CvOvVKy8LiuHjR9IKQIKgPUzM1Ez+AroRBR9yk5nqoispc1cYPA96dKFOseVzRlyRfdZvxl22P7qTJ3EIT6hU9WbqS0SV3wuia2qzMNIRhyCsn5ihbLh0Jk3zFXrKdtFgrL4dbrU+kXaddPujk4HKp4AghxFpJMrKCxgC0Qq1+t0p3KkouYeJ7cGSqzOl5i45Evafi1RNzHJ4ocGq+Rs32UFSVuKnz7uQUPWmTHT0pyjWPk7NlXD9A01Qc3yfEpzcV5Zot9epEYyR7Y6tBUxVqrs/O7gRDnXGGc4nm4rh40dzaGcNyPKZKDmXbozNhcN2WLJbrc2K2ihd4uH6IqWuYukbcDDk1W6M3bTb7MpKmzmTBImPqJCIanXGTjkSkfuuv6Ta3TpZPJJ0u1ZOP47NVfnpinp6UQSKqkzD15s2/jZjDMCQTqw9GS5h6sx/lQqoI3SmTj13R1ex1Wdwn0o7TLh90ciC3/gohNhr5LbaKlaaLThZrVN0AQ1eJGSqHx4u8M17i1JxFzQmouT6GBoT1iatl2+fwWBGVkGLNR1UVVAXius7Nw11s64ozXrDoTBrs7k83302v1my5klzC4P/f08fWzvjC/SoKt2zv5NCpeXZ0J+hKmhyeKGJ7Pr0ZE12JoSghN2/PNfsyGgt7I1GIRrTmZNaxvIUfhCtOJC3b9a2Z3f0pbM9nd38aLwh57eQ83alos0oA8OZoET8IKdsuYQipaOSCqwiKotCbidGbiZ31NfugTrvUkwN4e6xQPyrdGSMMw3XbqpFTPUKIjUaSkVUsf5cchiE3bcuhqhq6qjBXsXh3sowXhFQdF9uvJymhojBfsUku3MhLGKIpCpl4gKopWK7Ph7ZmOfCxYXRdX3FBOdc79JW2BX7tuv7maZCJok3CjJCMRkiYOnu3ZNnaESMZjSyZHdJYLFda2KF+yd3Z3vEnTR1dVVFR6U7WB4d5QYihaWdcjNd4nNdGahDC1X3pllcR2nXapTtlMpCNMVGwMDSN0fnaGYlbK8mpHiHERiPJyCpWakpcfOJDVWCsYDFdtrE8H9uBMAJxQ+GqvlS9Z8PxycQNijWHmKFxTcqgavv8f9f2omnaGYnIatNGl1ttW2DxO+Z4RGW24jBTdhjqjLOrL4W6MBV1rc85YWhn3Q5ofL+S5XLtlhQV2yNfdSnUXEbnq0vul2k8TsLQKFker43MNb/H5a5xAqs7FT3jNZGTL0IIcW6SjKxiqmjxwtGZ5iJy284cc1WXV0/MMl1yqdoOYRhiaAqdcRMrEpCJ1weJ/cpVPVzVn+G/3hxnrFBDVxSyCYP+TIz+TIzBzkRz26JxodxsxVlyk+7eweyq76xX6xlovGPuDkMOjxd5fSSPoWk4XrCmd+rLKy57tqTPuh3QfIeejjJVtBgvFAgAdeE5LO5zaTxOzfF4e6xI1fYoVF1OztbHuF/ui/Rqr4mcfBFCiHOTZGQVI3NV3puukI1FmCxWSJk602WHQ6eKTJYsVMCMqGzJxkmbBm+NFYioMJCN0pWO0RmPEIZgOwG5bIyetMnVfWl296cpWe6SysbJ2Qqvnpzn5GyV3kyUIAg4OVs564CwsyUJ0yWb10fynJ6vNT93tnfqUE++Xjw2y6n5KtcNZLC8gIrjs6M7uabtgMXHjcfyteYU1obGtsKx6TLpmEFPOsZPj89yeKJE0fIv+0V6tddETr4IIcS5STKygjAMma84zFcc9IWJp/XL0xQMXaVYddmai+N6AX4QkIppbM3F8P2QK3vSWK7Pm6NFqk5AIqpzOl8jlzKXNKkubngsVB0mijZ+GHJkooTnhxgRjbmKe0GTQMu2h64qdC2czDEXTUVd6Z06wAtHZ3jjdIGpks10yWbvYPa8Tmms9YRH49+dmCkDq9+Qe7lZ7TWRky9CCHFu8ptxBdMlm6LlEtHg1FyF/myMzoRRP+abNknM6syXHdLxCIZev9Pk2i0dHJ+ucM1AmiBUsFyPuKGSjsbQVZud3UuP5i5ueJwp1wjDkP50/d/2pg0Spn7B76aTpk5u4RhuLKItmYq60jv1xscHMlEyC6dolo9VX8s497Wc8Gj8u0xMJzlXXfWG3I1CTr4IIcS5bcwV4CKVbY9kNMJNw5389PgscVPD8nz6M1H8IKRSs5muOFzbn0FdaF5UqVdNClWXXNLkip4kXhBStj2Gu+JcP5hdMpp8ccPj22MhiqoQN3SGu+rNpuMF+6zvps+WHNQXwOyKn1vtnXp9iJgN1IeIDecSS5KNc/U+rPWER7OvJWUynEts+EVaTr4IIcS5STKygsaR1cmqRTZusmdLFssNGMtb/HysRL4WMJ63MdQKfVmTvmyMlKkz0BGlL22Sjhl0JQ26U9E1XUffmTDYM5hpDgqrf61z1oX6bMnB2RbA1d6przZErKHVvQ+ySAshhGiQZGQFq20lVGwPxw/oy5iMzFVIxVTS0QgjM5XmTI8re5LNpOBsi+25Bput16VuqyUBK80aWVx9sVy/fpxZeh82JDmCLIRoJ1lRFln8CzlhaGzteH9xHuqMM1O2+dnpAkcmyvghlO2AQs2lbPtEdJ3JUoXhXHzFAWLLXWxl4INojFxafYEtHbEzxryvt1YvkhfyeJthoZYjyEKIdpJkZJHFv5CXjy1XFIXd/Wk+sr1GXFeImRGqtkvc0PCDgJLlkq86zFeddR0F3vBBNEYur75EIxo7upMt/z5n0+pF8kIebzMs1HIEWQjRTmcfybnJLP6FXLY9KrZHfybKXNnh8HiRmbLD3sEMnckoo/NVyo5PLKIRN3VmyjYRVeXUbI1XTswxVbQIw/CM7xGGIVNFi2PT5VX/zVo0Kis7upMr3pLbCpfCsdTFr4m/0BD8QT9eq2O4FF0Kr7UQYvOS3ziLLP6FXL8cj/pFePNVQkJcP6Q/Y2J7PgHQkTCImzq5pEkmapCNGxyZKPL2eJFCzVvxHfTl9C77UjiW2upF8kIebzMs1JfCay2E2Lw23m/Vi7DS3S5vjhZIx3R60yYn56pUbJfOhImha0yXbPwAruxNMZa3GJ2vEgKZqM474wUqtstHduSWVC4up3L4pXDipdWL5IU83mZYqC+F11oIsXlJMrLI4rtdfj6a5/tvjDOWr1GouRyZKNGdMvH9kFRUIwhCohGV4Vycq3uTdCVNMjGdIAx5/VSeqZLDdMXG9QOuGXj/2G798rkP7rr5D8J6Nni2epG8kMeThVoIIdaXJCPLhAuXzD3x0givnpwnDMDxQ1w/YHtXgrmKTRAYmLpCJhan5vjMVtzmIC+AuYpDytRRFJW3x4pMlSxyiSiuH3DDUJb+TPQDu25+vYWLLuVbyyV/QgghxHKSjCwzXbJ57eQ8kwULxw0wIipxTcULQn56bJaYoaOpFYY643xkZ5KJfI3D40WA5lTR4VyVN0cLTJWqmDqUHA/HC7HcAEVRuKo3SXcqSn8myjvjpSVffyEVhXYePW38vE7P1+hadimfEEIIsRaSjCxTtj0MTWN7d5JTc1XKtkfK1IkZKumozlV9GV45PssvJktMFCyiEQ0FBdcP2TuYoTtl8vEru4hoCqfmqwxmY/z0+BzjhRp9mRjzFZv5ioECvHRsjhOzZYZrCRwv4PqtF1ZRaGdTbOPn1b1wKV9sYTtKCCGEWCtZNZZJGBphGGA5HumojrLwsYrtoyoq704WURSFvkyUubJLoeYyVapRsjy25WL0pKP0ZmLs29lF/FSeuYpDR9yg6njMVxySUZ2i5dKXiVF2XBRFQVEV5isuJcsFOO8KR6MptlWVlvORNHU6EhEATF1dcimfEEIIsRaSjKxgsmwxMm9RcXzKrk/M1Jkp2LheSDxSrwLEDR1S8ObpKj85NkdnwmDXQIqdPfUtk5LlEjM03GLAUC7OXNkmAPZsyVJz/YXkIUYyGmGmZBPVVSzX5/WRPBXbI2HqfPzKrjVNc20cPV1+DPmDqJB0p0yu37rypXxCCCHEWkgyskzF8XG9kK6kiabA3FiR2bIFqGiaSmJhrkgqpuMXAzrjJnsGs+SrDq7nL2nmdH2fiKaxuz/NS8fmKDsuEwWLXLJ+kd50ycJyPTJxnRuGslRsj2MzFbIx47xGyzeOnh4eLxISsnsgzXjeWrF3o9X9JXLSRAghxMWSZGSZpKnTEY/w1liRUs0lG9fxghDbC5kt2wxkouyMR/jQ1iz5mgvM4Xg+2bhBRNeWNHOGQYhiqrwzXiJfc8jEIrh+wEA2Rmc8AiikzPoFe11Jk6rjL0RxflNZGwkBgOuHjOetVYdzXU5D14QQQmwOkows050yuWV7JzNlm7mKg+V6qCjkLY+R2QoTRYtc3uCagQw7uhLEDR3X84noGq7nYzkBXUmDmZLNYEeMG4ayTJfsJRWLaESj6gakohGu7kszlq9RcXyGOuPs7E5Qtj12JhMMdcbPO/bGcK5670vIsenykgrIpT50baNcSrdRnocQQnwQzvtumueff567776bgYEBFEXh6aefPufX/PjHP+bGG2/ENE2uuOIKHn/88QsI9YOhKApxM8K2XIoretNous58zUNT4KreFEOdCSzP583TeY5OVbDcgN5MDMsNmCo55C0HQoXBjhg3Dnewuz/N7v40uaTJ2HyNkuUyW7axXB9NZcmI8Z50lI9f2d38c74Vi8X31SiKwpujRX4xWeaN0wWmSzZw6Y82b1Rulsd9qVt+59BU0bosn4cQQrTDea9ElUqF66+/ngMHDvDJT37ynP/++PHj3HXXXXz2s5/lu9/9Ls8++yy///u/T39/P3fccccFBb3eEobGbMXi8EQRQoW4oeH4AZqqUXU9ooHKeMFiIBtnsmhRthwsL4AQ/CCkKxVh386u5hj4RsXi5GyFiuMxW3HIV122dMSak1kb75xb1X+xWgXkUh9t3o7KTSuqGMu3vzIx/ZKuQAkhxKXkvJORO++8kzvvvHPN//5b3/oW27dv52tf+xoAu3fv5oUXXuBv/uZvLtlkBMDQNKq2R7Hms3drhu6kgaoo2H7AYEec10fmefHYLB0Jg4LlMDZvMV9zURWF7qTBbMWh4vjNxa0nHaVse8xV3OYCFY1o7OhOnndsa1k8V6uAXOoNpxdaubmYhKIVfTTLkyjgkq5ACSHEpWTdf0P+5Cc/4fbbb1/ysTvuuIP77rtvvb/1BWskEdu7krw9XmS2ZHN1b4prt2QYna8xV3GIaAoxQ+OWbR2cmCmTj2hkYgamrlJxPF47OU9kYXLrDUNZdven17zQnmthXcviealXQFZzoXFfTELRimrM8td2qDPe7NG5nH7+QgjRDuuejExMTNDb27vkY729vRSLRWq1GrHYmUdXbdvGtt/fYy8Wi+sd5hJJU8cNAlRV5cPbO9FVhW1dCXb1pQCYKtn0ZuIUqg5TRYdUzGCwU2Gm4uCFITFVo+YFWF7ATMkmDOtHhde60J5tYQ3DkJOzFUbzVbblEtRcf8XF81KvgKzmQuO+mISiFX00K722iqJcdj9/IYRoh0uydvzQQw/xxS9+sW3fvztlcuNwB4qiNC9/G84lUFWVaESjK2nSn41yeKxIb8ZkV1+KMAw5NV8vz8cNjddH8pyer9GdMjE0rb44pqNrWmjPtrBOl2xOzlaZLNpMFm12didImvqmP71xMQlFK6pIl2vyJ4QQl4J1T0b6+vqYnJxc8rHJyUnS6fSKVRGABx54gPvvv7/592KxyNatW9c1zuVyCYOreuv9HEOd8eYC1Vj0xvMWuaTJ7v50s2rRl60fxQ3DsJkIGJpGRyJyXotj0tRRFTg8VsTxfbZ2xpqP2Vgwb92e48RMuRnbZp8fcjEJhSQSQgjRXuuejOzbt4//+q//WvKxZ555hn379q36NaZpYprt22OfLtm8OVpsLuyKojSTi7UseoqisLs/TVfSvKDFsTtlsqUjxlTJJqKpjOVrdCXrTbBJU0fX6qPjt3TEGc4lVpwfcqH33FyuJKEQQojL13knI+VymaNHjzb/fvz4cQ4dOkRnZydDQ0M88MADjI6O8g//8A8AfPazn+Xhhx/mj//4jzlw4AA/+tGPePLJJ/n+97/fumfRYmfbJlm+6DXmSyxf9C9mcVQUpbkdtDy5KFkuA9kopq6SikbOqNg0tilsL+D4Jq6UCCGEuHycdzLyyiuv8Cu/8ivNvze2U/bv38/jjz/O+Pg4IyMjzc9v376d73//+3z+85/n61//OoODg3znO9+5pI/1rtR/sFpPxrm2R87Wy3G2z51vcrG8YlOyXJlzIYQQ4rKghGF4fhehtEGxWCSTyVAoFEin0+v+/VZKEhYnHapCc2DZbNlmtuywpSPOWL7Glb1JdnQnm49xcrbCyFyVhKmjq+qSJKIxpXO1UzOLYyhZLkenKgxkY4zmq+QSBrmkueoWzNkeWwghhPggrHX9viRP07TbSlssi7du3h4rcHS6RNzQCcKQpBE54xRHI3kZna8yWbK5dXsnlhssqVCcz3YQvD9Eq2J7lK36ALWNNmdECCHE5iPJyBot3jaZKVmcmK/SGTOwPJ+P7sxxZW9yyaLfSDS2dSWZLNmcmK2wJRtfcqrmfI6jLk4uGtWYs23BSEOnEEKIy4UkI2u0OBkoVB3eGi/i+1BzfRSU5lj3RkPrbNmmZLkEQcCOrgTDufrJl8UViq6kwUA2ynTJpitpEATBGbfsNixOLpKmTqHmyahxIYQQG4KsYmu0OBmYKVn0jJtEdQ3L88nG9OaJGsv1GZ2v4YchigJdKZObFpKQ5X0dM2WHsbyFH4S8M1EiDCEVjSzZelmpf0W2YIQQQmwkkoxcgOFcgr2DWUqWSxjCfM1l5N1pkqbObMUhoqrsHkgzlq+RW5gPspLFPSOvjdQghKv70ku2XlY7rSNbMEIIITYKtd0BXI66U/XJqx1xAxQYz1scm6kQM3R0VcHxfUbnq5Qsl9myzVTRYqVDS4t7RpKmTsLUGc1XKdvvf93iI7p+EFK2vTY8YyGEEGL9SGVkjRZvl1iuz1i+Rr7qMl1yuLovxVTZ5s3T82TjBtu6EhiaQsXxmK04FGreOU+8JAwNgJG5KmXLY7Zc/7qBbFSuohdCCLGhycq2Rou3S6ZLFrqqkI0bHJkocmpWpTtpYrk+hqZRc3yMmI7n16shjWbW5cnISideKo7PXMVtnpQxdZU9W9KMzFWBelLUuKdms1+OJ4QQYmOQZGSNFvd3FKous9X6cV03CMlXHXpSUfrSUQY7E82qyen5GsdnKkQ0lT2DmTV9n+XHfVPRCACFWv37F2pF9i4kMZv9cjwhhBAbgyQja7Q4SehIRMgmdN6dDIhGNGpuwGzVRl2URKSjOoMdMULqp2/KlrvkNt/VrHRS5vhMZcXhaGcbmiaEEEJcLiQZWaPlSUJ9nojN6fka3SmTpKkxnIs3R7SHYcjIXI1jMxUATs3X2NZlr+nemuVbN6sNRzufoWlCCCHEpUpWrzVaniQEQcC2rgQzZZswCMklDIZziSV3ywzn4lQcj225BDXXP6Ny0dhm8YKAiu0x1Pn+YLTFFZTV5orIvBEhhBAbgSQjF2i6ZDNRqKGrCvmaQxDGljSXKorCcC5BoeZhuQG6qp5RuWhss8QiGm+cLlC2vBVP3qw22l1GvgshhNgIJBm5QCNzVY7NVNEVheMzFWIRDU3Vms2lcO7KRWOb5cRsBQjJxiOM5qtkYnIyRgghxOYhycgaLO/t6EoazFcc8lUbVVEIQuhORZtDyc528+5ijWQlE9MJFkbCK4pCwqgu2fIRQgghNjJJRtZg+RHagWyUQs0lomkUqg7ZeIQwDM+7ibSRrDQqJm+PFsgmTOYrNidnK1IdEUIIsSlIMrIGy4/QTpdsUtEIv7qrl+PTJQayMXb2JElFIxfURNroLzk5W+XIZAmA1JxUR4QQQmwOkoyswfIjtN0pk7G8heX6DHYmWjJsrDtlnvP0jRBCCLERSTKyBssbUbuSBl1Js6VHatdy+kYIIYTYiGS1W4OVGlHX40itzA0RQgixGUkycgmRuSFCCCE2I7XdAQghhBBic5NkRAghhBBtJcmIEEIIIdpKkhEhhBBCtJUkI0IIIYRoK0lGhBBCCNFWkowIIYQQoq0kGRFCCCFEW0kyIoQQQoi2kmRECCGEEG0lyYgQQggh2kqSESGEEEK01WVxUV4YhgAUi8U2RyKEEEKItWqs2411fDWXRTJSKpUA2Lp1a5sjEUIIIcT5KpVKZDKZVT+vhOdKVy4BQRAwNjZGKpVCUZSWPW6xWGTr1q2cOnWKdDrdsse9VGz05wcb/znK87u8yfO7vMnzu3hhGFIqlRgYGEBVV+8MuSwqI6qqMjg4uG6Pn06nN+R/aA0b/fnBxn+O8vwub/L8Lm/y/C7O2SoiDdLAKoQQQoi2kmRECCGEEG21qZMR0zR58MEHMU2z3aGsi43+/GDjP0d5fpc3eX6XN3l+H5zLooFVCCGEEBvXpq6MCCGEEKL9JBkRQgghRFtJMiKEEEKItpJkRAghhBBttamTkW9+85ts27aNaDTKrbfeyksvvdTukFrm+eef5+6772ZgYABFUXj66afbHVLLPPTQQ3z4wx8mlUrR09PDb/zGb3DkyJF2h9UyjzzyCHv37m0OItq3bx8/+MEP2h3WuvnKV76Coijcd9997Q6lZf7iL/4CRVGW/Nm1a1e7w2qp0dFRfvu3f5tcLkcsFmPPnj288sor7Q6rJbZt23bG66coCgcPHmx3aC3h+z5//ud/zvbt24nFYuzcuZO//Mu/POf9Metp0yYj//RP/8T999/Pgw8+yGuvvcb111/PHXfcwdTUVLtDa4lKpcL111/PN7/5zXaH0nLPPfccBw8e5MUXX+SZZ57BdV1+7dd+jUql0u7QWmJwcJCvfOUrvPrqq7zyyiv86q/+Kr/+67/Oz3/+83aH1nIvv/wy3/72t9m7d2+7Q2m5a6+9lvHx8eafF154od0htcz8/Dy33XYbkUiEH/zgB7z99tt87Wtfo6Ojo92htcTLL7+85LV75plnAPjUpz7V5sha46tf/SqPPPIIDz/8MIcPH+arX/0qf/VXf8U3vvGN9gUVblK33HJLePDgwebffd8PBwYGwoceeqiNUa0PIHzqqafaHca6mZqaCoHwueeea3co66ajoyP8zne+0+4wWqpUKoVXXnll+Mwzz4S//Mu/HN57773tDqllHnzwwfD6669vdxjr5k/+5E/Cj33sY+0O4wNz7733hjt37gyDIGh3KC1x1113hQcOHFjysU9+8pPhPffc06aIwnBTVkYcx+HVV1/l9ttvb35MVVVuv/12fvKTn7QxMnEhCoUCAJ2dnW2OpPV83+d73/selUqFffv2tTucljp48CB33XXXkv8PN5Jf/OIXDAwMsGPHDu655x5GRkbaHVLL/Pu//zs333wzn/rUp+jp6eGGG27g7/7u79od1rpwHId//Md/5MCBAy29qLWdPvrRj/Lss8/y7rvvAvCzn/2MF154gTvvvLNtMV0WF+W12szMDL7v09vbu+Tjvb29vPPOO22KSlyIIAi47777uO2227juuuvaHU7LvPnmm+zbtw/Lskgmkzz11FNcc8017Q6rZb73ve/x2muv8fLLL7c7lHVx66238vjjj3P11VczPj7OF7/4RT7+8Y/z1ltvkUql2h3eRTt27BiPPPII999/P3/6p3/Kyy+/zB/8wR9gGAb79+9vd3gt9fTTT5PP5/md3/mddofSMl/4whcoFovs2rULTdPwfZ8vfelL3HPPPW2LaVMmI2LjOHjwIG+99daG2o8HuPrqqzl06BCFQoF//ud/Zv/+/Tz33HMbIiE5deoU9957L8888wzRaLTd4ayLxe8w9+7dy6233srw8DBPPvkkv/d7v9fGyFojCAJuvvlmvvzlLwNwww038NZbb/Gtb31rwyUjf//3f8+dd97JwMBAu0NpmSeffJLvfve7PPHEE1x77bUcOnSI++67j4GBgba9fpsyGenq6kLTNCYnJ5d8fHJykr6+vjZFJc7X5z73Of7zP/+T559/nsHBwXaH01KGYXDFFVcAcNNNN/Hyyy/z9a9/nW9/+9ttjuzivfrqq0xNTXHjjTc2P+b7Ps8//zwPP/wwtm2jaVobI2y9bDbLVVddxdGjR9sdSkv09/efkRjv3r2bf/mXf2lTROvj5MmT/M///A//+q//2u5QWuqP/uiP+MIXvsBv/uZvArBnzx5OnjzJQw891LZkZFP2jBiGwU033cSzzz7b/FgQBDz77LMbbl9+IwrDkM997nM89dRT/OhHP2L79u3tDmndBUGAbdvtDqMlPvGJT/Dmm29y6NCh5p+bb76Ze+65h0OHDm24RASgXC7z3nvv0d/f3+5QWuK222474zj9u+++y/DwcJsiWh+PPfYYPT093HXXXe0OpaWq1SqqunT51zSNIAjaFNEmrYwA3H///ezfv5+bb76ZW265hb/927+lUqnwu7/7u+0OrSXK5fKSd2HHjx/n0KFDdHZ2MjQ01MbILt7Bgwd54okn+Ld/+zdSqRQTExMAZDIZYrFYm6O7eA888AB33nknQ0NDlEolnnjiCX784x/zwx/+sN2htUQqlTqjvyeRSJDL5TZM388f/uEfcvfddzM8PMzY2BgPPvggmqbxW7/1W+0OrSU+//nP89GPfpQvf/nLfPrTn+all17i0Ucf5dFHH213aC0TBAGPPfYY+/fvR9c31lJ5991386UvfYmhoSGuvfZaXn/9df76r/+aAwcOtC+otp3juQR84xvfCIeGhkLDMMJbbrklfPHFF9sdUsv87//+bwic8Wf//v3tDu2irfS8gPCxxx5rd2gtceDAgXB4eDg0DCPs7u4OP/GJT4T//d//3e6w1tVGO9r7mc98Juzv7w8Nwwi3bNkSfuYznwmPHj3a7rBa6j/+4z/C6667LjRNM9y1a1f46KOPtjuklvrhD38YAuGRI0faHUrLFYvF8N577w2HhobCaDQa7tixI/yzP/uz0LbttsWkhGEbR64JIYQQYtPblD0jQgghhLh0SDIihBBCiLaSZEQIIYQQbSXJiBBCCCHaSpIRIYQQQrSVJCNCCCGEaCtJRoQQQgjRVpKMCCGEEKKtJBkRQgghRFtJMiKEEEKItpJkRAghhBBtJcmIEEIIIdrq/wHRnL6tjOlowAAAAABJRU5ErkJggg==", - "text/plain": [ - "
    " - ] - }, - "metadata": {}, - "output_type": "display_data" + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.6" } - ], - "source": [ - "# similarly, for bruise\n", - "sns.regplot(\n", - " x=\"new_cases_percent_of_pop\",\n", - " y=\"search_trends_bruise\",\n", - " data=weekly_data,\n", - " scatter_kws={'alpha': 0.2, \"s\" :5}\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Hd2A8707Uhz2" - }, - "source": [ - "We see that the slope of the line is positive in the graphs for cough and fever, but flat for bruise. That means that in places with increasing new cases of COVID-19, we saw increasing searches for cough and fever, but we didn't see increasing searches for unrelated symptoms like bruises. Interesting!" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Recap" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We used matplotlib to draw a line graph of COVID-19 cases over time in the USA. Then, we used downsampling to download only a portion of the available data, used seaborn to plot lines of best fit to observe corellation between COVID-19 cases and searches for related versus unrelated symptoms.\n", - "\n", - "Thank you for using BigQuery DataFrames!" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/notebooks/visualization/tutorial.ipynb b/notebooks/visualization/tutorial.ipynb index 89a5ed87b8f..ab838a89f0b 100644 --- a/notebooks/visualization/tutorial.ipynb +++ b/notebooks/visualization/tutorial.ipynb @@ -27,7 +27,7 @@ "id": "e661697d", "metadata": {}, "source": [ - "# BigQuery DataFrame Visualization Tutorials", + "## BigQuery DataFrame Visualization Tutorials\n", "\n", "\n", "\n", diff --git a/noxfile.py b/noxfile.py index 839e6e0e115..7626a49adfe 100644 --- a/noxfile.py +++ b/noxfile.py @@ -28,14 +28,14 @@ import nox import nox.sessions -PROJECT_ID_OVERRIDE = os.getenv("BIGFRAMES_TEST_PROJECT") -ENV_OVERRIDES = ( - {"GOOGLE_CLOUD_PROJECT": PROJECT_ID_OVERRIDE} if PROJECT_ID_OVERRIDE else {} -) - -RUFF_VERSION = "ruff==0.14.14" +BLACK_VERSION = "black==22.3.0" +FLAKE8_VERSION = "flake8==7.1.2" +ISORT_VERSION = "isort==5.12.0" MYPY_VERSION = "mypy==1.15.0" +# TODO: switch to 3.13 once remote functions / cloud run adds a runtime for it (internal issue 333742751) +LATEST_FULLY_SUPPORTED_PYTHON = "3.12" + # Notebook tests should match colab and BQ Studio. # Check with import sys; sys.version_info # on a fresh notebook runtime. @@ -58,17 +58,23 @@ "setup.py", ] -DEFAULT_PYTHON_VERSION = "3.14" +DEFAULT_PYTHON_VERSION = "3.10" -ALL_PYTHON = ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] +# Cloud Run Functions supports Python versions up to 3.12 +# https://cloud.google.com/run/docs/runtimes/python +E2E_TEST_PYTHON_VERSION = "3.12" + +UNIT_TEST_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", + "asyncmock", PYTEST_VERSION, + "pytest-asyncio", "pytest-cov", + "pytest-mock", "pytest-timeout", - "pluggy", ] -UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] +UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = [] UNIT_TEST_DEPENDENCIES: List[str] = [] UNIT_TEST_EXTRAS: List[str] = ["tests"] UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = { @@ -77,14 +83,13 @@ # Make sure we leave some versions without "extras" so we know those # dependencies are actually optional. "3.13": ["tests", "polars", "scikit-learn", "anywidget"], - "3.14": ["tests", "polars", "scikit-learn", "anywidget"], } # 3.11 is used by colab. # 3.10 is needed for Windows tests as it is the only version installed in the # bigframes-windows container image. For more information, search # bigframes/windows-docker, internally. -SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ALL_PYTHON +SYSTEM_TEST_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] SYSTEM_TEST_STANDARD_DEPENDENCIES = [ "jinja2", "mock", @@ -101,14 +106,15 @@ SYSTEM_TEST_EXTERNAL_DEPENDENCIES = [ "google-cloud-bigquery", ] -SYSTEM_TEST_EXTRAS: List[str] = [] +SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [] +SYSTEM_TEST_DEPENDENCIES: List[str] = [] +SYSTEM_TEST_EXTRAS: List[str] = ["tests"] SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = { # Make sure we leave some versions without "extras" so we know those # dependencies are actually optional. "3.10": ["tests", "scikit-learn", "anywidget"], - "3.12": ["tests", "scikit-learn", "polars", "anywidget"], + LATEST_FULLY_SUPPORTED_PYTHON: ["tests", "scikit-learn", "polars", "anywidget"], "3.13": ["tests", "polars", "anywidget"], - "3.14": ["tests", "polars", "anywidget"], } LOGGING_NAME_ENV_VAR = "BIGFRAMES_PERFORMANCE_LOG_NAME" @@ -123,7 +129,8 @@ # TODO(tswast): Consider removing this when unit_noextras and cover is run # from GitHub actions. "unit_noextras", - "system-3.12", # No extras. + "system-3.9", # No extras. + f"system-{LATEST_FULLY_SUPPORTED_PYTHON}", # All extras. "cover", # TODO(b/401609005): remove "cleanup", @@ -140,46 +147,26 @@ def lint(session): Returns a failure if the linters find linting errors or sufficiently serious code quality issues. """ - session.install(RUFF_VERSION) - - # Check imports + session.install(FLAKE8_VERSION, BLACK_VERSION, ISORT_VERSION) session.run( - "ruff", - "check", - "--select", - "I,F", - f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", - "--line-length=88", # Standard Black line length + "isort", + "--check", *LINT_PATHS, ) - - # Check formatting session.run( - "ruff", - "format", + "black", "--check", - f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", - "--line-length=88", *LINT_PATHS, ) + session.run("flake8", *LINT_PATHS) -# Use a python runtime which is available in the owlbot post processor here -# https://github.com/googleapis/synthtool/blob/master/docker/owlbot/python/Dockerfile @nox.session(python=DEFAULT_PYTHON_VERSION) def blacken(session): - """(Deprecated) Legacy session. Please use 'nox -s format'.""" - session.log( - "WARNING: The 'blacken' session is deprecated and will be removed in a future release. Please use 'nox -s format' in the future." - ) - - # Just run the ruff formatter (keeping legacy behavior of only formatting, not sorting imports) - session.install(RUFF_VERSION) + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) session.run( - "ruff", - "format", - f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", - "--line-length=88", + "black", *LINT_PATHS, ) @@ -187,31 +174,18 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) def format(session): """ - Run ruff to sort imports and format code. + Run isort to sort imports. Then run black + to format code to uniform standard. """ - # 1. Install ruff (skipped automatically if you run with --no-venv) - session.install(RUFF_VERSION) - - # 2. Run Ruff to fix imports - # check --select I: Enables strict import sorting - # --fix: Applies the changes automatically + session.install(BLACK_VERSION, ISORT_VERSION) + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections session.run( - "ruff", - "check", - "--select", - "I,F", - "--fix", - f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", - "--line-length=88", # Standard Black line length + "isort", *LINT_PATHS, ) - - # 3. Run Ruff to format code session.run( - "ruff", - "format", - f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", - "--line-length=88", # Standard Black line length + "black", *LINT_PATHS, ) @@ -232,20 +206,20 @@ def lint_setup_py(session): def install_unittest_dependencies(session, install_test_extra, *constraints): - extras = [] + standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES + session.install(*standard_deps, *constraints) + + if UNIT_TEST_LOCAL_DEPENDENCIES: + session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints) + if install_test_extra: if session.python in UNIT_TEST_EXTRAS_BY_PYTHON: extras = UNIT_TEST_EXTRAS_BY_PYTHON[session.python] else: extras = UNIT_TEST_EXTRAS - - session.install( - *UNIT_TEST_STANDARD_DEPENDENCIES, - *UNIT_TEST_DEPENDENCIES, - "-e", - f".[{','.join(extras)}]" if extras else ".", - *constraints, - ) + session.install("-e", f".[{','.join(extras)}]", *constraints) + else: + session.install("-e", ".", *constraints) def run_unit(session, install_test_extra): @@ -280,25 +254,76 @@ def run_unit(session, install_test_extra): ) -@nox.session(python=ALL_PYTHON) -@nox.parametrize("test_extra", [True, False]) -def unit(session, test_extra): - if session.python == "3.15": - session.skip( - "Skipping 3.15 until wheels are available for pyarrow. Also pyproj wheels are needed for dependency geopandas." - ) - if test_extra: - run_unit(session, install_test_extra=test_extra) - else: - unit_noextras(session) +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +def unit(session): + run_unit(session, install_test_extra=True) -@nox.session(python=ALL_PYTHON[-1]) +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1]) def unit_noextras(session): run_unit(session, install_test_extra=False) +@nox.session(python=DEFAULT_PYTHON_VERSION) +def mypy(session): + """Run type checks with mypy.""" + # Editable mode is not compatible with mypy when there are multiple + # package directories. See: + # https://github.com/python/mypy/issues/10564#issuecomment-851687749 + session.install(".") + + # Just install the dependencies' type info directly, since "mypy --install-types" + # might require an additional pass. + deps = ( + set( + [ + MYPY_VERSION, + # TODO: update to latest pandas-stubs once we resolve bigframes issues. + "pandas-stubs<=2.2.3.241126", + "types-protobuf", + "types-python-dateutil", + "types-requests", + "types-setuptools", + "types-tabulate", + "types-PyYAML", + "polars", + "anywidget", + ] + ) + | set(SYSTEM_TEST_STANDARD_DEPENDENCIES) + | set(UNIT_TEST_STANDARD_DEPENDENCIES) + ) + + session.install(*deps) + shutil.rmtree(".mypy_cache", ignore_errors=True) + session.run( + "mypy", + "bigframes", + os.path.join("tests", "system"), + os.path.join("tests", "unit"), + "--check-untyped-defs", + "--explicit-package-bases", + '--exclude="^third_party"', + ) + + def install_systemtest_dependencies(session, install_test_extra, *constraints): + # Use pre-release gRPC for system tests. + # Exclude version 1.49.0rc1 which has a known issue. + # See https://github.com/grpc/grpc/pull/30642 + session.install("--pre", "grpcio!=1.49.0rc1") + + session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_EXTERNAL_DEPENDENCIES: + session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_LOCAL_DEPENDENCIES: + session.install("-e", *SYSTEM_TEST_LOCAL_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_DEPENDENCIES: + session.install("-e", *SYSTEM_TEST_DEPENDENCIES, *constraints) + if install_test_extra and SYSTEM_TEST_EXTRAS_BY_PYTHON: extras = SYSTEM_TEST_EXTRAS_BY_PYTHON.get(session.python, []) elif install_test_extra and SYSTEM_TEST_EXTRAS: @@ -306,19 +331,10 @@ def install_systemtest_dependencies(session, install_test_extra, *constraints): else: extras = [] - # Use pre-release gRPC for system tests. - # Exclude version 1.49.0rc1 which has a known issue. - # See https://github.com/grpc/grpc/pull/30642 - - session.install( - "--pre", - "grpcio!=1.49.0rc1", - *SYSTEM_TEST_STANDARD_DEPENDENCIES, - *SYSTEM_TEST_EXTERNAL_DEPENDENCIES, - "-e", - f".[{','.join(extras)}]" if extras else ".", - *constraints, - ) + if extras: + session.install("-e", f".[{','.join(extras)}]", *constraints) + else: + session.install("-e", ".", *constraints) def run_system( @@ -355,7 +371,6 @@ def run_system( "py.test", "-v", f"-n={num_workers}", - "--dist=worksteal", # Any individual test taking longer than 15 mins will be terminated. f"--timeout={timeout_seconds}", # Log 20 slowest tests @@ -381,10 +396,14 @@ def run_system( ) pytest_cmd.extend(extra_pytest_options) - session.run(*pytest_cmd, *session.posargs, test_folder, env=ENV_OVERRIDES) + session.run( + *pytest_cmd, + *session.posargs, + test_folder, + ) -@nox.session(python="3.12") +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session: nox.sessions.Session): """Run the system test suite.""" run_system( @@ -395,7 +414,7 @@ def system(session: nox.sessions.Session): ) -@nox.session(python=DEFAULT_PYTHON_VERSION) +@nox.session(python=LATEST_FULLY_SUPPORTED_PYTHON) def system_noextras(session: nox.sessions.Session): """Run the system test suite.""" run_system( @@ -406,10 +425,9 @@ def system_noextras(session: nox.sessions.Session): ) -@nox.session(python="3.12") +@nox.session(python=LATEST_FULLY_SUPPORTED_PYTHON) def doctest(session: nox.sessions.Session): """Run the system test suite.""" - run_system( session=session, prefix_name="doctest", @@ -426,18 +444,6 @@ def doctest(session: nox.sessions.Session): "bigframes/testing", "--ignore", "bigframes/display/anywidget.py", - "--ignore", - "bigframes/bigquery/_operations/ai.py", - "--ignore", - "bigframes/bigquery/ai.py", - "--ignore", - "bigframes/ml", - "--ignore", - "bigframes/operations/ai.py", - "--ignore", - "bigframes/operations/semantics.py", - "--ignore", - "third_party/bigframes_vendored/sklearn", ), test_folder="bigframes", check_cov=True, @@ -445,7 +451,7 @@ def doctest(session: nox.sessions.Session): ) -@nox.session(python=DEFAULT_PYTHON_VERSION) +@nox.session(python=E2E_TEST_PYTHON_VERSION) def e2e(session: nox.sessions.Session): """Run the large tests in system test suite.""" run_system( @@ -475,16 +481,13 @@ def cover(session): This outputs the coverage report aggregating coverage from the test runs (including system test runs), and then erases coverage data. """ - # TODO: Remove this skip when the issue is resolved. - # https://github.com/googleapis/google-cloud-python/issues/16635 - session.skip("Temporarily skip coverage session") - session.install("coverage", "pytest-cov") # Create a coverage report that includes only the product code. omitted_paths = [ # non-prod, unit tested "bigframes/core/compile/polars/*", + "bigframes/core/compile/sqlglot/*", # untested "bigframes/streaming/*", # utils @@ -507,27 +510,25 @@ def cover(session): "report", "--show-missing", "--include=tests/system/small/*", - # Some tests only run under old pandas, some only under new pandas version - "--fail-under=98", + # TODO(b/353775058) resume coverage to 100 when the issue is fixed. + "--fail-under=99", ) session.run("coverage", "erase") -@nox.session(python="3.10") +@nox.session(python="3.13") def docs(session): """Build the docs for this library.""" session.install("-e", ".[scikit-learn]") session.install( - "sphinx", - "sphinx-sitemap", - "myst-parser", - "myst-nb", - "pydata-sphinx-theme", + "sphinx==8.2.3", + "sphinx-sitemap==2.9.0", + "myst-parser==4.0.1", + "pydata-sphinx-theme==0.16.1", ) shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) - session.run("python", "-m", "pip", "freeze") session.run( "python", @@ -548,7 +549,7 @@ def docs(session): ) -@nox.session(python="3.10") +@nox.session(python=DEFAULT_PYTHON_VERSION) def docfx(session): """Build the docfx yaml files for this library.""" @@ -558,7 +559,6 @@ def docfx(session): "sphinx-sitemap==2.9.0", "pydata-sphinx-theme==0.13.3", "myst-parser==0.18.1", - "myst-nb", "gcp-sphinx-docfx-yaml==3.2.4", "anywidget", ) @@ -599,36 +599,101 @@ def prerelease(session: nox.sessions.Session, tests_path, extra_pytest_options=( constraints_path = str( CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) - session.install( - *set(UNIT_TEST_STANDARD_DEPENDENCIES + SYSTEM_TEST_STANDARD_DEPENDENCIES), - "-c", - constraints_path, - "-e", - ".", - ) + + # Ignore officially released versions of certain packages specified in + # testing/constraints-*.txt and install a more recent, pre-release versions + # directly + already_installed = set() # PyArrow prerelease packages are published to an alternative PyPI host. # https://arrow.apache.org/docs/python/install.html#installing-nightly-packages session.install( - "--no-deps", - "--upgrade", "--extra-index-url", "https://pypi.fury.io/arrow-nightlies/", + "--prefer-binary", + "--pre", + "--upgrade", "pyarrow", + ) + already_installed.add("pyarrow") + + session.install( + "--prefer-binary", + "--pre", + "--upgrade", # We exclude each version individually so that we can continue to test # some prerelease packages. See: - # https://github.com/googleapis/google-cloud-python/pull/268#discussion_r1423205172 + # https://github.com/googleapis/python-bigquery-dataframes/pull/268#discussion_r1423205172 # "pandas!=2.1.4, !=2.2.0rc0, !=2.2.0, !=2.2.1", "pandas", - # Workaround https://github.com/googleapis/python-db-dtypes-pandas/issues/178 - "db-dtypes", - # Ensure we catch breaking changes in the client libraries early. - "git+https://github.com/googleapis/google-cloud-python.git#egg=google-cloud-bigquery&subdirectory=packages/google-cloud-bigquery", + ) + already_installed.add("pandas") + + # Try to avoid a cap on our SQLGlot so that bigframes + # can be integrated with SQLMesh. See: + # https://github.com/googleapis/python-bigquery-dataframes/issues/942 + # If SQLGlot introduces something that breaks us, lets file an issue + # upstream and/or make sure we fix bigframes to work with it. + session.install( + "--upgrade", + "git+https://github.com/tobymao/sqlglot.git#egg=sqlglot", + ) + already_installed.add("sqlglot") + + # Workaround https://github.com/googleapis/python-db-dtypes-pandas/issues/178 + session.install("--no-deps", "db-dtypes") + already_installed.add("db-dtypes") + + # Ensure we catch breaking changes in the client libraries early. + session.install( + "--upgrade", + "git+https://github.com/googleapis/python-bigquery.git#egg=google-cloud-bigquery", + ) + already_installed.add("google-cloud-bigquery") + session.install( "--upgrade", "-e", "git+https://github.com/googleapis/google-cloud-python.git#egg=google-cloud-bigquery-storage&subdirectory=packages/google-cloud-bigquery-storage", - "git+https://github.com/googleapis/google-cloud-python.git#egg=pandas-gbq&subdirectory=packages/pandas-gbq", ) + already_installed.add("google-cloud-bigquery-storage") + session.install( + "--upgrade", + "git+https://github.com/googleapis/python-bigquery-pandas.git#egg=pandas-gbq", + ) + already_installed.add("pandas-gbq") + + session.install( + *set(UNIT_TEST_STANDARD_DEPENDENCIES + SYSTEM_TEST_STANDARD_DEPENDENCIES), + "-c", + constraints_path, + ) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + if match.group(1) not in already_installed + ] + + print(already_installed) + + # We use --no-deps to ensure that pre-release versions aren't overwritten + # by the version ranges in setup.py. + session.install(*deps) + session.install("--no-deps", "-e", ".") # Print out prerelease package versions. session.run("python", "-m", "pip", "freeze") @@ -650,11 +715,10 @@ def prerelease(session: nox.sessions.Session, tests_path, extra_pytest_options=( tests_path, *extra_pytest_options, *session.posargs, - env=ENV_OVERRIDES, ) -@nox.session(python=ALL_PYTHON[-1]) +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1]) def unit_prerelease(session: nox.sessions.Session): """Run the unit test suite with prerelease dependencies.""" prerelease(session, os.path.join("tests", "unit")) @@ -685,7 +749,7 @@ def system_prerelease(session: nox.sessions.Session): @nox.session(python=COLAB_AND_BQ_STUDIO_PYTHON_VERSIONS) def notebook(session: nox.Session): - google_cloud_project = PROJECT_ID_OVERRIDE or os.getenv("GOOGLE_CLOUD_PROJECT") + google_cloud_project = os.getenv("GOOGLE_CLOUD_PROJECT") if not google_cloud_project: session.error( "Set GOOGLE_CLOUD_PROJECT environment variable to run notebook session." @@ -725,10 +789,11 @@ def notebook(session: nox.Session): # bq_dataframes_llm_code_generation creates a bucket in the sample. "notebooks/generative_ai/bq_dataframes_llm_code_generation.ipynb", # Needs BUCKET_URI. "notebooks/generative_ai/sentiment_analysis.ipynb", # Too slow + "notebooks/generative_ai/bq_dataframes_llm_gemini_2.ipynb", # Gemini 2.0 backend hasn't ready in prod. "notebooks/generative_ai/bq_dataframes_llm_vector_search.ipynb", # Limited quota for vector index ddl statements on table. "notebooks/generative_ai/bq_dataframes_ml_drug_name_generation.ipynb", # Needs CONNECTION. - "notebooks/generative_ai/ai_movie_poster.ipynb", # Needs CONNECTION. # TODO(b/366290533): to protect BQML quota + "notebooks/generative_ai/bq_dataframes_llm_claude3_museum_art.ipynb", "notebooks/vertex_sdk/sdk2_bigframes_pytorch.ipynb", # Needs BUCKET_URI. "notebooks/vertex_sdk/sdk2_bigframes_sklearn.ipynb", # Needs BUCKET_URI. "notebooks/vertex_sdk/sdk2_bigframes_tensorflow.ipynb", # Needs BUCKET_URI. @@ -744,10 +809,22 @@ def notebook(session: nox.Session): # This anywidget notebook uses deferred execution, so it won't # produce metrics for the performance benchmark script. "notebooks/dataframes/anywidget_mode.ipynb", - # Needs a connection - "notebooks/remote_functions/remote_function_vertex_claude_model.ipynb", ] + # TODO: remove exception for Python 3.13 cloud run adds a runtime for it (internal issue 333742751) + # TODO: remove exception for Python 3.13 if nbmake adds support for + # sys.exit(0) or pytest.skip(...). + # See: https://github.com/treebeardtech/nbmake/issues/134 + if session.python == "3.13": + denylist.extend( + [ + "notebooks/getting_started/getting_started_bq_dataframes.ipynb", + "notebooks/remote_functions/remote_function_usecases.ipynb", + "notebooks/remote_functions/remote_function_vertex_claude_model.ipynb", + "notebooks/remote_functions/remote_function.ipynb", + ] + ) + # Convert each Path notebook object to a string using a list comprehension, # and remove tests that we choose not to test. notebooks = [str(nb) for nb in notebooks_list] @@ -757,7 +834,11 @@ def notebook(session: nox.Session): notebooks_reg = { "regionalized.ipynb": [ "asia-southeast1", + "eu", + "europe-west4", + "southamerica-west1", "us", + "us-central1", ] } notebooks_reg = { @@ -784,7 +865,6 @@ def notebook(session: nox.Session): "python", CURRENT_DIRECTORY / "scripts" / "notebooks_fill_params.py", *notebooks, - env=ENV_OVERRIDES, ) processes = [] @@ -797,7 +877,8 @@ def notebook(session: nox.Session): ) if multi_process_mode: process = multiprocessing.Process( - target=session.run, args=args, kwargs={"env": ENV_OVERRIDES} + target=session.run, + args=args, ) process.start() processes.append(process) @@ -805,7 +886,7 @@ def notebook(session: nox.Session): # process to avoid potential race conditions。 time.sleep(1) else: - session.run(*args, env=ENV_OVERRIDES) + session.run(*args) for notebook, regions in notebooks_reg.items(): for region in regions: @@ -820,7 +901,6 @@ def notebook(session: nox.Session): process = multiprocessing.Process( target=session.run, args=region_args, - kwargs={"env": ENV_OVERRIDES}, ) process.start() processes.append(process) @@ -828,7 +908,7 @@ def notebook(session: nox.Session): # process to avoid potential race conditions。 time.sleep(1) else: - session.run(*region_args, env=ENV_OVERRIDES) + session.run(*region_args) for process in processes: process.join() @@ -845,7 +925,6 @@ def notebook(session: nox.Session): "scripts/run_and_publish_benchmark.py", "--notebook", "--publish-benchmarks=notebooks/", - env=ENV_OVERRIDES, ) @@ -909,7 +988,6 @@ def benchmark(session: nox.Session): "scripts/run_and_publish_benchmark.py", f"--benchmark-path={benchmark}", f"--iterations={args.iterations}", - env=ENV_OVERRIDES, ) finally: session.run( @@ -918,11 +996,10 @@ def benchmark(session: nox.Session): f"--publish-benchmarks={base_path}", f"--iterations={args.iterations}", f"--output-csv={args.output_csv}", - env=ENV_OVERRIDES, ) -@nox.session(python=DEFAULT_PYTHON_VERSION) +@nox.session(python="3.10") def release_dry_run(session): env = {} @@ -939,7 +1016,7 @@ def release_dry_run(session): @nox.session(python=DEFAULT_PYTHON_VERSION) def cleanup(session): """Clean up stale and/or temporary resources in the test project.""" - google_cloud_project = PROJECT_ID_OVERRIDE or os.getenv("GOOGLE_CLOUD_PROJECT") + google_cloud_project = os.getenv("GOOGLE_CLOUD_PROJECT") cleanup_options = [] if google_cloud_project: cleanup_options.append(f"--project-id={google_cloud_project}") @@ -961,132 +1038,3 @@ def cleanup(session): session.install("-e", ".") session.run("python", "scripts/manage_cloud_functions.py", *cleanup_options) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -@nox.parametrize( - "protobuf_implementation", - ["python", "upb"], -) -def core_deps_from_source(session, protobuf_implementation): - """Run all tests with core dependencies installed from source - rather than pulling the dependencies from PyPI. - """ - - # Install all dependencies - session.install("-e", ".") - - # Install dependencies for the unit test environment - unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES - session.install(*unit_deps_all) - - # Install dependencies for the system test environment - system_deps_all = ( - SYSTEM_TEST_STANDARD_DEPENDENCIES - + SYSTEM_TEST_EXTERNAL_DEPENDENCIES - + SYSTEM_TEST_EXTRAS - ) - session.install(*system_deps_all) - - # Because we test minimum dependency versions on the minimum Python - # version, the first version we test with in the unit tests sessions has a - # constraints file containing all dependencies and extras. - with open( - CURRENT_DIRECTORY / "testing" / "constraints-3.10.txt", - encoding="utf-8", - ) as constraints_file: - constraints_text = constraints_file.read() - - # Ignore leading whitespace and comment lines. - # Fiona fails to build on GitHub CI because gdal-config is missing and no Python 3.14 wheels are available. - constraints_deps = [ - match.group(1) - for match in re.finditer( - r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE - ) - if match.group(1) != "fiona" - ] - - # Install dependencies specified in `testing/constraints-X.txt`. - session.install(*constraints_deps) - - # TODO(https://github.com/googleapis/gapic-generator-python/issues/2358): `grpcio` and - # `grpcio-status` should be added to the list below so that they are installed from source, - # rather than PyPI. - # TODO(https://github.com/googleapis/gapic-generator-python/issues/2357): `protobuf` should be - # added to the list below so that it is installed from source, rather than PyPI - # Note: If a dependency is added to the `core_dependencies_from_source` list, - # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. - core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", - ] - - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") - - session.run( - "py.test", - "tests/unit", - env={ - "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, - }, - ) - - -@nox.session(python=ALL_PYTHON[-1]) -def prerelease_deps(session): - """Run all tests with prerelease versions of dependencies installed.""" - # TODO(https://github.com/googleapis/google-cloud-python/issues/16014): - # Add prerelease deps tests - unit_prerelease(session) - system_prerelease(session) - - -# NOTE: this is based on mypy session that came directly from the bigframes split repo -# the split repo used 3.10, the monorepo uses 3.14 -@nox.session(python="3.14") -def mypy(session): - """Run type checks with mypy.""" - # Editable mode is not compatible with mypy when there are multiple - # package directories. See: - # https://github.com/python/mypy/issues/10564#issuecomment-851687749 - session.install("--no-cache-dir", ".") - - # Just install the dependencies' type info directly, since "mypy --install-types" - # might require an additional pass. - deps = ( - set( - [ - MYPY_VERSION, - # TODO: update to latest pandas-stubs once we resolve bigframes issues. - "pandas-stubs<=2.2.3.241126", - "types-protobuf", - "types-python-dateutil", - "types-requests", - "types-setuptools", - "types-tabulate", - "types-PyYAML", - "polars", - "anywidget", - ] - ) - | set(SYSTEM_TEST_STANDARD_DEPENDENCIES) - | set(UNIT_TEST_STANDARD_DEPENDENCIES) - ) - - session.install(*deps) - shutil.rmtree(".mypy_cache", ignore_errors=True) - session.run( - "mypy", - "bigframes", - os.path.join("tests", "system"), - os.path.join("tests", "unit"), - "--check-untyped-defs", - "--explicit-package-bases", - '--exclude="^third_party"', - ) diff --git a/pyproject.toml b/pyproject.toml index e7d9c326a93..fed528d4a7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,3 @@ [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" - -[tool.ruff.lint.isort] -known-first-party = ["bigframes"] diff --git a/release-procedure.md b/release-procedure.md deleted file mode 100644 index aeb87862fe6..00000000000 --- a/release-procedure.md +++ /dev/null @@ -1,51 +0,0 @@ -# BigQuery DataFrames (bigframes) release procedure - -*(Note: bigframes releases are marked with `skip_release: true` in `librarian.yaml` and must be kicked off manually using legacylibrarian.)* - -## Setup (First Time Only) - -* Install `legacylibrarian`: - - go install github.com/googleapis/librarian/cmd/legacylibrarian@latest - -* Authenticate with GitHub CLI: - - gh auth login - -## Release Steps - -* Obtain GitHub token: - - export LIBRARIAN_GITHUB_TOKEN=$(gh auth token) - -* Stash changes (repo must be clean): - - git stash -u - -* Fetch and checkout base: - - git fetch origin main - git fetch origin --tags - git checkout origin/main - -* Check image updates: - - legacylibrarian update-image --push - -* Create release PR: - - # Option A: Push directly - legacylibrarian release stage --repo=https://github.com/googleapis/google-cloud-python --library=bigframes --library-version=X.X.X --push - - # Option B: Manual edit first (omit --push, edit files in /tmp/librarian-*, commit/push from there) - legacylibrarian release stage --repo=https://github.com/googleapis/google-cloud-python --library=bigframes --library-version=X.X.X - # In /tmp repository: - git commit -a -m "chore: create release" --no-verify # keep librarian config pristine - git push origin HEAD - gh pr create --fill --label "release:pending" - -* Post-release restore: - - # Move back any stashed/relocated files (like .vscode) - git checkout main - git stash pop diff --git a/renovate.json b/renovate.json new file mode 100644 index 00000000000..c7875c469bd --- /dev/null +++ b/renovate.json @@ -0,0 +1,12 @@ +{ + "extends": [ + "config:base", + "group:all", + ":preserveSemverRanges", + ":disableDependencyDashboard" + ], + "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py", ".github/workflows/unittest.yml"], + "pip_requirements": { + "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] + } +} diff --git a/samples/polars/noxfile.py b/samples/polars/noxfile.py index 63e742993f9..494639d2fa5 100644 --- a/samples/polars/noxfile.py +++ b/samples/polars/noxfile.py @@ -86,8 +86,9 @@ def get_pytest_env_vars() -> Dict[str, str]: return ret +# DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.10", "3.11", "3.12", "3.13", "3.14"] +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/samples/polars/requirements-test.txt b/samples/polars/requirements-test.txt index ce5e1b9e702..cbac5e3f126 100644 --- a/samples/polars/requirements-test.txt +++ b/samples/polars/requirements-test.txt @@ -1,3 +1,3 @@ # samples/snippets should be runnable with no "extras" -google-cloud-testutils==1.8.0 -pytest==9.0.3 +google-cloud-testutils==1.4.0 +pytest==8.3.2 diff --git a/samples/polars/requirements.txt b/samples/polars/requirements.txt index 218e674b9ed..16269825362 100644 --- a/samples/polars/requirements.txt +++ b/samples/polars/requirements.txt @@ -1,3 +1,3 @@ -bigframes==2.39.0 -polars==1.40.1 -pyarrow==24.0.0 +bigframes==2.25.0 +polars==1.24.0 +pyarrow==21.0.0 diff --git a/samples/snippets/multimodal_test.py b/samples/snippets/multimodal_test.py index ce04d511346..033fead33e0 100644 --- a/samples/snippets/multimodal_test.py +++ b/samples/snippets/multimodal_test.py @@ -123,404 +123,3 @@ def test_multimodal_dataframe(gcs_bucket_snippets: str) -> None: assert answer_alt is not None assert embeddings is not None assert chunked is not None - - -def test_multimodal_example(gcs_bucket_snippets: str) -> None: - BUCKET = gcs_bucket_snippets - # [START bigquery_dataframes_multimodal_load] - import bigframes.bigquery as bbq - import bigframes.pandas as bpd - - bbq.load_data( - "cymbal_pets.products", - write_disposition="OVERWRITE", - from_files_options={ - "format": "avro", - "uris": [ - "gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/tables/products/products_*.avro" - ], - }, - ) - # [END bigquery_dataframes_multimodal_load] - - # [START bigquery_dataframes_multimodal_create_images] - bbq.create_external_table( - "cymbal_pets.product_images", - replace=True, - connection_name="us.cymbal_conn", - options={ - "object_metadata": "SIMPLE", - "uris": [ - "gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/images/*.png" - ], - }, - ) - # [END bigquery_dataframes_multimodal_create_images] - - # [START bigquery_dataframes_multimodal_create_manuals] - bbq.create_external_table( - "cymbal_pets.product_manuals", - replace=True, - connection_name="us.cymbal_conn", - options={ - "object_metadata": "SIMPLE", - "uris": [ - "gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/documents/*.pdf" - ], - }, - ) - # [END bigquery_dataframes_multimodal_create_manuals] - - # [START bigquery_dataframes_multimodal_create_gemini] - gemini_model = bbq.ml.create_model( - "cymbal_pets.gemini", - replace=True, - connection_name="us.cymbal_conn", - options={"endpoint": "gemini-2.5-flash"}, - ) - # [END bigquery_dataframes_multimodal_create_gemini] - - # [START bigquery_dataframes_multimodal_create_embedding] - embedding_model = bbq.ml.create_model( - "cymbal_pets.embedding_model", - replace=True, - connection_name="us.cymbal_conn", - options={"endpoint": "multimodalembedding@001"}, - ) - # [END bigquery_dataframes_multimodal_create_embedding] - - # [START bigquery_dataframes_multimodal_create_df_products_mm] - df_images = bpd.read_gbq("SELECT * FROM cymbal_pets.product_images") - df_products = bpd.read_gbq("cymbal_pets.products") - - df_products_mm = df_images.merge(df_products, on="uri").drop(columns="uri") - df_products_mm = df_products_mm.rename(columns={"ref": "image"}) - # [END bigquery_dataframes_multimodal_create_df_products_mm] - - # [START bigquery_dataframes_multimodal_show_df_products_mm] - df_products_mm[["product_name", "image"]] - # [END bigquery_dataframes_multimodal_show_df_products_mm] - - # [START bigquery_dataframes_multimodal_image_description] - df_products_mm["url"] = bbq.obj.get_access_url( - df_products_mm["image"], "R" - ).to_frame() - df_products_mm["prompt0"] = "Can you describe the following image?" - - df_products_mm["prompt"] = bbq.struct(df_products_mm[["prompt0", "url"]]) - df_products_mm = bbq.ai.generate_table( - gemini_model, df_products_mm, output_schema={"image_description": "STRING"} - ) - - df_products_mm = df_products_mm[ - [ - "product_id", - "product_name", - "brand", - "category", - "subcategory", - "animal_type", - "search_keywords", - "price", - "description", - "inventory_level", - "supplier_id", - "average_rating", - "image", - "image_description", - ] - ] - # [END bigquery_dataframes_multimodal_image_description] - - # [START bigquery_dataframes_multimodal_generate_animal_type] - df_prompt = bbq.obj.get_access_url(df_products_mm["image"], "R").to_frame() - df_prompt[ - "prompt0" - ] = "For the image of a pet product, concisely generate the following metadata: 1) animal_type and 2) 5 SEO search keywords, and 3) product subcategory." - - df_products_mm["prompt"] = bbq.struct(df_prompt[["prompt0", "image"]]) - - df_products_mm = df_products_mm.drop( - columns=["animal_type", "search_keywords", "subcategory"] - ) - df_products_mm = bbq.ai.generate_table( - gemini_model, - df_products_mm, - output_schema="animal_type STRING, search_keywords ARRAY, subcategory STRING", - ) - # [END bigquery_dataframes_multimodal_generate_animal_type] - - # [START bigquery_dataframes_multimodal_show_animal_type] - df_products_mm[ - [ - "product_name", - "image_description", - "animal_type", - "search_keywords", - "subcategory", - ] - ] - # [END bigquery_dataframes_multimodal_show_animal_type] - - # [START bigquery_dataframes_multimodal_brand_description] - df_agg = df_products_mm[ - ["image", "description", "category", "subcategory", "brand"] - ] - df_agg["image"] = bbq.obj.get_access_url(df_products_mm["image"], "R") - df_agg = bbq.array_agg(df_agg.groupby(by=["brand"])) - - df_agg["cnt"] = bbq.array_length(df_agg["image"]) - - df_prompt = df_agg[["image", "description", "category", "subcategory"]] - df_prompt[ - "prompt0" - ] = "Use the images and text to give one concise brand description for a website brand page. Return the description only. " - - df_agg["prompt"] = bbq.struct( - df_prompt[["prompt0", "image", "description", "category", "subcategory"]] - ) - - df_agg = df_agg.reset_index() - - df_agg = bbq.ai.generate_table( - gemini_model, df_agg, output_schema={"brand_description": "STRING"} - ) - df_agg[["brand", "brand_description", "cnt"]] - # [END bigquery_dataframes_multimodal_brand_description] - - # [START bigquery_dataframes_multimodal_define_to_grayscale] - @bpd.udf( - dataset="cymbal_pets", - name="to_grayscale", - packages=["numpy", "opencv-python"], - bigquery_connection="us.cymbal_conn", - max_batching_rows=1, - ) - def to_grayscale(src_ref: str, dst_ref: str) -> str: - import json - from urllib.request import Request, urlopen - - import cv2 as cv - import numpy as np - - src_json = json.loads(src_ref) - srcUrl = src_json["access_urls"]["read_url"] - - dst_json = json.loads(dst_ref) - dstUrl = dst_json["access_urls"]["write_url"] - - req = urlopen(srcUrl) - arr = np.asarray(bytearray(req.read()), dtype=np.uint8) - img = cv.imdecode(arr, -1) # 'Load it as it is' - - # Convert the image to grayscale - gray_image = cv.cvtColor(img, cv.COLOR_BGR2GRAY) - - # Send POST request to the URL - _, img_encoded = cv.imencode(".png", gray_image) - - req = Request( - url=dstUrl, - data=img_encoded.tobytes(), - method="PUT", - headers={ - "Content-Type": "image/png", - }, - ) - with urlopen(req): - pass - return dst_ref - - # [END bigquery_dataframes_multimodal_define_to_grayscale] - - # [START bigquery_dataframes_multimodal_apply_to_grayscale] - df_grayscale = df_products_mm[["product_id", "product_name", "image"]] - df_grayscale[ - "gray_image_uri" - ] = f"gs://{BUCKET}/cymbal-pets-images/grayscale/" + df_grayscale[ - "image" - ].struct.field( - "uri" - ).str.extract( - r"([^/]+)$" - ) - - df_grayscale["gray_image"] = bbq.obj.make_ref( - df_grayscale["gray_image_uri"], "us.cymbal_conn" - ) - - df_grayscale["image_url"] = bbq.to_json_string( - bbq.obj.get_access_url(df_grayscale["image"], "r") - ) - df_grayscale["gray_image_url"] = bbq.to_json_string( - bbq.obj.get_access_url(df_grayscale["gray_image"], "rw") - ) - - df_grayscale[["image_url", "gray_image_url"]].apply(to_grayscale, axis=1) - # [END bigquery_dataframes_multimodal_apply_to_grayscale] - - # [START bigquery_dataframes_multimodal_define_chunk_pdf] - @bpd.udf( - dataset="cymbal_pets", - name="chunk_pdf", - packages=["pypdf"], - bigquery_connection="us.cymbal_conn", - max_batching_rows=1, - ) - def chunk_pdf(src_ref: str, chunk_size: int, overlap_size: int) -> list[str]: - import io - import json - from urllib.request import urlopen - - from pypdf import PdfReader # type: ignore - - src_json = json.loads(src_ref) - srcUrl = src_json["access_urls"]["read_url"] - - req = urlopen(srcUrl) - pdf_file = io.BytesIO(bytearray(req.read())) - reader = PdfReader(pdf_file, strict=False) - - # extract and chunk text simultaneously - all_text_chunks = [] - curr_chunk = "" - for page in reader.pages: - page_text = page.extract_text() - if page_text: - curr_chunk += page_text - # split the accumulated text into chunks of a specific size with overlaop - # this loop implements a sliding window approach to create chunks - while len(curr_chunk) >= chunk_size: - split_idx = curr_chunk.rfind(" ", 0, chunk_size) - if split_idx == -1: - split_idx = chunk_size - actual_chunk = curr_chunk[:split_idx] - all_text_chunks.append(actual_chunk) - overlap = curr_chunk[split_idx + 1 : split_idx + 1 + overlap_size] - curr_chunk = overlap + curr_chunk[split_idx + 1 + overlap_size :] - if curr_chunk: - all_text_chunks.append(curr_chunk) - - return all_text_chunks - - # [END bigquery_dataframes_multimodal_define_chunk_pdf] - - # [START bigquery_dataframes_multimodal_apply_chunk_pdf] - df_manuals = bpd.read_gbq("SELECT * FROM cymbal_pets.product_manuals") - df_manuals["url"] = bbq.to_json_string( - bbq.obj.get_access_url(df_manuals["ref"], "R") - ) - - df_manuals["chunk_size"] = 1000 - df_manuals["overlap_size"] = 100 - - df_manuals["chunked"] = df_manuals[["url", "chunk_size", "overlap_size"]].apply( - chunk_pdf, axis=1 - ) - # [END bigquery_dataframes_multimodal_apply_chunk_pdf] - - # [START bigquery_dataframes_multimodal_analyze_pdf] - df_chunked = df_manuals["chunked"].explode().to_frame() - df_chunked[ - "prompt0" - ] = "Can you summarize the product manual as bullet points? Highlight the legal clauses" - - df_chunked["prompt"] = bbq.struct(df_chunked[["prompt0", "chunked"]]) - - result = bbq.ai.generate_text(gemini_model, df_chunked["prompt"]) - result - # [END bigquery_dataframes_multimodal_analyze_pdf] - - # [START bigquery_dataframes_multimodal_create_embed_table] - df_products_mm["content"] = bbq.obj.get_access_url(df_products_mm["image"], "R") - df_embed = bbq.ai.generate_embedding( - embedding_model, df_products_mm[["content", "product_id"]] - ) - - df_embed.to_gbq("cymbal_pets.products_embedding", if_exists="replace") - # [END bigquery_dataframes_multimodal_create_embed_table] - - # [START bigquery_dataframes_multimodal_vector_search] - df_image = bpd.DataFrame( - { - "uri": [ - "gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/images/cozy-naps-cat-scratching-post-with-condo.png" - ] - } - ).cache() - df_image["image"] = bbq.obj.make_ref(df_image["uri"], "us.cymbal_conn") - df_search = bbq.ai.generate_embedding( - embedding_model, - bbq.obj.get_access_url(bbq.obj.fetch_metadata(df_image["image"]), "R"), - ) - - search_result = bbq.vector_search( - "cymbal_pets.products_embedding", "embedding", df_search["embedding"] - ) - search_result - # [END bigquery_dataframes_multimodal_vector_search] - - # [START bigquery_dataframes_create_external_table_all] - bbq.create_external_table( - "cymbal_pets.product_manuals_all", - replace=True, - connection_name="us.cymbal_conn", - options={ - "object_metadata": "SIMPLE", - "uris": [ - "gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/documents/*.pdf", - "gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/document_chunks/*.pdf", - ], - }, - ) - # [END bigquery_dataframes_create_external_table_all] - - # [START bigquery_dataframes_create_manual_to_chunks] - df1 = bpd.read_gbq("SELECT * FROM cymbal_pets.product_manuals_all").sort_values( - "uri" - ) - df2 = df1.copy() - df1["name"] = df1["uri"].str.extract(r".*/([^.]*).[^/]+") - df2["name"] = df2["uri"].str.extract(r".*/([^.]*)_page[0-9]+.[^/]+") - df_manuals_all = df1.merge(df2, on="name") - df_manuals_agg = ( - bbq.array_agg(df_manuals_all[["ref_x", "uri_x"]].groupby("uri_x"))["ref_x"] - .str[0] - .to_frame() - ) - df_manuals_agg["chunks"] = bbq.array_agg( - df_manuals_all[["ref_y", "uri_x"]].groupby("uri_x") - )["ref_y"] - # [END bigquery_dataframes_create_manual_to_chunks] - - # [START bigquery_dataframes_show_manual_to_chunks] - df_manuals_agg - # [END bigquery_dataframes_show_manual_to_chunks] - - # [START bigquery_dataframes_generate_pages_summary] - df_manuals_agg["chunks_url"] = bbq.array_agg( - bbq.obj.get_access_url(df_manuals_agg.explode("chunks")["chunks"], "R").groupby( - "uri_x" - ) - ) - df_manuals_agg[ - "prompt0" - ] = "Can you provide a page by page summary for the first 3 pages of the attached manual? Only write one line for each page. The pages are provided in serial order" - df_manuals_agg["prompt"] = bbq.struct(df_manuals_agg[["prompt0", "chunks_url"]]) - - result = bbq.ai.generate_text(gemini_model, df_manuals_agg["prompt"])["result"] - result - # [END bigquery_dataframes_generate_pages_summary] - - # [START bigquery_dataframes_generate_each_page_summary] - result = bbq.ai.generate_table( - gemini_model, - df_manuals_agg["prompt"], - output_schema={ - "page1_summary": "STRING", - "page2_summary": "STRING", - "page3_summary": "STRING", - }, - )[["page1_summary", "page2_summary", "page3_summary"]] - result - # [END bigquery_dataframes_generate_each_page_summary] diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 63e742993f9..494639d2fa5 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -86,8 +86,9 @@ def get_pytest_env_vars() -> Dict[str, str]: return ret +# DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.10", "3.11", "3.12", "3.13", "3.14"] +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/scripts/bigquery_generator/constants.py b/scripts/bigquery_generator/constants.py deleted file mode 100644 index 78c3fc60c2b..00000000000 --- a/scripts/bigquery_generator/constants.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pathlib - -SCRIPTS_DIRECTORY = pathlib.Path(__file__).parent.parent.absolute() -PACKAGE_ROOT = SCRIPTS_DIRECTORY.parent -CODE_ROOT = PACKAGE_ROOT / "bigframes" -SCRIPT_PATH_RELATIVE = ( - pathlib.Path(__file__).relative_to(PACKAGE_ROOT).parent.parent - / "generate_bigframes_bigquery.py" -) - - -# Directory containing the YAML files -DATA_DIR = SCRIPTS_DIRECTORY / "data" / "sql-functions" -# Directory where the generated Python files will be placed -OUTPUT_DIR = CODE_ROOT / "operations" / "googlesql" -# Directory where the generated test files will be placed -TEST_OUTPUT_DIR = PACKAGE_ROOT / "tests" / "unit" / "bigquery" / "generated" - -PYTHON_BUILTINS = { - "abs", - "all", - "any", - "ascii", - "bin", - "bool", - "breakpoint", - "bytearray", - "bytes", - "callable", - "chr", - "classmethod", - "compile", - "complex", - "delattr", - "dict", - "dir", - "divmod", - "enumerate", - "eval", - "exec", - "filter", - "float", - "format", - "frozenset", - "getattr", - "globals", - "hasattr", - "hash", - "help", - "hex", - "id", - "input", - "int", - "isinstance", - "issubclass", - "iter", - "len", - "list", - "locals", - "map", - "max", - "memoryview", - "min", - "next", - "object", - "oct", - "open", - "ord", - "pow", - "print", - "property", - "range", - "repr", - "reversed", - "round", - "set", - "setattr", - "slice", - "sorted", - "staticmethod", - "str", - "sum", - "super", - "tuple", - "type", - "vars", - "zip", -} - -DTYPE_MAP = { - "binary": "dtypes.BYTES_DTYPE", - "string": "dtypes.STRING_DTYPE", - "int64": "dtypes.INT_DTYPE", - "i64": "dtypes.INT_DTYPE", - "float64": "dtypes.FLOAT_DTYPE", - "fp64": "dtypes.FLOAT_DTYPE", - "bool": "dtypes.BOOL_DTYPE", - "boolean": "dtypes.BOOL_DTYPE", - "geography": "dtypes.GEO_DTYPE", - "json": "dtypes.JSON_DTYPE", - "date": "dtypes.DATE_DTYPE", - "time": "dtypes.TIME_DTYPE", - "datetime": "dtypes.DATETIME_DTYPE", - "timestamp": "dtypes.TIMESTAMP_DTYPE", - "decimal<38,9>": "dtypes.NUMERIC_DTYPE", - "decimal<76,38>": "dtypes.BIGNUMERIC_DTYPE", -} - -PY_TYPE_MAP = { - "binary": "bytes", - "string": "str", - "int64": "int", - "i64": "int", - "float64": "float", - "fp64": "float", - "bool": "bool", - "boolean": "bool", - "geography": "Any", - "json": "Any", - "date": "datetime.date", - "time": "datetime.time", - "datetime": "datetime.datetime", - "timestamp": "datetime.datetime", - "struct": "dict", - "decimal<38,9>": "decimal.Decimal", - "decimal<76,38>": "decimal.Decimal", - "interval_day": "datetime.timedelta", -} diff --git a/scripts/bigquery_generator/data_models.py b/scripts/bigquery_generator/data_models.py deleted file mode 100644 index e0cffd566a0..00000000000 --- a/scripts/bigquery_generator/data_models.py +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Data models for BigQuery code generator. - -`BQ*` models the Substrait YAML extension structure of BigQuery SQL functions, -while `BigFrames*` models the Jinja template outputs. - -BQ* Class Relations: -==================== -+-------------------+ -| BQModule | -+-------------------+ - | - | functions: list[BQFunc] - v -+-------------------+ -| BQFunc | -+-------------------+ - | - | impls: list[BQFuncImpl] - v -+-------------------+ -| BQFuncImpl | -+-------------------+ - | - | args: list[BQFuncArg] - v -+-------------------+ -| BQFuncArg | -+-------------------+ - -BigFrames* Class Relations: -================================= - +--------------------------------+ - | Accessor |<---+ children: list[Accessor] - +--------------------------------+----+ (nested namespace hierarchy) - | - | functions: list[BigFramesFunc] - v - +--------------------------------+ - | BigFramesFunc | - +--------------------------------+ - | - | args: list[BigFramesFuncArg] - v - +--------------------------------+ - | BigFramesFuncArg | - +--------------------------------+ - - ---------------------------------- - - +--------------------------------+ - | BigFramesOp | (Standalone data model for op defs) - +--------------------------------+ -""" - -from __future__ import annotations - -import dataclasses -import pathlib - -from . import constants - - -@dataclasses.dataclass(frozen=True) -class BQFuncArg: - """ - Represents an argument of a SQL function loaded from a yaml file. - """ - - name: str - value: str # The type of the arg - optional: bool - keyword_only: bool - - -@dataclasses.dataclass(frozen=True) -class BQFuncImpl: - """ - Represents an implementation (i.e. signature) for some SQL function loaded - from a yaml file. - """ - - args: tuple[BQFuncArg, ...] - return_type: str - - @property - def requires_generic_types(self) -> bool: - if "any1" in self.return_type: - return True - - return any("any1" in arg.value for arg in self.args) - - -@dataclasses.dataclass(frozen=True) -class BQFunc: - """ - Represents a SQL function loaded from a yaml file. - """ - - name: str - description: str - impls: tuple[BQFuncImpl, ...] - series_accessor_arg: str | None - - @property - def op_base_name(self) -> str: - return self.name.split(".")[-1] - - -@dataclasses.dataclass(frozen=True) -class BQModule: - """ - Represents the data loaded from a yaml file with SQL functions info. - """ - - yaml_file: pathlib.Path - functions: tuple[BQFunc, ...] - - @property - def module_path(self) -> pathlib.Path: - return self.yaml_file.relative_to(constants.DATA_DIR).with_suffix("") - - @property - def namespace(self) -> tuple[str, ...]: - parts = self.module_path.parts - if "global_namespace" in parts: - return tuple() - return parts - - @property - def is_global(self) -> bool: - return "global_namespace" in self.module_path.parts - - -@dataclasses.dataclass(frozen=True) -class BigFramesOp: - """ - Represents a BigFrames GoogleScalarOp impl to be defined in the code base. - """ - - internal_name: str - sql_name: str - arg_specs: str - signature: str - signature_definition: str | None - - -@dataclasses.dataclass(frozen=True) -class BigFramesFuncArg: - """ - Represents an argument of a BigFrames BigQuery function to be defined in the code base. - """ - - name: str - types: frozenset[str] - optional: bool - keyword_only: bool - - @property - def type_hint(self) -> str: - types = [constants.PY_TYPE_MAP.get(t, "Any") for t in sorted(self.types)] + [ - "Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]" - ] - - if len(types) > 1: - return "Union[" + ", ".join(sorted(set(types))) + "]" - - return types[0] - - @property - def default(self) -> str | None: - if self.optional: - return "sentinels.Sentinel.ARGUMENT_DEFAULT" - return None - - -@dataclasses.dataclass -class BigFramesFuncArgBuilder: - name: str - types: set[str] - optional: bool - keyword_only: bool - - def build(self) -> BigFramesFuncArg: - return BigFramesFuncArg( - name=self.name, - types=frozenset(self.types), - optional=self.optional, - keyword_only=self.keyword_only, - ) - - -@dataclasses.dataclass(frozen=True) -class BigFramesFunc: - """ - Represents a BigFrames BigQuery function to be defined in the codebase. - """ - - name: str - op_name: str - description: str - args: tuple[BigFramesFuncArg, ...] - series_accessor_arg: str | None - import_module: str | None = None - - -@dataclasses.dataclass -class Accessor: - """ - Represents the accessor extensions to be defined for pandas and BigFrames. - It consists of multiple functions bundled under the different namespaces. - - This class is designed to be mutable because it has a recursive data structure. - Mutability makes it easier to build the data structure trees from the top. - """ - - class_name: str - bigframes_class_name: str - pandas_class_name: str - is_root: bool - description: str - children: list[Accessor] - functions: list[BigFramesFunc] - prop_name: str | None = None diff --git a/scripts/bigquery_generator/file_generator.py b/scripts/bigquery_generator/file_generator.py deleted file mode 100644 index 3109afc3f8b..00000000000 --- a/scripts/bigquery_generator/file_generator.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pathlib -import subprocess -import sys -from typing import Sequence - -from . import constants, data_models, template_renderer - - -def _ensure_init_py(directory: pathlib.Path, limit_dir: pathlib.Path) -> None: - """Ensures __init__.py exists in the directory and its parents up to limit_dir.""" - curr = directory - while curr != limit_dir and curr != curr.parent: - init_file = curr / "__init__.py" - if not init_file.exists(): - print(f" Creating {init_file}") - with open(init_file, "w", encoding="utf-8") as f: - f.write(template_renderer.render_license()) - curr = curr.parent - - -def _write_file( - content: str, output_file: pathlib.Path, limit_dir: pathlib.Path -) -> None: - output_file.parent.mkdir(parents=True, exist_ok=True) - _ensure_init_py(output_file.parent, limit_dir) - - with open(output_file, "w", encoding="utf-8") as f: - f.write(content) - print(f" Generated {output_file}") - - -def _run_ruff() -> None: - targets = [ - constants.OUTPUT_DIR, - constants.TEST_OUTPUT_DIR, - constants.CODE_ROOT / "extensions", - ] - ruff_common_args = [ - "--target-version=py310", - "--line-length=88", - ] - - ruff_check_args = [ - "check", - "--select", - "I,F", - "--fix", - ] + ruff_common_args - subprocess.run( - [sys.executable, "-m", "ruff"] + ruff_check_args + targets, - check=True, - ) - - ruff_format_args = [ - "format", - ] + ruff_common_args - subprocess.run( - [sys.executable, "-m", "ruff"] + ruff_format_args + targets, - check=True, - ) - - -def _generate_op_defs(bq_module: data_models.BQModule) -> None: - if not bq_module.functions: - # If there are no function definitions, do not generate file without Python code. - return - - content = template_renderer.render_operation(bq_module) - output_file = constants.OUTPUT_DIR.joinpath(bq_module.module_path).with_suffix( - ".py" - ) - - _write_file(content, output_file, constants.OUTPUT_DIR.parent) - - -def _generate_tests(bq_module: data_models.BQModule) -> None: - if not bq_module.functions: - # If there are no function definitions, do not generate file without Python code. - return - - content = template_renderer.render_tests(bq_module) - output_file = constants.TEST_OUTPUT_DIR.joinpath( - bq_module.module_path.with_name(f"test_{bq_module.module_path.name}") - ).with_suffix(".py") - - _write_file(content, output_file, constants.TEST_OUTPUT_DIR.parent) - - -def _generate_accesor(bq_modules: Sequence[data_models.BQModule]) -> None: - (core_content, pd_content, bf_content) = template_renderer.render_accessor( - bq_modules - ) - - core_output_file = ( - constants.CODE_ROOT / "extensions" / "core" / "series_accessor.py" - ) - _write_file(core_content, core_output_file, constants.CODE_ROOT) - - pd_output_file = ( - constants.CODE_ROOT / "extensions" / "pandas" / "series_accessor.py" - ) - _write_file(pd_content, pd_output_file, constants.CODE_ROOT) - - bf_output_file = ( - constants.CODE_ROOT / "extensions" / "bigframes" / "series_accessor.py" - ) - _write_file(bf_content, bf_output_file, constants.CODE_ROOT) - - -def generate(bq_modules: Sequence[data_models.BQModule]) -> None: - for bq_module in bq_modules: - _generate_op_defs(bq_module) - _generate_tests(bq_module) - - _generate_accesor(bq_modules) - - # Ruff format - _run_ruff() diff --git a/scripts/bigquery_generator/template_renderer.py b/scripts/bigquery_generator/template_renderer.py deleted file mode 100644 index a4e8461286d..00000000000 --- a/scripts/bigquery_generator/template_renderer.py +++ /dev/null @@ -1,334 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -Renders jinja template with module data parsed from yaml. -""" - -from typing import Sequence - -import jinja2 - -from . import constants, data_models - - -def _load_templates() -> dict[str, jinja2.Template]: - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(constants.SCRIPTS_DIRECTORY / "templates"), - trim_blocks=True, - lstrip_blocks=True, - ) - return { - "operation": env.get_template("operation.py.j2"), - "test_operation": env.get_template("test_operation.py.j2"), - "license": env.get_template("license.py.j2"), - "signature_def": env.get_template("signature_def.py.j2"), - "core_series_accessor": env.get_template("core_series_accessor.py.j2"), - "bigframes_series_accessor": env.get_template( - "bigframes_series_accessor.py.j2" - ), - "pandas_series_accessor": env.get_template("pandas_series_accessor.py.j2"), - } - - -TEMPLATES: dict[str, jinja2.Template] = _load_templates() - - -def _unwrap_list_type(yaml_type: str) -> str | None: - if yaml_type.startswith("list<") and yaml_type.endswith(">"): - return yaml_type[5:-1] - return None - - -def _try_get_concrete_type_expr(yaml_type: str) -> str | None: - if yaml_type in constants.DTYPE_MAP: - return constants.DTYPE_MAP[yaml_type] - inner = _unwrap_list_type(yaml_type) - if inner and inner in constants.DTYPE_MAP: - # TODO (b/540011825): Support recursive type parsing - return f"dtypes.list_type({constants.DTYPE_MAP[inner]})" - return None - - -def _get_concrete_type_expr(yaml_type: str) -> str: - expr = _try_get_concrete_type_expr(yaml_type) - if expr is None: - raise ValueError(f"Not a concrete type: {yaml_type}") - return expr - - -def _is_concrete_type(yaml_type: str) -> bool: - return _try_get_concrete_type_expr(yaml_type) is not None - - -def _validate_type(yaml_type: str) -> None: - if yaml_type in ("any1", "struct") or yaml_type in constants.DTYPE_MAP: - return - inner = _unwrap_list_type(yaml_type) - if inner is not None: - if inner == "any1" or inner in constants.DTYPE_MAP: - return - raise ValueError(f"Unsupported inner type: {inner}") - raise ValueError(f"Unsupported type: {yaml_type}") - - -def _validate_types(impls: Sequence[data_models.BQFuncImpl]) -> None: - for impl in impls: - for arg in impl.args: - _validate_type(arg.value) - _validate_type(impl.return_type) - - -def render_signature_def( - bq_func: data_models.BQFunc, -) -> tuple[str, str | None]: - """ - Returns the signature function name and its definition. - If the signature function can be inlined, the first return value is the lambda, - and the second value is None. - - Examples: - Inlined signature function: - ("lambda *args: dtypes.FLOAT64_DTYPE", None) - - Custom signature function definition: - ("_ABS_SIG", "def _ABS_SIG(*args): ...") - """ - return_types = {impl.return_type for impl in bq_func.impls} - # Optimization: if all impls return the same concrete type, - # inline the signature function as a lambda - if len(return_types) == 1: - ret_type = next(iter(return_types)) - if _is_concrete_type(ret_type): - sig_expr = f"lambda *args: {_get_concrete_type_expr(ret_type)}" - return sig_expr, None - - _validate_types(bq_func.impls) - - sig_func_name = f"_{bq_func.op_base_name.upper()}_SIG" - - max_args = max(len(impl.args) for impl in bq_func.impls) - - rendered = TEMPLATES["signature_def"].render( - func_name=sig_func_name, - max_args=max_args, - impls=bq_func.impls, - sql_name=bq_func.name, - dtype_map=constants.DTYPE_MAP, - ) - - return sig_func_name, rendered - - -def _get_bigframes_func_args( - bq_func: data_models.BQFunc, -) -> tuple[data_models.BigFramesFuncArg, ...]: - """ - Coalesces arguments from all the signatures of this function, - and return them in the order of appearance in the yaml file - """ - args_by_name: dict[str, data_models.BigFramesFuncArgBuilder] = {} - arg_order: list[str] = [] - arg_appearances: dict[str, int] = {} - for impl in bq_func.impls: - seen_in_impl = set() - for bq_func_arg in impl.args: - name = bq_func_arg.name - seen_in_impl.add(name) - if name not in args_by_name: - args_by_name[name] = data_models.BigFramesFuncArgBuilder( - name=name, - types=set(), - optional=bq_func_arg.optional, - keyword_only=bq_func_arg.keyword_only, - ) - arg_order.append(name) - else: - # If it was marked optional or keyword_only in any previous impl, keep it. - # Or if this signature marks it as optional/keyword_only, update it. - if bq_func_arg.optional: - args_by_name[name].optional = True - if bq_func_arg.keyword_only: - args_by_name[name].keyword_only = True - args_by_name[name].types.add(bq_func_arg.value) - for name in seen_in_impl: - arg_appearances[name] = arg_appearances.get(name, 0) + 1 - - # If an argument is not in all impls, it must be optional overall - num_impls = len(bq_func.impls) - for name, count in arg_appearances.items(): - if count < num_impls: - args_by_name[name].optional = True - - return tuple(args_by_name[name].build() for name in arg_order) - - -def _to_bigframes_op(bq_func: data_models.BQFunc) -> data_models.BigFramesOp: - arg_specs = [] - for bf_func_arg in _get_bigframes_func_args(bq_func): - spec = "googlesql.ArgSpec(" - if bf_func_arg.keyword_only: - spec += f'arg_name="{bf_func_arg.name}", ' - if bf_func_arg.optional: - spec += "optional=True, " - spec = spec.rstrip(", ") + ")" - arg_specs.append(spec) - - arg_specs_str = ", ".join(arg_specs) - if len(arg_specs) == 1: - arg_specs_str += "," - - (signature, signature_definition) = render_signature_def(bq_func) - - return data_models.BigFramesOp( - internal_name=f"_{bq_func.op_base_name.upper()}_OP", - sql_name=bq_func.name.upper(), - arg_specs=arg_specs_str, - signature=signature, - signature_definition=signature_definition, - ) - - -def _to_bigframes_func( - bq_func: data_models.BQFunc, import_module: str | None = None -) -> data_models.BigFramesFunc: - python_name = bq_func.op_base_name - if python_name in constants.PYTHON_BUILTINS: - python_name = python_name + "_" - - return data_models.BigFramesFunc( - name=python_name, - op_name=f"_{bq_func.op_base_name.upper()}_OP", - description=bq_func.description, - args=_get_bigframes_func_args(bq_func), - series_accessor_arg=bq_func.series_accessor_arg, - import_module=import_module, - ) - - -def render_license() -> str: - return TEMPLATES["license"].render() - - -def render_operation( - bq_module: data_models.BQModule, -) -> str: - ops: list[data_models.BigFramesOp] = [] - functions: list[data_models.BigFramesFunc] = [] - - for bq_func in bq_module.functions: - ops.append(_to_bigframes_op(bq_func)) - functions.append(_to_bigframes_func(bq_func)) - - return TEMPLATES["operation"].render( - yaml_path=bq_module.yaml_file.relative_to(constants.PACKAGE_ROOT), - script_path=constants.SCRIPT_PATH_RELATIVE, - ops=ops, - functions=functions, - ) - - -def render_tests(bq_module: data_models.BQModule) -> str: - import_path = "bigframes.operations.googlesql." + ".".join( - bq_module.module_path.parts - ) - functions: list[data_models.BigFramesFunc] = [] - for bq_func in bq_module.functions: - functions.append(_to_bigframes_func(bq_func)) - - return TEMPLATES["test_operation"].render( - yaml_path=bq_module.yaml_file.relative_to(constants.PACKAGE_ROOT), - script_path=constants.SCRIPT_PATH_RELATIVE, - import_path=import_path, - short_name=bq_module.module_path.name, - is_global=bq_module.is_global, - functions=functions, - ) - - -def _create_accessor_class_name(namespace: tuple[str, ...], prefix: str = "") -> str: - if not namespace: - return f"{prefix}BigQuerySeriesAccessor" - camel_parts = [part.capitalize() for part in namespace] - return f"{prefix}{''.join(camel_parts)}SeriesAccessor" - - -def render_accessor( - bq_modules: Sequence[data_models.BQModule], -) -> tuple[str, str, str]: - """ - Returns the content for core accessor, pandas accessor and BF accessor - """ - - namespaces: set[tuple[str, ...]] = set() - for bq_module in bq_modules: - for i in range(len(bq_module.namespace) + 1): - namespaces.add(bq_module.namespace[:i]) - - sorted_namespaces = sorted(list(namespaces), key=lambda ns: (len(ns), ns)) - - accessors: list[data_models.Accessor] = [] - accessor_lookup_table: dict[tuple[str, ...], data_models.Accessor] = {} - for namespace in sorted_namespaces: - accessor = data_models.Accessor( - class_name=_create_accessor_class_name(namespace), - bigframes_class_name=_create_accessor_class_name( - namespace, prefix="Bigframes" - ), - pandas_class_name=_create_accessor_class_name(namespace, prefix="Pandas"), - is_root=len(namespace) == 0, - description=( - f"Series accessor for BigQuery {'.'.join(namespace)} functions." - if namespace - else "Series accessor for BigQuery functions." - ), - children=[], - functions=[], - ) - accessors.append(accessor) - accessor_lookup_table[namespace] = accessor - - # Establish parent-child relations - if len(namespace) > 0: - accessor.prop_name = namespace[-1] - parent_namespace = namespace[:-1] - accessor_lookup_table[parent_namespace].children.append(accessor) - - # Arrange functions by namespaces - for bq_module in bq_modules: - module_parts = bq_module.module_path.parts - for bq_func in bq_module.functions: - if bq_func.series_accessor_arg is None: - continue - bf_func = _to_bigframes_func( - bq_func, - import_module=f"bigframes.operations.googlesql.{'.'.join(module_parts)}", - ) - accessor_lookup_table[bq_module.namespace].functions.append(bf_func) - - core_content = TEMPLATES["core_series_accessor"].render( - script_path=constants.SCRIPT_PATH_RELATIVE, - namespaces=accessors, - ) - - pandas_content = TEMPLATES["pandas_series_accessor"].render( - script_path=constants.SCRIPT_PATH_RELATIVE, namespaces=accessors - ) - - bigframes_content = TEMPLATES["bigframes_series_accessor"].render( - script_path=constants.SCRIPT_PATH_RELATIVE, namespaces=accessors - ) - - return core_content, pandas_content, bigframes_content diff --git a/scripts/bigquery_generator/yaml_parser.py b/scripts/bigquery_generator/yaml_parser.py deleted file mode 100644 index 6ef1d5d9236..00000000000 --- a/scripts/bigquery_generator/yaml_parser.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import pathlib -from typing import Any - -import yaml - -from . import data_models - - -def _build_func_arg_ir(arg_data: Any) -> data_models.BQFuncArg: - return data_models.BQFuncArg( - name=arg_data["name"], - value=arg_data["value"], - optional=arg_data["optional"], - keyword_only=arg_data["keyword_only"], - ) - - -def _build_func_impl_ir(impl_data: Any) -> data_models.BQFuncImpl: - return data_models.BQFuncImpl( - args=tuple(_build_func_arg_ir(arg) for arg in impl_data["args"]), - return_type=impl_data["return"], - ) - - -def _build_func_ir(func_data: Any) -> data_models.BQFunc: - return data_models.BQFunc( - name=func_data["name"], - description=func_data["description"], - impls=tuple(_build_func_impl_ir(impl) for impl in func_data["impls"]), - series_accessor_arg=func_data.get("series_accessor_arg", None), - ) - - -def parse_yaml(yaml_file: pathlib.Path) -> data_models.BQModule: - print(f"Parsing {yaml_file}...") - - with open(yaml_file, "r", encoding="utf-8") as f: - data = yaml.safe_load(f) - - functions: tuple[data_models.BQFunc, ...] = () - if isinstance(data, dict) and "scalar_functions" in data: - functions = tuple( - _build_func_ir(func_data) for func_data in data["scalar_functions"] - ) - - return data_models.BQModule( - yaml_file=yaml_file, - functions=functions, - ) diff --git a/scripts/conftest.py b/scripts/conftest.py index 0d55bd4b478..83fd2b19aff 100644 --- a/scripts/conftest.py +++ b/scripts/conftest.py @@ -1,5 +1,5 @@ -import sys from pathlib import Path +import sys # inserts scripts into path so that tests can import project_root = Path(__file__).parent.parent diff --git a/scripts/create_gcs.py b/scripts/create_gcs.py index bdb8a23ddc9..8a94bfd8865 100644 --- a/scripts/create_gcs.py +++ b/scripts/create_gcs.py @@ -16,12 +16,12 @@ # bigframes.streaming testing if they don't already exist import os -import sys from pathlib import Path +import sys import google.cloud.exceptions as exceptions -import google.cloud.storage as gcs from google.cloud.storage import transfer_manager +import google.cloud.storage as gcs PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") diff --git a/scripts/create_read_gbq_colab_benchmark_tables.py b/scripts/create_read_gbq_colab_benchmark_tables.py index 727a1e116ac..63419bc6604 100644 --- a/scripts/create_read_gbq_colab_benchmark_tables.py +++ b/scripts/create_read_gbq_colab_benchmark_tables.py @@ -23,8 +23,8 @@ import time from typing import Any, Iterable, MutableSequence, Sequence -import numpy as np from google.cloud import bigquery +import numpy as np # --- Input Data --- # Generated by querying bigquery-magics usage. See internal issue b/420984164. @@ -463,7 +463,7 @@ def worker_initializer(project_id: str | None): def worker_process_item( - work_item: tuple[str, Sequence[tuple[str, str, int | None]], int], + work_item: tuple[str, Sequence[tuple[str, str, int | None]], int] ): global worker_client, worker_rng diff --git a/scripts/create_read_gbq_colab_benchmark_tables_test.py b/scripts/create_read_gbq_colab_benchmark_tables_test.py index 56c9cb2bc56..89c49e42435 100644 --- a/scripts/create_read_gbq_colab_benchmark_tables_test.py +++ b/scripts/create_read_gbq_colab_benchmark_tables_test.py @@ -20,9 +20,6 @@ import math import re -import numpy as np -import pytest - # Assuming the script to be tested is in the same directory or accessible via PYTHONPATH from create_read_gbq_colab_benchmark_tables import ( BIGQUERY_DATA_TYPE_SIZES, @@ -30,6 +27,8 @@ generate_work_items, get_bq_schema, ) +import numpy as np +import pytest # Helper function to calculate estimated row size from schema diff --git a/scripts/data/sql-functions/aead.yaml b/scripts/data/sql-functions/aead.yaml deleted file mode 100644 index 198248782d7..00000000000 --- a/scripts/data/sql-functions/aead.yaml +++ /dev/null @@ -1,134 +0,0 @@ -urn: extension:google:bq_scalar_functions -scalar_functions: - - name: "aead.decrypt_bytes" - description: "Uses the matching key from keyset to decrypt ciphertext and verifies the integrity of the data using additional_data. Returns an error if decryption or verification fails." - series_accessor_arg: keyset - impls: - # Signature: aead.decrypt_bytes:vbin_vbin_vbin - - args: - - name: "keyset" - value: binary - optional: false - keyword_only: false - - name: "ciphertext" - value: binary - optional: false - keyword_only: false - - name: "additional_data" - value: binary - optional: false - keyword_only: false - return: binary - # Signature: aead.decrypt_bytes:struct_vbin_vbin - - args: - - name: "keyset" - value: struct - optional: false - keyword_only: false - - name: "ciphertext" - value: binary - optional: false - keyword_only: false - - name: "additional_data" - value: binary - optional: false - keyword_only: false - return: binary - - name: "aead.decrypt_string" - description: "Like AEAD.DECRYPT_BYTES, but where additional_data is of type STRING." - series_accessor_arg: keyset - impls: - # Signature: aead.decrypt_string:vbin_vbin_str - - args: - - name: "keyset" - value: binary - optional: false - keyword_only: false - - name: "ciphertext" - value: binary - optional: false - keyword_only: false - - name: "additional_data" - value: string - optional: false - keyword_only: false - return: string - # Signature: aead.decrypt_string:struct_vbin_str - - args: - - name: "keyset" - value: struct - optional: false - keyword_only: false - - name: "ciphertext" - value: binary - optional: false - keyword_only: false - - name: "additional_data" - value: string - optional: false - keyword_only: false - return: string - - name: "aead.encrypt" - description: "Encrypts plaintext using the primary cryptographic key in keyset. The algorithm of the primary key must be AEAD_AES_GCM_256. Binds the ciphertext to the context defined by additional_data. Returns NULL if any input is NULL." - series_accessor_arg: keyset - impls: - # Signature: aead.encrypt:vbin_str_str - - args: - - name: "keyset" - value: binary - optional: false - keyword_only: false - - name: "plaintext" - value: string - optional: false - keyword_only: false - - name: "additional_data" - value: string - optional: false - keyword_only: false - return: binary - # Signature: aead.encrypt:vbin_vbin_vbin - - args: - - name: "keyset" - value: binary - optional: false - keyword_only: false - - name: "plaintext" - value: binary - optional: false - keyword_only: false - - name: "additional_data" - value: binary - optional: false - keyword_only: false - return: binary - # Signature: aead.encrypt:struct_str_str - - args: - - name: "keyset" - value: struct - optional: false - keyword_only: false - - name: "plaintext" - value: string - optional: false - keyword_only: false - - name: "additional_data" - value: string - optional: false - keyword_only: false - return: binary - # Signature: aead.encrypt:struct_vbin_vbin - - args: - - name: "keyset" - value: struct - optional: false - keyword_only: false - - name: "plaintext" - value: binary - optional: false - keyword_only: false - - name: "additional_data" - value: binary - optional: false - keyword_only: false - return: binary diff --git a/scripts/data/sql-functions/ai.yaml b/scripts/data/sql-functions/ai.yaml deleted file mode 100644 index f3238c8178b..00000000000 --- a/scripts/data/sql-functions/ai.yaml +++ /dev/null @@ -1 +0,0 @@ -urn: extension:google:bq_scalar_functions diff --git a/scripts/data/sql-functions/global_namespace/aead_encryption.yaml b/scripts/data/sql-functions/global_namespace/aead_encryption.yaml deleted file mode 100644 index 1e62de0f2a6..00000000000 --- a/scripts/data/sql-functions/global_namespace/aead_encryption.yaml +++ /dev/null @@ -1,134 +0,0 @@ -urn: extension:google:bq_scalar_functions -scalar_functions: - - name: "deterministic_decrypt_bytes" - description: "Uses the matching key from `keyset` to decrypt `ciphertext` and verifies the integrity of the data using `additional_data`. Returns an error if decryption fails." - series_accessor_arg: keyset - impls: - # Signature: deterministic_decrypt_bytes:vbin_vbin_vbin - - args: - - name: "keyset" - value: binary - optional: false - keyword_only: false - - name: "ciphertext" - value: binary - optional: false - keyword_only: false - - name: "additional_data" - value: binary - optional: false - keyword_only: false - return: binary - # Signature: deterministic_decrypt_bytes:struct_vbin_vbin - - args: - - name: "keyset" - value: struct - optional: false - keyword_only: false - - name: "ciphertext" - value: binary - optional: false - keyword_only: false - - name: "additional_data" - value: binary - optional: false - keyword_only: false - return: binary - - name: "deterministic_decrypt_string" - description: "Like `DETERMINISTIC_DECRYPT_BYTES`, but where plaintext is of type STRING." - series_accessor_arg: keyset - impls: - # Signature: deterministic_decrypt_string:vbin_vbin_str - - args: - - name: "keyset" - value: binary - optional: false - keyword_only: false - - name: "ciphertext" - value: binary - optional: false - keyword_only: false - - name: "additional_data" - value: string - optional: false - keyword_only: false - return: string - # Signature: deterministic_decrypt_string:struct_vbin_str - - args: - - name: "keyset" - value: struct - optional: false - keyword_only: false - - name: "ciphertext" - value: binary - optional: false - keyword_only: false - - name: "additional_data" - value: string - optional: false - keyword_only: false - return: string - - name: "deterministic_encrypt" - description: "Encrypts `plaintext` using the primary cryptographic key in `keyset` using deterministic AEAD. The algorithm of the primary key must be `DETERMINISTIC_AEAD_AES_SIV_CMAC_256`. Binds the ciphertext to the context defined by `additional_data`. Returns `NULL` if any input is `NULL`." - series_accessor_arg: keyset - impls: - # Signature: deterministic_encrypt:vbin_str_str - - args: - - name: "keyset" - value: binary - optional: false - keyword_only: false - - name: "plaintext" - value: string - optional: false - keyword_only: false - - name: "additional_data" - value: string - optional: false - keyword_only: false - return: binary - # Signature: deterministic_encrypt:vbin_vbin_vbin - - args: - - name: "keyset" - value: binary - optional: false - keyword_only: false - - name: "plaintext" - value: binary - optional: false - keyword_only: false - - name: "additional_data" - value: binary - optional: false - keyword_only: false - return: binary - # Signature: deterministic_encrypt:struct_str_str - - args: - - name: "keyset" - value: struct - optional: false - keyword_only: false - - name: "plaintext" - value: string - optional: false - keyword_only: false - - name: "additional_data" - value: string - optional: false - keyword_only: false - return: binary - # Signature: deterministic_encrypt:struct_vbin_vbin - - args: - - name: "keyset" - value: struct - optional: false - keyword_only: false - - name: "plaintext" - value: binary - optional: false - keyword_only: false - - name: "additional_data" - value: binary - optional: false - keyword_only: false - return: binary diff --git a/scripts/data/sql-functions/global_namespace/array.yaml b/scripts/data/sql-functions/global_namespace/array.yaml deleted file mode 100644 index aa9230c251b..00000000000 --- a/scripts/data/sql-functions/global_namespace/array.yaml +++ /dev/null @@ -1,341 +0,0 @@ -urn: extension:google:bq_scalar_functions -scalar_functions: - - name: "array_concat" - description: "Concatenates one or more arrays with the same element type into a single array." - series_accessor_arg: array_expression_1 - impls: - # Signature: array_concat:list_list - - args: - - name: "array_expression_1" - value: list - optional: false - keyword_only: false - - name: "array_expression_2" - value: list - optional: false - keyword_only: false - return: list - - name: "array_first" - description: "Takes an array and returns the first element in the array." - series_accessor_arg: array_expression - impls: - # Signature: array_first:list - - args: - - name: "array_expression" - value: list - optional: false - keyword_only: false - return: any1 - - name: "array_first_n" - description: "Returns a prefix of `input_array` consisting of the first `n` elements." - series_accessor_arg: input_array - impls: - # Signature: array_first_n:list_i64 - - args: - - name: "input_array" - value: list - optional: false - keyword_only: false - - name: "n" - value: i64 - optional: false - keyword_only: false - return: list - - name: "array_includes" - description: "Takes an array and returns `TRUE` if there is an element in the array that is equal to the search_value." - series_accessor_arg: array_to_search - impls: - # Signature: array_includes:list_any - - args: - - name: "array_to_search" - value: list - optional: false - keyword_only: false - - name: "search_value" - value: any1 - optional: false - keyword_only: false - return: boolean - - name: "array_includes_all" - description: "Takes an array to search and an array of search values. Returns `TRUE` if all search values are in the array to search, otherwise returns `FALSE`." - series_accessor_arg: array_to_search - impls: - # Signature: array_includes_all:list_list - - args: - - name: "array_to_search" - value: list - optional: false - keyword_only: false - - name: "search_values" - value: list - optional: false - keyword_only: false - return: boolean - - name: "array_includes_any" - description: "Takes an array to search and an array of search values. Returns `TRUE` if any search values are in the array to search, otherwise returns `FALSE`." - series_accessor_arg: array_to_search - impls: - # Signature: array_includes_any:list_list - - args: - - name: "array_to_search" - value: list - optional: false - keyword_only: false - - name: "search_values" - value: list - optional: false - keyword_only: false - return: boolean - - name: "array_is_distinct" - description: "Returns `TRUE` if the array contains no repeated elements, using the same equality comparison logic as `SELECT DISTINCT`." - series_accessor_arg: array_expression - impls: - # Signature: array_is_distinct:list - - args: - - name: "array_expression" - value: list - optional: false - keyword_only: false - return: boolean - - name: "array_last" - description: "Takes an array and returns the last element in the array." - series_accessor_arg: array_expression - impls: - # Signature: array_last:list - - args: - - name: "array_expression" - value: list - optional: false - keyword_only: false - return: any1 - - name: "array_length" - description: | - Compute the length of each array element in the Series. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - - >>> s = bpd.Series([[1, 2, 8, 3], [], [3, 4]]) - >>> bbq.array_length(s) - 0 4 - 1 0 - 2 2 - dtype: Int64 - - You can call this function using the Series `bigquery` accessor. - - >>> s.bigquery.array_length() - 0 4 - 1 0 - 2 2 - dtype: Int64 - - You can also use this accessor on a pandas Series after importing bigframes. - - >>> import bigframes - >>> import pandas as pd - >>> ps = pd.Series([[1, 2, 8, 3], [], [3, 4]]) - >>> ps.bigquery.array_length() - 0 4 - 1 0 - 2 2 - dtype: Int64 - - You can also apply this function directly to Series using `apply`. - - >>> s.apply(bbq.array_length, by_row=False) - 0 4 - 1 0 - 2 2 - dtype: Int64 - - Args: - series (bigframes.series.Series): A Series with array columns. - - Returns: - bigframes.series.Series: A Series of integer values indicating - the length of each element in the Series. - series_accessor_arg: series - impls: - # Signature: array_length:list - - args: - - name: "series" - value: list - optional: false - keyword_only: false - return: i64 - - name: "array_reverse" - description: "Returns the input `ARRAY` with elements in reverse order." - series_accessor_arg: value - impls: - # Signature: array_reverse:list - - args: - - name: "value" - value: list - optional: false - keyword_only: false - return: list - - name: "array_slice" - description: "Returns an array containing zero or more consecutive elements from the input array." - series_accessor_arg: array_to_slice - impls: - # Signature: array_slice:list_i64_i64 - - args: - - name: "array_to_slice" - value: list - optional: false - keyword_only: false - - name: "start_offset" - value: i64 - optional: false - keyword_only: false - - name: "end_offset" - value: i64 - optional: false - keyword_only: false - return: list - - name: "array_to_string" - description: | - Converts array elements within a Series into delimited strings. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> import bigframes.bigquery as bbq - - >>> s = bpd.Series([["H", "i", "!"], ["Hello", "World"], np.nan, [], ["Hi"]]) - >>> bbq.array_to_string(s, delimiter=", ") - 0 H, i, ! - 1 Hello, World - 2 - 3 - 4 Hi - dtype: string - - You can call this function using the Series `bigquery` accessor. - - >>> s.bigquery.array_to_string(delimiter=", ") - 0 H, i, ! - 1 Hello, World - 2 - 3 - 4 Hi - dtype: string - - You can also use this accessor on a pandas Series after importing bigframes. - - >>> import bigframes - >>> import pandas as pd - >>> ps = pd.Series([["H", "i", "!"], ["Hello", "World"], None, [], ["Hi"]]) - >>> ps.bigquery.array_to_string(delimiter=", ") - 0 H, i, ! - 1 Hello, World - 2 - 3 - 4 Hi - dtype: string - - Args: - series (bigframes.series.Series): A Series containing arrays. - delimiter (str): The string used to separate array elements. - null_text (str, optional): The string to replace any NULL values in the array with. - - Returns: - bigframes.series.Series: A Series containing delimited strings. - series_accessor_arg: series - impls: - # Signature: array_to_string:list_str_str - - args: - - name: "series" - value: list - optional: false - keyword_only: false - - name: "delimiter" - value: string - optional: false - keyword_only: false - - name: "null_text" - value: string - optional: true - keyword_only: false - return: string - # Signature: array_to_string:list_vbin_vbin - - args: - - name: "series" - value: list - optional: false - keyword_only: false - - name: "delimiter" - value: binary - optional: false - keyword_only: false - - name: "null_text" - value: binary - optional: true - keyword_only: false - return: binary - - name: "flatten" - description: "Takes an array of nested data and flattens a specific part of it into a single, flat array with the [array elements field access operator][array-el-field-operator]. Returns `NULL` if the input value is `NULL`." - series_accessor_arg: array_to_flatten - impls: - # Signature: flatten:list_i64 - - args: - - name: "array_to_flatten" - value: list - optional: false - keyword_only: false - - name: "depth" - value: i64 - optional: true - keyword_only: true - return: list - - name: "generate_array" - description: "Returns an array of values. The `start_expression` and `end_expression` parameters determine the inclusive start and end of the array." - impls: - # Signature: generate_array:i64_i64_i64 - - args: - - name: "start_expression" - value: i64 - optional: false - keyword_only: false - - name: "end_expression" - value: i64 - optional: false - keyword_only: false - - name: "step_expression" - value: i64 - optional: true - keyword_only: false - return: list - # Signature: generate_array:dec_dec_dec - - args: - - name: "start_expression" - value: decimal<38,9> - optional: false - keyword_only: false - - name: "end_expression" - value: decimal<38,9> - optional: false - keyword_only: false - - name: "step_expression" - value: decimal<38,9> - optional: true - keyword_only: false - return: list> - # Signature: generate_array:fp64_fp64_fp64 - - args: - - name: "start_expression" - value: fp64 - optional: false - keyword_only: false - - name: "end_expression" - value: fp64 - optional: false - keyword_only: false - - name: "step_expression" - value: fp64 - optional: true - keyword_only: false - return: list diff --git a/scripts/data/sql-functions/global_namespace/bit.yaml b/scripts/data/sql-functions/global_namespace/bit.yaml deleted file mode 100644 index fe14eae7b64..00000000000 --- a/scripts/data/sql-functions/global_namespace/bit.yaml +++ /dev/null @@ -1,27 +0,0 @@ -urn: extension:google:bq_scalar_functions -scalar_functions: - - name: "bit_count" - description: "The input, `expression`, must be an integer or `BYTES`. Returns the number of bits that are set in the input expression. For signed integers, this is the number of bits in two's complement form." - series_accessor_arg: expression - impls: - # Signature: bit_count:i32 - - args: - - name: "expression" - value: i32 - optional: false - keyword_only: false - return: i64 - # Signature: bit_count:i64 - - args: - - name: "expression" - value: i64 - optional: false - keyword_only: false - return: i64 - # Signature: bit_count:vbin - - args: - - name: "expression" - value: binary - optional: false - keyword_only: false - return: i64 diff --git a/scripts/data/sql-functions/global_namespace/conversion.yaml b/scripts/data/sql-functions/global_namespace/conversion.yaml deleted file mode 100644 index c39724427de..00000000000 --- a/scripts/data/sql-functions/global_namespace/conversion.yaml +++ /dev/null @@ -1,119 +0,0 @@ -urn: extension:google:bq_scalar_functions -scalar_functions: - - name: "bool" - description: "Converts a JSON boolean to a SQL BOOL value." - series_accessor_arg: json_string_expression - impls: - # Signature: bool:str - - args: - - name: "json_string_expression" - value: string - optional: false - keyword_only: false - return: boolean - - name: "double" - description: "Converts a JSON number to a SQL FLOAT64 value." - series_accessor_arg: json_string_expression - impls: - # Signature: double:str_str - - args: - - name: "json_string_expression" - value: string - optional: false - keyword_only: false - - name: "wide_number_mode" - value: string - optional: true - keyword_only: true - return: fp64 - - name: "float64" - description: "Converts a JSON number to a SQL FLOAT64 value." - series_accessor_arg: json_string_expression - impls: - # Signature: float64:str_str - - args: - - name: "json_string_expression" - value: string - optional: false - keyword_only: false - - name: "wide_number_mode" - value: string - optional: true - keyword_only: true - return: fp64 - - name: "int64" - description: "Converts a JSON number to a SQL INT64 value." - series_accessor_arg: json_string_expression - impls: - # Signature: int64:str - - args: - - name: "json_string_expression" - value: string - optional: false - keyword_only: false - return: i64 - - name: "parse_bignumeric" - description: "Converts a STRING to a BIGNUMERIC value." - series_accessor_arg: string_expression - impls: - # Signature: parse_bignumeric:str - - args: - - name: "string_expression" - value: string - optional: false - keyword_only: false - return: decimal<76,38> - - name: "parse_numeric" - description: "Converts a STRING to a NUMERIC value." - series_accessor_arg: string_expression - impls: - # Signature: parse_numeric:str - - args: - - name: "string_expression" - value: string - optional: false - keyword_only: false - return: decimal<38,9> - - name: "string" - description: "Converts a value to a STRING value." - series_accessor_arg: expression - impls: - # Signature: string:pts_str - - args: - - name: "expression" - value: timestamp - optional: false - keyword_only: false - - name: "timezone" - value: string - optional: true - keyword_only: false - return: string - # Signature: string:date - - args: - - name: "expression" - value: date - optional: false - keyword_only: false - return: string - # Signature: string:pt - - args: - - name: "expression" - value: time - optional: false - keyword_only: false - return: string - # Signature: string:pts - - args: - - name: "expression" - value: timestamp - optional: false - keyword_only: false - return: string - # Signature: string:str - - args: - - name: "expression" - value: string - optional: false - keyword_only: false - return: string diff --git a/scripts/data/sql-functions/global_namespace/date.yaml b/scripts/data/sql-functions/global_namespace/date.yaml deleted file mode 100644 index 8d1dfc95284..00000000000 --- a/scripts/data/sql-functions/global_namespace/date.yaml +++ /dev/null @@ -1,277 +0,0 @@ -urn: extension:google:bq_scalar_functions -scalar_functions: - - name: "current_date" - description: "Returns the current date as a DATE object. Parentheses are optional when called with no arguments." - impls: - # Signature: current_date:str - - args: - - name: "time_zone_expression" - value: string - optional: true - keyword_only: false - return: date - - name: "date" - description: "Constructs or extracts a date." - series_accessor_arg: expression - impls: - # Signature: date:pts_str - - args: - - name: "expression" - value: timestamp - optional: false - keyword_only: false - - name: "time_zone_expression" - value: string - optional: true - keyword_only: false - return: date - # Signature: date:pts - - args: - - name: "expression" - value: timestamp - optional: false - keyword_only: false - return: date - # Signature: date:i64_i64_i64 - - args: - - name: "year" - value: i64 - optional: false - keyword_only: false - - name: "month" - value: i64 - optional: false - keyword_only: false - - name: "day" - value: i64 - optional: false - keyword_only: false - return: date - # Signature: date:date - - args: - - name: "expression" - value: date - optional: false - keyword_only: false - return: date - # Signature: date:str - - args: - - name: "expression" - value: string - optional: false - keyword_only: false - return: date - - name: "date_add" - description: "Adds a specified time interval to a DATE." - series_accessor_arg: date_expression - impls: - # Signature: date_add:date_i64_any - - args: - - name: "date_expression" - value: date - optional: false - keyword_only: false - - name: "int64_expression" - value: i64 - optional: false - keyword_only: false - - name: "date_part" - value: any1 - optional: false - keyword_only: false - return: date - # TODO(b/527093666): add support for date_bucket when we add an INTERVAL dtype - - name: "date_diff" - description: "Gets the number of unit boundaries between two DATE values (end_date - start_date) at a particular time granularity." - series_accessor_arg: end_date - impls: - # Signature: date_diff:date_date_any - - args: - - name: "end_date" - value: date - optional: false - keyword_only: false - - name: "start_date" - value: date - optional: false - keyword_only: false - - name: "granularity" - value: any1 - optional: false - keyword_only: false - return: i64 - - name: "date_from_unix_date" - description: "Interprets an INT64 expression as the number of days since 1970-01-01." - series_accessor_arg: int64_expression - impls: - # Signature: date_from_unix_date:i64 - - args: - - name: "int64_expression" - value: i64 - optional: false - keyword_only: false - return: date - - name: "date_sub" - description: "Subtracts a specified time interval from a DATE." - series_accessor_arg: date_expression - impls: - # Signature: date_sub:date_i64_any - - args: - - name: "date_expression" - value: date - optional: false - keyword_only: false - - name: "int64_expression" - value: i64 - optional: false - keyword_only: false - - name: "date_part" - value: any1 - optional: false - keyword_only: false - return: date - - name: "date_trunc" - description: "Truncates a DATE, DATETIME, or TIMESTAMP value at a particular granularity." - series_accessor_arg: date_value - impls: - # Signature: date_trunc:date_any - - args: - - name: "date_value" - value: date - optional: false - keyword_only: false - - name: "granularity" - value: any1 - optional: false - keyword_only: false - return: date - - name: "extract" - description: "Returns the value corresponding to the specified date part." - series_accessor_arg: date_expression - impls: - # Signature: extract:date_any - - args: - - name: "date_expression" - value: date - optional: false - keyword_only: false - - name: "part" - value: any1 - optional: false - keyword_only: false - return: i64 - # Signature: extract:pts_any_str - - args: - - name: "date_expression" - value: timestamp - optional: false - keyword_only: false - - name: "part" - value: any1 - optional: false - keyword_only: false - - name: "time_zone" - value: string - optional: true - keyword_only: false - return: i64 - # Signature: extract:pts_any - - args: - - name: "date_expression" - value: timestamp - optional: false - keyword_only: false - - name: "part" - value: any1 - optional: false - keyword_only: false - return: i64 - # Signature: extract:pt_any - - args: - - name: "date_expression" - value: time - optional: false - keyword_only: false - - name: "part" - value: any1 - optional: false - keyword_only: false - return: i64 - - name: "format_date" - description: "Formats a DATE value according to a specified format string." - series_accessor_arg: date_expr - impls: - # Signature: format_date:str_date - - args: - - name: "format_string" - value: string - optional: false - keyword_only: false - - name: "date_expr" - value: date - optional: false - keyword_only: false - return: string - - name: "generate_date_array" - description: "Generates an array of dates in a range." - impls: - # Signature: generate_date_array:date_date_i64_any - - args: - - name: "start_date" - value: date - optional: false - keyword_only: false - - name: "end_date" - value: date - optional: false - keyword_only: false - - name: "int64_expression" - value: i64 - optional: true - keyword_only: false - - name: "date_part" - value: any1 - optional: true - keyword_only: false - return: list - - name: "last_day" - description: "Returns the last day from a date expression. This is commonly used to return the last day of the month." - series_accessor_arg: date_expression - impls: - # Signature: last_day:date_any - - args: - - name: "date_expression" - value: date - optional: false - keyword_only: false - - name: "date_part" - value: any1 - optional: true - keyword_only: false - return: date - - name: "parse_date" - description: "Converts a STRING value to a DATE value." - series_accessor_arg: date_string - impls: - # Signature: parse_date:str_str - - args: - - name: "format_string" - value: string - optional: false - keyword_only: false - - name: "date_string" - value: string - optional: false - keyword_only: false - return: date - - name: "unix_date" - description: "Returns the number of days since 1970-01-01." - series_accessor_arg: date_expression - impls: - # Signature: unix_date:date - - args: - - name: "date_expression" - value: date - optional: false - keyword_only: false - return: i64 diff --git a/scripts/decrypt-secrets.sh b/scripts/decrypt-secrets.sh new file mode 100755 index 00000000000..120b0ddc436 --- /dev/null +++ b/scripts/decrypt-secrets.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2024 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT=$( dirname "$DIR" ) + +# Work from the project root. +cd $ROOT + +# Prevent it from overriding files. +# We recommend that sample authors use their own service account files and cloud project. +# In that case, they are supposed to prepare these files by themselves. +if [[ -f "testing/test-env.sh" ]] || \ + [[ -f "testing/service-account.json" ]] || \ + [[ -f "testing/client-secrets.json" ]]; then + echo "One or more target files exist, aborting." + exit 1 +fi + +# Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources. +PROJECT_ID="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}" + +gcloud secrets versions access latest --secret="python-docs-samples-test-env" \ + --project="${PROJECT_ID}" \ + > testing/test-env.sh +gcloud secrets versions access latest \ + --secret="python-docs-samples-service-account" \ + --project="${PROJECT_ID}" \ + > testing/service-account.json +gcloud secrets versions access latest \ + --secret="python-docs-samples-client-secrets" \ + --project="${PROJECT_ID}" \ + > testing/client-secrets.json diff --git a/scripts/dev-utils/tpcds_upload_helper.py b/scripts/dev-utils/tpcds_upload_helper.py index dec5b39768f..52bb553cd81 100644 --- a/scripts/dev-utils/tpcds_upload_helper.py +++ b/scripts/dev-utils/tpcds_upload_helper.py @@ -9,10 +9,11 @@ def preprocess_csv(input_file_path, output_file_path): try: - with ( - open(input_file_path, mode="r", newline="", encoding="utf-8") as infile, - open(output_file_path, mode="w", newline="", encoding="utf-8") as outfile, - ): + with open( + input_file_path, mode="r", newline="", encoding="utf-8" + ) as infile, open( + output_file_path, mode="w", newline="", encoding="utf-8" + ) as outfile: reader = csv.reader(infile, delimiter="|") writer = csv.writer(outfile, delimiter="|") diff --git a/scripts/generate_bigframes_bigquery.py b/scripts/generate_bigframes_bigquery.py deleted file mode 100755 index fe90dc43472..00000000000 --- a/scripts/generate_bigframes_bigquery.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env -S uv run --active --script -# -# /// script -# dependencies = [ -# "jinja2", -# "pyyaml", -# "ruff==0.14.14", -# ] -# /// -# -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pathlib -import sys - -scripts_dir = pathlib.Path(__file__).parent -if str(scripts_dir) not in sys.path: - sys.path.insert(0, str(scripts_dir)) - -from bigquery_generator import constants, file_generator, yaml_parser # noqa: E402 - - -def main() -> None: - modules = [] - - for yaml_file in sorted(constants.DATA_DIR.glob("**/*.yaml")): - modules.append(yaml_parser.parse_yaml(yaml_file)) - - file_generator.generate(modules) - - -if __name__ == "__main__": - main() diff --git a/scripts/manage_cloud_functions.py b/scripts/manage_cloud_functions.py index c92be4ebadb..ccf588bde7c 100644 --- a/scripts/manage_cloud_functions.py +++ b/scripts/manage_cloud_functions.py @@ -67,7 +67,6 @@ def get_bigframes_functions(project, region): functions = GCF_CLIENT.list_functions( functions_v2.ListFunctionsRequest(parent=parent) ) - # Filter bigframes created functions functions = [ function diff --git a/scripts/readme-gen/readme_gen.py b/scripts/readme-gen/readme_gen.py new file mode 100644 index 00000000000..ceb1eada7c2 --- /dev/null +++ b/scripts/readme-gen/readme_gen.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generates READMEs using configuration defined in yaml.""" + +import argparse +import io +import os +import subprocess + +import jinja2 +import yaml + +jinja_env = jinja2.Environment( + trim_blocks=True, + loader=jinja2.FileSystemLoader( + os.path.abspath(os.path.join(os.path.dirname(__file__), "templates")) + ), + autoescape=True, +) + +README_TMPL = jinja_env.get_template("README.tmpl.rst") + + +def get_help(file): + return subprocess.check_output(["python", file, "--help"]).decode() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("source") + parser.add_argument("--destination", default="README.rst") + + args = parser.parse_args() + + source = os.path.abspath(args.source) + root = os.path.dirname(source) + destination = os.path.join(root, args.destination) + + jinja_env.globals["get_help"] = get_help + + with io.open(source, "r") as f: + config = yaml.load(f) + + # This allows get_help to execute in the right directory. + os.chdir(root) + + output = README_TMPL.render(config) + + with io.open(destination, "w") as f: + f.write(output) + + +if __name__ == "__main__": + main() diff --git a/scripts/readme-gen/templates/README.tmpl.rst b/scripts/readme-gen/templates/README.tmpl.rst new file mode 100644 index 00000000000..4fd239765b0 --- /dev/null +++ b/scripts/readme-gen/templates/README.tmpl.rst @@ -0,0 +1,87 @@ +{# The following line is a lie. BUT! Once jinja2 is done with it, it will + become truth! #} +.. This file is automatically generated. Do not edit this file directly. + +{{product.name}} Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/README.rst + + +This directory contains samples for {{product.name}}. {{product.description}} + +{{description}} + +.. _{{product.name}}: {{product.url}} + +{% if required_api_url %} +To run the sample, you need to enable the API at: {{required_api_url}} +{% endif %} + +{% if required_role %} +To run the sample, you need to have `{{required_role}}` role. +{% endif %} + +{{other_required_steps}} + +{% if setup %} +Setup +------------------------------------------------------------------------------- + +{% for section in setup %} + +{% include section + '.tmpl.rst' %} + +{% endfor %} +{% endif %} + +{% if samples %} +Samples +------------------------------------------------------------------------------- + +{% for sample in samples %} +{{sample.name}} ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +{% if not sample.hide_cloudshell_button %} +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/{{sample.file}},{{folder}}/README.rst +{% endif %} + + +{{sample.description}} + +To run this sample: + +.. code-block:: bash + + $ python {{sample.file}} +{% if sample.show_help %} + + {{get_help(sample.file)|indent}} +{% endif %} + + +{% endfor %} +{% endif %} + +{% if cloud_client_library %} + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + +{% endif %} + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/scripts/readme-gen/templates/auth.tmpl.rst b/scripts/readme-gen/templates/auth.tmpl.rst new file mode 100644 index 00000000000..1446b94a5e3 --- /dev/null +++ b/scripts/readme-gen/templates/auth.tmpl.rst @@ -0,0 +1,9 @@ +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started diff --git a/scripts/readme-gen/templates/auth_api_key.tmpl.rst b/scripts/readme-gen/templates/auth_api_key.tmpl.rst new file mode 100644 index 00000000000..11957ce2714 --- /dev/null +++ b/scripts/readme-gen/templates/auth_api_key.tmpl.rst @@ -0,0 +1,14 @@ +Authentication +++++++++++++++ + +Authentication for this service is done via an `API Key`_. To obtain an API +Key: + +1. Open the `Cloud Platform Console`_ +2. Make sure that billing is enabled for your project. +3. From the **Credentials** page, create a new **API Key** or use an existing + one for your project. + +.. _API Key: + https://developers.google.com/api-client-library/python/guide/aaa_apikeys +.. _Cloud Console: https://console.cloud.google.com/project?_ diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst new file mode 100644 index 00000000000..6f069c6c87a --- /dev/null +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -0,0 +1,29 @@ +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 3.7+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ diff --git a/scripts/readme-gen/templates/install_portaudio.tmpl.rst b/scripts/readme-gen/templates/install_portaudio.tmpl.rst new file mode 100644 index 00000000000..5ea33d18c00 --- /dev/null +++ b/scripts/readme-gen/templates/install_portaudio.tmpl.rst @@ -0,0 +1,35 @@ +Install PortAudio ++++++++++++++++++ + +Install `PortAudio`_. This is required by the `PyAudio`_ library to stream +audio from your computer's microphone. PyAudio depends on PortAudio for cross-platform compatibility, and is installed differently depending on the +platform. + +* For Mac OS X, you can use `Homebrew`_:: + + brew install portaudio + + **Note**: if you encounter an error when running `pip install` that indicates + it can't find `portaudio.h`, try running `pip install` with the following + flags:: + + pip install --global-option='build_ext' \ + --global-option='-I/usr/local/include' \ + --global-option='-L/usr/local/lib' \ + pyaudio + +* For Debian / Ubuntu Linux:: + + apt-get install portaudio19-dev python-all-dev + +* Windows may work without having to install PortAudio explicitly (it will get + installed with PyAudio). + +For more details, see the `PyAudio installation`_ page. + + +.. _PyAudio: https://people.csail.mit.edu/hubert/pyaudio/ +.. _PortAudio: http://www.portaudio.com/ +.. _PyAudio installation: + https://people.csail.mit.edu/hubert/pyaudio/#downloads +.. _Homebrew: http://brew.sh diff --git a/scripts/run_doctest.sh b/scripts/run_doctest.sh deleted file mode 100755 index d5fd7256ece..00000000000 --- a/scripts/run_doctest.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -set -eo pipefail - -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 - -# Assume we are running from the repo root or we need to find it. -# If this script is in packages/bigframes/scripts/run_doctest.sh, -# then repo root is 3 levels up. -export PROJECT_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")/../../..") -cd "$PROJECT_ROOT" - -git config --global --add safe.directory "$(realpath .)" - -package_name="bigframes" -package_path="packages/${package_name}" -files_to_check="${package_path}" - -# Use the IF block to handle the case where KOKORO vars are missing -# (e.g. local testing) -if [[ -n "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}" && -n "${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" ]]; then - echo "checking changes with 'git diff ${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT} -- ${files_to_check}'" - - package_modified=$(git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- "${files_to_check}" | wc -l) -else - # If not a PR (like a local run or a different CI trigger), - # we treat it as 0 so it falls through to the "continuous" check. - package_modified=0 -fi - -# Check if modified OR if it's a continuous build -if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then - echo "------------------------------------------------------------" - echo "Running doctest for: ${package_name}" - echo "------------------------------------------------------------" - - # Ensure credentials are set for system tests in Kokoro - if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS}" && -f "${KOKORO_GFILE_DIR}/service-account.json" ]]; then - export GOOGLE_APPLICATION_CREDENTIALS="${KOKORO_GFILE_DIR}/service-account.json" - fi - - export GOOGLE_CLOUD_PROJECT="bigframes-testing" - NOX_SESSION=("cleanup" "doctest") - - cd "${package_path}" - python3 -m nox -s "${NOX_SESSION[@]}" -else - echo "No changes in ${package_name} and not a continuous build, skipping." -fi \ No newline at end of file diff --git a/scripts/templates/bigframes_series_accessor.py.j2 b/scripts/templates/bigframes_series_accessor.py.j2 deleted file mode 100644 index 8ce37d67321..00000000000 --- a/scripts/templates/bigframes_series_accessor.py.j2 +++ /dev/null @@ -1,44 +0,0 @@ -{% include 'license.py.j2' %} - -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated by the script: {{ script_path }} -# - -from __future__ import annotations - -from typing import cast, Optional, TypeVar - -from bigframes.core.logging import log_adapter -from bigframes.extensions.core import series_accessor as core_accessor -from bigframes import series, dataframe, session - -T = TypeVar("T", bound="dataframe.DataFrame") -S = TypeVar("S", bound="series.Series") - - -{% for ns in namespaces %} -@log_adapter.class_logger -class {{ ns.bigframes_class_name }}(core_accessor.{{ ns.class_name }}[T, S]): - def __init__(self, bf_obj: S): - super().__init__(bf_obj) - - def _bf_from_series( - self, session: Optional[session.Session] = None - ) -> series.Series: - return self._obj - - def _to_dataframe(self, bf_df: dataframe.DataFrame) -> T: - return cast(T, bf_df) - - def _to_series(self, bf_series: series.Series) -> S: - return cast(S, bf_series) - - {% for child in ns.children %} - @property - def {{ child.prop_name }}(self) -> {{ child.bigframes_class_name }}[T, S]: - return {{ child.bigframes_class_name }}(self._obj) - - {% endfor %} - -{% endfor %} diff --git a/scripts/templates/core_series_accessor.py.j2 b/scripts/templates/core_series_accessor.py.j2 deleted file mode 100644 index 89decdcbe1e..00000000000 --- a/scripts/templates/core_series_accessor.py.j2 +++ /dev/null @@ -1,81 +0,0 @@ -{% include 'license.py.j2' %} - -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated by the script: {{ script_path }} -# - -from __future__ import annotations - -import abc -import datetime -from typing import ( - Any, - Literal, - Optional, - TypeVar, - Union, - cast, -) - -from bigframes import series, session -from bigframes.core import col, sentinels -from bigframes.extensions.core import abstract_series_accessor, series_mixins - -T = TypeVar("T") -S = TypeVar("S") - - -{% for ns in namespaces %} -{% if ns.class_name == "AiSeriesAccessor" %} -class {{ ns.class_name }}(series_mixins.AIMixin[T, S]): -{% else %} -class {{ ns.class_name }}(abstract_series_accessor.AbstractBigQuerySeriesAccessor[T, S]): -{% endif %} - """{{ ns.description }}""" - - {% for child in ns.children %} - @property - @abc.abstractmethod - def {{ child.prop_name }}(self) -> {{ child.class_name }}[T, S]: - """Accessor for BigQuery {{ child.prop_name }} functions.""" - - {% endfor %} - {% for func in ns.functions %} - def {{ func.name }}( - self, - {% for arg in func.args if arg.name != func.series_accessor_arg %} - {{ arg.name }}: Union[series.Series, col.Expression, {{ arg.type_hint }}]{% if arg.default %} = {{ arg.default }}{% endif %}, - {% endfor %} - *, - session: Optional[session.Session] = None, - ) -> S: - """{{ func.description | indent(8) }}""" - from {{ func.import_module }} import {{ func.name }} as {{ func.name }}_impl - {% if func.args | length > 1 %} - - # Resolve session from other arguments if not passed - if session is None: - from bigframes.core import googlesql - session = googlesql._find_session( - {% for arg in func.args if arg.name != func.series_accessor_arg %} - {{ arg.name }}, - {% endfor %} - ) - {% endif %} - - bf_series = self._bf_from_series(session) - result = {{ func.name }}_impl( - {% for arg in func.args %} - {% if arg.name == func.series_accessor_arg %} - bf_series, - {% else %} - {{ arg.name }}, - {% endif %} - {% endfor %} - ) - return self._to_series(cast(series.Series, result)) - - {% endfor %} - -{% endfor %} diff --git a/scripts/templates/license.py.j2 b/scripts/templates/license.py.j2 deleted file mode 100644 index 58d482ea386..00000000000 --- a/scripts/templates/license.py.j2 +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/scripts/templates/operation.py.j2 b/scripts/templates/operation.py.j2 deleted file mode 100644 index 720d867986e..00000000000 --- a/scripts/templates/operation.py.j2 +++ /dev/null @@ -1,50 +0,0 @@ -{% include 'license.py.j2' %} - -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: {{ yaml_path }} -# by the script: {{ script_path }} - -from __future__ import annotations - -import datetime -import decimal -from typing import Any, Literal, Optional, TypeVar, Union - -from bigframes import dtypes -import bigframes.core.col -import bigframes.core.expression as ex -import bigframes.core.googlesql -import bigframes.core.sentinels as sentinels -from bigframes.operations import googlesql -import bigframes.operations as ops -import bigframes.series as series - -{% for op in ops %} -{% if op.signature_definition %} -{{ op.signature_definition }} - - -{% endif %} -{{ op.internal_name }} = googlesql.GoogleSqlScalarOp( - "{{ op.sql_name }}", - args=({{ op.arg_specs }}), - signature={{ op.signature }}, -) -{% endfor %} -{% for func in functions %} - - -def {{ func.name }}( -{% for arg in func.args %} - {{ arg.name }}: Union[series.Series, bigframes.core.col.Expression, {{ arg.type_hint }}]{% if arg.default %} = {{ arg.default }}{% endif %}, -{% endfor %} -) -> Union[series.Series, bigframes.core.col.Expression]: - """{{ func.description | indent(4) }}""" - return bigframes.core.googlesql.apply_googlesql_scalar_op( - {{ func.op_name }}, -{% for arg in func.args %} - {{ arg.name }}, -{% endfor %} - ) -{% endfor %} diff --git a/scripts/templates/pandas_series_accessor.py.j2 b/scripts/templates/pandas_series_accessor.py.j2 deleted file mode 100644 index 15054665561..00000000000 --- a/scripts/templates/pandas_series_accessor.py.j2 +++ /dev/null @@ -1,53 +0,0 @@ -{% include 'license.py.j2' %} - -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated by the script: {{ script_path }} -# - -from __future__ import annotations - -from typing import cast, Optional, TypeVar - -import pandas -import pandas.api.extensions - -from bigframes import dataframe, series, session -from bigframes.core import global_session as bf_session -from bigframes.core.logging import log_adapter -from bigframes.extensions.core import series_accessor as core_accessor - -T = TypeVar("T", bound="pandas.DataFrame") -S = TypeVar("S", bound="pandas.Series") - - -{% for ns in namespaces %} -{% if ns.is_root %} -@pandas.api.extensions.register_series_accessor("bigquery") -{% endif %} -@log_adapter.class_logger -class {{ ns.pandas_class_name }}(core_accessor.{{ ns.class_name }}[T, S]): - def __init__(self, pandas_obj: S): - super().__init__(pandas_obj) - - def _bf_from_series( - self, session: Optional[session.Session] = None - ) -> series.Series: - if session is None: - session = bf_session.get_global_session() - return cast(series.Series, session.read_pandas(self._obj)) - - def _to_dataframe(self, bf_df: dataframe.DataFrame) -> T: - return cast(T, bf_df.to_pandas(ordered=True)) - - def _to_series(self, bf_series: series.Series) -> S: - return cast(S, bf_series.to_pandas(ordered=True)) - - {% for child in ns.children %} - @property - def {{ child.prop_name }}(self) -> {{ child.pandas_class_name }}[T, S]: - return {{ child.pandas_class_name }}(self._obj) - - {% endfor %} - -{% endfor %} diff --git a/scripts/templates/signature_def.py.j2 b/scripts/templates/signature_def.py.j2 deleted file mode 100644 index b00c95e3383..00000000000 --- a/scripts/templates/signature_def.py.j2 +++ /dev/null @@ -1,76 +0,0 @@ -def {{ func_name }}(*args): - # Pad args with None to match max expected args - args = args + (None,) * ({{ max_args }} - len(args)) - {% for impl in impls %} - # Try matching impl {{ loop.index0 }} - {% if impl.requires_generic_types %} - any1_val = None - {% endif %} - match_ok = True - {% for arg in impl.args %} - {% set idx = loop.index0 %} - if match_ok and args[{{ idx }}] is not None: - {% if arg.value == "any1" %} - if any1_val is not None: - try: - any1_val = dtypes.coerce_to_common(any1_val, args[{{ idx }}]) - except TypeError: - match_ok = False - else: - any1_val = args[{{ idx }}] - {% elif arg.value.startswith("list<") and arg.value.endswith(">") %} - {% set inner_type = arg.value[5:-1] %} - if not dtypes.is_array_like(args[{{ idx }}]): - match_ok = False - else: - inner = dtypes.get_array_inner_type(args[{{ idx }}]) - {% if inner_type == "any1" %} - if any1_val is not None: - try: - any1_val = dtypes.coerce_to_common(any1_val, inner) - except TypeError: - match_ok = False - else: - any1_val = inner - {% else %} - {% set dtype_expr = dtype_map[inner_type] %} - try: - if dtypes.coerce_to_common(inner, {{ dtype_expr }}) != {{ dtype_expr }}: - match_ok = False - except TypeError: - match_ok = False - {% endif %} - {% elif arg.value == "struct" %} - if not dtypes.is_struct_like(args[{{ idx }}]): - match_ok = False - {% else %} - {% set dtype_expr = dtype_map[arg.value] %} - try: - if dtypes.coerce_to_common(args[{{ idx }}], {{ dtype_expr }}) != {{ dtype_expr }}: - match_ok = False - except TypeError: - match_ok = False - {% endif %} - {% endfor %} - if match_ok: - {% set return_type_yaml = impl.return_type %} - {% if return_type_yaml == "any1" %} - return any1_val - {% elif return_type_yaml.startswith("list<") and return_type_yaml.endswith(">") %} - {% set inner_type = return_type_yaml[5:-1] %} - {% if inner_type == "any1" %} - if any1_val is not None: - return dtypes.list_type(any1_val) - else: - return None - {% else %} - {% set dtype_expr = dtype_map[inner_type] %} - return dtypes.list_type({{ dtype_expr }}) - {% endif %} - {% else %} - {% set dtype_expr = dtype_map[return_type_yaml] %} - return {{ dtype_expr }} - {% endif %} - - {% endfor %} - raise TypeError(f"Could not find matching signature for {{ sql_name }} with argument types: {[str(t) for t in args]}") diff --git a/scripts/templates/test_operation.py.j2 b/scripts/templates/test_operation.py.j2 deleted file mode 100644 index 6aee365cded..00000000000 --- a/scripts/templates/test_operation.py.j2 +++ /dev/null @@ -1,44 +0,0 @@ -{% include 'license.py.j2' %} - -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: {{ yaml_path }} -# by the script: {{ script_path }} - -import bigframes.core.col -import bigframes.core.expression as ex -import bigframes.pandas as bpd -import {{ import_path }} as {{ short_name }}_op -import bigframes.bigquery as bbq - - -{% for func in functions %} -def test_{{ func.name }}_expression(): - # Call the function with col() expressions -{% if is_global %} - result = bbq.{{ func.name }}( -{% else %} - result = bbq.{{ short_name }}.{{ func.name }}( -{% endif %} - {% for arg in func.args %} - bpd.col("{{ arg.name }}"), - {% endfor %} - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == {{ short_name }}_op.{{ func.op_name }} - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == {{ func.args | length }} - {% for arg in func.args %} - assert isinstance(expr.inputs[{{ loop.index0 }}], ex.UnboundVariableExpression) - assert expr.inputs[{{ loop.index0 }}].id == "{{ arg.name }}" - {% endfor %} - - -{% endfor %} diff --git a/scripts/test_publish_api_coverage.py b/scripts/test_publish_api_coverage.py index 167cf5917b0..6e366b6854e 100644 --- a/scripts/test_publish_api_coverage.py +++ b/scripts/test_publish_api_coverage.py @@ -15,8 +15,8 @@ import sys import pandas -import pytest from publish_api_coverage import build_api_coverage_table +import pytest pytest.importorskip("sklearn") @@ -31,8 +31,10 @@ def api_coverage_df(): reason="Issues with installing sklearn for this test in python 3.13", ) def test_api_coverage_produces_expected_schema(api_coverage_df): - # Older pandas has different timestamp default precision - pytest.importorskip("pandas", minversion="2.0.0") + if sys.version.split(".")[:2] == ["3", "9"]: + pytest.skip( + "Python 3.9 uses older pandas without good microsecond timestamp support." + ) pandas.testing.assert_series_equal( api_coverage_df.dtypes, @@ -54,8 +56,6 @@ def test_api_coverage_produces_expected_schema(api_coverage_df): "release_version": "string", }, ), - # String dtype behavior not consistent across pandas versions - check_dtype=False, ) diff --git a/scripts/tpch_result_verify.py b/scripts/tpch_result_verify.py new file mode 100644 index 00000000000..0c932f6eac8 --- /dev/null +++ b/scripts/tpch_result_verify.py @@ -0,0 +1,128 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import re + +from google.cloud import bigquery +import pandas as pd +from tqdm import tqdm + +import bigframes + +project_id = "bigframes-dev-perf" +dataset_id = "tpch_0001g" +dataset = { + "line_item_ds": f"bigframes-dev-perf.{dataset_id}.LINEITEM", + "region_ds": f"bigframes-dev-perf.{dataset_id}.REGION", + "nation_ds": f"bigframes-dev-perf.{dataset_id}.NATION", + "supplier_ds": f"bigframes-dev-perf.{dataset_id}.SUPPLIER", + "part_ds": f"bigframes-dev-perf.{dataset_id}.PART", + "part_supp_ds": f"bigframes-dev-perf.{dataset_id}.PARTSUPP", + "customer_ds": f"bigframes-dev-perf.{dataset_id}.CUSTOMER", + "orders_ds": f"bigframes-dev-perf.{dataset_id}.ORDERS", +} + + +def _execute_query(query): + client = bigquery.Client() + job_config = bigquery.QueryJobConfig(use_query_cache=False) + query_job = client.query(query, job_config=job_config) + query_job.result() + df = query_job.to_dataframe() + df.columns = df.columns.str.upper() + return df + + +def _initialize_session(ordered: bool): + context = bigframes.BigQueryOptions( + location="US", ordering_mode="strict" if ordered else "partial" + ) + session = bigframes.Session(context=context) + return session + + +def _verify_result(bigframes_query, sql_result): + exec_globals = {"_initialize_session": _initialize_session} + exec(bigframes_query, exec_globals) + bigframes_result = exec_globals.get("result") + if isinstance(bigframes_result, pd.DataFrame): + pd.testing.assert_frame_equal( + sql_result.reset_index(drop=True), + bigframes_result.reset_index(drop=True), + check_dtype=False, + ) + else: + assert sql_result.shape == (1, 1) + sql_scalar = sql_result.iloc[0, 0] + assert sql_scalar == bigframes_result + + +def verify(query_num=None): + range_iter = range(1, 23) if query_num is None else [query_num] + for i in tqdm(range_iter, desc="Processing queries"): + if query_num is not None and i != query_num: + continue + + # Execute SQL: + sql_file_path = f"third_party/bigframes_vendored/tpch/sql_queries/q{i}.sql" + with open(sql_file_path, "r") as f: + sql_query = f.read() + sql_query = sql_query.format(**dataset) + file_path = f"third_party/bigframes_vendored/tpch/queries/q{i}.py" + if os.path.exists(file_path): + with open(file_path, "r") as file: + file_content = file.read() + + file_content = re.sub( + r"next\((\w+)\.to_pandas_batches\((.*?)\)\)", + r"return \1.to_pandas()", + file_content, + ) + file_content = re.sub(r"_\s*=\s*(\w+)", r"return \1", file_content) + sql_result = _execute_query(sql_query) + + print(f"Checking {file_path} in ordered session") + bigframes_query = ( + file_content + + f"\nresult = q('{project_id}', '{dataset_id}', _initialize_session(ordered=True))" + ) + _verify_result(bigframes_query, sql_result) + + print(f"Checking {file_path} in unordered session") + bigframes_query = ( + file_content + + f"\nresult = q('{project_id}', '{dataset_id}', _initialize_session(ordered=False))" + ) + _verify_result(bigframes_query, sql_result) + + else: + raise FileNotFoundError(f"File {file_path} not found.") + + +if __name__ == "__main__": + """ + Runs verification of TPCH benchmark script outputs to ensure correctness for a specified query or all queries + with 1GB dataset. + + Example: + python scripts/tpch_result_verify.py -q 15 # Verifies TPCH query number 15 + python scripts/tpch_result_verify.py # Verifies all TPCH queries from 1 to 22 + """ + parser = argparse.ArgumentParser() + parser.add_argument("-q", "--query_number", type=int, default=None) + args = parser.parse_args() + + verify(args.query_number) diff --git a/setup.py b/setup.py index e2717fbe5e4..720687952c4 100644 --- a/setup.py +++ b/setup.py @@ -33,33 +33,32 @@ # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 5 - Production/Stable" dependencies = [ - # please keep these in sync with the minimum versions in testing/constraints-3.10.txt + # please keep these in sync with the minimum versions in testing/constraints-3.9.txt "cloudpickle >= 2.0.0", "fsspec >=2023.3.0", - "gcsfs >=2023.3.0, !=2025.5.0, !=2026.2.0, !=2026.3.0", + "gcsfs >=2023.3.0, !=2025.5.0", "geopandas >=0.12.2", - "google-auth[pyopenssl] >=2.15.0,<3.0", + "google-auth >=2.15.0,<3.0", "google-cloud-bigquery[bqstorage,pandas] >=3.36.0", # 2.30 needed for arrow support. "google-cloud-bigquery-storage >= 2.30.0, < 3.0.0", - "google-cloud-functions >=1.20.2", - "google-cloud-bigquery-connection >=1.18.2", - "google-cloud-resource-manager >=1.14.2", + "google-cloud-functions >=1.12.0", + "google-cloud-bigquery-connection >=1.12.0", + "google-cloud-resource-manager >=1.10.3", "google-cloud-storage >=2.0.0", - "google-crc32c >=1.0.0,<2.0.0", "grpc-google-iam-v1 >= 0.14.2", "numpy >=1.24.0", "pandas >=1.5.3", "pandas-gbq >=0.26.1", - "pyarrow >=23.0.1", + "pyarrow >=15.0.2", "pydata-google-auth >=1.8.2", "requests >=2.27.1", "shapely >=1.8.5", "tabulate >=0.9", + "ipywidgets >=7.7.1", "humanize >=4.6.0", "matplotlib >=3.7.1", "db-dtypes >=1.4.2", - "pyiceberg >= 0.7.1", # For vendored ibis-framework. "atpublic>=2.3,<6", "python-dateutil>=2.8.2,<3", @@ -73,9 +72,8 @@ "tests": [ "freezegun", "pytest-snapshot", - "google-cloud-bigtable >=2.30.0", - "google-cloud-pubsub >=2.29.0", - "tzdata", + "google-cloud-bigtable >=2.24.0", + "google-cloud-pubsub >=2.21.4", ], # used for local engine "polars": ["polars >= 1.21.0"], @@ -124,29 +122,23 @@ name=name, version=version_id, description=description, - download_url="https://github.com/googleapis/google-cloud-python/tree/main/packages/bigframes/releases", long_description=readme, long_description_content_type="text/x-rst", author="Google LLC", author_email="bigframes-feedback@google.com", license="Apache 2.0", - url="https://dataframes.bigquery.dev", - project_urls={ - "Source": "https://github.com/googleapis/google-cloud-python/tree/main/packages/bigframes", - "Changelog": "https://dataframes.bigquery.dev/changelog.html", - "Issues": "https://github.com/googleapis/google-cloud-python/tree/main/packages/bigframes/issues", - }, + url="https://github.com/googleapis/python-bigquery-dataframes", classifiers=[ release_status, "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", "Operating System :: OS Independent", "Topic :: Internet", ], @@ -158,7 +150,7 @@ "bigframes_vendored": "third_party/bigframes_vendored", }, packages=packages, - python_requires=">=3.10", + python_requires=">=3.9", include_package_data=True, zip_safe=False, ) diff --git a/specs/bigframes-bigquery-contributing.md b/specs/bigframes-bigquery-contributing.md deleted file mode 100644 index 10931af0755..00000000000 --- a/specs/bigframes-bigquery-contributing.md +++ /dev/null @@ -1,501 +0,0 @@ -# bigframes.bigquery inputs and outputs policies - -The goal of the [bigframes.bigquery -APIs](https://dataframes.bigquery.dev/reference/api/bigframes.bigquery.html#module-bigframes.bigquery) -is to provide the simplest possible mapping from BigQuery (GoogleSQL) -[functions](https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/functions-all) -and -[operations](https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax) -to Python. "Simplest" is somewhat ambiguous though, when it comes to the types -involved and behaviors, so this document aims to expand on that vision with -specific examples. - -## SQL and BigFrames expression types - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    SQL expression type(s) - Python type(s) - Notes - Examples -
    Column expression (usable in a SELECT clause) - - - Both Python Series and column expression should be supported as inputs, - with the output reflecting the users input. Use a TypeVar - rather than directly using union types to make type checking easier. -

    -Special considerations for Series inputs: -

    -If an input and output are both a Series with the same number of rows, make sure -the output Series is implicitly (row identity) alignable with the original -input. In other words, don't generate a table expression. -

    -If there are multiple Series inputs, they should be implicitly aligned if -possible so as not to generate unnecessary table expressions. -

    		Most scalar functions accept one or more column expressions as input. -
    Scalar values - - - Theoretically, we could try to get the type system to help the user - disambiguate between this case and the "Column expression" case, but I think - that's more trouble than it it's worth with regards to the expectations of - Python users. - - -
    Table expression - bpd.DataFrame -

    -All columns are included as normal columns in the input table expression, -including named index columns. If column names aren't unique or contain -characters not compatible with BigQuery flexible column names, raise an error. -

    -Outputs are unordered and unindexed to allow for cleaner mapping with SQL. -

    		Most APIs that take a table expression as input, also output a table - expression with the same number of rows and passing through all unused - columns. - -

    This should be used to pass through any index or ordering columns (as well - as all other columns, if that's the SQL behavior), to allow for easy joining - with the original input DataFrame. -

    		Same number of rows as the input, so we should preserve index and ordering: - - - -

    - Different number of rows in output, so no need to preserve index or ordering. - Default index / ordering should be specified with the Session's - configuration: - -

    - -

    - Possible to have the same number of rows as the input, but joining with the original goes against the purpose of the feature: - -

    - -
    Table name - string (referring to fully-qualified table ID, e.g. project.dataset.table / project.catalog.namespace.table) - Some SQL APIs do not support or have limitations with arbitrary table expressions, instead taking in a table ID, such as TABLESAMPLE expression. -

    -Also, SEARCH and VECTOR_SEARCH, if you want the indexes attached to the table to actually apply. -

    -For outputs, it might be preferable to output a table ID instead of a DataFrame, if the user is explicitly creating a table. For example, to_gbq() returns a string with the table name, which is useful for the case where BigFrame generates the table ID for the user. -

    		All of the items from the "Table expression" row above. APIs that require a table expression, but don't take a table ID can trivially take a table ID through a (SELECT * FROM table) subquery. -

    -Some APIs only take a table ID and not an arbitrary table expression:

    - -
    Aggregated table expression - DataFrameGroupBy - - - -
    Analytic table expression -
      - -
    • DataFrameGroupBy - feasibility TBD -
    • Deferred column Expression with a Window applied.
    - -     		- - -
    Column name (unqualified*) \ - \ -*I've only encountered examples where the table name / table expression is passed in separately. - string, -

    -For cases where the column name is used as an alias and we aren't using named Series: -

    -dict[str, Expression] -

    		Often a table expression input is paired with a column name input, as is the case with the CREATE MODEL and VECTOR_SEARCH APIs -

    -If SQL expects a column name rather than a column expression, do not attempt to change this in Python. For example, don't allow a Series as a substitute for DataFrames + Column name. \ - \ -If the associated table expression is input as a DataFrame, validate that these map cleanly to SQL and raise a ValueError if not. For example: \ -

      - -
    • Duplicate column names (excluding unnamed index columns). -
    • Column names that are some hashable value other than integer (which maps cleanly to a column name) or string. -
    • Any column name containing a punctuation mark that is not allowed by BigQuery flexible column names, such as ! or $.
    - -     		- -
    Literal values - corresponding literal Python value (e.g. int, float, string) - For cases where scalar values are also supported, it should be safe to start with this and then expand to support expressions without a breaking change, as is done in https://github.com/googleapis/google-cloud-python/pull/16606. - Most scalar functions accept one or more literal values as input. -
    Scalar subqueries - Not supported yet, except implicitly in some aggregation use cases. -

    -Would need some sort of bigframes deferred expression that can be tied to a table expression. -

    -(Possibly DataFrame with 1 column?) -

    		- -
    - -## Python policies - -### Naming - -Take the SQL function name, keyword name (used as a function name in Python), or argument name and transform them to lower_snake_case to reflect Python conventions. - -### Internal expressions - -Prefer creating deferred BigFrames expression objects where feasible. For -example, all scalar outputting functions should return a -`bigframes.pandas.Series` or `bigframes.core.col.Expression` that wraps a -`bigframes.core.expression.Expression`. - -Prefer returning a `bigframes.pandas.DataFrame` that wraps a -`bigframes.bigframes.core.bigframe_node.BigFrameNode`. See `from_bq_data_source` in -`bigframes.core.array_value.ArrayValue`, as an example. - -Exceptions to this are cases where the output schema is likely to evolve or -differ in ways that are difficult to model, such as the `ML.PREDICT` SQL -function, where output columns differ based on the model type and support for -model types are frequently added to BigQuery. In these exceptional cases, the -generated query should run immediately and the returned value should wrap the -results. - -### Argument syntax details - -Arguments in Python can be one of: - -* Positional - * Supported by `*args` in Python, but not recommended. Positional arguments in SQL should map to named positional or keyword arguments in Python. -* Positional or keyword - * Required positional arguments should be positional, just like they are in SQL. -* Keyword-only - * All other arguments should be keyword-only. Use `, * ,` Python syntax to achieve this. - -For optional parameters, use an optional sentinel (see: ) and omit the value from the generated SQL if the user doesn't explicitly provide one. This ensures that an explicit NULL / None value can be passed in. - -``` - -from enum import Enum - -class Default(Enum): - token = 0 - -DEFAULT = Default.token - -def spam(*, ham: list[str] | None | Default = DEFAULT): - op_kwargs = {} - - if ham is not DEFAULT: - op_kwargs['ham'] = "prosciutto" - - ... - -``` - -### Scalar operations types policies - -Many operations output a table expression. For these, the output type is always a DataFrame, regardless of the input types. - -For scalar operations, there are three cases to consider when determining the output types: - - - - - - - - - - - - - - - - - - -
    Scalar ops - Input type(s) - Scalar ops - Output type -
    Expression - Expression -
    Series / DataFrame - Series / DataFrame -

    -Preserve ordering and index(es). Join inputs as needed before applying the operation. -

    Mix of Expression and Series / DataFrame - Series / DataFrame -

    -Preserve ordering and index(es). Join inputs as needed before applying the operation. -

    - -## Examples - -### PIVOT SQL operator - -SQL syntax ([docs](https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#pivot_operator)): - -``` -FROM from_item[, ...] pivot_operator - -pivot_operator: - PIVOT( - aggregate_function_call [as_alias][, ...] - FOR input_column - IN ( pivot_column [as_alias][, ...] ) - ) [AS alias] - -as_alias: - [AS] alias - -``` - -SQL example: - -``` -WITH Produce AS ( - SELECT 'Kale' as product, 51 as sales, 'Q1' as quarter, 2020 as year UNION ALL - SELECT 'Kale', 23, 'Q2', 2020 UNION ALL - SELECT 'Kale', 45, 'Q3', 2020 UNION ALL - SELECT 'Kale', 3, 'Q4', 2020 UNION ALL - SELECT 'Kale', 70, 'Q1', 2021 UNION ALL - SELECT 'Kale', 85, 'Q2', 2021 UNION ALL - SELECT 'Apple', 77, 'Q1', 2020 UNION ALL - SELECT 'Apple', 0, 'Q2', 2020 UNION ALL - SELECT 'Apple', 1, 'Q1', 2021) -SELECT * FROM Produce - -/*---------+-------+---------+------+ - | product | sales | quarter | year | - +---------+-------+---------+------| - | Kale | 51 | Q1 | 2020 | - | Kale | 23 | Q2 | 2020 | - | Kale | 45 | Q3 | 2020 | - | Kale | 3 | Q4 | 2020 | - | Kale | 70 | Q1 | 2021 | - | Kale | 85 | Q2 | 2021 | - | Apple | 77 | Q1 | 2020 | - | Apple | 0 | Q2 | 2020 | - | Apple | 1 | Q1 | 2021 | - +---------+-------+---------+------*/ - - -SELECT * FROM - Produce - PIVOT(SUM(sales) FOR quarter IN ('Q1', 'Q2', 'Q3', 'Q4')) - -/*---------+------+----+------+------+------+ - | product | year | Q1 | Q2 | Q3 | Q4 | - +---------+------+----+------+------+------+ - | Apple | 2020 | 77 | 0 | NULL | NULL | - | Apple | 2021 | 1 | NULL | NULL | NULL | - | Kale | 2020 | 51 | 23 | 45 | 3 | - | Kale | 2021 | 70 | 85 | NULL | NULL | - +---------+------+----+------+------+------*/ - -``` - -Python definition: - -``` -def pivot( - table_expression: bpd.DataFrame, - *, - aggregation: Expression | dict[str, Expression], - input_column: str, - pivot_columns: dict[str, float | str | ...] | Sequence[float | str | ...], -) -> bpd.DataFrame: - ... -``` - -Since pivot creates a table expression, we run immediately. - - \ -Python usage: - -``` -pivotted = bbq.pivot( - my_produce_dataframe, - aggregation=bpd.col("sales").sum(), - input_column="quarter", - pivot_columns=["Q1", "Q2", "Q3", "Q4"], -) -``` - -### UNPIVOT SQL operator - -SQL syntax ([docs](https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#unpivot_operator)): - -``` -FROM from_item[, ...] unpivot_operator - -unpivot_operator: - UNPIVOT [ { INCLUDE NULLS | EXCLUDE NULLS } ] ( - { single_column_unpivot | multi_column_unpivot } - ) [unpivot_alias] - -single_column_unpivot: - values_column - FOR name_column - IN (columns_to_unpivot) - -multi_column_unpivot: - values_column_set - FOR name_column - IN (column_sets_to_unpivot) - -values_column_set: - (values_column[, ...]) - -columns_to_unpivot: - unpivot_column [row_value_alias][, ...] - -column_sets_to_unpivot: - (unpivot_column [row_value_alias][, ...]) - -unpivot_alias and row_value_alias: - [AS] alias -``` - -SQL example: - -``` -WITH Produce AS ( - SELECT 'Kale' as product, 51 as Q1, 23 as Q2, 45 as Q3, 3 as Q4 UNION ALL - SELECT 'Apple', 77, 0, 25, 2) - --- SELECT * FROM Produce -/*---------+----+----+----+----+ - | product | Q1 | Q2 | Q3 | Q4 | - +---------+----+----+----+----+ - | Kale | 51 | 23 | 45 | 3 | - | Apple | 77 | 0 | 25 | 2 | - +---------+----+----+----+----*/ - -SELECT * FROM Produce -UNPIVOT(sales FOR quarter IN (Q1, Q2, Q3, Q4)) -- single_column_unpivot - -/*---------+-------+---------+ - | product | sales | quarter | - +---------+-------+---------+ - | Kale | 51 | Q1 | - | Kale | 23 | Q2 | - | Kale | 45 | Q3 | - | Kale | 3 | Q4 | - | Apple | 77 | Q1 | - | Apple | 0 | Q2 | - | Apple | 25 | Q3 | - | Apple | 2 | Q4 | - +---------+-------+---------*/ -``` - -Python definition: - -``` -def unpivot( - table_expression: bpd.DataFrame, - *, - exclude_nulls: bool = True, - values_column: str | Sequence[str], - name_column: str, - columns_to_unpivot: dict[str, str | int] | Sequence[str], -) -> bpd.DataFrame: - ... -``` - -Since unpivot creates a table expression, we run immediately. - - \ -Python usage: - -``` -unpivotted = bbq.unpivot( - my_produce_dataframe, - values_column="sales", - name_column="quarter", - columns_to_unpivot=["Q1", "Q2", "Q3", "Q4"], -) -``` diff --git a/specs/bigframes-bigquery-generator.md b/specs/bigframes-bigquery-generator.md deleted file mode 100644 index 1078bbd05a3..00000000000 --- a/specs/bigframes-bigquery-generator.md +++ /dev/null @@ -1,100 +0,0 @@ -# Code generation for bigframes.bigquery - -This document describes code generation for the `bigframes.bigquery` modules. -For detailed specifications on input and output types, refer to -[Contributing to bigframes.bigquery](./bigframes-bigquery-contributing.md). - -## Overview - -The script at `packages/bigframes/scripts/generate_bigframes_bigquery.py` -generates python submodules for the `bigframes.bigquery` module. When run -without any arguments, it iterates through all yaml files at -`packages/bigframes/scripts/data/sql-functions/**/*.yaml` to generate the code. - -The script also generates a unit test that verifies that the functions have been -included in the `bigframes.bigquery` module, which is important to check, as the -`__init__.py` file requires manual updates. - -## Running the generator - -Since the dependencies for the script differ from that of bigframes -and its test suite, use the self-contained Python script technique described at -https://docs.astral.sh/uv/guides/scripts/ -to automatically manage dependencies using `uv`. Therefore, the header of the -script will look something like: - -```python -#!/usr/bin/env -S uv run --script -# -# /// script -# dependencies = [ -# "jinja2", -# "pyyaml", -# ] -# /// -# -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# ... -``` - -To run the script: - -```bash -cd packages/bigframes -uv run scripts/generate_bigframes_bigquery.py -``` - -To improve reproducibility, we also check in the uv lock file generated by -running `uv lock --script scripts/generate_bigframes_bigquery.py`. - -## Generated code organization - -The `generate_bigframes_bigquery.py` script generates submodules of -`bigframes.bigquery._operations`, with the full path reflecting the organization -of the YAML files. For example, a YAML file at -`packages/bigframes/scripts/data/sql-functions/aead.yaml` corresponds to a -generated Python module at `bigframes.bigquery._operations.aead`. Likewise, -`packages/bigframes/scripts/data/sql-functions/builtins/bit.yaml` corresponds -to the `bigframes.bigquery._operations.builtins.bit` submodule. - -## Generated module implementation - -Each generated module has all functions defined in the YAML file converted to -the equivalent Python definition, including keyword arguments and docstrings. - -### Code generation - -The code will be templated using the jinja2 template engine. This allows -proposed changes to the templated code to be reviewed more easily. - -### Handling optional arguments - -When the user calls a Python function without specifying the optional -argument, that argument is omitted from the SQL text. To allow for explicit -NULL values to be passed in (None in Python), the default value is specified -to be a default sentinel value enum `bigframes.core.sentinels.DEFAULT`. For -example: - -```python -import bigframes.core.sentinels - -def current_date( - time_zone_expression: str | bigframes.core.sentinels.Default = bigframes.core.sentinels.DEFAULT, -): - ... -``` - -### Input and output types - -Refer to the table in -[Contributing to bigframes.bigquery](./bigframes-bigquery-contributing.md). - -### Internal bigframes operator - -Scalar functions should generate an expression using the `GoogleSqlScalarOp`. -This keeps the implementation as scalar SQL functions consistent. - -Aggregate, analytic, and table-valued functions currently require custom ops. As -such, those functions are currently out of scope for this generator. diff --git a/testing/constraints-3.10.txt b/testing/constraints-3.10.txt index 1dcdd64baa0..1695a4806b8 100644 --- a/testing/constraints-3.10.txt +++ b/testing/constraints-3.10.txt @@ -1,124 +1,19 @@ -# Please keep these in sync with the minimum versions in setup.py -cloudpickle==2.0.0 -fsspec==2023.3.0 -gcsfs==2023.3.0 -geopandas==0.12.2 -google-auth==2.15.0 -google-cloud-bigtable==2.30.0 -google-cloud-pubsub==2.29.0 -google-cloud-bigquery==3.36.0 -google-cloud-functions==1.20.2 -google-cloud-bigquery-connection==1.18.2 -google-cloud-iam==2.18.2 -google-cloud-resource-manager==1.14.2 -google-cloud-storage==2.0.0 -grpc-google-iam-v1==0.14.2 -numpy==1.24.0 -pandas==1.5.3 -pandas-gbq==0.26.1 -pyarrow==23.0.1 -pydata-google-auth==1.8.2 -pyiceberg==0.7.1 -requests==2.27.1 -scikit-learn==1.2.2 -shapely==1.8.5 -tabulate==0.9 -humanize==4.6.0 +# When we drop Python 3.9, +# please keep these in sync with the minimum versions in setup.py +google-auth==2.27.0 +ipykernel==5.5.6 +ipython==7.34.0 +notebook==6.5.5 +pandas==2.1.4 +pandas-stubs==2.1.4.231227 +portpicker==1.5.2 +requests==2.32.3 +tornado==6.3.3 +absl-py==1.4.0 +debugpy==1.6.6 +ipywidgets==7.7.1 matplotlib==3.7.1 -db-dtypes==1.4.2 -# For vendored ibis-framework. -atpublic==2.3 -python-dateutil==2.8.2 -pytz==2022.7 -toolz==0.11 -typing-extensions==4.6.1 -rich==12.4.4 -# For anywidget mode -anywidget>=0.9.18 -traitlets==5.0.0 -# constrained dependencies to give pip a helping hand -aiohappyeyeballs==2.6.1 -aiohttp==3.13.3 -aiosignal==1.4.0 -anywidget==0.9.21 -asttokens==3.0.1 -async-timeout==5.0.1 -attrs==25.4.0 -cachetools==5.5.2 -certifi==2026.1.4 -charset-normalizer==2.0.12 -click==8.3.1 -click-plugins==1.1.1.2 -cligj==0.7.2 -comm==0.2.3 -commonmark==0.9.1 -contourpy==1.3.2 -coverage==7.13.3 -cycler==0.12.1 -db-dtypes==1.4.2 -decorator==5.2.1 -exceptiongroup==1.2.2 -executing==2.2.1 -fiona==1.10.1 -fonttools==4.61.1 -freezegun==1.5.5 -frozenlist==1.8.0 -google-api-core==2.29.0 -google-auth-oauthlib==1.2.4 -google-cloud-bigquery-storage==2.36.0 -google-cloud-core==2.5.0 -google-crc32c==1.8.0 -google-resumable-media==2.8.0 -googleapis-common-protos==1.72.0 -grpc-google-iam-v1==0.14.2 -grpcio==1.74.0 -grpcio-status==1.62.3 -idna==3.11 -iniconfig2.3.0 -ipython==8.21.0 -ipython-genutils==0.2.0 -ipywidgets==8.1.8 -jedi==0.19.2 -joblib==1.5.3 -jupyterlab_widgets==3.0.16 -kiwisolver==1.4.9 -matplotlib-inline==0.2.1 -mock==5.2.0 -moc==5.2.0 -multidict==6.7.1 -oauthlib==3.3.1 -packaging==26.0 -parso==0.8.5 -pexpect==4.9.0 -pillow==12.1.0 -pluggy==1.6.0 -prompt_toolkit==3.0.52 -propcache==0.4.1 -proto-plus==1.27.1 -protobuf==6.33.5 -psygnal==0.15.1 -ptyprocess==0.7.0 -pure_eval==0.2.3 -pyasn1==0.6.2 -pyasn1_modules==0.4.2 -Pygments==2.19.2 -pyparsing==3.3.2 -pyproj==3.7.1 -pytest==8.4.2 -pytest-cov==7.0.0 -pytest-snapshot==0.9.0 -pytest-timeout==2.4.0 -python-dateutil==2.8.2 -requests-oauthlib==2.0.0 -rsa==4.9.1 -scipy==1.15.3 -setuptools==80.9.0 -six==1.17.0 -stack-data==0.6.3 -threadpoolctl==3.6.0 -tomli==2.4.0 -urllib3==1.26.20 -wcwidth==0.6.0 -wheel==0.45.1 -widgetsnbextension==4.0.15 -yarl==1.22.0 +psutil==5.9.5 +seaborn==0.13.1 +traitlets==5.7.1 +polars==1.21.0 diff --git a/testing/constraints-3.11.txt b/testing/constraints-3.11.txt index 17854fda96f..831d22b0ff7 100644 --- a/testing/constraints-3.11.txt +++ b/testing/constraints-3.11.txt @@ -133,7 +133,7 @@ fsspec==2025.3.0 future==1.0.0 gast==0.6.0 gcsfs==2025.3.0 -GDAL==3.13.1 +GDAL==3.8.4 gdown==5.2.0 geemap==0.35.3 geocoder==1.38.1 @@ -145,7 +145,7 @@ gitdb==4.0.12 GitPython==3.1.45 glob2==0.7 google==2.0.3 -google-ai-generativelanguage==0.6.17 +google-ai-generativelanguage==0.6.15 google-api-core==2.25.1 google-api-python-client==2.177.0 google-auth==2.38.0 @@ -172,7 +172,7 @@ google-pasta==0.2.0 google-resumable-media==2.7.2 googleapis-common-protos==1.70.0 googledrivedownloader==1.1.0 -gradio==6.15.1 +gradio==5.39.0 gradio_client==1.11.0 graphviz==0.21 greenlet==3.2.3 @@ -180,7 +180,7 @@ groovy==0.1.2 grpc-google-iam-v1==0.14.2 grpc-interceptor==0.15.4 grpcio==1.74.0 -grpcio-status==1.72.1 +grpcio-status==1.71.2 grpclib==0.4.8 gspread==6.2.1 gspread-dataframe==4.0.0 @@ -269,7 +269,7 @@ langchain==0.3.27 langchain-core==0.3.72 langchain-text-splitters==0.3.9 langcodes==3.5.0 -langsmith==0.8.18 +langsmith==0.4.10 language_data==1.3.0 launchpadlib==1.10.16 lazr.restfulclient==0.14.4 @@ -303,7 +303,7 @@ mdit-py-plugins==0.4.2 mdurl==0.1.2 miniKanren==1.0.5 missingno==0.5.2 -mistune==3.3.0 +mistune==3.1.3 mizani==0.13.5 mkl==2025.2.0 ml_dtypes==0.5.3 @@ -311,7 +311,7 @@ mlxtend==0.23.4 more-itertools==10.7.0 moviepy==1.0.3 mpmath==1.3.0 -msgpack==1.2.1 +msgpack==1.1.1 multidict==6.6.3 multipledispatch==1.0.0 multiprocess==0.70.16 @@ -401,14 +401,14 @@ prompt_toolkit==3.0.51 propcache==0.3.2 prophet==1.1.7 proto-plus==1.26.1 -protobuf==6.33.5 +protobuf==5.29.5 psutil==5.9.5 psycopg2==2.9.10 psygnal==0.14.0 ptyprocess==0.7.0 py-cpuinfo==9.0.0 py4j==0.10.9.7 -pyarrow==23.0.1 +pyarrow==18.1.0 pyasn1==0.6.1 pyasn1_modules==0.4.2 pycairo==1.28.0 @@ -444,7 +444,7 @@ pyproj==3.7.1 pyproject_hooks==1.2.0 pyshp==2.3.1 PySocks==1.7.1 -pyspark==3.5.2 +pyspark==3.5.1 pytensor==2.31.7 python-apt==0.0.0 python-box==7.3.2 @@ -506,7 +506,7 @@ sniffio==1.3.1 snowballstemmer==3.0.1 sortedcontainers==2.4.0 soundfile==0.13.1 -soupsieve==2.8.4 +soupsieve==2.7 soxr==0.5.0.post1 spacy==3.8.7 spacy-legacy==3.0.12 @@ -569,7 +569,7 @@ tornado==6.4.2 tqdm==4.67.1 traitlets==5.7.1 traittypes==0.2.1 -transformers==5.5.0 +transformers==4.54.1 treelite==4.4.1 treescope==0.1.9 triton==3.2.0 diff --git a/testing/constraints-3.9.txt b/testing/constraints-3.9.txt new file mode 100644 index 00000000000..9865d3b364a --- /dev/null +++ b/testing/constraints-3.9.txt @@ -0,0 +1,38 @@ +# please keep these in sync with the minimum versions in setup.py +cloudpickle==2.0.0 +fsspec==2023.3.0 +gcsfs==2023.3.0 +geopandas==0.12.2 +google-auth==2.15.0 +google-cloud-bigtable==2.24.0 +google-cloud-pubsub==2.21.4 +google-cloud-bigquery==3.36.0 +google-cloud-functions==1.12.0 +google-cloud-bigquery-connection==1.12.0 +google-cloud-iam==2.12.1 +google-cloud-resource-manager==1.10.3 +google-cloud-storage==2.0.0 +grpc-google-iam-v1==0.14.2 +numpy==1.24.0 +pandas==1.5.3 +pandas-gbq==0.26.1 +pyarrow==15.0.2 +pydata-google-auth==1.8.2 +requests==2.27.1 +scikit-learn==1.2.2 +shapely==1.8.5 +tabulate==0.9 +ipywidgets==7.7.1 +humanize==4.6.0 +matplotlib==3.7.1 +db-dtypes==1.4.2 +# For vendored ibis-framework. +atpublic==2.3 +python-dateutil==2.8.2 +pytz==2022.7 +toolz==0.11 +typing-extensions==4.5.0 +rich==12.4.4 +# For anywidget mode +anywidget>=0.9.18 +traitlets==5.0.0 diff --git a/tests/data/nested_structs.jsonl b/tests/data/nested_structs.jsonl index 97e230c9197..f57214b0b3c 100644 --- a/tests/data/nested_structs.jsonl +++ b/tests/data/nested_structs.jsonl @@ -1,6 +1,2 @@ -{"id": 1, "person": {"name": "Alice", "age": 30, "address": {"city": "New York", "country": "USA"}}, "bool_col": true, "int64_col": "123456789", "float64_col": 1.25, "string_col": "Hello World", "json_col": {"a": 1, "b": [1, 2]}, "date_col": "2026-06-24", "time_col": "12:34:56.789012", "datetime_col": "2026-06-24 12:34:56.789012", "timestamp_col": "2026-06-24T12:34:56.789012Z", "bytes_col": "SGVsbG8=", "numeric_col": "123456.789", "bignumeric_col": "123456.7890123456789", "geography_col": "POINT(30 10)", "duration_col": "1000"} -{"id": 2, "person": {"name": "", "age": -1, "address": {"city": "", "country": ""}}, "bool_col": false, "int64_col": "-9223372036854775808", "float64_col": "-Infinity", "string_col": "", "json_col": {}, "date_col": "0001-01-01", "time_col": "00:00:00", "datetime_col": "0001-01-02 00:00:00", "timestamp_col": "0001-01-02T00:00:00Z", "bytes_col": "", "numeric_col": "-99999999999999999999999999999.999999999", "bignumeric_col": "-99999999999999999999999999999999999999.99999999999999999999999999999999999999", "geography_col": "POINT(0 0)", "duration_col": "-9223372036854775"} -{"id": 3, "person": {"name": "Very Long Name...", "age": 150, "address": {"city": "City", "country": "Country"}}, "bool_col": true, "int64_col": "9223372036854775807", "float64_col": "Infinity", "string_col": "Unicode: 🚀 Spark ✨", "json_col": {"max": true, "nested": {"val": 999}}, "date_col": "9999-12-31", "time_col": "23:59:59.999999", "datetime_col": "9999-12-31 23:59:59.999999", "timestamp_col": "9999-12-31T23:59:59.999999Z", "bytes_col": "dmVyeSBsb25nIGJ5dGVzIHZhbHVl", "numeric_col": "99999999999999999999999999999.999999999", "bignumeric_col": "99999999999999999999999999999999999999.99999999999999999999999999999999999999", "geography_col": "POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))", "duration_col": "9223372036854775"} -{"id": 4, "person": null, "bool_col": null, "int64_col": null, "float64_col": null, "string_col": null, "date_col": null, "time_col": null, "datetime_col": null, "timestamp_col": null, "bytes_col": null, "numeric_col": null, "bignumeric_col": null, "geography_col": null, "duration_col": null} -{"id": 5, "person": {"name": "Bob", "age": 0, "address": null}, "bool_col": false, "int64_col": "0", "float64_col": "NaN", "string_col": "Line 1\nLine 2\n\"Quotes\"", "json_col": [1, "two", null], "date_col": "1970-01-01", "time_col": "12:00:00", "datetime_col": "1970-01-01 12:00:00", "timestamp_col": "1970-01-01T12:00:00Z", "bytes_col": "AA==", "numeric_col": "0", "bignumeric_col": "0", "geography_col": "LINESTRING(0 0, 1 1, 2 2)", "duration_col": "0"} -{"id": 6, "person": null, "bool_col": null, "int64_col": null, "float64_col": null, "string_col": null, "json_col": null, "date_col": null, "time_col": null, "datetime_col": null, "timestamp_col": null, "bytes_col": null, "numeric_col": null, "bignumeric_col": null, "geography_col": null, "duration_col": null} +{"id": 1, "person": {"name": "Alice", "age":30, "address": {"city": "New York", "country": "USA"}}} +{"id": 2, "person": {"name": "Bob", "age":25, "address": {"city": "London", "country": "UK"}}} \ No newline at end of file diff --git a/tests/data/nested_structs_schema.json b/tests/data/nested_structs_schema.json index 06e4a3e5275..6692615ceff 100644 --- a/tests/data/nested_structs_schema.json +++ b/tests/data/nested_structs_schema.json @@ -7,7 +7,6 @@ { "name": "person", "type": "RECORD", - "mode": "NULLABLE", "fields": [ { "name": "name", @@ -22,7 +21,6 @@ { "name": "address", "type": "RECORD", - "mode": "NULLABLE", "fields": [ { "name": "city", @@ -37,76 +35,5 @@ ] } ] - }, - { - "name": "bool_col", - "type": "BOOLEAN", - "mode": "NULLABLE" - }, - { - "name": "int64_col", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "float64_col", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "string_col", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "json_col", - "type": "JSON", - "mode": "NULLABLE" - }, - { - "name": "date_col", - "type": "DATE", - "mode": "NULLABLE" - }, - { - "name": "time_col", - "type": "TIME", - "mode": "NULLABLE" - }, - { - "name": "datetime_col", - "type": "DATETIME", - "mode": "NULLABLE" - }, - { - "name": "timestamp_col", - "type": "TIMESTAMP", - "mode": "NULLABLE" - }, - { - "name": "bytes_col", - "type": "BYTES", - "mode": "NULLABLE" - }, - { - "name": "numeric_col", - "type": "NUMERIC", - "mode": "NULLABLE" - }, - { - "name": "bignumeric_col", - "type": "BIGNUMERIC", - "mode": "NULLABLE" - }, - { - "name": "geography_col", - "type": "GEOGRAPHY", - "mode": "NULLABLE" - }, - { - "name": "duration_col", - "type": "INTEGER", - "mode": "NULLABLE", - "description": "#microseconds" } ] diff --git a/tests/js/package-lock.json b/tests/js/package-lock.json index 241ebd2a8d5..5526e0581e2 100644 --- a/tests/js/package-lock.json +++ b/tests/js/package-lock.json @@ -11,9 +11,9 @@ "devDependencies": { "@babel/preset-env": "^7.24.7", "@testing-library/jest-dom": "^6.4.6", - "jest": "^30.0.0", - "jest-environment-jsdom": "^30.2.0", - "jsdom": "^29.0.0" + "jest": "^29.7.0", + "jest-environment-jsdom": "^29.7.0", + "jsdom": "^24.1.0" } }, "node_modules/@adobe/css-tools": { @@ -37,48 +37,14 @@ "lru-cache": "^10.4.3" } }, - "node_modules/@asamuzakjp/dom-selector": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.1.1.tgz", - "integrity": "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@asamuzakjp/generational-cache": "^1.0.1", - "@asamuzakjp/nwsapi": "^2.3.9", - "bidi-js": "^1.0.3", - "css-tree": "^3.2.1", - "is-potential-custom-element-name": "^1.0.1" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - } - }, - "node_modules/@asamuzakjp/generational-cache": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@asamuzakjp/generational-cache/-/generational-cache-1.0.1.tgz", - "integrity": "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - } - }, - "node_modules/@asamuzakjp/nwsapi": { - "version": "2.3.9", - "resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz", - "integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==", - "dev": true, - "license": "MIT" - }, "node_modules/@babel/code-frame": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", - "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.28.5", + "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" }, @@ -94,9 +60,9 @@ "license": "MIT" }, "node_modules/@babel/compat-data": { - "version": "7.29.3", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.3.tgz", - "integrity": "sha512-LIVqM46zQWZhj17qA8wb4nW/ixr2y1Nw+r1etiAWgRM6U1IqP+LNhL1yg440jYZR72jCWcWbLWzIosH+uP1fqg==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.5.tgz", + "integrity": "sha512-6uFXyCayocRbqhZOB+6XcuZbkMNimwfVGFji8CTZnCzOHVGvDqzvitu1re2AU5LROliz7eQPhB8CpAMvnx9EjA==", "dev": true, "license": "MIT", "engines": { @@ -145,14 +111,14 @@ } }, "node_modules/@babel/generator": { - "version": "7.29.1", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.1.tgz", - "integrity": "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.5.tgz", + "integrity": "sha512-3EwLFhZ38J4VyIP6WNtt2kUdW9dokXA9Cr4IVIFHuCpZ3H8/YFOl5JjZHisrn1fATPBmKKqXzDFvh9fUwHz6CQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.29.0", - "@babel/types": "^7.29.0", + "@babel/parser": "^7.28.5", + "@babel/types": "^7.28.5", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" @@ -175,13 +141,13 @@ } }, "node_modules/@babel/helper-compilation-targets": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", - "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", + "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/compat-data": "^7.28.6", + "@babel/compat-data": "^7.27.2", "@babel/helper-validator-option": "^7.27.1", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", @@ -212,18 +178,18 @@ } }, "node_modules/@babel/helper-create-class-features-plugin": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.28.6.tgz", - "integrity": "sha512-dTOdvsjnG3xNT9Y0AUg1wAl38y+4Rl4sf9caSQZOXdNqVn+H+HbbJ4IyyHaIqNR6SW9oJpA/RuRjsjCw2IdIow==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.28.5.tgz", + "integrity": "sha512-q3WC4JfdODypvxArsJQROfupPBq9+lMwjKq7C33GhbFYJsufD0yd/ziwD+hJucLeWsnFPWZjsU2DNFqBPE7jwQ==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-annotate-as-pure": "^7.27.3", "@babel/helper-member-expression-to-functions": "^7.28.5", "@babel/helper-optimise-call-expression": "^7.27.1", - "@babel/helper-replace-supers": "^7.28.6", + "@babel/helper-replace-supers": "^7.27.1", "@babel/helper-skip-transparent-expression-wrappers": "^7.27.1", - "@babel/traverse": "^7.28.6", + "@babel/traverse": "^7.28.5", "semver": "^6.3.1" }, "engines": { @@ -272,17 +238,17 @@ } }, "node_modules/@babel/helper-define-polyfill-provider": { - "version": "0.6.8", - "resolved": "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.6.8.tgz", - "integrity": "sha512-47UwBLPpQi1NoWzLuHNjRoHlYXMwIJoBf7MFou6viC/sIHWYygpvr0B6IAyh5sBdA2nr2LPIRww8lfaUVQINBA==", + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.6.5.tgz", + "integrity": "sha512-uJnGFcPsWQK8fvjgGP5LZUZZsYGIoPeRjSF5PGwrelYgq7Q15/Ft9NGFp1zglwgIv//W0uG4BevRuSJRyylZPg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-compilation-targets": "^7.28.6", - "@babel/helper-plugin-utils": "^7.28.6", - "debug": "^4.4.3", + "@babel/helper-compilation-targets": "^7.27.2", + "@babel/helper-plugin-utils": "^7.27.1", + "debug": "^4.4.1", "lodash.debounce": "^4.0.8", - "resolve": "^1.22.11" + "resolve": "^1.22.10" }, "peerDependencies": { "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" @@ -313,29 +279,29 @@ } }, "node_modules/@babel/helper-module-imports": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", - "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", + "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", "dev": true, "license": "MIT", "dependencies": { - "@babel/traverse": "^7.28.6", - "@babel/types": "^7.28.6" + "@babel/traverse": "^7.27.1", + "@babel/types": "^7.27.1" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-transforms": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", - "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.3.tgz", + "integrity": "sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-module-imports": "^7.28.6", - "@babel/helper-validator-identifier": "^7.28.5", - "@babel/traverse": "^7.28.6" + "@babel/helper-module-imports": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1", + "@babel/traverse": "^7.28.3" }, "engines": { "node": ">=6.9.0" @@ -358,9 +324,9 @@ } }, "node_modules/@babel/helper-plugin-utils": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.28.6.tgz", - "integrity": "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.27.1.tgz", + "integrity": "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==", "dev": true, "license": "MIT", "engines": { @@ -386,15 +352,15 @@ } }, "node_modules/@babel/helper-replace-supers": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.28.6.tgz", - "integrity": "sha512-mq8e+laIk94/yFec3DxSjCRD2Z0TAjhVbEJY3UQrlwVo15Lmt7C2wAUbK4bjnTs4APkwsYLTahXRraQXhb1WCg==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.27.1.tgz", + "integrity": "sha512-7EHz6qDZc8RYS5ElPoShMheWvEgERonFCs7IAonWLLUTXW59DP14bCZt89/GKyreYn8g3S83m21FelHKbeDCKA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-member-expression-to-functions": "^7.28.5", + "@babel/helper-member-expression-to-functions": "^7.27.1", "@babel/helper-optimise-call-expression": "^7.27.1", - "@babel/traverse": "^7.28.6" + "@babel/traverse": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -477,13 +443,13 @@ } }, "node_modules/@babel/parser": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", - "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.5.tgz", + "integrity": "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.29.0" + "@babel/types": "^7.28.5" }, "bin": { "parser": "bin/babel-parser.js" @@ -541,23 +507,6 @@ "@babel/core": "^7.0.0" } }, - "node_modules/@babel/plugin-bugfix-safari-rest-destructuring-rhs-array": { - "version": "7.29.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-rest-destructuring-rhs-array/-/plugin-bugfix-safari-rest-destructuring-rhs-array-7.29.3.tgz", - "integrity": "sha512-SRS46DFR4HqzUzCVgi90/xMoL+zeBDBvWdKYXSEzh79kXswNFEglUpMKxR04//dPqwYXWUBJ3mpUd933ru9Kmg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6", - "@babel/helper-skip-transparent-expression-wrappers": "^7.27.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, "node_modules/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.27.1.tgz", @@ -577,14 +526,14 @@ } }, "node_modules/@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly/-/plugin-bugfix-v8-static-class-fields-redefine-readonly-7.28.6.tgz", - "integrity": "sha512-a0aBScVTlNaiUe35UtfxAN7A/tehvvG4/ByO6+46VPKTRSlfnAFsgKy0FUh+qAkQrDTmhDkT+IBOKlOoMUxQ0g==", + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly/-/plugin-bugfix-v8-static-class-fields-redefine-readonly-7.28.3.tgz", + "integrity": "sha512-b6YTX108evsvE4YgWyQ921ZAFFQm3Bn+CA3+ZXlNVnPhx+UfsVURoPjfGAPCjBgrqo30yX/C2nZGX96DxvR9Iw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6", - "@babel/traverse": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1", + "@babel/traverse": "^7.28.3" }, "engines": { "node": ">=6.9.0" @@ -662,13 +611,13 @@ } }, "node_modules/@babel/plugin-syntax-import-assertions": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.28.6.tgz", - "integrity": "sha512-pSJUpFHdx9z5nqTSirOCMtYVP2wFgoWhP0p3g8ONK/4IHhLIBd0B9NYqAvIUAhq+OkhO4VM1tENCt0cjlsNShw==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.27.1.tgz", + "integrity": "sha512-UT/Jrhw57xg4ILHLFnzFpPDlMbcdEicaAtjPQpbj9wa8T4r5KVWCimHcL/460g8Ht0DMxDyjsLgiWSkVjnwPFg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -678,13 +627,13 @@ } }, "node_modules/@babel/plugin-syntax-import-attributes": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.28.6.tgz", - "integrity": "sha512-jiLC0ma9XkQT3TKJ9uYvlakm66Pamywo+qwL+oL8HJOvc6TWdZXVfhqJr8CCzbSGUAbDOzlGHJC1U+vRfLQDvw==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.27.1.tgz", + "integrity": "sha512-oFT0FrKHgF53f4vOsZGi2Hh3I35PfSmVs4IBFLFj4dnafP+hIWDLg3VyKmUHfLoLHlyxY4C7DGtmHuJgn+IGww==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -720,13 +669,13 @@ } }, "node_modules/@babel/plugin-syntax-jsx": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.28.6.tgz", - "integrity": "sha512-wgEmr06G6sIpqr8YDwA2dSRTE3bJ+V0IfpzfSY3Lfgd7YWOaAdlykvJi13ZKBt8cZHfgH1IXN+CL656W3uUa4w==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.27.1.tgz", + "integrity": "sha512-y8YTNIeKoyhGd9O0Jiyzyyqk8gdjnumGTQPsz0xOZOQ2RmkVJeZ1vmmfIvFEKqucBG6axJGBZDE/7iI5suUI/w==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -846,13 +795,13 @@ } }, "node_modules/@babel/plugin-syntax-typescript": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.28.6.tgz", - "integrity": "sha512-+nDNmQye7nlnuuHDboPbGm00Vqg3oO8niRRL27/4LYHUsHYh0zJ1xWOz0uRwNFmM1Avzk8wZbc6rdiYhomzv/A==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.27.1.tgz", + "integrity": "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -895,15 +844,15 @@ } }, "node_modules/@babel/plugin-transform-async-generator-functions": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-generator-functions/-/plugin-transform-async-generator-functions-7.29.0.tgz", - "integrity": "sha512-va0VdWro4zlBr2JsXC+ofCPB2iG12wPtVGTWFx2WLDOM3nYQZZIGP82qku2eW/JR83sD+k2k+CsNtyEbUqhU6w==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-generator-functions/-/plugin-transform-async-generator-functions-7.28.0.tgz", + "integrity": "sha512-BEOdvX4+M765icNPZeidyADIvQ1m1gmunXufXxvRESy/jNNyfovIqUyE7MVgGBjWktCoJlzvFA1To2O4ymIO3Q==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6", + "@babel/helper-plugin-utils": "^7.27.1", "@babel/helper-remap-async-to-generator": "^7.27.1", - "@babel/traverse": "^7.29.0" + "@babel/traverse": "^7.28.0" }, "engines": { "node": ">=6.9.0" @@ -913,14 +862,14 @@ } }, "node_modules/@babel/plugin-transform-async-to-generator": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.28.6.tgz", - "integrity": "sha512-ilTRcmbuXjsMmcZ3HASTe4caH5Tpo93PkTxF9oG2VZsSWsahydmcEHhix9Ik122RcTnZnUzPbmux4wh1swfv7g==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.27.1.tgz", + "integrity": "sha512-NREkZsZVJS4xmTr8qzE5y8AfIPqsdQfRuUiLRTEzb7Qii8iFWCyDKaUV2c0rCuh4ljDZ98ALHP/PetiBV2nddA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-module-imports": "^7.28.6", - "@babel/helper-plugin-utils": "^7.28.6", + "@babel/helper-module-imports": "^7.27.1", + "@babel/helper-plugin-utils": "^7.27.1", "@babel/helper-remap-async-to-generator": "^7.27.1" }, "engines": { @@ -947,13 +896,13 @@ } }, "node_modules/@babel/plugin-transform-block-scoping": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.28.6.tgz", - "integrity": "sha512-tt/7wOtBmwHPNMPu7ax4pdPz6shjFrmHDghvNC+FG9Qvj7D6mJcoRQIF5dy4njmxR941l6rgtvfSB2zX3VlUIw==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.28.5.tgz", + "integrity": "sha512-45DmULpySVvmq9Pj3X9B+62Xe+DJGov27QravQJU1LLcapR6/10i+gYVAucGGJpHBp5mYxIMK4nDAT/QDLr47g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -963,14 +912,14 @@ } }, "node_modules/@babel/plugin-transform-class-properties": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-class-properties/-/plugin-transform-class-properties-7.28.6.tgz", - "integrity": "sha512-dY2wS3I2G7D697VHndN91TJr8/AAfXQNt5ynCTI/MpxMsSzHp+52uNivYT5wCPax3whc47DR8Ba7cmlQMg24bw==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-class-properties/-/plugin-transform-class-properties-7.27.1.tgz", + "integrity": "sha512-D0VcalChDMtuRvJIu3U/fwWjf8ZMykz5iZsg77Nuj821vCKI3zCyRLwRdWbsuJ/uRwZhZ002QtCqIkwC/ZkvbA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-create-class-features-plugin": "^7.28.6", - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-create-class-features-plugin": "^7.27.1", + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -980,14 +929,14 @@ } }, "node_modules/@babel/plugin-transform-class-static-block": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-class-static-block/-/plugin-transform-class-static-block-7.28.6.tgz", - "integrity": "sha512-rfQ++ghVwTWTqQ7w8qyDxL1XGihjBss4CmTgGRCTAC9RIbhVpyp4fOeZtta0Lbf+dTNIVJer6ych2ibHwkZqsQ==", + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-class-static-block/-/plugin-transform-class-static-block-7.28.3.tgz", + "integrity": "sha512-LtPXlBbRoc4Njl/oh1CeD/3jC+atytbnf/UqLoqTDcEYGUPj022+rvfkbDYieUrSj3CaV4yHDByPE+T2HwfsJg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-create-class-features-plugin": "^7.28.6", - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-create-class-features-plugin": "^7.28.3", + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -997,18 +946,18 @@ } }, "node_modules/@babel/plugin-transform-classes": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.28.6.tgz", - "integrity": "sha512-EF5KONAqC5zAqT783iMGuM2ZtmEBy+mJMOKl2BCvPZ2lVrwvXnB6o+OBWCS+CoeCCpVRF2sA2RBKUxvT8tQT5Q==", + "version": "7.28.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.28.4.tgz", + "integrity": "sha512-cFOlhIYPBv/iBoc+KS3M6et2XPtbT2HiCRfBXWtfpc9OAyostldxIf9YAYB6ypURBBbx+Qv6nyrLzASfJe+hBA==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-annotate-as-pure": "^7.27.3", - "@babel/helper-compilation-targets": "^7.28.6", + "@babel/helper-compilation-targets": "^7.27.2", "@babel/helper-globals": "^7.28.0", - "@babel/helper-plugin-utils": "^7.28.6", - "@babel/helper-replace-supers": "^7.28.6", - "@babel/traverse": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1", + "@babel/helper-replace-supers": "^7.27.1", + "@babel/traverse": "^7.28.4" }, "engines": { "node": ">=6.9.0" @@ -1018,14 +967,14 @@ } }, "node_modules/@babel/plugin-transform-computed-properties": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.28.6.tgz", - "integrity": "sha512-bcc3k0ijhHbc2lEfpFHgx7eYw9KNXqOerKWfzbxEHUGKnS3sz9C4CNL9OiFN1297bDNfUiSO7DaLzbvHQQQ1BQ==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.27.1.tgz", + "integrity": "sha512-lj9PGWvMTVksbWiDT2tW68zGS/cyo4AkZ/QTp0sQT0mjPopCmrSkzxeXkznjqBxzDI6TclZhOJbBmbBLjuOZUw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6", - "@babel/template": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1", + "@babel/template": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1052,14 +1001,14 @@ } }, "node_modules/@babel/plugin-transform-dotall-regex": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.28.6.tgz", - "integrity": "sha512-SljjowuNKB7q5Oayv4FoPzeB74g3QgLt8IVJw9ADvWy3QnUb/01aw8I4AVv8wYnPvQz2GDDZ/g3GhcNyDBI4Bg==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.27.1.tgz", + "integrity": "sha512-gEbkDVGRvjj7+T1ivxrfgygpT7GUd4vmODtYpbs0gZATdkX8/iSnOtZSxiZnsgm1YjTgjI6VKBGSJJevkrclzw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-create-regexp-features-plugin": "^7.28.5", - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-create-regexp-features-plugin": "^7.27.1", + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1085,14 +1034,14 @@ } }, "node_modules/@babel/plugin-transform-duplicate-named-capturing-groups-regex": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-named-capturing-groups-regex/-/plugin-transform-duplicate-named-capturing-groups-regex-7.29.0.tgz", - "integrity": "sha512-zBPcW2lFGxdiD8PUnPwJjag2J9otbcLQzvbiOzDxpYXyCuYX9agOwMPGn1prVH0a4qzhCKu24rlH4c1f7yA8rw==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-named-capturing-groups-regex/-/plugin-transform-duplicate-named-capturing-groups-regex-7.27.1.tgz", + "integrity": "sha512-hkGcueTEzuhB30B3eJCbCYeCaaEQOmQR0AdvzpD4LoN0GXMWzzGSuRrxR2xTnCrvNbVwK9N6/jQ92GSLfiZWoQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-create-regexp-features-plugin": "^7.28.5", - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-create-regexp-features-plugin": "^7.27.1", + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1118,14 +1067,14 @@ } }, "node_modules/@babel/plugin-transform-explicit-resource-management": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-explicit-resource-management/-/plugin-transform-explicit-resource-management-7.28.6.tgz", - "integrity": "sha512-Iao5Konzx2b6g7EPqTy40UZbcdXE126tTxVFr/nAIj+WItNxjKSYTEw3RC+A2/ZetmdJsgueL1KhaMCQHkLPIg==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-explicit-resource-management/-/plugin-transform-explicit-resource-management-7.28.0.tgz", + "integrity": "sha512-K8nhUcn3f6iB+P3gwCv/no7OdzOZQcKchW6N389V6PD8NUWKZHzndOd9sPDVbMoBsbmjMqlB4L9fm+fEFNVlwQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6", - "@babel/plugin-transform-destructuring": "^7.28.5" + "@babel/helper-plugin-utils": "^7.27.1", + "@babel/plugin-transform-destructuring": "^7.28.0" }, "engines": { "node": ">=6.9.0" @@ -1135,13 +1084,13 @@ } }, "node_modules/@babel/plugin-transform-exponentiation-operator": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.28.6.tgz", - "integrity": "sha512-WitabqiGjV/vJ0aPOLSFfNY1u9U3R7W36B03r5I2KoNix+a3sOhJ3pKFB3R5It9/UiK78NiO0KE9P21cMhlPkw==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.28.5.tgz", + "integrity": "sha512-D4WIMaFtwa2NizOp+dnoFjRez/ClKiC2BqqImwKd1X28nqBtZEyCYJ2ozQrrzlxAFrcrjxo39S6khe9RNDlGzw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1202,13 +1151,13 @@ } }, "node_modules/@babel/plugin-transform-json-strings": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-json-strings/-/plugin-transform-json-strings-7.28.6.tgz", - "integrity": "sha512-Nr+hEN+0geQkzhbdgQVPoqr47lZbm+5fCUmO70722xJZd0Mvb59+33QLImGj6F+DkK3xgDi1YVysP8whD6FQAw==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-json-strings/-/plugin-transform-json-strings-7.27.1.tgz", + "integrity": "sha512-6WVLVJiTjqcQauBhn1LkICsR2H+zm62I3h9faTDKt1qP4jn2o72tSvqMwtGFKGTpojce0gJs+76eZ2uCHRZh0Q==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1234,13 +1183,13 @@ } }, "node_modules/@babel/plugin-transform-logical-assignment-operators": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-logical-assignment-operators/-/plugin-transform-logical-assignment-operators-7.28.6.tgz", - "integrity": "sha512-+anKKair6gpi8VsM/95kmomGNMD0eLz1NQ8+Pfw5sAwWH9fGYXT50E55ZpV0pHUHWf6IUTWPM+f/7AAff+wr9A==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-logical-assignment-operators/-/plugin-transform-logical-assignment-operators-7.28.5.tgz", + "integrity": "sha512-axUuqnUTBuXyHGcJEVVh9pORaN6wC5bYfE7FGzPiaWa3syib9m7g+/IT/4VgCOe2Upef43PHzeAvcrVek6QuuA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1283,14 +1232,14 @@ } }, "node_modules/@babel/plugin-transform-modules-commonjs": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.28.6.tgz", - "integrity": "sha512-jppVbf8IV9iWWwWTQIxJMAJCWBuuKx71475wHwYytrRGQ2CWiDvYlADQno3tcYpS/T2UUWFQp3nVtYfK/YBQrA==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.27.1.tgz", + "integrity": "sha512-OJguuwlTYlN0gBZFRPqwOGNWssZjfIUdS7HMYtN8c1KmwpwHFBwTeFZrg9XZa+DFTitWOW5iTAG7tyCUPsCCyw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-module-transforms": "^7.28.6", - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-module-transforms": "^7.27.1", + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1300,16 +1249,16 @@ } }, "node_modules/@babel/plugin-transform-modules-systemjs": { - "version": "7.29.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.29.4.tgz", - "integrity": "sha512-N7QmZ0xRZfjHOfZeQLJjwgX2zS9pdGHSVl/cjSGlo4dXMqvurfxXDMKY4RqEKzPozV78VMcd0lxyG13mlbKc4w==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.28.5.tgz", + "integrity": "sha512-vn5Jma98LCOeBy/KpeQhXcV2WZgaRUtjwQmjoBuLNlOmkg0fB5pdvYVeWRYI69wWKwK2cD1QbMiUQnoujWvrew==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-module-transforms": "^7.28.6", - "@babel/helper-plugin-utils": "^7.28.6", + "@babel/helper-module-transforms": "^7.28.3", + "@babel/helper-plugin-utils": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5", - "@babel/traverse": "^7.29.0" + "@babel/traverse": "^7.28.5" }, "engines": { "node": ">=6.9.0" @@ -1336,14 +1285,14 @@ } }, "node_modules/@babel/plugin-transform-named-capturing-groups-regex": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.29.0.tgz", - "integrity": "sha512-1CZQA5KNAD6ZYQLPw7oi5ewtDNxH/2vuCh+6SmvgDfhumForvs8a1o9n0UrEoBD8HU4djO2yWngTQlXl1NDVEQ==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.27.1.tgz", + "integrity": "sha512-SstR5JYy8ddZvD6MhV0tM/j16Qds4mIpJTOd1Yu9J9pJjH93bxHECF7pgtc28XvkzTD6Pxcm/0Z73Hvk7kb3Ng==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-create-regexp-features-plugin": "^7.28.5", - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-create-regexp-features-plugin": "^7.27.1", + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1369,13 +1318,13 @@ } }, "node_modules/@babel/plugin-transform-nullish-coalescing-operator": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-nullish-coalescing-operator/-/plugin-transform-nullish-coalescing-operator-7.28.6.tgz", - "integrity": "sha512-3wKbRgmzYbw24mDJXT7N+ADXw8BC/imU9yo9c9X9NKaLF1fW+e5H1U5QjMUBe4Qo4Ox/o++IyUkl1sVCLgevKg==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-nullish-coalescing-operator/-/plugin-transform-nullish-coalescing-operator-7.27.1.tgz", + "integrity": "sha512-aGZh6xMo6q9vq1JGcw58lZ1Z0+i0xB2x0XaauNIUXd6O1xXc3RwoWEBlsTQrY4KQ9Jf0s5rgD6SiNkaUdJegTA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1385,13 +1334,13 @@ } }, "node_modules/@babel/plugin-transform-numeric-separator": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-numeric-separator/-/plugin-transform-numeric-separator-7.28.6.tgz", - "integrity": "sha512-SJR8hPynj8outz+SlStQSwvziMN4+Bq99it4tMIf5/Caq+3iOc0JtKyse8puvyXkk3eFRIA5ID/XfunGgO5i6w==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-numeric-separator/-/plugin-transform-numeric-separator-7.27.1.tgz", + "integrity": "sha512-fdPKAcujuvEChxDBJ5c+0BTaS6revLV7CJL08e4m3de8qJfNIuCc2nc7XJYOjBoTMJeqSmwXJ0ypE14RCjLwaw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1401,17 +1350,17 @@ } }, "node_modules/@babel/plugin-transform-object-rest-spread": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-rest-spread/-/plugin-transform-object-rest-spread-7.28.6.tgz", - "integrity": "sha512-5rh+JR4JBC4pGkXLAcYdLHZjXudVxWMXbB6u6+E9lRL5TrGVbHt1TjxGbZ8CkmYw9zjkB7jutzOROArsqtncEA==", + "version": "7.28.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-rest-spread/-/plugin-transform-object-rest-spread-7.28.4.tgz", + "integrity": "sha512-373KA2HQzKhQCYiRVIRr+3MjpCObqzDlyrM6u4I201wL8Mp2wHf7uB8GhDwis03k2ti8Zr65Zyyqs1xOxUF/Ew==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-compilation-targets": "^7.28.6", - "@babel/helper-plugin-utils": "^7.28.6", - "@babel/plugin-transform-destructuring": "^7.28.5", + "@babel/helper-compilation-targets": "^7.27.2", + "@babel/helper-plugin-utils": "^7.27.1", + "@babel/plugin-transform-destructuring": "^7.28.0", "@babel/plugin-transform-parameters": "^7.27.7", - "@babel/traverse": "^7.28.6" + "@babel/traverse": "^7.28.4" }, "engines": { "node": ">=6.9.0" @@ -1438,13 +1387,13 @@ } }, "node_modules/@babel/plugin-transform-optional-catch-binding": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-optional-catch-binding/-/plugin-transform-optional-catch-binding-7.28.6.tgz", - "integrity": "sha512-R8ja/Pyrv0OGAvAXQhSTmWyPJPml+0TMqXlO5w+AsMEiwb2fg3WkOvob7UxFSL3OIttFSGSRFKQsOhJ/X6HQdQ==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-optional-catch-binding/-/plugin-transform-optional-catch-binding-7.27.1.tgz", + "integrity": "sha512-txEAEKzYrHEX4xSZN4kJ+OfKXFVSWKB2ZxM9dpcE3wT7smwkNmXo5ORRlVzMVdJbD+Q8ILTgSD7959uj+3Dm3Q==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1454,13 +1403,13 @@ } }, "node_modules/@babel/plugin-transform-optional-chaining": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-optional-chaining/-/plugin-transform-optional-chaining-7.28.6.tgz", - "integrity": "sha512-A4zobikRGJTsX9uqVFdafzGkqD30t26ck2LmOzAuLL8b2x6k3TIqRiT2xVvA9fNmFeTX484VpsdgmKNA0bS23w==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-optional-chaining/-/plugin-transform-optional-chaining-7.28.5.tgz", + "integrity": "sha512-N6fut9IZlPnjPwgiQkXNhb+cT8wQKFlJNqcZkWlcTqkcqx6/kU4ynGmLFoa4LViBSirn05YAwk+sQBbPfxtYzQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6", + "@babel/helper-plugin-utils": "^7.27.1", "@babel/helper-skip-transparent-expression-wrappers": "^7.27.1" }, "engines": { @@ -1487,14 +1436,14 @@ } }, "node_modules/@babel/plugin-transform-private-methods": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-private-methods/-/plugin-transform-private-methods-7.28.6.tgz", - "integrity": "sha512-piiuapX9CRv7+0st8lmuUlRSmX6mBcVeNQ1b4AYzJxfCMuBfB0vBXDiGSmm03pKJw1v6cZ8KSeM+oUnM6yAExg==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-private-methods/-/plugin-transform-private-methods-7.27.1.tgz", + "integrity": "sha512-10FVt+X55AjRAYI9BrdISN9/AQWHqldOeZDUoLyif1Kn05a56xVBXb8ZouL8pZ9jem8QpXaOt8TS7RHUIS+GPA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-create-class-features-plugin": "^7.28.6", - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-create-class-features-plugin": "^7.27.1", + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1504,15 +1453,15 @@ } }, "node_modules/@babel/plugin-transform-private-property-in-object": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-private-property-in-object/-/plugin-transform-private-property-in-object-7.28.6.tgz", - "integrity": "sha512-b97jvNSOb5+ehyQmBpmhOCiUC5oVK4PMnpRvO7+ymFBoqYjeDHIU9jnrNUuwHOiL9RpGDoKBpSViarV+BU+eVA==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-private-property-in-object/-/plugin-transform-private-property-in-object-7.27.1.tgz", + "integrity": "sha512-5J+IhqTi1XPa0DXF83jYOaARrX+41gOewWbkPyjMNRDqgOCqdffGh8L3f/Ek5utaEBZExjSAzcyjmV9SSAWObQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-annotate-as-pure": "^7.27.3", - "@babel/helper-create-class-features-plugin": "^7.28.6", - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-annotate-as-pure": "^7.27.1", + "@babel/helper-create-class-features-plugin": "^7.27.1", + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1538,13 +1487,13 @@ } }, "node_modules/@babel/plugin-transform-regenerator": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.29.0.tgz", - "integrity": "sha512-FijqlqMA7DmRdg/aINBSs04y8XNTYw/lr1gJ2WsmBnnaNw1iS43EPkJW+zK7z65auG3AWRFXWj+NcTQwYptUog==", + "version": "7.28.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.28.4.tgz", + "integrity": "sha512-+ZEdQlBoRg9m2NnzvEeLgtvBMO4tkFBw5SQIUgLICgTrumLoU7lr+Oghi6km2PFj+dbUt2u1oby2w3BDO9YQnA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1554,14 +1503,14 @@ } }, "node_modules/@babel/plugin-transform-regexp-modifiers": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regexp-modifiers/-/plugin-transform-regexp-modifiers-7.28.6.tgz", - "integrity": "sha512-QGWAepm9qxpaIs7UM9FvUSnCGlb8Ua1RhyM4/veAxLwt3gMat/LSGrZixyuj4I6+Kn9iwvqCyPTtbdxanYoWYg==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regexp-modifiers/-/plugin-transform-regexp-modifiers-7.27.1.tgz", + "integrity": "sha512-TtEciroaiODtXvLZv4rmfMhkCv8jx3wgKpL68PuiPh2M4fvz5jhsA7697N1gMvkvr/JTF13DrFYyEbY9U7cVPA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-create-regexp-features-plugin": "^7.28.5", - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-create-regexp-features-plugin": "^7.27.1", + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1603,13 +1552,13 @@ } }, "node_modules/@babel/plugin-transform-spread": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.28.6.tgz", - "integrity": "sha512-9U4QObUC0FtJl05AsUcodau/RWDytrU6uKgkxu09mLR9HLDAtUMoPuuskm5huQsoktmsYpI+bGmq+iapDcriKA==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.27.1.tgz", + "integrity": "sha512-kpb3HUqaILBJcRFVhFUs6Trdd4mkrzcGXss+6/mxUd273PfbWqSDHRzMT2234gIg2QYfAjvXLSquP1xECSg09Q==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6", + "@babel/helper-plugin-utils": "^7.27.1", "@babel/helper-skip-transparent-expression-wrappers": "^7.27.1" }, "engines": { @@ -1684,14 +1633,14 @@ } }, "node_modules/@babel/plugin-transform-unicode-property-regex": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-property-regex/-/plugin-transform-unicode-property-regex-7.28.6.tgz", - "integrity": "sha512-4Wlbdl/sIZjzi/8St0evF0gEZrgOswVO6aOzqxh1kDZOl9WmLrHq2HtGhnOJZmHZYKP8WZ1MDLCt5DAWwRo57A==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-property-regex/-/plugin-transform-unicode-property-regex-7.27.1.tgz", + "integrity": "sha512-uW20S39PnaTImxp39O5qFlHLS9LJEmANjMG7SxIhap8rCHqu0Ik+tLEPX5DKmHn6CsWQ7j3lix2tFOa5YtL12Q==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-create-regexp-features-plugin": "^7.28.5", - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-create-regexp-features-plugin": "^7.27.1", + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1718,14 +1667,14 @@ } }, "node_modules/@babel/plugin-transform-unicode-sets-regex": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-sets-regex/-/plugin-transform-unicode-sets-regex-7.28.6.tgz", - "integrity": "sha512-/wHc/paTUmsDYN7SZkpWxogTOBNnlx7nBQYfy6JJlCT7G3mVhltk3e++N7zV0XfgGsrqBxd4rJQt9H16I21Y1Q==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-sets-regex/-/plugin-transform-unicode-sets-regex-7.27.1.tgz", + "integrity": "sha512-EtkOujbc4cgvb0mlpQefi4NTPBzhSIevblFevACNLUspmrALgmEBdL/XfnyyITfd8fKBZrZys92zOWcik7j9Tw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-create-regexp-features-plugin": "^7.28.5", - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-create-regexp-features-plugin": "^7.27.1", + "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -1735,82 +1684,81 @@ } }, "node_modules/@babel/preset-env": { - "version": "7.29.5", - "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.29.5.tgz", - "integrity": "sha512-/69t2aEzGKHD76DyLbHysF/QH2LJOB8iFnYO37unDTKBTubzcMRv0f3H5EiN1Q6ajOd/eB7dAInF0qdFVS06kA==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.28.5.tgz", + "integrity": "sha512-S36mOoi1Sb6Fz98fBfE+UZSpYw5mJm0NUHtIKrOuNcqeFauy1J6dIvXm2KRVKobOSaGq4t/hBXdN4HGU3wL9Wg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/compat-data": "^7.29.3", - "@babel/helper-compilation-targets": "^7.28.6", - "@babel/helper-plugin-utils": "^7.28.6", + "@babel/compat-data": "^7.28.5", + "@babel/helper-compilation-targets": "^7.27.2", + "@babel/helper-plugin-utils": "^7.27.1", "@babel/helper-validator-option": "^7.27.1", "@babel/plugin-bugfix-firefox-class-in-computed-class-key": "^7.28.5", "@babel/plugin-bugfix-safari-class-field-initializer-scope": "^7.27.1", "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "^7.27.1", - "@babel/plugin-bugfix-safari-rest-destructuring-rhs-array": "^7.29.3", "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "^7.27.1", - "@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly": "^7.28.6", + "@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly": "^7.28.3", "@babel/plugin-proposal-private-property-in-object": "7.21.0-placeholder-for-preset-env.2", - "@babel/plugin-syntax-import-assertions": "^7.28.6", - "@babel/plugin-syntax-import-attributes": "^7.28.6", + "@babel/plugin-syntax-import-assertions": "^7.27.1", + "@babel/plugin-syntax-import-attributes": "^7.27.1", "@babel/plugin-syntax-unicode-sets-regex": "^7.18.6", "@babel/plugin-transform-arrow-functions": "^7.27.1", - "@babel/plugin-transform-async-generator-functions": "^7.29.0", - "@babel/plugin-transform-async-to-generator": "^7.28.6", + "@babel/plugin-transform-async-generator-functions": "^7.28.0", + "@babel/plugin-transform-async-to-generator": "^7.27.1", "@babel/plugin-transform-block-scoped-functions": "^7.27.1", - "@babel/plugin-transform-block-scoping": "^7.28.6", - "@babel/plugin-transform-class-properties": "^7.28.6", - "@babel/plugin-transform-class-static-block": "^7.28.6", - "@babel/plugin-transform-classes": "^7.28.6", - "@babel/plugin-transform-computed-properties": "^7.28.6", + "@babel/plugin-transform-block-scoping": "^7.28.5", + "@babel/plugin-transform-class-properties": "^7.27.1", + "@babel/plugin-transform-class-static-block": "^7.28.3", + "@babel/plugin-transform-classes": "^7.28.4", + "@babel/plugin-transform-computed-properties": "^7.27.1", "@babel/plugin-transform-destructuring": "^7.28.5", - "@babel/plugin-transform-dotall-regex": "^7.28.6", + "@babel/plugin-transform-dotall-regex": "^7.27.1", "@babel/plugin-transform-duplicate-keys": "^7.27.1", - "@babel/plugin-transform-duplicate-named-capturing-groups-regex": "^7.29.0", + "@babel/plugin-transform-duplicate-named-capturing-groups-regex": "^7.27.1", "@babel/plugin-transform-dynamic-import": "^7.27.1", - "@babel/plugin-transform-explicit-resource-management": "^7.28.6", - "@babel/plugin-transform-exponentiation-operator": "^7.28.6", + "@babel/plugin-transform-explicit-resource-management": "^7.28.0", + "@babel/plugin-transform-exponentiation-operator": "^7.28.5", "@babel/plugin-transform-export-namespace-from": "^7.27.1", "@babel/plugin-transform-for-of": "^7.27.1", "@babel/plugin-transform-function-name": "^7.27.1", - "@babel/plugin-transform-json-strings": "^7.28.6", + "@babel/plugin-transform-json-strings": "^7.27.1", "@babel/plugin-transform-literals": "^7.27.1", - "@babel/plugin-transform-logical-assignment-operators": "^7.28.6", + "@babel/plugin-transform-logical-assignment-operators": "^7.28.5", "@babel/plugin-transform-member-expression-literals": "^7.27.1", "@babel/plugin-transform-modules-amd": "^7.27.1", - "@babel/plugin-transform-modules-commonjs": "^7.28.6", - "@babel/plugin-transform-modules-systemjs": "^7.29.4", + "@babel/plugin-transform-modules-commonjs": "^7.27.1", + "@babel/plugin-transform-modules-systemjs": "^7.28.5", "@babel/plugin-transform-modules-umd": "^7.27.1", - "@babel/plugin-transform-named-capturing-groups-regex": "^7.29.0", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.27.1", "@babel/plugin-transform-new-target": "^7.27.1", - "@babel/plugin-transform-nullish-coalescing-operator": "^7.28.6", - "@babel/plugin-transform-numeric-separator": "^7.28.6", - "@babel/plugin-transform-object-rest-spread": "^7.28.6", + "@babel/plugin-transform-nullish-coalescing-operator": "^7.27.1", + "@babel/plugin-transform-numeric-separator": "^7.27.1", + "@babel/plugin-transform-object-rest-spread": "^7.28.4", "@babel/plugin-transform-object-super": "^7.27.1", - "@babel/plugin-transform-optional-catch-binding": "^7.28.6", - "@babel/plugin-transform-optional-chaining": "^7.28.6", + "@babel/plugin-transform-optional-catch-binding": "^7.27.1", + "@babel/plugin-transform-optional-chaining": "^7.28.5", "@babel/plugin-transform-parameters": "^7.27.7", - "@babel/plugin-transform-private-methods": "^7.28.6", - "@babel/plugin-transform-private-property-in-object": "^7.28.6", + "@babel/plugin-transform-private-methods": "^7.27.1", + "@babel/plugin-transform-private-property-in-object": "^7.27.1", "@babel/plugin-transform-property-literals": "^7.27.1", - "@babel/plugin-transform-regenerator": "^7.29.0", - "@babel/plugin-transform-regexp-modifiers": "^7.28.6", + "@babel/plugin-transform-regenerator": "^7.28.4", + "@babel/plugin-transform-regexp-modifiers": "^7.27.1", "@babel/plugin-transform-reserved-words": "^7.27.1", "@babel/plugin-transform-shorthand-properties": "^7.27.1", - "@babel/plugin-transform-spread": "^7.28.6", + "@babel/plugin-transform-spread": "^7.27.1", "@babel/plugin-transform-sticky-regex": "^7.27.1", "@babel/plugin-transform-template-literals": "^7.27.1", "@babel/plugin-transform-typeof-symbol": "^7.27.1", "@babel/plugin-transform-unicode-escapes": "^7.27.1", - "@babel/plugin-transform-unicode-property-regex": "^7.28.6", + "@babel/plugin-transform-unicode-property-regex": "^7.27.1", "@babel/plugin-transform-unicode-regex": "^7.27.1", - "@babel/plugin-transform-unicode-sets-regex": "^7.28.6", + "@babel/plugin-transform-unicode-sets-regex": "^7.27.1", "@babel/preset-modules": "0.1.6-no-external-plugins", - "babel-plugin-polyfill-corejs2": "^0.4.15", - "babel-plugin-polyfill-corejs3": "^0.14.0", - "babel-plugin-polyfill-regenerator": "^0.6.6", - "core-js-compat": "^3.48.0", + "babel-plugin-polyfill-corejs2": "^0.4.14", + "babel-plugin-polyfill-corejs3": "^0.13.0", + "babel-plugin-polyfill-regenerator": "^0.6.5", + "core-js-compat": "^3.43.0", "semver": "^6.3.1" }, "engines": { @@ -1846,33 +1794,33 @@ } }, "node_modules/@babel/template": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", - "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", + "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.28.6", - "@babel/parser": "^7.28.6", - "@babel/types": "^7.28.6" + "@babel/code-frame": "^7.27.1", + "@babel/parser": "^7.27.2", + "@babel/types": "^7.27.1" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.0.tgz", - "integrity": "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.5.tgz", + "integrity": "sha512-TCCj4t55U90khlYkVV/0TfkJkAkUg3jZFA3Neb7unZT8CPok7iiRfaX0F+WnqWqt7OxhOn0uBKXCw4lbL8W0aQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.29.0", - "@babel/generator": "^7.29.0", + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.5", "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.29.0", - "@babel/template": "^7.28.6", - "@babel/types": "^7.29.0", + "@babel/parser": "^7.28.5", + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.5", "debug": "^4.3.1" }, "engines": { @@ -1880,9 +1828,9 @@ } }, "node_modules/@babel/types": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", - "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.5.tgz", + "integrity": "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==", "dev": true, "license": "MIT", "dependencies": { @@ -1900,19 +1848,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@bramus/specificity": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/@bramus/specificity/-/specificity-2.4.2.tgz", - "integrity": "sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==", - "dev": true, - "license": "MIT", - "dependencies": { - "css-tree": "^3.0.0" - }, - "bin": { - "specificity": "bin/cli.js" - } - }, "node_modules/@csstools/color-helpers": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.1.0.tgz", @@ -2008,31 +1943,6 @@ "@csstools/css-tokenizer": "^3.0.4" } }, - "node_modules/@csstools/css-syntax-patches-for-csstree": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.3.tgz", - "integrity": "sha512-SH60bMfrRCJF3morcdk57WklujF4Jr/EsQUzqkarfHXEFcAR1gg7fS/chAE922Sehgzc1/+Tz5H3Ypa1HiEKrg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT-0", - "peerDependencies": { - "css-tree": "^3.2.1" - }, - "peerDependenciesMeta": { - "css-tree": { - "optional": true - } - } - }, "node_modules/@csstools/css-tokenizer": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz", @@ -2053,76 +1963,6 @@ "node": ">=18" } }, - "node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@exodus/bytes": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.0.tgz", - "integrity": "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - }, - "peerDependencies": { - "@noble/hashes": "^1.8.0 || ^2.0.0" - }, - "peerDependenciesMeta": { - "@noble/hashes": { - "optional": true - } - } - }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -2140,10 +1980,20 @@ "node": ">=8" } }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/@istanbuljs/schema": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz", - "integrity": "sha512-+Sg6GCR/wy1oSmQDFq4LQDAhm3ETKnorxN+y5nbLULOR3P0c14f2Wurzj3/xqPXtasLFfHd5iRFQ7AJt4KH2cw==", + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", "dev": true, "license": "MIT", "engines": { @@ -2151,61 +2001,61 @@ } }, "node_modules/@jest/console": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/console/-/console-30.4.1.tgz", - "integrity": "sha512-v3bhyxUh9Hgmo5p6hAOXe14/R3ZxZDOsvHleh4B07z3m/x4/ngPUXEm9XwK4sF4u+f+P2ORb0Ge+MgpaqRMVDA==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", + "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.4.1", + "@jest/types": "^29.6.3", "@types/node": "*", - "chalk": "^4.1.2", - "jest-message-util": "30.4.1", - "jest-util": "30.4.1", + "chalk": "^4.0.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", "slash": "^3.0.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/core": { - "version": "30.4.2", - "resolved": "https://registry.npmjs.org/@jest/core/-/core-30.4.2.tgz", - "integrity": "sha512-TZJA6cPJUFxoWhxaLo8t0VX/MZX2wPWr0uIDvLSHIvN4gu9h02vSzqI2kBADG1ExqQlC+cY09xKMSreivvrChQ==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", + "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "30.4.1", - "@jest/pattern": "30.4.0", - "@jest/reporters": "30.4.1", - "@jest/test-result": "30.4.1", - "@jest/transform": "30.4.1", - "@jest/types": "30.4.1", + "@jest/console": "^29.7.0", + "@jest/reporters": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", "@types/node": "*", - "ansi-escapes": "^4.3.2", - "chalk": "^4.1.2", - "ci-info": "^4.2.0", - "exit-x": "^0.2.2", - "fast-json-stable-stringify": "^2.1.0", - "graceful-fs": "^4.2.11", - "jest-changed-files": "30.4.1", - "jest-config": "30.4.2", - "jest-haste-map": "30.4.1", - "jest-message-util": "30.4.1", - "jest-regex-util": "30.4.0", - "jest-resolve": "30.4.1", - "jest-resolve-dependencies": "30.4.2", - "jest-runner": "30.4.2", - "jest-runtime": "30.4.2", - "jest-snapshot": "30.4.1", - "jest-util": "30.4.1", - "jest-validate": "30.4.1", - "jest-watcher": "30.4.1", - "pretty-format": "30.4.1", - "slash": "^3.0.0" + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-changed-files": "^29.7.0", + "jest-config": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-resolve-dependencies": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "jest-watcher": "^29.7.0", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-ansi": "^6.0.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" @@ -2216,178 +2066,140 @@ } } }, - "node_modules/@jest/diff-sequences": { - "version": "30.4.0", - "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.4.0.tgz", - "integrity": "sha512-zOpzlfUs45l6u7jm39qr87JCHUDsaeCtvL+kQe/Vn9jSnRB4/5IPXISm0h9I1vZW/o00Kn4UTJ2MOlhnUGwv3g==", + "node_modules/@jest/core/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "license": "MIT", "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">=8" } }, - "node_modules/@jest/environment": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.4.1.tgz", - "integrity": "sha512-AK9yNRqgKxiabqMoe4oW+3/TSSeV8vkdC7BGaxZdU0AFXfOpofTLqdru2GXKZghP3sdgwE9XXpnVwfZ8JnFV4w==", + "node_modules/@jest/core/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "license": "MIT", "dependencies": { - "@jest/fake-timers": "30.4.1", - "@jest/types": "30.4.1", - "@types/node": "*", - "jest-mock": "30.4.1" + "ansi-regex": "^5.0.1" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">=8" } }, - "node_modules/@jest/environment-jsdom-abstract": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/environment-jsdom-abstract/-/environment-jsdom-abstract-30.4.1.tgz", - "integrity": "sha512-dSlKrqug3siYNHVnjwIldShY12wAH3spwRltO/+8VOjg0X+xEq7vOs3DbBs4LRKsu7OH+NUb9kuZUNBF9Ho3TA==", + "node_modules/@jest/environment": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", + "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.4.1", - "@jest/fake-timers": "30.4.1", - "@jest/types": "30.4.1", - "@types/jsdom": "^21.1.7", + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", "@types/node": "*", - "jest-mock": "30.4.1", - "jest-util": "30.4.1" + "jest-mock": "^29.7.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "peerDependencies": { - "canvas": "^3.0.0", - "jsdom": "*" - }, - "peerDependenciesMeta": { - "canvas": { - "optional": true - } + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/expect": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-30.4.1.tgz", - "integrity": "sha512-ginrj6TMgh2GshLUGCjO94Ptx9HhdZA/I6A9iUfyeLKFtdAjnKzHDgzgP9HYQgbxM1lbXScQ2eUBz2lGeVDPWA==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", "dev": true, "license": "MIT", "dependencies": { - "expect": "30.4.1", - "jest-snapshot": "30.4.1" + "expect": "^29.7.0", + "jest-snapshot": "^29.7.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/expect-utils": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.4.1.tgz", - "integrity": "sha512-ZBn5CglH8fBsQsvs4VWNzD4aWfUYks+IdOOQU3MEK71ol/BcVm+P+rtb1KpiFBpSWSCE27uOahyyf1vfqOVbcQ==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", + "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0" + "jest-get-type": "^29.6.3" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/fake-timers": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.4.1.tgz", - "integrity": "sha512-iW5umdmfPeWzehrVhugFQZqCchSCud5S1l2YT0O9ZhjRR0ExclANDZkiSBwzqtnlOn0J1JXvO+HZ6rkuyOVOgQ==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", + "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.4.1", - "@sinonjs/fake-timers": "^15.4.0", + "@jest/types": "^29.6.3", + "@sinonjs/fake-timers": "^10.0.2", "@types/node": "*", - "jest-message-util": "30.4.1", - "jest-mock": "30.4.1", - "jest-util": "30.4.1" + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/get-type": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", - "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/globals": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-30.4.1.tgz", - "integrity": "sha512-ZbuY4cmXC8DkxYjfvT2DbcHWL2T6vmsMhXCDcmTB2T0y0gaezBI77ufq5ZAIdcRkYZ7NEQEDg1xFeKbxUJ5v5Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "30.4.1", - "@jest/expect": "30.4.1", - "@jest/types": "30.4.1", - "jest-mock": "30.4.1" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/pattern": { - "version": "30.4.0", - "resolved": "https://registry.npmjs.org/@jest/pattern/-/pattern-30.4.0.tgz", - "integrity": "sha512-RAWn3+f9u8BsHijKJ71uHcFp6vmyEt6VvoWXkl6hKF3qVIuWNmudVjg12DlBPGup/frIl5UcUlH5HfEuvHpEXg==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", + "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", "dev": true, "license": "MIT", "dependencies": { - "@types/node": "*", - "jest-regex-util": "30.4.0" + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/types": "^29.6.3", + "jest-mock": "^29.7.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/reporters": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-30.4.1.tgz", - "integrity": "sha512-/SnkPCzEQpUaBH81kjdEdDdo2WZl5hxw+BmLDGWjRkm8o7XlhjwsU36cqwe5PGBE5WYpBvDzRSdXx9rbGuJtNA==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", + "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^0.2.3", - "@jest/console": "30.4.1", - "@jest/test-result": "30.4.1", - "@jest/transform": "30.4.1", - "@jest/types": "30.4.1", - "@jridgewell/trace-mapping": "^0.3.25", + "@jest/console": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", "@types/node": "*", - "chalk": "^4.1.2", - "collect-v8-coverage": "^1.0.2", - "exit-x": "^0.2.2", - "glob": "^10.5.0", - "graceful-fs": "^4.2.11", + "chalk": "^4.0.0", + "collect-v8-coverage": "^1.0.0", + "exit": "^0.1.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", "istanbul-lib-coverage": "^3.0.0", "istanbul-lib-instrument": "^6.0.0", "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^5.0.0", + "istanbul-lib-source-maps": "^4.0.0", "istanbul-reports": "^3.1.3", - "jest-message-util": "30.4.1", - "jest-util": "30.4.1", - "jest-worker": "30.4.1", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", "slash": "^3.0.0", - "string-length": "^4.0.2", + "string-length": "^4.0.1", + "strip-ansi": "^6.0.0", "v8-to-istanbul": "^9.0.1" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" @@ -2398,125 +2210,178 @@ } } }, - "node_modules/@jest/schemas": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.4.1.tgz", - "integrity": "sha512-i6b4qw5qnP8c5FEeBJg/uZQ4ddrkN6Ca8qISJh0pr7a5hfn3h3v5x60BEbOC7OYAGZNMs1LfFLwnW2CuK8F57Q==", + "node_modules/@jest/reporters/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/reporters/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/@jest/reporters/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@jest/reporters/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/@jest/reporters/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "license": "MIT", "dependencies": { - "@sinclair/typebox": "^0.34.0" + "ansi-regex": "^5.0.1" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">=8" } }, - "node_modules/@jest/snapshot-utils": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/snapshot-utils/-/snapshot-utils-30.4.1.tgz", - "integrity": "sha512-ObY4ljvQ95mt6iwKtVLetR/4yXiAgl3H4nJxhztr0MTjrN97TwDYrnCp/kF60Ec9HdhkWTHSu+Hg05aXfngpOA==", + "node_modules/@jest/schemas": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.4.1", - "chalk": "^4.1.2", - "graceful-fs": "^4.2.11", - "natural-compare": "^1.4.0" + "@sinclair/typebox": "^0.27.8" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/source-map": { - "version": "30.0.1", - "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-30.0.1.tgz", - "integrity": "sha512-MIRWMUUR3sdbP36oyNyhbThLHyJ2eEDClPCiHVbrYAe5g3CHRArIVpBw7cdSB5fr+ofSfIb2Tnsw8iEHL0PYQg==", + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", + "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", "dev": true, "license": "MIT", "dependencies": { - "@jridgewell/trace-mapping": "^0.3.25", - "callsites": "^3.1.0", - "graceful-fs": "^4.2.11" + "@jridgewell/trace-mapping": "^0.3.18", + "callsites": "^3.0.0", + "graceful-fs": "^4.2.9" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/test-result": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-30.4.1.tgz", - "integrity": "sha512-/ZG7pgEiOmmWkN9TplKbOu4id2N5lh7FHwRwlkgBVAzGdRH+OkkQ8wX/kIxg4zmd3ZQvAL1RwL2yWsvNYYECTw==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", + "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "30.4.1", - "@jest/types": "30.4.1", - "@types/istanbul-lib-coverage": "^2.0.6", - "collect-v8-coverage": "^1.0.2" + "@jest/console": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "collect-v8-coverage": "^1.0.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/test-sequencer": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-30.4.1.tgz", - "integrity": "sha512-PeYE+4td5rKjoRPxztObrXU+H8hsjZfxKMXOcmrr34JerSyB/ROOxbbicz8B7A5j9R9VayDnVPvBmedqCsFCdw==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", + "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/test-result": "30.4.1", - "graceful-fs": "^4.2.11", - "jest-haste-map": "30.4.1", + "@jest/test-result": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", "slash": "^3.0.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/transform": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-30.4.1.tgz", - "integrity": "sha512-Wz0LyktlTvRefoymh+n64hQ84KNXsRGcwdoZ8CSa0Ea+fgYcHZlnk+hDP7v2MS7il2bQ5uTEIxf4/NNfhMN4KQ==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", + "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/core": "^7.27.4", - "@jest/types": "30.4.1", - "@jridgewell/trace-mapping": "^0.3.25", - "babel-plugin-istanbul": "^7.0.1", - "chalk": "^4.1.2", + "@babel/core": "^7.11.6", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "babel-plugin-istanbul": "^6.1.1", + "chalk": "^4.0.0", "convert-source-map": "^2.0.0", "fast-json-stable-stringify": "^2.1.0", - "graceful-fs": "^4.2.11", - "jest-haste-map": "30.4.1", - "jest-regex-util": "30.4.0", - "jest-util": "30.4.1", - "pirates": "^4.0.7", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "micromatch": "^4.0.4", + "pirates": "^4.0.4", "slash": "^3.0.0", - "write-file-atomic": "^5.0.1" + "write-file-atomic": "^4.0.2" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/types": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.4.1.tgz", - "integrity": "sha512-f1x/vJXIfjOlEmejYpbkbgw1gOqpPECwMvMEtBqe47j7H2Hg8h8w3o3ikhSXq3MI15kg+oQ0exWO0uCtTNJLoQ==", + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", + "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/pattern": "30.4.0", - "@jest/schemas": "30.4.1", - "@types/istanbul-lib-coverage": "^2.0.6", - "@types/istanbul-reports": "^3.0.4", + "@jest/schemas": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", "@types/node": "*", - "@types/yargs": "^17.0.33", - "chalk": "^4.1.2" + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jridgewell/gen-mapping": { @@ -2569,47 +2434,10 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "0.2.12", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", - "integrity": "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "^1.4.3", - "@emnapi/runtime": "^1.4.3", - "@tybys/wasm-util": "^0.10.0" - } - }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=14" - } - }, - "node_modules/@pkgr/core": { - "version": "0.2.9", - "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.9.tgz", - "integrity": "sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/pkgr" - } - }, "node_modules/@sinclair/typebox": { - "version": "0.34.49", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.49.tgz", - "integrity": "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A==", + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", "dev": true, "license": "MIT" }, @@ -2624,13 +2452,13 @@ } }, "node_modules/@sinonjs/fake-timers": { - "version": "15.4.0", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-15.4.0.tgz", - "integrity": "sha512-DsG+8/LscQIQg68J6Ef3dv10u6nVyetYn923s3/sus5eaGfTo1of5WMZSLf0UJc9KDuKPilPH0UDJCjvNbDNCA==", + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", "dev": true, "license": "BSD-3-Clause", "dependencies": { - "@sinonjs/commons": "^3.0.1" + "@sinonjs/commons": "^3.0.0" } }, "node_modules/@testing-library/jest-dom": { @@ -2653,15 +2481,14 @@ "yarn": ">=1" } }, - "node_modules/@tybys/wasm-util": { - "version": "0.10.2", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", - "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", + "node_modules/@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", "dev": true, "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" + "engines": { + "node": ">= 10" } }, "node_modules/@types/babel__core": { @@ -2709,6 +2536,16 @@ "@babel/types": "^7.28.2" } }, + "node_modules/@types/graceful-fs": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", + "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", @@ -2737,9 +2574,9 @@ } }, "node_modules/@types/jsdom": { - "version": "21.1.7", - "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.7.tgz", - "integrity": "sha512-yOriVnggzrnQ3a9OKOCxaVuSug3w3/SbOj5i7VwXWZEyUNl3bLF9V3MfxGbZKuwqJOQyRfqXyROBB1CoZLFWzA==", + "version": "20.0.1", + "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-20.0.1.tgz", + "integrity": "sha512-d0r18sZPmMQr1eG35u12FZfhIXNrnsPU/g5wvRKCUf/tOGilKKwYMYGqh33BNR6ba+2gkHw1EUiHoN3mn7E5IQ==", "dev": true, "license": "MIT", "dependencies": { @@ -2789,281 +2626,50 @@ "dev": true, "license": "MIT" }, - "node_modules/@ungap/structured-clone": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.1.tgz", - "integrity": "sha512-mUFwbeTqrVgDQxFveS+df2yfap6iuP20NAKAsBt5jDEoOTDew+zwLAOilHCeQJOVSvmgCX4ogqIrA0mnyr08yQ==", + "node_modules/abab": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", + "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", + "deprecated": "Use your platform's native atob() and btoa() methods instead", "dev": true, - "license": "ISC" + "license": "BSD-3-Clause" }, - "node_modules/@unrs/resolver-binding-android-arm-eabi": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm-eabi/-/resolver-binding-android-arm-eabi-1.11.1.tgz", - "integrity": "sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@unrs/resolver-binding-android-arm64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm64/-/resolver-binding-android-arm64-1.11.1.tgz", - "integrity": "sha512-lCxkVtb4wp1v+EoN+HjIG9cIIzPkX5OtM03pQYkG+U5O/wL53LC4QbIeazgiKqluGeVEeBlZahHalCaBvU1a2g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@unrs/resolver-binding-darwin-arm64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-arm64/-/resolver-binding-darwin-arm64-1.11.1.tgz", - "integrity": "sha512-gPVA1UjRu1Y/IsB/dQEsp2V1pm44Of6+LWvbLc9SDk1c2KhhDRDBUkQCYVWe6f26uJb3fOK8saWMgtX8IrMk3g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@unrs/resolver-binding-darwin-x64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-x64/-/resolver-binding-darwin-x64-1.11.1.tgz", - "integrity": "sha512-cFzP7rWKd3lZaCsDze07QX1SC24lO8mPty9vdP+YVa3MGdVgPmFc59317b2ioXtgCMKGiCLxJ4HQs62oz6GfRQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@unrs/resolver-binding-freebsd-x64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-freebsd-x64/-/resolver-binding-freebsd-x64-1.11.1.tgz", - "integrity": "sha512-fqtGgak3zX4DCB6PFpsH5+Kmt/8CIi4Bry4rb1ho6Av2QHTREM+47y282Uqiu3ZRF5IQioJQ5qWRV6jduA+iGw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] - }, - "node_modules/@unrs/resolver-binding-linux-arm-gnueabihf": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-gnueabihf/-/resolver-binding-linux-arm-gnueabihf-1.11.1.tgz", - "integrity": "sha512-u92mvlcYtp9MRKmP+ZvMmtPN34+/3lMHlyMj7wXJDeXxuM0Vgzz0+PPJNsro1m3IZPYChIkn944wW8TYgGKFHw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-arm-musleabihf": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-musleabihf/-/resolver-binding-linux-arm-musleabihf-1.11.1.tgz", - "integrity": "sha512-cINaoY2z7LVCrfHkIcmvj7osTOtm6VVT16b5oQdS4beibX2SYBwgYLmqhBjA1t51CarSaBuX5YNsWLjsqfW5Cw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-arm64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-gnu/-/resolver-binding-linux-arm64-gnu-1.11.1.tgz", - "integrity": "sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-arm64-musl": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-musl/-/resolver-binding-linux-arm64-musl-1.11.1.tgz", - "integrity": "sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-ppc64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-ppc64-gnu/-/resolver-binding-linux-ppc64-gnu-1.11.1.tgz", - "integrity": "sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-riscv64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-gnu/-/resolver-binding-linux-riscv64-gnu-1.11.1.tgz", - "integrity": "sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-riscv64-musl": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-musl/-/resolver-binding-linux-riscv64-musl-1.11.1.tgz", - "integrity": "sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-s390x-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-s390x-gnu/-/resolver-binding-linux-s390x-gnu-1.11.1.tgz", - "integrity": "sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==", - "cpu": [ - "s390x" - ], + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-x64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-gnu/-/resolver-binding-linux-x64-gnu-1.11.1.tgz", - "integrity": "sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-linux-x64-musl": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-musl/-/resolver-binding-linux-x64-musl-1.11.1.tgz", - "integrity": "sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@unrs/resolver-binding-wasm32-wasi": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-wasm32-wasi/-/resolver-binding-wasm32-wasi-1.11.1.tgz", - "integrity": "sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@napi-rs/wasm-runtime": "^0.2.11" + "bin": { + "acorn": "bin/acorn" }, "engines": { - "node": ">=14.0.0" + "node": ">=0.4.0" } }, - "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.11.1.tgz", - "integrity": "sha512-nRcz5Il4ln0kMhfL8S3hLkxI85BXs3o8EYoattsJNdsX4YUU89iOkVn7g0VHSRxFuVMdM4Q1jEpIId1Ihim/Uw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@unrs/resolver-binding-win32-ia32-msvc": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-ia32-msvc/-/resolver-binding-win32-ia32-msvc-1.11.1.tgz", - "integrity": "sha512-DCEI6t5i1NmAZp6pFonpD5m7i6aFrpofcp4LA2i8IIq60Jyo28hamKBxNrZcyOwVOZkgsRp9O2sXWBWP8MnvIQ==", - "cpu": [ - "ia32" - ], + "node_modules/acorn-globals": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-7.0.1.tgz", + "integrity": "sha512-umOSDSDrfHbTNPuNpC2NSnnA3LUrqpevPb4T9jRx4MagXNS0rs+gwiTcAvqCRmsD6utzsrzNt+ebm00SNWiC3Q==", "dev": true, "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@unrs/resolver-binding-win32-x64-msvc": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-x64-msvc/-/resolver-binding-win32-x64-msvc-1.11.1.tgz", - "integrity": "sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g==", - "cpu": [ - "x64" - ], + "dependencies": { + "acorn": "^8.1.0", + "acorn-walk": "^8.0.2" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", "dev": true, "license": "MIT", - "optional": true, - "os": [ - "win32" - ] + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } }, "node_modules/agent-base": { "version": "7.1.4", @@ -3091,19 +2697,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, "node_modules/ansi-styles": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", @@ -3131,19 +2724,6 @@ "node": ">= 8" } }, - "node_modules/anymatch/node_modules/picomatch": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", - "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -3164,70 +2744,104 @@ "node": ">= 0.4" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true, + "license": "MIT" + }, "node_modules/babel-jest": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.4.1.tgz", - "integrity": "sha512-fATAbM8piYxkiXQp3RBXmZHxZVNJZAVXXfyeyCN2Tida3+qJ8ea9UxhiJ2y4fLO90ZImKt6k9FlcH2+rLkJGhw==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", + "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/transform": "30.4.1", - "@types/babel__core": "^7.20.5", - "babel-plugin-istanbul": "^7.0.1", - "babel-preset-jest": "30.4.0", - "chalk": "^4.1.2", - "graceful-fs": "^4.2.11", + "@jest/transform": "^29.7.0", + "@types/babel__core": "^7.1.14", + "babel-plugin-istanbul": "^6.1.1", + "babel-preset-jest": "^29.6.3", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", "slash": "^3.0.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { - "@babel/core": "^7.11.0 || ^8.0.0-0" + "@babel/core": "^7.8.0" } }, "node_modules/babel-plugin-istanbul": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-7.0.1.tgz", - "integrity": "sha512-D8Z6Qm8jCvVXtIRkBnqNHX0zJ37rQcFJ9u8WOS6tkYOsRdHBzypCstaxWiu5ZIlqQtviRYbgnRLSoCEvjqcqbA==", + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", + "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", "dev": true, "license": "BSD-3-Clause", - "workspaces": [ - "test/babel-8" - ], "dependencies": { "@babel/helper-plugin-utils": "^7.0.0", "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.3", - "istanbul-lib-instrument": "^6.0.2", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-instrument": "^5.0.4", "test-exclude": "^6.0.0" }, "engines": { - "node": ">=12" + "node": ">=8" + } + }, + "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", + "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.12.3", + "@babel/parser": "^7.14.7", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-istanbul/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" } }, "node_modules/babel-plugin-jest-hoist": { - "version": "30.4.0", - "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-30.4.0.tgz", - "integrity": "sha512-9EdtWM/sSfXLOGLwSn+GS6pIXyBnL07/8gyJlwFXjWy4DxMOyItqyUT29d4lQiS380EZwYlX7/At4PgBS+m2aA==", + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", + "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", "dev": true, "license": "MIT", "dependencies": { - "@types/babel__core": "^7.20.5" + "@babel/template": "^7.3.3", + "@babel/types": "^7.3.3", + "@types/babel__core": "^7.1.14", + "@types/babel__traverse": "^7.0.6" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/babel-plugin-polyfill-corejs2": { - "version": "0.4.17", - "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.4.17.tgz", - "integrity": "sha512-aTyf30K/rqAsNwN76zYrdtx8obu0E4KoUME29B1xj+B3WxgvWkp943vYQ+z8Mv3lw9xHXMHpvSPOBxzAkIa94w==", + "version": "0.4.14", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.4.14.tgz", + "integrity": "sha512-Co2Y9wX854ts6U8gAAPXfn0GmAyctHuK8n0Yhfjd6t30g7yvKjspvvOo9yG+z52PZRgFErt7Ka2pYnXCjLKEpg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/compat-data": "^7.28.6", - "@babel/helper-define-polyfill-provider": "^0.6.8", + "@babel/compat-data": "^7.27.7", + "@babel/helper-define-polyfill-provider": "^0.6.5", "semver": "^6.3.1" }, "peerDependencies": { @@ -3245,27 +2859,27 @@ } }, "node_modules/babel-plugin-polyfill-corejs3": { - "version": "0.14.2", - "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.14.2.tgz", - "integrity": "sha512-coWpDLJ410R781Npmn/SIBZEsAetR4xVi0SxLMXPaMO4lSf1MwnkGYMtkFxew0Dn8B3/CpbpYxN0JCgg8mn67g==", + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.13.0.tgz", + "integrity": "sha512-U+GNwMdSFgzVmfhNm8GJUX88AadB3uo9KpJqS3FaqNIPKgySuvMb+bHPsOmmuWyIcuqZj/pzt1RUIUZns4y2+A==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-define-polyfill-provider": "^0.6.8", - "core-js-compat": "^3.48.0" + "@babel/helper-define-polyfill-provider": "^0.6.5", + "core-js-compat": "^3.43.0" }, "peerDependencies": { "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" } }, "node_modules/babel-plugin-polyfill-regenerator": { - "version": "0.6.8", - "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.6.8.tgz", - "integrity": "sha512-M762rNHfSF1EV3SLtnCJXFoQbbIIz0OyRwnCmV0KPC7qosSfCO0QLTSuJX3ayAebubhE6oYBAYPrBA5ljowaZg==", + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.6.5.tgz", + "integrity": "sha512-ISqQ2frbiNU9vIJkzg7dlPpznPZ4jOiUQ1uSmB0fEHeowtN3COYRsXr/xexn64NpU13P06jc/L5TgiJXOgrbEg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-define-polyfill-provider": "^0.6.8" + "@babel/helper-define-polyfill-provider": "^0.6.5" }, "peerDependencies": { "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" @@ -3299,20 +2913,20 @@ } }, "node_modules/babel-preset-jest": { - "version": "30.4.0", - "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-30.4.0.tgz", - "integrity": "sha512-lBY4jxsNmCnSiu7kquw8ZC9F4+XLMOKypT3RnNHPvU2Kpd4W0xaPuLr5ZkRyOsvLYAY4yaW1ZwTW4xB7NIiZzg==", + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", + "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", "dev": true, "license": "MIT", "dependencies": { - "babel-plugin-jest-hoist": "30.4.0", - "babel-preset-current-node-syntax": "^1.2.0" + "babel-plugin-jest-hoist": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { - "@babel/core": "^7.11.0 || ^8.0.0-beta.1" + "@babel/core": "^7.0.0" } }, "node_modules/balanced-match": { @@ -3323,42 +2937,32 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.10.23", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.23.tgz", - "integrity": "sha512-xwVXGqevyKPsiuQdLj+dZMVjidjJV508TBqexND5HrF89cGdCYCJFB3qhcxRHSeMctdCfbR1jrxBajhDy7o29g==", + "version": "2.8.30", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.30.tgz", + "integrity": "sha512-aTUKW4ptQhS64+v2d6IkPzymEzzhw+G0bA1g3uBRV3+ntkH+svttKseW5IOR4Ed6NUVKqnY7qT3dKvzQ7io4AA==", "dev": true, "license": "Apache-2.0", "bin": { - "baseline-browser-mapping": "dist/cli.cjs" - }, - "engines": { - "node": ">=6.0.0" + "baseline-browser-mapping": "dist/cli.js" } }, - "node_modules/bidi-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", - "integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==", - "dev": true, - "license": "MIT", - "dependencies": { - "require-from-string": "^2.0.2" - } - }, - "node_modules/brace-expansion": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", - "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "license": "MIT", "dependencies": { - "balanced-match": "^1.0.0" + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" } }, "node_modules/browserslist": { - "version": "4.28.2", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", - "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz", + "integrity": "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==", "dev": true, "funding": [ { @@ -3376,11 +2980,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.10.12", - "caniuse-lite": "^1.0.30001782", - "electron-to-chromium": "^1.5.328", - "node-releases": "^2.0.36", - "update-browserslist-db": "^1.2.3" + "baseline-browser-mapping": "^2.8.25", + "caniuse-lite": "^1.0.30001754", + "electron-to-chromium": "^1.5.249", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.1.4" }, "bin": { "browserslist": "cli.js" @@ -3406,6 +3010,20 @@ "dev": true, "license": "MIT" }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -3417,19 +3035,22 @@ } }, "node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "dev": true, "license": "MIT", "engines": { - "node": ">=6" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/caniuse-lite": { - "version": "1.0.30001791", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001791.tgz", - "integrity": "sha512-yk0l/YSrOnFZk3UROpDLQD9+kC1l4meK/wed583AXrzoarMGJcbRi2Q4RaUYbKxYAsZ8sWmaSa/DsLmdBeI1vQ==", + "version": "1.0.30001756", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001756.tgz", + "integrity": "sha512-4HnCNKbMLkLdhJz3TToeVWHSnfJvPaq6vu/eRP0Ahub/07n484XHhBF5AJoSGHdVrS8tKFauUQz8Bp9P7LVx7A==", "dev": true, "funding": [ { @@ -3491,9 +3112,9 @@ } }, "node_modules/ci-info": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.4.0.tgz", - "integrity": "sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==", + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", "dev": true, "funding": [ { @@ -3507,9 +3128,9 @@ } }, "node_modules/cjs-module-lexer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", - "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", + "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", "dev": true, "license": "MIT" }, @@ -3645,6 +3266,19 @@ "dev": true, "license": "MIT" }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -3660,19 +3294,41 @@ "license": "MIT" }, "node_modules/core-js-compat": { - "version": "3.49.0", - "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.49.0.tgz", - "integrity": "sha512-VQXt1jr9cBz03b331DFDCCP90b3fanciLkgiOoy8SBHy06gNf+vQ1A3WFLqG7I8TipYIKeYK9wxd0tUrvHcOZA==", + "version": "3.47.0", + "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.47.0.tgz", + "integrity": "sha512-IGfuznZ/n7Kp9+nypamBhvwdwLsW6KC8IOaURw2doAK5e98AG3acVLdh0woOnEqCfUtS+Vu882JE4k/DAm3ItQ==", "dev": true, "license": "MIT", "dependencies": { - "browserslist": "^4.28.1" + "browserslist": "^4.28.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/core-js" } }, + "node_modules/create-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", + "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "prompts": "^2.0.1" + }, + "bin": { + "create-jest": "bin/create-jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -3688,20 +3344,6 @@ "node": ">= 8" } }, - "node_modules/css-tree": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", - "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", - "dev": true, - "license": "MIT", - "dependencies": { - "mdn-data": "2.27.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" - } - }, "node_modules/css.escape": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz", @@ -3709,6 +3351,13 @@ "dev": true, "license": "MIT" }, + "node_modules/cssom": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==", + "dev": true, + "license": "MIT" + }, "node_modules/cssstyle": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.6.0.tgz", @@ -3723,6 +3372,13 @@ "node": ">=18" } }, + "node_modules/cssstyle/node_modules/rrweb-cssom": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", + "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "dev": true, + "license": "MIT" + }, "node_modules/data-urls": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", @@ -3763,9 +3419,9 @@ "license": "MIT" }, "node_modules/dedent": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.2.tgz", - "integrity": "sha512-WzMx3mW98SN+zn3hgemf4OzdmyNhhhKz5Ay0pUfQiMQ3e1g+xmTJWp/pKdwKVXhdSkAEGIIzqeuWrL3mV/AXbA==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.0.tgz", + "integrity": "sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ==", "dev": true, "license": "MIT", "peerDependencies": { @@ -3787,6 +3443,16 @@ "node": ">=0.10.0" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -3797,6 +3463,16 @@ "node": ">=8" } }, + "node_modules/diff-sequences": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", + "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, "node_modules/dom-accessibility-api": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.6.3.tgz", @@ -3804,17 +3480,39 @@ "dev": true, "license": "MIT" }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "node_modules/domexception": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz", + "integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==", + "deprecated": "Use your platform's native DOMException instead", "dev": true, - "license": "MIT" + "license": "MIT", + "dependencies": { + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } }, "node_modules/electron-to-chromium": { - "version": "1.5.344", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.344.tgz", - "integrity": "sha512-4MxfbmNDm+KPh066EZy+eUnkcDPcZ35wNmOWzFuh/ijvHsve6kbLTLURy88uCNK5FbpN+yk2nQY6BYh1GEt+wg==", + "version": "1.5.259", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.259.tgz", + "integrity": "sha512-I+oLXgpEJzD6Cwuwt1gYjxsDmu/S/Kd41mmLA3O+/uH2pFRO/DvOjUyGozL8j3KeLV6WyZ7ssPwELMsXCcsJAQ==", "dev": true, "license": "ISC" }, @@ -3831,13 +3529,6 @@ "url": "https://github.com/sindresorhus/emittery?sponsor=1" } }, - "node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "dev": true, - "license": "MIT" - }, "node_modules/entities": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", @@ -3861,6 +3552,55 @@ "is-arrayish": "^0.2.1" } }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -3871,6 +3611,28 @@ "node": ">=6" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, "node_modules/esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", @@ -3885,6 +3647,16 @@ "node": ">=4" } }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", @@ -3926,32 +3698,30 @@ "dev": true, "license": "ISC" }, - "node_modules/exit-x": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/exit-x/-/exit-x-0.2.2.tgz", - "integrity": "sha512-+I6B/IkJc1o/2tiURyz/ivu/O0nKNEArIUB5O7zBrlDVJr22SCLH3xTeEry428LvFhRzIA1g8izguxJ/gbNcVQ==", + "node_modules/exit": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", + "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.8.0" } }, "node_modules/expect": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/expect/-/expect-30.4.1.tgz", - "integrity": "sha512-PMARsyh/JtqC20HoGqlFcIlQAyqUtW4PlI1rup1uhYJtKuwAjbvWi3GQMAn+STdHum/dk8xrKfUM1+5SAwpolA==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/expect-utils": "30.4.1", - "@jest/get-type": "30.1.0", - "jest-matcher-utils": "30.4.1", - "jest-message-util": "30.4.1", - "jest-mock": "30.4.1", - "jest-util": "30.4.1" + "@jest/expect-utils": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/fast-json-stable-stringify": { @@ -3971,6 +3741,19 @@ "bser": "2.1.1" } }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/find-up": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", @@ -3985,21 +3768,21 @@ "node": ">=8" } }, - "node_modules/foreground-child": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", - "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", "dev": true, - "license": "ISC", + "license": "MIT", "dependencies": { - "cross-spawn": "^7.0.6", - "signal-exit": "^4.0.1" + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" }, "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" + "node": ">= 6" } }, "node_modules/fs.realpath": { @@ -4054,6 +3837,31 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/get-package-type": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", @@ -4064,6 +3872,20 @@ "node": ">=8.0.0" } }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -4077,26 +3899,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/glob": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", - "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", - "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", "dev": true, - "license": "ISC", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^3.1.2", - "minimatch": "^9.0.4", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^1.11.1" - }, - "bin": { - "glob": "dist/esm/bin.mjs" + "license": "MIT", + "engines": { + "node": ">= 0.4" }, "funding": { - "url": "https://github.com/sponsors/isaacs" + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/graceful-fs": { @@ -4116,6 +3929,35 @@ "node": ">=8" } }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -4302,6 +4144,16 @@ "node": ">=6" } }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, "node_modules/is-potential-custom-element-name": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", @@ -4372,15 +4224,15 @@ } }, "node_modules/istanbul-lib-source-maps": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-5.0.6.tgz", - "integrity": "sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", + "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", "dev": true, "license": "BSD-3-Clause", "dependencies": { - "@jridgewell/trace-mapping": "^0.3.23", "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0" + "istanbul-lib-coverage": "^3.0.0", + "source-map": "^0.6.1" }, "engines": { "node": ">=10" @@ -4400,39 +4252,23 @@ "node": ">=8" } }, - "node_modules/jackspeak": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", - "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" - } - }, "node_modules/jest": { - "version": "30.4.2", - "resolved": "https://registry.npmjs.org/jest/-/jest-30.4.2.tgz", - "integrity": "sha512-Yi1jqNC/Oq0N4hBgNH/YvBpP1P57QqundgytzYqy3yqAa7NZPNjSoi4SGbRAXDMdBzNE6xBCi5U7RgfrvMEUVQ==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", + "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/core": "30.4.2", - "@jest/types": "30.4.1", - "import-local": "^3.2.0", - "jest-cli": "30.4.2" + "@jest/core": "^29.7.0", + "@jest/types": "^29.6.3", + "import-local": "^3.0.2", + "jest-cli": "^29.7.0" }, "bin": { "jest": "bin/jest.js" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" @@ -4444,75 +4280,76 @@ } }, "node_modules/jest-changed-files": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-30.4.1.tgz", - "integrity": "sha512-IuctmYrxi21iOSOaIXpJWalHyPAsVv0GeBHKDn8C1CA4W5htHn7INL+wdnL4Bo0+olEndvAFkmb++tIQJG+vvg==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", + "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", "dev": true, "license": "MIT", "dependencies": { - "execa": "^5.1.1", - "jest-util": "30.4.1", + "execa": "^5.0.0", + "jest-util": "^29.7.0", "p-limit": "^3.1.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-circus": { - "version": "30.4.2", - "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-30.4.2.tgz", - "integrity": "sha512-rvHH7VlY6LgbJXJTQ87GW62g1FntOtbhh0zT+v04kC+pgL6aBKyYINXxWukCpj3dcIBMw5/XUbtDS9dU9JTXeQ==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", + "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.4.1", - "@jest/expect": "30.4.1", - "@jest/test-result": "30.4.1", - "@jest/types": "30.4.1", + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", "@types/node": "*", - "chalk": "^4.1.2", + "chalk": "^4.0.0", "co": "^4.6.0", - "dedent": "^1.6.0", - "is-generator-fn": "^2.1.0", - "jest-each": "30.4.1", - "jest-matcher-utils": "30.4.1", - "jest-message-util": "30.4.1", - "jest-runtime": "30.4.2", - "jest-snapshot": "30.4.1", - "jest-util": "30.4.1", + "dedent": "^1.0.0", + "is-generator-fn": "^2.0.0", + "jest-each": "^29.7.0", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", "p-limit": "^3.1.0", - "pretty-format": "30.4.1", - "pure-rand": "^7.0.0", + "pretty-format": "^29.7.0", + "pure-rand": "^6.0.0", "slash": "^3.0.0", - "stack-utils": "^2.0.6" + "stack-utils": "^2.0.3" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-cli": { - "version": "30.4.2", - "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-30.4.2.tgz", - "integrity": "sha512-jfA2ocvVHMXS2QijrJ0d31ektP+d/W0T5RpcTX2Pq+3sVqHlsXVCM2+FmwpL+bdY8OfHpIg9xMxLF17Zg0U49Q==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", + "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/core": "30.4.2", - "@jest/test-result": "30.4.1", - "@jest/types": "30.4.1", - "chalk": "^4.1.2", - "exit-x": "^0.2.2", - "import-local": "^3.2.0", - "jest-config": "30.4.2", - "jest-util": "30.4.1", - "jest-validate": "30.4.1", - "yargs": "^17.7.2" + "@jest/core": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "create-jest": "^29.7.0", + "exit": "^0.1.2", + "import-local": "^3.0.2", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "yargs": "^17.3.1" }, "bin": { "jest": "bin/jest.js" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" @@ -4524,158 +4361,164 @@ } }, "node_modules/jest-config": { - "version": "30.4.2", - "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-30.4.2.tgz", - "integrity": "sha512-rNHAShJQqQwFNoL0hbf3BphSBOWnpOUAKvidLS/AjNVLPfoj5mSf4jQMfW3cYOs6hXeZC7nF7mDHaBnbxELOzg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.27.4", - "@jest/get-type": "30.1.0", - "@jest/pattern": "30.4.0", - "@jest/test-sequencer": "30.4.1", - "@jest/types": "30.4.1", - "babel-jest": "30.4.1", - "chalk": "^4.1.2", - "ci-info": "^4.2.0", - "deepmerge": "^4.3.1", - "glob": "^10.5.0", - "graceful-fs": "^4.2.11", - "jest-circus": "30.4.2", - "jest-docblock": "30.4.0", - "jest-environment-node": "30.4.1", - "jest-regex-util": "30.4.0", - "jest-resolve": "30.4.1", - "jest-runner": "30.4.2", - "jest-util": "30.4.1", - "jest-validate": "30.4.1", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", + "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/test-sequencer": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-jest": "^29.7.0", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "deepmerge": "^4.2.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-circus": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "micromatch": "^4.0.4", "parse-json": "^5.2.0", - "pretty-format": "30.4.1", + "pretty-format": "^29.7.0", "slash": "^3.0.0", "strip-json-comments": "^3.1.1" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { "@types/node": "*", - "esbuild-register": ">=3.4.0", "ts-node": ">=9.0.0" }, "peerDependenciesMeta": { "@types/node": { "optional": true }, - "esbuild-register": { - "optional": true - }, "ts-node": { "optional": true } } }, - "node_modules/jest-diff": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.4.1.tgz", - "integrity": "sha512-CRpFK0RtLriVDGcPPAnR6HMVI8bSR2jnUIgralhauzYQZIb4RH9AtEInTuQr65LmmGggGcRT6HIASxwqsVsmlA==", + "node_modules/jest-config/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/diff-sequences": "30.4.0", - "@jest/get-type": "30.1.0", - "chalk": "^4.1.2", - "pretty-format": "30.4.1" + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/jest-config/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-config/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "*" } }, - "node_modules/jest-docblock": { - "version": "30.4.0", - "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-30.4.0.tgz", - "integrity": "sha512-ZPMabUZCx5MpbZ2eBYSvZ0J8fvo3dR9oM+eeUpb3aKNQFuS2tu3Duw1TNlMoP8k3WQgKGJuhcMFvwcVuq6T7oA==", + "node_modules/jest-diff": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", + "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", "dev": true, "license": "MIT", "dependencies": { - "detect-newline": "^3.1.0" + "chalk": "^4.0.0", + "diff-sequences": "^29.6.3", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jest-each": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-30.4.1.tgz", - "integrity": "sha512-/8MJbH6fuj48TstjrMf+u/pd06Qezz5xOXvZA6442heNOWr8bdeoGZX2d9fCn028CoMgYmroH9//zky5GfyYmA==", + "node_modules/jest-docblock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", + "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0", - "@jest/types": "30.4.1", - "chalk": "^4.1.2", - "jest-util": "30.4.1", - "pretty-format": "30.4.1" + "detect-newline": "^3.0.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jest-environment-jsdom": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-environment-jsdom/-/jest-environment-jsdom-30.4.1.tgz", - "integrity": "sha512-o3nfaN4zej7qgk2X0j8Jhq/S9nAVKs2xK3QeQxeHVvpkEPxaA1yxDGydR+iVI7zPy7Cp62Aq2h3Ja46QvfWHGA==", + "node_modules/jest-each": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", + "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.4.1", - "@jest/environment-jsdom-abstract": "30.4.1", - "jsdom": "^26.1.0" + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "jest-util": "^29.7.0", + "pretty-format": "^29.7.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "peerDependencies": { - "canvas": "^3.0.0" - }, - "peerDependenciesMeta": { - "canvas": { - "optional": true - } + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jest-environment-jsdom/node_modules/jsdom": { - "version": "26.1.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-26.1.0.tgz", - "integrity": "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==", + "node_modules/jest-environment-jsdom": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-environment-jsdom/-/jest-environment-jsdom-29.7.0.tgz", + "integrity": "sha512-k9iQbsf9OyOfdzWH8HDmrRT0gSIcX+FLNW7IQq94tFX0gynPwqDTW0Ho6iMVNjGz/nb+l/vW3dWM2bbLLpkbXA==", "dev": true, "license": "MIT", "dependencies": { - "cssstyle": "^4.2.1", - "data-urls": "^5.0.0", - "decimal.js": "^10.5.0", - "html-encoding-sniffer": "^4.0.0", - "http-proxy-agent": "^7.0.2", - "https-proxy-agent": "^7.0.6", - "is-potential-custom-element-name": "^1.0.1", - "nwsapi": "^2.2.16", - "parse5": "^7.2.1", - "rrweb-cssom": "^0.8.0", - "saxes": "^6.0.0", - "symbol-tree": "^3.2.4", - "tough-cookie": "^5.1.1", - "w3c-xmlserializer": "^5.0.0", - "webidl-conversions": "^7.0.0", - "whatwg-encoding": "^3.1.1", - "whatwg-mimetype": "^4.0.0", - "whatwg-url": "^14.1.1", - "ws": "^8.18.0", - "xml-name-validator": "^5.0.0" + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/jsdom": "^20.0.0", + "@types/node": "*", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0", + "jsdom": "^20.0.0" }, "engines": { - "node": ">=18" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { - "canvas": "^3.0.0" + "canvas": "^2.5.0" }, "peerDependenciesMeta": { "canvas": { @@ -4683,711 +4526,684 @@ } } }, - "node_modules/jest-environment-node": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.4.1.tgz", - "integrity": "sha512-4FZYVOk85hz2AyT6BbarKy9u37g6DbrDyCdFhsnDdXqyrueYQvB+0zO4f/kqLCRD0BsPRXPMNJeQwihKZV8naw==", + "node_modules/jest-environment-jsdom/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.4.1", - "@jest/fake-timers": "30.4.1", - "@jest/types": "30.4.1", - "@types/node": "*", - "jest-mock": "30.4.1", - "jest-util": "30.4.1", - "jest-validate": "30.4.1" + "debug": "4" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">= 6.0.0" } }, - "node_modules/jest-haste-map": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-30.4.1.tgz", - "integrity": "sha512-rFrcONd8jeFsyw+Z9CrScJgglRf2+NFmNam8dKu7n+SoHqNYT47mn0DdEcVUZJpvh7Iz6/si7f7yUH7GJHVgnw==", + "node_modules/jest-environment-jsdom/node_modules/cssstyle": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz", + "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.4.1", - "@types/node": "*", - "anymatch": "^3.1.3", - "fb-watchman": "^2.0.2", - "graceful-fs": "^4.2.11", - "jest-regex-util": "30.4.0", - "jest-util": "30.4.1", - "jest-worker": "30.4.1", - "picomatch": "^4.0.3", - "walker": "^1.0.8" + "cssom": "~0.3.6" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "optionalDependencies": { - "fsevents": "^2.3.3" + "node": ">=8" } }, - "node_modules/jest-leak-detector": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.4.1.tgz", - "integrity": "sha512-IpmyiioeHxiWDhesHnUFmOxcTzwCwKpgACgWajtAP+nYQXiY7DakTxB6Bx9JFiRMljr0AX1PvnQdaU1KFoz6NQ==", + "node_modules/jest-environment-jsdom/node_modules/cssstyle/node_modules/cssom": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-environment-jsdom/node_modules/data-urls": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", + "integrity": "sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0", - "pretty-format": "30.4.1" + "abab": "^2.0.6", + "whatwg-mimetype": "^3.0.0", + "whatwg-url": "^11.0.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">=12" } }, - "node_modules/jest-matcher-utils": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.4.1.tgz", - "integrity": "sha512-zvYfX5CaeEkFrrLS9suWe9rvJrm9J1Iv3ua8kIBv9GEPzcnsfBf0bob37la7s67fs0nlBC3EuvkOLnXQKxtx4A==", + "node_modules/jest-environment-jsdom/node_modules/html-encoding-sniffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", + "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0", - "chalk": "^4.1.2", - "jest-diff": "30.4.1", - "pretty-format": "30.4.1" + "whatwg-encoding": "^2.0.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">=12" } }, - "node_modules/jest-message-util": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.4.1.tgz", - "integrity": "sha512-kwCKIvq0MCW1HzLoGola9Te6JUdzgV0loyKJ3Qghrkz9i5/RRIHsL95BMQc2HBBhlBKC4j22K9p11TGHH8RBpQ==", + "node_modules/jest-environment-jsdom/node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.27.1", - "@jest/types": "30.4.1", - "@types/stack-utils": "^2.0.3", - "chalk": "^4.1.2", - "graceful-fs": "^4.2.11", - "jest-util": "30.4.1", - "picomatch": "^4.0.3", - "pretty-format": "30.4.1", - "slash": "^3.0.0", - "stack-utils": "^2.0.6" + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">= 6" } }, - "node_modules/jest-mock": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.4.1.tgz", - "integrity": "sha512-/i8SVb8/NSB7RfNi8gfqu8gxLV23KaL5EpAttyb9iz8qWRIqXRLflycz/32wXsYkOnaUlx8NAKnJYtpsmXUmfw==", + "node_modules/jest-environment-jsdom/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.4.1", - "@types/node": "*", - "jest-util": "30.4.1" + "agent-base": "6", + "debug": "4" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">= 6" } }, - "node_modules/jest-pnp-resolver": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", - "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", - "dev": true, - "license": "MIT", + "node_modules/jest-environment-jsdom/node_modules/jsdom": { + "version": "20.0.3", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-20.0.3.tgz", + "integrity": "sha512-SYhBvTh89tTfCD/CRdSOm13mOBa42iTaTyfyEWBdKcGdPxPtLFBXuHR8XHb33YNYaP+lLbmSvBTsnoesCNJEsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "abab": "^2.0.6", + "acorn": "^8.8.1", + "acorn-globals": "^7.0.0", + "cssom": "^0.5.0", + "cssstyle": "^2.3.0", + "data-urls": "^3.0.2", + "decimal.js": "^10.4.2", + "domexception": "^4.0.0", + "escodegen": "^2.0.0", + "form-data": "^4.0.0", + "html-encoding-sniffer": "^3.0.0", + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.1", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.2", + "parse5": "^7.1.1", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^4.1.2", + "w3c-xmlserializer": "^4.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^2.0.0", + "whatwg-mimetype": "^3.0.0", + "whatwg-url": "^11.0.0", + "ws": "^8.11.0", + "xml-name-validator": "^4.0.0" + }, "engines": { - "node": ">=6" + "node": ">=14" }, "peerDependencies": { - "jest-resolve": "*" + "canvas": "^2.5.0" }, "peerDependenciesMeta": { - "jest-resolve": { + "canvas": { "optional": true } } }, - "node_modules/jest-regex-util": { - "version": "30.4.0", - "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-30.4.0.tgz", - "integrity": "sha512-mWlvLviKIgIQ8VCuM1xRdD0TWp3zlzionlmDBjuXVBs+VkmXq6FgW9T4Emr7oGz/Rk6feDCGyiugolcQEyp3mg==", + "node_modules/jest-environment-jsdom/node_modules/tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", "dev": true, "license": "MIT", + "dependencies": { + "punycode": "^2.1.1" + }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">=12" } }, - "node_modules/jest-resolve": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-30.4.1.tgz", - "integrity": "sha512-Zry8Yq/yJcNAZ7dJ5F2heic8AheXvbFZ7XI5V+h28nrYZ7Qoyy4dItq8OodjnYD270mvX+ZudmrNV9cysqhW5Q==", + "node_modules/jest-environment-jsdom/node_modules/w3c-xmlserializer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz", + "integrity": "sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw==", "dev": true, "license": "MIT", "dependencies": { - "chalk": "^4.1.2", - "graceful-fs": "^4.2.11", - "jest-haste-map": "30.4.1", - "jest-pnp-resolver": "^1.2.3", - "jest-util": "30.4.1", - "jest-validate": "30.4.1", - "slash": "^3.0.0", - "unrs-resolver": "^1.7.11" + "xml-name-validator": "^4.0.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">=14" } }, - "node_modules/jest-resolve-dependencies": { - "version": "30.4.2", - "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-30.4.2.tgz", - "integrity": "sha512-gDiVh1I+GxYzz9oXlyw+1wv6VOYX1WYxMOfjsA3iGKePV2oxmbHhwxfkALxNxYy1ciw6APWwkW2zZONwP97aEQ==", + "node_modules/jest-environment-jsdom/node_modules/whatwg-encoding": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", + "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", "dev": true, "license": "MIT", "dependencies": { - "jest-regex-util": "30.4.0", - "jest-snapshot": "30.4.1" + "iconv-lite": "0.6.3" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">=12" } }, - "node_modules/jest-runner": { - "version": "30.4.2", - "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-30.4.2.tgz", - "integrity": "sha512-2dw0PslVYXxffXGpLo+Ejad+KcI1Qkjn7f4X4619gf21oCUmL+SPfjqIa/losUem3yEOvfNZe/F1HWUcNpODcg==", + "node_modules/jest-environment-jsdom/node_modules/whatwg-mimetype": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", "dev": true, "license": "MIT", - "dependencies": { - "@jest/console": "30.4.1", - "@jest/environment": "30.4.1", - "@jest/test-result": "30.4.1", - "@jest/transform": "30.4.1", - "@jest/types": "30.4.1", - "@types/node": "*", - "chalk": "^4.1.2", - "emittery": "^0.13.1", - "exit-x": "^0.2.2", - "graceful-fs": "^4.2.11", - "jest-docblock": "30.4.0", - "jest-environment-node": "30.4.1", - "jest-haste-map": "30.4.1", - "jest-leak-detector": "30.4.1", - "jest-message-util": "30.4.1", - "jest-resolve": "30.4.1", - "jest-runtime": "30.4.2", - "jest-util": "30.4.1", - "jest-watcher": "30.4.1", - "jest-worker": "30.4.1", - "p-limit": "^3.1.0", - "source-map-support": "0.5.13" - }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">=12" } }, - "node_modules/jest-runtime": { - "version": "30.4.2", - "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-30.4.2.tgz", - "integrity": "sha512-3/5e8iPz2k/VLqlr8DgTftYyLUv8Su3FkCAO2/Od81UsUTpSxOrS6O5x5KkoQwyUjmpYyDJKeyAvg2T2nvpNkQ==", + "node_modules/jest-environment-jsdom/node_modules/whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.4.1", - "@jest/fake-timers": "30.4.1", - "@jest/globals": "30.4.1", - "@jest/source-map": "30.0.1", - "@jest/test-result": "30.4.1", - "@jest/transform": "30.4.1", - "@jest/types": "30.4.1", - "@types/node": "*", - "chalk": "^4.1.2", - "cjs-module-lexer": "^2.1.0", - "collect-v8-coverage": "^1.0.2", - "glob": "^10.5.0", - "graceful-fs": "^4.2.11", - "jest-haste-map": "30.4.1", - "jest-message-util": "30.4.1", - "jest-mock": "30.4.1", - "jest-regex-util": "30.4.0", - "jest-resolve": "30.4.1", - "jest-snapshot": "30.4.1", - "jest-util": "30.4.1", - "slash": "^3.0.0", - "strip-bom": "^4.0.0" + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">=12" } }, - "node_modules/jest-snapshot": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-30.4.1.tgz", - "integrity": "sha512-tEOkkfOMppUyeiHwjZswOQ3lcnoTnws/q5FnGIaeIh/jmoU0ZlgMYRR8sTlTj+nNGCoJ0RDq6SfxGxCsyMTPmw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.27.4", - "@babel/generator": "^7.27.5", - "@babel/plugin-syntax-jsx": "^7.27.1", - "@babel/plugin-syntax-typescript": "^7.27.1", - "@babel/types": "^7.27.3", - "@jest/expect-utils": "30.4.1", - "@jest/get-type": "30.1.0", - "@jest/snapshot-utils": "30.4.1", - "@jest/transform": "30.4.1", - "@jest/types": "30.4.1", - "babel-preset-current-node-syntax": "^1.2.0", - "chalk": "^4.1.2", - "expect": "30.4.1", - "graceful-fs": "^4.2.11", - "jest-diff": "30.4.1", - "jest-matcher-utils": "30.4.1", - "jest-message-util": "30.4.1", - "jest-util": "30.4.1", - "pretty-format": "30.4.1", - "semver": "^7.7.2", - "synckit": "^0.11.8" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node_modules/jest-environment-jsdom/node_modules/xml-name-validator": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz", + "integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12" } }, - "node_modules/jest-util": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.4.1.tgz", - "integrity": "sha512-vjQb1sACEiv13DKJMDToJpzVW0joCsIQrmbg0fi7CyOOt+g9jTuQl2A216pWRBYhOVt53XbL/2LbMKg1BECWOw==", + "node_modules/jest-environment-node": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", + "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.4.1", + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", "@types/node": "*", - "chalk": "^4.1.2", - "ci-info": "^4.2.0", - "graceful-fs": "^4.2.11", - "picomatch": "^4.0.3" + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jest-validate": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.4.1.tgz", - "integrity": "sha512-PDWi4SOwLnwqNDfHZjOcsEFyZ4fc/2W2gVL3DEoyqnB6jCQMLRtfBong8s6omIw3lI0HWOus12xfnFmQtjW3fw==", + "node_modules/jest-get-type": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", + "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", "dev": true, "license": "MIT", - "dependencies": { - "@jest/get-type": "30.1.0", - "@jest/types": "30.4.1", - "camelcase": "^6.3.0", - "chalk": "^4.1.2", - "leven": "^3.1.0", - "pretty-format": "30.4.1" - }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jest-validate/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "node_modules/jest-haste-map": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", + "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", "dev": true, "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/graceful-fs": "^4.1.3", + "@types/node": "*", + "anymatch": "^3.0.3", + "fb-watchman": "^2.0.0", + "graceful-fs": "^4.2.9", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "micromatch": "^4.0.4", + "walker": "^1.0.8" + }, "engines": { - "node": ">=10" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "optionalDependencies": { + "fsevents": "^2.3.2" } }, - "node_modules/jest-watcher": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-30.4.1.tgz", - "integrity": "sha512-/l9UonmvCwjHH7d2h3iAwIloLc1H0S8mJZ/LNK3i86hqwPAz8otUJjP9MfYtz9Tt77Su5FD2xGjZn8d31IZHlw==", + "node_modules/jest-leak-detector": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", + "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/test-result": "30.4.1", - "@jest/types": "30.4.1", - "@types/node": "*", - "ansi-escapes": "^4.3.2", - "chalk": "^4.1.2", - "emittery": "^0.13.1", - "jest-util": "30.4.1", - "string-length": "^4.0.2" + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jest-worker": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.4.1.tgz", - "integrity": "sha512-SHynN/q/QD++iNyvMdy+WMmbCGk8jIsNcRxycXbWubSOhvo6T+j2afcfUSl+3hYsiBebOTo0cT7c2H7CXugu1g==", + "node_modules/jest-matcher-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", + "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", "dev": true, "license": "MIT", "dependencies": { - "@types/node": "*", - "@ungap/structured-clone": "^1.3.0", - "jest-util": "30.4.1", - "merge-stream": "^2.0.0", - "supports-color": "^8.1.1" + "chalk": "^4.0.0", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jest-worker/node_modules/supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "node_modules/jest-message-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", + "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", "dev": true, "license": "MIT", "dependencies": { - "has-flag": "^4.0.0" + "@babel/code-frame": "^7.12.13", + "@jest/types": "^29.6.3", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" }, "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "node_modules/jest-mock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", + "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", "dev": true, "license": "MIT", "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-util": "^29.7.0" }, - "bin": { - "js-yaml": "bin/js-yaml.js" + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jsdom": { - "version": "29.1.1", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.1.1.tgz", - "integrity": "sha512-ECi4Fi2f7BdJtUKTflYRTiaMxIB0O6zfR1fX0GXpUrf6flp8QIYn1UT20YQqdSOfk2dfkCwS8LAFoJDEppNK5Q==", + "node_modules/jest-pnp-resolver": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", + "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", "dev": true, "license": "MIT", - "dependencies": { - "@asamuzakjp/css-color": "^5.1.11", - "@asamuzakjp/dom-selector": "^7.1.1", - "@bramus/specificity": "^2.4.2", - "@csstools/css-syntax-patches-for-csstree": "^1.1.3", - "@exodus/bytes": "^1.15.0", - "css-tree": "^3.2.1", - "data-urls": "^7.0.0", - "decimal.js": "^10.6.0", - "html-encoding-sniffer": "^6.0.0", - "is-potential-custom-element-name": "^1.0.1", - "lru-cache": "^11.3.5", - "parse5": "^8.0.1", - "saxes": "^6.0.0", - "symbol-tree": "^3.2.4", - "tough-cookie": "^6.0.1", - "undici": "^7.25.0", - "w3c-xmlserializer": "^5.0.0", - "webidl-conversions": "^8.0.1", - "whatwg-mimetype": "^5.0.0", - "whatwg-url": "^16.0.1", - "xml-name-validator": "^5.0.0" - }, "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24.0.0" + "node": ">=6" }, "peerDependencies": { - "canvas": "^3.0.0" + "jest-resolve": "*" }, "peerDependenciesMeta": { - "canvas": { + "jest-resolve": { "optional": true } } }, - "node_modules/jsdom/node_modules/@asamuzakjp/css-color": { - "version": "5.1.11", - "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.11.tgz", - "integrity": "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==", + "node_modules/jest-regex-util": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", + "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", "dev": true, "license": "MIT", - "dependencies": { - "@asamuzakjp/generational-cache": "^1.0.1", - "@csstools/css-calc": "^3.2.0", - "@csstools/css-color-parser": "^4.1.0", - "@csstools/css-parser-algorithms": "^4.0.0", - "@csstools/css-tokenizer": "^4.0.0" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - } - }, - "node_modules/jsdom/node_modules/@csstools/color-helpers": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.0.2.tgz", - "integrity": "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], - "license": "MIT-0", "engines": { - "node": ">=20.19.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jsdom/node_modules/@csstools/css-calc": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.2.0.tgz", - "integrity": "sha512-bR9e6o2BDB12jzN/gIbjHa5wLJ4UjD1CB9pM7ehlc0ddk6EBz+yYS1EV2MF55/HUxrHcB/hehAyt5vhsA3hx7w==", + "node_modules/jest-resolve": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", + "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], "license": "MIT", - "engines": { - "node": ">=20.19.0" + "dependencies": { + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-pnp-resolver": "^1.2.2", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "resolve": "^1.20.0", + "resolve.exports": "^2.0.0", + "slash": "^3.0.0" }, - "peerDependencies": { - "@csstools/css-parser-algorithms": "^4.0.0", - "@csstools/css-tokenizer": "^4.0.0" + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jsdom/node_modules/@csstools/css-color-parser": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.1.0.tgz", - "integrity": "sha512-U0KhLYmy2GVj6q4T3WaAe6NPuFYCPQoE3b0dRGxejWDgcPp8TP7S5rVdM5ZrFaqu4N67X8YaPBw14dQSYx3IyQ==", + "node_modules/jest-resolve-dependencies": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", + "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], "license": "MIT", "dependencies": { - "@csstools/color-helpers": "^6.0.2", - "@csstools/css-calc": "^3.2.0" + "jest-regex-util": "^29.6.3", + "jest-snapshot": "^29.7.0" }, "engines": { - "node": ">=20.19.0" - }, - "peerDependencies": { - "@csstools/css-parser-algorithms": "^4.0.0", - "@csstools/css-tokenizer": "^4.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jsdom/node_modules/@csstools/css-parser-algorithms": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.0.tgz", - "integrity": "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w==", + "node_modules/jest-runner": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", + "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], "license": "MIT", - "engines": { - "node": ">=20.19.0" + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/environment": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "graceful-fs": "^4.2.9", + "jest-docblock": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-leak-detector": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-resolve": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-util": "^29.7.0", + "jest-watcher": "^29.7.0", + "jest-worker": "^29.7.0", + "p-limit": "^3.1.0", + "source-map-support": "0.5.13" }, - "peerDependencies": { - "@csstools/css-tokenizer": "^4.0.0" + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jsdom/node_modules/@csstools/css-tokenizer": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-4.0.0.tgz", - "integrity": "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==", + "node_modules/jest-runtime": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", + "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/csstools" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/csstools" - } - ], "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/globals": "^29.7.0", + "@jest/source-map": "^29.6.3", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "cjs-module-lexer": "^1.0.0", + "collect-v8-coverage": "^1.0.0", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0", + "strip-bom": "^4.0.0" + }, "engines": { - "node": ">=20.19.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jsdom/node_modules/data-urls": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", - "integrity": "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==", + "node_modules/jest-runtime/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", "dependencies": { - "whatwg-mimetype": "^5.0.0", - "whatwg-url": "^16.0.0" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" } }, - "node_modules/jsdom/node_modules/entities": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz", - "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==", + "node_modules/jest-runtime/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", "dev": true, - "license": "BSD-2-Clause", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, "engines": { - "node": ">=20.19.0" + "node": "*" }, "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/jsdom/node_modules/html-encoding-sniffer": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz", - "integrity": "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==", + "node_modules/jest-runtime/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, - "license": "MIT", + "license": "ISC", "dependencies": { - "@exodus/bytes": "^1.6.0" + "brace-expansion": "^1.1.7" }, "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + "node": "*" } }, - "node_modules/jsdom/node_modules/lru-cache": { - "version": "11.3.6", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.6.tgz", - "integrity": "sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A==", - "dev": true, - "license": "BlueOak-1.0.0", + "node_modules/jest-snapshot": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", + "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@babel/generator": "^7.7.2", + "@babel/plugin-syntax-jsx": "^7.7.2", + "@babel/plugin-syntax-typescript": "^7.7.2", + "@babel/types": "^7.3.3", + "@jest/expect-utils": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0", + "chalk": "^4.0.0", + "expect": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "natural-compare": "^1.4.0", + "pretty-format": "^29.7.0", + "semver": "^7.5.3" + }, "engines": { - "node": "20 || >=22" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jsdom/node_modules/parse5": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", - "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", + "node_modules/jest-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", + "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", "dev": true, "license": "MIT", "dependencies": { - "entities": "^8.0.0" + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "graceful-fs": "^4.2.9", + "picomatch": "^2.2.3" }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jsdom/node_modules/tldts": { - "version": "7.0.30", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.30.tgz", - "integrity": "sha512-ELrFxuqsDdHUwoh0XxDbxuLD3Wnz49Z57IFvTtvWy1hJdcMZjXLIuonjilCiWHlT2GbE4Wlv1wKVTzDFnXH1aw==", + "node_modules/jest-validate": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", + "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", "dev": true, "license": "MIT", "dependencies": { - "tldts-core": "^7.0.30" + "@jest/types": "^29.6.3", + "camelcase": "^6.2.0", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "leven": "^3.1.0", + "pretty-format": "^29.7.0" }, - "bin": { - "tldts": "bin/cli.js" + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jsdom/node_modules/tldts-core": { - "version": "7.0.30", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.30.tgz", - "integrity": "sha512-uiHN8PIB1VmWyS98eZYja4xzlYqeFZVjb4OuYlJQnZAuJhMw4PbKQOKgHKhBdJR3FE/t5mUQ1Kd80++B+qhD1Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/jsdom/node_modules/tough-cookie": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.1.tgz", - "integrity": "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==", + "node_modules/jest-watcher": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", + "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", "dev": true, - "license": "BSD-3-Clause", + "license": "MIT", "dependencies": { - "tldts": "^7.0.5" + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "jest-util": "^29.7.0", + "string-length": "^4.0.1" }, "engines": { - "node": ">=16" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jsdom/node_modules/tr46": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz", - "integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==", + "node_modules/jest-worker": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", + "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", "dev": true, "license": "MIT", "dependencies": { - "punycode": "^2.3.1" + "@types/node": "*", + "jest-util": "^29.7.0", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" }, "engines": { - "node": ">=20" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jsdom/node_modules/webidl-conversions": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz", - "integrity": "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==", + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", "dev": true, - "license": "BSD-2-Clause", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, "engines": { - "node": ">=20" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" } }, - "node_modules/jsdom/node_modules/whatwg-mimetype": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz", - "integrity": "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==", + "node_modules/js-yaml": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "dev": true, "license": "MIT", - "engines": { - "node": ">=20" + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" } }, - "node_modules/jsdom/node_modules/whatwg-url": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", - "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", + "node_modules/jsdom": { + "version": "24.1.3", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-24.1.3.tgz", + "integrity": "sha512-MyL55p3Ut3cXbeBEG7Hcv0mVM8pp8PBNWxRqchZnSfAiES1v1mRnMeFfaHWIPULpwsYfvO+ZmMZz5tGCnjzDUQ==", "dev": true, "license": "MIT", "dependencies": { - "@exodus/bytes": "^1.11.0", - "tr46": "^6.0.0", - "webidl-conversions": "^8.0.1" + "cssstyle": "^4.0.1", + "data-urls": "^5.0.0", + "decimal.js": "^10.4.3", + "form-data": "^4.0.0", + "html-encoding-sniffer": "^4.0.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.5", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.12", + "parse5": "^7.1.2", + "rrweb-cssom": "^0.7.1", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^4.1.4", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^3.1.1", + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.0.0", + "ws": "^8.18.0", + "xml-name-validator": "^5.0.0" }, "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + "node": ">=18" + }, + "peerDependencies": { + "canvas": "^2.11.2" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } } }, "node_modules/jsesc": { @@ -5423,6 +5239,16 @@ "node": ">=6" } }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -5493,12 +5319,15 @@ "tmpl": "1.0.5" } }, - "node_modules/mdn-data": { - "version": "2.27.1", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", - "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", "dev": true, - "license": "CC0-1.0" + "license": "MIT", + "engines": { + "node": ">= 0.4" + } }, "node_modules/merge-stream": { "version": "2.0.0", @@ -5507,50 +5336,61 @@ "dev": true, "license": "MIT" }, - "node_modules/mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, "engines": { - "node": ">=6" + "node": ">=8.6" } }, - "node_modules/min-indent": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", - "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", "dev": true, "license": "MIT", "engines": { - "node": ">=4" + "node": ">= 0.6" } }, - "node_modules/minimatch": { - "version": "9.0.9", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", - "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "dev": true, - "license": "ISC", + "license": "MIT", "dependencies": { - "brace-expansion": "^2.0.2" + "mime-db": "1.52.0" }, "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" } }, - "node_modules/minipass": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", - "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "node_modules/min-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", + "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", "dev": true, - "license": "BlueOak-1.0.0", + "license": "MIT", "engines": { - "node": ">=16 || 14 >=14.17" + "node": ">=4" } }, "node_modules/ms": { @@ -5560,22 +5400,6 @@ "dev": true, "license": "MIT" }, - "node_modules/napi-postinstall": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.3.4.tgz", - "integrity": "sha512-PHI5f1O0EP5xJ9gQmFGMS6IZcrVvTjpXjz7Na41gTE7eE2hK11lg04CECCYEEjdc17EV4DO+fkGEtt7TpTaTiQ==", - "dev": true, - "license": "MIT", - "bin": { - "napi-postinstall": "lib/cli.js" - }, - "engines": { - "node": "^12.20.0 || ^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/napi-postinstall" - } - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -5591,9 +5415,9 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.38", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.38.tgz", - "integrity": "sha512-3qT/88Y3FbH/Kx4szpQQ4HzUbVrHPKTLVpVocKiLfoYvw9XSGOX2FmD2d6DrXbVYyAQTF2HeF6My8jmzx7/CRw==", + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", "dev": true, "license": "MIT" }, @@ -5708,13 +5532,6 @@ "node": ">=6" } }, - "node_modules/package-json-from-dist": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", - "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "dev": true, - "license": "BlueOak-1.0.0" - }, "node_modules/parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", @@ -5784,23 +5601,6 @@ "dev": true, "license": "MIT" }, - "node_modules/path-scurry": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", - "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - }, - "engines": { - "node": ">=16 || 14 >=14.18" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -5809,13 +5609,13 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true, "license": "MIT", "engines": { - "node": ">=12" + "node": ">=8.6" }, "funding": { "url": "https://github.com/sponsors/jonschlinkert" @@ -5845,19 +5645,45 @@ } }, "node_modules/pretty-format": { - "version": "30.4.1", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.4.1.tgz", - "integrity": "sha512-K6KiKMHTL4jjX4u3Kir2EW07nRfcqVTXIImx50wbjHQTcZPgg+gjVeNTIT3l3L1Rd4UefxfogquC9J37SoFyyw==", + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", + "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", "dev": true, "license": "MIT", "dependencies": { - "@jest/schemas": "30.4.1", - "ansi-styles": "^5.2.0", - "react-is-18": "npm:react-is@^18.3.1", - "react-is-19": "npm:react-is@^19.2.5" + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" }, "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + "node": ">= 6" + } + }, + "node_modules/psl": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz", + "integrity": "sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "funding": { + "url": "https://github.com/sponsors/lupomontero" } }, "node_modules/punycode": { @@ -5871,9 +5697,9 @@ } }, "node_modules/pure-rand": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-7.0.1.tgz", - "integrity": "sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", + "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", "dev": true, "funding": [ { @@ -5887,19 +5713,17 @@ ], "license": "MIT" }, - "node_modules/react-is-18": { - "name": "react-is", - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", "dev": true, "license": "MIT" }, - "node_modules/react-is-19": { - "name": "react-is", - "version": "19.2.6", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.2.6.tgz", - "integrity": "sha512-XjBR15BhXuylgWGuslhDKqlSayuqvqBX91BP8pauG8kd1zY8kotkNWbXksTCNRarse4kuGbe2kIY05ARtwNIvw==", + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", "dev": true, "license": "MIT" }, @@ -5985,15 +5809,12 @@ "node": ">=0.10.0" } }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } + "license": "MIT" }, "node_modules/resolve": { "version": "1.22.11", @@ -6039,10 +5860,20 @@ "node": ">=8" } }, - "node_modules/rrweb-cssom": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", - "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "node_modules/resolve.exports": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", + "integrity": "sha512-OcXjMsGdhL4XnbShKpAcSqPMzQoYkYyhbEaeSko47MjRP9NfEQMhZkXL1DoFlt9LWQn4YttrdnV6X2OiyzBi+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/rrweb-cssom": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.7.1.tgz", + "integrity": "sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg==", "dev": true, "license": "MIT" }, @@ -6067,9 +5898,9 @@ } }, "node_modules/semver": { - "version": "7.8.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.0.tgz", - "integrity": "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==", + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", "dev": true, "license": "ISC", "bin": { @@ -6102,18 +5933,12 @@ "node": ">=8" } }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } + "license": "MIT" }, "node_modules/slash": { "version": "3.0.0", @@ -6135,16 +5960,6 @@ "node": ">=0.10.0" } }, - "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/source-map-support": { "version": "0.5.13", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", @@ -6223,110 +6038,6 @@ "node": ">=8" } }, - "node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/string-width-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", - "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.2.2" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/strip-bom": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", @@ -6406,22 +6117,6 @@ "dev": true, "license": "MIT" }, - "node_modules/synckit": { - "version": "0.11.12", - "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.12.tgz", - "integrity": "sha512-Bh7QjT8/SuKUIfObSXNHNSK6WHo6J1tHCqJsuaFDP7gP0fkzSfTxI8y85JrppZ0h8l0maIgc2tfuZQ6/t3GtnQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@pkgr/core": "^0.2.9" - }, - "engines": { - "node": "^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/synckit" - } - }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -6438,9 +6133,9 @@ } }, "node_modules/test-exclude/node_modules/brace-expansion": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", - "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", "dependencies": { @@ -6452,7 +6147,7 @@ "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "deprecated": "Glob versions prior to v9 are no longer supported", "dev": true, "license": "ISC", "dependencies": { @@ -6471,9 +6166,9 @@ } }, "node_modules/test-exclude/node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, "license": "ISC", "dependencies": { @@ -6483,26 +6178,6 @@ "node": "*" } }, - "node_modules/tldts": { - "version": "6.1.86", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", - "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "tldts-core": "^6.1.86" - }, - "bin": { - "tldts": "bin/cli.js" - } - }, - "node_modules/tldts-core": { - "version": "6.1.86", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", - "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", - "dev": true, - "license": "MIT" - }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -6510,17 +6185,33 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, "node_modules/tough-cookie": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", - "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.4.tgz", + "integrity": "sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==", "dev": true, "license": "BSD-3-Clause", "dependencies": { - "tldts": "^6.1.32" + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" }, "engines": { - "node": ">=16" + "node": ">=6" } }, "node_modules/tr46": { @@ -6536,14 +6227,6 @@ "node": ">=18" } }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD", - "optional": true - }, "node_modules/type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", @@ -6567,16 +6250,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/undici": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", - "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", @@ -6628,45 +6301,20 @@ "node": ">=4" } }, - "node_modules/unrs-resolver": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/unrs-resolver/-/unrs-resolver-1.11.1.tgz", - "integrity": "sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg==", + "node_modules/universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", "dev": true, - "hasInstallScript": true, "license": "MIT", - "dependencies": { - "napi-postinstall": "^0.3.0" - }, - "funding": { - "url": "https://opencollective.com/unrs-resolver" - }, - "optionalDependencies": { - "@unrs/resolver-binding-android-arm-eabi": "1.11.1", - "@unrs/resolver-binding-android-arm64": "1.11.1", - "@unrs/resolver-binding-darwin-arm64": "1.11.1", - "@unrs/resolver-binding-darwin-x64": "1.11.1", - "@unrs/resolver-binding-freebsd-x64": "1.11.1", - "@unrs/resolver-binding-linux-arm-gnueabihf": "1.11.1", - "@unrs/resolver-binding-linux-arm-musleabihf": "1.11.1", - "@unrs/resolver-binding-linux-arm64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-arm64-musl": "1.11.1", - "@unrs/resolver-binding-linux-ppc64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-riscv64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-riscv64-musl": "1.11.1", - "@unrs/resolver-binding-linux-s390x-gnu": "1.11.1", - "@unrs/resolver-binding-linux-x64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-x64-musl": "1.11.1", - "@unrs/resolver-binding-wasm32-wasi": "1.11.1", - "@unrs/resolver-binding-win32-arm64-msvc": "1.11.1", - "@unrs/resolver-binding-win32-ia32-msvc": "1.11.1", - "@unrs/resolver-binding-win32-x64-msvc": "1.11.1" + "engines": { + "node": ">= 4.0.0" } }, "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.4.tgz", + "integrity": "sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A==", "dev": true, "funding": [ { @@ -6694,6 +6342,17 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "node_modules/v8-to-istanbul": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", @@ -6795,117 +6454,6 @@ "node": ">= 8" } }, - "node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/wrap-ansi-cjs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi/node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -6914,23 +6462,30 @@ "license": "ISC" }, "node_modules/write-file-atomic": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.1.tgz", - "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", + "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", "dev": true, "license": "ISC", "dependencies": { "imurmurhash": "^0.1.4", - "signal-exit": "^4.0.1" + "signal-exit": "^3.0.7" }, "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/write-file-atomic/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, "node_modules/ws": { - "version": "8.21.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", - "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", "dev": true, "license": "MIT", "engines": { diff --git a/tests/js/package.json b/tests/js/package.json index 86031e068c1..d34c5a065aa 100644 --- a/tests/js/package.json +++ b/tests/js/package.json @@ -12,9 +12,9 @@ "license": "ISC", "devDependencies": { "@babel/preset-env": "^7.24.7", - "jest": "^30.0.0", - "jest-environment-jsdom": "^30.2.0", + "jest": "^29.7.0", + "jest-environment-jsdom": "^29.7.0", "@testing-library/jest-dom": "^6.4.6", - "jsdom": "^29.0.0" + "jsdom": "^24.1.0" } } diff --git a/tests/js/table_widget_angular.test.js b/tests/js/table_widget_angular.test.js deleted file mode 100644 index 1e7d0275c5d..00000000000 --- a/tests/js/table_widget_angular.test.js +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { jest } from '@jest/globals'; - -describe('TableWidgetAngular', () => { - let render; - - beforeEach(async () => { - jest.resetModules(); - const tableWidgetAngular = ( - await import('../../bigframes/display/table_widget_angular.js') - ).default; - render = tableWidgetAngular.render; - }); - - it('should have a render function', () => { - expect(render).toBeDefined(); - }); - - it( - 'should bootstrap multiple widgets independently ' + - 'on their respective elements', - async () => { - const el1 = document.createElement('div'); - document.body.appendChild(el1); - - const model1 = { - get: jest.fn((prop) => { - if (prop === 'table_html') { - return '
    Widget 1 Content
    '; - } - if (prop === 'page_size') return 10; - if (prop === 'page') return 0; - if (prop === 'row_count') return 100; - if (prop === 'max_columns') return 20; - return null; - }), - set: jest.fn(), - save_changes: jest.fn(), - on: jest.fn(), - }; - - const el2 = document.createElement('div'); - document.body.appendChild(el2); - - const model2 = { - get: jest.fn((prop) => { - if (prop === 'table_html') { - return '
    Widget 2 Content
    '; - } - if (prop === 'page_size') return 25; - if (prop === 'page') return 0; - if (prop === 'row_count') return 200; - if (prop === 'max_columns') return 20; - return null; - }), - set: jest.fn(), - save_changes: jest.fn(), - on: jest.fn(), - }; - - render({ model: model1, el: el1 }); - render({ model: model2, el: el2 }); - - // Wait for async angular bootstrap to complete - await new Promise((resolve) => setTimeout(resolve, 200)); - - const appRoot1 = el1.querySelector('.bigframes-widget'); - expect(appRoot1).not.toBeNull(); - expect(el1.textContent).toContain('Widget 1 Content'); - expect(el1.textContent).toContain('100 total rows'); - expect(el1.textContent).toContain('Page 1 of 10'); - - const appRoot2 = el2.querySelector('.bigframes-widget'); - expect(appRoot2).not.toBeNull(); - expect(el2.textContent).toContain('Widget 2 Content'); - expect(el2.textContent).toContain('200 total rows'); - expect(el2.textContent).toContain('Page 1 of 8'); - - document.body.removeChild(el1); - document.body.removeChild(el2); - }); - - it( - 'should render deferred card and trigger execution on click', - async () => { - // Arrange - const el = document.createElement('div'); - document.body.appendChild(el); - - const state = { - is_deferred_mode: true, - dry_run_info: 'Estimated cost: $0.05', - start_execution: false, - table_html: '', - page_size: 10, - page: 0, - row_count: 0, - max_columns: 20, - }; - - const listeners = {}; - const model = { - get: jest.fn((prop) => state[prop]), - set: jest.fn((prop, val) => { - state[prop] = val; - }), - save_changes: jest.fn(), - on: jest.fn((event, callback) => { - listeners[event] = callback; - }), - }; - - // Act - render({ model, el }); - await new Promise((resolve) => setTimeout(resolve, 200)); - - // Assert (Initial state) - const estimate = el.querySelector('.deferred-estimate'); - expect(estimate).not.toBeNull(); - expect(estimate.textContent).toContain('Estimated cost: $0.05'); - - const runButton = el.querySelector('.run-query-button'); - expect(runButton).not.toBeNull(); - expect(runButton.textContent).toContain('Run Query'); - expect(el.querySelector('.table-container')).toBeNull(); - - // Act (Click Run Query) - runButton.click(); - await new Promise((resolve) => setTimeout(resolve, 50)); - - // Assert (Execution requested) - expect(model.set).toHaveBeenCalledWith('start_execution', true); - expect(model.save_changes).toHaveBeenCalled(); - expect(runButton.disabled).toBe(true); - expect(el.querySelector('.spinner')).not.toBeNull(); - - // Act (Simulate Python load completion) - state.is_deferred_mode = false; - state.table_html = '
    Data Loaded
    '; - state.row_count = 50; - - if (listeners['change:is_deferred_mode']) { - listeners['change:is_deferred_mode'](); - } - if (listeners['change:table_html']) { - listeners['change:table_html'](); - } - if (listeners['change:row_count']) { - listeners['change:row_count'](); - } - await new Promise((resolve) => setTimeout(resolve, 200)); - - // Assert (Transition to loaded state) - expect(el.querySelector('.deferred-container')).toBeNull(); - const tableContainer = el.querySelector('.table-container'); - expect(tableContainer).not.toBeNull(); - expect(el.textContent).toContain('Data Loaded'); - expect(el.textContent).toContain('50 total rows'); - - // Clean up - document.body.removeChild(el); - }); -}); diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 0b30c331a1e..9c4fcf58b17 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -12,11 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64 -import datetime -import decimal +from datetime import datetime import hashlib -import json import logging import math import pathlib @@ -25,21 +22,15 @@ import typing from typing import Dict, Generator, Optional -import db_dtypes # type: ignore[import-untyped] -import fsspec # type: ignore[import-untyped] -import gcsfs # type: ignore[import-untyped] -import geopandas as gpd # type: ignore[import-untyped] import google.api_core.exceptions import google.cloud.bigquery as bigquery import google.cloud.bigquery_connection_v1 as bigquery_connection_v1 -import google.cloud.bigquery_storage_v1 import google.cloud.exceptions import google.cloud.functions_v2 as functions_v2 import google.cloud.resourcemanager_v3 as resourcemanager_v3 import google.cloud.storage as storage # type: ignore import numpy as np import pandas as pd -import pandas.arrays import pyarrow as pa import pytest import pytz @@ -79,15 +70,6 @@ def _hash_digest_file(hasher, filepath): hasher.update(chunk) -@pytest.fixture(scope="session", autouse=True) -def configure_gcsfs(): - # gcsfs by default uses a cache that can be stale, causing file loads to - # fail if the file was uploaded indirectly (eg via bq export job) during the - # course of the tests. disable the cache to avoid this. - fsspec.config.conf["gcs"] = {"use_listings_cache": False} - gcsfs.GCSFileSystem.clear_instance_cache() - - @pytest.fixture(scope="session") def tokyo_location() -> str: return TOKYO_LOCATION @@ -121,13 +103,6 @@ def bigquery_client(session: bigframes.Session) -> bigquery.Client: return session.bqclient -@pytest.fixture(scope="session") -def bigquery_storage_read_client( - session: bigframes.Session, -) -> google.cloud.bigquery_storage_v1.BigQueryReadClient: - return session.bqstoragereadclient - - @pytest.fixture(scope="session") def bigquery_client_tokyo(session_tokyo: bigframes.Session) -> bigquery.Client: return session_tokyo.bqclient @@ -502,213 +477,14 @@ def nested_structs_df( @pytest.fixture(scope="session") def nested_structs_pandas_df(nested_structs_pandas_type: pd.ArrowDtype) -> pd.DataFrame: - """pd.DataFrame pointing at test data. - - Manually parses using json.loads to preserve data types. - """ - with open(DATA_DIR / "nested_structs.jsonl") as f: - raw_rows = [json.loads(line) for line in f] - - ids = [row["id"] for row in raw_rows] - - def get_val(row, col_name): - return row.get(col_name) - - # person - person_struct_schema = nested_structs_pandas_type.pyarrow_dtype - processed_person: list[Optional[dict[str, typing.Any]]] = [] - for row in raw_rows: - x = get_val(row, "person") - if x is None: - processed_person.append(None) - else: - d = dict(x) - if "age" in d and d["age"] is not None: - d["age"] = int(d["age"]) - processed_person.append(d) - person_arr = pa.array(processed_person, type=person_struct_schema) - person_ser = pd.Series(person_arr, index=ids, dtype=nested_structs_pandas_type) - - # bool_col - bool_vals = [ - bool(get_val(row, "bool_col")) if get_val(row, "bool_col") is not None else None - for row in raw_rows - ] - bool_ser = pd.Series(bool_vals, index=ids, dtype=pd.BooleanDtype()) - - # int64_col - int64_vals = [ - int(get_val(row, "int64_col")) - if get_val(row, "int64_col") is not None - else None - for row in raw_rows - ] - int64_ser = pd.Series(int64_vals, index=ids, dtype=pd.Int64Dtype()) - - # float64_col - float64_vals = [ - float(get_val(row, "float64_col")) - if get_val(row, "float64_col") is not None - else None - for row in raw_rows - ] - np_vals = np.array( - [x if x is not None else np.nan for x in float64_vals], dtype=np.float64 - ) - mask = np.array([x is None for x in float64_vals], dtype=bool) - float64_arr = pd.arrays.FloatingArray(np_vals, mask) # type: ignore - float64_ser = pd.Series(float64_arr, index=ids) - - # string_col - string_vals = [ - str(get_val(row, "string_col")) - if get_val(row, "string_col") is not None - else None - for row in raw_rows - ] - string_ser = pd.Series( - string_vals, index=ids, dtype=pd.StringDtype(storage="pyarrow") - ) - - # json_col - json_strs: list[Optional[str]] = [] - for row in raw_rows: - if "json_col" not in row: - json_strs.append(None) - elif row["json_col"] is None: - json_strs.append("null") - else: - json_strs.append( - json.dumps(row["json_col"], sort_keys=True, separators=(",", ":")) - ) - json_arr = pa.array(json_strs, type=db_dtypes.JSONArrowType()) - json_ser = pd.Series( - json_arr, index=ids, dtype=pd.ArrowDtype(db_dtypes.JSONArrowType()) - ) - - # date_col - date_vals = [ - datetime.date.fromisoformat(get_val(row, "date_col")) - if get_val(row, "date_col") is not None - else None - for row in raw_rows - ] - date_arr = pa.array(date_vals, type=pa.date32()) - date_ser = pd.Series(date_arr, index=ids, dtype=pd.ArrowDtype(pa.date32())) - - # time_col - time_vals = [ - datetime.time.fromisoformat(get_val(row, "time_col")) - if get_val(row, "time_col") is not None - else None - for row in raw_rows - ] - time_arr = pa.array(time_vals, type=pa.time64("us")) - time_ser = pd.Series(time_arr, index=ids, dtype=pd.ArrowDtype(pa.time64("us"))) - - # datetime_col - datetime_vals: list[Optional[datetime.datetime]] = [] - for row in raw_rows: - val = get_val(row, "datetime_col") - if val is None: - datetime_vals.append(None) - else: - datetime_vals.append(datetime.datetime.fromisoformat(val.replace(" ", "T"))) - datetime_arr = pa.array(datetime_vals, type=pa.timestamp("us")) - datetime_ser = pd.Series( - datetime_arr, index=ids, dtype=pd.ArrowDtype(pa.timestamp("us")) - ) - - # timestamp_col - timestamp_vals = [ - datetime.datetime.fromisoformat( - get_val(row, "timestamp_col").replace("Z", "+00:00") - ) - if get_val(row, "timestamp_col") is not None - else None - for row in raw_rows - ] - timestamp_arr = pa.array(timestamp_vals, type=pa.timestamp("us", tz="UTC")) - timestamp_ser = pd.Series( - timestamp_arr, index=ids, dtype=pd.ArrowDtype(pa.timestamp("us", tz="UTC")) - ) - - # bytes_col - bytes_vals: list[Optional[bytes]] = [] - for row in raw_rows: - val = get_val(row, "bytes_col") - if val is None: - bytes_vals.append(None) - elif val == "": - bytes_vals.append(b"") - else: - bytes_vals.append(base64.b64decode(val)) - bytes_arr = pa.array(bytes_vals, type=pa.binary()) - bytes_ser = pd.Series(bytes_arr, index=ids, dtype=pd.ArrowDtype(pa.binary())) - - # numeric_col - numeric_vals = [ - decimal.Decimal(str(get_val(row, "numeric_col"))) - if get_val(row, "numeric_col") is not None - else None - for row in raw_rows - ] - numeric_arr = pa.array(numeric_vals, type=pa.decimal128(38, 9)) - numeric_ser = pd.Series( - numeric_arr, index=ids, dtype=pd.ArrowDtype(pa.decimal128(38, 9)) - ) - - # bignumeric_col - bignumeric_vals = [ - decimal.Decimal(str(get_val(row, "bignumeric_col"))) - if get_val(row, "bignumeric_col") is not None - else None - for row in raw_rows - ] - bignumeric_arr = pa.array(bignumeric_vals, type=pa.decimal256(76, 38)) - bignumeric_ser = pd.Series( - bignumeric_arr, index=ids, dtype=pd.ArrowDtype(pa.decimal256(76, 38)) - ) - - # geography_col - geo_vals = [get_val(row, "geography_col") for row in raw_rows] - geo_ser = gpd.GeoSeries.from_wkt(geo_vals) - geo_ser.index = ids - - # duration_col - duration_vals = [ - int(get_val(row, "duration_col")) - if get_val(row, "duration_col") is not None - else None - for row in raw_rows - ] - duration_arr = pa.array(duration_vals, type=pa.duration("us")) - duration_ser = pd.Series( - duration_arr, index=ids, dtype=pd.ArrowDtype(pa.duration("us")) - ) + """pd.DataFrame pointing at test data.""" - df = pd.DataFrame( - { - "person": person_ser, - "bool_col": bool_ser, - "int64_col": int64_ser, - "float64_col": float64_ser, - "string_col": string_ser, - "json_col": json_ser, - "date_col": date_ser, - "time_col": time_ser, - "datetime_col": datetime_ser, - "timestamp_col": timestamp_ser, - "bytes_col": bytes_ser, - "numeric_col": numeric_ser, - "bignumeric_col": bignumeric_ser, - "geography_col": geo_ser, - "duration_col": duration_ser, - }, - index=ids, + df = pd.read_json( + DATA_DIR / "nested_structs.jsonl", + lines=True, ) - df.index.name = "id" - + df = df.set_index("id") + df["person"] = df["person"].astype(nested_structs_pandas_type) return df @@ -1039,9 +815,9 @@ def new_time_series_pandas_df(): return pd.DataFrame( { "parsed_date": [ - datetime.datetime(2017, 8, 2, tzinfo=utc), - datetime.datetime(2017, 8, 3, tzinfo=utc), - datetime.datetime(2017, 8, 4, tzinfo=utc), + datetime(2017, 8, 2, tzinfo=utc), + datetime(2017, 8, 3, tzinfo=utc), + datetime(2017, 8, 4, tzinfo=utc), ], "total_visits": [2500, 2500, 2500], } @@ -1060,12 +836,12 @@ def new_time_series_pandas_df_w_id(): return pd.DataFrame( { "parsed_date": [ - datetime.datetime(2017, 8, 2, tzinfo=utc), - datetime.datetime(2017, 8, 2, tzinfo=utc), - datetime.datetime(2017, 8, 3, tzinfo=utc), - datetime.datetime(2017, 8, 3, tzinfo=utc), - datetime.datetime(2017, 8, 4, tzinfo=utc), - datetime.datetime(2017, 8, 4, tzinfo=utc), + datetime(2017, 8, 2, tzinfo=utc), + datetime(2017, 8, 2, tzinfo=utc), + datetime(2017, 8, 3, tzinfo=utc), + datetime(2017, 8, 3, tzinfo=utc), + datetime(2017, 8, 4, tzinfo=utc), + datetime(2017, 8, 4, tzinfo=utc), ], "id": ["1", "2", "1", "2", "1", "2"], "total_visits": [2500, 2500, 2500, 2500, 2500, 2500], @@ -1530,14 +1306,6 @@ def usa_names_grouped_table( return session.bqclient.get_table(table_id) -@pytest.fixture(scope="session", autouse=True) -def use_sqlglot_compiler(): - original_setting = bigframes.options.experiments.sql_compiler - bigframes.options.experiments.sql_compiler = "experimental" - yield - bigframes.options.experiments.sql_compiler = original_setting - - @pytest.fixture() def restore_sampling_settings(): enable_downsampling = bigframes.options.sampling.enable_downsampling @@ -1678,7 +1446,7 @@ def cleanup_cloud_functions(session, cloudfunctions_client, dataset_id_permanent continue # Ignore the functions less than one day old - age = datetime.datetime.now() - datetime.datetime.fromtimestamp( + age = datetime.now() - datetime.fromtimestamp( cloud_function.update_time.timestamp() ) if age.days <= 0: @@ -1731,6 +1499,16 @@ def images_uris() -> list[str]: ] +@pytest.fixture(scope="session") +def images_mm_df( + images_uris, session: bigframes.Session, bq_connection: str +) -> bpd.DataFrame: + blob_series = bpd.Series(images_uris, session=session).str.to_blob( + connection=bq_connection + ) + return blob_series.rename("blob_col").to_frame() + + @pytest.fixture() def reset_default_session_and_location(): bpd.close_session() @@ -1738,3 +1516,29 @@ def reset_default_session_and_location(): yield bpd.close_session() bpd.options.bigquery.location = None + + +@pytest.fixture(scope="session") +def pdf_gcs_path() -> str: + return "gs://bigframes_blob_test/pdfs/*" + + +@pytest.fixture(scope="session") +def pdf_mm_df( + pdf_gcs_path, session: bigframes.Session, bq_connection: str +) -> bpd.DataFrame: + return session.from_glob_path(pdf_gcs_path, name="pdf", connection=bq_connection) + + +@pytest.fixture(scope="session") +def audio_gcs_path() -> str: + return "gs://bigframes_blob_test/audio/*" + + +@pytest.fixture(scope="session") +def audio_mm_df( + audio_gcs_path, session: bigframes.Session, bq_connection: str +) -> bpd.DataFrame: + return session.from_glob_path( + audio_gcs_path, name="audio", connection=bq_connection + ) diff --git a/tests/system/large/bigquery/test_ai.py b/tests/system/large/bigquery/test_ai.py deleted file mode 100644 index 504fe5aa389..00000000000 --- a/tests/system/large/bigquery/test_ai.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - -import bigframes.pandas as bpd -from bigframes.bigquery import ai, ml - - -@pytest.fixture(scope="session") -def embedding_model(bq_connection, dataset_id): - model_name = f"{dataset_id}.embedding_model" - return ml.create_model( - model_name=model_name, - options={"endpoint": "gemini-embedding-001"}, - connection_name=bq_connection, - ) - - -@pytest.fixture(scope="session") -def text_model(bq_connection, dataset_id): - model_name = f"{dataset_id}.text_model" - return ml.create_model( - model_name=model_name, - options={"endpoint": "gemini-2.5-flash"}, - connection_name=bq_connection, - ) - - -def test_generate_embedding(embedding_model): - df = bpd.DataFrame( - { - "content": [ - "What is BigQuery?", - "What is BQML?", - ] - } - ) - - result = ai.generate_embedding(embedding_model, df) - - assert len(result) == 2 - assert "embedding" in result.columns - assert "statistics" in result.columns - assert "status" in result.columns - - -def test_generate_embedding_with_options(embedding_model): - df = bpd.DataFrame( - { - "content": [ - "What is BigQuery?", - "What is BQML?", - ] - } - ) - - result = ai.generate_embedding( - embedding_model, df, task_type="RETRIEVAL_DOCUMENT", output_dimensionality=256 - ) - - assert len(result) == 2 - embedding = result["embedding"].to_pandas() - assert len(embedding[0]) == 256 - - -def test_generate_text(text_model): - df = bpd.DataFrame({"prompt": ["Dog", "Cat"]}) - - result = ai.generate_text(text_model, df) - - assert len(result) == 2 - assert "result" in result.columns - assert "statistics" in result.columns - assert "full_response" in result.columns - assert "status" in result.columns - - -def test_generate_text_with_options(text_model): - df = bpd.DataFrame({"prompt": ["Dog", "Cat"]}) - - result = ai.generate_text(text_model, df, max_output_tokens=1) - - # It basically asserts that the results are still returned. - assert len(result) == 2 - - -def test_generate_table(text_model): - df = bpd.DataFrame( - {"prompt": ["Generate a table of 2 programming languages and their creators."]} - ) - - result = ai.generate_table( - text_model, - df, - output_schema="language STRING, creator STRING", - ) - - assert "language" in result.columns - assert "creator" in result.columns - # The model may not always return the exact number of rows requested. - assert len(result) > 0 - - -def test_generate_table_with_mapping_schema(text_model): - df = bpd.DataFrame( - {"prompt": ["Generate a table of 2 programming languages and their creators."]} - ) - - result = ai.generate_table( - text_model, - df, - output_schema={"language": "STRING", "creator": "STRING"}, - ) - - assert "language" in result.columns - assert "creator" in result.columns - # The model may not always return the exact number of rows requested. - assert len(result) > 0 diff --git a/tests/system/large/bigquery/test_io.py b/tests/system/large/bigquery/test_io.py deleted file mode 100644 index 024c6174709..00000000000 --- a/tests/system/large/bigquery/test_io.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for for the specific language governing permissions and -# limitations under the License. - -import bigframes.bigquery as bbq - - -def test_load_data(session, dataset_id): - table_name = f"{dataset_id}.test_load_data" - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv" - - # Create the external table - table = bbq.load_data( - table_name, - columns={ - "name": "STRING", - "post_abbr": "STRING", - }, - from_files_options={"format": "CSV", "uris": [uri], "skip_leading_rows": 1}, - session=session, - ) - assert table is not None - - # Read the table to verify - import bigframes.pandas as bpd - - bf_df = bpd.read_gbq(table_name) - pd_df = bf_df.to_pandas() - assert len(pd_df) > 0 diff --git a/tests/system/large/bigquery/test_ml.py b/tests/system/large/bigquery/test_ml.py deleted file mode 100644 index f0f7d4f6917..00000000000 --- a/tests/system/large/bigquery/test_ml.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - -import bigframes.bigquery.ml as ml -import bigframes.pandas as bpd - - -@pytest.fixture(scope="session") -def embedding_model(bq_connection, dataset_id): - model_name = f"{dataset_id}.embedding_model" - return ml.create_model( - model_name=model_name, - options={"endpoint": "gemini-embedding-001"}, - connection_name=bq_connection, - ) - - -def test_generate_embedding(embedding_model): - df = bpd.DataFrame( - { - "content": [ - "What is BigQuery?", - "What is BQML?", - ] - } - ) - - result = ml.generate_embedding(embedding_model, df) - assert len(result) == 2 - assert "ml_generate_embedding_result" in result.columns - assert "ml_generate_embedding_status" in result.columns - - -def test_generate_embedding_with_options(embedding_model): - df = bpd.DataFrame( - { - "content": [ - "What is BigQuery?", - "What is BQML?", - ] - } - ) - - result = ml.generate_embedding( - embedding_model, df, task_type="RETRIEVAL_DOCUMENT", output_dimensionality=256 - ) - assert len(result) == 2 - assert "ml_generate_embedding_result" in result.columns - assert "ml_generate_embedding_status" in result.columns - embedding = result["ml_generate_embedding_result"].to_pandas() - assert len(embedding[0]) == 256 - - -def test_get_insights(dataset_id): - df = bpd.DataFrame( - { - "dim1": ["a", "a", "b", "b", "a", "a", "b", "b"], - "dim2": ["x", "y", "x", "y", "x", "y", "x", "y"], - "metric": [10, 20, 30, 40, 12, 25, 35, 45], - "is_test": [False, False, False, False, True, True, True, True], - } - ) - model_name = f"{dataset_id}.contribution_analysis_model" - - ml.create_model( - model_name=model_name, - options={ - "model_type": "CONTRIBUTION_ANALYSIS", - "contribution_metric": "SUM(metric)", - "is_test_col": "is_test", - }, - training_data=df, - ) - - result = ml.get_insights(model_name) - assert len(result) > 0 - assert "contributors" in result.columns - - -def test_create_model_linear_regression(dataset_id): - df = bpd.DataFrame({"x": [1, 2, 3], "y": [2, 4, 6]}) - model_name = f"{dataset_id}.linear_regression_model" - - result = ml.create_model( - model_name=model_name, - options={"model_type": "LINEAR_REG", "input_label_cols": ["y"]}, - training_data=df, - ) - - assert result["modelType"] == "LINEAR_REGRESSION" - - -def test_create_model_with_transform(dataset_id): - df = bpd.DataFrame({"x": [1, 2, 3], "y": [2, 4, 6]}) - model_name = f"{dataset_id}.transform_model" - - result = ml.create_model( - model_name=model_name, - options={"model_type": "LINEAR_REG", "input_label_cols": ["y"]}, - training_data=df, - transform=["x * 2 AS x_doubled", "y"], - ) - - assert result["modelType"] == "LINEAR_REGRESSION" diff --git a/tests/system/large/bigquery/test_obj.py b/tests/system/large/bigquery/test_obj.py new file mode 100644 index 00000000000..dcca7580b14 --- /dev/null +++ b/tests/system/large/bigquery/test_obj.py @@ -0,0 +1,41 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import bigframes.bigquery as bbq + + +@pytest.fixture() +def objectrefs(bq_connection): + return bbq.obj.make_ref( + [ + "gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/images/tick-terminator-for-dogs.png" + ], + bq_connection, + ) + + +def test_obj_fetch_metadata(objectrefs): + metadata = bbq.obj.fetch_metadata(objectrefs) + + result = metadata.to_pandas() + assert len(result) == len(objectrefs) + + +def test_obj_get_access_url(objectrefs): + access = bbq.obj.get_access_url(objectrefs, "r") + + result = access.to_pandas() + assert len(result) == len(objectrefs) diff --git a/tests/system/large/blob/test_function.py b/tests/system/large/blob/test_function.py new file mode 100644 index 00000000000..7963fabd0b6 --- /dev/null +++ b/tests/system/large/blob/test_function.py @@ -0,0 +1,851 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import traceback +from typing import Generator +import uuid + +from google.cloud import storage +import pandas as pd +import pytest + +import bigframes +from bigframes import dtypes +import bigframes.pandas as bpd + + +@pytest.fixture(scope="function") +def images_output_folder() -> Generator[str, None, None]: + id = uuid.uuid4().hex + folder = os.path.join("gs://bigframes_blob_test/output/", id) + yield folder + + # clean up + try: + cloud_storage_client = storage.Client() + bucket = cloud_storage_client.bucket("bigframes_blob_test") + blobs = bucket.list_blobs(prefix="output/" + id) + for blob in blobs: + blob.delete() + except Exception as exc: + traceback.print_exception(type(exc), exc, None) + + +@pytest.fixture(scope="function") +def images_output_uris(images_output_folder: str) -> list[str]: + return [ + os.path.join(images_output_folder, "img0.jpg"), + os.path.join(images_output_folder, "img1.jpg"), + ] + + +def test_blob_exif( + bq_connection: str, + session: bigframes.Session, +): + exif_image_df = session.from_glob_path( + "gs://bigframes_blob_test/images_exif/*", + name="blob_col", + connection=bq_connection, + ) + + actual = exif_image_df["blob_col"].blob.exif( + engine="pillow", connection=bq_connection, verbose=False + ) + expected = bpd.Series( + ['{"ExifOffset": 47, "Make": "MyCamera"}'], + session=session, + dtype=dtypes.JSON_DTYPE, + ) + pd.testing.assert_series_equal( + actual.to_pandas(), + expected.to_pandas(), + check_dtype=False, + check_index_type=False, + ) + + +def test_blob_exif_verbose( + bq_connection: str, + session: bigframes.Session, +): + exif_image_df = session.from_glob_path( + "gs://bigframes_blob_test/images_exif/*", + name="blob_col", + connection=bq_connection, + ) + + actual = exif_image_df["blob_col"].blob.exif( + engine="pillow", connection=bq_connection, verbose=True + ) + assert hasattr(actual, "struct") + actual_exploded = actual.struct.explode() + assert "status" in actual_exploded.columns + assert "content" in actual_exploded.columns + + status_series = actual_exploded["status"] + assert status_series.dtype == dtypes.STRING_DTYPE + + content_series = actual_exploded["content"] + assert content_series.dtype == dtypes.JSON_DTYPE + + +def test_blob_image_blur_to_series( + images_mm_df: bpd.DataFrame, + bq_connection: str, + images_output_uris: list[str], + session: bigframes.Session, +): + series = bpd.Series(images_output_uris, session=session).str.to_blob( + connection=bq_connection + ) + + actual = images_mm_df["blob_col"].blob.image_blur( + (8, 8), dst=series, connection=bq_connection, engine="opencv", verbose=False + ) + + expected_df = pd.DataFrame( + { + "uri": images_output_uris, + "version": [None, None], + "authorizer": [bq_connection.casefold(), bq_connection.casefold()], + "details": [None, None], + } + ) + pd.testing.assert_frame_equal( + actual.struct.explode().to_pandas(), + expected_df, + check_dtype=False, + check_index_type=False, + ) + + # verify the files exist + assert not actual.blob.size().isna().any() + + +def test_blob_image_blur_to_series_verbose( + images_mm_df: bpd.DataFrame, + bq_connection: str, + images_output_uris: list[str], + session: bigframes.Session, +): + series = bpd.Series(images_output_uris, session=session).str.to_blob( + connection=bq_connection + ) + + actual = images_mm_df["blob_col"].blob.image_blur( + (8, 8), dst=series, connection=bq_connection, engine="opencv", verbose=True + ) + + assert hasattr(actual, "struct") + actual_exploded = actual.struct.explode() + assert "status" in actual_exploded.columns + assert "content" in actual_exploded.columns + + status_series = actual_exploded["status"] + assert status_series.dtype == dtypes.STRING_DTYPE + + # Content should be blob objects for GCS destination + # verify the files exist + assert not actual.blob.size().isna().any() + + +def test_blob_image_blur_to_folder( + images_mm_df: bpd.DataFrame, + bq_connection: str, + images_output_folder: str, + images_output_uris: list[str], +): + actual = images_mm_df["blob_col"].blob.image_blur( + (8, 8), + dst=images_output_folder, + connection=bq_connection, + engine="opencv", + verbose=False, + ) + expected_df = pd.DataFrame( + { + "uri": images_output_uris, + "version": [None, None], + "authorizer": [bq_connection.casefold(), bq_connection.casefold()], + "details": [None, None], + } + ) + pd.testing.assert_frame_equal( + actual.struct.explode().to_pandas(), + expected_df, + check_dtype=False, + check_index_type=False, + ) + + # verify the files exist + assert not actual.blob.size().isna().any() + + +def test_blob_image_blur_to_folder_verbose( + images_mm_df: bpd.DataFrame, + bq_connection: str, + images_output_folder: str, + images_output_uris: list[str], +): + actual = images_mm_df["blob_col"].blob.image_blur( + (8, 8), + dst=images_output_folder, + connection=bq_connection, + engine="opencv", + verbose=True, + ) + assert hasattr(actual, "struct") + actual_exploded = actual.struct.explode() + assert "status" in actual_exploded.columns + assert "content" in actual_exploded.columns + + status_series = actual_exploded["status"] + assert status_series.dtype == dtypes.STRING_DTYPE + + content_series = actual_exploded["content"] + # Content should be blob objects for GCS destination + assert hasattr(content_series, "blob") + + # verify the files exist + assert not actual.blob.size().isna().any() + + +def test_blob_image_blur_to_bq(images_mm_df: bpd.DataFrame, bq_connection: str): + actual = images_mm_df["blob_col"].blob.image_blur( + (8, 8), connection=bq_connection, engine="opencv", verbose=False + ) + + assert isinstance(actual, bpd.Series) + assert len(actual) == 2 + assert actual.dtype == dtypes.BYTES_DTYPE + + +def test_blob_image_blur_to_bq_verbose(images_mm_df: bpd.DataFrame, bq_connection: str): + actual = images_mm_df["blob_col"].blob.image_blur( + (8, 8), connection=bq_connection, engine="opencv", verbose=True + ) + + assert isinstance(actual, bpd.Series) + assert len(actual) == 2 + + assert hasattr(actual, "struct") + actual_exploded = actual.struct.explode() + assert "status" in actual_exploded.columns + assert "content" in actual_exploded.columns + + status_series = actual_exploded["status"] + assert status_series.dtype == dtypes.STRING_DTYPE + + content_series = actual_exploded["content"] + assert content_series.dtype == dtypes.BYTES_DTYPE + + +def test_blob_image_resize_to_series( + images_mm_df: bpd.DataFrame, + bq_connection: str, + images_output_uris: list[str], + session: bigframes.Session, +): + series = bpd.Series(images_output_uris, session=session).str.to_blob( + connection=bq_connection + ) + + actual = images_mm_df["blob_col"].blob.image_resize( + (200, 300), + dst=series, + connection=bq_connection, + engine="opencv", + verbose=False, + ) + + expected_df = pd.DataFrame( + { + "uri": images_output_uris, + "version": [None, None], + "authorizer": [bq_connection.casefold(), bq_connection.casefold()], + "details": [None, None], + } + ) + pd.testing.assert_frame_equal( + actual.struct.explode().to_pandas(), + expected_df, + check_dtype=False, + check_index_type=False, + ) + + # verify the files exist + assert not actual.blob.size().isna().any() + + +def test_blob_image_resize_to_series_verbose( + images_mm_df: bpd.DataFrame, + bq_connection: str, + images_output_uris: list[str], + session: bigframes.Session, +): + series = bpd.Series(images_output_uris, session=session).str.to_blob( + connection=bq_connection + ) + + actual = images_mm_df["blob_col"].blob.image_resize( + (200, 300), + dst=series, + connection=bq_connection, + engine="opencv", + verbose=True, + ) + + assert hasattr(actual, "struct") + actual_exploded = actual.struct.explode() + assert "status" in actual_exploded.columns + assert "content" in actual_exploded.columns + + status_series = actual_exploded["status"] + assert status_series.dtype == dtypes.STRING_DTYPE + + content_series = actual_exploded["content"] + # Content should be blob objects for GCS destination + assert hasattr(content_series, "blob") + + # verify the files exist + assert not actual.blob.size().isna().any() + + +def test_blob_image_resize_to_folder( + images_mm_df: bpd.DataFrame, + bq_connection: str, + images_output_folder: str, + images_output_uris: list[str], +): + actual = images_mm_df["blob_col"].blob.image_resize( + (200, 300), + dst=images_output_folder, + connection=bq_connection, + engine="opencv", + verbose=False, + ) + + expected_df = pd.DataFrame( + { + "uri": images_output_uris, + "version": [None, None], + "authorizer": [bq_connection.casefold(), bq_connection.casefold()], + "details": [None, None], + } + ) + pd.testing.assert_frame_equal( + actual.struct.explode().to_pandas(), + expected_df, + check_dtype=False, + check_index_type=False, + ) + + # verify the files exist + assert not actual.blob.size().isna().any() + + +def test_blob_image_resize_to_folder_verbose( + images_mm_df: bpd.DataFrame, + bq_connection: str, + images_output_folder: str, + images_output_uris: list[str], +): + actual = images_mm_df["blob_col"].blob.image_resize( + (200, 300), + dst=images_output_folder, + connection=bq_connection, + engine="opencv", + verbose=True, + ) + + assert hasattr(actual, "struct") + actual_exploded = actual.struct.explode() + assert "status" in actual_exploded.columns + assert "content" in actual_exploded.columns + + status_series = actual_exploded["status"] + assert status_series.dtype == dtypes.STRING_DTYPE + + content_series = actual_exploded["content"] + # Content should be blob objects for GCS destination + assert hasattr(content_series, "blob") + + # verify the files exist + assert not content_series.blob.size().isna().any() + + +def test_blob_image_resize_to_bq(images_mm_df: bpd.DataFrame, bq_connection: str): + actual = images_mm_df["blob_col"].blob.image_resize( + (200, 300), connection=bq_connection, engine="opencv", verbose=False + ) + + assert isinstance(actual, bpd.Series) + assert len(actual) == 2 + assert actual.dtype == dtypes.BYTES_DTYPE + + +def test_blob_image_resize_to_bq_verbose( + images_mm_df: bpd.DataFrame, bq_connection: str +): + actual = images_mm_df["blob_col"].blob.image_resize( + (200, 300), connection=bq_connection, engine="opencv", verbose=True + ) + + assert isinstance(actual, bpd.Series) + assert len(actual) == 2 + + assert hasattr(actual, "struct") + actual_exploded = actual.struct.explode() + assert "status" in actual_exploded.columns + assert "content" in actual_exploded.columns + + status_series = actual_exploded["status"] + assert status_series.dtype == dtypes.STRING_DTYPE + + content_series = actual_exploded["content"] + assert content_series.dtype == dtypes.BYTES_DTYPE + + +def test_blob_image_normalize_to_series( + images_mm_df: bpd.DataFrame, + bq_connection: str, + images_output_uris: list[str], + session: bigframes.Session, +): + series = bpd.Series(images_output_uris, session=session).str.to_blob( + connection=bq_connection + ) + + actual = images_mm_df["blob_col"].blob.image_normalize( + alpha=50.0, + beta=150.0, + norm_type="minmax", + dst=series, + connection=bq_connection, + engine="opencv", + verbose=False, + ) + + expected_df = pd.DataFrame( + { + "uri": images_output_uris, + "version": [None, None], + "authorizer": [bq_connection.casefold(), bq_connection.casefold()], + "details": [None, None], + } + ) + pd.testing.assert_frame_equal( + actual.struct.explode().to_pandas(), + expected_df, + check_dtype=False, + check_index_type=False, + ) + + # verify the files exist + assert not actual.blob.size().isna().any() + + +def test_blob_image_normalize_to_series_verbose( + images_mm_df: bpd.DataFrame, + bq_connection: str, + images_output_uris: list[str], + session: bigframes.Session, +): + series = bpd.Series(images_output_uris, session=session).str.to_blob( + connection=bq_connection + ) + + actual = images_mm_df["blob_col"].blob.image_normalize( + alpha=50.0, + beta=150.0, + norm_type="minmax", + dst=series, + connection=bq_connection, + engine="opencv", + verbose=True, + ) + + assert hasattr(actual, "struct") + actual_exploded = actual.struct.explode() + assert "status" in actual_exploded.columns + assert "content" in actual_exploded.columns + + status_series = actual_exploded["status"] + assert status_series.dtype == dtypes.STRING_DTYPE + + content_series = actual_exploded["content"] + # Content should be blob objects for GCS destination + assert hasattr(content_series, "blob") + + +def test_blob_image_normalize_to_folder( + images_mm_df: bpd.DataFrame, + bq_connection: str, + images_output_folder: str, + images_output_uris: list[str], +): + actual = images_mm_df["blob_col"].blob.image_normalize( + alpha=50.0, + beta=150.0, + norm_type="minmax", + dst=images_output_folder, + connection=bq_connection, + engine="opencv", + verbose=False, + ) + + expected_df = pd.DataFrame( + { + "uri": images_output_uris, + "version": [None, None], + "authorizer": [bq_connection.casefold(), bq_connection.casefold()], + "details": [None, None], + } + ) + pd.testing.assert_frame_equal( + actual.struct.explode().to_pandas(), + expected_df, + check_dtype=False, + check_index_type=False, + ) + + # verify the files exist + assert not actual.blob.size().isna().any() + + +def test_blob_image_normalize_to_folder_verbose( + images_mm_df: bpd.DataFrame, + bq_connection: str, + images_output_folder: str, + images_output_uris: list[str], +): + actual = images_mm_df["blob_col"].blob.image_normalize( + alpha=50.0, + beta=150.0, + norm_type="minmax", + dst=images_output_folder, + connection=bq_connection, + engine="opencv", + verbose=True, + ) + + assert hasattr(actual, "struct") + actual_exploded = actual.struct.explode() + assert "status" in actual_exploded.columns + assert "content" in actual_exploded.columns + + status_series = actual_exploded["status"] + assert status_series.dtype == dtypes.STRING_DTYPE + + content_series = actual_exploded["content"] + # Content should be blob objects for GCS destination + assert hasattr(content_series, "blob") + + +def test_blob_image_normalize_to_bq(images_mm_df: bpd.DataFrame, bq_connection: str): + actual = images_mm_df["blob_col"].blob.image_normalize( + alpha=50.0, + beta=150.0, + norm_type="minmax", + connection=bq_connection, + engine="opencv", + verbose=False, + ) + + assert isinstance(actual, bpd.Series) + assert len(actual) == 2 + assert actual.dtype == dtypes.BYTES_DTYPE + + +def test_blob_image_normalize_to_bq_verbose( + images_mm_df: bpd.DataFrame, bq_connection: str +): + actual = images_mm_df["blob_col"].blob.image_normalize( + alpha=50.0, + beta=150.0, + norm_type="minmax", + connection=bq_connection, + engine="opencv", + verbose=True, + ) + + assert isinstance(actual, bpd.Series) + assert len(actual) == 2 + + assert hasattr(actual, "struct") + actual_exploded = actual.struct.explode() + assert "status" in actual_exploded.columns + assert "content" in actual_exploded.columns + + status_series = actual_exploded["status"] + assert status_series.dtype == dtypes.STRING_DTYPE + + content_series = actual_exploded["content"] + assert content_series.dtype == dtypes.BYTES_DTYPE + + +def test_blob_pdf_extract( + pdf_mm_df: bpd.DataFrame, + bq_connection: str, +): + actual = ( + pdf_mm_df["pdf"] + .blob.pdf_extract(connection=bq_connection, verbose=False, engine="pypdf") + .explode() + .to_pandas() + ) + + # check relative length + expected_text = "Sample PDF This is a testing file. Some dummy messages are used for testing purposes." + expected_len = len(expected_text) + + actual_text = actual[actual != ""].iloc[0] + actual_len = len(actual_text) + + relative_length_tolerance = 0.25 + min_acceptable_len = expected_len * (1 - relative_length_tolerance) + max_acceptable_len = expected_len * (1 + relative_length_tolerance) + assert min_acceptable_len <= actual_len <= max_acceptable_len, ( + f"Item (verbose=False): Extracted text length {actual_len} is outside the acceptable range " + f"[{min_acceptable_len:.0f}, {max_acceptable_len:.0f}]. " + f"Expected reference length was {expected_len}. " + ) + + # check for major keywords + major_keywords = ["Sample", "PDF", "testing", "dummy", "messages"] + for keyword in major_keywords: + assert ( + keyword.lower() in actual_text.lower() + ), f"Item (verbose=False): Expected keyword '{keyword}' not found in extracted text. " + + +def test_blob_pdf_extract_verbose( + pdf_mm_df: bpd.DataFrame, + bq_connection: str, +): + actual = ( + pdf_mm_df["pdf"] + .blob.pdf_extract(connection=bq_connection, verbose=True, engine="pypdf") + .explode() + .to_pandas() + ) + + # check relative length + expected_text = "Sample PDF This is a testing file. Some dummy messages are used for testing purposes." + expected_len = len(expected_text) + + # The first entry is for a file that doesn't exist, so we check the second one + successful_results = actual[actual.apply(lambda x: x["status"] == "")] + actual_text = successful_results.apply(lambda x: x["content"]).iloc[0] + actual_len = len(actual_text) + + relative_length_tolerance = 0.25 + min_acceptable_len = expected_len * (1 - relative_length_tolerance) + max_acceptable_len = expected_len * (1 + relative_length_tolerance) + assert min_acceptable_len <= actual_len <= max_acceptable_len, ( + f"Item (verbose=True): Extracted text length {actual_len} is outside the acceptable range " + f"[{min_acceptable_len:.0f}, {max_acceptable_len:.0f}]. " + f"Expected reference length was {expected_len}. " + ) + + # check for major keywords + major_keywords = ["Sample", "PDF", "testing", "dummy", "messages"] + for keyword in major_keywords: + assert ( + keyword.lower() in actual_text.lower() + ), f"Item (verbose=True): Expected keyword '{keyword}' not found in extracted text. " + + +def test_blob_pdf_chunk(pdf_mm_df: bpd.DataFrame, bq_connection: str): + actual = ( + pdf_mm_df["pdf"] + .blob.pdf_chunk( + connection=bq_connection, + chunk_size=50, + overlap_size=10, + verbose=False, + engine="pypdf", + ) + .explode() + .to_pandas() + ) + + # check relative length + expected_text = "Sample PDF This is a testing file. Some dummy messages are used for testing purposes." + expected_len = len(expected_text) + + # First entry is NA + actual_text = "".join(actual.dropna()) + actual_len = len(actual_text) + + relative_length_tolerance = 0.25 + min_acceptable_len = expected_len * (1 - relative_length_tolerance) + max_acceptable_len = expected_len * (1 + relative_length_tolerance) + assert min_acceptable_len <= actual_len <= max_acceptable_len, ( + f"Item (verbose=False): Extracted text length {actual_len} is outside the acceptable range " + f"[{min_acceptable_len:.0f}, {max_acceptable_len:.0f}]. " + f"Expected reference length was {expected_len}. " + ) + + # check for major keywords + major_keywords = ["Sample", "PDF", "testing", "dummy", "messages"] + for keyword in major_keywords: + assert ( + keyword.lower() in actual_text.lower() + ), f"Item (verbose=False): Expected keyword '{keyword}' not found in extracted text. " + + +def test_blob_pdf_chunk_verbose(pdf_mm_df: bpd.DataFrame, bq_connection: str): + actual = ( + pdf_mm_df["pdf"] + .blob.pdf_chunk( + connection=bq_connection, + chunk_size=50, + overlap_size=10, + verbose=True, + engine="pypdf", + ) + .explode() + .to_pandas() + ) + + # check relative length + expected_text = "Sample PDF This is a testing file. Some dummy messages are used for testing purposes." + expected_len = len(expected_text) + + # The first entry is for a file that doesn't exist, so we check the second one + successful_results = actual[actual.apply(lambda x: x["status"] == "")] + actual_text = "".join(successful_results.apply(lambda x: x["content"]).iloc[0]) + actual_len = len(actual_text) + + relative_length_tolerance = 0.25 + min_acceptable_len = expected_len * (1 - relative_length_tolerance) + max_acceptable_len = expected_len * (1 + relative_length_tolerance) + assert min_acceptable_len <= actual_len <= max_acceptable_len, ( + f"Item (verbose=True): Extracted text length {actual_len} is outside the acceptable range " + f"[{min_acceptable_len:.0f}, {max_acceptable_len:.0f}]. " + f"Expected reference length was {expected_len}. " + ) + + # check for major keywords + major_keywords = ["Sample", "PDF", "testing", "dummy", "messages"] + for keyword in major_keywords: + assert ( + keyword.lower() in actual_text.lower() + ), f"Item (verbose=True): Expected keyword '{keyword}' not found in extracted text. " + + +@pytest.mark.parametrize( + "model_name", + [ + "gemini-2.0-flash-001", + "gemini-2.0-flash-lite-001", + ], +) +def test_blob_transcribe( + audio_mm_df: bpd.DataFrame, + model_name: str, +): + actual = ( + audio_mm_df["audio"] + .blob.audio_transcribe( + model_name=model_name, # type: ignore + verbose=False, + ) + .to_pandas() + ) + + # check relative length + expected_text = "Now, as all books not primarily intended as picture-books consist principally of types composed to form letterpress" + expected_len = len(expected_text) + + actual_text = actual[0] + + if pd.isna(actual_text) or actual_text == "": + # Ensure the tests are robust to flakes in the model, which isn't + # particularly useful information for the bigframes team. + logging.warning(f"blob_transcribe() model {model_name} verbose=False failure") + return + + actual_len = len(actual_text) + + relative_length_tolerance = 0.2 + min_acceptable_len = expected_len * (1 - relative_length_tolerance) + max_acceptable_len = expected_len * (1 + relative_length_tolerance) + assert min_acceptable_len <= actual_len <= max_acceptable_len, ( + f"Item (verbose=False): Transcribed text length {actual_len} is outside the acceptable range " + f"[{min_acceptable_len:.0f}, {max_acceptable_len:.0f}]. " + f"Expected reference length was {expected_len}. " + ) + + # check for major keywords + major_keywords = ["book", "picture"] + for keyword in major_keywords: + assert ( + keyword.lower() in actual_text.lower() + ), f"Item (verbose=False): Expected keyword '{keyword}' not found in transcribed text. " + + +@pytest.mark.parametrize( + "model_name", + [ + "gemini-2.0-flash-001", + "gemini-2.0-flash-lite-001", + ], +) +def test_blob_transcribe_verbose( + audio_mm_df: bpd.DataFrame, + model_name: str, +): + actual = ( + audio_mm_df["audio"] + .blob.audio_transcribe( + model_name=model_name, # type: ignore + verbose=True, + ) + .to_pandas() + ) + + # check relative length + expected_text = "Now, as all books not primarily intended as picture-books consist principally of types composed to form letterpress" + expected_len = len(expected_text) + + actual_text = actual[0]["content"] + + if pd.isna(actual_text) or actual_text == "": + # Ensure the tests are robust to flakes in the model, which isn't + # particularly useful information for the bigframes team. + logging.warning(f"blob_transcribe() model {model_name} verbose=True failure") + return + + actual_len = len(actual_text) + + relative_length_tolerance = 0.2 + min_acceptable_len = expected_len * (1 - relative_length_tolerance) + max_acceptable_len = expected_len * (1 + relative_length_tolerance) + assert min_acceptable_len <= actual_len <= max_acceptable_len, ( + f"Item (verbose=True): Transcribed text length {actual_len} is outside the acceptable range " + f"[{min_acceptable_len:.0f}, {max_acceptable_len:.0f}]. " + f"Expected reference length was {expected_len}. " + ) + + # check for major keywords + major_keywords = ["book", "picture"] + for keyword in major_keywords: + assert ( + keyword.lower() in actual_text.lower() + ), f"Item (verbose=True): Expected keyword '{keyword}' not found in transcribed text. " diff --git a/tests/system/large/functions/test_managed_function.py b/tests/system/large/functions/test_managed_function.py index 888852edd4d..732123ec847 100644 --- a/tests/system/large/functions/test_managed_function.py +++ b/tests/system/large/functions/test_managed_function.py @@ -16,6 +16,7 @@ import google.api_core.exceptions import pandas +import pyarrow import pytest import test_utils.prefixer @@ -24,333 +25,405 @@ import bigframes.dtypes import bigframes.exceptions as bfe import bigframes.pandas as bpd +from bigframes.testing.utils import cleanup_function_assets prefixer = test_utils.prefixer.Prefixer("bigframes", "") -@pytest.fixture -def function_id(dataset_id, session): - name = prefixer.create_prefix() - yield name +def test_managed_function_array_output(session, scalars_dfs, dataset_id): try: - session.bqclient.delete_routine(f"{dataset_id}.{name}") - # some tests, like test_managed_function_options_errors, should not actually create the function. - # so we ignore the not found error. - except google.api_core.exceptions.NotFound: - pass + with warnings.catch_warnings(record=True) as record: + + @session.udf( + dataset=dataset_id, + name=prefixer.create_prefix(), + ) + def featurize(x: int) -> list[float]: + return [float(i) for i in [x, x + 1, x + 2]] + + # No following conflict warning when there is no redundant type hints. + input_type_warning = "Conflicting input types detected" + return_type_warning = "Conflicting return type detected" + assert not any(input_type_warning in str(warning.message) for warning in record) + assert not any( + return_type_warning in str(warning.message) for warning in record + ) -def test_managed_function_array_output(session, scalars_dfs, dataset_id, function_id): - with warnings.catch_warnings(record=True) as record: + scalars_df, scalars_pandas_df = scalars_dfs - @session.udf( - dataset=dataset_id, - name=function_id, - ) - def featurize(x: int) -> list[float]: - return [float(i) for i in [x, x + 1, x + 2]] + bf_int64_col = scalars_df["int64_too"] + bf_result = bf_int64_col.apply(featurize).to_pandas() - # No following conflict warning when there is no redundant type hints. - input_type_warning = "Conflicting input types detected" - return_type_warning = "Conflicting return type detected" - assert not any(input_type_warning in str(warning.message) for warning in record) - assert not any(return_type_warning in str(warning.message) for warning in record) + pd_int64_col = scalars_pandas_df["int64_too"] + pd_result = pd_int64_col.apply(featurize) - scalars_df, scalars_pandas_df = scalars_dfs + # Ignore any dtype disparity. + pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) - bf_int64_col = scalars_df["int64_too"] - bf_result = bf_int64_col.apply(featurize).to_pandas() + # Make sure the read_gbq_function path works for this function. + featurize_ref = session.read_gbq_function(featurize.bigframes_bigquery_function) - pd_int64_col = scalars_pandas_df["int64_too"] - pd_result = pd_int64_col.apply(featurize) + assert hasattr(featurize_ref, "bigframes_bigquery_function") + assert featurize_ref.bigframes_remote_function is None + assert ( + featurize_ref.bigframes_bigquery_function + == featurize.bigframes_bigquery_function + ) - # Ignore any dtype disparity. - pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) + # Test on the function from read_gbq_function. + got = featurize_ref(10) + assert got == [10.0, 11.0, 12.0] - # Make sure the read_gbq_function path works for this function. - featurize_ref = session.read_gbq_function(f"{dataset_id}.{function_id}") + bf_result_gbq = bf_int64_col.apply(featurize_ref).to_pandas() + pandas.testing.assert_series_equal(bf_result_gbq, pd_result, check_dtype=False) - # Test on the function from read_gbq_function. - got = featurize_ref(10) - assert got == [10.0, 11.0, 12.0] + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets(featurize, session.bqclient, ignore_failures=False) - bf_result_gbq = bf_int64_col.apply(featurize_ref).to_pandas() - pandas.testing.assert_series_equal(bf_result_gbq, pd_result, check_dtype=False) +def test_managed_function_series_apply(session, dataset_id, scalars_dfs): + try: -def test_managed_function_series_apply(session, dataset_id, scalars_dfs, function_id): - @session.udf(dataset=dataset_id, name=function_id) - def foo(x: int) -> bytes: - return bytes(abs(x)) + # An explicit name with "def" in it is used to test the robustness of + # the user code extraction logic, which depends on that term. + bq_name = f"{prefixer.create_prefix()}_def_to_test_code_extraction" + assert "def" in bq_name, "The substring 'def' was not found in 'bq_name'" - # Function should still work normally. - assert foo(-2) == bytes(2) + @session.udf(dataset=dataset_id, name=bq_name) + def foo(x: int) -> bytes: + return bytes(abs(x)) - scalars_df, scalars_pandas_df = scalars_dfs + # Function should still work normally. + assert foo(-2) == bytes(2) - bf_result_col = scalars_df["int64_too"].apply(foo) - bf_result = ( - scalars_df["int64_too"].to_frame().assign(result=bf_result_col).to_pandas() - ) + assert hasattr(foo, "bigframes_bigquery_function") + assert hasattr(foo, "input_dtypes") + assert hasattr(foo, "output_dtype") + assert hasattr(foo, "bigframes_bigquery_function_output_dtype") - pd_result_col = scalars_pandas_df["int64_too"].apply(foo) - pd_result = scalars_pandas_df["int64_too"].to_frame().assign(result=pd_result_col) + scalars_df, scalars_pandas_df = scalars_dfs - pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) + bf_result_col = scalars_df["int64_too"].apply(foo) + bf_result = ( + scalars_df["int64_too"].to_frame().assign(result=bf_result_col).to_pandas() + ) - # Make sure the read_gbq_function path works for this function. - foo_ref = session.read_gbq_function(f"{dataset_id}.{function_id}") + pd_result_col = scalars_pandas_df["int64_too"].apply(foo) + pd_result = ( + scalars_pandas_df["int64_too"].to_frame().assign(result=pd_result_col) + ) - bf_result_col_gbq = scalars_df["int64_too"].apply(foo_ref) - bf_result_gbq = ( - scalars_df["int64_too"].to_frame().assign(result=bf_result_col_gbq).to_pandas() - ) + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) - pandas.testing.assert_frame_equal(bf_result_gbq, pd_result, check_dtype=False) + # Make sure the read_gbq_function path works for this function. + foo_ref = session.read_gbq_function( + function_name=foo.bigframes_bigquery_function, # type: ignore + ) + assert hasattr(foo_ref, "bigframes_bigquery_function") + assert foo_ref.bigframes_remote_function is None + assert foo.bigframes_bigquery_function == foo_ref.bigframes_bigquery_function # type: ignore + + bf_result_col_gbq = scalars_df["int64_too"].apply(foo_ref) + bf_result_gbq = ( + scalars_df["int64_too"] + .to_frame() + .assign(result=bf_result_col_gbq) + .to_pandas() + ) + + pandas.testing.assert_frame_equal(bf_result_gbq, pd_result, check_dtype=False) + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets(foo, session.bqclient, ignore_failures=False) def test_managed_function_series_apply_array_output( session, dataset_id, scalars_dfs, - function_id, ): - with pytest.warns(bfe.PreviewWarning, match="udf is in preview."): - - @session.udf(dataset=dataset_id, name=function_id) - def foo_list(x: int) -> list[float]: - return [float(abs(x)), float(abs(x) + 1)] - - scalars_df, scalars_pandas_df = scalars_dfs - - bf_result_col = scalars_df["int64_too"].apply(foo_list) - bf_result = ( - scalars_df["int64_too"].to_frame().assign(result=bf_result_col).to_pandas() - ) - - pd_result_col = scalars_pandas_df["int64_too"].apply(foo_list) - pd_result = scalars_pandas_df["int64_too"].to_frame().assign(result=pd_result_col) + try: - # Ignore any dtype difference. - pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) + with pytest.warns(bfe.PreviewWarning, match="udf is in preview."): + @session.udf(dataset=dataset_id, name=prefixer.create_prefix()) + def foo_list(x: int) -> list[float]: + return [float(abs(x)), float(abs(x) + 1)] -def test_managed_function_series_combine(session, dataset_id, scalars_dfs, function_id): - # This function is deliberately written to not work with NA input. - def add(x: int, y: int) -> int: - return x + y + scalars_df, scalars_pandas_df = scalars_dfs - scalars_df, scalars_pandas_df = scalars_dfs - int_col_name_with_nulls = "int64_col" - int_col_name_no_nulls = "int64_too" - bf_df = scalars_df[[int_col_name_with_nulls, int_col_name_no_nulls]] - pd_df = scalars_pandas_df[[int_col_name_with_nulls, int_col_name_no_nulls]] + bf_result_col = scalars_df["int64_too"].apply(foo_list) + bf_result = ( + scalars_df["int64_too"].to_frame().assign(result=bf_result_col).to_pandas() + ) - # make sure there are NA values in the test column. - assert any([pandas.isna(val) for val in bf_df[int_col_name_with_nulls]]) + pd_result_col = scalars_pandas_df["int64_too"].apply(foo_list) + pd_result = ( + scalars_pandas_df["int64_too"].to_frame().assign(result=pd_result_col) + ) - add_managed_func = session.udf(dataset=dataset_id, name=function_id)(add) + # Ignore any dtype difference. + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets(foo_list, session.bqclient, ignore_failures=False) - # with nulls in the series the managed function application would fail. - with pytest.raises( - google.api_core.exceptions.BadRequest, match="unsupported operand" - ): - bf_df[int_col_name_with_nulls].combine( - bf_df[int_col_name_no_nulls], add_managed_func - ).to_pandas() - # after filtering out nulls the managed function application should work - # similar to pandas. - pd_filter = pd_df[int_col_name_with_nulls].notnull() - pd_result = pd_df[pd_filter][int_col_name_with_nulls].combine( - pd_df[pd_filter][int_col_name_no_nulls], add - ) - bf_filter = bf_df[int_col_name_with_nulls].notnull() - bf_result = ( - bf_df[bf_filter][int_col_name_with_nulls] - .combine(bf_df[bf_filter][int_col_name_no_nulls], add_managed_func) - .to_pandas() - ) +def test_managed_function_series_combine(session, dataset_id, scalars_dfs): + try: + # This function is deliberately written to not work with NA input. + def add(x: int, y: int) -> int: + return x + y + + scalars_df, scalars_pandas_df = scalars_dfs + int_col_name_with_nulls = "int64_col" + int_col_name_no_nulls = "int64_too" + bf_df = scalars_df[[int_col_name_with_nulls, int_col_name_no_nulls]] + pd_df = scalars_pandas_df[[int_col_name_with_nulls, int_col_name_no_nulls]] + + # make sure there are NA values in the test column. + assert any([pandas.isna(val) for val in bf_df[int_col_name_with_nulls]]) + + add_managed_func = session.udf( + dataset=dataset_id, name=prefixer.create_prefix() + )(add) + + # with nulls in the series the managed function application would fail. + with pytest.raises( + google.api_core.exceptions.BadRequest, match="unsupported operand" + ): + bf_df[int_col_name_with_nulls].combine( + bf_df[int_col_name_no_nulls], add_managed_func + ).to_pandas() + + # after filtering out nulls the managed function application should work + # similar to pandas. + pd_filter = pd_df[int_col_name_with_nulls].notnull() + pd_result = pd_df[pd_filter][int_col_name_with_nulls].combine( + pd_df[pd_filter][int_col_name_no_nulls], add + ) + bf_filter = bf_df[int_col_name_with_nulls].notnull() + bf_result = ( + bf_df[bf_filter][int_col_name_with_nulls] + .combine(bf_df[bf_filter][int_col_name_no_nulls], add_managed_func) + .to_pandas() + ) - # ignore any dtype difference. - pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) + # ignore any dtype difference. + pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) - # Make sure the read_gbq_function path works for this function. - add_managed_func_ref = session.read_gbq_function(f"{dataset_id}.{function_id}") - bf_result = ( - bf_df[bf_filter][int_col_name_with_nulls] - .combine(bf_df[bf_filter][int_col_name_no_nulls], add_managed_func_ref) - .to_pandas() - ) - pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) + # Make sure the read_gbq_function path works for this function. + add_managed_func_ref = session.read_gbq_function( + add_managed_func.bigframes_bigquery_function + ) + bf_result = ( + bf_df[bf_filter][int_col_name_with_nulls] + .combine(bf_df[bf_filter][int_col_name_no_nulls], add_managed_func_ref) + .to_pandas() + ) + pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets( + add_managed_func, session.bqclient, ignore_failures=False + ) -def test_managed_function_series_combine_array_output( - session, dataset_id, scalars_dfs, function_id -): - # The type hints in this function's signature has conflicts. The - # `input_types` and `output_type` arguments from udf decorator take - # precedence and will be used instead. - def add_list(x, y: bool) -> list[bool]: - return [x, y] +def test_managed_function_series_combine_array_output(session, dataset_id, scalars_dfs): + try: - scalars_df, scalars_pandas_df = scalars_dfs - int_col_name_with_nulls = "int64_col" - int_col_name_no_nulls = "int64_too" - bf_df = scalars_df[[int_col_name_with_nulls, int_col_name_no_nulls]] - pd_df = scalars_pandas_df[[int_col_name_with_nulls, int_col_name_no_nulls]] + # The type hints in this function's signature has conflicts. The + # `input_types` and `output_type` arguments from udf decorator take + # precedence and will be used instead. + def add_list(x, y: bool) -> list[bool]: + return [x, y] + + scalars_df, scalars_pandas_df = scalars_dfs + int_col_name_with_nulls = "int64_col" + int_col_name_no_nulls = "int64_too" + bf_df = scalars_df[[int_col_name_with_nulls, int_col_name_no_nulls]] + pd_df = scalars_pandas_df[[int_col_name_with_nulls, int_col_name_no_nulls]] + + # Make sure there are NA values in the test column. + assert any([pandas.isna(val) for val in bf_df[int_col_name_with_nulls]]) + + with warnings.catch_warnings(record=True) as record: + add_list_managed_func = session.udf( + input_types=[int, int], + output_type=list[int], + dataset=dataset_id, + name=prefixer.create_prefix(), + )(add_list) + + input_type_warning = "Conflicting input types detected" + assert any(input_type_warning in str(warning.message) for warning in record) + return_type_warning = "Conflicting return type detected" + assert any(return_type_warning in str(warning.message) for warning in record) + + # After filtering out nulls the managed function application should work + # similar to pandas. + pd_filter = pd_df[int_col_name_with_nulls].notnull() + pd_result = pd_df[pd_filter][int_col_name_with_nulls].combine( + pd_df[pd_filter][int_col_name_no_nulls], add_list + ) + bf_filter = bf_df[int_col_name_with_nulls].notnull() + bf_result = ( + bf_df[bf_filter][int_col_name_with_nulls] + .combine(bf_df[bf_filter][int_col_name_no_nulls], add_list_managed_func) + .to_pandas() + ) - # Make sure there are NA values in the test column. - assert any([pandas.isna(val) for val in bf_df[int_col_name_with_nulls]]) + # Ignore any dtype difference. + pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) - with warnings.catch_warnings(record=True) as record: - add_list_managed_func = session.udf( - input_types=[int, int], - output_type=list[int], - dataset=dataset_id, - name=function_id, - )(add_list) - - input_type_warning = "Conflicting input types detected" - assert any(input_type_warning in str(warning.message) for warning in record) - return_type_warning = "Conflicting return type detected" - assert any(return_type_warning in str(warning.message) for warning in record) - - # After filtering out nulls the managed function application should work - # similar to pandas. - pd_filter = pd_df[int_col_name_with_nulls].notnull() - pd_result = pd_df[pd_filter][int_col_name_with_nulls].combine( - pd_df[pd_filter][int_col_name_no_nulls], add_list - ) - bf_filter = bf_df[int_col_name_with_nulls].notnull() - bf_result = ( - bf_df[bf_filter][int_col_name_with_nulls] - .combine(bf_df[bf_filter][int_col_name_no_nulls], add_list_managed_func) - .to_pandas() - ) + # Make sure the read_gbq_function path works for this function. + add_list_managed_func_ref = session.read_gbq_function( + function_name=add_list_managed_func.bigframes_bigquery_function, # type: ignore + ) - # Ignore any dtype difference. - pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) + assert hasattr(add_list_managed_func_ref, "bigframes_bigquery_function") + assert add_list_managed_func_ref.bigframes_remote_function is None + assert ( + add_list_managed_func_ref.bigframes_bigquery_function + == add_list_managed_func.bigframes_bigquery_function + ) - # Make sure the read_gbq_function path works for this function. - add_list_managed_func_ref = session.read_gbq_function(f"{dataset_id}.{function_id}") + # Test on the function from read_gbq_function. + got = add_list_managed_func_ref(10, 38) + assert got == [10, 38] - # Test on the function from read_gbq_function. - got = add_list_managed_func_ref(10, 38) - assert got == [10, 38] + bf_result_gbq = ( + bf_df[bf_filter][int_col_name_with_nulls] + .combine(bf_df[bf_filter][int_col_name_no_nulls], add_list_managed_func_ref) + .to_pandas() + ) - bf_result_gbq = ( - bf_df[bf_filter][int_col_name_with_nulls] - .combine(bf_df[bf_filter][int_col_name_no_nulls], add_list_managed_func_ref) - .to_pandas() - ) + pandas.testing.assert_series_equal(bf_result_gbq, pd_result, check_dtype=False) + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets( + add_list_managed_func, session.bqclient, ignore_failures=False + ) - pandas.testing.assert_series_equal(bf_result_gbq, pd_result, check_dtype=False) +def test_managed_function_dataframe_map(session, dataset_id, scalars_dfs): + try: -def test_managed_function_dataframe_map(session, dataset_id, scalars_dfs, function_id): - def add_one(x): - return x + 1 + def add_one(x): + return x + 1 - mf_add_one = session.udf( - input_types=[int], - output_type=int, - dataset=dataset_id, - name=function_id, - )(add_one) + mf_add_one = session.udf( + input_types=[int], + output_type=int, + dataset=dataset_id, + name=prefixer.create_prefix(), + )(add_one) - scalars_df, scalars_pandas_df = scalars_dfs - int64_cols = ["int64_col", "int64_too"] + scalars_df, scalars_pandas_df = scalars_dfs + int64_cols = ["int64_col", "int64_too"] - bf_int64_df = scalars_df[int64_cols] - bf_int64_df_filtered = bf_int64_df.dropna() - bf_result = bf_int64_df_filtered.map(mf_add_one).to_pandas() + bf_int64_df = scalars_df[int64_cols] + bf_int64_df_filtered = bf_int64_df.dropna() + bf_result = bf_int64_df_filtered.map(mf_add_one).to_pandas() - pd_int64_df = scalars_pandas_df[int64_cols] - pd_int64_df_filtered = pd_int64_df.dropna() - pd_result = pd_int64_df_filtered.map(add_one) - # TODO(shobs): Figure why pandas .map() changes the dtype, i.e. - # pd_int64_df_filtered.dtype is Int64Dtype() - # pd_int64_df_filtered.map(lambda x: x).dtype is int64. - # For this test let's force the pandas dtype to be same as input. - for col in pd_result: - pd_result[col] = pd_result[col].astype(pd_int64_df_filtered[col].dtype) + pd_int64_df = scalars_pandas_df[int64_cols] + pd_int64_df_filtered = pd_int64_df.dropna() + pd_result = pd_int64_df_filtered.map(add_one) + # TODO(shobs): Figure why pandas .map() changes the dtype, i.e. + # pd_int64_df_filtered.dtype is Int64Dtype() + # pd_int64_df_filtered.map(lambda x: x).dtype is int64. + # For this test let's force the pandas dtype to be same as input. + for col in pd_result: + pd_result[col] = pd_result[col].astype(pd_int64_df_filtered[col].dtype) - pandas.testing.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets(mf_add_one, session.bqclient, ignore_failures=False) -def test_managed_function_dataframe_map_array_output( - session, scalars_dfs, dataset_id, function_id -): - def add_one_list(x): - return [x + 1] * 3 +def test_managed_function_dataframe_map_array_output(session, scalars_dfs, dataset_id): + try: - mf_add_one_list = session.udf( - input_types=[int], - output_type=list[int], - dataset=dataset_id, - name=function_id, - )(add_one_list) + def add_one_list(x): + return [x + 1] * 3 - scalars_df, scalars_pandas_df = scalars_dfs - int64_cols = ["int64_col", "int64_too"] + mf_add_one_list = session.udf( + input_types=[int], + output_type=list[int], + dataset=dataset_id, + name=prefixer.create_prefix(), + )(add_one_list) - bf_int64_df = scalars_df[int64_cols] - bf_int64_df_filtered = bf_int64_df.dropna() - bf_result = bf_int64_df_filtered.map(mf_add_one_list).to_pandas() + scalars_df, scalars_pandas_df = scalars_dfs + int64_cols = ["int64_col", "int64_too"] - pd_int64_df = scalars_pandas_df[int64_cols] - pd_int64_df_filtered = pd_int64_df.dropna() - pd_result = pd_int64_df_filtered.map(add_one_list) + bf_int64_df = scalars_df[int64_cols] + bf_int64_df_filtered = bf_int64_df.dropna() + bf_result = bf_int64_df_filtered.map(mf_add_one_list).to_pandas() - # Ignore any dtype difference. - pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pd_int64_df = scalars_pandas_df[int64_cols] + pd_int64_df_filtered = pd_int64_df.dropna() + pd_result = pd_int64_df_filtered.map(add_one_list) - # Make sure the read_gbq_function path works for this function. - mf_add_one_list_ref = session.read_gbq_function(f"{dataset_id}.{function_id}") + # Ignore any dtype difference. + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) - bf_result_gbq = bf_int64_df_filtered.map(mf_add_one_list_ref).to_pandas() - pandas.testing.assert_frame_equal(bf_result_gbq, pd_result, check_dtype=False) + # Make sure the read_gbq_function path works for this function. + mf_add_one_list_ref = session.read_gbq_function( + function_name=mf_add_one_list.bigframes_bigquery_function, # type: ignore + ) + bf_result_gbq = bf_int64_df_filtered.map(mf_add_one_list_ref).to_pandas() + pandas.testing.assert_frame_equal(bf_result_gbq, pd_result, check_dtype=False) + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets( + mf_add_one_list, session.bqclient, ignore_failures=False + ) -def test_managed_function_dataframe_apply_axis_1( - session, dataset_id, scalars_dfs, function_id -): - scalars_df, scalars_pandas_df = scalars_dfs - series = scalars_df["int64_too"] - series_pandas = scalars_pandas_df["int64_too"] - def add_ints(x, y): - return x + y +def test_managed_function_dataframe_apply_axis_1(session, dataset_id, scalars_dfs): + try: + scalars_df, scalars_pandas_df = scalars_dfs + series = scalars_df["int64_too"] + series_pandas = scalars_pandas_df["int64_too"] - add_ints_mf = session.udf( - input_types=[int, int], - output_type=int, - dataset=dataset_id, - name=function_id, - )(add_ints) + def add_ints(x, y): + return x + y - with pytest.warns( - bigframes.exceptions.PreviewWarning, match="axis=1 scenario is in preview." - ): - bf_result = ( - bpd.DataFrame({"x": series, "y": series}) - .apply(add_ints_mf, axis=1) - .to_pandas() + add_ints_mf = session.udf( + input_types=[int, int], + output_type=int, + dataset=dataset_id, + name=prefixer.create_prefix(), + )(add_ints) + assert add_ints_mf.bigframes_bigquery_function # type: ignore + + with pytest.warns( + bigframes.exceptions.PreviewWarning, match="axis=1 scenario is in preview." + ): + bf_result = ( + bpd.DataFrame({"x": series, "y": series}) + .apply(add_ints_mf, axis=1) + .to_pandas() + ) + + pd_result = pandas.DataFrame({"x": series_pandas, "y": series_pandas}).apply( + lambda row: add_ints(row["x"], row["y"]), axis=1 ) - pd_result = pandas.DataFrame({"x": series_pandas, "y": series_pandas}).apply( - lambda row: add_ints(row["x"], row["y"]), axis=1 - ) - - pandas.testing.assert_series_equal( - pd_result, bf_result, check_dtype=False, check_exact=True - ) + pandas.testing.assert_series_equal( + pd_result, bf_result, check_dtype=False, check_exact=True + ) + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets(add_ints_mf, session.bqclient, ignore_failures=False) -def test_managed_function_dataframe_apply_axis_1_array_output( - session, dataset_id, function_id -): +def test_managed_function_dataframe_apply_axis_1_array_output(session, dataset_id): bf_df = bigframes.dataframe.DataFrame( { "Id": [1, 2, 3], @@ -372,69 +445,92 @@ def test_managed_function_dataframe_apply_axis_1_array_output( input_types=[int, float, str], output_type=list[str], dataset=dataset_id, - name=function_id, + name=prefixer.create_prefix(), ) def foo(x, y, z): return [str(x), str(y), z] - # Fails to apply on dataframe with incompatible number of columns. - with pytest.raises( - ValueError, - match="^Parameter count mismatch:.* expected 3 parameters but received 2 DataFrame columns.", - ): - bf_df[["Id", "Age"]].apply(foo, axis=1) + try: - with pytest.raises( - ValueError, - match="^Parameter count mismatch:.* expected 3 parameters but received 4 DataFrame columns.", - ): - bf_df.assign(Country="lalaland").apply(foo, axis=1) + assert getattr(foo, "is_row_processor") is False + assert getattr(foo, "input_dtypes") == expected_dtypes + assert getattr(foo, "output_dtype") == pandas.ArrowDtype( + pyarrow.list_( + bigframes.dtypes.bigframes_dtype_to_arrow_dtype( + bigframes.dtypes.STRING_DTYPE + ) + ) + ) + assert getattr(foo, "output_dtype") == getattr( + foo, "bigframes_bigquery_function_output_dtype" + ) - # Fails to apply on dataframe with incompatible column datatypes. - with pytest.raises( - ValueError, - match="^Data type mismatch for DataFrame columns: Expected .* Received .*", - ): - bf_df.assign(Age=bf_df["Age"].astype("Int64")).apply(foo, axis=1) + # Fails to apply on dataframe with incompatible number of columns. + with pytest.raises( + ValueError, + match="^Parameter count mismatch:.* expected 3 parameters but received 2 DataFrame columns.", + ): + bf_df[["Id", "Age"]].apply(foo, axis=1) + + with pytest.raises( + ValueError, + match="^Parameter count mismatch:.* expected 3 parameters but received 4 DataFrame columns.", + ): + bf_df.assign(Country="lalaland").apply(foo, axis=1) + + # Fails to apply on dataframe with incompatible column datatypes. + with pytest.raises( + ValueError, + match="^Data type mismatch for DataFrame columns: Expected .* Received .*", + ): + bf_df.assign(Age=bf_df["Age"].astype("Int64")).apply(foo, axis=1) + + # Successfully applies to dataframe with matching number of columns. + # and their datatypes. + with pytest.warns( + bigframes.exceptions.PreviewWarning, + match="axis=1 scenario is in preview.", + ): + bf_result = bf_df.apply(foo, axis=1).to_pandas() + + # Since this scenario is not pandas-like, let's handcraft the + # expected result. + expected_result = pandas.Series( + [ + ["1", "22.5", "alpha"], + ["2", "23.0", "beta"], + ["3", "23.5", "gamma"], + ] + ) - # Successfully applies to dataframe with matching number of columns. - # and their datatypes. - with pytest.warns( - bigframes.exceptions.PreviewWarning, - match="axis=1 scenario is in preview.", - ): - bf_result = bf_df.apply(foo, axis=1).to_pandas() - - # Since this scenario is not pandas-like, let's handcraft the - # expected result. - expected_result = pandas.Series( - [ - ["1", "22.5", "alpha"], - ["2", "23.0", "beta"], - ["3", "23.5", "gamma"], - ] - ) + pandas.testing.assert_series_equal( + expected_result, bf_result, check_dtype=False, check_index_type=False + ) - pandas.testing.assert_series_equal( - expected_result, bf_result, check_dtype=False, check_index_type=False - ) + # Make sure the read_gbq_function path works for this function. + foo_ref = session.read_gbq_function(foo.bigframes_bigquery_function) - # Make sure the read_gbq_function path works for this function. - foo_ref = session.read_gbq_function(f"{dataset_id}.{function_id}") + assert hasattr(foo_ref, "bigframes_bigquery_function") + assert foo_ref.bigframes_remote_function is None + assert foo_ref.bigframes_bigquery_function == foo.bigframes_bigquery_function - # Test on the function from read_gbq_function. - got = foo_ref(10, 38, "hello") - assert got == ["10", "38.0", "hello"] + # Test on the function from read_gbq_function. + got = foo_ref(10, 38, "hello") + assert got == ["10", "38.0", "hello"] - with pytest.warns( - bigframes.exceptions.PreviewWarning, - match="axis=1 scenario is in preview.", - ): - bf_result_gbq = bf_df.apply(foo_ref, axis=1).to_pandas() + with pytest.warns( + bigframes.exceptions.PreviewWarning, + match="axis=1 scenario is in preview.", + ): + bf_result_gbq = bf_df.apply(foo_ref, axis=1).to_pandas() - pandas.testing.assert_series_equal( - bf_result_gbq, expected_result, check_dtype=False, check_index_type=False - ) + pandas.testing.assert_series_equal( + bf_result_gbq, expected_result, check_dtype=False, check_index_type=False + ) + + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets(foo, session.bqclient, ignore_failures=False) @pytest.mark.parametrize( @@ -445,78 +541,95 @@ def foo(x, y, z): ], ) def test_managed_function_with_connection( - session, scalars_dfs, dataset_id, request, connection_fixture, function_id + session, scalars_dfs, dataset_id, request, connection_fixture ): - bigquery_connection = request.getfixturevalue(connection_fixture) + try: + bigquery_connection = request.getfixturevalue(connection_fixture) - @session.udf( - bigquery_connection=bigquery_connection, - dataset=dataset_id, - name=function_id, - ) - def foo(x: int) -> int: - return x + 10 + @session.udf( + bigquery_connection=bigquery_connection, + dataset=dataset_id, + name=prefixer.create_prefix(), + ) + def foo(x: int) -> int: + return x + 10 - # Function should still work normally. - assert foo(-2) == 8 + # Function should still work normally. + assert foo(-2) == 8 - scalars_df, scalars_pandas_df = scalars_dfs + scalars_df, scalars_pandas_df = scalars_dfs - bf_result_col = scalars_df["int64_too"].apply(foo) - bf_result = ( - scalars_df["int64_too"].to_frame().assign(result=bf_result_col).to_pandas() - ) + bf_result_col = scalars_df["int64_too"].apply(foo) + bf_result = ( + scalars_df["int64_too"].to_frame().assign(result=bf_result_col).to_pandas() + ) - pd_result_col = scalars_pandas_df["int64_too"].apply(foo) - pd_result = scalars_pandas_df["int64_too"].to_frame().assign(result=pd_result_col) + pd_result_col = scalars_pandas_df["int64_too"].apply(foo) + pd_result = ( + scalars_pandas_df["int64_too"].to_frame().assign(result=pd_result_col) + ) - pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets(foo, session.bqclient, ignore_failures=False) -def test_managed_function_options(session, dataset_id, scalars_dfs, function_id): - def multiply_five(x: int) -> int: - return x * 5 +def test_managed_function_options(session, dataset_id, scalars_dfs): + try: - mf_multiply_five = session.udf( - dataset=dataset_id, - name=function_id, - max_batching_rows=100, - container_cpu=2, - container_memory="2Gi", - )(multiply_five) + def multiply_five(x: int) -> int: + return x * 5 - scalars_df, scalars_pandas_df = scalars_dfs + mf_multiply_five = session.udf( + dataset=dataset_id, + name=prefixer.create_prefix(), + max_batching_rows=100, + container_cpu=2, + container_memory="2Gi", + )(multiply_five) - bf_int64_df = scalars_df["int64_col"] - bf_int64_df_filtered = bf_int64_df.dropna() - bf_result = bf_int64_df_filtered.apply(mf_multiply_five).to_pandas() + scalars_df, scalars_pandas_df = scalars_dfs - pd_int64_df = scalars_pandas_df["int64_col"] - pd_int64_df_filtered = pd_int64_df.dropna() - pd_result = pd_int64_df_filtered.apply(multiply_five) + bf_int64_df = scalars_df["int64_col"] + bf_int64_df_filtered = bf_int64_df.dropna() + bf_result = bf_int64_df_filtered.apply(mf_multiply_five).to_pandas() - pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) + pd_int64_df = scalars_pandas_df["int64_col"] + pd_int64_df_filtered = pd_int64_df.dropna() + pd_result = pd_int64_df_filtered.apply(multiply_five) - # Make sure the read_gbq_function path works for this function. - multiply_five_ref = session.read_gbq_function( - function_name=f"{dataset_id}.{function_id}" # type: ignore - ) + pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) - bf_result = bf_int64_df_filtered.apply(multiply_five_ref).to_pandas() - pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) + # Make sure the read_gbq_function path works for this function. + multiply_five_ref = session.read_gbq_function( + function_name=mf_multiply_five.bigframes_bigquery_function, # type: ignore + ) + assert mf_multiply_five.bigframes_bigquery_function == multiply_five_ref.bigframes_bigquery_function # type: ignore - # Retrieve the routine and validate its runtime configuration. - routine = session.bqclient.get_routine(f"{dataset_id}.{function_id}") + bf_result = bf_int64_df_filtered.apply(multiply_five_ref).to_pandas() + pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) - # TODO(jialuo): Use the newly exposed class properties instead of - # accessing the hidden _properties after resolve of this issue: - # https://github.com/googleapis/python-bigquery/issues/2240. - assert routine._properties["externalRuntimeOptions"]["maxBatchingRows"] == "100" - assert routine._properties["externalRuntimeOptions"]["containerCpu"] == 2 - assert routine._properties["externalRuntimeOptions"]["containerMemory"] == "2Gi" + # Retrieve the routine and validate its runtime configuration. + routine = session.bqclient.get_routine( + mf_multiply_five.bigframes_bigquery_function + ) + + # TODO(jialuo): Use the newly exposed class properties instead of + # accessing the hidden _properties after resolve of this issue: + # https://github.com/googleapis/python-bigquery/issues/2240. + assert routine._properties["externalRuntimeOptions"]["maxBatchingRows"] == "100" + assert routine._properties["externalRuntimeOptions"]["containerCpu"] == 2 + assert routine._properties["externalRuntimeOptions"]["containerMemory"] == "2Gi" + + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets( + mf_multiply_five, session.bqclient, ignore_failures=False + ) -def test_managed_function_options_errors(session, dataset_id, function_id): +def test_managed_function_options_errors(session, dataset_id): def foo(x: int) -> int: return 0 @@ -527,7 +640,7 @@ def foo(x: int) -> int: ): session.udf( dataset=dataset_id, - name=function_id, + name=prefixer.create_prefix(), max_batching_rows=100, container_cpu=2.5, container_memory="2Gi", @@ -540,7 +653,7 @@ def foo(x: int) -> int: ): session.udf( dataset=dataset_id, - name=function_id, + name=prefixer.create_prefix(), max_batching_rows=100, container_cpu=0.10, container_memory="512Mi", @@ -553,117 +666,131 @@ def foo(x: int) -> int: ): session.udf( dataset=dataset_id, - name=function_id, + name=prefixer.create_prefix(), max_batching_rows=100, container_cpu=2, container_memory="64Mi", )(foo) -def test_managed_function_df_apply_axis_1( - session, dataset_id, scalars_dfs, function_id -): +def test_managed_function_df_apply_axis_1(session, dataset_id, scalars_dfs): columns = ["bool_col", "int64_col", "int64_too", "float64_col", "string_col"] scalars_df, scalars_pandas_df = scalars_dfs + try: - def serialize_row(row): - # TODO(b/435021126): Remove explicit type conversion of the field - # "name" after the issue has been addressed. It is added only to - # accept partial pandas parity for the time being. - custom = { - "name": int(row.name), - "index": [idx for idx in row.index], - "values": [ - val.item() if hasattr(val, "item") else val for val in row.values - ], - } - - return str( - { - "default": row.to_json(), - "split": row.to_json(orient="split"), - "records": row.to_json(orient="records"), - "index": row.to_json(orient="index"), - "table": row.to_json(orient="table"), - "custom": custom, + def serialize_row(row): + # TODO(b/435021126): Remove explicit type conversion of the field + # "name" after the issue has been addressed. It is added only to + # accept partial pandas parity for the time being. + custom = { + "name": int(row.name), + "index": [idx for idx in row.index], + "values": [ + val.item() if hasattr(val, "item") else val for val in row.values + ], } - ) - with pytest.raises( - TypeError, - match="Argument type hint must be Pandas Series, not BigFrames Series.", - ): + return str( + { + "default": row.to_json(), + "split": row.to_json(orient="split"), + "records": row.to_json(orient="records"), + "index": row.to_json(orient="index"), + "table": row.to_json(orient="table"), + "custom": custom, + } + ) + + with pytest.raises( + TypeError, + match="Argument type hint must be Pandas Series, not BigFrames Series.", + ): + serialize_row_mf = session.udf( + input_types=bigframes.series.Series, + output_type=str, + dataset=dataset_id, + name=prefixer.create_prefix(), + )(serialize_row) + serialize_row_mf = session.udf( - input_types=bigframes.series.Series, + input_types=pandas.Series, output_type=str, dataset=dataset_id, - name=function_id, + name=prefixer.create_prefix(), )(serialize_row) - serialize_row_mf = session.udf( - input_types=pandas.Series, - output_type=str, - dataset=dataset_id, - name=function_id, - )(serialize_row) + assert getattr(serialize_row_mf, "is_row_processor") - bf_result = scalars_df[columns].apply(serialize_row_mf, axis=1).to_pandas() - pd_result = scalars_pandas_df[columns].apply(serialize_row, axis=1) + bf_result = scalars_df[columns].apply(serialize_row_mf, axis=1).to_pandas() + pd_result = scalars_pandas_df[columns].apply(serialize_row, axis=1) - # bf_result.dtype is 'string[pyarrow]' while pd_result.dtype is 'object' - # , ignore this mismatch by using check_dtype=False. - pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) + # bf_result.dtype is 'string[pyarrow]' while pd_result.dtype is 'object' + # , ignore this mismatch by using check_dtype=False. + pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) - # Let's make sure the read_gbq_function path works for this function. - serialize_row_reuse = session.read_gbq_function( - f"{dataset_id}.{function_id}", is_row_processor=True - ) - bf_result = scalars_df[columns].apply(serialize_row_reuse, axis=1).to_pandas() - pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) + # Let's make sure the read_gbq_function path works for this function. + serialize_row_reuse = session.read_gbq_function( + serialize_row_mf.bigframes_bigquery_function, is_row_processor=True + ) + bf_result = scalars_df[columns].apply(serialize_row_reuse, axis=1).to_pandas() + pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) + finally: + # clean up the gcp assets created for the managed function. + cleanup_function_assets( + serialize_row_mf, session.bqclient, ignore_failures=False + ) -def test_managed_function_df_apply_axis_1_aggregates( - session, dataset_id, scalars_dfs, function_id -): + +def test_managed_function_df_apply_axis_1_aggregates(session, dataset_id, scalars_dfs): columns = ["int64_col", "int64_too", "float64_col"] scalars_df, scalars_pandas_df = scalars_dfs - def analyze(row): - # TODO(b/435021126): Remove explicit type conversion of the fields - # after the issue has been addressed. It is added only to accept - # partial pandas parity for the time being. - return str( - { - "dtype": row.dtype, - "count": int(row.count()), - "min": int(row.min()), - "max": int(row.max()), - "mean": float(row.mean()), - "std": float(row.std()), - "var": float(row.var()), - } - ) + try: - with pytest.warns( - bfe.FunctionPackageVersionWarning, - match=( - "numpy, pandas, and pyarrow versions in the function execution" - "\nenvironment may not precisely match your local environment." - ), - ): - analyze_mf = session.udf( - input_types=pandas.Series, - output_type=str, - dataset=dataset_id, - name=function_id, - )(analyze) + def analyze(row): + # TODO(b/435021126): Remove explicit type conversion of the fields + # after the issue has been addressed. It is added only to accept + # partial pandas parity for the time being. + return str( + { + "dtype": row.dtype, + "count": int(row.count()), + "min": int(row.min()), + "max": int(row.max()), + "mean": float(row.mean()), + "std": float(row.std()), + "var": float(row.var()), + } + ) + + with pytest.warns( + bfe.FunctionPackageVersionWarning, + match=( + "numpy, pandas, and pyarrow versions in the function execution" + "\nenvironment may not precisely match your local environment." + ), + ): + + analyze_mf = session.udf( + input_types=pandas.Series, + output_type=str, + dataset=dataset_id, + name=prefixer.create_prefix(), + )(analyze) + + assert getattr(analyze_mf, "is_row_processor") - bf_result = scalars_df[columns].dropna().apply(analyze_mf, axis=1).to_pandas() - pd_result = scalars_pandas_df[columns].dropna().apply(analyze, axis=1) + bf_result = scalars_df[columns].dropna().apply(analyze_mf, axis=1).to_pandas() + pd_result = scalars_pandas_df[columns].dropna().apply(analyze, axis=1) - # bf_result.dtype is 'string[pyarrow]' while pd_result.dtype is 'object' - # , ignore this mismatch by using check_dtype=False. - pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) + # bf_result.dtype is 'string[pyarrow]' while pd_result.dtype is 'object' + # , ignore this mismatch by using check_dtype=False. + pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) + + finally: + # clean up the gcp assets created for the managed function. + cleanup_function_assets(analyze_mf, session.bqclient, ignore_failures=False) @pytest.mark.parametrize( @@ -733,50 +860,58 @@ def analyze(row): ), ], ) -def test_managed_function_df_apply_axis_1_complex( - session, dataset_id, pd_df, function_id -): +def test_managed_function_df_apply_axis_1_complex(session, dataset_id, pd_df): bf_df = session.read_pandas(pd_df) - def serialize_row(row): - # TODO(b/435021126): Remove explicit type conversion of the field - # "name" after the issue has been addressed. It is added only to - # accept partial pandas parity for the time being. - custom = { - "name": int(row.name), - "index": [idx for idx in row.index], - "values": [ - val.item() if hasattr(val, "item") else val for val in row.values - ], - } - return str( - { - "default": row.to_json(), - "split": row.to_json(orient="split"), - "records": row.to_json(orient="records"), - "index": row.to_json(orient="index"), - "custom": custom, + try: + + def serialize_row(row): + # TODO(b/435021126): Remove explicit type conversion of the field + # "name" after the issue has been addressed. It is added only to + # accept partial pandas parity for the time being. + custom = { + "name": int(row.name), + "index": [idx for idx in row.index], + "values": [ + val.item() if hasattr(val, "item") else val for val in row.values + ], } - ) + return str( + { + "default": row.to_json(), + "split": row.to_json(orient="split"), + "records": row.to_json(orient="records"), + "index": row.to_json(orient="index"), + "custom": custom, + } + ) - serialize_row_mf = session.udf( - input_types=pandas.Series, - output_type=str, - dataset=dataset_id, - name=function_id, - )(serialize_row) + serialize_row_mf = session.udf( + input_types=pandas.Series, + output_type=str, + dataset=dataset_id, + name=prefixer.create_prefix(), + )(serialize_row) - bf_result = bf_df.apply(serialize_row_mf, axis=1).to_pandas() - pd_result = pd_df.apply(serialize_row, axis=1) + assert getattr(serialize_row_mf, "is_row_processor") - # ignore known dtype difference between pandas and bigframes. - pandas.testing.assert_series_equal( - pd_result, bf_result, check_dtype=False, check_index_type=False - ) + bf_result = bf_df.apply(serialize_row_mf, axis=1).to_pandas() + pd_result = pd_df.apply(serialize_row, axis=1) + + # ignore known dtype difference between pandas and bigframes. + pandas.testing.assert_series_equal( + pd_result, bf_result, check_dtype=False, check_index_type=False + ) + + finally: + # clean up the gcp assets created for the managed function. + cleanup_function_assets( + serialize_row_mf, session.bqclient, ignore_failures=False + ) @pytest.mark.skip(reason="Revert after this bug b/435018880 is fixed.") -def test_managed_function_df_apply_axis_1_na_nan_inf(dataset_id, session, function_id): +def test_managed_function_df_apply_axis_1_na_nan_inf(dataset_id, session): """This test is for special cases of float values, to make sure any (nan, inf, -inf) produced by user code is honored. """ @@ -800,400 +935,399 @@ def test_managed_function_df_apply_axis_1_na_nan_inf(dataset_id, session, functi pd_df = bf_df.to_pandas() - def float_parser(row: pandas.Series): - import numpy as mynp - import pandas as mypd - - if row["text"] == "pandas na": - return mypd.NA - if row["text"] == "numpy nan": - return mynp.nan - return float(row["text"]) - - float_parser_mf = session.udf( - input_types=pandas.Series, - output_type=float, - dataset=dataset_id, - name=function_id, - )(float_parser) + try: - pd_result = pd_df.apply(float_parser, axis=1) - bf_result = bf_df.apply(float_parser_mf, axis=1).to_pandas() + def float_parser(row: pandas.Series): + import numpy as mynp + import pandas as mypd - # bf_result.dtype is 'Float64' while pd_result.dtype is 'object' - # , ignore this mismatch by using check_dtype=False. - pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) + if row["text"] == "pandas na": + return mypd.NA + if row["text"] == "numpy nan": + return mynp.nan + return float(row["text"]) - # Let's also assert that the data is consistent in this round trip - # (BQ -> BigFrames -> BQ -> GCF -> BQ -> BigFrames) w.r.t. their - # expected values in BQ. - bq_result = bf_df["num"].to_pandas() - bq_result.name = None - pandas.testing.assert_series_equal(bq_result, bf_result) + float_parser_mf = session.udf( + input_types=pandas.Series, + output_type=float, + dataset=dataset_id, + name=prefixer.create_prefix(), + )(float_parser) + + assert getattr(float_parser_mf, "is_row_processor") + + pd_result = pd_df.apply(float_parser, axis=1) + bf_result = bf_df.apply(float_parser_mf, axis=1).to_pandas() + + # bf_result.dtype is 'Float64' while pd_result.dtype is 'object' + # , ignore this mismatch by using check_dtype=False. + pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) + + # Let's also assert that the data is consistent in this round trip + # (BQ -> BigFrames -> BQ -> GCF -> BQ -> BigFrames) w.r.t. their + # expected values in BQ. + bq_result = bf_df["num"].to_pandas() + bq_result.name = None + pandas.testing.assert_series_equal(bq_result, bf_result) + finally: + # clean up the gcp assets created for the managed function. + cleanup_function_assets( + float_parser_mf, session.bqclient, ignore_failures=False + ) -def test_managed_function_df_apply_axis_1_args( - session, dataset_id, scalars_dfs, function_id -): +def test_managed_function_df_apply_axis_1_args(session, dataset_id, scalars_dfs): columns = ["int64_col", "int64_too"] scalars_df, scalars_pandas_df = scalars_dfs - def the_sum(s1, s2, x): - return s1 + s2 + x - - the_sum_mf = session.udf( - input_types=[int, int, int], - output_type=int, - dataset=dataset_id, - name=function_id, - )(the_sum) + try: - args1 = (1,) + def the_sum(s1, s2, x): + return s1 + s2 + x - # Fails to apply on dataframe with incompatible number of columns and args. - with pytest.raises( - ValueError, - match="^Parameter count mismatch:.* expected 3 parameters but received 4 values \\(3 DataFrame columns and 1 args\\)", - ): - scalars_df[columns + ["float64_col"]].apply(the_sum_mf, axis=1, args=args1) - - # Fails to apply on dataframe with incompatible column datatypes. - with pytest.raises( - ValueError, - match="^Data type mismatch for DataFrame columns: Expected .* Received .*", - ): - scalars_df[columns].assign( - int64_col=lambda df: df["int64_col"].astype("Float64") - ).apply(the_sum_mf, axis=1, args=args1) + the_sum_mf = session.udf( + input_types=[int, int, int], + output_type=int, + dataset=dataset_id, + name=prefixer.create_prefix(), + )(the_sum) + + args1 = (1,) + + # Fails to apply on dataframe with incompatible number of columns and args. + with pytest.raises( + ValueError, + match="^Parameter count mismatch:.* expected 3 parameters but received 4 values \\(3 DataFrame columns and 1 args\\)", + ): + scalars_df[columns + ["float64_col"]].apply(the_sum_mf, axis=1, args=args1) + + # Fails to apply on dataframe with incompatible column datatypes. + with pytest.raises( + ValueError, + match="^Data type mismatch for DataFrame columns: Expected .* Received .*", + ): + scalars_df[columns].assign( + int64_col=lambda df: df["int64_col"].astype("Float64") + ).apply(the_sum_mf, axis=1, args=args1) + + # Fails to apply on dataframe with incompatible args datatypes. + with pytest.raises( + ValueError, + match="^Data type mismatch for 'args' parameter: Expected .* Received .*", + ): + scalars_df[columns].apply(the_sum_mf, axis=1, args=(1.3,)) - # Fails to apply on dataframe with incompatible args datatypes. - with pytest.raises( - ValueError, - match="^Data type mismatch for 'args' parameter: Expected .* Received .*", - ): - scalars_df[columns].apply(the_sum_mf, axis=1, args=(1.3,)) + bf_result = ( + scalars_df[columns] + .dropna() + .apply(the_sum_mf, axis=1, args=args1) + .to_pandas() + ) + pd_result = scalars_pandas_df[columns].dropna().apply(sum, axis=1, args=args1) - bf_result = ( - scalars_df[columns].dropna().apply(the_sum_mf, axis=1, args=args1).to_pandas() - ) - pd_result = scalars_pandas_df[columns].dropna().apply(sum, axis=1, args=args1) + pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) - pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) + finally: + # clean up the gcp assets created for the managed function. + cleanup_function_assets(the_sum_mf, session.bqclient, ignore_failures=False) -def test_managed_function_df_apply_axis_1_series_args( - session, dataset_id, scalars_dfs, function_id -): +def test_managed_function_df_apply_axis_1_series_args(session, dataset_id, scalars_dfs): columns = ["int64_col", "float64_col"] scalars_df, scalars_pandas_df = scalars_dfs - def analyze(s: pandas.Series, x: bool, y: float) -> str: - value = f"value is {s['int64_col']} and {s['float64_col']}" - if x: - return f"{value}, x is True!" - if y > 0: - return f"{value}, x is False, y is positive!" - return f"{value}, x is False, y is non-positive!" - - analyze_mf = session.udf( - dataset=dataset_id, - name=function_id, - )(analyze) - - args1 = (True, 10.0) - bf_result = ( - scalars_df[columns].dropna().apply(analyze_mf, axis=1, args=args1).to_pandas() - ) - pd_result = scalars_pandas_df[columns].dropna().apply(analyze, axis=1, args=args1) - - pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) - - args2 = (False, -10.0) - analyze_mf_ref = session.read_gbq_function( - f"{dataset_id}.{function_id}", is_row_processor=True - ) - bf_result = ( - scalars_df[columns] - .dropna() - .apply(analyze_mf_ref, axis=1, args=args2) - .to_pandas() - ) - pd_result = scalars_pandas_df[columns].dropna().apply(analyze, axis=1, args=args2) - - pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) - + try: -def test_managed_function_df_where_mask(session, dataset_id, scalars_dfs, function_id): - # The return type has to be bool type for callable where condition. - def is_sum_positive(a, b): - return a + b > 0 + def analyze(s: pandas.Series, x: bool, y: float) -> str: + value = f"value is {s['int64_col']} and {s['float64_col']}" + if x: + return f"{value}, x is True!" + if y > 0: + return f"{value}, x is False, y is positive!" + return f"{value}, x is False, y is non-positive!" - is_sum_positive_mf = session.udf( - input_types=[int, int], - output_type=bool, - dataset=dataset_id, - name=function_id, - )(is_sum_positive) + analyze_mf = session.udf( + dataset=dataset_id, + name=prefixer.create_prefix(), + )(analyze) - scalars_df, scalars_pandas_df = scalars_dfs - int64_cols = ["int64_col", "int64_too"] + args1 = (True, 10.0) + bf_result = ( + scalars_df[columns] + .dropna() + .apply(analyze_mf, axis=1, args=args1) + .to_pandas() + ) + pd_result = ( + scalars_pandas_df[columns].dropna().apply(analyze, axis=1, args=args1) + ) - bf_int64_df = scalars_df[int64_cols] - bf_int64_df_filtered = bf_int64_df.dropna() - pd_int64_df = scalars_pandas_df[int64_cols] - pd_int64_df_filtered = pd_int64_df.dropna() + pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) - # Test callable condition in dataframe.where method. - bf_result = bf_int64_df_filtered.where(is_sum_positive_mf).to_pandas() - # Pandas doesn't support such case, use following as workaround. - pd_result = pd_int64_df_filtered.where(pd_int64_df_filtered.sum(axis=1) > 0) + args2 = (False, -10.0) + analyze_mf_ref = session.read_gbq_function( + analyze_mf.bigframes_bigquery_function, is_row_processor=True + ) + bf_result = ( + scalars_df[columns] + .dropna() + .apply(analyze_mf_ref, axis=1, args=args2) + .to_pandas() + ) + pd_result = ( + scalars_pandas_df[columns].dropna().apply(analyze, axis=1, args=args2) + ) - # Ignore any dtype difference. - pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pandas.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) - # Make sure the read_gbq_function path works for dataframe.where method. - is_sum_positive_ref = session.read_gbq_function(f"{dataset_id}.{function_id}") + finally: + # clean up the gcp assets created for the managed function. + cleanup_function_assets(analyze_mf, session.bqclient, ignore_failures=False) - bf_result_gbq = bf_int64_df_filtered.where( - is_sum_positive_ref, -bf_int64_df_filtered - ).to_pandas() - pd_result_gbq = pd_int64_df_filtered.where( - pd_int64_df_filtered.sum(axis=1) > 0, -pd_int64_df_filtered - ) - # Ignore any dtype difference. - pandas.testing.assert_frame_equal(bf_result_gbq, pd_result_gbq, check_dtype=False) +def test_managed_function_df_where_mask(session, dataset_id, scalars_dfs): + try: - # Test callable condition in dataframe.mask method. - bf_result_gbq = bf_int64_df_filtered.mask( - is_sum_positive_ref, -bf_int64_df_filtered - ).to_pandas() - pd_result_gbq = pd_int64_df_filtered.mask( - pd_int64_df_filtered.sum(axis=1) > 0, -pd_int64_df_filtered - ) + # The return type has to be bool type for callable where condition. + def is_sum_positive(a, b): + return a + b > 0 - # Ignore any dtype difference. - pandas.testing.assert_frame_equal(bf_result_gbq, pd_result_gbq, check_dtype=False) + is_sum_positive_mf = session.udf( + input_types=[int, int], + output_type=bool, + dataset=dataset_id, + name=prefixer.create_prefix(), + )(is_sum_positive) + scalars_df, scalars_pandas_df = scalars_dfs + int64_cols = ["int64_col", "int64_too"] -def test_managed_function_df_where_mask_series( - session, dataset_id, scalars_dfs, function_id -): - # The return type has to be bool type for callable where condition. - def is_sum_positive_series(s): - return s["int64_col"] + s["int64_too"] > 0 + bf_int64_df = scalars_df[int64_cols] + bf_int64_df_filtered = bf_int64_df.dropna() + pd_int64_df = scalars_pandas_df[int64_cols] + pd_int64_df_filtered = pd_int64_df.dropna() - is_sum_positive_series_mf = session.udf( - input_types=pandas.Series, - output_type=bool, - dataset=dataset_id, - name=function_id, - )(is_sum_positive_series) + # Test callable condition in dataframe.where method. + bf_result = bf_int64_df_filtered.where(is_sum_positive_mf).to_pandas() + # Pandas doesn't support such case, use following as workaround. + pd_result = pd_int64_df_filtered.where(pd_int64_df_filtered.sum(axis=1) > 0) - scalars_df, scalars_pandas_df = scalars_dfs - int64_cols = ["int64_col", "int64_too"] + # Ignore any dtype difference. + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) - bf_int64_df = scalars_df[int64_cols] - bf_int64_df_filtered = bf_int64_df.dropna() - pd_int64_df = scalars_pandas_df[int64_cols] - pd_int64_df_filtered = pd_int64_df.dropna() + # Make sure the read_gbq_function path works for dataframe.where method. + is_sum_positive_ref = session.read_gbq_function( + function_name=is_sum_positive_mf.bigframes_bigquery_function + ) - # Test callable condition in dataframe.where method. - bf_result = bf_int64_df_filtered.where(is_sum_positive_series_mf).to_pandas() - pd_result = pd_int64_df_filtered.where(is_sum_positive_series) + bf_result_gbq = bf_int64_df_filtered.where( + is_sum_positive_ref, -bf_int64_df_filtered + ).to_pandas() + pd_result_gbq = pd_int64_df_filtered.where( + pd_int64_df_filtered.sum(axis=1) > 0, -pd_int64_df_filtered + ) - # Ignore any dtype difference. - pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) + # Ignore any dtype difference. + pandas.testing.assert_frame_equal( + bf_result_gbq, pd_result_gbq, check_dtype=False + ) - # Make sure the read_gbq_function path works for dataframe.where method. - is_sum_positive_series_ref = session.read_gbq_function( - f"{dataset_id}.{function_id}", is_row_processor=True - ) + # Test callable condition in dataframe.mask method. + bf_result_gbq = bf_int64_df_filtered.mask( + is_sum_positive_ref, -bf_int64_df_filtered + ).to_pandas() + pd_result_gbq = pd_int64_df_filtered.mask( + pd_int64_df_filtered.sum(axis=1) > 0, -pd_int64_df_filtered + ) - # This is for callable `other` arg in dataframe.where method. - def func_for_other(x): - return -x + # Ignore any dtype difference. + pandas.testing.assert_frame_equal( + bf_result_gbq, pd_result_gbq, check_dtype=False + ) - bf_result_gbq = bf_int64_df_filtered.where( - is_sum_positive_series_ref, func_for_other - ).to_pandas() - pd_result_gbq = pd_int64_df_filtered.where(is_sum_positive_series, func_for_other) + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets( + is_sum_positive_mf, session.bqclient, ignore_failures=False + ) - # Ignore any dtype difference. - pandas.testing.assert_frame_equal(bf_result_gbq, pd_result_gbq, check_dtype=False) - # Test callable condition in dataframe.mask method. - bf_result_gbq = bf_int64_df_filtered.mask( - is_sum_positive_series_ref, func_for_other - ).to_pandas() - pd_result_gbq = pd_int64_df_filtered.mask(is_sum_positive_series, func_for_other) +def test_managed_function_df_where_mask_series(session, dataset_id, scalars_dfs): + try: - # Ignore any dtype difference. - pandas.testing.assert_frame_equal(bf_result_gbq, pd_result_gbq, check_dtype=False) + # The return type has to be bool type for callable where condition. + def is_sum_positive_series(s): + return s["int64_col"] + s["int64_too"] > 0 + is_sum_positive_series_mf = session.udf( + input_types=pandas.Series, + output_type=bool, + dataset=dataset_id, + name=prefixer.create_prefix(), + )(is_sum_positive_series) -def test_managed_function_df_where_other_issue( - session, dataset_id, scalars_df_index, function_id -): - def the_sum(s: pandas.Series) -> int: - return s["int64_col"] + s["int64_too"] + scalars_df, scalars_pandas_df = scalars_dfs + int64_cols = ["int64_col", "int64_too"] - the_sum_mf = session.udf( - dataset=dataset_id, - name=function_id, - )(the_sum) + bf_int64_df = scalars_df[int64_cols] + bf_int64_df_filtered = bf_int64_df.dropna() + pd_int64_df = scalars_pandas_df[int64_cols] + pd_int64_df_filtered = pd_int64_df.dropna() - int64_cols = ["int64_col", "int64_too"] + # Test callable condition in dataframe.where method. + bf_result = bf_int64_df_filtered.where(is_sum_positive_series_mf).to_pandas() + pd_result = pd_int64_df_filtered.where(is_sum_positive_series) - bf_int64_df = scalars_df_index[int64_cols] - bf_int64_df_filtered = bf_int64_df.dropna() + # Ignore any dtype difference. + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) - with pytest.raises( - ValueError, - match="Seires is not a supported replacement type!", - ): - # The execution of the callable other=the_sum_mf will return a - # Series, which is not a supported replacement type. - bf_int64_df_filtered.where(cond=bf_int64_df_filtered, other=the_sum_mf) + # Make sure the read_gbq_function path works for dataframe.where method. + is_sum_positive_series_ref = session.read_gbq_function( + function_name=is_sum_positive_series_mf.bigframes_bigquery_function, + is_row_processor=True, + ) + # This is for callable `other` arg in dataframe.where method. + def func_for_other(x): + return -x -def test_managed_function_series_where_mask_map( - session, dataset_id, scalars_dfs, function_id -): - # The return type has to be bool type for callable where condition. - def _is_positive(s): - return s + 1000 > 0 + bf_result_gbq = bf_int64_df_filtered.where( + is_sum_positive_series_ref, func_for_other + ).to_pandas() + pd_result_gbq = pd_int64_df_filtered.where( + is_sum_positive_series, func_for_other + ) - is_positive_mf = session.udf( - input_types=int, - output_type=bool, - dataset=dataset_id, - name=function_id, - )(_is_positive) + # Ignore any dtype difference. + pandas.testing.assert_frame_equal( + bf_result_gbq, pd_result_gbq, check_dtype=False + ) - scalars, scalars_pandas = scalars_dfs + # Test callable condition in dataframe.mask method. + bf_result_gbq = bf_int64_df_filtered.mask( + is_sum_positive_series_ref, func_for_other + ).to_pandas() + pd_result_gbq = pd_int64_df_filtered.mask( + is_sum_positive_series, func_for_other + ) - bf_int64 = scalars["int64_col"] - bf_int64_filtered = bf_int64.dropna() - pd_int64 = scalars_pandas["int64_col"] - pd_int64_filtered = pd_int64.dropna() + # Ignore any dtype difference. + pandas.testing.assert_frame_equal( + bf_result_gbq, pd_result_gbq, check_dtype=False + ) - # Test series.where method: the cond is a callable (managed function) - # and the other is not a callable. - bf_result = bf_int64_filtered.where( - cond=is_positive_mf, other=-bf_int64_filtered - ).to_pandas() - pd_result = pd_int64_filtered.where(cond=_is_positive, other=-pd_int64_filtered) + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets( + is_sum_positive_series_mf, session.bqclient, ignore_failures=False + ) - # Ignore any dtype difference. - pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) - # Test series.mask method: the cond is a callable (managed function) - # and the other is not a callable. - bf_result = bf_int64_filtered.mask( - cond=is_positive_mf, other=-bf_int64_filtered - ).to_pandas() - pd_result = pd_int64_filtered.mask(cond=_is_positive, other=-pd_int64_filtered) +def test_managed_function_df_where_other_issue(session, dataset_id, scalars_df_index): + try: - # Ignore any dtype difference. - pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) + def the_sum(s: pandas.Series) -> int: + return s["int64_col"] + s["int64_too"] - # Test series.map method. - bf_result = bf_int64_filtered.map(is_positive_mf).to_pandas() - pd_result = pd_int64_filtered.map(_is_positive) + the_sum_mf = session.udf( + dataset=dataset_id, + name=prefixer.create_prefix(), + )(the_sum) - # Ignore any dtype difference. - pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) + int64_cols = ["int64_col", "int64_too"] + bf_int64_df = scalars_df_index[int64_cols] + bf_int64_df_filtered = bf_int64_df.dropna() -def test_managed_function_series_apply_args( - session, dataset_id, scalars_dfs, function_id -): - with pytest.warns(bfe.PreviewWarning, match="udf is in preview."): + with pytest.raises( + ValueError, + match="Seires is not a supported replacement type!", + ): + # The execution of the callable other=the_sum_mf will return a + # Series, which is not a supported replacement type. + bf_int64_df_filtered.where(cond=bf_int64_df_filtered, other=the_sum_mf) - @session.udf(dataset=dataset_id, name=function_id) - def foo_list(x: int, y0: float, y1: bytes, y2: bool) -> list[str]: - return [str(x), str(y0), str(y1), str(y2)] + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets(the_sum_mf, session.bqclient, ignore_failures=False) - scalars_df, scalars_pandas_df = scalars_dfs - bf_result = ( - scalars_df["int64_too"] - .apply(foo_list, args=(12.34, b"hello world", False)) - .to_pandas() - ) - pd_result = scalars_pandas_df["int64_too"].apply( - foo_list, args=(12.34, b"hello world", False) - ) +def test_managed_function_series_where_mask_map(session, dataset_id, scalars_dfs): + try: - # Ignore any dtype difference. - pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) + # The return type has to be bool type for callable where condition. + def _is_positive(s): + return s + 1000 > 0 + is_positive_mf = session.udf( + input_types=int, + output_type=bool, + dataset=dataset_id, + name=prefixer.create_prefix(), + )(_is_positive) -def test_deferred_unnamed_udf_execution(session, scalars_dfs): - import bigframes.functions.udf_def as udf_def + scalars, scalars_pandas = scalars_dfs - # Create an unnamed UDF (name=None) - @session.udf() - def unnamed_multiplier(x: int) -> int: - return x * 3 + bf_int64 = scalars["int64_col"] + bf_int64_filtered = bf_int64.dropna() + pd_int64 = scalars_pandas["int64_col"] + pd_int64_filtered = pd_int64.dropna() - assert isinstance(unnamed_multiplier.udf_def, udf_def.PythonUdf) + # Test series.where method: the cond is a callable (managed function) + # and the other is not a callable. + bf_result = bf_int64_filtered.where( + cond=is_positive_mf, other=-bf_int64_filtered + ).to_pandas() + pd_result = pd_int64_filtered.where(cond=_is_positive, other=-pd_int64_filtered) - scalars_df, scalars_pandas_df = scalars_dfs - bf_series = scalars_df["int64_too"] - pd_series = scalars_pandas_df["int64_too"] + # Ignore any dtype difference. + pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) - bf_result = bf_series.apply(unnamed_multiplier).to_pandas() - pd_result = pd_series.apply(lambda x: x * 3) + # Test series.mask method: the cond is a callable (managed function) + # and the other is not a callable. + bf_result = bf_int64_filtered.mask( + cond=is_positive_mf, other=-bf_int64_filtered + ).to_pandas() + pd_result = pd_int64_filtered.mask(cond=_is_positive, other=-pd_int64_filtered) - pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) + # Ignore any dtype difference. + pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) - import bigframes.functions._function_session as functions_sessions + # Test series.map method. + bf_result = bf_int64_filtered.map(is_positive_mf).to_pandas() + pd_result = pd_int64_filtered.map(_is_positive) - config = unnamed_multiplier.udf_def.to_managed_function_config() - expected_routine_name = functions_sessions.get_managed_function_name( - config, session.session_id - ) - routine = session.bqclient.get_routine( - f"{session._anonymous_dataset.project}.{session._anonymous_dataset.dataset_id}.{expected_routine_name}" - ) - assert routine is not None + # Ignore any dtype difference. + pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets(is_positive_mf, session.bqclient, ignore_failures=False) -def test_deferred_udf_with_runtime_requirements(session, scalars_dfs): - import bigframes.functions.udf_def as udf_def - # Create an unnamed UDF with custom options - @session.udf( - container_cpu=1, - container_memory="2Gi", - max_batching_rows=25, - ) - def heavy_unnamed_udf(x: int) -> int: - return x + 100 +def test_managed_function_series_apply_args(session, dataset_id, scalars_dfs): + try: - assert isinstance(heavy_unnamed_udf.udf_def, udf_def.PythonUdf) + with pytest.warns(bfe.PreviewWarning, match="udf is in preview."): - scalars_df, scalars_pandas_df = scalars_dfs - bf_series = scalars_df["int64_too"] - pd_series = scalars_pandas_df["int64_too"] + @session.udf(dataset=dataset_id, name=prefixer.create_prefix()) + def foo_list(x: int, y0: float, y1: bytes, y2: bool) -> list[str]: + return [str(x), str(y0), str(y1), str(y2)] - bf_result = bf_series.apply(heavy_unnamed_udf).to_pandas() - pd_result = pd_series.apply(lambda x: x + 100) + scalars_df, scalars_pandas_df = scalars_dfs - pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) + bf_result = ( + scalars_df["int64_too"] + .apply(foo_list, args=(12.34, b"hello world", False)) + .to_pandas() + ) + pd_result = scalars_pandas_df["int64_too"].apply( + foo_list, args=(12.34, b"hello world", False) + ) - # Verify it was deployed with the correct runtime options - import bigframes.functions._function_session as functions_sessions + # Ignore any dtype difference. + pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) - config = heavy_unnamed_udf.udf_def.to_managed_function_config() - expected_routine_name = functions_sessions.get_managed_function_name( - config, session.session_id - ) - routine = session.bqclient.get_routine( - f"{session._anonymous_dataset.project}.{session._anonymous_dataset.dataset_id}.{expected_routine_name}" - ) - assert routine._properties["externalRuntimeOptions"]["containerCpu"] == 1 - assert routine._properties["externalRuntimeOptions"]["containerMemory"] == "2Gi" - assert routine._properties["externalRuntimeOptions"]["maxBatchingRows"] == "25" + finally: + # Clean up the gcp assets created for the managed function. + cleanup_function_assets(foo_list, session.bqclient, ignore_failures=False) diff --git a/tests/system/large/functions/test_remote_function.py b/tests/system/large/functions/test_remote_function.py index 69769a1a846..2591c0c13a2 100644 --- a/tests/system/large/functions/test_remote_function.py +++ b/tests/system/large/functions/test_remote_function.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from datetime import datetime import importlib.util import inspect import math # must keep this at top level to test udf referring global import @@ -19,20 +20,19 @@ import shutil import tempfile import textwrap -import uuid import warnings -from datetime import datetime import google.api_core.exceptions +from google.cloud import bigquery, functions_v2, storage import pandas import pytest import test_utils.prefixer -from google.cloud import bigquery, functions_v2, storage import bigframes import bigframes.dataframe import bigframes.dtypes import bigframes.exceptions +import bigframes.functions._utils as bff_utils import bigframes.pandas as bpd import bigframes.series from bigframes.testing.utils import ( @@ -526,6 +526,24 @@ def add_one(x): # Make a unique udf add_one_uniq, add_one_uniq_dir = make_uniq_udf(add_one) + # Expected cloud function name for the unique udf + package_requirements = bff_utils.get_updated_package_requirements() + add_one_uniq_hash = bff_utils.get_hash(add_one_uniq, package_requirements) + add_one_uniq_cf_name = bff_utils.get_cloud_function_name( + add_one_uniq_hash, session.session_id + ) + + # There should be no cloud function yet for the unique udf + cloud_functions = list( + get_cloud_functions( + session.cloudfunctionsclient, + session.bqclient.project, + session.bqclient.location, + name=add_one_uniq_cf_name, + ) + ) + assert len(cloud_functions) == 0 + # The first time both the cloud function and the bq remote function don't # exist and would be created remote_add_one = session.remote_function( @@ -537,9 +555,6 @@ def add_one(x): cloud_function_service_account="default", )(add_one_uniq) - assert remote_add_one.bigframes_cloud_function is not None - add_one_uniq_cf_name = remote_add_one.bigframes_cloud_function.split("/")[-1] - # There should have been excactly one cloud function created at this point cloud_functions = list( get_cloud_functions( @@ -827,6 +842,7 @@ def test_remote_function_with_external_package_dependencies( session, scalars_dfs, dataset_id, bq_cf_connection ): try: + # The return type hint in this function's signature has conflict. The # `output_type` argument from remote_function decorator takes precedence # and will be used instead. @@ -881,6 +897,7 @@ def test_remote_function_with_explicit_name_reuse( session, scalars_dfs, dataset_id, bq_cf_connection ): try: + dirs_to_cleanup = [] # Define a user code @@ -1215,7 +1232,7 @@ def square(x): @pytest.mark.flaky(retries=2, delay=120) -def test_remote_function_via_session_custom_sa(scalars_pandas_df_index): +def test_remote_function_via_session_custom_sa(scalars_dfs): # TODO(shobs): Automate the following set-up during testing in the test project. # # For upfront convenience, the following set up has been statically created @@ -1235,39 +1252,42 @@ def test_remote_function_via_session_custom_sa(scalars_pandas_df_index): try: + # TODO(shobs): Figure out why the default ingress setting + # (internal-only) does not work here @rf_session.remote_function( input_types=[int], output_type=int, reuse=False, cloud_function_service_account=gcf_service_account, - cloud_function_ingress_settings="internal-and-gclb", + cloud_function_ingress_settings="all", ) - def double_num(x): + def square_num(x): if x is None: return x - return x + x + return x * x # assert that the GCF is created with the intended SA gcf = rf_session.cloudfunctionsclient.get_function( - name=double_num.bigframes_cloud_function + name=square_num.bigframes_cloud_function ) assert gcf.service_config.service_account_email == gcf_service_account # assert that the function works as expected on data + scalars_df, scalars_pandas_df = scalars_dfs - bf_int64_col = rf_session.read_pandas(scalars_pandas_df_index.int64_col) - bf_result_col = bf_int64_col.apply(double_num) + bf_int64_col = scalars_df["int64_col"] + bf_result_col = bf_int64_col.apply(square_num) bf_result = bf_int64_col.to_frame().assign(result=bf_result_col).to_pandas() - pd_int64_col = scalars_pandas_df_index.int64_col - pd_result_col = pd_int64_col.apply(lambda x: x if x is None else x + x) + pd_int64_col = scalars_pandas_df["int64_col"] + pd_result_col = pd_int64_col.apply(lambda x: x if x is None else x * x) pd_result = pd_int64_col.to_frame().assign(result=pd_result_col) assert_frame_equal(bf_result, pd_result, check_dtype=False) finally: # clean up the gcp assets created for the remote function cleanup_function_assets( - double_num, rf_session.bqclient, rf_session.cloudfunctionsclient + square_num, rf_session.bqclient, rf_session.cloudfunctionsclient ) @@ -1286,7 +1306,7 @@ def double_num(x): ) @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_via_session_custom_build_sa( - set_build_service_account, scalars_pandas_df_index + scalars_dfs, set_build_service_account ): # TODO(shobs): Automate the following set-up during testing in the test project. # @@ -1305,38 +1325,43 @@ def test_remote_function_via_session_custom_build_sa( try: + # TODO(shobs): Figure out why the default ingress setting + # (internal-only) does not work here @rf_session.remote_function( input_types=[int], output_type=int, reuse=False, cloud_function_service_account="default", cloud_build_service_account=set_build_service_account, - cloud_function_ingress_settings="internal-and-gclb", + cloud_function_ingress_settings="all", ) - def double_num(x): + def square_num(x): if x is None: return x - return x + x + return x * x # assert that the GCF is created with the intended SA gcf = rf_session.cloudfunctionsclient.get_function( - name=double_num.bigframes_cloud_function + name=square_num.bigframes_cloud_function ) assert gcf.build_config.service_account == expected_build_service_account - bf_int64_col = rf_session.read_pandas(scalars_pandas_df_index.int64_col) - bf_result_col = bf_int64_col.apply(double_num) + # assert that the function works as expected on data + scalars_df, scalars_pandas_df = scalars_dfs + + bf_int64_col = scalars_df["int64_col"] + bf_result_col = bf_int64_col.apply(square_num) bf_result = bf_int64_col.to_frame().assign(result=bf_result_col).to_pandas() - pd_int64_col = scalars_pandas_df_index.int64_col - pd_result_col = pd_int64_col.apply(lambda x: x if x is None else x + x) + pd_int64_col = scalars_pandas_df["int64_col"] + pd_result_col = pd_int64_col.apply(lambda x: x if x is None else x * x) pd_result = pd_int64_col.to_frame().assign(result=pd_result_col) assert_frame_equal(bf_result, pd_result, check_dtype=False) finally: # clean up the gcp assets created for the remote function cleanup_function_assets( - double_num, rf_session.bqclient, rf_session.cloudfunctionsclient + square_num, rf_session.bqclient, rf_session.cloudfunctionsclient ) @@ -1415,7 +1440,7 @@ def square_num(x): @pytest.mark.flaky(retries=2, delay=120) -def test_remote_function_via_session_vpc(scalars_pandas_df_index): +def test_remote_function_via_session_vpc(scalars_dfs): # TODO(shobs): Automate the following set-up during testing in the test project. # # For upfront convenience, the following set up has been statically created @@ -1440,45 +1465,50 @@ def test_remote_function_via_session_vpc(scalars_pandas_df_index): try: - def double_num(x): + def square_num(x): if x is None: return x - return x + x + return x * x - double_num_remote = rf_session.remote_function( + # TODO(shobs): See if the test vpc can be configured to make this flow + # work with the default ingress setting (internal-only) + square_num_remote = rf_session.remote_function( input_types=[int], output_type=int, reuse=False, cloud_function_service_account="default", cloud_function_vpc_connector=gcf_vpc_connector, cloud_function_vpc_connector_egress_settings="all", - cloud_function_ingress_settings="internal-and-gclb", - )(double_num) + cloud_function_ingress_settings="all", + )(square_num) gcf = rf_session.cloudfunctionsclient.get_function( - name=double_num_remote.bigframes_cloud_function + name=square_num_remote.bigframes_cloud_function ) - # assert that the GCF test_remote_function_via_session_custom_sais created with the intended vpc connector and + # assert that the GCF is created with the intended vpc connector and # egress settings. assert gcf.service_config.vpc_connector == gcf_vpc_connector # The value is since we set # cloud_function_vpc_connector_egress_settings="all" earlier. assert gcf.service_config.vpc_connector_egress_settings == 2 - bf_int64_col = rf_session.read_pandas(scalars_pandas_df_index.int64_col) - bf_result_col = bf_int64_col.apply(double_num_remote) + # assert that the function works as expected on data + scalars_df, scalars_pandas_df = scalars_dfs + + bf_int64_col = scalars_df["int64_col"] + bf_result_col = bf_int64_col.apply(square_num_remote) bf_result = bf_int64_col.to_frame().assign(result=bf_result_col).to_pandas() - pd_int64_col = scalars_pandas_df_index.int64_col - pd_result_col = pd_int64_col.apply(double_num) + pd_int64_col = scalars_pandas_df["int64_col"] + pd_result_col = pd_int64_col.apply(square_num) pd_result = pd_int64_col.to_frame().assign(result=pd_result_col) assert_frame_equal(bf_result, pd_result, check_dtype=False) finally: # clean up the gcp assets created for the remote function cleanup_function_assets( - double_num_remote, rf_session.bqclient, rf_session.cloudfunctionsclient + square_num_remote, rf_session.bqclient, rf_session.cloudfunctionsclient ) @@ -1547,9 +1577,7 @@ def square(x): bq_routine = session.bqclient.get_routine( square_remote.bigframes_bigquery_function ) - assert bq_routine.remote_function_options.max_batching_rows == ( - max_batching_rows or 1000 - ) + assert bq_routine.remote_function_options.max_batching_rows == max_batching_rows scalars_df, scalars_pandas_df = scalars_dfs @@ -1669,51 +1697,6 @@ def square(x): ) -@pytest.mark.flaky(retries=2, delay=120) -def test_remote_function_reflects_config_change_with_reuse(session): - square_remote = None - square_remote_2 = None - try: - - def square(x): - return x * x - - # random alphanumeric name starting with a letter - deploy_name = "a" + str(uuid.uuid4().hex) - square_remote = session.remote_function( - input_types=[int], - name=deploy_name, - output_type=int, - reuse=True, - cloud_function_service_account="default", - cloud_function_cpus=1, - )(square) - square_remote_2 = session.remote_function( - input_types=[int], - name=deploy_name, - output_type=int, - reuse=True, - cloud_function_service_account="default", - cloud_function_cpus=2, - )(square) - - # Assert that the GCF is created with the intended max instance count - gcf = session.cloudfunctionsclient.get_function( - name=square_remote_2.bigframes_cloud_function - ) - assert float(gcf.service_config.available_cpu) == 2.0 - finally: - # clean up the gcp assets created for the remote function - if square_remote is not None: - cleanup_function_assets( - square_remote, session.bqclient, session.cloudfunctionsclient - ) - if square_remote_2 is not None: - cleanup_function_assets( - square_remote_2, session.bqclient, session.cloudfunctionsclient - ) - - @pytest.mark.flaky(retries=2, delay=120) def test_df_apply_axis_1(session, scalars_dfs): columns = ["bool_col", "int64_col", "int64_too", "float64_col", "string_col"] @@ -2110,40 +2093,19 @@ def foo_list(x: pandas.Series, y0: float, y1, y2) -> list[str]: @pytest.mark.parametrize( - ( - "memory_mib_args", - "expected_memory", - "expected_cpus", - ), + ("memory_mib_args", "expected_memory"), [ - pytest.param({}, "1024Mi", None, id="no-set"), - pytest.param( - {"cloud_function_memory_mib": None}, "1024Mi", None, id="set-None" - ), - pytest.param({"cloud_function_memory_mib": 128}, "128Mi", None, id="set-128"), - pytest.param( - {"cloud_function_memory_mib": 512, "cloud_function_cpus": 0.6}, - "512Mi", - "0.6", - id="set-512", - ), - pytest.param( - {"cloud_function_memory_mib": 1024}, "1024Mi", None, id="set-1024" - ), - pytest.param( - {"cloud_function_memory_mib": 4096, "cloud_function_cpus": 4}, - "4096Mi", - "4", - id="set-4096", - ), - pytest.param( - {"cloud_function_memory_mib": 32768}, "32768Mi", None, id="set-32768" - ), + pytest.param({}, "1024Mi", id="no-set"), + pytest.param({"cloud_function_memory_mib": None}, "256M", id="set-None"), + pytest.param({"cloud_function_memory_mib": 128}, "128Mi", id="set-128"), + pytest.param({"cloud_function_memory_mib": 1024}, "1024Mi", id="set-1024"), + pytest.param({"cloud_function_memory_mib": 4096}, "4096Mi", id="set-4096"), + pytest.param({"cloud_function_memory_mib": 32768}, "32768Mi", id="set-32768"), ], ) @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_gcf_memory( - session, scalars_dfs, memory_mib_args, expected_memory, expected_cpus + session, scalars_dfs, memory_mib_args, expected_memory ): try: @@ -2159,12 +2121,6 @@ def square(x: int) -> int: name=square_remote.bigframes_cloud_function ) assert gcf.service_config.available_memory == expected_memory - if expected_cpus is not None: - assert gcf.service_config.available_cpu == expected_cpus - if float(gcf.service_config.available_cpu) >= 1.0: - assert gcf.service_config.max_instance_request_concurrency >= float( - gcf.service_config.available_cpu - ) scalars_df, scalars_pandas_df = scalars_dfs @@ -2186,8 +2142,12 @@ def square(x: int) -> int: pytest.param(32769, id="set-32769-too-high"), ], ) +@pytest.mark.flaky(retries=2, delay=120) def test_remote_function_gcf_memory_unsupported(session, memory_mib): - with pytest.raises(ValueError, match="Cloud run supports"): + with pytest.raises( + google.api_core.exceptions.InvalidArgument, + match="Invalid value specified for container memory", + ): @session.remote_function( reuse=False, @@ -2650,6 +2610,12 @@ def generate_stats(row: pandas.Series) -> list[int]: True, id="set-none", ), + pytest.param( + {"cloud_function_ingress_settings": "all"}, + functions_v2.ServiceConfig.IngressSettings.ALLOW_ALL, + False, + id="set-all", + ), pytest.param( {"cloud_function_ingress_settings": "internal-only"}, functions_v2.ServiceConfig.IngressSettings.ALLOW_INTERNAL_ONLY, @@ -3041,6 +3007,7 @@ def foo(x: int) -> int: @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_df_where_mask(session, dataset_id, scalars_dfs): try: + # The return type has to be bool type for callable where condition. def is_sum_positive(a, b): return a + b > 0 @@ -3119,6 +3086,7 @@ def the_sum(a, b): @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_df_where_mask_series(session, dataset_id, scalars_dfs): try: + # The return type has to be bool type for callable where condition. def is_sum_positive_series(s: pandas.Series) -> bool: return s["int64_col"] + s["int64_too"] > 0 diff --git a/tests/system/large/ml/conftest.py b/tests/system/large/ml/conftest.py index ffb02e8beb8..7735f3eff56 100644 --- a/tests/system/large/ml/conftest.py +++ b/tests/system/large/ml/conftest.py @@ -14,9 +14,9 @@ import hashlib import logging +from google.cloud import bigquery import google.cloud.exceptions import pytest -from google.cloud import bigquery import bigframes from bigframes.ml import core, linear_model diff --git a/tests/system/large/ml/test_ensemble.py b/tests/system/large/ml/test_ensemble.py index eabd36ab387..c2e9036eed7 100644 --- a/tests/system/large/ml/test_ensemble.py +++ b/tests/system/large/ml/test_ensemble.py @@ -155,7 +155,7 @@ def test_xgbclassifier_default_params(penguins_df_default_index, dataset_id): ) -@pytest.mark.flaky(retries=2) +# @pytest.mark.flaky(retries=2) def test_xgbclassifier_dart_booster_multiple_params( penguins_df_default_index, dataset_id ): diff --git a/tests/system/large/ml/test_forecasting.py b/tests/system/large/ml/test_forecasting.py index 8500ad9d5f1..72a0ee469b5 100644 --- a/tests/system/large/ml/test_forecasting.py +++ b/tests/system/large/ml/test_forecasting.py @@ -88,7 +88,6 @@ def test_arima_plus_model_fit_score( result, columns=expected_columns, index=2 if id_col_name else 1, - col_exact=False, ) # save, load to ensure configuration was kept diff --git a/tests/system/large/ml/test_linear_model.py b/tests/system/large/ml/test_linear_model.py index 60edc717a5a..d7bb122772e 100644 --- a/tests/system/large/ml/test_linear_model.py +++ b/tests/system/large/ml/test_linear_model.py @@ -15,8 +15,8 @@ import pandas as pd import pytest -import bigframes.ml.linear_model from bigframes.ml import model_selection +import bigframes.ml.linear_model from bigframes.testing import utils diff --git a/tests/system/large/ml/test_llm.py b/tests/system/large/ml/test_llm.py index 638e151ca14..1daaebb8cb8 100644 --- a/tests/system/large/ml/test_llm.py +++ b/tests/system/large/ml/test_llm.py @@ -12,27 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable -from unittest import mock - import pandas as pd import pyarrow as pa import pytest +from bigframes.ml import llm import bigframes.pandas as bpd -from bigframes.ml import core, llm from bigframes.testing import utils @pytest.mark.parametrize( "model_name", ( + "gemini-2.0-flash-exp", + "gemini-2.0-flash-001", + "gemini-2.0-flash-lite-001", "gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite", ), ) -@pytest.mark.flaky(retries=2) +@pytest.mark.flaky( + retries=2 +) # usually create model shouldn't be flaky, but this one due to the limited quota of gemini-2.0-flash-exp. def test_create_load_gemini_text_generator_model( dataset_id, model_name, session, bq_connection ): @@ -54,12 +56,15 @@ def test_create_load_gemini_text_generator_model( @pytest.mark.parametrize( "model_name", ( + "gemini-2.0-flash-exp", + "gemini-2.0-flash-001", + "gemini-2.0-flash-lite-001", "gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite", ), ) -@pytest.mark.flaky(retries=2) +# @pytest.mark.flaky(retries=2) def test_gemini_text_generator_predict_default_params_success( llm_text_df, model_name, session, bq_connection ): @@ -75,6 +80,9 @@ def test_gemini_text_generator_predict_default_params_success( @pytest.mark.parametrize( "model_name", ( + "gemini-2.0-flash-exp", + "gemini-2.0-flash-001", + "gemini-2.0-flash-lite-001", "gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite", @@ -98,6 +106,9 @@ def test_gemini_text_generator_predict_with_params_success( @pytest.mark.parametrize( "model_name", ( + "gemini-2.0-flash-exp", + "gemini-2.0-flash-001", + "gemini-2.0-flash-lite-001", "gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite", @@ -123,6 +134,9 @@ def test_gemini_text_generator_multi_cols_predict_success( @pytest.mark.parametrize( "model_name", ( + "gemini-2.0-flash-exp", + "gemini-2.0-flash-001", + "gemini-2.0-flash-lite-001", "gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite", @@ -166,8 +180,8 @@ def test_gemini_text_generator_predict_output_schema_success( @pytest.mark.parametrize( "model_name", ( - "gemini-2.5-flash", - "gemini-2.5-flash-lite", + "gemini-2.0-flash-001", + "gemini-2.0-flash-lite-001", ), ) def test_llm_gemini_score(llm_fine_tune_df_default_index, model_name): @@ -188,15 +202,14 @@ def test_llm_gemini_score(llm_fine_tune_df_default_index, model_name): "evaluation_status", ], index=1, - col_exact=False, ) @pytest.mark.parametrize( "model_name", ( - "gemini-2.5-flash", - "gemini-2.5-flash-lite", + "gemini-2.0-flash-001", + "gemini-2.0-flash-lite-001", ), ) def test_llm_gemini_pro_score_params(llm_fine_tune_df_default_index, model_name): @@ -217,583 +230,4 @@ def test_llm_gemini_pro_score_params(llm_fine_tune_df_default_index, model_name) "label", "evaluation_status", ], - col_exact=False, - ) - - -@pytest.mark.parametrize( - "model_name", - ("text-embedding-005", "text-embedding-004", "text-multilingual-embedding-002"), -) -def test_create_load_text_embedding_generator_model( - dataset_id, model_name, session, bq_connection -): - text_embedding_model = llm.TextEmbeddingGenerator( - model_name=model_name, connection_name=bq_connection, session=session - ) - assert text_embedding_model is not None - assert text_embedding_model._bqml_model is not None - - # save, load to ensure configuration was kept - reloaded_model = text_embedding_model.to_gbq( - f"{dataset_id}.temp_text_model", replace=True - ) - assert f"{dataset_id}.temp_text_model" == reloaded_model._bqml_model.model_name - assert reloaded_model.connection_name == bq_connection - assert reloaded_model.model_name == model_name - - -@pytest.mark.parametrize( - "model_name", - ("text-embedding-005", "text-embedding-004", "text-multilingual-embedding-002"), -) -@pytest.mark.flaky(retries=2) -def test_text_embedding_generator_predict_default_params_success( - llm_text_df, model_name, session, bq_connection -): - text_embedding_model = llm.TextEmbeddingGenerator( - model_name=model_name, connection_name=bq_connection, session=session - ) - df = text_embedding_model.predict(llm_text_df).to_pandas() - utils.check_pandas_df_schema_and_index( - df, columns=utils.ML_GENERATE_EMBEDDING_OUTPUT, index=3, col_exact=False - ) - assert len(df["ml_generate_embedding_result"][0]) == 768 - - -@pytest.mark.parametrize( - "model_name", - ("text-embedding-005", "text-embedding-004", "text-multilingual-embedding-002"), -) -@pytest.mark.flaky(retries=2) -def test_text_embedding_generator_multi_cols_predict_success( - llm_text_df: bpd.DataFrame, model_name, session, bq_connection -): - df = llm_text_df.assign(additional_col=1) - df = df.rename(columns={"prompt": "content"}) - text_embedding_model = llm.TextEmbeddingGenerator( - model_name=model_name, connection_name=bq_connection, session=session - ) - pd_df = text_embedding_model.predict(df).to_pandas() - utils.check_pandas_df_schema_and_index( - pd_df, - columns=utils.ML_GENERATE_EMBEDDING_OUTPUT + ["additional_col"], - index=3, - col_exact=False, - ) - assert len(pd_df["ml_generate_embedding_result"][0]) == 768 - - -def test_create_load_multimodal_embedding_generator_model( - dataset_id, session, bq_connection -): - mm_embedding_model = llm.MultimodalEmbeddingGenerator( - connection_name=bq_connection, session=session - ) - assert mm_embedding_model is not None - assert mm_embedding_model._bqml_model is not None - - # save, load to ensure configuration was kept - reloaded_model = mm_embedding_model.to_gbq( - f"{dataset_id}.temp_mm_model", replace=True - ) - assert f"{dataset_id}.temp_mm_model" == reloaded_model._bqml_model.model_name - assert reloaded_model.connection_name == bq_connection - - -# Overrides __eq__ function for comparing as mock.call parameter -class EqCmpAllDataFrame(bpd.DataFrame): - def __eq__(self, other): - return self.equals(other) - - -@pytest.mark.skip("b/436340035 test failed") -@pytest.mark.parametrize( - ( - "model_class", - "options", - ), - [ - ( - llm.GeminiTextGenerator, - { - "temperature": 0.9, - "max_output_tokens": 8192, - "top_p": 1.0, - "ground_with_google_search": False, - }, - ), - ( - llm.Claude3TextGenerator, - { - "max_output_tokens": 128, - "top_k": 40, - "top_p": 0.95, - }, - ), - ], -) -def test_text_generator_retry_success( - session, - model_class, - options, - bq_connection, -): - # Requests. - df0 = EqCmpAllDataFrame( - { - "prompt": [ - "What is BigQuery?", - "What is BQML?", - "What is BigQuery DataFrame?", - ] - }, - index=[0, 1, 2], - session=session, - ) - df1 = EqCmpAllDataFrame( - { - "ml_generate_text_status": ["error", "error"], - "prompt": [ - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[1, 2], - session=session, - ) - df2 = EqCmpAllDataFrame( - { - "ml_generate_text_status": ["error"], - "prompt": [ - "What is BQML?", - ], - }, - index=[1], - session=session, ) - - mock_generate_text = mock.create_autospec( - Callable[[core.BqmlModel, bpd.DataFrame, dict], bpd.DataFrame] - ) - mock_bqml_model = mock.create_autospec(spec=core.BqmlModel) - type(mock_bqml_model).session = mock.PropertyMock(return_value=session) - generate_text_tvf = core.BqmlModel.TvfDef( - mock_generate_text, "ml_generate_text_status" - ) - # Responses. Retry twice then all succeeded. - mock_generate_text.side_effect = [ - EqCmpAllDataFrame( - { - "ml_generate_text_status": ["", "error", "error"], - "prompt": [ - "What is BigQuery?", - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[0, 1, 2], - session=session, - ), - EqCmpAllDataFrame( - { - "ml_generate_text_status": ["error", ""], - "prompt": [ - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[1, 2], - session=session, - ), - EqCmpAllDataFrame( - { - "ml_generate_text_status": [""], - "prompt": [ - "What is BQML?", - ], - }, - index=[1], - session=session, - ), - ] - - text_generator_model = model_class(connection_name=bq_connection, session=session) - text_generator_model._bqml_model = mock_bqml_model - - with mock.patch.object(core.BqmlModel, "generate_text_tvf", generate_text_tvf): - # 3rd retry isn't triggered - result = text_generator_model.predict(df0, max_retries=3) - - mock_generate_text.assert_has_calls( - [ - mock.call(mock_bqml_model, df0, options), - mock.call(mock_bqml_model, df1, options), - mock.call(mock_bqml_model, df2, options), - ] - ) - pd.testing.assert_frame_equal( - result.to_pandas(), - pd.DataFrame( - { - "ml_generate_text_status": ["", "", ""], - "prompt": [ - "What is BigQuery?", - "What is BigQuery DataFrame?", - "What is BQML?", - ], - }, - index=[0, 2, 1], - ), - check_dtype=False, - check_index_type=False, - ) - - -@pytest.mark.skip("b/436340035 test failed") -@pytest.mark.parametrize( - ( - "model_class", - "options", - ), - [ - ( - llm.GeminiTextGenerator, - { - "temperature": 0.9, - "max_output_tokens": 8192, - "top_p": 1.0, - "ground_with_google_search": False, - }, - ), - ( - llm.Claude3TextGenerator, - { - "max_output_tokens": 128, - "top_k": 40, - "top_p": 0.95, - }, - ), - ], -) -def test_text_generator_retry_no_progress(session, model_class, options, bq_connection): - # Requests. - df0 = EqCmpAllDataFrame( - { - "prompt": [ - "What is BigQuery?", - "What is BQML?", - "What is BigQuery DataFrame?", - ] - }, - index=[0, 1, 2], - session=session, - ) - df1 = EqCmpAllDataFrame( - { - "ml_generate_text_status": ["error", "error"], - "prompt": [ - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[1, 2], - session=session, - ) - - mock_generate_text = mock.create_autospec( - Callable[[core.BqmlModel, bpd.DataFrame, dict], bpd.DataFrame] - ) - mock_bqml_model = mock.create_autospec(spec=core.BqmlModel) - type(mock_bqml_model).session = mock.PropertyMock(return_value=session) - generate_text_tvf = core.BqmlModel.TvfDef( - mock_generate_text, "ml_generate_text_status" - ) - # Responses. Retry once, no progress, just stop. - mock_generate_text.side_effect = [ - EqCmpAllDataFrame( - { - "ml_generate_text_status": ["", "error", "error"], - "prompt": [ - "What is BigQuery?", - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[0, 1, 2], - session=session, - ), - EqCmpAllDataFrame( - { - "ml_generate_text_status": ["error", "error"], - "prompt": [ - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[1, 2], - session=session, - ), - ] - - text_generator_model = model_class(connection_name=bq_connection, session=session) - text_generator_model._bqml_model = mock_bqml_model - - with mock.patch.object(core.BqmlModel, "generate_text_tvf", generate_text_tvf): - # No progress, only conduct retry once - result = text_generator_model.predict(df0, max_retries=3) - - mock_generate_text.assert_has_calls( - [ - mock.call(mock_bqml_model, df0, options), - mock.call(mock_bqml_model, df1, options), - ] - ) - pd.testing.assert_frame_equal( - result.to_pandas(), - pd.DataFrame( - { - "ml_generate_text_status": ["", "error", "error"], - "prompt": [ - "What is BigQuery?", - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[0, 1, 2], - ), - check_dtype=False, - check_index_type=False, - ) - - -@pytest.mark.skip("b/436340035 test failed") -def test_text_embedding_generator_retry_success(session, bq_connection): - # Requests. - df0 = EqCmpAllDataFrame( - { - "content": [ - "What is BigQuery?", - "What is BQML?", - "What is BigQuery DataFrame?", - ] - }, - index=[0, 1, 2], - session=session, - ) - df1 = EqCmpAllDataFrame( - { - "ml_generate_embedding_status": ["error", "error"], - "content": [ - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[1, 2], - session=session, - ) - df2 = EqCmpAllDataFrame( - { - "ml_generate_embedding_status": ["error"], - "content": [ - "What is BQML?", - ], - }, - index=[1], - session=session, - ) - - mock_generate_embedding = mock.create_autospec( - Callable[[core.BqmlModel, bpd.DataFrame, dict], bpd.DataFrame] - ) - mock_bqml_model = mock.create_autospec(spec=core.BqmlModel) - type(mock_bqml_model).session = mock.PropertyMock(return_value=session) - generate_embedding_tvf = core.BqmlModel.TvfDef( - mock_generate_embedding, "ml_generate_embedding_status" - ) - - # Responses. Retry twice then all succeeded. - mock_generate_embedding.side_effect = [ - EqCmpAllDataFrame( - { - "ml_generate_embedding_status": ["", "error", "error"], - "content": [ - "What is BigQuery?", - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[0, 1, 2], - session=session, - ), - EqCmpAllDataFrame( - { - "ml_generate_embedding_status": ["error", ""], - "content": [ - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[1, 2], - session=session, - ), - EqCmpAllDataFrame( - { - "ml_generate_embedding_status": [""], - "content": [ - "What is BQML?", - ], - }, - index=[1], - session=session, - ), - ] - options: dict = {} - - text_embedding_model = llm.TextEmbeddingGenerator( - connection_name=bq_connection, session=session - ) - text_embedding_model._bqml_model = mock_bqml_model - - with mock.patch.object( - core.BqmlModel, "generate_embedding_tvf", generate_embedding_tvf - ): - # 3rd retry isn't triggered - result = text_embedding_model.predict(df0, max_retries=3) - - mock_generate_embedding.assert_has_calls( - [ - mock.call(mock_bqml_model, df0, options), - mock.call(mock_bqml_model, df1, options), - mock.call(mock_bqml_model, df2, options), - ] - ) - pd.testing.assert_frame_equal( - result.to_pandas(), - pd.DataFrame( - { - "ml_generate_embedding_status": ["", "", ""], - "content": [ - "What is BigQuery?", - "What is BigQuery DataFrame?", - "What is BQML?", - ], - }, - index=[0, 2, 1], - ), - check_dtype=False, - check_index_type=False, - ) - - -def test_text_embedding_generator_retry_no_progress(session, bq_connection): - # Requests. - df0 = EqCmpAllDataFrame( - { - "content": [ - "What is BigQuery?", - "What is BQML?", - "What is BigQuery DataFrame?", - ] - }, - index=[0, 1, 2], - session=session, - ) - df1 = EqCmpAllDataFrame( - { - "ml_generate_embedding_status": ["error", "error"], - "content": [ - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[1, 2], - session=session, - ) - - mock_generate_embedding = mock.create_autospec( - Callable[[core.BqmlModel, bpd.DataFrame, dict], bpd.DataFrame] - ) - mock_bqml_model = mock.create_autospec(spec=core.BqmlModel) - type(mock_bqml_model).session = mock.PropertyMock(return_value=session) - generate_embedding_tvf = core.BqmlModel.TvfDef( - mock_generate_embedding, "ml_generate_embedding_status" - ) - - # Responses. Retry once, no progress, just stop. - mock_generate_embedding.side_effect = [ - EqCmpAllDataFrame( - { - "ml_generate_embedding_status": ["", "error", "error"], - "content": [ - "What is BigQuery?", - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[0, 1, 2], - session=session, - ), - EqCmpAllDataFrame( - { - "ml_generate_embedding_status": ["error", "error"], - "content": [ - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[1, 2], - session=session, - ), - ] - options: dict = {} - - text_embedding_model = llm.TextEmbeddingGenerator( - connection_name=bq_connection, session=session - ) - text_embedding_model._bqml_model = mock_bqml_model - - with mock.patch.object( - core.BqmlModel, "generate_embedding_tvf", generate_embedding_tvf - ): - # No progress, only conduct retry once - result = text_embedding_model.predict(df0, max_retries=3) - - mock_generate_embedding.assert_has_calls( - [ - mock.call(mock_bqml_model, df0, options), - mock.call(mock_bqml_model, df1, options), - ] - ) - pd.testing.assert_frame_equal( - result.to_pandas(), - pd.DataFrame( - { - "ml_generate_embedding_status": ["", "error", "error"], - "content": [ - "What is BigQuery?", - "What is BQML?", - "What is BigQuery DataFrame?", - ], - }, - index=[0, 1, 2], - ), - check_dtype=False, - check_index_type=False, - ) - - -# b/436340035 temp disable the test to unblock presumbit -@pytest.mark.parametrize( - "model_class", - [ - llm.TextEmbeddingGenerator, - llm.MultimodalEmbeddingGenerator, - llm.GeminiTextGenerator, - # llm.Claude3TextGenerator, - ], -) -def test_text_embedding_generator_no_default_model_warning(model_class): - message = "Since upgrading the default model can cause unintended breakages, the\ndefault model will be removed in BigFrames 3.0. Please supply an\nexplicit model to avoid this message." - with pytest.warns(FutureWarning, match=message): - model_class(model_name=None) diff --git a/tests/system/large/ml/test_multimodal_llm.py b/tests/system/large/ml/test_multimodal_llm.py new file mode 100644 index 00000000000..03fdddf6654 --- /dev/null +++ b/tests/system/large/ml/test_multimodal_llm.py @@ -0,0 +1,45 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from bigframes.ml import llm +import bigframes.pandas as bpd +from bigframes.testing import utils + + +@pytest.mark.parametrize( + "model_name", + ( + "gemini-2.0-flash-exp", + "gemini-2.0-flash-001", + "gemini-2.0-flash-lite-001", + ), +) +@pytest.mark.flaky(retries=2) +def test_gemini_text_generator_multimodal_input( + images_mm_df: bpd.DataFrame, model_name, session, bq_connection +): + gemini_text_generator_model = llm.GeminiTextGenerator( + model_name=model_name, connection_name=bq_connection, session=session + ) + pd_df = gemini_text_generator_model.predict( + images_mm_df, prompt=["Describe", images_mm_df["blob_col"]] + ).to_pandas() + utils.check_pandas_df_schema_and_index( + pd_df, + columns=utils.ML_GENERATE_TEXT_OUTPUT + ["blob_col"], + index=2, + col_exact=False, + ) diff --git a/scripts/bigquery_generator/__init__.py b/tests/system/large/operations/__init__.py similarity index 95% rename from scripts/bigquery_generator/__init__.py rename to tests/system/large/operations/__init__.py index 58d482ea386..6d5e14bcf4a 100644 --- a/scripts/bigquery_generator/__init__.py +++ b/tests/system/large/operations/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/system/large/bigquery/test_table.py b/tests/system/large/operations/conftest.py similarity index 51% rename from tests/system/large/bigquery/test_table.py rename to tests/system/large/operations/conftest.py index dd956b3a040..6f64c7552f3 100644 --- a/tests/system/large/bigquery/test_table.py +++ b/tests/system/large/operations/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,25 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import bigframes.bigquery as bbq +import pytest +import bigframes.ml.llm as llm -def test_create_external_table(session, dataset_id, bq_connection): - table_name = f"{dataset_id}.test_object_table" - uri = "gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/images/*" - # Create the external table - table = bbq.create_external_table( - table_name, - connection_name=bq_connection, - options={"object_metadata": "SIMPLE", "uris": [uri]}, +@pytest.fixture(scope="session") +def gemini_flash_model(session, bq_connection) -> llm.GeminiTextGenerator: + return llm.GeminiTextGenerator( session=session, + connection_name=bq_connection, + model_name="gemini-2.0-flash-001", ) - assert table is not None - # Read the table to verify - import bigframes.pandas as bpd - bf_df = bpd.read_gbq(table_name) - pd_df = bf_df.to_pandas() - assert len(pd_df) > 0 +@pytest.fixture(scope="session") +def text_embedding_generator(session, bq_connection) -> llm.TextEmbeddingGenerator: + return llm.TextEmbeddingGenerator( + session=session, connection_name=bq_connection, model_name="text-embedding-005" + ) diff --git a/tests/system/large/operations/test_ai.py b/tests/system/large/operations/test_ai.py new file mode 100644 index 00000000000..86b30d9c657 --- /dev/null +++ b/tests/system/large/operations/test_ai.py @@ -0,0 +1,918 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contextlib import nullcontext +from unittest.mock import patch + +import pandas as pd +import pandas.testing +import pytest + +import bigframes +from bigframes import dataframe, exceptions, series + +AI_OP_EXP_OPTION = "experiments.ai_operators" +BLOB_EXP_OPTION = "experiments.blob" +THRESHOLD_OPTION = "compute.ai_ops_confirmation_threshold" + + +def test_filter(session, gemini_flash_model): + df = dataframe.DataFrame( + data={ + "country": ["USA", "Germany"], + "city": ["Seattle", "Berlin"], + "year": [2023, 2024], + }, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_df = df.ai.filter( + "{city} is the capital of {country} in {year}", gemini_flash_model + ).to_pandas() + + expected_df = pd.DataFrame( + {"country": ["Germany"], "city": ["Berlin"], "year": [2024]}, index=[1] + ) + pandas.testing.assert_frame_equal( + actual_df, expected_df, check_dtype=False, check_index_type=False + ) + + +def test_filter_multi_model(session, gemini_flash_model): + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + BLOB_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + df = session.from_glob_path( + "gs://bigframes-dev-testing/a_multimodel/images/*", name="image" + ) + df["prey"] = series.Series( + ["building", "cross road", "rock", "squirrel", "rabbit"], session=session + ) + result = df.ai.filter( + "The object in {image} feeds on {prey}", + gemini_flash_model, + ).to_pandas() + + assert len(result) <= len(df) + + +@pytest.mark.parametrize( + ("reply"), + [ + pytest.param("y"), + pytest.param( + "n", marks=pytest.mark.xfail(raises=exceptions.OperationAbortedError) + ), + ], +) +def test_filter_with_confirmation(session, gemini_flash_model, reply, monkeypatch): + df = dataframe.DataFrame( + data={ + "country": ["USA", "Germany"], + "city": ["Seattle", "Berlin"], + "year": [2023, 2024], + }, + session=session, + ) + monkeypatch.setattr("builtins.input", lambda: reply) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 0, + ): + df.ai.filter("{city} is the capital of {country} in {year}", gemini_flash_model) + + +def test_filter_single_column_reference(session, gemini_flash_model): + df = dataframe.DataFrame( + data={"country": ["USA", "Germany"], "city": ["Seattle", "Berlin"]}, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_df = df.ai.filter( + "{country} is in Europe", gemini_flash_model + ).to_pandas() + + expected_df = pd.DataFrame({"country": ["Germany"], "city": ["Berlin"]}, index=[1]) + pandas.testing.assert_frame_equal( + actual_df, expected_df, check_dtype=False, check_index_type=False + ) + + +@pytest.mark.parametrize( + "instruction", + [ + pytest.param( + "No column reference", + id="zero_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "{city} is in the {non_existing_column}", + id="non_existing_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "{id}", + id="invalid_type", + marks=pytest.mark.xfail(raises=TypeError), + ), + ], +) +def test_filter_invalid_instruction_raise_error(instruction, gemini_flash_model): + df = dataframe.DataFrame({"id": [1, 2], "city": ["Seattle", "Berlin"]}) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df.ai.filter(instruction, gemini_flash_model) + + +def test_filter_invalid_model_raise_error(): + df = dataframe.DataFrame( + {"country": ["USA", "Germany"], "city": ["Seattle", "Berlin"]} + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(TypeError): + df.ai.filter("{city} is the capital of {country}", None) + + +@pytest.mark.parametrize( + ("output_schema", "output_col"), + [ + pytest.param(None, "ml_generate_text_llm_result", id="default_schema"), + pytest.param({"food": "string"}, "food", id="non_default_schema"), + ], +) +def test_map(session, gemini_flash_model, output_schema, output_col): + df = dataframe.DataFrame( + data={ + "ingredient_1": ["Burger Bun", "Soy Bean"], + "ingredient_2": ["Beef Patty", "Bittern"], + "gluten-free": [True, True], + }, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_df = df.ai.map( + "What is the {gluten-free} food made from {ingredient_1} and {ingredient_2}? One word only.", + gemini_flash_model, + output_schema=output_schema, + ).to_pandas() + # Result sanitation + actual_df[output_col] = actual_df[output_col].str.strip().str.lower() + + expected_df = pd.DataFrame( + { + "ingredient_1": ["Burger Bun", "Soy Bean"], + "ingredient_2": ["Beef Patty", "Bittern"], + "gluten-free": [True, True], + output_col: ["burger", "tofu"], + } + ) + pandas.testing.assert_frame_equal( + actual_df, + expected_df, + check_dtype=False, + check_index_type=False, + check_column_type=False, + ) + + +def test_map_multimodel(session, gemini_flash_model): + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + BLOB_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + df = session.from_glob_path( + "gs://bigframes-dev-testing/a_multimodel/images/*", name="image" + ) + df["scenario"] = series.Series( + ["building", "cross road", "tree", "squirrel", "rabbit"], session=session + ) + result = df.ai.map( + "What is the object in {image} combined with {scenario}? One word only.", + gemini_flash_model, + output_schema={"object": "string"}, + ).to_pandas() + + assert len(result) == len(df) + + +@pytest.mark.parametrize( + ("reply"), + [ + pytest.param("y"), + pytest.param( + "n", marks=pytest.mark.xfail(raises=exceptions.OperationAbortedError) + ), + ], +) +def test_map_with_confirmation(session, gemini_flash_model, reply, monkeypatch): + df = dataframe.DataFrame( + data={ + "ingredient_1": ["Burger Bun", "Soy Bean"], + "ingredient_2": ["Beef Patty", "Bittern"], + "gluten-free": [True, True], + }, + session=session, + ) + monkeypatch.setattr("builtins.input", lambda: reply) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 0, + ): + df.ai.map( + "What is the {gluten-free} food made from {ingredient_1} and {ingredient_2}? One word only.", + gemini_flash_model, + ) + + +@pytest.mark.parametrize( + "instruction", + [ + pytest.param( + "No column reference", + id="zero_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "What is the food made from {ingredient_1} and {non_existing_column}?}", + id="non_existing_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "{id}", + id="invalid_type", + marks=pytest.mark.xfail(raises=TypeError), + ), + ], +) +def test_map_invalid_instruction_raise_error(instruction, gemini_flash_model): + df = dataframe.DataFrame( + data={ + "id": [1, 2], + "ingredient_1": ["Burger Bun", "Soy Bean"], + "ingredient_2": ["Beef Patty", "Bittern"], + } + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df.ai.map(instruction, gemini_flash_model, output_schema={"food": "string"}) + + +def test_map_invalid_model_raise_error(): + df = dataframe.DataFrame( + data={ + "ingredient_1": ["Burger Bun", "Soy Bean"], + "ingredient_2": ["Beef Patty", "Bittern"], + }, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(TypeError): + df.ai.map( + "What is the food made from {ingredient_1} and {ingredient_2}? One word only.", + None, + ) + + +def test_classify(gemini_flash_model, session): + df = dataframe.DataFrame(data={"creature": ["dog", "rose"]}, session=session) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_result = df.ai.classify( + "{creature}", + gemini_flash_model, + labels=["animal", "plant"], + output_column="result", + ).to_pandas() + + expected_result = pd.DataFrame( + { + "creature": ["dog", "rose"], + "result": ["animal", "plant"], + } + ) + pandas.testing.assert_frame_equal( + actual_result, expected_result, check_index_type=False, check_dtype=False + ) + + +@pytest.mark.parametrize( + "instruction", + [ + pytest.param("{city} is in {country}", id="no_dataframe_reference"), + pytest.param("{left.city} is in {country}", id="has_left_dataframe_reference"), + pytest.param( + "{city} is in {right.country}", + id="has_right_dataframe_reference", + ), + pytest.param( + "{left.city} is in {right.country}", id="has_both_dataframe_references" + ), + ], +) +def test_join(instruction, session, gemini_flash_model): + cities = dataframe.DataFrame( + data={ + "city": ["Seattle", "Berlin"], + }, + session=session, + ) + countries = dataframe.DataFrame( + data={"country": ["USA", "UK", "Germany"]}, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_df = cities.ai.join( + countries, + instruction, + gemini_flash_model, + ).to_pandas() + + expected_df = pd.DataFrame( + { + "city": ["Seattle", "Berlin"], + "country": ["USA", "Germany"], + } + ) + pandas.testing.assert_frame_equal( + actual_df, + expected_df, + check_dtype=False, + check_index_type=False, + check_column_type=False, + ) + + +@pytest.mark.parametrize( + ("reply"), + [ + pytest.param("y"), + pytest.param( + "n", marks=pytest.mark.xfail(raises=exceptions.OperationAbortedError) + ), + ], +) +def test_join_with_confirmation(session, gemini_flash_model, reply, monkeypatch): + cities = dataframe.DataFrame( + data={ + "city": ["Seattle", "Berlin"], + }, + session=session, + ) + countries = dataframe.DataFrame( + data={"country": ["USA", "UK", "Germany"]}, + session=session, + ) + monkeypatch.setattr("builtins.input", lambda: reply) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 0, + ): + cities.ai.join( + countries, + "{city} is in {country}", + gemini_flash_model, + ) + + +def test_self_join(session, gemini_flash_model): + animals = dataframe.DataFrame( + data={ + "animal": ["ant", "elephant"], + }, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_df = animals.ai.join( + animals, + "{left.animal} is heavier than {right.animal}", + gemini_flash_model, + ).to_pandas() + + expected_df = pd.DataFrame( + { + "animal_left": ["elephant"], + "animal_right": ["ant"], + } + ) + pandas.testing.assert_frame_equal( + actual_df, + expected_df, + check_dtype=False, + check_index_type=False, + check_column_type=False, + ) + + +@pytest.mark.parametrize( + ("instruction", "error_pattern"), + [ + ("No column reference", "No column references"), + pytest.param( + "{city} is in {continent}", r"Column .+ not found", id="non_existing_column" + ), + pytest.param( + "{city} is in {country}", + r"Ambiguous column reference: .+", + id="ambiguous_column", + ), + pytest.param( + "{right.city} is in {country}", r"Column .+ not found", id="wrong_prefix" + ), + pytest.param( + "{city} is in {right.continent}", + r"Column .+ not found", + id="prefix_on_non_existing_column", + ), + ], +) +def test_join_invalid_instruction_raise_error( + instruction, error_pattern, gemini_flash_model +): + df1 = dataframe.DataFrame( + {"city": ["Seattle", "Berlin"], "country": ["USA", "Germany"]} + ) + df2 = dataframe.DataFrame( + { + "country": ["USA", "UK", "Germany"], + "region": ["North America", "Europe", "Europe"], + } + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError, match=error_pattern): + df1.ai.join(df2, instruction, gemini_flash_model) + + +def test_join_invalid_model_raise_error(): + cities = dataframe.DataFrame({"city": ["Seattle", "Berlin"]}) + countries = dataframe.DataFrame({"country": ["USA", "UK", "Germany"]}) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(TypeError): + cities.ai.join(countries, "{city} is in {country}", None) + + +@pytest.mark.parametrize( + "score_column", + [ + pytest.param(None, id="no_score_column"), + pytest.param("distance", id="has_score_column"), + ], +) +def test_search(session, text_embedding_generator, score_column): + df = dataframe.DataFrame( + data={"creatures": ["salmon", "sea urchin", "baboons", "frog", "chimpanzee"]}, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_result = df.ai.search( + "creatures", + "monkey", + top_k=2, + model=text_embedding_generator, + score_column=score_column, + ).to_pandas() + + expected_result = pd.Series( + ["baboons", "chimpanzee"], index=[2, 4], name="creatures" + ) + pandas.testing.assert_series_equal( + actual_result["creatures"], + expected_result, + check_dtype=False, + check_index_type=False, + ) + + if score_column is None: + assert len(actual_result.columns) == 1 + else: + assert score_column in actual_result.columns + + +@pytest.mark.parametrize( + ("reply"), + [ + pytest.param("y"), + pytest.param( + "n", marks=pytest.mark.xfail(raises=exceptions.OperationAbortedError) + ), + ], +) +def test_search_with_confirmation( + session, text_embedding_generator, reply, monkeypatch +): + df = dataframe.DataFrame( + data={"creatures": ["salmon", "sea urchin", "baboons", "frog", "chimpanzee"]}, + session=session, + ) + monkeypatch.setattr("builtins.input", lambda: reply) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 0, + ): + df.ai.search( + "creatures", + "monkey", + top_k=2, + model=text_embedding_generator, + ) + + +def test_search_invalid_column_raises_error(session, text_embedding_generator): + df = dataframe.DataFrame( + data={"creatures": ["salmon", "sea urchin", "baboons", "frog", "chimpanzee"]}, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df.ai.search("whatever", "monkey", top_k=2, model=text_embedding_generator) + + +def test_search_invalid_model_raises_error(session): + df = dataframe.DataFrame( + data={"creatures": ["salmon", "sea urchin", "baboons", "frog", "chimpanzee"]}, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(TypeError): + df.ai.search("creatures", "monkey", top_k=2, model=None) + + +def test_search_invalid_top_k_raises_error(session, text_embedding_generator): + df = dataframe.DataFrame( + data={"creatures": ["salmon", "sea urchin", "baboons", "frog", "chimpanzee"]}, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df.ai.search("creatures", "monkey", top_k=0, model=text_embedding_generator) + + +@pytest.mark.parametrize( + "score_column", + [ + pytest.param(None, id="no_score_column"), + pytest.param("distance", id="has_score_column"), + ], +) +def test_sim_join(session, text_embedding_generator, score_column): + df1 = dataframe.DataFrame( + data={"creatures": ["salmon", "cat"]}, + session=session, + ) + df2 = dataframe.DataFrame( + data={"creatures": ["dog", "tuna"]}, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_result = df1.ai.sim_join( + df2, + left_on="creatures", + right_on="creatures", + model=text_embedding_generator, + top_k=1, + score_column=score_column, + ).to_pandas() + + expected_result = pd.DataFrame( + {"creatures": ["salmon", "cat"], "creatures_1": ["tuna", "dog"]} + ) + pandas.testing.assert_frame_equal( + actual_result[["creatures", "creatures_1"]], + expected_result, + check_dtype=False, + check_index_type=False, + ) + + if score_column is None: + assert len(actual_result.columns) == 2 + else: + assert score_column in actual_result.columns + + +@pytest.mark.parametrize( + ("reply"), + [ + pytest.param("y"), + pytest.param( + "n", marks=pytest.mark.xfail(raises=exceptions.OperationAbortedError) + ), + ], +) +def test_sim_join_with_confirmation( + session, text_embedding_generator, reply, monkeypatch +): + df1 = dataframe.DataFrame( + data={"creatures": ["salmon", "cat"]}, + session=session, + ) + df2 = dataframe.DataFrame( + data={"creatures": ["dog", "tuna"]}, + session=session, + ) + monkeypatch.setattr("builtins.input", lambda: reply) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 0, + ): + df1.ai.sim_join( + df2, + left_on="creatures", + right_on="creatures", + model=text_embedding_generator, + top_k=1, + ) + + +@pytest.mark.parametrize( + ("left_on", "right_on"), + [ + pytest.param("whatever", "creatures", id="incorrect_left_column"), + pytest.param("creatures", "whatever", id="incorrect_right_column"), + ], +) +def test_sim_join_invalid_column_raises_error( + session, text_embedding_generator, left_on, right_on +): + df1 = dataframe.DataFrame( + data={"creatures": ["salmon", "cat"]}, + session=session, + ) + df2 = dataframe.DataFrame( + data={"creatures": ["dog", "tuna"]}, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df1.ai.sim_join( + df2, left_on=left_on, right_on=right_on, model=text_embedding_generator + ) + + +def test_sim_join_invalid_model_raises_error(session): + df1 = dataframe.DataFrame( + data={"creatures": ["salmon", "cat"]}, + session=session, + ) + df2 = dataframe.DataFrame( + data={"creatures": ["dog", "tuna"]}, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(TypeError): + df1.ai.sim_join(df2, left_on="creatures", right_on="creatures", model=None) + + +def test_sim_join_invalid_top_k_raises_error(session, text_embedding_generator): + df1 = dataframe.DataFrame( + data={"creatures": ["salmon", "cat"]}, + session=session, + ) + df2 = dataframe.DataFrame( + data={"creatures": ["dog", "tuna"]}, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df1.ai.sim_join( + df2, + left_on="creatures", + right_on="creatures", + top_k=0, + model=text_embedding_generator, + ) + + +def test_sim_join_data_too_large_raises_error(session, text_embedding_generator): + df1 = dataframe.DataFrame( + data={"creatures": ["salmon", "cat"]}, + session=session, + ) + df2 = dataframe.DataFrame( + data={"creatures": ["dog", "tuna"]}, + session=session, + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df1.ai.sim_join( + df2, + left_on="creatures", + right_on="creatures", + model=text_embedding_generator, + max_rows=1, + ) + + +@patch("builtins.input", return_value="") +def test_confirm_operation__below_threshold_do_not_confirm(mock_input): + df = dataframe.DataFrame({}) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 3, + ): + df.ai._confirm_operation(1) + + mock_input.assert_not_called() + + +@patch("builtins.input", return_value="") +def test_confirm_operation__threshold_is_none_do_not_confirm(mock_input): + df = dataframe.DataFrame({}) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + None, + ): + df.ai._confirm_operation(100) + + mock_input.assert_not_called() + + +@patch("builtins.input", return_value="") +def test_confirm_operation__threshold_autofail_do_not_confirm(mock_input): + df = dataframe.DataFrame({}) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 1, + "compute.ai_ops_threshold_autofail", + True, + ), pytest.raises(exceptions.OperationAbortedError): + df.ai._confirm_operation(100) + + mock_input.assert_not_called() + + +@pytest.mark.parametrize( + ("reply", "expectation"), + [ + ("y", nullcontext()), + ("yes", nullcontext()), + ("", nullcontext()), + ("n", pytest.raises(exceptions.OperationAbortedError)), + ("something", pytest.raises(exceptions.OperationAbortedError)), + ], +) +def test_confirm_operation__above_threshold_confirm(reply, expectation, monkeypatch): + monkeypatch.setattr("builtins.input", lambda: reply) + df = dataframe.DataFrame({}) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 3, + ), expectation as e: + assert df.ai._confirm_operation(4) == e diff --git a/tests/system/large/operations/test_semantics.py b/tests/system/large/operations/test_semantics.py new file mode 100644 index 00000000000..7ae78a5c53a --- /dev/null +++ b/tests/system/large/operations/test_semantics.py @@ -0,0 +1,1293 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contextlib import nullcontext +from unittest.mock import patch + +import pandas as pd +import pandas.testing +import pytest + +import bigframes +from bigframes import dataframe, dtypes, exceptions, series + +pytest.skip( + "Semantics namespace is deprecated. ", + allow_module_level=True, +) + +SEM_OP_EXP_OPTION = "experiments.semantic_operators" +BLOB_EXP_OPTION = "experiments.blob" +THRESHOLD_OPTION = "compute.semantic_ops_confirmation_threshold" + + +def test_semantics_experiment_off_raise_error(): + df = dataframe.DataFrame( + {"country": ["USA", "Germany"], "city": ["Seattle", "Berlin"]} + ) + + with bigframes.option_context(SEM_OP_EXP_OPTION, False), pytest.raises( + NotImplementedError + ): + df.semantics + + +@pytest.mark.parametrize( + ("max_agg_rows", "cluster_column"), + [ + pytest.param(1, None, id="one", marks=pytest.mark.xfail(raises=ValueError)), + pytest.param(2, None, id="two"), + pytest.param(3, None, id="three"), + pytest.param(4, None, id="four"), + pytest.param(5, "Years", id="two_w_cluster_column"), + pytest.param(6, "Years", id="three_w_cluster_column"), + pytest.param(7, "Years", id="four_w_cluster_column"), + ], +) +def test_agg(session, gemini_flash_model, max_agg_rows, cluster_column): + df = dataframe.DataFrame( + data={ + "Movies": [ + "Titanic", + "The Wolf of Wall Street", + "Killers of the Flower Moon", + "The Revenant", + "Inception", + "Shuttle Island", + "The Great Gatsby", + ], + "Years": [1997, 2013, 2023, 2015, 2010, 2010, 2013], + }, + session=session, + ) + instruction = "Find the shared first name of actors in {Movies}. One word answer." + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 50, + ): + actual_s = df.semantics.agg( + instruction, + model=gemini_flash_model, + max_agg_rows=max_agg_rows, + cluster_column=cluster_column, + ).to_pandas() + + expected_s = pd.Series(["Leonardo\n"], dtype=dtypes.STRING_DTYPE) + expected_s.name = "Movies" + pandas.testing.assert_series_equal(actual_s, expected_s, check_index_type=False) + + +@pytest.mark.parametrize( + ("reply"), + [ + pytest.param("y"), + pytest.param( + "n", marks=pytest.mark.xfail(raises=exceptions.OperationAbortedError) + ), + ], +) +def test_agg_with_confirmation(session, gemini_flash_model, reply, monkeypatch): + df = dataframe.DataFrame( + data={ + "Movies": [ + "Titanic", + "The Wolf of Wall Street", + "Killers of the Flower Moon", + "The Revenant", + "Inception", + "Shuttle Island", + "The Great Gatsby", + ], + "Years": [1997, 2013, 2023, 2015, 2010, 2010, 2013], + }, + session=session, + ) + instruction = "Find the shared first name of actors in {Movies}. One word answer." + monkeypatch.setattr("builtins.input", lambda: reply) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 0, + ): + df.semantics.agg( + instruction, + model=gemini_flash_model, + ) + + +def test_agg_w_int_column(session, gemini_flash_model): + df = dataframe.DataFrame( + data={ + "Movies": [ + "Killers of the Flower Moon", + "The Great Gatsby", + "The Wolf of Wall Street", + ], + "Years": [2023, 2013, 2013], + }, + session=session, + ) + instruction = "Find the {Years} Leonardo DiCaprio acted in the most movies. Your answer should be the four-digit year, returned as a string." + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_s = df.semantics.agg( + instruction, + model=gemini_flash_model, + ).to_pandas() + + expected_s = pd.Series(["2013\n"], dtype=dtypes.STRING_DTYPE) + expected_s.name = "Years" + pandas.testing.assert_series_equal(actual_s, expected_s, check_index_type=False) + + +@pytest.mark.parametrize( + "instruction", + [ + pytest.param( + "No column reference", + id="zero_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "{Movies} is good", + id="non_existing_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "{Movies} is better than {Movies}", + id="two_columns", + marks=pytest.mark.xfail(raises=NotImplementedError), + ), + ], +) +def test_agg_invalid_instruction_raise_error(instruction, gemini_flash_model): + df = dataframe.DataFrame( + data={ + "Movies": [ + "Titanic", + "The Wolf of Wall Street", + "Killers of the Flower Moon", + ], + "Year": [1997, 2013, 2023], + }, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + df.semantics.agg(instruction, gemini_flash_model) + + +@pytest.mark.parametrize( + "cluster_column", + [ + pytest.param( + "non_existing_column", + id="non_existing_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "Movies", id="non_int_column", marks=pytest.mark.xfail(raises=TypeError) + ), + ], +) +def test_agg_invalid_cluster_column_raise_error(gemini_flash_model, cluster_column): + df = dataframe.DataFrame( + data={ + "Movies": [ + "Titanic", + "The Wolf of Wall Street", + "Killers of the Flower Moon", + "The Revenant", + ], + }, + ) + instruction = "Find the shared first name of actors in {Movies}. One word answer." + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + df.semantics.agg(instruction, gemini_flash_model, cluster_column=cluster_column) + + +@pytest.mark.parametrize( + ("n_clusters"), + [ + pytest.param(1, id="one", marks=pytest.mark.xfail(raises=ValueError)), + pytest.param(2, id="two"), + ], +) +def test_cluster_by(session, text_embedding_generator, n_clusters): + df = dataframe.DataFrame( + ( + { + "Item": [ + "Orange", + "Cantaloupe", + "Watermelon", + "Chicken", + "Duck", + "Hen", + "Rooster", + ] + } + ), + session=session, + ) + output_column = "cluster id" + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + result = df.semantics.cluster_by( + "Item", + output_column, + text_embedding_generator, + n_clusters=n_clusters, + ) + + assert output_column in result + # In rare cases, it's possible to have fewer than K clusters due to randomness. + assert len(result[output_column].unique()) <= n_clusters + + +@pytest.mark.parametrize( + ("reply"), + [ + pytest.param("y"), + pytest.param( + "n", marks=pytest.mark.xfail(raises=exceptions.OperationAbortedError) + ), + ], +) +def test_cluster_by_with_confirmation( + session, text_embedding_generator, reply, monkeypatch +): + df = dataframe.DataFrame( + ( + { + "Item": [ + "Orange", + "Cantaloupe", + "Watermelon", + "Chicken", + "Duck", + "Hen", + "Rooster", + ] + } + ), + session=session, + ) + monkeypatch.setattr("builtins.input", lambda: reply) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 0, + ): + df.semantics.cluster_by( + "Item", + "cluster id", + text_embedding_generator, + n_clusters=2, + ) + + +def test_cluster_by_invalid_column(session, text_embedding_generator): + df = dataframe.DataFrame( + ({"Product": ["Smartphone", "Laptop", "Coffee Maker", "T-shirt", "Jeans"]}), + session=session, + ) + output_column = "cluster id" + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df.semantics.cluster_by( + "unknown_column", + output_column, + text_embedding_generator, + n_clusters=3, + ) + + +def test_cluster_by_invalid_model(session, gemini_flash_model): + df = dataframe.DataFrame( + ({"Product": ["Smartphone", "Laptop", "Coffee Maker", "T-shirt", "Jeans"]}), + session=session, + ) + output_column = "cluster id" + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(TypeError): + df.semantics.cluster_by( + "Product", + output_column, + gemini_flash_model, + n_clusters=3, + ) + + +def test_filter(session, gemini_flash_model): + df = dataframe.DataFrame( + data={ + "country": ["USA", "Germany"], + "city": ["Seattle", "Berlin"], + "year": [2023, 2024], + }, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_df = df.semantics.filter( + "{city} is the capital of {country} in {year}", gemini_flash_model + ).to_pandas() + + expected_df = pd.DataFrame( + {"country": ["Germany"], "city": ["Berlin"], "year": [2024]}, index=[1] + ) + pandas.testing.assert_frame_equal( + actual_df, expected_df, check_dtype=False, check_index_type=False + ) + + +def test_filter_multi_model(session, gemini_flash_model): + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + BLOB_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + df = session.from_glob_path( + "gs://bigframes-dev-testing/a_multimodel/images/*", name="image" + ) + df["prey"] = series.Series( + ["building", "cross road", "rock", "squirrel", "rabbit"], session=session + ) + result = df.semantics.filter( + "The object in {image} feeds on {prey}", + gemini_flash_model, + ).to_pandas() + + assert len(result) <= len(df) + + +@pytest.mark.parametrize( + ("reply"), + [ + pytest.param("y"), + pytest.param( + "n", marks=pytest.mark.xfail(raises=exceptions.OperationAbortedError) + ), + ], +) +def test_filter_with_confirmation(session, gemini_flash_model, reply, monkeypatch): + df = dataframe.DataFrame( + data={ + "country": ["USA", "Germany"], + "city": ["Seattle", "Berlin"], + "year": [2023, 2024], + }, + session=session, + ) + monkeypatch.setattr("builtins.input", lambda: reply) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 0, + ): + df.semantics.filter( + "{city} is the capital of {country} in {year}", gemini_flash_model + ) + + +def test_filter_single_column_reference(session, gemini_flash_model): + df = dataframe.DataFrame( + data={"country": ["USA", "Germany"], "city": ["Seattle", "Berlin"]}, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_df = df.semantics.filter( + "{country} is in Europe", gemini_flash_model + ).to_pandas() + + expected_df = pd.DataFrame({"country": ["Germany"], "city": ["Berlin"]}, index=[1]) + pandas.testing.assert_frame_equal( + actual_df, expected_df, check_dtype=False, check_index_type=False + ) + + +@pytest.mark.parametrize( + "instruction", + [ + pytest.param( + "No column reference", + id="zero_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "{city} is in the {non_existing_column}", + id="non_existing_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "{id}", + id="invalid_type", + marks=pytest.mark.xfail(raises=TypeError), + ), + ], +) +def test_filter_invalid_instruction_raise_error(instruction, gemini_flash_model): + df = dataframe.DataFrame({"id": [1, 2], "city": ["Seattle", "Berlin"]}) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df.semantics.filter(instruction, gemini_flash_model) + + +def test_filter_invalid_model_raise_error(): + df = dataframe.DataFrame( + {"country": ["USA", "Germany"], "city": ["Seattle", "Berlin"]} + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(TypeError): + df.semantics.filter("{city} is the capital of {country}", None) + + +def test_map(session, gemini_flash_model): + df = dataframe.DataFrame( + data={ + "ingredient_1": ["Burger Bun", "Soy Bean"], + "ingredient_2": ["Beef Patty", "Bittern"], + "gluten-free": [True, True], + }, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_df = df.semantics.map( + "What is the {gluten-free} food made from {ingredient_1} and {ingredient_2}? One word only.", + "food", + gemini_flash_model, + ).to_pandas() + # Result sanitation + actual_df["food"] = actual_df["food"].str.strip().str.lower() + + expected_df = pd.DataFrame( + { + "ingredient_1": ["Burger Bun", "Soy Bean"], + "ingredient_2": ["Beef Patty", "Bittern"], + "gluten-free": [True, True], + "food": ["burger", "tofu"], + } + ) + pandas.testing.assert_frame_equal( + actual_df, + expected_df, + check_dtype=False, + check_index_type=False, + check_column_type=False, + ) + + +def test_map_multimodel(session, gemini_flash_model): + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + BLOB_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + df = session.from_glob_path( + "gs://bigframes-dev-testing/a_multimodel/images/*", name="image" + ) + df["scenario"] = series.Series( + ["building", "cross road", "tree", "squirrel", "rabbit"], session=session + ) + result = df.semantics.map( + "What is the object in {image} combined with {scenario}? One word only.", + "object", + gemini_flash_model, + ).to_pandas() + + assert len(result) == len(df) + + +@pytest.mark.parametrize( + ("reply"), + [ + pytest.param("y"), + pytest.param( + "n", marks=pytest.mark.xfail(raises=exceptions.OperationAbortedError) + ), + ], +) +def test_map_with_confirmation(session, gemini_flash_model, reply, monkeypatch): + df = dataframe.DataFrame( + data={ + "ingredient_1": ["Burger Bun", "Soy Bean"], + "ingredient_2": ["Beef Patty", "Bittern"], + "gluten-free": [True, True], + }, + session=session, + ) + monkeypatch.setattr("builtins.input", lambda: reply) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 0, + ): + df.semantics.map( + "What is the {gluten-free} food made from {ingredient_1} and {ingredient_2}? One word only.", + "food", + gemini_flash_model, + ) + + +@pytest.mark.parametrize( + "instruction", + [ + pytest.param( + "No column reference", + id="zero_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "What is the food made from {ingredient_1} and {non_existing_column}?}", + id="non_existing_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "{id}", + id="invalid_type", + marks=pytest.mark.xfail(raises=TypeError), + ), + ], +) +def test_map_invalid_instruction_raise_error(instruction, gemini_flash_model): + df = dataframe.DataFrame( + data={ + "id": [1, 2], + "ingredient_1": ["Burger Bun", "Soy Bean"], + "ingredient_2": ["Beef Patty", "Bittern"], + } + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df.semantics.map(instruction, "food", gemini_flash_model) + + +def test_map_invalid_model_raise_error(): + df = dataframe.DataFrame( + data={ + "ingredient_1": ["Burger Bun", "Soy Bean"], + "ingredient_2": ["Beef Patty", "Bittern"], + }, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(TypeError): + df.semantics.map( + "What is the food made from {ingredient_1} and {ingredient_2}? One word only.", + "food", + None, + ) + + +@pytest.mark.parametrize( + "instruction", + [ + pytest.param("{city} is in {country}", id="no_dataframe_reference"), + pytest.param("{left.city} is in {country}", id="has_left_dataframe_reference"), + pytest.param( + "{city} is in {right.country}", + id="has_right_dataframe_reference", + ), + pytest.param( + "{left.city} is in {right.country}", id="has_both_dataframe_references" + ), + ], +) +def test_join(instruction, session, gemini_flash_model): + cities = dataframe.DataFrame( + data={ + "city": ["Seattle", "Berlin"], + }, + session=session, + ) + countries = dataframe.DataFrame( + data={"country": ["USA", "UK", "Germany"]}, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_df = cities.semantics.join( + countries, + instruction, + gemini_flash_model, + ).to_pandas() + + expected_df = pd.DataFrame( + { + "city": ["Seattle", "Berlin"], + "country": ["USA", "Germany"], + } + ) + pandas.testing.assert_frame_equal( + actual_df, + expected_df, + check_dtype=False, + check_index_type=False, + check_column_type=False, + ) + + +@pytest.mark.parametrize( + ("reply"), + [ + pytest.param("y"), + pytest.param( + "n", marks=pytest.mark.xfail(raises=exceptions.OperationAbortedError) + ), + ], +) +def test_join_with_confirmation(session, gemini_flash_model, reply, monkeypatch): + cities = dataframe.DataFrame( + data={ + "city": ["Seattle", "Berlin"], + }, + session=session, + ) + countries = dataframe.DataFrame( + data={"country": ["USA", "UK", "Germany"]}, + session=session, + ) + monkeypatch.setattr("builtins.input", lambda: reply) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 0, + ): + cities.semantics.join( + countries, + "{city} is in {country}", + gemini_flash_model, + ) + + +def test_self_join(session, gemini_flash_model): + animals = dataframe.DataFrame( + data={ + "animal": ["ant", "elephant"], + }, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_df = animals.semantics.join( + animals, + "{left.animal} is heavier than {right.animal}", + gemini_flash_model, + ).to_pandas() + + expected_df = pd.DataFrame( + { + "animal_left": ["elephant"], + "animal_right": ["ant"], + } + ) + pandas.testing.assert_frame_equal( + actual_df, + expected_df, + check_dtype=False, + check_index_type=False, + check_column_type=False, + ) + + +@pytest.mark.parametrize( + ("instruction", "error_pattern"), + [ + ("No column reference", "No column references"), + pytest.param( + "{city} is in {continent}", r"Column .+ not found", id="non_existing_column" + ), + pytest.param( + "{city} is in {country}", + r"Ambiguous column reference: .+", + id="ambiguous_column", + ), + pytest.param( + "{right.city} is in {country}", r"Column .+ not found", id="wrong_prefix" + ), + pytest.param( + "{city} is in {right.continent}", + r"Column .+ not found", + id="prefix_on_non_existing_column", + ), + ], +) +def test_join_invalid_instruction_raise_error( + instruction, error_pattern, gemini_flash_model +): + df1 = dataframe.DataFrame( + {"city": ["Seattle", "Berlin"], "country": ["USA", "Germany"]} + ) + df2 = dataframe.DataFrame( + { + "country": ["USA", "UK", "Germany"], + "region": ["North America", "Europe", "Europe"], + } + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError, match=error_pattern): + df1.semantics.join(df2, instruction, gemini_flash_model) + + +def test_join_invalid_model_raise_error(): + cities = dataframe.DataFrame({"city": ["Seattle", "Berlin"]}) + countries = dataframe.DataFrame({"country": ["USA", "UK", "Germany"]}) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(TypeError): + cities.semantics.join(countries, "{city} is in {country}", None) + + +@pytest.mark.parametrize( + "score_column", + [ + pytest.param(None, id="no_score_column"), + pytest.param("distance", id="has_score_column"), + ], +) +def test_search(session, text_embedding_generator, score_column): + df = dataframe.DataFrame( + data={"creatures": ["salmon", "sea urchin", "baboons", "frog", "chimpanzee"]}, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_result = df.semantics.search( + "creatures", + "monkey", + top_k=2, + model=text_embedding_generator, + score_column=score_column, + ).to_pandas() + + expected_result = pd.Series( + ["baboons", "chimpanzee"], index=[2, 4], name="creatures" + ) + pandas.testing.assert_series_equal( + actual_result["creatures"], + expected_result, + check_dtype=False, + check_index_type=False, + ) + + if score_column is None: + assert len(actual_result.columns) == 1 + else: + assert score_column in actual_result.columns + + +@pytest.mark.parametrize( + ("reply"), + [ + pytest.param("y"), + pytest.param( + "n", marks=pytest.mark.xfail(raises=exceptions.OperationAbortedError) + ), + ], +) +def test_search_with_confirmation( + session, text_embedding_generator, reply, monkeypatch +): + df = dataframe.DataFrame( + data={"creatures": ["salmon", "sea urchin", "baboons", "frog", "chimpanzee"]}, + session=session, + ) + monkeypatch.setattr("builtins.input", lambda: reply) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 0, + ): + df.semantics.search( + "creatures", + "monkey", + top_k=2, + model=text_embedding_generator, + ) + + +def test_search_invalid_column_raises_error(session, text_embedding_generator): + df = dataframe.DataFrame( + data={"creatures": ["salmon", "sea urchin", "baboons", "frog", "chimpanzee"]}, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df.semantics.search( + "whatever", "monkey", top_k=2, model=text_embedding_generator + ) + + +def test_search_invalid_model_raises_error(session): + df = dataframe.DataFrame( + data={"creatures": ["salmon", "sea urchin", "baboons", "frog", "chimpanzee"]}, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(TypeError): + df.semantics.search("creatures", "monkey", top_k=2, model=None) + + +def test_search_invalid_top_k_raises_error(session, text_embedding_generator): + df = dataframe.DataFrame( + data={"creatures": ["salmon", "sea urchin", "baboons", "frog", "chimpanzee"]}, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df.semantics.search( + "creatures", "monkey", top_k=0, model=text_embedding_generator + ) + + +@pytest.mark.parametrize( + "score_column", + [ + pytest.param(None, id="no_score_column"), + pytest.param("distance", id="has_score_column"), + ], +) +def test_sim_join(session, text_embedding_generator, score_column): + df1 = dataframe.DataFrame( + data={"creatures": ["salmon", "cat"]}, + session=session, + ) + df2 = dataframe.DataFrame( + data={"creatures": ["dog", "tuna"]}, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + actual_result = df1.semantics.sim_join( + df2, + left_on="creatures", + right_on="creatures", + model=text_embedding_generator, + top_k=1, + score_column=score_column, + ).to_pandas() + + expected_result = pd.DataFrame( + {"creatures": ["salmon", "cat"], "creatures_1": ["tuna", "dog"]} + ) + pandas.testing.assert_frame_equal( + actual_result[["creatures", "creatures_1"]], + expected_result, + check_dtype=False, + check_index_type=False, + ) + + if score_column is None: + assert len(actual_result.columns) == 2 + else: + assert score_column in actual_result.columns + + +@pytest.mark.parametrize( + ("reply"), + [ + pytest.param("y"), + pytest.param( + "n", marks=pytest.mark.xfail(raises=exceptions.OperationAbortedError) + ), + ], +) +def test_sim_join_with_confirmation( + session, text_embedding_generator, reply, monkeypatch +): + df1 = dataframe.DataFrame( + data={"creatures": ["salmon", "cat"]}, + session=session, + ) + df2 = dataframe.DataFrame( + data={"creatures": ["dog", "tuna"]}, + session=session, + ) + monkeypatch.setattr("builtins.input", lambda: reply) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 0, + ): + df1.semantics.sim_join( + df2, + left_on="creatures", + right_on="creatures", + model=text_embedding_generator, + top_k=1, + ) + + +@pytest.mark.parametrize( + ("left_on", "right_on"), + [ + pytest.param("whatever", "creatures", id="incorrect_left_column"), + pytest.param("creatures", "whatever", id="incorrect_right_column"), + ], +) +def test_sim_join_invalid_column_raises_error( + session, text_embedding_generator, left_on, right_on +): + df1 = dataframe.DataFrame( + data={"creatures": ["salmon", "cat"]}, + session=session, + ) + df2 = dataframe.DataFrame( + data={"creatures": ["dog", "tuna"]}, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df1.semantics.sim_join( + df2, left_on=left_on, right_on=right_on, model=text_embedding_generator + ) + + +def test_sim_join_invalid_model_raises_error(session): + df1 = dataframe.DataFrame( + data={"creatures": ["salmon", "cat"]}, + session=session, + ) + df2 = dataframe.DataFrame( + data={"creatures": ["dog", "tuna"]}, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(TypeError): + df1.semantics.sim_join( + df2, left_on="creatures", right_on="creatures", model=None + ) + + +def test_sim_join_invalid_top_k_raises_error(session, text_embedding_generator): + df1 = dataframe.DataFrame( + data={"creatures": ["salmon", "cat"]}, + session=session, + ) + df2 = dataframe.DataFrame( + data={"creatures": ["dog", "tuna"]}, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df1.semantics.sim_join( + df2, + left_on="creatures", + right_on="creatures", + top_k=0, + model=text_embedding_generator, + ) + + +def test_sim_join_data_too_large_raises_error(session, text_embedding_generator): + df1 = dataframe.DataFrame( + data={"creatures": ["salmon", "cat"]}, + session=session, + ) + df2 = dataframe.DataFrame( + data={"creatures": ["dog", "tuna"]}, + session=session, + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df1.semantics.sim_join( + df2, + left_on="creatures", + right_on="creatures", + model=text_embedding_generator, + max_rows=1, + ) + + +@pytest.mark.parametrize( + "instruction", + [ + pytest.param( + "No column reference", + id="zero_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "{Animals}", + id="non_existing_column", + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + "{Animals} and {Animals}", + id="two_columns", + marks=pytest.mark.xfail(raises=NotImplementedError), + ), + pytest.param( + "{index}", + id="preserved", + marks=pytest.mark.xfail(raises=ValueError), + ), + ], +) +def test_top_k_invalid_instruction_raise_error(instruction, gemini_flash_model): + df = dataframe.DataFrame( + { + "Animals": ["Dog", "Cat", "Bird", "Horse"], + "ID": [1, 2, 3, 4], + "index": ["a", "b", "c", "d"], + } + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ): + df.semantics.top_k(instruction, model=gemini_flash_model, k=2) + + +def test_top_k_invalid_k_raise_error(gemini_flash_model): + df = dataframe.DataFrame({"Animals": ["Dog", "Cat", "Bird", "Horse"]}) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 10, + ), pytest.raises(ValueError): + df.semantics.top_k( + "{Animals} are more popular as pets", + gemini_flash_model, + k=0, + ) + + +@patch("builtins.input", return_value="") +def test_confirm_operation__below_threshold_do_not_confirm(mock_input): + df = dataframe.DataFrame({}) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 3, + ): + df.semantics._confirm_operation(1) + + mock_input.assert_not_called() + + +@patch("builtins.input", return_value="") +def test_confirm_operation__threshold_is_none_do_not_confirm(mock_input): + df = dataframe.DataFrame({}) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + None, + ): + df.semantics._confirm_operation(100) + + mock_input.assert_not_called() + + +@patch("builtins.input", return_value="") +def test_confirm_operation__threshold_autofail_do_not_confirm(mock_input): + df = dataframe.DataFrame({}) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 1, + "compute.semantic_ops_threshold_autofail", + True, + ), pytest.raises(exceptions.OperationAbortedError): + df.semantics._confirm_operation(100) + + mock_input.assert_not_called() + + +@pytest.mark.parametrize( + ("reply", "expectation"), + [ + ("y", nullcontext()), + ("yes", nullcontext()), + ("", nullcontext()), + ("n", pytest.raises(exceptions.OperationAbortedError)), + ("something", pytest.raises(exceptions.OperationAbortedError)), + ], +) +def test_confirm_operation__above_threshold_confirm(reply, expectation, monkeypatch): + monkeypatch.setattr("builtins.input", lambda: reply) + df = dataframe.DataFrame({}) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 3, + ), expectation as e: + assert df.semantics._confirm_operation(4) == e diff --git a/tests/system/large/streaming/test_bigtable.py b/tests/system/large/streaming/test_bigtable.py index f10c534404e..38e01f44bc2 100644 --- a/tests/system/large/streaming/test_bigtable.py +++ b/tests/system/large/streaming/test_bigtable.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time -import uuid from datetime import datetime, timedelta +import time from typing import Generator +import uuid import pytest diff --git a/tests/system/large/streaming/test_pubsub.py b/tests/system/large/streaming/test_pubsub.py index cdc27ae65cf..9ff965fd775 100644 --- a/tests/system/large/streaming/test_pubsub.py +++ b/tests/system/large/streaming/test_pubsub.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import uuid from concurrent import futures from datetime import datetime, timedelta from typing import Generator +import uuid import pytest diff --git a/tests/system/large/test_dataframe_io.py b/tests/system/large/test_dataframe_io.py index c352d618d6a..c60940109d3 100644 --- a/tests/system/large/test_dataframe_io.py +++ b/tests/system/large/test_dataframe_io.py @@ -22,9 +22,8 @@ def test_to_pandas_batches_raise_when_large_result_not_allowed(session): - with ( - bigframes.option_context(LARGE_TABLE_OPTION, False), - pytest.raises(google.api_core.exceptions.Forbidden), + with bigframes.option_context(LARGE_TABLE_OPTION, False), pytest.raises( + google.api_core.exceptions.Forbidden ): df = session.read_gbq(WIKIPEDIA_TABLE) next(df.to_pandas_batches(page_size=500, max_results=1500)) @@ -60,9 +59,8 @@ def test_to_pandas_batches_override_global_option( def test_to_pandas_raise_when_large_result_not_allowed(session): - with ( - bigframes.option_context(LARGE_TABLE_OPTION, False), - pytest.raises(google.api_core.exceptions.Forbidden), + with bigframes.option_context(LARGE_TABLE_OPTION, False), pytest.raises( + google.api_core.exceptions.Forbidden ): df = session.read_gbq(WIKIPEDIA_TABLE) next(df.to_pandas()) diff --git a/tests/system/large/test_location.py b/tests/system/large/test_location.py index 3127d5865a9..3ebe2bb040e 100644 --- a/tests/system/large/test_location.py +++ b/tests/system/large/test_location.py @@ -13,9 +13,7 @@ # limitations under the License. import typing -import unittest.mock as mock -import google.auth.credentials import pandas import pandas.testing import pytest @@ -178,12 +176,8 @@ def test_bq_rep_endpoints(bigquery_location): def test_clients_provider_no_location(): - credentials = mock.create_autospec(google.auth.credentials.Credentials) - with pytest.raises(ValueError, match="Must set location to use regional endpoints"): - bigframes.session.clients.ClientsProvider( - project="", credentials=credentials, use_regional_endpoints=True - ) + bigframes.session.clients.ClientsProvider(use_regional_endpoints=True) @pytest.mark.parametrize( @@ -192,16 +186,12 @@ def test_clients_provider_no_location(): sorted(bigframes.constants.REP_NOT_ENABLED_BIGQUERY_LOCATIONS), ) def test_clients_provider_use_regional_endpoints_non_rep_locations(bigquery_location): - credentials = mock.create_autospec(google.auth.credentials.Credentials) with pytest.raises( ValueError, match=f"not .*available in the location {bigquery_location}", ): bigframes.session.clients.ClientsProvider( - project="", - credentials=credentials, - location=bigquery_location, - use_regional_endpoints=True, + location=bigquery_location, use_regional_endpoints=True ) diff --git a/tests/system/large/test_session.py b/tests/system/large/test_session.py index 937b3c9e274..48c2b9e1b3f 100644 --- a/tests/system/large/test_session.py +++ b/tests/system/large/test_session.py @@ -52,8 +52,7 @@ def large_pd_df(): [ ("bigquery_load"), ("bigquery_streaming"), - # TODO(b/502298527): Reenable bigquery_write test - # ("bigquery_write"), + ("bigquery_write"), ], ) def test_read_pandas_large_df(session, large_pd_df, write_engine: str): diff --git a/tests/system/large/test_tpch.py b/tests/system/large/test_tpch.py deleted file mode 100644 index de630ce0dd4..00000000000 --- a/tests/system/large/test_tpch.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re - -import pandas as pd -import pytest -from google.cloud import bigquery - -TPCH_PATH = "third_party/bigframes_vendored/tpch" -PROJECT_ID = "bigframes-dev-perf" -DATASET_ID = "tpch_0001g" -DATASET = { - "line_item_ds": f"{PROJECT_ID}.{DATASET_ID}.LINEITEM", - "region_ds": f"{PROJECT_ID}.{DATASET_ID}.REGION", - "nation_ds": f"{PROJECT_ID}.{DATASET_ID}.NATION", - "supplier_ds": f"{PROJECT_ID}.{DATASET_ID}.SUPPLIER", - "part_ds": f"{PROJECT_ID}.{DATASET_ID}.PART", - "part_supp_ds": f"{PROJECT_ID}.{DATASET_ID}.PARTSUPP", - "customer_ds": f"{PROJECT_ID}.{DATASET_ID}.CUSTOMER", - "orders_ds": f"{PROJECT_ID}.{DATASET_ID}.ORDERS", -} - - -def _execute_sql_query(bigquery_client, sql_query): - sql_query = sql_query.format(**DATASET) - - job_config = bigquery.QueryJobConfig(use_query_cache=False) - query_job = bigquery_client.query(sql_query, job_config=job_config) - query_job.result() - df = query_job.to_dataframe() - df.columns = df.columns.str.upper() - return df - - -def _execute_bigframes_script(session, bigframes_script): - bigframes_script = re.sub( - r"next\((\w+)\.to_pandas_batches\((.*?)\)\)", - r"return \1.to_pandas()", - bigframes_script, - ) - bigframes_script = re.sub(r"_\s*=\s*(\w+)", r"return \1", bigframes_script) - - bigframes_script = ( - bigframes_script - + f"\nresult = q('{PROJECT_ID}', '{DATASET_ID}', _initialize_session)" - ) - exec_globals = {"_initialize_session": session} - exec(bigframes_script, exec_globals) - bigframes_result = exec_globals.get("result") - return bigframes_result - - -def _verify_result(bigframes_result, sql_result): - if isinstance(bigframes_result, pd.DataFrame): - pd.testing.assert_frame_equal( - sql_result.reset_index(drop=True), - bigframes_result.reset_index(drop=True), - check_dtype=False, - ) - else: - assert sql_result.shape == (1, 1) - sql_scalar = sql_result.iloc[0, 0] - assert sql_scalar == bigframes_result - - -@pytest.mark.parametrize("query_num", range(1, 23)) -@pytest.mark.parametrize("ordered", [True, False]) -def test_tpch_correctness(session, unordered_session, query_num, ordered): - """Runs verification of TPCH benchmark script outputs to ensure correctness.""" - # Execute SQL: - sql_file_path = f"{TPCH_PATH}/sql_queries/q{query_num}.sql" - assert os.path.exists(sql_file_path) - with open(sql_file_path, "r") as f: - sql_query = f.read() - - sql_result = _execute_sql_query(session.bqclient, sql_query) - - # Execute BigFrames: - file_path = f"{TPCH_PATH}/queries/q{query_num}.py" - assert os.path.exists(file_path) - with open(file_path, "r") as file: - bigframes_script = file.read() - - bigframes_result = _execute_bigframes_script( - session if ordered else unordered_session, bigframes_script - ) - - _verify_result(bigframes_result, sql_result) diff --git a/tests/system/load/test_llm.py b/tests/system/load/test_llm.py index eec76cf9b67..25cde92c133 100644 --- a/tests/system/load/test_llm.py +++ b/tests/system/load/test_llm.py @@ -41,8 +41,8 @@ def llm_remote_text_df(session, llm_remote_text_pandas_df): @pytest.mark.parametrize( "model_name", ( - "gemini-2.5-flash", - "gemini-2.5-flash-lite", + "gemini-2.0-flash-001", + "gemini-2.0-flash-lite-001", ), ) def test_llm_gemini_configure_fit( @@ -79,7 +79,7 @@ def test_llm_gemini_configure_fit( @pytest.mark.flaky(retries=2) def test_llm_gemini_w_ground_with_google_search(llm_remote_text_df): - model = llm.GeminiTextGenerator(model_name="gemini-2.5-flash", max_iterations=1) + model = llm.GeminiTextGenerator(model_name="gemini-2.0-flash-001", max_iterations=1) df = model.predict( llm_remote_text_df["prompt"], ground_with_google_search=True, @@ -98,10 +98,18 @@ def test_llm_gemini_w_ground_with_google_search(llm_remote_text_df): # (b/366290533): Claude models are of extremely low capacity. The tests should reside in small tests. Moving these here just to protect BQML's shared capacity(as load test only runs once per day.) and make sure we still have minimum coverage. +@pytest.mark.parametrize( + "model_name", + ("claude-3-haiku", "claude-3-5-sonnet"), +) @pytest.mark.flaky(retries=3, delay=120) -def test_claude3_text_generator_create_load(dataset_id, session, bq_connection): +def test_claude3_text_generator_create_load( + dataset_id, model_name, session, session_us_east5, bq_connection +): + if model_name in ("claude-3-5-sonnet",): + session = session_us_east5 claude3_text_generator_model = llm.Claude3TextGenerator( - model_name="claude-3-haiku", connection_name=bq_connection, session=session + model_name=model_name, connection_name=bq_connection, session=session ) assert claude3_text_generator_model is not None assert claude3_text_generator_model._bqml_model is not None @@ -112,15 +120,21 @@ def test_claude3_text_generator_create_load(dataset_id, session, bq_connection): ) assert f"{dataset_id}.temp_text_model" == reloaded_model._bqml_model.model_name assert reloaded_model.connection_name == bq_connection - assert reloaded_model.model_name == "claude-3-haiku" + assert reloaded_model.model_name == model_name +@pytest.mark.parametrize( + "model_name", + ("claude-3-haiku", "claude-3-5-sonnet"), +) @pytest.mark.flaky(retries=3, delay=120) def test_claude3_text_generator_predict_default_params_success( - llm_text_df, session, bq_connection + llm_text_df, model_name, session, session_us_east5, bq_connection ): + if model_name in ("claude-3-5-sonnet",): + session = session_us_east5 claude3_text_generator_model = llm.Claude3TextGenerator( - model_name="claude-3-haiku", connection_name=bq_connection, session=session + model_name=model_name, connection_name=bq_connection, session=session ) df = claude3_text_generator_model.predict(llm_text_df).to_pandas() utils.check_pandas_df_schema_and_index( @@ -128,12 +142,18 @@ def test_claude3_text_generator_predict_default_params_success( ) +@pytest.mark.parametrize( + "model_name", + ("claude-3-haiku", "claude-3-5-sonnet"), +) @pytest.mark.flaky(retries=3, delay=120) def test_claude3_text_generator_predict_with_params_success( - llm_text_df, session, bq_connection + llm_text_df, model_name, session, session_us_east5, bq_connection ): + if model_name in ("claude-3-5-sonnet",): + session = session_us_east5 claude3_text_generator_model = llm.Claude3TextGenerator( - model_name="claude-3-haiku", connection_name=bq_connection, session=session + model_name=model_name, connection_name=bq_connection, session=session ) df = claude3_text_generator_model.predict( llm_text_df, max_output_tokens=100, top_k=20, top_p=0.5 @@ -143,13 +163,20 @@ def test_claude3_text_generator_predict_with_params_success( ) +@pytest.mark.parametrize( + "model_name", + ("claude-3-haiku", "claude-3-5-sonnet"), +) @pytest.mark.flaky(retries=3, delay=120) def test_claude3_text_generator_predict_multi_col_success( - llm_text_df, session, bq_connection + llm_text_df, model_name, session, session_us_east5, bq_connection ): + if model_name in ("claude-3-5-sonnet",): + session = session_us_east5 + llm_text_df["additional_col"] = 1 claude3_text_generator_model = llm.Claude3TextGenerator( - model_name="claude-3-haiku", connection_name=bq_connection, session=session + model_name=model_name, connection_name=bq_connection, session=session ) df = claude3_text_generator_model.predict(llm_text_df).to_pandas() utils.check_pandas_df_schema_and_index( diff --git a/tests/system/small/bigquery/test_ai.py b/tests/system/small/bigquery/test_ai.py index 0b6738dec80..b4dc3d2508d 100644 --- a/tests/system/small/bigquery/test_ai.py +++ b/tests/system/small/bigquery/test_ai.py @@ -12,62 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import annotations - -import uuid from unittest import mock -import google.cloud.bigquery import pandas as pd import pyarrow as pa import pytest +from bigframes import dataframe, dtypes, series import bigframes.bigquery as bbq import bigframes.pandas as bpd -from bigframes import dataframe, dtypes, series from bigframes.testing import utils as test_utils -@pytest.fixture -def use_ibis_compiler(): - original_setting = bpd.options.experiments.sql_compiler - bpd.options.experiments.sql_compiler = "legacy" - try: - yield - finally: - bpd.options.experiments.sql_compiler = original_setting - - -def _create_mock_obj_ref_df(session, uris, name="image", connection=None): - df = bpd.DataFrame({name: uris}, session=session) - # Convert string URIs to ObjectRef structs - if connection is None: - connection = "us.bigframes-rf-conn" - df[name] = bbq.obj.make_ref(df[name], authorizer=connection) - - table_id = f"bigframes-dev.bigframes_tests_sys.tmp_obj_ref_{uuid.uuid4().hex}" - df.to_gbq(table_id, if_exists="replace") - - client = session.bqclient - table = client.get_table(table_id) - schema = list(table.schema) - for i, field in enumerate(schema): - if field.name == name: - schema[i] = google.cloud.bigquery.SchemaField( - name=field.name, - field_type=field.field_type, - mode=field.mode, - description="bigframes_dtype: OBJ_REF_DTYPE", - fields=field.fields, - ) - break - table.schema = schema - client.update_table(table, ["schema"]) - - return session.read_gbq(table_id) - - -def test_ai_function_pandas_tuple_input(session): +def test_ai_function_pandas_input(session): s1 = pd.Series(["apple", "bear"]) s2 = bpd.Series(["fruit", "tree"], session=session) prompt = (s1, " is a ", s2) @@ -86,17 +43,6 @@ def test_ai_function_pandas_tuple_input(session): ) -def test_ai_function_pandas_series_input(session): - s = pd.Series(["cat", "lavender"]) - - result = bbq.ai.classify( - s, categories=["animal", "plant"], endpoint="gemini-2.5-flash" - ) - - assert len(result) == len(s) - assert result.dtype == dtypes.STRING_DTYPE - - def test_ai_function_string_input(session): with mock.patch( "bigframes.core.global_session.get_global_session" @@ -158,19 +104,6 @@ def test_ai_generate(session): ) -def test_ai_generate_access_full_response_with_ibis(session, use_ibis_compiler): - country = bpd.Series(["Japan", "Canada"], session=session) - prompt = ("What's the capital city of ", country, "? one word only") - - result = ( - bbq.ai.generate(prompt, endpoint="gemini-2.5-flash") - .struct.field("full_response") - .to_pandas() - ) - - assert _contains_no_nulls(result) - - def test_ai_generate_with_output_schema(session): country = bpd.Series(["Japan", "Canada"], session=session) prompt = ("Describe ", country) @@ -225,30 +158,12 @@ def test_ai_generate_bool(session): ) -def test_ai_generate_bool_access_full_response_with_ibis(session, use_ibis_compiler): - s1 = bpd.Series(["apple", "bear"], session=session) - s2 = bpd.Series(["fruit", "tree"], session=session) - prompt = (s1, " is a ", s2) - - result = ( - bbq.ai.generate_bool(prompt, endpoint="gemini-2.5-flash") - .struct.field("full_response") - .to_pandas() - ) - - assert _contains_no_nulls(result) - - -def test_ai_generate_bool_multi_model(session, bq_connection): - df = _create_mock_obj_ref_df( - session, - ["gs://cloud-samples-data/vision/ocr/sign.jpg"], - name="image", - connection=bq_connection, +def test_ai_generate_bool_multi_model(session): + df = session.from_glob_path( + "gs://bigframes-dev-testing/a_multimodel/images/*", name="image" ) - image_runtime = bbq.obj.get_access_url(df["image"], mode="R") - result = bbq.ai.generate_bool((image_runtime, " contains an animal")) + result = bbq.ai.generate_bool((df["image"], " contains an animal")) assert _contains_no_nulls(result) assert result.dtype == pd.ArrowDtype( @@ -280,30 +195,13 @@ def test_ai_generate_int(session): ) -def test_ai_generate_int_access_full_response_with_ibis(session, use_ibis_compiler): - s = bpd.Series(["Cat"], session=session) - prompt = ("How many legs does a ", s, " have?") - - result = ( - bbq.ai.generate_int(prompt, endpoint="gemini-2.5-flash") - .struct.field("full_response") - .to_pandas() +def test_ai_generate_int_multi_model(session): + df = session.from_glob_path( + "gs://bigframes-dev-testing/a_multimodel/images/*", name="image" ) - assert _contains_no_nulls(result) - - -def test_ai_generate_int_multi_model(session, bq_connection): - df = _create_mock_obj_ref_df( - session, - ["gs://cloud-samples-data/vision/ocr/sign.jpg"], - name="image", - connection=bq_connection, - ) - - image_runtime = bbq.obj.get_access_url(df["image"], mode="R") result = bbq.ai.generate_int( - ("How many animals are there in the picture ", image_runtime) + ("How many animals are there in the picture ", df["image"]) ) assert _contains_no_nulls(result) @@ -336,30 +234,13 @@ def test_ai_generate_double(session): ) -def test_ai_generate_double_access_full_response_with_ibis(session, use_ibis_compiler): - s = bpd.Series(["Cat"], session=session) - prompt = ("How many legs does a ", s, " have?") - - result = ( - bbq.ai.generate_double(prompt, endpoint="gemini-2.5-flash") - .struct.field("full_response") - .to_pandas() - ) - - assert _contains_no_nulls(result) - - -def test_ai_generate_double_multi_model(session, bq_connection): - df = _create_mock_obj_ref_df( - session, - ["gs://cloud-samples-data/vision/ocr/sign.jpg"], - name="image", - connection=bq_connection, +def test_ai_generate_double_multi_model(session): + df = session.from_glob_path( + "gs://bigframes-dev-testing/a_multimodel/images/*", name="image" ) - image_runtime = bbq.obj.get_access_url(df["image"], mode="R") result = bbq.ai.generate_double( - ("How many animals are there in the picture ", image_runtime) + ("How many animals are there in the picture ", df["image"]) ) assert _contains_no_nulls(result) @@ -374,68 +255,27 @@ def test_ai_generate_double_multi_model(session, bq_connection): ) -def test_ai_embed_series_content(session): - content = bpd.Series(["dog"], session=session) - - result = bbq.ai.embed(content, endpoint="text-embedding-005") - - assert _contains_no_nulls(result) - assert result.dtype == pd.ArrowDtype( - pa.struct( - ( - pa.field("result", pa.list_(pa.float64())), - pa.field("status", pa.string()), - ) - ) - ) - - -def test_ai_embed_string_content(session): - with mock.patch( - "bigframes.core.global_session.get_global_session" - ) as mock_get_session: - mock_get_session.return_value = session - - result = bbq.ai.embed("dog", endpoint="text-embedding-005") - - assert _contains_no_nulls(result) - assert result.dtype == pd.ArrowDtype( - pa.struct( - ( - pa.field("result", pa.list_(pa.float64())), - pa.field("status", pa.string()), - ) - ) - ) - - def test_ai_if(session): s1 = bpd.Series(["apple", "bear"], session=session) s2 = bpd.Series(["fruit", "tree"], session=session) prompt = (s1, " is a ", s2) - result = bbq.ai.if_( - prompt, - optimization_mode="maximize_quality", - max_error_ratio=0.5, - ) + result = bbq.ai.if_(prompt) - assert len(result) == len(s1) + assert _contains_no_nulls(result) assert result.dtype == dtypes.BOOL_DTYPE def test_ai_if_multi_model(session, bq_connection): - df = _create_mock_obj_ref_df( - session, - ["gs://cloud-samples-data/vision/ocr/sign.jpg"], + df = session.from_glob_path( + "gs://bigframes-dev-testing/a_multimodel/images/*", name="image", connection=bq_connection, ) - image_runtime = bbq.obj.get_access_url(df["image"], mode="R") - result = bbq.ai.if_((image_runtime, " contains an animal")) + result = bbq.ai.if_((df["image"], " contains an animal")) - assert len(result) == len(df) + assert _contains_no_nulls(result) assert result.dtype == dtypes.BOOL_DTYPE @@ -444,42 +284,20 @@ def test_ai_classify(session): result = bbq.ai.classify(s, ["animal", "plant"]) - assert len(result) == len(s) - assert result.dtype == dtypes.STRING_DTYPE - - -def test_ai_classify_with_examples(session): - s = bpd.Series(["cat", "orchid"], session=session) - - result = bbq.ai.classify(s, ["animal", "plant"], examples=[("dog", "animal")]) - - assert len(result) == len(s) + assert _contains_no_nulls(result) assert result.dtype == dtypes.STRING_DTYPE -def test_ai_classify_output_mode(session, bq_connection): - s = bpd.Series(["cat", "orchid"], session=session) - - result = bbq.ai.classify( - s, ["animal", "plant"], output_mode="multi", examples=[("dog", ["animal"])] - ) - - assert len(result) == len(s) - assert result.dtype == dtypes.list_type(dtypes.STRING_DTYPE) - - def test_ai_classify_multi_model(session, bq_connection): - df = _create_mock_obj_ref_df( - session, - ["gs://cloud-samples-data/vision/ocr/sign.jpg"], + df = session.from_glob_path( + "gs://bigframes-dev-testing/a_multimodel/images/*", name="image", connection=bq_connection, ) - image_runtime = bbq.obj.get_access_url(df["image"], mode="R") - result = bbq.ai.classify(image_runtime, ["photo", "cartoon"]) + result = bbq.ai.classify(df["image"], ["photo", "cartoon"]) - assert len(result) == len(df) + assert _contains_no_nulls(result) assert result.dtype == dtypes.STRING_DTYPE @@ -489,23 +307,19 @@ def test_ai_score(session): result = bbq.ai.score(prompt) - assert len(result) == len(s) + assert _contains_no_nulls(result) assert result.dtype == dtypes.FLOAT_DTYPE -def test_ai_score_multi_model(session, bq_connection): - df = _create_mock_obj_ref_df( - session, - ["gs://cloud-samples-data/vision/ocr/sign.jpg"], - name="image", - connection=bq_connection, +def test_ai_score_multi_model(session): + df = session.from_glob_path( + "gs://bigframes-dev-testing/a_multimodel/images/*", name="image" ) - image_runtime = bbq.obj.get_access_url(df["image"], mode="R") - prompt = ("Rank the liveliness of ", image_runtime, "on the scale from 1 to 3") + prompt = ("Rank the liveliness of ", df["image"], "on the scale from 1 to 3") result = bbq.ai.score(prompt) - assert len(result) == len(df) + assert _contains_no_nulls(result) assert result.dtype == dtypes.FLOAT_DTYPE @@ -556,35 +370,5 @@ def test_forecast_w_params(time_series_df_default_index: dataframe.DataFrame): ) -def test_ai_similarity(session): - s1 = bpd.Series(["happy", "sad"], session=session) - s2 = pd.Series(["glad", "angry"]) - - result = bbq.ai.similarity(s1, s2, endpoint="text-embedding-005") - - assert _contains_no_nulls(result) - assert result.dtype == dtypes.FLOAT_DTYPE - - -def test_ai_similarity_one_content_is_string_literal(session): - s1 = "happy" - s2 = bpd.Series(["glad", "angry"], session=session) - - result = bbq.ai.similarity(s1, s2, model="embeddinggemma-300m") - - assert _contains_no_nulls(result) - assert result.dtype == dtypes.FLOAT_DTYPE - - -def test_ai_similarity_both_contents_are_string_literals(session): - s1 = "happy" - s2 = "glad" - - result = bbq.ai.similarity(s1, s2, endpoint="text-embedding-005") - - assert _contains_no_nulls(result) - assert result.dtype == dtypes.FLOAT_DTYPE - - -def _contains_no_nulls(s: series.Series | pd.Series) -> bool: +def _contains_no_nulls(s: series.Series) -> bool: return len(s) == s.count() diff --git a/tests/system/small/bigquery/test_array.py b/tests/system/small/bigquery/test_array.py index c8c69f7457e..2ceb90e22c8 100644 --- a/tests/system/small/bigquery/test_array.py +++ b/tests/system/small/bigquery/test_array.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import cast - import numpy as np import pandas as pd import pytest @@ -68,15 +66,10 @@ ], ) def test_array_length(input_data, expected): - series = pd.Series(input_data) - expected = pd.Series( - expected, - index=pd.Index(range(len(input_data)), dtype="Int64"), - dtype=bigframes.dtypes.INT_DTYPE, - ) - result = cast(bpd.Series, bbq.array_length(series)) + series = bpd.Series(input_data) + expected = pd.Series(expected, dtype=bigframes.dtypes.INT_DTYPE) pd.testing.assert_series_equal( - result.to_pandas(), + bbq.array_length(series).to_pandas(), expected, check_index_type=False, ) diff --git a/tests/system/small/bigquery/test_datetime.py b/tests/system/small/bigquery/test_datetime.py index 58e07928f0c..dc68e7b892d 100644 --- a/tests/system/small/bigquery/test_datetime.py +++ b/tests/system/small/bigquery/test_datetime.py @@ -18,7 +18,6 @@ import pyarrow as pa import pytest -import bigframes.testing.utils from bigframes import bigquery _TIMESTAMP_DTYPE = pd.ArrowDtype(pa.timestamp("us", tz="UTC")) @@ -41,7 +40,7 @@ def test_unix_seconds(scalars_dfs): .apply(lambda ts: _to_unix_epoch(ts, "s")) .astype("Int64") ) - bigframes.testing.utils.assert_series_equal(actual_res, expected_res) + pd.testing.assert_series_equal(actual_res, expected_res) def test_unix_seconds_after_type_casting(int_series): @@ -54,9 +53,7 @@ def test_unix_seconds_after_type_casting(int_series): .apply(lambda ts: _to_unix_epoch(ts, "s")) .astype("Int64") ) - bigframes.testing.utils.assert_series_equal( - actual_res, expected_res, check_index_type=False - ) + pd.testing.assert_series_equal(actual_res, expected_res, check_index_type=False) def test_unix_seconds_incorrect_input_type_raise_error(scalars_dfs): @@ -76,7 +73,7 @@ def test_unix_millis(scalars_dfs): .apply(lambda ts: _to_unix_epoch(ts, "ms")) .astype("Int64") ) - bigframes.testing.utils.assert_series_equal(actual_res, expected_res) + pd.testing.assert_series_equal(actual_res, expected_res) def test_unix_millis_after_type_casting(int_series): @@ -89,9 +86,7 @@ def test_unix_millis_after_type_casting(int_series): .apply(lambda ts: _to_unix_epoch(ts, "ms")) .astype("Int64") ) - bigframes.testing.utils.assert_series_equal( - actual_res, expected_res, check_index_type=False - ) + pd.testing.assert_series_equal(actual_res, expected_res, check_index_type=False) def test_unix_millis_incorrect_input_type_raise_error(scalars_dfs): @@ -111,7 +106,7 @@ def test_unix_micros(scalars_dfs): .apply(lambda ts: _to_unix_epoch(ts, "us")) .astype("Int64") ) - bigframes.testing.utils.assert_series_equal(actual_res, expected_res) + pd.testing.assert_series_equal(actual_res, expected_res) def test_unix_micros_after_type_casting(int_series): @@ -124,9 +119,7 @@ def test_unix_micros_after_type_casting(int_series): .apply(lambda ts: _to_unix_epoch(ts, "us")) .astype("Int64") ) - bigframes.testing.utils.assert_series_equal( - actual_res, expected_res, check_index_type=False - ) + pd.testing.assert_series_equal(actual_res, expected_res, check_index_type=False) def test_unix_micros_incorrect_input_type_raise_error(scalars_dfs): diff --git a/tests/system/small/bigquery/test_geo.py b/tests/system/small/bigquery/test_geo.py index 16df467d24a..28db58c7112 100644 --- a/tests/system/small/bigquery/test_geo.py +++ b/tests/system/small/bigquery/test_geo.py @@ -28,11 +28,10 @@ Polygon, ) +from bigframes.bigquery import st_length import bigframes.bigquery as bbq import bigframes.geopandas import bigframes.session -import bigframes.testing.utils -from bigframes.bigquery import st_length def test_geo_st_area(session: bigframes.session.Session): @@ -57,7 +56,7 @@ def test_geo_st_area(session: bigframes.session.Session): geobf_s_result = bbq.st_area(geobf_s).to_pandas().round(-3) assert geobf_s_result.iloc[0] >= 1000 - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( geobf_s_result, geopd_s_result, check_dtype=False, @@ -110,7 +109,7 @@ def test_st_length_various_geometries(session): # Test default use_spheroid result_default = st_length(geoseries).to_pandas() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( result_default, expected_lengths, rtol=1e-3, @@ -119,7 +118,7 @@ def test_st_length_various_geometries(session): # Test explicit use_spheroid=False result_explicit_false = st_length(geoseries, use_spheroid=False).to_pandas() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( result_explicit_false, expected_lengths, rtol=1e-3, @@ -153,7 +152,7 @@ def test_geo_st_difference_with_geometry_objects(session: bigframes.session.Sess index=[0, 1, 2], dtype=geopandas.array.GeometryDtype(), ) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( geobf_s_result, expected, check_index_type=False, @@ -192,7 +191,7 @@ def test_geo_st_difference_with_single_geometry_object( index=[0, 1, 2], dtype=geopandas.array.GeometryDtype(), ) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( geobf_s_result, expected, check_index_type=False, @@ -218,7 +217,7 @@ def test_geo_st_difference_with_similar_geometry_objects( index=[0, 1, 2], dtype=geopandas.array.GeometryDtype(), ) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( geobf_s_result, expected, check_index_type=False, @@ -274,7 +273,7 @@ def test_geo_st_distance_with_geometry_objects(session: bigframes.session.Sessio index=[0, 1, 2, 3], dtype="Float64", ) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( geobf_s_result, expected, check_index_type=False, @@ -321,7 +320,7 @@ def test_geo_st_distance_with_single_geometry_object( ], dtype="Float64", ) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( geobf_s_result, expected, check_index_type=False, @@ -356,7 +355,7 @@ def test_geo_st_intersection_with_geometry_objects(session: bigframes.session.Se index=[0, 1, 2], dtype=geopandas.array.GeometryDtype(), ) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( geobf_s_result, expected, check_index_type=False, @@ -395,7 +394,7 @@ def test_geo_st_intersection_with_single_geometry_object( index=[0, 1, 2], dtype=geopandas.array.GeometryDtype(), ) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( geobf_s_result, expected, check_index_type=False, @@ -425,7 +424,7 @@ def test_geo_st_intersection_with_similar_geometry_objects( index=[0, 1, 2], dtype=geopandas.array.GeometryDtype(), ) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( geobf_s_result, expected, check_index_type=False, @@ -444,7 +443,9 @@ def test_geo_st_isclosed(session: bigframes.session.Session): GeometryCollection(), # Empty GeometryCollection bigframes.geopandas.GeoSeries.from_wkt( ["GEOMETRYCOLLECTION EMPTY"], session=session - ).iloc[0], # Also empty + ).iloc[ + 0 + ], # Also empty None, # Should be filtered out by dropna ], index=[0, 1, 2, 3, 4, 5, 6], @@ -464,7 +465,7 @@ def test_geo_st_isclosed(session: bigframes.session.Session): ] expected_series = pd.Series(data=expected_data, dtype="boolean") - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, expected_series, # We default to Int64 (nullable) dtype, but pandas defaults to int64 index. diff --git a/tests/system/small/bigquery/test_json.py b/tests/system/small/bigquery/test_json.py index 2d97172e7b5..d2ebb73972a 100644 --- a/tests/system/small/bigquery/test_json.py +++ b/tests/system/small/bigquery/test_json.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - import geopandas as gpd # type: ignore import pandas as pd import pyarrow as pa @@ -390,7 +389,7 @@ def test_parse_json_w_invalid_series_type(): def test_to_json_from_int(): s = bpd.Series([1, 2, None, 3]) actual = bbq.to_json(s) - expected = bpd.Series(["1.0", "2.0", None, "3.0"], dtype=dtypes.JSON_DTYPE) + expected = bpd.Series(["1.0", "2.0", "null", "3.0"], dtype=dtypes.JSON_DTYPE) pd.testing.assert_series_equal(actual.to_pandas(), expected.to_pandas()) @@ -405,7 +404,7 @@ def test_to_json_from_struct(): actual = bbq.to_json(s) expected = bpd.Series( - ['{"version":1,"project":"pandas"}', '{"version":2,"project":"numpy"}'], + ['{"project":"pandas","version":1}', '{"project":"numpy","version":2}'], dtype=dtypes.JSON_DTYPE, ) @@ -430,7 +429,7 @@ def test_to_json_string_from_struct(): actual = bbq.to_json_string(s) expected = bpd.Series( - ['{"version":1,"project":"pandas"}', '{"version":2,"project":"numpy"}'], + ['{"project":"pandas","version":1}', '{"project":"numpy","version":2}'], dtype=dtypes.STRING_DTYPE, ) diff --git a/tests/system/small/bigquery/test_mathematical.py b/tests/system/small/bigquery/test_mathematical.py deleted file mode 100644 index 66aef96e57d..00000000000 --- a/tests/system/small/bigquery/test_mathematical.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import bigframes.bigquery as bbq - - -def test_rand(scalars_df_index): - df = scalars_df_index - - # Apply rand - df = df.assign(random=bbq.rand()) - result = df["random"] - - # Eagerly evaluate - result_pd = result.to_pandas() - - # Check length - assert len(result_pd) == len(df) - - # Check values in [0, 1) - assert (result_pd >= 0).all() - assert (result_pd < 1).all() - - # Check not all values are equal (unlikely collision for random) - if len(result_pd) > 1: - assert result_pd.nunique() > 1 diff --git a/tests/system/small/bigquery/test_sql.py b/tests/system/small/bigquery/test_sql.py index c0f7eed938e..c519b427faf 100644 --- a/tests/system/small/bigquery/test_sql.py +++ b/tests/system/small/bigquery/test_sql.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pandas as pd import pytest import bigframes.bigquery as bbq import bigframes.dtypes as dtypes import bigframes.pandas as bpd -import bigframes.testing.utils def test_sql_scalar_for_all_scalar_types(scalars_df_null_index): @@ -59,10 +59,8 @@ def test_sql_scalar_for_bool_series(scalars_df_index): series: bpd.Series = scalars_df_index["bool_col"] result = bbq.sql_scalar("CAST({0} AS INT64)", [series]) expected = series.astype(dtypes.INT_DTYPE) - expected.name = result.name - bigframes.testing.utils.assert_series_equal( - result.to_pandas(), expected.to_pandas() - ) + expected.name = None + pd.testing.assert_series_equal(result.to_pandas(), expected.to_pandas()) @pytest.mark.parametrize( @@ -85,10 +83,8 @@ def test_sql_scalar_outputs_all_scalar_types(scalars_df_index, column_name): series: bpd.Series = scalars_df_index[column_name] result = bbq.sql_scalar("{0}", [series]) expected = series - expected.name = result.name - bigframes.testing.utils.assert_series_equal( - result.to_pandas(), expected.to_pandas() - ) + expected.name = None + pd.testing.assert_series_equal(result.to_pandas(), expected.to_pandas()) def test_sql_scalar_for_array_series(repeated_df): @@ -118,18 +114,14 @@ def test_sql_scalar_for_array_series(repeated_df): + repeated_df["numeric_list_col"].list.len() + repeated_df["string_list_col"].list.len() ) - bigframes.testing.utils.assert_series_equal( - result.to_pandas(), expected.to_pandas() - ) + pd.testing.assert_series_equal(result.to_pandas(), expected.to_pandas()) def test_sql_scalar_outputs_array_series(repeated_df): result = bbq.sql_scalar("{0}", [repeated_df["int_list_col"]]) expected = repeated_df["int_list_col"] - expected.name = result.name - bigframes.testing.utils.assert_series_equal( - result.to_pandas(), expected.to_pandas() - ) + expected.name = None + pd.testing.assert_series_equal(result.to_pandas(), expected.to_pandas()) def test_sql_scalar_for_struct_series(nested_structs_df): @@ -140,18 +132,14 @@ def test_sql_scalar_for_struct_series(nested_structs_df): expected = nested_structs_df["person"].struct.field( "name" ).str.len() + nested_structs_df["person"].struct.field("age") - bigframes.testing.utils.assert_series_equal( - result.to_pandas(), expected.to_pandas() - ) + pd.testing.assert_series_equal(result.to_pandas(), expected.to_pandas()) def test_sql_scalar_outputs_struct_series(nested_structs_df): result = bbq.sql_scalar("{0}", [nested_structs_df["person"]]) expected = nested_structs_df["person"] - expected.name = result.name - bigframes.testing.utils.assert_series_equal( - result.to_pandas(), expected.to_pandas() - ) + expected.name = None + pd.testing.assert_series_equal(result.to_pandas(), expected.to_pandas()) def test_sql_scalar_for_json_series(json_df): @@ -162,16 +150,12 @@ def test_sql_scalar_for_json_series(json_df): ], ) expected = bbq.json_value(json_df["json_col"], "$.int_value") - expected.name = result.name - bigframes.testing.utils.assert_series_equal( - result.to_pandas(), expected.to_pandas() - ) + expected.name = None + pd.testing.assert_series_equal(result.to_pandas(), expected.to_pandas()) def test_sql_scalar_outputs_json_series(json_df): result = bbq.sql_scalar("{0}", [json_df["json_col"]]) expected = json_df["json_col"] - expected.name = result.name - bigframes.testing.utils.assert_series_equal( - result.to_pandas(), expected.to_pandas() - ) + expected.name = None + pd.testing.assert_series_equal(result.to_pandas(), expected.to_pandas()) diff --git a/tests/system/small/bigquery/test_struct.py b/tests/system/small/bigquery/test_struct.py index 85404969605..58c822f642f 100644 --- a/tests/system/small/bigquery/test_struct.py +++ b/tests/system/small/bigquery/test_struct.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pandas as pd import pytest import bigframes.bigquery as bbq import bigframes.series as series -import bigframes.testing.utils @pytest.mark.parametrize( @@ -53,10 +53,9 @@ def test_struct_from_dataframe(columns_arg): srs = series.Series( columns_arg, ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( srs.to_pandas(), bbq.struct(srs.struct.explode()).to_pandas(), check_index_type=False, check_dtype=False, - check_names=False, # None vs nan version dependent ) diff --git a/tests/system/small/bigquery/test_vector_search.py b/tests/system/small/bigquery/test_vector_search.py index b8ad4c0df22..ff320731e25 100644 --- a/tests/system/small/bigquery/test_vector_search.py +++ b/tests/system/small/bigquery/test_vector_search.py @@ -13,7 +13,7 @@ # limitations under the License. import random -from typing import Any, Dict, Iterable, cast +from typing import Any, cast, Dict, Iterable import google.cloud.bigquery import numpy as np diff --git a/tests/system/small/blob/test_io.py b/tests/system/small/blob/test_io.py new file mode 100644 index 00000000000..5ada4fabb0e --- /dev/null +++ b/tests/system/small/blob/test_io.py @@ -0,0 +1,127 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import IPython.display +import pandas as pd + +import bigframes +import bigframes.pandas as bpd + + +def test_blob_create_from_uri_str( + bq_connection: str, session: bigframes.Session, images_uris +): + uri_series = bpd.Series(images_uris, session=session) + blob_series = uri_series.str.to_blob(connection=bq_connection) + + pd_blob_df = blob_series.struct.explode().to_pandas() + expected_pd_df = pd.DataFrame( + { + "uri": images_uris, + "version": [None, None], + "authorizer": [bq_connection.casefold(), bq_connection.casefold()], + "details": [None, None], + } + ) + + pd.testing.assert_frame_equal( + pd_blob_df, expected_pd_df, check_dtype=False, check_index_type=False + ) + + +def test_blob_create_from_glob_path( + bq_connection: str, session: bigframes.Session, images_gcs_path, images_uris +): + blob_df = session.from_glob_path( + images_gcs_path, connection=bq_connection, name="blob_col" + ) + pd_blob_df = ( + blob_df["blob_col"] + .struct.explode() + .to_pandas() + .sort_values("uri") + .reset_index(drop=True) + ) + + expected_df = pd.DataFrame( + { + "uri": images_uris, + "version": [None, None], + "authorizer": [bq_connection.casefold(), bq_connection.casefold()], + "details": [None, None], + } + ) + + pd.testing.assert_frame_equal( + pd_blob_df, expected_df, check_dtype=False, check_index_type=False + ) + + +def test_blob_create_read_gbq_object_table( + bq_connection: str, session: bigframes.Session, images_gcs_path, images_uris +): + obj_table = session._create_object_table(images_gcs_path, bq_connection) + + blob_df = session.read_gbq_object_table(obj_table, name="blob_col") + pd_blob_df = ( + blob_df["blob_col"] + .struct.explode() + .to_pandas() + .sort_values("uri") + .reset_index(drop=True) + ) + expected_df = pd.DataFrame( + { + "uri": images_uris, + "version": [None, None], + "authorizer": [bq_connection.casefold(), bq_connection.casefold()], + "details": [None, None], + } + ) + + pd.testing.assert_frame_equal( + pd_blob_df, expected_df, check_dtype=False, check_index_type=False + ) + + +def test_display_images(monkeypatch, images_mm_df: bpd.DataFrame): + mock_display = mock.Mock() + monkeypatch.setattr(IPython.display, "display", mock_display) + + images_mm_df["blob_col"].blob.display() + + for call in mock_display.call_args_list: + args, _ = call + arg = args[0] + assert isinstance(arg, IPython.display.Image) + + +def test_display_nulls( + monkeypatch, + bq_connection: str, + session: bigframes.Session, +): + uri_series = bpd.Series([None, None, None], dtype="string", session=session) + blob_series = uri_series.str.to_blob(connection=bq_connection) + mock_display = mock.Mock() + monkeypatch.setattr(IPython.display, "display", mock_display) + + blob_series.blob.display() + + for call in mock_display.call_args_list: + args, _ = call + arg = args[0] + assert arg == "" diff --git a/tests/system/small/blob/test_properties.py b/tests/system/small/blob/test_properties.py index c3597b37116..47d4d2aa04f 100644 --- a/tests/system/small/blob/test_properties.py +++ b/tests/system/small/blob/test_properties.py @@ -13,17 +13,13 @@ # limitations under the License. import pandas as pd -import pytest -import bigframes.bigquery as bbq import bigframes.dtypes as dtypes import bigframes.pandas as bpd -pytest.skip("Skipping blob tests due to b/481790217", allow_module_level=True) - def test_blob_uri(images_uris: list[str], images_mm_df: bpd.DataFrame): - actual = images_mm_df["blob_col"].struct.field("uri").to_pandas() + actual = images_mm_df["blob_col"].blob.uri().to_pandas() expected = pd.Series(images_uris, name="uri") pd.testing.assert_series_equal( @@ -32,7 +28,7 @@ def test_blob_uri(images_uris: list[str], images_mm_df: bpd.DataFrame): def test_blob_authorizer(images_mm_df: bpd.DataFrame, bq_connection: str): - actual = images_mm_df["blob_col"].struct.field("authorizer").to_pandas() + actual = images_mm_df["blob_col"].blob.authorizer().to_pandas() expected = pd.Series( [bq_connection.casefold(), bq_connection.casefold()], name="authorizer" ) @@ -43,9 +39,7 @@ def test_blob_authorizer(images_mm_df: bpd.DataFrame, bq_connection: str): def test_blob_version(images_mm_df: bpd.DataFrame): - actual = bbq.json_value( - images_mm_df["blob_col"].struct.field("details"), "$.version" - ).to_pandas() + actual = images_mm_df["blob_col"].blob.version().to_pandas() expected = pd.Series(["1753907851152593", "1753907851111538"], name="version") pd.testing.assert_series_equal( @@ -54,7 +48,7 @@ def test_blob_version(images_mm_df: bpd.DataFrame): def test_blob_metadata(images_mm_df: bpd.DataFrame): - actual = images_mm_df["blob_col"].struct.field("details").to_pandas() + actual = images_mm_df["blob_col"].blob.metadata().to_pandas() expected = pd.Series( [ ( @@ -78,9 +72,7 @@ def test_blob_metadata(images_mm_df: bpd.DataFrame): def test_blob_content_type(images_mm_df: bpd.DataFrame): - actual = bbq.json_value( - images_mm_df["blob_col"].struct.field("details"), "$.content_type" - ).to_pandas() + actual = images_mm_df["blob_col"].blob.content_type().to_pandas() expected = pd.Series(["image/jpeg", "image/jpeg"], name="content_type") pd.testing.assert_series_equal( @@ -89,9 +81,7 @@ def test_blob_content_type(images_mm_df: bpd.DataFrame): def test_blob_md5_hash(images_mm_df: bpd.DataFrame): - actual = bbq.json_value( - images_mm_df["blob_col"].struct.field("details"), "$.md5_hash" - ).to_pandas() + actual = images_mm_df["blob_col"].blob.md5_hash().to_pandas() expected = pd.Series( ["e130ad042261a1883cd2cc06831cf748", "e2ae3191ff2b809fd0935f01a537c650"], name="md5_hash", @@ -103,11 +93,7 @@ def test_blob_md5_hash(images_mm_df: bpd.DataFrame): def test_blob_size(images_mm_df: bpd.DataFrame): - actual = ( - bbq.json_value(images_mm_df["blob_col"].struct.field("details"), "$.size") - .astype("Int64") - .to_pandas() - ) + actual = images_mm_df["blob_col"].blob.size().to_pandas() expected = pd.Series([338390, 43333], name="size") pd.testing.assert_series_equal( @@ -116,9 +102,7 @@ def test_blob_size(images_mm_df: bpd.DataFrame): def test_blob_updated(images_mm_df: bpd.DataFrame): - actual = bbq.json_value( - images_mm_df["blob_col"].struct.field("details"), "$.updated" - ).to_pandas() + actual = images_mm_df["blob_col"].blob.updated().to_pandas() expected = pd.Series( [ pd.Timestamp("2025-07-30 20:37:31", tz="UTC"), diff --git a/bigframes/extensions/bigframes/__init__.py b/tests/system/small/blob/test_urls.py similarity index 56% rename from bigframes/extensions/bigframes/__init__.py rename to tests/system/small/blob/test_urls.py index 439a8189ded..02a76587f5f 100644 --- a/bigframes/extensions/bigframes/__init__.py +++ b/tests/system/small/blob/test_urls.py @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,16 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from bigframes.extensions.bigframes.dataframe_accessor import ( - BigframesAIAccessor, - BigframesBigQueryDataFrameAccessor, -) -from bigframes.extensions.bigframes.series_accessor import ( - BigframesBigQuerySeriesAccessor, -) +import bigframes.pandas as bpd -__all__ = [ - "BigframesAIAccessor", - "BigframesBigQueryDataFrameAccessor", - "BigframesBigQuerySeriesAccessor", -] + +def test_blob_read_url(images_mm_df: bpd.DataFrame): + urls = images_mm_df["blob_col"].blob.read_url() + + assert urls.str.startswith("https://storage.googleapis.com/").all() + + +def test_blob_write_url(images_mm_df: bpd.DataFrame): + urls = images_mm_df["blob_col"].blob.write_url() + + assert urls.str.startswith("https://storage.googleapis.com/").all() diff --git a/tests/system/small/core/indexes/test_base.py b/tests/system/small/core/indexes/test_base.py index 3225f643299..05ea40cfb9c 100644 --- a/tests/system/small/core/indexes/test_base.py +++ b/tests/system/small/core/indexes/test_base.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from packaging import version import pandas as pd import pandas.testing import pytest -from packaging import version @pytest.mark.parametrize("level", [None, 0, 1, "level0", "level1"]) diff --git a/tests/system/small/core/logging/test_data_types.py b/tests/system/small/core/logging/test_data_types.py index d69e17cfff8..7e197a96727 100644 --- a/tests/system/small/core/logging/test_data_types.py +++ b/tests/system/small/core/logging/test_data_types.py @@ -17,9 +17,9 @@ import pandas as pd import pyarrow as pa -import bigframes.pandas as bpd from bigframes import dtypes from bigframes.core.logging import data_types +import bigframes.pandas as bpd def encode_types(inputs: Sequence[dtypes.Dtype]) -> str: diff --git a/tests/system/small/core/test_convert.py b/tests/system/small/core/test_convert.py index f63f945ad24..7ce0dd47ba2 100644 --- a/tests/system/small/core/test_convert.py +++ b/tests/system/small/core/test_convert.py @@ -13,9 +13,9 @@ # limitations under the License. +from pandas import testing import pandas as pd import pytest -from pandas import testing from bigframes import dataframe from bigframes.core import convert diff --git a/tests/system/small/core/test_reshape.py b/tests/system/small/core/test_reshape.py index aba9bf01859..0850bf50bb8 100644 --- a/tests/system/small/core/test_reshape.py +++ b/tests/system/small/core/test_reshape.py @@ -13,9 +13,9 @@ # limitations under the License. import pandas as pd +import pandas.testing import pytest -import bigframes.testing.utils from bigframes import session from bigframes.core.reshape import merge @@ -56,7 +56,7 @@ def test_join_with_index( how=how, ) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) diff --git a/tests/system/small/engines/conftest.py b/tests/system/small/engines/conftest.py index 823ba9806d5..a775731cde8 100644 --- a/tests/system/small/engines/conftest.py +++ b/tests/system/small/engines/conftest.py @@ -14,10 +14,9 @@ import pathlib from typing import Generator -import google.cloud.bigquery_storage_v1 +from google.cloud import bigquery import pandas as pd import pytest -from google.cloud import bigquery import bigframes from bigframes.core import ArrayValue, events, local_data @@ -45,53 +44,21 @@ def fake_session() -> Generator[bigframes.Session, None, None]: yield session -@pytest.fixture(scope="session") -def pyarrow_engine(): - return local_scan_executor.LocalScanExecutor() - - -@pytest.fixture(scope="session") -def polars_engine(): - return polars_executor.PolarsExecutor() - - -@pytest.fixture(scope="session") -def bq_engine( - bigquery_client: bigquery.Client, - bigquery_storage_read_client: google.cloud.bigquery_storage_v1.BigQueryReadClient, -): - return direct_gbq_execution.DirectGbqExecutor( - bigquery_client, - bqstoragereadclient=bigquery_storage_read_client, - publisher=events.Publisher(), - compiler="ibis", - ) - - -@pytest.fixture(scope="session") -def sqlglot_engine( - bigquery_client: bigquery.Client, - bigquery_storage_read_client: google.cloud.bigquery_storage_v1.BigQueryReadClient, -) -> semi_executor.SemiExecutor: - return direct_gbq_execution.DirectGbqExecutor( - bigquery_client, - bqstoragereadclient=bigquery_storage_read_client, - publisher=events.Publisher(), - ) - - @pytest.fixture(scope="session", params=["pyarrow", "polars", "bq", "bq-sqlglot"]) -def engine( - request, pyarrow_engine, polars_engine, bq_engine, sqlglot_engine -) -> semi_executor.SemiExecutor: +def engine(request, bigquery_client: bigquery.Client) -> semi_executor.SemiExecutor: if request.param == "pyarrow": - return pyarrow_engine + return local_scan_executor.LocalScanExecutor() if request.param == "polars": - return polars_engine + return polars_executor.PolarsExecutor() + publisher = events.Publisher() if request.param == "bq": - return bq_engine + return direct_gbq_execution.DirectGbqExecutor( + bigquery_client, publisher=publisher + ) if request.param == "bq-sqlglot": - return sqlglot_engine + return direct_gbq_execution.DirectGbqExecutor( + bigquery_client, compiler="sqlglot", publisher=publisher + ) raise ValueError(f"Unrecognized param: {request.param}") diff --git a/tests/system/small/engines/test_aggregation.py b/tests/system/small/engines/test_aggregation.py index 669eae9ebf7..4ed826d2aed 100644 --- a/tests/system/small/engines/test_aggregation.py +++ b/tests/system/small/engines/test_aggregation.py @@ -12,18 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest from google.cloud import bigquery +import pytest -import bigframes.operations.aggregations as agg_ops from bigframes.core import ( agg_expressions, array_value, + events, expression, identifiers, nodes, ) -from bigframes.session import polars_executor +import bigframes.operations.aggregations as agg_ops +from bigframes.session import direct_gbq_execution, polars_executor from bigframes.testing.engine_utils import assert_equivalence_execution pytest.importorskip("polars") @@ -127,14 +128,19 @@ def test_engines_unary_variance_aggregates( def test_sql_engines_median_op_aggregates( scalars_array_value: array_value.ArrayValue, bigquery_client: bigquery.Client, - bq_engine, - sqlglot_engine, ): node = apply_agg_to_all_valid( scalars_array_value, agg_ops.MedianOp(), ).node - assert_equivalence_execution(node, bq_engine, sqlglot_engine) + publisher = events.Publisher() + left_engine = direct_gbq_execution.DirectGbqExecutor( + bigquery_client, publisher=publisher + ) + right_engine = direct_gbq_execution.DirectGbqExecutor( + bigquery_client, compiler="sqlglot", publisher=publisher + ) + assert_equivalence_execution(node, left_engine, right_engine) @pytest.mark.parametrize("engine", ["polars", "bq", "bq-sqlglot"], indirect=True) diff --git a/tests/system/small/engines/test_array_ops.py b/tests/system/small/engines/test_array_ops.py index 159f23f48d6..3b80cb8854f 100644 --- a/tests/system/small/engines/test_array_ops.py +++ b/tests/system/small/engines/test_array_ops.py @@ -14,9 +14,9 @@ import pytest +from bigframes.core import array_value, expression import bigframes.operations as ops import bigframes.operations.aggregations as agg_ops -from bigframes.core import array_value, expression from bigframes.session import polars_executor from bigframes.testing.engine_utils import assert_equivalence_execution diff --git a/tests/system/small/engines/test_bool_ops.py b/tests/system/small/engines/test_bool_ops.py index a6ef702885b..a77d52b3560 100644 --- a/tests/system/small/engines/test_bool_ops.py +++ b/tests/system/small/engines/test_bool_ops.py @@ -16,8 +16,8 @@ import pytest -import bigframes.operations as ops from bigframes.core import array_value +import bigframes.operations as ops from bigframes.session import polars_executor from bigframes.testing.engine_utils import assert_equivalence_execution diff --git a/tests/system/small/engines/test_comparison_ops.py b/tests/system/small/engines/test_comparison_ops.py index cd6ece55863..0fcc48b10a9 100644 --- a/tests/system/small/engines/test_comparison_ops.py +++ b/tests/system/small/engines/test_comparison_ops.py @@ -12,15 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio import itertools import pytest -import bigframes.operations as ops from bigframes.core import array_value +import bigframes.operations as ops from bigframes.session import polars_executor -from bigframes.testing.engine_utils import SPEC, assert_equivalence_execution +from bigframes.testing.engine_utils import assert_equivalence_execution pytest.importorskip("polars") @@ -69,16 +68,3 @@ def test_engines_project_comparison_op( # bool col actually doesn't work properly for bq engine arr = apply_op_pairwise(scalars_array_value, op, excluded_cols=["string_col"]) assert_equivalence_execution(arr.node, REFERENCE_ENGINE, engine) - - -@pytest.mark.parametrize("engine", ["bq-sqlglot"], indirect=True) -def test_engines_precedence_like_and_in( - scalars_array_value: array_value.ArrayValue, engine -): - exprs = [ - ops.eq_op.as_expr("bool_col", ops.StrContainsOp("a").as_expr("string_col")), - ] - arr, _ = scalars_array_value.compute_values(exprs) - res = asyncio.run(engine.execute(arr.node, SPEC)) - assert res is not None - assert len(res.batches().to_pandas()) > 0 diff --git a/tests/system/small/engines/test_filtering.py b/tests/system/small/engines/test_filtering.py index fcb85aa8859..817bb4c3f7f 100644 --- a/tests/system/small/engines/test_filtering.py +++ b/tests/system/small/engines/test_filtering.py @@ -13,8 +13,8 @@ # limitations under the License. import pytest -import bigframes.operations as ops from bigframes.core import array_value, expression, nodes +import bigframes.operations as ops from bigframes.session import polars_executor from bigframes.testing.engine_utils import assert_equivalence_execution diff --git a/tests/system/small/engines/test_generic_ops.py b/tests/system/small/engines/test_generic_ops.py index 96beb51f99d..c0469ed97a7 100644 --- a/tests/system/small/engines/test_generic_ops.py +++ b/tests/system/small/engines/test_generic_ops.py @@ -16,9 +16,9 @@ import pytest +from bigframes.core import array_value, expression import bigframes.dtypes import bigframes.operations as ops -from bigframes.core import array_value, expression from bigframes.session import polars_executor from bigframes.testing.engine_utils import assert_equivalence_execution @@ -263,16 +263,16 @@ def test_engines_astype_time(scalars_array_value: array_value.ArrayValue, engine @pytest.mark.parametrize("engine", ["polars", "bq", "bq-sqlglot"], indirect=True) def test_engines_astype_from_json(scalars_array_value: array_value.ArrayValue, engine): exprs = [ - ops.JSONDecode(to_type=bigframes.dtypes.INT_DTYPE).as_expr( + ops.AsTypeOp(to_type=bigframes.dtypes.INT_DTYPE).as_expr( expression.const("5", bigframes.dtypes.JSON_DTYPE) ), - ops.JSONDecode(to_type=bigframes.dtypes.FLOAT_DTYPE).as_expr( + ops.AsTypeOp(to_type=bigframes.dtypes.FLOAT_DTYPE).as_expr( expression.const("5", bigframes.dtypes.JSON_DTYPE) ), - ops.JSONDecode(to_type=bigframes.dtypes.BOOL_DTYPE).as_expr( + ops.AsTypeOp(to_type=bigframes.dtypes.BOOL_DTYPE).as_expr( expression.const("true", bigframes.dtypes.JSON_DTYPE) ), - ops.JSONDecode(to_type=bigframes.dtypes.STRING_DTYPE).as_expr( + ops.AsTypeOp(to_type=bigframes.dtypes.STRING_DTYPE).as_expr( expression.const('"hello world"', bigframes.dtypes.JSON_DTYPE) ), ] @@ -284,32 +284,17 @@ def test_engines_astype_from_json(scalars_array_value: array_value.ArrayValue, e @pytest.mark.parametrize("engine", ["polars", "bq", "bq-sqlglot"], indirect=True) def test_engines_astype_to_json(scalars_array_value: array_value.ArrayValue, engine): exprs = [ - ops.ToJSON().as_expr(expression.deref("int64_col")), - ops.ToJSON().as_expr( - # Use a const since float to json has precision issues - expression.const(5.2, bigframes.dtypes.FLOAT_DTYPE) - ), - ops.ToJSON().as_expr(expression.deref("bool_col")), - ops.ToJSON().as_expr( - # Use a const since "str_col" has special chars. - expression.const('"hello world"', bigframes.dtypes.STRING_DTYPE) + ops.AsTypeOp(to_type=bigframes.dtypes.JSON_DTYPE).as_expr( + expression.deref("int64_col") ), - ] - arr, _ = scalars_array_value.compute_values(exprs) - - assert_equivalence_execution(arr.node, REFERENCE_ENGINE, engine) - - -@pytest.mark.parametrize("engine", ["polars", "bq", "bq-sqlglot"], indirect=True) -def test_engines_to_json_string(scalars_array_value: array_value.ArrayValue, engine): - exprs = [ - ops.ToJSONString().as_expr(expression.deref("int64_col")), - ops.ToJSONString().as_expr( + ops.AsTypeOp(to_type=bigframes.dtypes.JSON_DTYPE).as_expr( # Use a const since float to json has precision issues expression.const(5.2, bigframes.dtypes.FLOAT_DTYPE) ), - ops.ToJSONString().as_expr(expression.deref("bool_col")), - ops.ToJSONString().as_expr( + ops.AsTypeOp(to_type=bigframes.dtypes.JSON_DTYPE).as_expr( + expression.deref("bool_col") + ), + ops.AsTypeOp(to_type=bigframes.dtypes.JSON_DTYPE).as_expr( # Use a const since "str_col" has special chars. expression.const('"hello world"', bigframes.dtypes.STRING_DTYPE) ), @@ -424,39 +409,6 @@ def test_engines_notnull_op(scalars_array_value: array_value.ArrayValue, engine) assert_equivalence_execution(arr.node, REFERENCE_ENGINE, engine) -@pytest.mark.parametrize("engine", ["polars", "bq", "bq-sqlglot"], indirect=True) -def test_engines_coerce_to_bool_op_scalars( - scalars_array_value: array_value.ArrayValue, engine -): - arr, _ = scalars_array_value.compute_values( - [ - ops.coerce_to_bool_op.as_expr(expression.deref("bool_col")), - ops.coerce_to_bool_op.as_expr(expression.deref("int64_col")), - ops.coerce_to_bool_op.as_expr(expression.deref("float64_col")), - ops.coerce_to_bool_op.as_expr(expression.deref("string_col")), - ops.coerce_to_bool_op.as_expr(expression.deref("bytes_col")), - ] - ) - - assert_equivalence_execution(arr.node, REFERENCE_ENGINE, engine) - - -@pytest.mark.parametrize("engine", ["polars", "bq", "bq-sqlglot"], indirect=True) -def test_engines_coerce_to_bool_op_arrays( - arrays_array_value: array_value.ArrayValue, engine -): - arr, _ = arrays_array_value.compute_values( - [ - ops.coerce_to_bool_op.as_expr(expression.deref("int_list_col")), - ops.coerce_to_bool_op.as_expr(expression.deref("bool_list_col")), - ops.coerce_to_bool_op.as_expr(expression.deref("float_list_col")), - ops.coerce_to_bool_op.as_expr(expression.deref("string_list_col")), - ] - ) - - assert_equivalence_execution(arr.node, REFERENCE_ENGINE, engine) - - @pytest.mark.parametrize("engine", ["polars", "bq", "bq-sqlglot"], indirect=True) def test_engines_invert_op(scalars_array_value: array_value.ArrayValue, engine): arr, _ = scalars_array_value.compute_values( @@ -515,21 +467,3 @@ def test_engines_isin_op_nested_filter( arr = scalars_array_value.filter(filter_clause) assert_equivalence_execution(arr.node, REFERENCE_ENGINE, engine) - - -@pytest.mark.parametrize("engine", ["polars", "bq", "bq-sqlglot"], indirect=True) -def test_engines_getitem_ops(arrays_array_value: array_value.ArrayValue, engine): - arr, _ = arrays_array_value.compute_values( - [ - ops.GetItemOp(0).as_expr(expression.deref("float_list_col")), - ops.DynamicGetItemOp().as_expr( - expression.deref("float_list_col"), expression.const(0) - ), - ops.GetItemOp(0).as_expr(expression.deref("string_list_col")), - ops.DynamicGetItemOp().as_expr( - expression.deref("string_list_col"), expression.const(0) - ), - ] - ) - - assert_equivalence_execution(arr.node, REFERENCE_ENGINE, engine) diff --git a/tests/system/small/engines/test_googlesql_ops.py b/tests/system/small/engines/test_googlesql_ops.py deleted file mode 100644 index e47308fa355..00000000000 --- a/tests/system/small/engines/test_googlesql_ops.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import pytest - -import bigframes.operations.googlesql as gsql_ops -from bigframes.core import array_value -from bigframes.session import polars_executor -from bigframes.testing.engine_utils import assert_equivalence_execution - -polars = pytest.importorskip("polars") - -# Polars used as reference as its fast and local. Generally though, prefer gbq engine where they disagree. -REFERENCE_ENGINE = polars_executor.PolarsExecutor() - - -def test_engines_googlesql_st_area( - scalars_array_value: array_value.ArrayValue, bq_engine, sqlglot_engine -): - expr = gsql_ops.ST_AREA.as_expr("geography_col") - - arr, _ = scalars_array_value.compute_values([expr]) - - assert_equivalence_execution(arr.node, bq_engine, sqlglot_engine) diff --git a/tests/system/small/engines/test_numeric_ops.py b/tests/system/small/engines/test_numeric_ops.py index c188e37370c..ef0f8d9d0d8 100644 --- a/tests/system/small/engines/test_numeric_ops.py +++ b/tests/system/small/engines/test_numeric_ops.py @@ -17,8 +17,8 @@ import pytest -import bigframes.operations as ops from bigframes.core import array_value, expression +import bigframes.operations as ops from bigframes.session import polars_executor from bigframes.testing.engine_utils import assert_equivalence_execution @@ -53,47 +53,6 @@ def apply_op_pairwise( return new_arr -def apply_op( - array: array_value.ArrayValue, op: ops.UnaryOp, excluded_cols=[] -) -> array_value.ArrayValue: - exprs = [] - labels = [] - for arg in array.column_ids: - if arg in excluded_cols: - continue - try: - _ = op.output_type(array.get_column_type(arg)) - expr = op.as_expr(arg) - exprs.append(expr) - labels.append(f"{arg}_{op.name}") - except TypeError: - continue - assert len(exprs) > 0 - new_arr, ids = array.compute_values(exprs) - new_arr = new_arr.rename_columns( - {new_col: label for new_col, label in zip(ids, labels)} - ) - return new_arr - - -@pytest.mark.parametrize("engine", ["polars", "bq"], indirect=True) -def test_engines_project_ceil( - scalars_array_value: array_value.ArrayValue, - engine, -): - arr = apply_op(scalars_array_value, ops.ceil_op) - assert_equivalence_execution(arr.node, REFERENCE_ENGINE, engine) - - -@pytest.mark.parametrize("engine", ["polars", "bq"], indirect=True) -def test_engines_project_floor( - scalars_array_value: array_value.ArrayValue, - engine, -): - arr = apply_op(scalars_array_value, ops.floor_op) - assert_equivalence_execution(arr.node, REFERENCE_ENGINE, engine) - - @pytest.mark.parametrize("engine", ["polars", "bq", "bq-sqlglot"], indirect=True) def test_engines_project_add( scalars_array_value: array_value.ArrayValue, diff --git a/tests/system/small/engines/test_sorting.py b/tests/system/small/engines/test_sorting.py index cbb6215adae..ec1c0d95ee3 100644 --- a/tests/system/small/engines/test_sorting.py +++ b/tests/system/small/engines/test_sorting.py @@ -12,13 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio - import pytest -import bigframes.operations as bf_ops from bigframes.core import array_value, nodes, ordering -from bigframes.session import execution_spec, polars_executor +import bigframes.operations as bf_ops +from bigframes.session import polars_executor from bigframes.testing.engine_utils import assert_equivalence_execution pytest.importorskip("polars") @@ -98,10 +96,7 @@ def test_polars_engines_skips_unrecognized_order_expr( ), ) node = nodes.OrderByNode(node, ORDER_EXPRESSIONS) - result = asyncio.run( - engine.execute(node, execution_spec.ExecutionSpec(ordered=True)) - ) - assert result is None + assert engine.execute(node, ordered=True) is None def apply_reverse(node: nodes.BigFrameNode) -> nodes.BigFrameNode: diff --git a/tests/system/small/engines/test_strings.py b/tests/system/small/engines/test_strings.py index 32a8c4bcd78..d4504745048 100644 --- a/tests/system/small/engines/test_strings.py +++ b/tests/system/small/engines/test_strings.py @@ -14,8 +14,8 @@ import pytest -import bigframes.operations as ops from bigframes.core import array_value +import bigframes.operations as ops from bigframes.session import polars_executor from bigframes.testing.engine_utils import assert_equivalence_execution diff --git a/tests/system/small/engines/test_temporal_ops.py b/tests/system/small/engines/test_temporal_ops.py index 61b1b06b1f2..66edfeddcc3 100644 --- a/tests/system/small/engines/test_temporal_ops.py +++ b/tests/system/small/engines/test_temporal_ops.py @@ -12,17 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio - -import pandas as pd import pytest -import bigframes.operations as ops -from bigframes import dtypes from bigframes.core import array_value -from bigframes.core import expression as ex +import bigframes.operations as ops from bigframes.session import polars_executor -from bigframes.testing.engine_utils import SPEC, assert_equivalence_execution +from bigframes.testing.engine_utils import assert_equivalence_execution pytest.importorskip("polars") @@ -69,45 +64,3 @@ def test_engines_date_accessors(scalars_array_value: array_value.ArrayValue, eng arr, _ = scalars_array_value.compute_values(exprs) assert_equivalence_execution(arr.node, REFERENCE_ENGINE, engine) - - -@pytest.mark.parametrize("engine", ["bq", "bq-sqlglot"], indirect=True) -def test_engines_temporal_arithmetic( - scalars_array_value: array_value.ArrayValue, engine -): - exprs = [ - ops.timestamp_add_op.as_expr( - "timestamp_col", ex.const(pd.Timedelta(seconds=1), dtypes.TIMEDELTA_DTYPE) - ), - ops.timestamp_sub_op.as_expr( - "timestamp_col", ex.const(pd.Timedelta(seconds=1), dtypes.TIMEDELTA_DTYPE) - ), - ops.date_add_op.as_expr( - "date_col", ex.const(pd.Timedelta(days=1), dtypes.TIMEDELTA_DTYPE) - ), - ops.date_sub_op.as_expr( - "date_col", ex.const(pd.Timedelta(days=1), dtypes.TIMEDELTA_DTYPE) - ), - ops.timestamp_diff_op.as_expr("timestamp_col", "timestamp_col"), - ops.date_diff_op.as_expr("date_col", "date_col"), - ] - - arr, _ = scalars_array_value.compute_values(exprs) - res = asyncio.run(engine.execute(arr.node, SPEC)) - assert res is not None - assert len(res.batches().to_pandas()) > 0 - - -@pytest.mark.parametrize("engine", ["bq", "bq-sqlglot"], indirect=True) -def test_engines_to_datetime(scalars_array_value: array_value.ArrayValue, engine): - exprs = [ - ops.ToDatetimeOp().as_expr("timestamp_col"), - ] - arr, _ = scalars_array_value.compute_values(exprs) - res = asyncio.run(engine.execute(arr.node, SPEC)) - assert res is not None - df = res.batches().to_pandas() - # The input timestamp was: TIMESTAMP('2021-07-21T17:43:43.945289+00:00') - # The output should be naive DATETIME('2021-07-21T17:43:43.945289') - val = df.iloc[0, -1] - assert pd.Timestamp(val) == pd.Timestamp("2021-07-21T17:43:43.945289") diff --git a/tests/system/small/engines/test_windowing.py b/tests/system/small/engines/test_windowing.py index 8235fe0ef6b..5e4a94d9003 100644 --- a/tests/system/small/engines/test_windowing.py +++ b/tests/system/small/engines/test_windowing.py @@ -12,18 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +from google.cloud import bigquery import pytest -import bigframes.operations.aggregations as agg_ops from bigframes.core import ( agg_expressions, array_value, + events, expression, identifiers, nodes, window_spec, ) -from bigframes.session import polars_executor +import bigframes.operations.aggregations as agg_ops +from bigframes.session import direct_gbq_execution, polars_executor from bigframes.testing.engine_utils import assert_equivalence_execution pytest.importorskip("polars") @@ -44,9 +46,8 @@ def test_engines_with_offsets( @pytest.mark.parametrize("agg_op", [agg_ops.sum_op, agg_ops.count_op]) def test_engines_with_rows_window( scalars_array_value: array_value.ArrayValue, + bigquery_client: bigquery.Client, agg_op, - bq_engine, - sqlglot_engine, ): window = window_spec.WindowSpec( bounds=window_spec.RowsWindowBounds.from_window_size(3, "left"), @@ -61,4 +62,12 @@ def test_engines_with_rows_window( ), window_spec=window, ) - assert_equivalence_execution(window_node, bq_engine, sqlglot_engine) + + publisher = events.Publisher() + bq_executor = direct_gbq_execution.DirectGbqExecutor( + bigquery_client, publisher=publisher + ) + bq_sqlgot_executor = direct_gbq_execution.DirectGbqExecutor( + bigquery_client, compiler="sqlglot", publisher=publisher + ) + assert_equivalence_execution(window_node, bq_executor, bq_sqlgot_executor) diff --git a/tests/system/small/functions/test_remote_function.py b/tests/system/small/functions/test_remote_function.py index 869b26ca38c..1ee60dafd66 100644 --- a/tests/system/small/functions/test_remote_function.py +++ b/tests/system/small/functions/test_remote_function.py @@ -19,40 +19,26 @@ import bigframes_vendored.constants as constants import google.api_core.exceptions +from google.cloud import bigquery import pandas import pandas as pd import pyarrow import pytest import test_utils.prefixer -from google.cloud import bigquery import bigframes import bigframes.clients import bigframes.core.events import bigframes.dtypes import bigframes.exceptions -import bigframes.session._io.bigquery from bigframes.functions import _utils as bff_utils from bigframes.functions import function as bff -from bigframes.testing.utils import assert_frame_equal, assert_series_equal +import bigframes.session._io.bigquery +from bigframes.testing.utils import assert_frame_equal, get_function_name _prefixer = test_utils.prefixer.Prefixer("bigframes", "") -def get_function_name(func, package_requirements=None, is_row_processor=False): - """Get a bigframes function name for testing given a udf.""" - # Augment user package requirements with any internal package - # requirements. - package_requirements = bff_utils.get_updated_package_requirements( - package_requirements or [], is_row_processor - ) - - # Compute a unique hash representing the user code. - function_hash = bff_utils.get_hash(func, package_requirements) - - return f"bigframes_{function_hash}" - - @pytest.fixture(scope="module") def bq_cf_connection() -> str: """Pre-created BQ connection in the test project in US location, used to @@ -116,8 +102,12 @@ def get_bq_connection_id_path_format(connection_id_dot_format): return f"projects/{fields[0]}/locations/{fields[1]}/connections/{fields[2]}" -@pytest.mark.flaky(retries=2, delay=120) +# @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_direct_no_session_param( + bigquery_client, + bigqueryconnection_client, + cloudfunctions_client, + resourcemanager_client, scalars_dfs, dataset_id_permanent, bq_cf_connection, @@ -128,6 +118,10 @@ def square(x): square = bff.remote_function( input_types=int, output_type=int, + bigquery_client=bigquery_client, + bigquery_connection_client=bigqueryconnection_client, + cloud_functions_client=cloudfunctions_client, + resource_manager_client=resourcemanager_client, dataset=dataset_id_permanent, bigquery_connection=bq_cf_connection, # See e2e tests for tests that actually deploy the Cloud Function. @@ -178,9 +172,10 @@ def test_remote_function_connection_w_location( def square(x): return x * x - square = session.remote_function( + square = bff.remote_function( input_types=int, output_type=int, + session=session, dataset=dataset_id_permanent, bigquery_connection=bq_cf_connection_location, # See e2e tests for tests that actually deploy the Cloud Function. @@ -246,9 +241,10 @@ def square(x): "The location does not match BigQuery connection location:" ), ): - session.remote_function( + bff.remote_function( input_types=int, output_type=int, + session=session, dataset=dataset_id_permanent, bigquery_connection=connection_id, # See e2e tests for tests that actually deploy the Cloud Function. @@ -268,9 +264,10 @@ def test_remote_function_connection_w_location_project( def square(x): return x * x - square = session.remote_function( + square = bff.remote_function( input_types=int, output_type=int, + session=session, dataset=dataset_id_permanent, bigquery_connection=bq_cf_connection_location_project, # See e2e tests for tests that actually deploy the Cloud Function. @@ -338,9 +335,10 @@ def square(x): "The project_id does not match BigQuery connection gcp_project_id:" ), ): - session.remote_function( + bff.remote_function( input_types=int, output_type=int, + session=session, dataset=dataset_id_permanent, bigquery_connection=connection_id, # See e2e tests for tests that actually deploy the Cloud Function. @@ -350,6 +348,49 @@ def square(x): )(square) +@pytest.mark.flaky(retries=2, delay=120) +def test_remote_function_direct_session_param( + session_with_bq_connection, scalars_dfs, dataset_id_permanent +): + def square(x): + return x * x + + square = bff.remote_function( + input_types=int, + output_type=int, + session=session_with_bq_connection, + dataset=dataset_id_permanent, + name=get_function_name(square), + cloud_function_service_account="default", + )(square) + + # Function should still work normally. + assert square(2) == 4 + + scalars_df, scalars_pandas_df = scalars_dfs + + bf_int64_col = scalars_df["int64_col"] + bf_int64_col_filter = bf_int64_col.notnull() + bf_int64_col_filtered = bf_int64_col[bf_int64_col_filter] + bf_result_col = bf_int64_col_filtered.apply(square) + bf_result = ( + bf_int64_col_filtered.to_frame().assign(result=bf_result_col).to_pandas() + ) + + pd_int64_col = scalars_pandas_df["int64_col"] + pd_int64_col_filter = pd_int64_col.notnull() + pd_int64_col_filtered = pd_int64_col[pd_int64_col_filter] + pd_result_col = pd_int64_col_filtered.apply(lambda x: x * x) + # TODO(shobs): Figure why pandas .apply() changes the dtype, i.e. + # pd_int64_col_filtered.dtype is Int64Dtype() + # pd_int64_col_filtered.apply(lambda x: x * x).dtype is int64. + # For this test let's force the pandas dtype to be same as bigframes' dtype. + pd_result_col = pd_result_col.astype(pd.Int64Dtype()) + pd_result = pd_int64_col_filtered.to_frame().assign(result=pd_result_col) + + assert_frame_equal(bf_result, pd_result) + + @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_via_session_default( session_with_bq_connection, scalars_dfs, dataset_id_permanent @@ -468,12 +509,7 @@ def add_one(x): pd_int64_df = scalars_pandas_df[int64_cols] pd_int64_df_filtered = pd_int64_df.dropna() - - # TODO(swast): Remove when pandas 2.1.x+ is the minimum supported. - if hasattr(pd_int64_df_filtered, "map"): - pd_result = pd_int64_df_filtered.map(add_one) - else: - pd_result = pd_int64_df_filtered.applymap(add_one) + pd_result = pd_int64_df_filtered.applymap(add_one) # TODO(shobs): Figure why pandas .applymap() changes the dtype, i.e. # pd_int64_df_filtered.dtype is Int64Dtype() # pd_int64_df_filtered.applymap(lambda x: x).dtype is int64. @@ -508,13 +544,7 @@ def add_one(x): pd_int64_df = scalars_pandas_df[int64_cols] pd_int64_df_filtered = pd_int64_df[pd_int64_df["int64_col"].notnull()] - - # TODO(swast): Remove when pandas 2.1.x+ is the minimum supported. - if hasattr(pd_int64_df_filtered, "map"): - pd_result = pd_int64_df_filtered.map(add_one) - else: - pd_result = pd_int64_df_filtered.applymap(add_one) - + pd_result = pd_int64_df_filtered.applymap(add_one) # TODO(shobs): Figure why pandas .applymap() changes the dtype, i.e. # pd_int64_df_filtered.dtype is Int64Dtype() # pd_int64_df_filtered.applymap(lambda x: x).dtype is int64. @@ -547,13 +577,7 @@ def add_one(x): bf_result = bf_int64_df.applymap(remote_add_one, na_action="ignore").to_pandas() pd_int64_df = scalars_pandas_df[int64_cols] - - # TODO(swast): Remove when pandas 2.1.x+ is the minimum supported. - if hasattr(pd_int64_df, "map"): - pd_result = pd_int64_df.map(add_one, na_action="ignore") - else: - pd_result = pd_int64_df.applymap(add_one, na_action="ignore") - + pd_result = pd_int64_df.applymap(add_one, na_action="ignore") # TODO(shobs): Figure why pandas .applymap() changes the dtype, i.e. # pd_int64_df_filtered.dtype is Int64Dtype() # pd_int64_df_filtered.applymap(lambda x: x).dtype is int64. @@ -593,7 +617,7 @@ def bytes_to_hex(mybytes: bytes) -> bytes: )(bytes_to_hex) bf_result = scalars_df.bytes_col.map(remote_bytes_to_hex).to_pandas() - assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -643,8 +667,9 @@ def add_one(x): def test_read_gbq_function_detects_invalid_function(session, dataset_id): dataset_ref = bigquery.DatasetReference.from_string(dataset_id) with pytest.raises(ValueError) as e: - session.read_gbq_function( + bff.read_gbq_function( str(dataset_ref.routine("not_a_function")), + session=session, ) assert "Unknown function" in str(e.value) @@ -653,6 +678,10 @@ def test_read_gbq_function_detects_invalid_function(session, dataset_id): @pytest.mark.flaky(retries=2, delay=120) def test_read_gbq_function_like_original( session, + bigquery_client, + bigqueryconnection_client, + cloudfunctions_client, + resourcemanager_client, scalars_df_index, dataset_id_permanent, bq_cf_connection, @@ -663,7 +692,11 @@ def square1(x): square1 = bff.remote_function( input_types=[int], output_type=int, + bigquery_client=bigquery_client, + bigquery_connection_client=bigqueryconnection_client, dataset=dataset_id_permanent, + cloud_functions_client=cloudfunctions_client, + resource_manager_client=resourcemanager_client, bigquery_connection=bq_cf_connection, reuse=True, name=get_function_name(square1), @@ -673,8 +706,9 @@ def square1(x): # Function should still work normally. assert square1(2) == 4 - square2 = session.read_gbq_function( + square2 = bff.read_gbq_function( function_name=square1.bigframes_bigquery_function, # type: ignore + session=session, ) # The newly-created function (square1) should have a remote function AND a @@ -720,7 +754,7 @@ def test_read_gbq_function_runs_existing_udf_4_params(session): def test_read_gbq_function_runs_existing_udf_array_output(session, routine_id_unique): - bigframes.session._io.bigquery.start_query_with_job( + bigframes.session._io.bigquery.start_query_with_client( session.bqclient, textwrap.dedent( f""" @@ -736,6 +770,7 @@ def test_read_gbq_function_runs_existing_udf_array_output(session, routine_id_un project=None, timeout=None, metrics=None, + query_with_job=True, publisher=bigframes.core.events.Publisher(), ) func = session.read_gbq_function(routine_id_unique) @@ -750,7 +785,7 @@ def test_read_gbq_function_runs_existing_udf_array_output(session, routine_id_un pd_result = pd_s.apply(func) bf_result = bf_s.apply(func) assert bigframes.dtypes.is_array_string_like(bf_result.dtype) - assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result.to_pandas(), check_dtype=False, check_index_type=False ) @@ -758,7 +793,7 @@ def test_read_gbq_function_runs_existing_udf_array_output(session, routine_id_un def test_read_gbq_function_runs_existing_udf_2_params_array_output( session, routine_id_unique ): - bigframes.session._io.bigquery.start_query_with_job( + bigframes.session._io.bigquery.start_query_with_client( session.bqclient, textwrap.dedent( f""" @@ -774,6 +809,7 @@ def test_read_gbq_function_runs_existing_udf_2_params_array_output( project=None, timeout=None, metrics=None, + query_with_job=True, publisher=bigframes.core.events.Publisher(), ) func = session.read_gbq_function(routine_id_unique) @@ -790,7 +826,7 @@ def test_read_gbq_function_runs_existing_udf_2_params_array_output( pd_result = pd_df["col0"].combine(pd_df["col1"], func) bf_result = bf_df["col0"].combine(bf_df["col1"], func) assert bigframes.dtypes.is_array_string_like(bf_result.dtype) - assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result.to_pandas(), check_dtype=False, check_index_type=False ) @@ -798,7 +834,7 @@ def test_read_gbq_function_runs_existing_udf_2_params_array_output( def test_read_gbq_function_runs_existing_udf_4_params_array_output( session, routine_id_unique ): - bigframes.session._io.bigquery.start_query_with_job( + bigframes.session._io.bigquery.start_query_with_client( session.bqclient, textwrap.dedent( f""" @@ -814,6 +850,7 @@ def test_read_gbq_function_runs_existing_udf_4_params_array_output( project=None, timeout=None, metrics=None, + query_with_job=True, publisher=bigframes.core.events.Publisher(), ) func = session.read_gbq_function(routine_id_unique) @@ -844,7 +881,7 @@ def test_read_gbq_function_runs_existing_udf_4_params_array_output( ) bf_result = bf_df.apply(func, axis=1) assert bigframes.dtypes.is_array_string_like(bf_result.dtype) - assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result.to_pandas(), check_dtype=False, check_index_type=False ) @@ -874,8 +911,9 @@ def test_read_gbq_function_reads_udfs(session, bigquery_client, dataset_id): for routine in (sql_routine, js_routine): # Create the routine in BigQuery and read it back using read_gbq_function. bigquery_client.create_routine(routine, exists_ok=True) - square = session.read_gbq_function( + square = bff.read_gbq_function( str(routine.reference), + session=session, ) # It should point to the named routine and yield the expected results. @@ -949,23 +987,27 @@ def test_read_gbq_function_requires_explicit_types( bigquery_client.create_routine(only_arg_type_specified, exists_ok=True) bigquery_client.create_routine(neither_type_specified, exists_ok=True) - session.read_gbq_function( + bff.read_gbq_function( str(both_types_specified.reference), + session=session, ) with pytest.warns( bigframes.exceptions.UnknownDataTypeWarning, match=r"missing input data types[\s\S]*assume default data type", ): - session.read_gbq_function( + bff.read_gbq_function( str(only_return_type_specified.reference), + session=session, ) with pytest.raises(ValueError): - session.read_gbq_function( + bff.read_gbq_function( str(only_arg_type_specified.reference), + session=session, ) with pytest.raises(ValueError): - session.read_gbq_function( + bff.read_gbq_function( str(neither_type_specified.reference), + session=session, ) @@ -1010,7 +1052,7 @@ def test_read_gbq_function_respects_python_output_type( # Create the routine in BigQuery and read it back using read_gbq_function. bigquery_client.create_routine(sql_routine, exists_ok=True) - func = session.read_gbq_function(str(sql_routine.reference)) + func = bff.read_gbq_function(str(sql_routine.reference), session=session) # test that the function works as expected s = bigframes.series.Series([1, 10, 100]) @@ -1018,7 +1060,9 @@ def test_read_gbq_function_respects_python_output_type( actual = s.apply(func).to_pandas() # ignore type disparities, e.g. "int64" in pandas v/s "Int64" in bigframes - assert_series_equal(expected, actual, check_dtype=False, check_index_type=False) + pd.testing.assert_series_equal( + expected, actual, check_dtype=False, check_index_type=False + ) @pytest.mark.parametrize( @@ -1056,7 +1100,7 @@ def test_read_gbq_function_supports_python_output_type_only_for_string_outputs( TypeError, match="An explicit output_type should be provided only for a BigQuery function with STRING output.", ): - session.read_gbq_function(str(sql_routine.reference)) + bff.read_gbq_function(str(sql_routine.reference), session=session) @pytest.mark.parametrize( @@ -1087,7 +1131,7 @@ def test_read_gbq_function_supported_python_output_type( # Create the routine in BigQuery and read it back using read_gbq_function. bigquery_client.create_routine(sql_routine, exists_ok=True) - session.read_gbq_function(str(sql_routine.reference)) + bff.read_gbq_function(str(sql_routine.reference), session=session) @pytest.mark.flaky(retries=2, delay=120) @@ -1156,7 +1200,9 @@ def add_ints(row: pandas.Series) -> int: # bf_result.to_numpy() produces an array of numpy.float64's # (in system_prerelease tests), while pd_result.to_numpy() produces an # array of ints, ignore this mismatch by using check_exact=False. - assert_series_equal(pd_result, bf_result, check_dtype=False, check_exact=False) + pd.testing.assert_series_equal( + pd_result, bf_result, check_dtype=False, check_exact=False + ) # Read back the deployed BQ remote function using read_gbq_function. func_ref = session.read_gbq_function( @@ -1164,17 +1210,14 @@ def add_ints(row: pandas.Series) -> int: is_row_processor=True, ) - assert ( - func_ref.bigframes_remote_function == add_ints_remote.bigframes_remote_function - ) # type: ignore - assert ( - func_ref.bigframes_bigquery_function - == add_ints_remote.bigframes_bigquery_function - ) # type: ignore + assert func_ref.bigframes_remote_function == add_ints_remote.bigframes_remote_function # type: ignore + assert func_ref.bigframes_bigquery_function == add_ints_remote.bigframes_bigquery_function # type: ignore assert func_ref.bigframes_remote_function == func_ref.bigframes_bigquery_function # type: ignore bf_result_gbq = scalars_df[columns].apply(func_ref, axis=1).to_pandas() - assert_series_equal(pd_result, bf_result_gbq, check_dtype=False, check_exact=False) + pd.testing.assert_series_equal( + pd_result, bf_result_gbq, check_dtype=False, check_exact=False + ) @pytest.mark.flaky(retries=2, delay=120) @@ -1210,7 +1253,9 @@ def add_ints(row: pandas.Series) -> int: # bf_result.to_numpy() produces an array of numpy.float64's # (in system_prerelease tests), while pd_result.to_numpy() produces an # array of ints, ignore this mismatch by using check_exact=False. - assert_series_equal(pd_result, bf_result, check_dtype=False, check_exact=False) + pd.testing.assert_series_equal( + pd_result, bf_result, check_dtype=False, check_exact=False + ) @pytest.mark.flaky(retries=2, delay=120) @@ -1241,7 +1286,9 @@ def add_numbers(row): # bf_result.index[0].dtype is 'string[pyarrow]' while # pd_result.index[0].dtype is 'object', ignore this mismatch by using # check_index_type=False. - assert_series_equal(pd_result, bf_result, check_dtype=False, check_index_type=False) + pd.testing.assert_series_equal( + pd_result, bf_result, check_dtype=False, check_index_type=False + ) def test_df_apply_axis_1_unsupported_callable(scalars_dfs): @@ -1290,17 +1337,13 @@ def echo_len(row): dtype = scalars_df[column].dtype - with ( - pytest.raises( - NotImplementedError, - match=re.escape( - f"DataFrame has a column of dtype '{dtype}' which is not supported with axis=1. Supported dtypes are (" - ), - ), - pytest.warns( - bigframes.exceptions.PreviewWarning, - match="axis=1 scenario is in preview.", + with pytest.raises( + NotImplementedError, + match=re.escape( + f"DataFrame has a column of dtype '{dtype}' which is not supported with axis=1. Supported dtypes are (" ), + ), pytest.warns( + bigframes.exceptions.PreviewWarning, match="axis=1 scenario is in preview." ): scalars_df[[column]].apply(echo_len_remote, axis=1) @@ -1409,7 +1452,7 @@ def is_odd(x: int) -> bool: bf_result = bf_method(is_odd_remote).to_pandas() # ignore any dtype difference - assert_series_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) @pytest.mark.flaky(retries=2, delay=120) @@ -1458,7 +1501,7 @@ def add(x: int, y: int) -> int: ) # ignore any dtype difference - assert_series_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) @pytest.mark.flaky(retries=2, delay=120) @@ -1520,7 +1563,7 @@ def add_pandas(s: pd.Series) -> float: bf_result = bf_df[bf_filter].apply(add_remote, axis=1).to_pandas() # ignore any dtype difference - assert_series_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) @pytest.mark.parametrize( @@ -1605,9 +1648,10 @@ def func_tuple(x): ValueError, match=r"must be one of the supported types", ): - session.remote_function( + bff.remote_function( input_types=int, output_type=Sequence[int], + session=session, dataset=dataset_id_permanent, bigquery_connection=bq_cf_connection, reuse=True, diff --git a/tests/system/small/geopandas/test_geoseries.py b/tests/system/small/geopandas/test_geoseries.py index 9f1f830dc68..a2f0759161d 100644 --- a/tests/system/small/geopandas/test_geoseries.py +++ b/tests/system/small/geopandas/test_geoseries.py @@ -18,11 +18,11 @@ import bigframes_vendored.constants as constants import geopandas # type: ignore +from geopandas.array import GeometryDtype # type:ignore import geopandas.testing # type:ignore import google.api_core.exceptions import pandas as pd import pytest -from geopandas.array import GeometryDtype # type:ignore from shapely.geometry import ( # type: ignore GeometryCollection, LineString, diff --git a/tests/system/small/ml/conftest.py b/tests/system/small/ml/conftest.py index 2f84b351e04..c735dbc76b8 100644 --- a/tests/system/small/ml/conftest.py +++ b/tests/system/small/ml/conftest.py @@ -13,8 +13,8 @@ # limitations under the License. import os -import uuid from typing import cast +import uuid import pandas as pd import pytest diff --git a/tests/system/small/ml/test_cluster.py b/tests/system/small/ml/test_cluster.py index 2bf334e84df..2a5e979b30d 100644 --- a/tests/system/small/ml/test_cluster.py +++ b/tests/system/small/ml/test_cluster.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np import pandas as pd -import bigframes.pandas as bpd from bigframes.ml import cluster +import bigframes.pandas as bpd from bigframes.testing.utils import assert_frame_equal _PD_NEW_PENGUINS = pd.DataFrame.from_dict( @@ -142,26 +141,6 @@ def test_kmeans_cluster_centers(penguins_kmeans_model: cluster.KMeans): .sort_values(["centroid_id", "feature"]) .reset_index(drop=True) ) - - # FIX: Helper to ignore row order inside categorical_value lists - # and sign flipping of values inside numerical_value list. - # This prevents the test from failing if BQML returns [MALE, FEMALE] instead of [FEMALE, MALE] - # or 0.197 versus -0.197. - def sort_and_abs_categorical(val): - # Accept BOTH python lists AND numpy arrays - if isinstance(val, (list, np.ndarray)) and len(val) > 0: - # Take abs of value first, then sort - processed = [ - {"category": x["category"], "value": abs(x["value"])} for x in val - ] - return sorted(processed, key=lambda x: x["category"]) - return val - - result["numerical_value"] = result["numerical_value"].abs() - result["categorical_value"] = result["categorical_value"].apply( - sort_and_abs_categorical - ) - expected = ( pd.DataFrame( { @@ -219,18 +198,11 @@ def sort_and_abs_categorical(val): .sort_values(["centroid_id", "feature"]) .reset_index(drop=True) ) - - # Sort and sign flip expected values to match the output of the model. - expected["numerical_value"] = expected["numerical_value"].abs() - expected["categorical_value"] = expected["categorical_value"].apply( - sort_and_abs_categorical - ) - pd.testing.assert_frame_equal( result, expected, check_exact=False, - rtol=0.1, # Keep or slightly increase if numerical drift persists + rtol=0.1, # int64 Index by default in pandas versus Int64 (nullable) Index in BigQuery DataFrame check_index_type=False, check_dtype=False, diff --git a/tests/system/small/ml/test_core.py b/tests/system/small/ml/test_core.py index c32ba80b30e..9add4a4a537 100644 --- a/tests/system/small/ml/test_core.py +++ b/tests/system/small/ml/test_core.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import typing from datetime import datetime +import typing -import numpy as np import pandas as pd import pyarrow as pa import pytest @@ -79,16 +78,6 @@ def test_model_eval_with_data(penguins_bqml_linear_model, penguins_df_default_in def test_model_centroids(penguins_bqml_kmeans_model: core.BqmlModel): result = penguins_bqml_kmeans_model.centroids().to_pandas() - - # FIX: Helper to ignore row order inside categorical_value lists - # This prevents the test from failing if BQML returns [MALE, FEMALE] instead of [FEMALE, MALE] - def sort_categorical(val): - if isinstance(val, (list, np.ndarray)) and len(val) > 0: - return sorted(val, key=lambda x: x["category"]) - return val - - result["categorical_value"] = result["categorical_value"].apply(sort_categorical) - expected = ( pd.DataFrame( { @@ -146,12 +135,6 @@ def sort_categorical(val): .sort_values(["centroid_id", "feature"]) .reset_index(drop=True) ) - - # Sort expected values to match the output of the model. - expected["categorical_value"] = expected["categorical_value"].apply( - sort_categorical - ) - pd.testing.assert_frame_equal( result, expected, @@ -169,26 +152,6 @@ def test_pca_model_principal_components(penguins_bqml_pca_model: core.BqmlModel) # result is too long, only check the first principal component here. result = result.head(7) - - # FIX: Helper to ignore row order inside categorical_value lists - # and sign flipping of values inside numerical_value list. - # This prevents the test from failing if BQML returns [MALE, FEMALE] instead of [FEMALE, MALE] - # or 0.197 versus -0.197. - def sort_and_abs_categorical(val): - # Accept BOTH python lists AND numpy arrays - if isinstance(val, (list, np.ndarray)) and len(val) > 0: - # Take abs of value first, then sort - processed = [ - {"category": x["category"], "value": abs(x["value"])} for x in val - ] - return sorted(processed, key=lambda x: x["category"]) - return val - - result["numerical_value"] = result["numerical_value"].abs() - result["categorical_value"] = result["categorical_value"].apply( - sort_and_abs_categorical - ) - expected = ( pd.DataFrame( { @@ -248,12 +211,6 @@ def sort_and_abs_categorical(val): .reset_index(drop=True) ) - # Sort and sign flip expected values to match the output of the model. - expected["numerical_value"] = expected["numerical_value"].abs() - expected["categorical_value"] = expected["categorical_value"].apply( - sort_and_abs_categorical - ) - utils.assert_pandas_df_equal_pca_components( result, expected, diff --git a/tests/system/small/ml/test_decomposition.py b/tests/system/small/ml/test_decomposition.py index 36abfe55adf..297ee49739a 100644 --- a/tests/system/small/ml/test_decomposition.py +++ b/tests/system/small/ml/test_decomposition.py @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np import pandas as pd +from bigframes.ml import decomposition import bigframes.pandas as bpd import bigframes.testing.utils -from bigframes.ml import decomposition def test_pca_predict( @@ -35,7 +34,7 @@ def test_pca_predict( ) bigframes.testing.utils.assert_pandas_df_equal_pca( - predictions, expected, check_exact=False, rtol=0.2 + predictions, expected, check_exact=False, rtol=0.1 ) @@ -56,7 +55,7 @@ def test_pca_detect_anomalies( expected, check_exact=False, check_dtype=False, - rtol=0.2, + rtol=0.1, ) @@ -79,7 +78,7 @@ def test_pca_detect_anomalies_params( expected, check_exact=False, check_dtype=False, - rtol=0.2, + rtol=0.1, ) @@ -93,7 +92,7 @@ def test_pca_score(penguins_pca_model: decomposition.PCA): result, expected, check_exact=False, - rtol=0.2, + rtol=0.1, check_index_type=False, ) @@ -103,26 +102,6 @@ def test_pca_components_(penguins_pca_model: decomposition.PCA): # result is too long, only check the first principal component here. result = result.head(7) - - # FIX: Helper to ignore row order inside categorical_value lists - # and sign flipping of values inside numerical_value list. - # This prevents the test from failing if BQML returns [MALE, FEMALE] instead of [FEMALE, MALE] - # or 0.197 versus -0.197. - def sort_and_abs_categorical(val): - # Accept BOTH python lists AND numpy arrays - if isinstance(val, (list, np.ndarray)) and len(val) > 0: - # Take abs of value first, then sort - processed = [ - {"category": x["category"], "value": abs(x["value"])} for x in val - ] - return sorted(processed, key=lambda x: x["category"]) - return val - - result["numerical_value"] = result["numerical_value"].abs() - result["categorical_value"] = result["categorical_value"].apply( - sort_and_abs_categorical - ) - expected = ( pd.DataFrame( { @@ -182,17 +161,11 @@ def sort_and_abs_categorical(val): .reset_index(drop=True) ) - # Sort and sign flip expected values to match the output of the model. - expected["numerical_value"] = expected["numerical_value"].abs() - expected["categorical_value"] = expected["categorical_value"].apply( - sort_and_abs_categorical - ) - bigframes.testing.utils.assert_pandas_df_equal_pca_components( result, expected, check_exact=False, - rtol=0.2, # FIX: Slightly increased rtol for numerical drift (from 0.1) + rtol=0.1, check_index_type=False, check_dtype=False, ) @@ -211,7 +184,7 @@ def test_pca_explained_variance_(penguins_pca_model: decomposition.PCA): result, expected, check_exact=False, - rtol=0.2, + rtol=0.1, check_index_type=False, check_dtype=False, ignore_order=True, @@ -231,7 +204,7 @@ def test_pca_explained_variance_ratio_(penguins_pca_model: decomposition.PCA): result, expected, check_exact=False, - rtol=0.2, + rtol=0.1, check_index_type=False, check_dtype=False, ignore_order=True, diff --git a/tests/system/small/ml/test_forecasting.py b/tests/system/small/ml/test_forecasting.py index 23487983ee3..134f82e96e9 100644 --- a/tests/system/small/ml/test_forecasting.py +++ b/tests/system/small/ml/test_forecasting.py @@ -493,7 +493,7 @@ def test_arima_plus_score( dtype="Float64", ) pd.testing.assert_frame_equal( - result[expected.columns], + result, expected, rtol=0.1, check_index_type=False, @@ -594,7 +594,7 @@ def test_arima_plus_score_series( dtype="Float64", ) pd.testing.assert_frame_equal( - result[expected.columns], + result, expected, rtol=0.1, check_index_type=False, diff --git a/tests/system/small/ml/test_llm.py b/tests/system/small/ml/test_llm.py new file mode 100644 index 00000000000..d15c5d31605 --- /dev/null +++ b/tests/system/small/ml/test_llm.py @@ -0,0 +1,611 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable +from unittest import mock + +import pandas as pd +import pytest + +from bigframes import exceptions +from bigframes.ml import core, llm +import bigframes.pandas as bpd +from bigframes.testing import utils + + +@pytest.mark.parametrize( + "model_name", + ("text-embedding-005", "text-embedding-004", "text-multilingual-embedding-002"), +) +def test_create_load_text_embedding_generator_model( + dataset_id, model_name, session, bq_connection +): + text_embedding_model = llm.TextEmbeddingGenerator( + model_name=model_name, connection_name=bq_connection, session=session + ) + assert text_embedding_model is not None + assert text_embedding_model._bqml_model is not None + + # save, load to ensure configuration was kept + reloaded_model = text_embedding_model.to_gbq( + f"{dataset_id}.temp_text_model", replace=True + ) + assert f"{dataset_id}.temp_text_model" == reloaded_model._bqml_model.model_name + assert reloaded_model.connection_name == bq_connection + assert reloaded_model.model_name == model_name + + +@pytest.mark.parametrize( + "model_name", + ("text-embedding-005", "text-embedding-004", "text-multilingual-embedding-002"), +) +@pytest.mark.flaky(retries=2) +def test_text_embedding_generator_predict_default_params_success( + llm_text_df, model_name, session, bq_connection +): + text_embedding_model = llm.TextEmbeddingGenerator( + model_name=model_name, connection_name=bq_connection, session=session + ) + df = text_embedding_model.predict(llm_text_df).to_pandas() + utils.check_pandas_df_schema_and_index( + df, columns=utils.ML_GENERATE_EMBEDDING_OUTPUT, index=3, col_exact=False + ) + assert len(df["ml_generate_embedding_result"][0]) == 768 + + +@pytest.mark.parametrize( + "model_name", + ("text-embedding-005", "text-embedding-004", "text-multilingual-embedding-002"), +) +@pytest.mark.flaky(retries=2) +def test_text_embedding_generator_multi_cols_predict_success( + llm_text_df: bpd.DataFrame, model_name, session, bq_connection +): + df = llm_text_df.assign(additional_col=1) + df = df.rename(columns={"prompt": "content"}) + text_embedding_model = llm.TextEmbeddingGenerator( + model_name=model_name, connection_name=bq_connection, session=session + ) + pd_df = text_embedding_model.predict(df).to_pandas() + utils.check_pandas_df_schema_and_index( + pd_df, + columns=utils.ML_GENERATE_EMBEDDING_OUTPUT + ["additional_col"], + index=3, + col_exact=False, + ) + assert len(pd_df["ml_generate_embedding_result"][0]) == 768 + + +def test_create_load_multimodal_embedding_generator_model( + dataset_id, session, bq_connection +): + mm_embedding_model = llm.MultimodalEmbeddingGenerator( + connection_name=bq_connection, session=session + ) + assert mm_embedding_model is not None + assert mm_embedding_model._bqml_model is not None + + # save, load to ensure configuration was kept + reloaded_model = mm_embedding_model.to_gbq( + f"{dataset_id}.temp_mm_model", replace=True + ) + assert f"{dataset_id}.temp_mm_model" == reloaded_model._bqml_model.model_name + assert reloaded_model.connection_name == bq_connection + + +# Overrides __eq__ function for comparing as mock.call parameter +class EqCmpAllDataFrame(bpd.DataFrame): + def __eq__(self, other): + return self.equals(other) + + +@pytest.mark.skip("b/436340035 test failed") +@pytest.mark.parametrize( + ( + "model_class", + "options", + ), + [ + ( + llm.GeminiTextGenerator, + { + "temperature": 0.9, + "max_output_tokens": 8192, + "top_p": 1.0, + "ground_with_google_search": False, + }, + ), + ( + llm.Claude3TextGenerator, + { + "max_output_tokens": 128, + "top_k": 40, + "top_p": 0.95, + }, + ), + ], +) +def test_text_generator_retry_success( + session, + model_class, + options, + bq_connection, +): + # Requests. + df0 = EqCmpAllDataFrame( + { + "prompt": [ + "What is BigQuery?", + "What is BQML?", + "What is BigQuery DataFrame?", + ] + }, + index=[0, 1, 2], + session=session, + ) + df1 = EqCmpAllDataFrame( + { + "ml_generate_text_status": ["error", "error"], + "prompt": [ + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[1, 2], + session=session, + ) + df2 = EqCmpAllDataFrame( + { + "ml_generate_text_status": ["error"], + "prompt": [ + "What is BQML?", + ], + }, + index=[1], + session=session, + ) + + mock_generate_text = mock.create_autospec( + Callable[[core.BqmlModel, bpd.DataFrame, dict], bpd.DataFrame] + ) + mock_bqml_model = mock.create_autospec(spec=core.BqmlModel) + type(mock_bqml_model).session = mock.PropertyMock(return_value=session) + generate_text_tvf = core.BqmlModel.TvfDef( + mock_generate_text, "ml_generate_text_status" + ) + # Responses. Retry twice then all succeeded. + mock_generate_text.side_effect = [ + EqCmpAllDataFrame( + { + "ml_generate_text_status": ["", "error", "error"], + "prompt": [ + "What is BigQuery?", + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[0, 1, 2], + session=session, + ), + EqCmpAllDataFrame( + { + "ml_generate_text_status": ["error", ""], + "prompt": [ + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[1, 2], + session=session, + ), + EqCmpAllDataFrame( + { + "ml_generate_text_status": [""], + "prompt": [ + "What is BQML?", + ], + }, + index=[1], + session=session, + ), + ] + + text_generator_model = model_class(connection_name=bq_connection, session=session) + text_generator_model._bqml_model = mock_bqml_model + + with mock.patch.object(core.BqmlModel, "generate_text_tvf", generate_text_tvf): + # 3rd retry isn't triggered + result = text_generator_model.predict(df0, max_retries=3) + + mock_generate_text.assert_has_calls( + [ + mock.call(mock_bqml_model, df0, options), + mock.call(mock_bqml_model, df1, options), + mock.call(mock_bqml_model, df2, options), + ] + ) + pd.testing.assert_frame_equal( + result.to_pandas(), + pd.DataFrame( + { + "ml_generate_text_status": ["", "", ""], + "prompt": [ + "What is BigQuery?", + "What is BigQuery DataFrame?", + "What is BQML?", + ], + }, + index=[0, 2, 1], + ), + check_dtype=False, + check_index_type=False, + ) + + +@pytest.mark.skip("b/436340035 test failed") +@pytest.mark.parametrize( + ( + "model_class", + "options", + ), + [ + ( + llm.GeminiTextGenerator, + { + "temperature": 0.9, + "max_output_tokens": 8192, + "top_p": 1.0, + "ground_with_google_search": False, + }, + ), + ( + llm.Claude3TextGenerator, + { + "max_output_tokens": 128, + "top_k": 40, + "top_p": 0.95, + }, + ), + ], +) +def test_text_generator_retry_no_progress(session, model_class, options, bq_connection): + # Requests. + df0 = EqCmpAllDataFrame( + { + "prompt": [ + "What is BigQuery?", + "What is BQML?", + "What is BigQuery DataFrame?", + ] + }, + index=[0, 1, 2], + session=session, + ) + df1 = EqCmpAllDataFrame( + { + "ml_generate_text_status": ["error", "error"], + "prompt": [ + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[1, 2], + session=session, + ) + + mock_generate_text = mock.create_autospec( + Callable[[core.BqmlModel, bpd.DataFrame, dict], bpd.DataFrame] + ) + mock_bqml_model = mock.create_autospec(spec=core.BqmlModel) + type(mock_bqml_model).session = mock.PropertyMock(return_value=session) + generate_text_tvf = core.BqmlModel.TvfDef( + mock_generate_text, "ml_generate_text_status" + ) + # Responses. Retry once, no progress, just stop. + mock_generate_text.side_effect = [ + EqCmpAllDataFrame( + { + "ml_generate_text_status": ["", "error", "error"], + "prompt": [ + "What is BigQuery?", + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[0, 1, 2], + session=session, + ), + EqCmpAllDataFrame( + { + "ml_generate_text_status": ["error", "error"], + "prompt": [ + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[1, 2], + session=session, + ), + ] + + text_generator_model = model_class(connection_name=bq_connection, session=session) + text_generator_model._bqml_model = mock_bqml_model + + with mock.patch.object(core.BqmlModel, "generate_text_tvf", generate_text_tvf): + # No progress, only conduct retry once + result = text_generator_model.predict(df0, max_retries=3) + + mock_generate_text.assert_has_calls( + [ + mock.call(mock_bqml_model, df0, options), + mock.call(mock_bqml_model, df1, options), + ] + ) + pd.testing.assert_frame_equal( + result.to_pandas(), + pd.DataFrame( + { + "ml_generate_text_status": ["", "error", "error"], + "prompt": [ + "What is BigQuery?", + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[0, 1, 2], + ), + check_dtype=False, + check_index_type=False, + ) + + +@pytest.mark.skip("b/436340035 test failed") +def test_text_embedding_generator_retry_success(session, bq_connection): + # Requests. + df0 = EqCmpAllDataFrame( + { + "content": [ + "What is BigQuery?", + "What is BQML?", + "What is BigQuery DataFrame?", + ] + }, + index=[0, 1, 2], + session=session, + ) + df1 = EqCmpAllDataFrame( + { + "ml_generate_embedding_status": ["error", "error"], + "content": [ + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[1, 2], + session=session, + ) + df2 = EqCmpAllDataFrame( + { + "ml_generate_embedding_status": ["error"], + "content": [ + "What is BQML?", + ], + }, + index=[1], + session=session, + ) + + mock_generate_embedding = mock.create_autospec( + Callable[[core.BqmlModel, bpd.DataFrame, dict], bpd.DataFrame] + ) + mock_bqml_model = mock.create_autospec(spec=core.BqmlModel) + type(mock_bqml_model).session = mock.PropertyMock(return_value=session) + generate_embedding_tvf = core.BqmlModel.TvfDef( + mock_generate_embedding, "ml_generate_embedding_status" + ) + + # Responses. Retry twice then all succeeded. + mock_generate_embedding.side_effect = [ + EqCmpAllDataFrame( + { + "ml_generate_embedding_status": ["", "error", "error"], + "content": [ + "What is BigQuery?", + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[0, 1, 2], + session=session, + ), + EqCmpAllDataFrame( + { + "ml_generate_embedding_status": ["error", ""], + "content": [ + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[1, 2], + session=session, + ), + EqCmpAllDataFrame( + { + "ml_generate_embedding_status": [""], + "content": [ + "What is BQML?", + ], + }, + index=[1], + session=session, + ), + ] + options: dict = {} + + text_embedding_model = llm.TextEmbeddingGenerator( + connection_name=bq_connection, session=session + ) + text_embedding_model._bqml_model = mock_bqml_model + + with mock.patch.object( + core.BqmlModel, "generate_embedding_tvf", generate_embedding_tvf + ): + # 3rd retry isn't triggered + result = text_embedding_model.predict(df0, max_retries=3) + + mock_generate_embedding.assert_has_calls( + [ + mock.call(mock_bqml_model, df0, options), + mock.call(mock_bqml_model, df1, options), + mock.call(mock_bqml_model, df2, options), + ] + ) + pd.testing.assert_frame_equal( + result.to_pandas(), + pd.DataFrame( + { + "ml_generate_embedding_status": ["", "", ""], + "content": [ + "What is BigQuery?", + "What is BigQuery DataFrame?", + "What is BQML?", + ], + }, + index=[0, 2, 1], + ), + check_dtype=False, + check_index_type=False, + ) + + +def test_text_embedding_generator_retry_no_progress(session, bq_connection): + # Requests. + df0 = EqCmpAllDataFrame( + { + "content": [ + "What is BigQuery?", + "What is BQML?", + "What is BigQuery DataFrame?", + ] + }, + index=[0, 1, 2], + session=session, + ) + df1 = EqCmpAllDataFrame( + { + "ml_generate_embedding_status": ["error", "error"], + "content": [ + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[1, 2], + session=session, + ) + + mock_generate_embedding = mock.create_autospec( + Callable[[core.BqmlModel, bpd.DataFrame, dict], bpd.DataFrame] + ) + mock_bqml_model = mock.create_autospec(spec=core.BqmlModel) + type(mock_bqml_model).session = mock.PropertyMock(return_value=session) + generate_embedding_tvf = core.BqmlModel.TvfDef( + mock_generate_embedding, "ml_generate_embedding_status" + ) + + # Responses. Retry once, no progress, just stop. + mock_generate_embedding.side_effect = [ + EqCmpAllDataFrame( + { + "ml_generate_embedding_status": ["", "error", "error"], + "content": [ + "What is BigQuery?", + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[0, 1, 2], + session=session, + ), + EqCmpAllDataFrame( + { + "ml_generate_embedding_status": ["error", "error"], + "content": [ + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[1, 2], + session=session, + ), + ] + options: dict = {} + + text_embedding_model = llm.TextEmbeddingGenerator( + connection_name=bq_connection, session=session + ) + text_embedding_model._bqml_model = mock_bqml_model + + with mock.patch.object( + core.BqmlModel, "generate_embedding_tvf", generate_embedding_tvf + ): + # No progress, only conduct retry once + result = text_embedding_model.predict(df0, max_retries=3) + + mock_generate_embedding.assert_has_calls( + [ + mock.call(mock_bqml_model, df0, options), + mock.call(mock_bqml_model, df1, options), + ] + ) + pd.testing.assert_frame_equal( + result.to_pandas(), + pd.DataFrame( + { + "ml_generate_embedding_status": ["", "error", "error"], + "content": [ + "What is BigQuery?", + "What is BQML?", + "What is BigQuery DataFrame?", + ], + }, + index=[0, 1, 2], + ), + check_dtype=False, + check_index_type=False, + ) + + +@pytest.mark.parametrize( + "model_name", + ("gemini-2.0-flash-exp",), +) +def test_gemini_preview_model_warnings(model_name): + with pytest.warns(exceptions.PreviewWarning): + llm.GeminiTextGenerator(model_name=model_name) + + +# b/436340035 temp disable the test to unblock presumbit +@pytest.mark.parametrize( + "model_class", + [ + llm.TextEmbeddingGenerator, + llm.MultimodalEmbeddingGenerator, + llm.GeminiTextGenerator, + # llm.Claude3TextGenerator, + ], +) +def test_text_embedding_generator_no_default_model_warning(model_class): + message = "Since upgrading the default model can cause unintended breakages, the\ndefault model will be removed in BigFrames 3.0. Please supply an\nexplicit model to avoid this message." + with pytest.warns(FutureWarning, match=message): + model_class(model_name=None) diff --git a/tests/system/small/ml/test_metrics.py b/tests/system/small/ml/test_metrics.py index ab9c3e4552c..040d4d97f64 100644 --- a/tests/system/small/ml/test_metrics.py +++ b/tests/system/small/ml/test_metrics.py @@ -19,7 +19,6 @@ import pytest import bigframes -import bigframes.testing.utils from bigframes.ml import metrics @@ -162,7 +161,7 @@ def test_roc_curve_binary_classification_prediction_returns_expected(session): pd_tpr = tpr.to_pandas() pd_thresholds = thresholds.to_pandas() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( # skip testing the first value, as it is redundant and inconsistent across sklearn versions pd_thresholds[1:], pd.Series( @@ -172,7 +171,7 @@ def test_roc_curve_binary_classification_prediction_returns_expected(session): ), check_index=False, ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_fpr, pd.Series( [0.0, 0.0, 0.0, 0.25, 0.25, 0.5, 0.5, 0.75, 0.75, 0.75, 1.0], @@ -181,7 +180,7 @@ def test_roc_curve_binary_classification_prediction_returns_expected(session): ), check_index_type=False, ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_tpr, pd.Series( [ @@ -262,7 +261,7 @@ def test_roc_curve_binary_classification_decision_returns_expected(session): pd_tpr = tpr.to_pandas() pd_thresholds = thresholds.to_pandas() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( # skip testing the first value, as it is redundant and inconsistent across sklearn versions pd_thresholds[1:], pd.Series( @@ -272,7 +271,7 @@ def test_roc_curve_binary_classification_decision_returns_expected(session): ), check_index=False, ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_fpr, pd.Series( [0.0, 0.0, 1.0], @@ -281,7 +280,7 @@ def test_roc_curve_binary_classification_decision_returns_expected(session): ), check_index_type=False, ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_tpr, pd.Series( [ @@ -354,7 +353,7 @@ def test_roc_curve_binary_classification_prediction_series(session): pd_tpr = tpr.to_pandas() pd_thresholds = thresholds.to_pandas() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( # skip testing the first value, as it is redundant and inconsistent across sklearn versions pd_thresholds[1:], pd.Series( @@ -364,7 +363,7 @@ def test_roc_curve_binary_classification_prediction_series(session): ), check_index=False, ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_fpr, pd.Series( [0.0, 0.0, 0.0, 0.25, 0.25, 0.5, 0.5, 0.75, 0.75, 0.75, 1.0], @@ -373,7 +372,7 @@ def test_roc_curve_binary_classification_prediction_series(session): ), check_index_type=False, ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_tpr, pd.Series( [ @@ -506,7 +505,7 @@ def test_confusion_matrix(session): 2: [0, 1, 2], } ).astype("int64") - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( confusion_matrix, expected_pd_df, check_index_type=False ) @@ -524,7 +523,7 @@ def test_confusion_matrix_column_index(session): {1: [1, 0, 1, 0], 2: [0, 0, 2, 0], 3: [0, 0, 0, 0], 4: [0, 1, 0, 1]}, index=[1, 2, 3, 4], ).astype("int64") - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( confusion_matrix, expected_pd_df, check_index_type=False ) @@ -543,7 +542,7 @@ def test_confusion_matrix_matches_sklearn(session): pd_df[["y_true"]], pd_df[["y_pred"]] ) expected_pd_df = pd.DataFrame(expected_confusion_matrix) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( confusion_matrix, expected_pd_df, check_index_type=False ) @@ -565,7 +564,7 @@ def test_confusion_matrix_str_matches_sklearn(session): expected_confusion_matrix, index=["ant", "bird", "cat"] ) expected_pd_df.columns = pd.Index(["ant", "bird", "cat"]) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( confusion_matrix, expected_pd_df, check_index_type=False ) @@ -586,7 +585,7 @@ def test_confusion_matrix_series(session): 2: [0, 1, 2], } ).astype("int64") - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( confusion_matrix, expected_pd_df, check_index_type=False ) @@ -606,9 +605,7 @@ def test_recall_score(session): expected_index = [0, 1, 2] expected_recall = pd.Series(expected_values, index=expected_index) - bigframes.testing.utils.assert_series_equal( - recall, expected_recall, check_index_type=False - ) + pd.testing.assert_series_equal(recall, expected_recall, check_index_type=False) def test_recall_score_matches_sklearn(session): @@ -626,9 +623,7 @@ def test_recall_score_matches_sklearn(session): ) expected_index = [0, 1, 2] expected_recall = pd.Series(expected_values, index=expected_index) - bigframes.testing.utils.assert_series_equal( - recall, expected_recall, check_index_type=False - ) + pd.testing.assert_series_equal(recall, expected_recall, check_index_type=False) def test_recall_score_str_matches_sklearn(session): @@ -646,9 +641,7 @@ def test_recall_score_str_matches_sklearn(session): ) expected_index = ["ant", "bird", "cat"] expected_recall = pd.Series(expected_values, index=expected_index) - bigframes.testing.utils.assert_series_equal( - recall, expected_recall, check_index_type=False - ) + pd.testing.assert_series_equal(recall, expected_recall, check_index_type=False) def test_recall_score_series(session): @@ -664,9 +657,7 @@ def test_recall_score_series(session): expected_index = [0, 1, 2] expected_recall = pd.Series(expected_values, index=expected_index) - bigframes.testing.utils.assert_series_equal( - recall, expected_recall, check_index_type=False - ) + pd.testing.assert_series_equal(recall, expected_recall, check_index_type=False) def test_precision_score(session): @@ -684,7 +675,7 @@ def test_precision_score(session): expected_index = [0, 1, 2] expected_precision = pd.Series(expected_values, index=expected_index) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( precision_score, expected_precision, check_index_type=False ) @@ -707,7 +698,7 @@ def test_precision_score_matches_sklearn(session): ) expected_index = [0, 1, 2] expected_precision = pd.Series(expected_values, index=expected_index) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( precision_score, expected_precision, check_index_type=False ) @@ -729,7 +720,7 @@ def test_precision_score_str_matches_sklearn(session): ) expected_index = ["ant", "bird", "cat"] expected_precision = pd.Series(expected_values, index=expected_index) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( precision_score, expected_precision, check_index_type=False ) @@ -747,7 +738,7 @@ def test_precision_score_series(session): expected_index = [0, 1, 2] expected_precision = pd.Series(expected_values, index=expected_index) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( precision_score, expected_precision, check_index_type=False ) @@ -807,6 +798,7 @@ def test_precision_score_binary_default_arguments(session): def test_precision_score_binary_invalid_input_raise_error( session, y_true, y_pred, pos_label ): + bf_y_true = session.read_pandas(y_true) bf_y_pred = session.read_pandas(y_pred) @@ -831,9 +823,7 @@ def test_f1_score(session): expected_index = [0, 1, 2] expected_f1 = pd.Series(expected_values, index=expected_index) - bigframes.testing.utils.assert_series_equal( - f1_score, expected_f1, check_index_type=False - ) + pd.testing.assert_series_equal(f1_score, expected_f1, check_index_type=False) def test_f1_score_matches_sklearn(session): @@ -851,9 +841,7 @@ def test_f1_score_matches_sklearn(session): ) expected_index = [0, 1, 2] expected_f1 = pd.Series(expected_values, index=expected_index) - bigframes.testing.utils.assert_series_equal( - f1_score, expected_f1, check_index_type=False - ) + pd.testing.assert_series_equal(f1_score, expected_f1, check_index_type=False) def test_f1_score_str_matches_sklearn(session): @@ -871,9 +859,7 @@ def test_f1_score_str_matches_sklearn(session): ) expected_index = ["ant", "bird", "cat"] expected_f1 = pd.Series(expected_values, index=expected_index) - bigframes.testing.utils.assert_series_equal( - f1_score, expected_f1, check_index_type=False - ) + pd.testing.assert_series_equal(f1_score, expected_f1, check_index_type=False) def test_f1_score_series(session): @@ -889,9 +875,7 @@ def test_f1_score_series(session): expected_index = [0, 1, 2] expected_f1 = pd.Series(expected_values, index=expected_index) - bigframes.testing.utils.assert_series_equal( - f1_score, expected_f1, check_index_type=False - ) + pd.testing.assert_series_equal(f1_score, expected_f1, check_index_type=False) def test_mean_squared_error(session: bigframes.Session): diff --git a/tests/system/small/ml/test_metrics_pairwise.py b/tests/system/small/ml/test_metrics_pairwise.py index 44f1ed671b7..d3798f7cae1 100644 --- a/tests/system/small/ml/test_metrics_pairwise.py +++ b/tests/system/small/ml/test_metrics_pairwise.py @@ -15,8 +15,8 @@ import numpy as np import pandas as pd -import bigframes.pandas as bpd from bigframes.ml import metrics +import bigframes.pandas as bpd def test_paired_cosine_distances(): diff --git a/tests/system/small/ml/test_model_selection.py b/tests/system/small/ml/test_model_selection.py index b7764a7d916..ebce6e405a5 100644 --- a/tests/system/small/ml/test_model_selection.py +++ b/tests/system/small/ml/test_model_selection.py @@ -18,9 +18,9 @@ import pandas as pd import pytest +from bigframes.ml import model_selection import bigframes.pandas as bpd import bigframes.session -from bigframes.ml import model_selection @pytest.mark.parametrize( @@ -323,13 +323,7 @@ def test_train_test_split_value_error(penguins_df_default_index, train_size, tes ) def test_train_test_split_stratify(df_fixture, request): df = request.getfixturevalue(df_fixture) - X = df[ - [ - "species", - "island", - "culmen_length_mm", - ] - ].rename( + X = df[["species", "island", "culmen_length_mm",]].rename( columns={"species": "x_species"} ) # Keep "species" col just for easy checking. Rename to avoid conflicts. y = df[["species"]] @@ -417,9 +411,8 @@ def test_KFold_split(df_fixture, n_splits, request): ] y = df["body_mass_g"] - len_test_upper, len_test_lower = ( - math.ceil(len(df) / n_splits), - math.floor(len(df) / n_splits), + len_test_upper, len_test_lower = math.ceil(len(df) / n_splits), math.floor( + len(df) / n_splits ) len_train_upper, len_train_lower = ( len(df) - len_test_lower, @@ -467,9 +460,8 @@ def test_KFold_split_X_only(df_fixture, n_splits, request): ] ] - len_test_upper, len_test_lower = ( - math.ceil(len(df) / n_splits), - math.floor(len(df) / n_splits), + len_test_upper, len_test_lower = math.ceil(len(df) / n_splits), math.floor( + len(df) / n_splits ) len_train_upper, len_train_lower = ( len(df) - len_test_lower, diff --git a/tests/system/small/ml/test_multimodal_llm.py b/tests/system/small/ml/test_multimodal_llm.py new file mode 100644 index 00000000000..e29669afd30 --- /dev/null +++ b/tests/system/small/ml/test_multimodal_llm.py @@ -0,0 +1,84 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pandas as pd +import pyarrow as pa +import pytest + +from bigframes.ml import llm +import bigframes.pandas as bpd +from bigframes.testing import utils + + +@pytest.mark.flaky(retries=2) +def test_multimodal_embedding_generator_predict_default_params_success( + images_mm_df, session, bq_connection +): + text_embedding_model = llm.MultimodalEmbeddingGenerator( + connection_name=bq_connection, session=session + ) + df = text_embedding_model.predict(images_mm_df).to_pandas() + utils.check_pandas_df_schema_and_index( + df, + columns=utils.ML_MULTIMODAL_GENERATE_EMBEDDING_OUTPUT, + index=2, + col_exact=False, + ) + assert len(df["ml_generate_embedding_result"][0]) == 1408 + + +@pytest.mark.parametrize( + "model_name", + ( + "gemini-2.0-flash-exp", + "gemini-2.0-flash-001", + ), +) +@pytest.mark.flaky(retries=2) +def test_gemini_text_generator_multimodal_structured_output( + images_mm_df: bpd.DataFrame, model_name, session, bq_connection +): + gemini_text_generator_model = llm.GeminiTextGenerator( + model_name=model_name, connection_name=bq_connection, session=session + ) + output_schema = { + "bool_output": "bool", + "int_output": "int64", + "float_output": "float64", + "str_output": "string", + "array_output": "array", + "struct_output": "struct", + } + df = gemini_text_generator_model.predict( + images_mm_df, + prompt=["Describe", images_mm_df["blob_col"]], + output_schema=output_schema, + ) + assert df["bool_output"].dtype == pd.BooleanDtype() + assert df["int_output"].dtype == pd.Int64Dtype() + assert df["float_output"].dtype == pd.Float64Dtype() + assert df["str_output"].dtype == pd.StringDtype(storage="pyarrow") + assert df["array_output"].dtype == pd.ArrowDtype(pa.list_(pa.int64())) + assert df["struct_output"].dtype == pd.ArrowDtype( + pa.struct([("number", pa.int64())]) + ) + + pd_df = df.to_pandas() + utils.check_pandas_df_schema_and_index( + pd_df, + columns=list(output_schema.keys()) + + ["blob_col", "prompt", "full_response", "status"], + index=2, + col_exact=False, + ) diff --git a/tests/system/small/ml/test_preprocessing.py b/tests/system/small/ml/test_preprocessing.py index ec63cc94f23..3280b16f42a 100644 --- a/tests/system/small/ml/test_preprocessing.py +++ b/tests/system/small/ml/test_preprocessing.py @@ -18,8 +18,8 @@ import pyarrow as pa import bigframes.features -import bigframes.pandas as bpd from bigframes.ml import preprocessing +import bigframes.pandas as bpd from bigframes.testing import utils ONE_HOT_ENCODED_DTYPE = ( diff --git a/tests/system/small/ml/test_utils.py b/tests/system/small/ml/test_utils.py index ec3bd315b13..b3aa4ed59bc 100644 --- a/tests/system/small/ml/test_utils.py +++ b/tests/system/small/ml/test_utils.py @@ -13,10 +13,10 @@ # limitations under the License. import pandas as pd +import pandas.testing import pytest import bigframes.ml.utils as utils -import bigframes.testing.utils _DATA_FRAME = pd.DataFrame({"column": [1, 2, 3]}) _SERIES = pd.Series([1, 2, 3], name="column") @@ -31,7 +31,7 @@ def test_convert_to_dataframe(session, data): (actual_result,) = utils.batch_convert_to_dataframe(bf_data) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( actual_result.to_pandas(), _DATA_FRAME, check_index_type=False, @@ -46,7 +46,7 @@ def test_convert_to_dataframe(session, data): def test_convert_pandas_to_dataframe(data, session): (actual_result,) = utils.batch_convert_to_dataframe(data, session=session) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( actual_result.to_pandas(), _DATA_FRAME, check_index_type=False, @@ -63,7 +63,7 @@ def test_convert_to_series(session, data): (actual_result,) = utils.batch_convert_to_series(bf_data) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result.to_pandas(), _SERIES, check_index_type=False, check_dtype=False ) @@ -75,6 +75,6 @@ def test_convert_to_series(session, data): def test_convert_pandas_to_series(data, session): (actual_result,) = utils.batch_convert_to_series(data, session=session) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result.to_pandas(), _SERIES, check_index_type=False, check_dtype=False ) diff --git a/tests/system/small/operations/test_ai.py b/tests/system/small/operations/test_ai.py new file mode 100644 index 00000000000..d6ec3cacadc --- /dev/null +++ b/tests/system/small/operations/test_ai.py @@ -0,0 +1,276 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Note that the tests in this files uses fake models for deterministic results. +# Tests that use real LLM models are under system/large/test_ai.py + +import pandas as pd +import pandas.testing +import pytest + +import bigframes +from bigframes import dataframe, dtypes +from bigframes.ml import llm +import bigframes.operations.ai +from bigframes.testing import utils + +AI_OP_EXP_OPTION = "experiments.ai_operators" +THRESHOLD_OPTION = "compute.ai_ops_confirmation_threshold" +AI_FORECAST_COLUMNS = [ + "forecast_timestamp", + "forecast_value", + "confidence_level", + "prediction_interval_lower_bound", + "prediction_interval_upper_bound", + "ai_forecast_status", +] + + +class FakeGeminiTextGenerator(llm.GeminiTextGenerator): + def __init__(self, prediction): + self.prediction = prediction + + def predict(self, *args, **kwargs): + return self.prediction + + +@pytest.mark.parametrize( + ("func", "kwargs"), + [ + pytest.param( + bigframes.operations.ai.AIAccessor.filter, + {"instruction": None, "model": None}, + id="filter", + ), + pytest.param( + bigframes.operations.ai.AIAccessor.map, + {"instruction": None, "model": None}, + id="map", + ), + pytest.param( + bigframes.operations.ai.AIAccessor.classify, + {"instruction": None, "model": None, "labels": None}, + id="classify", + ), + pytest.param( + bigframes.operations.ai.AIAccessor.join, + {"other": None, "instruction": None, "model": None}, + id="join", + ), + pytest.param( + bigframes.operations.ai.AIAccessor.search, + {"search_column": None, "query": None, "top_k": None, "model": None}, + id="search", + ), + pytest.param( + bigframes.operations.ai.AIAccessor.sim_join, + {"other": None, "left_on": None, "right_on": None, "model": None}, + id="sim_join", + ), + ], +) +def test_experiment_off_raise_error(session, func, kwargs): + df = dataframe.DataFrame( + {"country": ["USA", "Germany"], "city": ["Seattle", "Berlin"]}, session=session + ) + + with bigframes.option_context(AI_OP_EXP_OPTION, False), pytest.raises( + NotImplementedError + ): + func(df.ai, **kwargs) + + +def test_filter(session): + df = dataframe.DataFrame({"col": ["A", "B"]}, session=session) + model = FakeGeminiTextGenerator( + dataframe.DataFrame( + { + "answer": [True, False], + "full_response": _create_dummy_full_response(2), + }, + session=session, + ), + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 50, + ): + result = df.ai.filter( + "filter {col}", + model=model, + ).to_pandas() + + pandas.testing.assert_frame_equal( + result, + pd.DataFrame({"col": ["A"]}, dtype=dtypes.STRING_DTYPE), + check_index_type=False, + ) + + +def test_map(session): + df = dataframe.DataFrame({"col": ["A", "B"]}, session=session) + model = FakeGeminiTextGenerator( + dataframe.DataFrame( + { + "output": ["true", "false"], + "full_response": _create_dummy_full_response(2), + }, + session=session, + ), + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 50, + ): + result = df.ai.map( + "map {col}", model=model, output_schema={"output": "string"} + ).to_pandas() + + pandas.testing.assert_frame_equal( + result, + pd.DataFrame( + {"col": ["A", "B"], "output": ["true", "false"]}, dtype=dtypes.STRING_DTYPE + ), + check_index_type=False, + ) + + +def test_classify(session): + df = dataframe.DataFrame({"col": ["A", "B"]}, session=session) + model = FakeGeminiTextGenerator( + dataframe.DataFrame( + { + "result": ["A", "B"], + "full_response": _create_dummy_full_response(2), + }, + session=session, + ), + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 50, + ): + result = df.ai.classify( + "classify {col}", model=model, labels=["A", "B"] + ).to_pandas() + + pandas.testing.assert_frame_equal( + result, + pd.DataFrame( + {"col": ["A", "B"], "result": ["A", "B"]}, dtype=dtypes.STRING_DTYPE + ), + check_index_type=False, + ) + + +@pytest.mark.parametrize( + "labels", + [ + pytest.param([], id="empty-label"), + pytest.param(["A", "A", "B"], id="duplicate-labels"), + ], +) +def test_classify_invalid_labels_raise_error(session, labels): + df = dataframe.DataFrame({"col": ["A", "B"]}, session=session) + model = FakeGeminiTextGenerator( + dataframe.DataFrame( + { + "result": ["A", "B"], + "full_response": _create_dummy_full_response(2), + }, + session=session, + ), + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 50, + ), pytest.raises(ValueError): + df.ai.classify("classify {col}", model=model, labels=labels) + + +def test_join(session): + left_df = dataframe.DataFrame({"col_A": ["A"]}, session=session) + right_df = dataframe.DataFrame({"col_B": ["B"]}, session=session) + model = FakeGeminiTextGenerator( + dataframe.DataFrame( + { + "answer": [True], + "full_response": _create_dummy_full_response(1), + }, + session=session, + ), + ) + + with bigframes.option_context( + AI_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 50, + ): + result = left_df.ai.join( + right_df, "join {col_A} and {col_B}", model + ).to_pandas() + + pandas.testing.assert_frame_equal( + result, + pd.DataFrame({"col_A": ["A"], "col_B": ["B"]}, dtype=dtypes.STRING_DTYPE), + check_index_type=False, + ) + + +def test_forecast_default(time_series_df_default_index: dataframe.DataFrame): + df = time_series_df_default_index[time_series_df_default_index["id"] == "1"] + + result = df.ai.forecast(timestamp_column="parsed_date", data_column="total_visits") + + utils.check_pandas_df_schema_and_index( + result, + columns=AI_FORECAST_COLUMNS, + index=10, + ) + + +def test_forecast_w_params(time_series_df_default_index: dataframe.DataFrame): + result = time_series_df_default_index.ai.forecast( + timestamp_column="parsed_date", + data_column="total_visits", + id_columns=["id"], + horizon=20, + confidence_level=0.98, + ) + + utils.check_pandas_df_schema_and_index( + result, + columns=["id"] + AI_FORECAST_COLUMNS, + index=20 * 2, # 20 for each id + ) + + +def _create_dummy_full_response(row_count: int) -> pd.Series: + entry = """{"candidates": [{"avg_logprobs": -0.5}]}""" + + return pd.Series([entry] * row_count) diff --git a/tests/system/small/operations/test_dates.py b/tests/system/small/operations/test_dates.py index 3554322462b..9e8da642090 100644 --- a/tests/system/small/operations/test_dates.py +++ b/tests/system/small/operations/test_dates.py @@ -15,11 +15,11 @@ import datetime +from packaging import version import pandas as pd +import pandas.testing import pytest -from packaging import version -import bigframes.testing.utils from bigframes import dtypes @@ -35,7 +35,7 @@ def test_date_diff_between_series(session): actual_result = (bf_df["col_1"] - bf_df["col_2"]).to_pandas() expected_result = (pd_df["col_1"] - pd_df["col_2"]).astype(dtypes.TIMEDELTA_DTYPE) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result, expected_result, check_index_type=False ) @@ -47,7 +47,7 @@ def test_date_diff_literal_sub_series(scalars_dfs): actual_result = (literal - bf_df["date_col"]).to_pandas() expected_result = (literal - pd_df["date_col"]).astype(dtypes.TIMEDELTA_DTYPE) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result, expected_result, check_index_type=False ) @@ -59,7 +59,7 @@ def test_date_diff_series_sub_literal(scalars_dfs): actual_result = (bf_df["date_col"] - literal).to_pandas() expected_result = (pd_df["date_col"] - literal).astype(dtypes.TIMEDELTA_DTYPE) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result, expected_result, check_index_type=False ) @@ -70,7 +70,7 @@ def test_date_series_diff_agg(scalars_dfs): actual_result = bf_df["date_col"].diff().to_pandas() expected_result = pd_df["date_col"].diff().astype(dtypes.TIMEDELTA_DTYPE) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result, expected_result, check_index_type=False ) @@ -86,6 +86,6 @@ def test_date_can_cast_after_accessor(scalars_dfs): pd.to_datetime(pd_df["date_col"]).dt.isocalendar().week.astype("Int64") ) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result, expected_result, check_dtype=False, check_index_type=False ) diff --git a/tests/system/small/operations/test_datetimes.py b/tests/system/small/operations/test_datetimes.py index ebfe0414de0..0e023189d56 100644 --- a/tests/system/small/operations/test_datetimes.py +++ b/tests/system/small/operations/test_datetimes.py @@ -16,13 +16,14 @@ import typing import numpy +from packaging import version +from pandas import testing import pandas as pd import pytest -from packaging import version import bigframes.pandas as bpd import bigframes.series -from bigframes.testing.utils import assert_frame_equal, assert_series_equal +from bigframes.testing.utils import assert_series_equal DATETIME_COL_NAMES = [("datetime_col",), ("timestamp_col",)] DATE_COLUMNS = [ @@ -303,7 +304,7 @@ def test_dt_isocalendar(session): actual_result = bf_s.dt.isocalendar().to_pandas() expected_result = pd_s.dt.isocalendar() - assert_frame_equal( + testing.assert_frame_equal( actual_result, expected_result, check_dtype=False, check_index_type=False ) @@ -323,41 +324,6 @@ def test_dt_tz(scalars_dfs, col_name): assert bf_result == pd_result -@pytest.mark.parametrize( - ("col_name", "tz"), - [ - ("datetime_col", None), - ("timestamp_col", None), - ("datetime_col", "UTC"), - ], -) -def test_dt_tz_localize(scalars_dfs, col_name, tz): - pytest.importorskip("pandas", minversion="2.0.0") - scalars_df, scalars_pandas_df = scalars_dfs - bf_series = scalars_df[col_name] - - bf_result = bf_series.dt.tz_localize(tz) - pd_result = scalars_pandas_df[col_name].dt.tz_localize(tz) - - assert_series_equal(bf_result.to_pandas(), pd_result, check_index_type=False) - - -def test_dt_tz_localize_already_localized(scalars_dfs): - pytest.importorskip("pandas", minversion="2.0.0") - scalars_df, _ = scalars_dfs - - with pytest.raises(TypeError): - scalars_df["timestamp_col"].dt.tz_localize("UTC") - - -def test_dt_tz_localize_invalid_timezone(scalars_dfs): - pytest.importorskip("pandas", minversion="2.0.0") - scalars_df, _ = scalars_dfs - - with pytest.raises(ValueError): - scalars_df["datetime_col"].dt.tz_localize("US/Eastern") - - @pytest.mark.parametrize( ("col_name",), DATETIME_COL_NAMES, @@ -387,7 +353,7 @@ def test_dt_strftime(scalars_df_index, scalars_pandas_df_index, column, date_for pytest.importorskip("pandas", minversion="2.0.0") bf_result = scalars_df_index[column].dt.strftime(date_format).to_pandas() pd_result = scalars_pandas_df_index[column].dt.strftime(date_format) - assert_series_equal(bf_result, pd_result, check_dtype=False) + pd.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) assert bf_result.dtype == "string[pyarrow]" @@ -399,7 +365,7 @@ def test_dt_strftime_date(): expected_result = pd.Series(["08/15/2014", "08/15/2215", "02/29/2016"]) bf_result = bf_series.dt.strftime("%m/%d/%Y").to_pandas() - assert_series_equal( + pd.testing.assert_series_equal( bf_result, expected_result, check_index_type=False, check_dtype=False ) assert bf_result.dtype == "string[pyarrow]" @@ -415,7 +381,7 @@ def test_dt_strftime_time(): ) bf_result = bf_series.dt.strftime("%X").to_pandas() - assert_series_equal( + pd.testing.assert_series_equal( bf_result, expected_result, check_index_type=False, check_dtype=False ) assert bf_result.dtype == "string[pyarrow]" @@ -555,7 +521,7 @@ def test_timestamp_diff_two_dataframes(scalars_dfs): actual_result = (bf_df - bf_df).to_pandas() expected_result = pd_df - pd_df - assert_frame_equal(actual_result, expected_result) + testing.assert_frame_equal(actual_result, expected_result) def test_timestamp_diff_two_series_with_different_types_raise_error(scalars_dfs): @@ -577,12 +543,9 @@ def test_timestamp_diff_series_sub_literal(scalars_dfs, column, value): bf_series = bf_df[column] pd_series = pd_df[column] - # Pandas doesn't handle nulls properly here so we ffill - # overflows for no good reason - # related? https://github.com/apache/arrow/issues/43031 - actual_result = (bf_series.ffill() - value).to_pandas() + actual_result = (bf_series - value).to_pandas() - expected_result = pd_series.ffill() - value + expected_result = pd_series - value assert_series_equal(actual_result, expected_result) @@ -598,12 +561,9 @@ def test_timestamp_diff_literal_sub_series(scalars_dfs, column, value): bf_series = bf_df[column] pd_series = pd_df[column] - # Pandas doesn't handle nulls properly here so we ffill - # overflows for no good reason - # related? https://github.com/apache/arrow/issues/43031 - actual_result = (value - bf_series.ffill()).to_pandas() + actual_result = (value - bf_series).to_pandas() - expected_result = value - pd_series.ffill() + expected_result = value - pd_series assert_series_equal(actual_result, expected_result) @@ -615,12 +575,7 @@ def test_timestamp_series_diff_agg(scalars_dfs, column): actual_result = bf_series.diff().to_pandas() - # overflows for no good reason - # related? https://github.com/apache/arrow/issues/43031 - expected_result = pd_series.ffill().diff() - expected_result = expected_result.mask( - pd_series.isnull() | pd_series.shift(1).isnull() - ) + expected_result = pd_series.diff() assert_series_equal(actual_result, expected_result) @@ -675,6 +630,6 @@ def test_to_datetime(scalars_dfs, col): ).to_pandas() expected_result = pd.Series(pd.to_datetime(pd_df[col])) - assert_series_equal( + testing.assert_series_equal( actual_result, expected_result, check_dtype=False, check_index_type=False ) diff --git a/tests/system/small/operations/test_plotting.py b/tests/system/small/operations/test_plotting.py index e579c90b4df..2585ac8e813 100644 --- a/tests/system/small/operations/test_plotting.py +++ b/tests/system/small/operations/test_plotting.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from matplotlib.collections import PathCollection import numpy as np import pandas as pd import pandas._testing as tm import pytest -from matplotlib.collections import PathCollection import bigframes.operations._matplotlib.core as bf_mpl import bigframes.pandas as bpd diff --git a/tests/system/small/operations/test_semantics.py b/tests/system/small/operations/test_semantics.py new file mode 100644 index 00000000000..8b520d8c035 --- /dev/null +++ b/tests/system/small/operations/test_semantics.py @@ -0,0 +1,143 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Note that the tests in this files uses fake models for deterministic results. +# Tests that use real LLM models are under system/large/test_semantcs.py + +import pandas as pd +import pandas.testing +import pytest + +import bigframes +from bigframes import dataframe, dtypes +from bigframes.ml import llm + +SEM_OP_EXP_OPTION = "experiments.semantic_operators" +THRESHOLD_OPTION = "compute.semantic_ops_confirmation_threshold" + + +class FakeGeminiTextGenerator(llm.GeminiTextGenerator): + def __init__(self, prediction): + self.prediction = prediction + + def predict(self, *args, **kwargs): + return self.prediction + + +def test_semantics_experiment_off_raise_error(session): + df = dataframe.DataFrame( + {"country": ["USA", "Germany"], "city": ["Seattle", "Berlin"]}, session=session + ) + + with bigframes.option_context(SEM_OP_EXP_OPTION, False), pytest.raises( + NotImplementedError + ): + df.semantics + + +def test_filter(session): + df = dataframe.DataFrame({"col": ["A", "B"]}, session=session) + model = FakeGeminiTextGenerator( + dataframe.DataFrame( + {"ml_generate_text_llm_result": ["true", "false"]}, session=session + ), + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 50, + ): + result = df.semantics.filter( + "filter {col}", + model=model, + ).to_pandas() + + pandas.testing.assert_frame_equal( + result, + pd.DataFrame({"col": ["A"]}, dtype=dtypes.STRING_DTYPE), + check_index_type=False, + ) + + +def test_map(session): + df = dataframe.DataFrame({"col": ["A", "B"]}, session=session) + model = FakeGeminiTextGenerator( + dataframe.DataFrame( + {"ml_generate_text_llm_result": ["true", "false"]}, session=session + ), + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 50, + ): + result = df.semantics.map( + "map {col}", model=model, output_column="output" + ).to_pandas() + + pandas.testing.assert_frame_equal( + result, + pd.DataFrame( + {"col": ["A", "B"], "output": ["true", "false"]}, dtype=dtypes.STRING_DTYPE + ), + check_index_type=False, + ) + + +def test_join(session): + left_df = dataframe.DataFrame({"col_A": ["A"]}, session=session) + right_df = dataframe.DataFrame({"col_B": ["B"]}, session=session) + model = FakeGeminiTextGenerator( + dataframe.DataFrame({"ml_generate_text_llm_result": ["true"]}, session=session), + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 50, + ): + result = left_df.semantics.join( + right_df, "join {col_A} and {col_B}", model + ).to_pandas() + + pandas.testing.assert_frame_equal( + result, + pd.DataFrame({"col_A": ["A"], "col_B": ["B"]}, dtype=dtypes.STRING_DTYPE), + check_index_type=False, + ) + + +def test_top_k(session): + df = dataframe.DataFrame({"col": ["A", "B"]}, session=session) + model = FakeGeminiTextGenerator( + dataframe.DataFrame( + {"ml_generate_text_llm_result": ["Document 1"]}, session=session + ), + ) + + with bigframes.option_context( + SEM_OP_EXP_OPTION, + True, + THRESHOLD_OPTION, + 50, + ): + result = df.semantics.top_k("top k of {col}", model, k=1).to_pandas() + + assert len(result) == 1 diff --git a/tests/system/small/operations/test_strings.py b/tests/system/small/operations/test_strings.py index 94285cc7dc4..657fc231d18 100644 --- a/tests/system/small/operations/test_strings.py +++ b/tests/system/small/operations/test_strings.py @@ -317,7 +317,7 @@ def test_isnumeric(weird_strings, weird_strings_pd): pd.testing.assert_series_equal( bf_result, - pd_result.astype(pd.BooleanDtype()), + pd_result.astype(pd.BooleanDtype()) # the dtype here is a case of intentional diversion from pandas # see go/bigframes-dtypes ) @@ -329,7 +329,7 @@ def test_isalpha(weird_strings, weird_strings_pd): pd.testing.assert_series_equal( bf_result, - pd_result.astype(pd.BooleanDtype()), + pd_result.astype(pd.BooleanDtype()) # the dtype here is a case of intentional diversion from pandas # see go/bigframes-dtypes ) @@ -343,7 +343,7 @@ def test_isdigit(weird_strings, weird_strings_pd): pd.testing.assert_series_equal( bf_result, - pd_result.astype(pd.BooleanDtype()), + pd_result.astype(pd.BooleanDtype()) # the dtype here is a case of intentional diversion from pandas # see go/bigframes-dtypes ) @@ -355,7 +355,7 @@ def test_isdecimal(weird_strings, weird_strings_pd): pd.testing.assert_series_equal( bf_result, - pd_result.astype(pd.BooleanDtype()), + pd_result.astype(pd.BooleanDtype()) # the dtype here is a case of intentional diversion from pandas # see go/bigframes-dtypes ) @@ -367,7 +367,7 @@ def test_isalnum(weird_strings, weird_strings_pd): pd.testing.assert_series_equal( bf_result, - pd_result.astype(pd.BooleanDtype()), + pd_result.astype(pd.BooleanDtype()) # the dtype here is a case of intentional diversion from pandas # see go/bigframes-dtypes ) @@ -379,7 +379,7 @@ def test_isspace(weird_strings, weird_strings_pd): pd.testing.assert_series_equal( bf_result, - pd_result.astype(pd.BooleanDtype()), + pd_result.astype(pd.BooleanDtype()) # the dtype here is a case of intentional diversion from pandas # see go/bigframes-dtypes ) @@ -391,7 +391,7 @@ def test_islower(weird_strings, weird_strings_pd): assert_series_equal( bf_result, - pd_result.astype(pd.BooleanDtype()), + pd_result.astype(pd.BooleanDtype()) # the dtype here is a case of intentional diversion from pandas # see go/bigframes-dtypes ) @@ -403,7 +403,7 @@ def test_isupper(weird_strings, weird_strings_pd): assert_series_equal( bf_result, - pd_result.astype(pd.BooleanDtype()), + pd_result.astype(pd.BooleanDtype()) # the dtype here is a case of intentional diversion from pandas # see go/bigframes-dtypes ) diff --git a/tests/system/small/operations/test_timedeltas.py b/tests/system/small/operations/test_timedeltas.py index 9512950e168..18c88db8eb5 100644 --- a/tests/system/small/operations/test_timedeltas.py +++ b/tests/system/small/operations/test_timedeltas.py @@ -17,67 +17,51 @@ import operator import numpy as np +from packaging import version import pandas as pd +import pandas.testing import pyarrow as pa import pytest -from packaging import version -import bigframes.testing.utils from bigframes import dtypes -# Some methods/features used by this test don't exist in pandas 1.x -pytest.importorskip("pandas", minversion="2.0.0") - @pytest.fixture(scope="module") def temporal_dfs(session): pandas_df = pd.DataFrame( { - "datetime_col": pd.Series( - [ - pd.Timestamp("2025-02-01 01:00:01"), - pd.Timestamp("2019-01-02 02:00:00"), - pd.Timestamp("1997-01-01 19:00:00"), - ], - dtype=dtypes.DATETIME_DTYPE, - ), - "timestamp_col": pd.Series( - [ - pd.Timestamp("2023-01-01 01:00:01", tz="UTC"), - pd.Timestamp("2024-01-02 02:00:00", tz="UTC"), - pd.Timestamp("2005-03-05 02:00:00", tz="UTC"), - ], - dtype=dtypes.TIMESTAMP_DTYPE, - ), + "datetime_col": [ + pd.Timestamp("2025-02-01 01:00:01"), + pd.Timestamp("2019-01-02 02:00:00"), + pd.Timestamp("1997-01-01 19:00:00"), + ], + "timestamp_col": [ + pd.Timestamp("2023-01-01 01:00:01", tz="UTC"), + pd.Timestamp("2024-01-02 02:00:00", tz="UTC"), + pd.Timestamp("2005-03-05 02:00:00", tz="UTC"), + ], "date_col": pd.Series( [ datetime.date(2000, 1, 1), datetime.date(2001, 2, 3), datetime.date(2020, 9, 30), ], - dtype=dtypes.DATE_DTYPE, - ), - "timedelta_col_1": pd.Series( - [ - pd.Timedelta(5, "s"), - pd.Timedelta(-4, "m"), - pd.Timedelta(5, "h"), - ], - dtype=dtypes.TIMEDELTA_DTYPE, - ), - "timedelta_col_2": pd.Series( - [ - pd.Timedelta(3, "s"), - pd.Timedelta(-4, "m"), - pd.Timedelta(6, "h"), - ], - dtype=dtypes.TIMEDELTA_DTYPE, + dtype=pd.ArrowDtype(pa.date32()), ), - "float_col": pd.Series([1.5, 2, -3], dtype=dtypes.FLOAT_DTYPE), - "int_col": pd.Series([1, 2, -3], dtype="Int64"), - "positive_int_col": pd.Series([1, 2, 3], dtype="Int64"), - }, - index=pd.Index(range(3), dtype="Int64"), + "timedelta_col_1": [ + pd.Timedelta(5, "s"), + pd.Timedelta(-4, "m"), + pd.Timedelta(5, "h"), + ], + "timedelta_col_2": [ + pd.Timedelta(3, "s"), + pd.Timedelta(-4, "m"), + pd.Timedelta(6, "h"), + ], + "float_col": [1.5, 2, -3], + "int_col": [1, 2, -3], + "positive_int_col": [1, 2, 3], + } ) bigframes_df = session.read_pandas(pandas_df) @@ -86,106 +70,89 @@ def temporal_dfs(session): def _assert_series_equal(actual: pd.Series, expected: pd.Series): - """Helper function specifically for timedelta testing. Don't use it outside of this module.""" - bigframes.testing.utils.assert_series_equal( - actual, - expected, - check_index_type=False, - check_dtype=False, - ) + """Helper function specifically for timedelta testsing. Don't use it outside of this module.""" + if actual.dtype == dtypes.FLOAT_DTYPE: + pandas.testing.assert_series_equal( + actual, expected.astype("Float64"), check_index_type=False + ) + elif actual.dtype == dtypes.INT_DTYPE: + pandas.testing.assert_series_equal( + actual, expected.astype("Int64"), check_index_type=False + ) + else: + pandas.testing.assert_series_equal( + actual.astype("timedelta64[ns]"), + expected.dt.floor("us"), # in BF the precision is microsecond + check_index_type=False, + ) @pytest.mark.parametrize( - ("op", "col_1", "col_2", "arrow_supported"), + ("op", "col_1", "col_2"), [ - (operator.add, "timedelta_col_1", "timedelta_col_2", True), - (operator.sub, "timedelta_col_1", "timedelta_col_2", True), - (operator.truediv, "timedelta_col_1", "timedelta_col_2", True), - (operator.floordiv, "timedelta_col_1", "timedelta_col_2", True), - (operator.truediv, "timedelta_col_1", "float_col", False), - (operator.floordiv, "timedelta_col_1", "float_col", False), - (operator.mul, "timedelta_col_1", "float_col", False), - (operator.mul, "float_col", "timedelta_col_1", False), - (operator.mod, "timedelta_col_1", "timedelta_col_2", False), + (operator.add, "timedelta_col_1", "timedelta_col_2"), + (operator.sub, "timedelta_col_1", "timedelta_col_2"), + (operator.truediv, "timedelta_col_1", "timedelta_col_2"), + (operator.floordiv, "timedelta_col_1", "timedelta_col_2"), + (operator.truediv, "timedelta_col_1", "float_col"), + (operator.floordiv, "timedelta_col_1", "float_col"), + (operator.mul, "timedelta_col_1", "float_col"), + (operator.mul, "float_col", "timedelta_col_1"), + (operator.mod, "timedelta_col_1", "timedelta_col_2"), ], ) -def test_timedelta_binary_ops_between_series( - temporal_dfs, op, col_1, col_2, arrow_supported -): +def test_timedelta_binary_ops_between_series(temporal_dfs, op, col_1, col_2): bf_df, pd_df = temporal_dfs actual_result = op(bf_df[col_1], bf_df[col_2]).to_pandas() - if not arrow_supported: - expected_result = pd_df.apply(lambda x: op(x[col_1], x[col_2]), axis=1) - else: - expected_result = op(pd_df[col_1], pd_df[col_2]) + expected_result = op(pd_df[col_1], pd_df[col_2]) _assert_series_equal(actual_result, expected_result) @pytest.mark.parametrize( - ("op", "col", "literal", "arrow_supported"), + ("op", "col", "literal"), [ - (operator.add, "timedelta_col_1", pd.Timedelta(2, "s").as_unit("us"), True), - (operator.sub, "timedelta_col_1", pd.Timedelta(2, "s").as_unit("us"), True), - (operator.truediv, "timedelta_col_1", pd.Timedelta(2, "s").as_unit("us"), True), - ( - operator.floordiv, - "timedelta_col_1", - pd.Timedelta(2, "s").as_unit("us"), - False, - ), - (operator.truediv, "timedelta_col_1", 3, True), - (operator.floordiv, "timedelta_col_1", 3, False), - (operator.mul, "timedelta_col_1", 3, True), - (operator.mul, "float_col", pd.Timedelta(1, "s").as_unit("us"), True), - (operator.mod, "timedelta_col_1", pd.Timedelta(7, "s").as_unit("us"), False), + (operator.add, "timedelta_col_1", pd.Timedelta(2, "s")), + (operator.sub, "timedelta_col_1", pd.Timedelta(2, "s")), + (operator.truediv, "timedelta_col_1", pd.Timedelta(2, "s")), + (operator.floordiv, "timedelta_col_1", pd.Timedelta(2, "s")), + (operator.truediv, "timedelta_col_1", 3), + (operator.floordiv, "timedelta_col_1", 3), + (operator.mul, "timedelta_col_1", 3), + (operator.mul, "float_col", pd.Timedelta(1, "s")), + (operator.mod, "timedelta_col_1", pd.Timedelta(7, "s")), ], ) -def test_timedelta_binary_ops_series_and_literal( - temporal_dfs, op, col, literal, arrow_supported -): +def test_timedelta_binary_ops_series_and_literal(temporal_dfs, op, col, literal): bf_df, pd_df = temporal_dfs actual_result = op(bf_df[col], literal).to_pandas() - if not arrow_supported: - expected_result = pd_df[col].map(lambda x: op(x, literal)) - else: - expected_result = op(pd_df[col], literal) + expected_result = op(pd_df[col], literal) _assert_series_equal(actual_result, expected_result) @pytest.mark.parametrize( - ("op", "col", "literal", "arrow_supported"), + ("op", "col", "literal"), [ - (operator.add, "timedelta_col_1", pd.Timedelta(2, "s").as_unit("us"), True), - (operator.sub, "timedelta_col_1", pd.Timedelta(2, "s").as_unit("us"), True), - (operator.truediv, "timedelta_col_1", pd.Timedelta(2, "s").as_unit("us"), True), - ( - operator.floordiv, - "timedelta_col_1", - pd.Timedelta(2, "s").as_unit("us"), - True, - ), - (operator.truediv, "float_col", pd.Timedelta(2, "s").as_unit("us"), True), - (operator.floordiv, "float_col", pd.Timedelta(2, "s").as_unit("us"), True), - (operator.mul, "timedelta_col_1", 3, True), - (operator.mul, "float_col", pd.Timedelta(1, "s").as_unit("us"), False), - (operator.mod, "timedelta_col_1", pd.Timedelta(7, "s").as_unit("us"), False), + (operator.add, "timedelta_col_1", pd.Timedelta(2, "s")), + (operator.sub, "timedelta_col_1", pd.Timedelta(2, "s")), + (operator.truediv, "timedelta_col_1", pd.Timedelta(2, "s")), + (operator.floordiv, "timedelta_col_1", pd.Timedelta(2, "s")), + (operator.truediv, "float_col", pd.Timedelta(2, "s")), + (operator.floordiv, "float_col", pd.Timedelta(2, "s")), + (operator.mul, "timedelta_col_1", 3), + (operator.mul, "float_col", pd.Timedelta(1, "s")), + (operator.mod, "timedelta_col_1", pd.Timedelta(7, "s")), ], ) -def test_timedelta_binary_ops_literal_and_series( - temporal_dfs, op, col, literal, arrow_supported -): +def test_timedelta_binary_ops_literal_and_series(temporal_dfs, op, col, literal): bf_df, pd_df = temporal_dfs actual_result = op(literal, bf_df[col]).to_pandas() - if not arrow_supported: - expected_result = pd_df[col].map(lambda x: op(literal, x)) - else: - expected_result = op(literal, pd_df[col]) + expected_result = op(literal, pd_df[col]) _assert_series_equal(actual_result, expected_result) @@ -209,10 +176,12 @@ def test_timedelta_unary_ops(temporal_dfs, op): def test_timestamp_add__ts_series_plus_td_series(temporal_dfs, column, pd_dtype): bf_df, pd_df = temporal_dfs - actual_result = (bf_df[column] + bf_df["timedelta_col_1"]).to_pandas() + actual_result = ( + (bf_df[column] + bf_df["timedelta_col_1"]).to_pandas().astype(pd_dtype) + ) expected_result = pd_df[column] + pd_df["timedelta_col_1"] - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result, expected_result, check_index_type=False ) @@ -230,7 +199,7 @@ def test_timestamp_add__ts_series_plus_td_series__explicit_cast(temporal_dfs, co @pytest.mark.parametrize( "literal", [ - pytest.param(pd.Timedelta(1, unit="s").as_unit("us"), id="pandas"), + pytest.param(pd.Timedelta(1, unit="s"), id="pandas"), pytest.param(datetime.timedelta(seconds=1), id="python-datetime"), pytest.param(np.timedelta64(1, "s"), id="numpy"), ], @@ -238,10 +207,12 @@ def test_timestamp_add__ts_series_plus_td_series__explicit_cast(temporal_dfs, co def test_timestamp_add__ts_series_plus_td_literal(temporal_dfs, literal): bf_df, pd_df = temporal_dfs - actual_result = (bf_df["timestamp_col"] + literal).to_pandas() + actual_result = ( + (bf_df["timestamp_col"] + literal).to_pandas().astype("datetime64[ns, UTC]") + ) expected_result = pd_df["timestamp_col"] + literal - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result, expected_result, check_index_type=False ) @@ -256,22 +227,24 @@ def test_timestamp_add__ts_series_plus_td_literal(temporal_dfs, literal): def test_timestamp_add__td_series_plus_ts_series(temporal_dfs, column, pd_dtype): bf_df, pd_df = temporal_dfs - actual_result = (bf_df["timedelta_col_1"] + bf_df[column]).to_pandas() + actual_result = ( + (bf_df["timedelta_col_1"] + bf_df[column]).to_pandas().astype(pd_dtype) + ) expected_result = pd_df["timedelta_col_1"] + pd_df[column] - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result, expected_result, check_index_type=False ) def test_timestamp_add__td_literal_plus_ts_series(temporal_dfs): bf_df, pd_df = temporal_dfs - timedelta = pd.Timedelta(1, unit="s").as_unit("us") + timedelta = pd.Timedelta(1, unit="s") - actual_result = (timedelta + bf_df["datetime_col"]).to_pandas() + actual_result = (timedelta + bf_df["datetime_col"]).to_pandas().astype(" pd.Timedelta(1, "h")) - ].to_pandas() + actual_result = ( + bf_series[((bf_series - timestamp) > pd.Timedelta(1, "h"))] + .to_pandas() + .astype(" pd.Timedelta(1, "h")] - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result, expected_result, check_index_type=False ) @@ -561,30 +540,29 @@ def test_timedelta_filtering(session): def test_timedelta_ordering(session): pd_df = pd.DataFrame( { - "col_1": pd.Series( - [ - pd.Timestamp("2025-01-01 01:00:00"), - pd.Timestamp("2025-01-01 02:00:00"), - pd.Timestamp("2025-01-01 03:00:00"), - ], - dtype=dtypes.TIMESTAMP_DTYPE, - ), - "col_2": pd.Series( - [ - pd.Timestamp("2025-01-01 01:00:02"), - pd.Timestamp("2025-01-01 02:00:01"), - pd.Timestamp("2025-01-01 02:59:59"), - ], - dtype=dtypes.TIMESTAMP_DTYPE, - ), + "col_1": [ + pd.Timestamp("2025-01-01 01:00:00"), + pd.Timestamp("2025-01-01 02:00:00"), + pd.Timestamp("2025-01-01 03:00:00"), + ], + "col_2": [ + pd.Timestamp("2025-01-01 01:00:02"), + pd.Timestamp("2025-01-01 02:00:01"), + pd.Timestamp("2025-01-01 02:59:59"), + ], } ) bf_df = session.read_pandas(pd_df) - actual_result = (bf_df["col_2"] - bf_df["col_1"]).sort_values().to_pandas() + actual_result = ( + (bf_df["col_2"] - bf_df["col_1"]) + .sort_values() + .to_pandas() + .astype("timedelta64[ns]") + ) expected_result = (pd_df["col_2"] - pd_df["col_1"]).sort_values() - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result, expected_result, check_index_type=False ) @@ -615,7 +593,7 @@ def test_timedelta_agg__timedelta_result(temporal_dfs, agg_func): actual_result = agg_func(bf_df["timedelta_col_1"]) - expected_result = agg_func(pd_df["timedelta_col_1"]) + expected_result = agg_func(pd_df["timedelta_col_1"]).floor("us") assert actual_result == expected_result @@ -651,6 +629,6 @@ def test_timestamp_diff_after_type_casting(temporal_dfs): expected_result = pd_df["timestamp_col"] - pd_df["positive_int_col"].astype( "datetime64[us, UTC]" ) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( actual_result, expected_result, check_index_type=False, check_dtype=False ) diff --git a/tests/system/small/pandas/test_describe.py b/tests/system/small/pandas/test_describe.py index beb7a1968fc..6f288115128 100644 --- a/tests/system/small/pandas/test_describe.py +++ b/tests/system/small/pandas/test_describe.py @@ -15,8 +15,6 @@ import pandas.testing import pytest -import bigframes.pandas as bpd - def test_df_describe_non_temporal(scalars_dfs): # TODO: supply a reason why this isn't compatible with pandas 1.x @@ -354,62 +352,3 @@ def test_series_groupby_describe(scalars_dfs): check_dtype=False, check_index_type=False, ) - - -def test_describe_json_and_obj_ref_returns_count(session): - # Test describe() works on JSON and OBJ_REF types (without nunique, which fails) - import uuid - - import google.cloud.bigquery - - sql = """ - SELECT - PARSE_JSON('{"a": 1}') AS json_col, - 'gs://cloud-samples-data/vision/ocr/sign.jpg' AS uri_col - """ - df_init = session.read_gbq(sql) - - table_id = f"bigframes-dev.bigframes_tests_sys.tmp_obj_ref_{uuid.uuid4().hex}" - df_init.to_gbq(table_id, if_exists="replace") - - client = session.bqclient - table = client.get_table(table_id) - schema = list(table.schema) - for i, field in enumerate(schema): - if field.name == "uri_col": - schema[i] = google.cloud.bigquery.SchemaField( - name=field.name, - field_type=field.field_type, - mode=field.mode, - description="bigframes_dtype: OBJ_REF_DTYPE", - ) - break - table.schema = schema - client.update_table(table, ["schema"]) - - df = session.read_gbq(table_id) - df = df.rename(columns={"uri_col": "obj_ref_col"}) - - res = df.describe(include="all").to_pandas() - - assert "count" in res.index - assert res.loc["count", "json_col"] == 1.0 - assert res.loc["count", "obj_ref_col"] == 1.0 - - -def test_describe_with_unsupported_type_returns_empty_dataframe(session): - df = session.read_gbq("SELECT ST_GEOGPOINT(1.0, 2.0) AS geo_col") - - res = df.describe().to_pandas() - - assert len(res.columns) == 0 - assert len(res.index) == 1 - - -def test_describe_empty_dataframe_returns_empty_dataframe(session): - df = bpd.DataFrame() - - res = df.describe().to_pandas() - - assert len(res.columns) == 0 - assert len(res.index) == 1 diff --git a/tests/system/small/session/test_read_gbq_colab.py b/tests/system/small/session/test_read_gbq_colab.py index 6ba9c760847..65f47fe4e31 100644 --- a/tests/system/small/session/test_read_gbq_colab.py +++ b/tests/system/small/session/test_read_gbq_colab.py @@ -89,20 +89,11 @@ def test_read_gbq_colab_fresh_session_is_hybrid(): assert len(result) == 100 assert session._executor._enable_polars_execution is True # type: ignore - assert executions_before_python == 1 - assert executions_after == 2 - history = session.execution_history().to_dataframe() - assert history.iloc[-1]["job_type"] == "polars" + assert executions_after == executions_before_python == 1 def test_read_gbq_colab_peek_avoids_requery(maybe_ordered_session): - history_before = maybe_ordered_session.execution_history().to_dataframe() - queries_before = ( - len(history_before[history_before["job_type"] == "query"]) - if "job_type" in history_before.columns - else 0 - ) - + executions_before_sql = maybe_ordered_session._metrics.execution_count df = maybe_ordered_session._read_gbq_colab( """ SELECT @@ -116,36 +107,20 @@ def test_read_gbq_colab_peek_avoids_requery(maybe_ordered_session): LIMIT 300 """ ) - - history_after_read = maybe_ordered_session.execution_history().to_dataframe() - queries_after_read = len( - history_after_read[history_after_read["job_type"] == "query"] - ) - + executions_before_python = maybe_ordered_session._metrics.execution_count result = df.peek(100) - - history_after_peek = maybe_ordered_session.execution_history().to_dataframe() - queries_after_peek = len( - history_after_peek[history_after_peek["job_type"] == "query"] - ) + executions_after = maybe_ordered_session._metrics.execution_count # Ok, this isn't guaranteed by peek, but should happen with read api based impl # if starts failing, maybe stopped using read api? assert result["total"].is_monotonic_decreasing assert len(result) == 100 - assert queries_after_read == queries_before + 1 - assert queries_after_peek == queries_after_read + assert executions_after == executions_before_python == executions_before_sql + 1 def test_read_gbq_colab_repr_avoids_requery(maybe_ordered_session): - history_before = maybe_ordered_session.execution_history().to_dataframe() - queries_before = ( - len(history_before[history_before["job_type"] == "query"]) - if "job_type" in history_before.columns - else 0 - ) - + executions_before_sql = maybe_ordered_session._metrics.execution_count df = maybe_ordered_session._read_gbq_colab( """ SELECT @@ -159,21 +134,10 @@ def test_read_gbq_colab_repr_avoids_requery(maybe_ordered_session): LIMIT 300 """ ) - - history_after_read = maybe_ordered_session.execution_history().to_dataframe() - queries_after_read = len( - history_after_read[history_after_read["job_type"] == "query"] - ) - + executions_before_python = maybe_ordered_session._metrics.execution_count _ = repr(df) - - history_after_repr = maybe_ordered_session.execution_history().to_dataframe() - queries_after_repr = len( - history_after_repr[history_after_repr["job_type"] == "query"] - ) - - assert queries_after_read == queries_before + 1 - assert queries_after_repr == queries_after_read + executions_after = maybe_ordered_session._metrics.execution_count + assert executions_after == executions_before_python == executions_before_sql + 1 def test_read_gbq_colab_includes_formatted_scalars(session): diff --git a/tests/system/small/session/test_session_logging.py b/tests/system/small/session/test_session_logging.py deleted file mode 100644 index 4618e110687..00000000000 --- a/tests/system/small/session/test_session_logging.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from unittest import mock - -import bigframes.session._io.bigquery as bq_io -from bigframes.core.logging import data_types - - -def test_data_type_logging(scalars_df_index): - s = scalars_df_index["int64_col"] + 1.5 - - # We want to check the job_config passed to _query_and_wait_bigframes - with mock.patch( - "bigframes.session._io.bigquery.start_query_job_optional", - wraps=bq_io.start_query_job_optional, - ) as mock_query: - s.to_pandas() - - # Fetch job labels sent to the BQ client and verify their values - assert mock_query.called - call_args = mock_query.call_args - job_config = call_args.kwargs.get("job_config") - assert job_config is not None - job_labels = job_config.labels - assert "bigframes-dtypes" in job_labels - assert job_labels["bigframes-dtypes"] == data_types.encode_type_refs( - s._block._expr.node - ) diff --git a/tests/system/small/test_anywidget.py b/tests/system/small/test_anywidget.py index 70106f490b6..fad8f5b2b50 100644 --- a/tests/system/small/test_anywidget.py +++ b/tests/system/small/test_anywidget.py @@ -70,7 +70,7 @@ def table_widget(paginated_bf_df: bigframes.dataframe.DataFrame): from bigframes.display import TableWidget with bigframes.option_context( - "display.render_mode", "anywidget", "display.max_rows", 2 + "display.repr_mode", "anywidget", "display.max_rows", 2 ): # Delay context manager cleanup of `max_rows` until after tests finish. yield TableWidget(paginated_bf_df) @@ -100,7 +100,7 @@ def small_widget(small_bf_df): """Helper fixture for tests using a DataFrame smaller than the page size.""" from bigframes.display import TableWidget - with bf.option_context("display.render_mode", "anywidget", "display.max_rows", 5): + with bf.option_context("display.repr_mode", "anywidget", "display.max_rows", 5): yield TableWidget(small_bf_df) @@ -126,9 +126,7 @@ def unknown_row_count_widget(session): mock_batches.return_value = blocks.PandasBatches( batches_iterator, total_rows=None ) - with bf.option_context( - "display.render_mode", "anywidget", "display.max_rows", 2 - ): + with bf.option_context("display.repr_mode", "anywidget", "display.max_rows", 2): widget = TableWidget(bf_df) yield widget @@ -208,7 +206,7 @@ def test_widget_initialization_should_calculate_total_row_count( from bigframes.display import TableWidget with bigframes.option_context( - "display.render_mode", "anywidget", "display.max_rows", 2 + "display.repr_mode", "anywidget", "display.max_rows", 2 ): widget = TableWidget(paginated_bf_df) @@ -318,7 +316,7 @@ def test_widget_pagination_should_work_with_custom_page_size( ): """Test that a widget paginates correctly with a custom page size.""" with bigframes.option_context( - "display.render_mode", "anywidget", "display.max_rows", 3 + "display.repr_mode", "anywidget", "display.max_rows", 3 ): from bigframes.display import TableWidget @@ -372,7 +370,7 @@ def test_global_options_change_should_not_affect_existing_widget_page_size( then the widget's page size should remain unchanged. """ with bigframes.option_context( - "display.render_mode", "anywidget", "display.max_rows", 2 + "display.repr_mode", "anywidget", "display.max_rows", 2 ): from bigframes.display import TableWidget @@ -397,7 +395,7 @@ def test_widget_with_empty_dataframe_should_have_zero_row_count( then its row_count should be 0. """ - with bigframes.option_context("display.render_mode", "anywidget"): + with bigframes.option_context("display.repr_mode", "anywidget"): from bigframes.display import TableWidget widget = TableWidget(empty_bf_df) @@ -408,18 +406,31 @@ def test_widget_with_empty_dataframe_should_have_zero_row_count( def test_widget_with_empty_dataframe_should_render_table_headers( empty_bf_df: bf.dataframe.DataFrame, ): + """ + + Given an empty DataFrame, + + when a widget is created from it, + + then its HTML representation should still render the table headers. + + """ - with bigframes.option_context("display.render_mode", "anywidget"): + with bigframes.option_context("display.repr_mode", "anywidget"): + from bigframes.display import TableWidget widget = TableWidget(empty_bf_df) + html = widget.table_html + assert "]*>]*>([^<]*)", thead) assert match is not None, "Could not find table header cell in output." - assert match.group(1) == "", ( - f"Expected empty index header, but found: {match.group(1)}" - ) + assert ( + match.group(1) == "" + ), f"Expected empty index header, but found: {match.group(1)}" def test_widget_with_custom_index_should_display_index_column( @@ -1064,7 +1075,7 @@ def test_widget_with_custom_index_should_display_index_column( """ from bigframes.display.anywidget import TableWidget - with bf.option_context("display.render_mode", "anywidget", "display.max_rows", 2): + with bf.option_context("display.repr_mode", "anywidget", "display.max_rows", 2): widget = TableWidget(custom_index_bf_df) html = widget.table_html @@ -1084,7 +1095,7 @@ def test_widget_with_custom_index_pagination_preserves_index( """ from bigframes.display.anywidget import TableWidget - with bf.option_context("display.render_mode", "anywidget", "display.max_rows", 2): + with bf.option_context("display.repr_mode", "anywidget", "display.max_rows", 2): widget = TableWidget(custom_index_bf_df) widget.page = 1 # Navigate to page 2 @@ -1105,7 +1116,7 @@ def test_widget_with_custom_index_matches_pandas_output( """ from bigframes.display.anywidget import TableWidget - with bf.option_context("display.render_mode", "anywidget", "display.max_rows", 3): + with bf.option_context("display.repr_mode", "anywidget", "display.max_rows", 3): widget = TableWidget(custom_index_bf_df) html = widget.table_html @@ -1126,7 +1137,7 @@ def test_series_anywidget_integration_with_notebook_display( """Test Series display integration in Jupyter-like environment.""" pytest.importorskip("anywidget") - with bf.option_context("display.render_mode", "anywidget"): + with bf.option_context("display.repr_mode", "anywidget"): series = paginated_bf_df["value"] # Test the full display pipeline @@ -1151,7 +1162,7 @@ def test_series_different_data_types_anywidget(session: bf.Session): ) bf_df = session.read_pandas(test_data) - with bf.option_context("display.render_mode", "anywidget"): + with bf.option_context("display.repr_mode", "anywidget"): for col_name in test_data.columns: series = bf_df[col_name] widget = bigframes.display.TableWidget(series.to_frame()) diff --git a/tests/system/small/test_bq_sessions.py b/tests/system/small/test_bq_sessions.py index 99d2dfece3b..801346600d4 100644 --- a/tests/system/small/test_bq_sessions.py +++ b/tests/system/small/test_bq_sessions.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time from concurrent.futures import ThreadPoolExecutor +import time import google import google.api_core.exceptions -import pytest from google.cloud import bigquery +import pytest import bigframes.core.events from bigframes.session import bigquery_session diff --git a/tests/system/small/test_dataframe.py b/tests/system/small/test_dataframe.py index a109c33ffa6..0f7b782b66d 100644 --- a/tests/system/small/test_dataframe.py +++ b/tests/system/small/test_dataframe.py @@ -33,7 +33,6 @@ import bigframes.dtypes as dtypes import bigframes.pandas as bpd import bigframes.series as series -import bigframes.testing from bigframes.testing.utils import ( assert_dfs_equivalent, assert_frame_equal, @@ -84,8 +83,7 @@ def test_df_construct_pandas_default(scalars_dfs): ("bigquery_inline"), ("bigquery_load"), ("bigquery_streaming"), - # TODO(b/502298527): Reenable bigquery_write test - # ("bigquery_write"), + ("bigquery_write"), ], ) def test_read_pandas_all_nice_types( @@ -135,7 +133,7 @@ def test_df_construct_structs(session): ] ).to_frame() bf_series = session.read_pandas(pd_frame) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_series.to_pandas(), pd_frame, check_index_type=False, check_dtype=False ) @@ -145,7 +143,7 @@ def test_df_construct_local_concat_pd(scalars_pandas_df_index, session): bf_df = session.read_pandas(pd_df) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_df.to_pandas(), pd_df, check_index_type=False, check_dtype=False ) @@ -320,7 +318,7 @@ def test_df_nlargest(scalars_df_index, scalars_pandas_df_index, keep): 3, ["bool_col", "int64_too"], keep=keep ) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) @@ -338,7 +336,7 @@ def test_df_nsmallest(scalars_df_index, scalars_pandas_df_index, keep): bf_result = scalars_df_index.nsmallest(6, ["bool_col"], keep=keep) pd_result = scalars_pandas_df_index.nsmallest(6, ["bool_col"], keep=keep) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) @@ -357,7 +355,7 @@ def test_get_columns(scalars_dfs): col_names = ["bool_col", "float64_col", "int64_col"] df_subset = scalars_df.get(col_names) df_pandas = df_subset.to_pandas() - bigframes.testing.utils.assert_index_equal( + pd.testing.assert_index_equal( df_pandas.columns, scalars_pandas_df[col_names].columns ) @@ -404,9 +402,7 @@ def test_insert(scalars_dfs, loc, column, value, allow_duplicates): bf_df.insert(loc, column, value, allow_duplicates) pd_df.insert(loc, column, value, allow_duplicates) - bigframes.testing.utils.assert_frame_equal( - bf_df.to_pandas(), pd_df, check_dtype=False - ) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df, check_dtype=False) def test_mask_series_cond(scalars_df_index, scalars_pandas_df_index): @@ -600,7 +596,7 @@ def test_drop_column(scalars_dfs): scalars_df, scalars_pandas_df = scalars_dfs col_name = "int64_col" df_pandas = scalars_df.drop(columns=col_name).to_pandas() - bigframes.testing.utils.assert_index_equal( + pd.testing.assert_index_equal( df_pandas.columns, scalars_pandas_df.drop(columns=col_name).columns ) @@ -609,7 +605,7 @@ def test_drop_columns(scalars_dfs): scalars_df, scalars_pandas_df = scalars_dfs col_names = ["int64_col", "geography_col", "time_col"] df_pandas = scalars_df.drop(columns=col_names).to_pandas() - bigframes.testing.utils.assert_index_equal( + pd.testing.assert_index_equal( df_pandas.columns, scalars_pandas_df.drop(columns=col_names).columns ) @@ -621,7 +617,7 @@ def test_drop_labels_axis_1(scalars_dfs): pd_result = scalars_pandas_df.drop(labels=labels, axis=1) bf_result = scalars_df.drop(labels=labels, axis=1).to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result) + pd.testing.assert_frame_equal(pd_result, bf_result) def test_drop_with_custom_column_labels(scalars_dfs): @@ -648,7 +644,7 @@ def test_df_memory_usage(scalars_dfs): pd_result = scalars_pandas_df.memory_usage() bf_result = scalars_df.memory_usage() - bigframes.testing.utils.assert_series_equal(pd_result, bf_result, rtol=1.5) + pd.testing.assert_series_equal(pd_result, bf_result, rtol=1.5) def test_df_info(scalars_dfs): @@ -718,7 +714,9 @@ def test_df_info_no_cols(session): def test_df_info_no_cols_no_rows(session): expected = ( - "\nIndex: 0 entries\nEmpty DataFrame\n" + "\n" + "Index: 0 entries\n" + "Empty DataFrame\n" ) df = session.DataFrame({}) @@ -745,7 +743,7 @@ def test_select_dtypes(scalars_dfs, include, exclude): pd_result = scalars_pandas_df.select_dtypes(include=include, exclude=exclude) bf_result = scalars_df.select_dtypes(include=include, exclude=exclude).to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result) + pd.testing.assert_frame_equal(pd_result, bf_result) def test_drop_index(scalars_dfs): @@ -754,7 +752,7 @@ def test_drop_index(scalars_dfs): pd_result = scalars_pandas_df.drop(index=[4, 1, 2]) bf_result = scalars_df.drop(index=[4, 1, 2]).to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result) + pd.testing.assert_frame_equal(pd_result, bf_result) def test_drop_pandas_index(scalars_dfs): @@ -764,7 +762,7 @@ def test_drop_pandas_index(scalars_dfs): pd_result = scalars_pandas_df.drop(index=drop_index) bf_result = scalars_df.drop(index=drop_index).to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result) + pd.testing.assert_frame_equal(pd_result, bf_result) def test_drop_bigframes_index(scalars_dfs): @@ -775,12 +773,10 @@ def test_drop_bigframes_index(scalars_dfs): pd_result = scalars_pandas_df.drop(index=drop_pandas_index) bf_result = scalars_df.drop(index=drop_index).to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result) + pd.testing.assert_frame_equal(pd_result, bf_result) def test_drop_bigframes_index_with_na(scalars_dfs): - if pd.__version__.startswith("3"): - pytest.skip("Pandas 3.0 doesn't doesn't support drop with pd.NA values") scalars_df, scalars_pandas_df = scalars_dfs scalars_df = scalars_df.copy() scalars_pandas_df = scalars_pandas_df.copy() @@ -792,7 +788,7 @@ def test_drop_bigframes_index_with_na(scalars_dfs): pd_result = scalars_pandas_df.drop(index=drop_pandas_index) # drop_pandas_index) bf_result = scalars_df.drop(index=drop_index).to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result) + pd.testing.assert_frame_equal(pd_result, bf_result) def test_drop_bigframes_multiindex(scalars_dfs): @@ -813,7 +809,7 @@ def test_drop_bigframes_multiindex(scalars_dfs): bf_result = scalars_df.drop(index=drop_index).to_pandas() pd_result = scalars_pandas_df.drop(index=drop_pandas_index) - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result) + pd.testing.assert_frame_equal(pd_result, bf_result) def test_drop_labels_axis_0(scalars_dfs): @@ -822,7 +818,7 @@ def test_drop_labels_axis_0(scalars_dfs): pd_result = scalars_pandas_df.drop(labels=[4, 1, 2], axis=0) bf_result = scalars_df.drop(labels=[4, 1, 2], axis=0).to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result) + pd.testing.assert_frame_equal(pd_result, bf_result) def test_drop_index_and_columns(scalars_dfs): @@ -831,14 +827,14 @@ def test_drop_index_and_columns(scalars_dfs): pd_result = scalars_pandas_df.drop(index=[4, 1, 2], columns="int64_col") bf_result = scalars_df.drop(index=[4, 1, 2], columns="int64_col").to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result) + pd.testing.assert_frame_equal(pd_result, bf_result) def test_rename(scalars_dfs): scalars_df, scalars_pandas_df = scalars_dfs col_name_dict = {"bool_col": 1.2345} df_pandas = scalars_df.rename(columns=col_name_dict).to_pandas() - bigframes.testing.utils.assert_index_equal( + pd.testing.assert_index_equal( df_pandas.columns, scalars_pandas_df.rename(columns=col_name_dict).columns ) @@ -848,9 +844,7 @@ def test_df_peek(scalars_dfs_maybe_ordered): peek_result = scalars_df.peek(n=3, force=False, allow_large_results=True) - bigframes.testing.utils.assert_index_equal( - scalars_pandas_df.columns, peek_result.columns - ) + pd.testing.assert_index_equal(scalars_pandas_df.columns, peek_result.columns) assert len(peek_result) == 3 @@ -859,18 +853,14 @@ def test_df_peek_with_large_results_not_allowed(scalars_dfs_maybe_ordered): peek_result = scalars_df.peek(n=3, force=False, allow_large_results=False) - bigframes.testing.utils.assert_index_equal( - scalars_pandas_df.columns, peek_result.columns - ) + pd.testing.assert_index_equal(scalars_pandas_df.columns, peek_result.columns) assert len(peek_result) == 3 def test_df_peek_filtered(scalars_dfs): scalars_df, scalars_pandas_df = scalars_dfs peek_result = scalars_df[scalars_df.int64_col != 0].peek(n=3, force=False) - bigframes.testing.utils.assert_index_equal( - scalars_pandas_df.columns, peek_result.columns - ) + pd.testing.assert_index_equal(scalars_pandas_df.columns, peek_result.columns) assert len(peek_result) == 3 @@ -885,7 +875,7 @@ def test_df_peek_exception(scalars_dfs): def test_df_peek_force_default(scalars_dfs): scalars_df, scalars_pandas_df = scalars_dfs peek_result = scalars_df[["int64_col", "int64_too"]].cumsum().peek(n=3) - bigframes.testing.utils.assert_index_equal( + pd.testing.assert_index_equal( scalars_pandas_df[["int64_col", "int64_too"]].columns, peek_result.columns ) assert len(peek_result) == 3 @@ -896,7 +886,7 @@ def test_df_peek_reset_index(scalars_dfs): peek_result = ( scalars_df[["int64_col", "int64_too"]].reset_index(drop=True).peek(n=3) ) - bigframes.testing.utils.assert_index_equal( + pd.testing.assert_index_equal( scalars_pandas_df[["int64_col", "int64_too"]].columns, peek_result.columns ) assert len(peek_result) == 3 @@ -945,55 +935,42 @@ def test_join_repr(scalars_dfs_maybe_ordered): def test_repr_w_display_options(scalars_dfs, session): + metrics = session._metrics scalars_df, _ = scalars_dfs # get a pandas df of the expected format df, _ = scalars_df._block.to_pandas() pandas_df = df.set_axis(scalars_df._block.column_labels, axis=1) pandas_df.index.name = scalars_df.index.name - history_pre = session.execution_history().to_dataframe() - queries_pre = ( - len(history_pre[history_pre["job_type"] == "query"]) - if "job_type" in history_pre.columns - else 0 - ) - + executions_pre = metrics.execution_count with bigframes.option_context( "display.max_rows", 10, "display.max_columns", 5, "display.max_colwidth", 10 ): + # When there are 10 or fewer rows, the outputs should be identical except for the extra note. actual = scalars_df.head(10).__repr__() - - history_post = session.execution_history().to_dataframe() - queries_post = len(history_post[history_post["job_type"] == "query"]) + executions_post = metrics.execution_count with display_options.pandas_repr(bigframes.options.display): pandas_repr = pandas_df.head(10).__repr__() assert actual == pandas_repr - assert (queries_post - queries_pre) <= 2 + assert (executions_post - executions_pre) <= 3 def test_mimebundle_html_repr_w_all_rows(scalars_dfs, session): + metrics = session._metrics scalars_df, _ = scalars_dfs # get a pandas df of the expected format df, _ = scalars_df._block.to_pandas() pandas_df = df.set_axis(scalars_df._block.column_labels, axis=1) pandas_df.index.name = scalars_df.index.name - history_pre = session.execution_history().to_dataframe() - queries_pre = ( - len(history_pre[history_pre["job_type"] == "query"]) - if "job_type" in history_pre.columns - else 0 - ) - + executions_pre = metrics.execution_count # When there are 10 or fewer rows, the outputs should be identical except for the extra note. bundle = scalars_df.head(10)._repr_mimebundle_() actual = bundle["text/html"] - - history_post = session.execution_history().to_dataframe() - queries_post = len(history_post[history_post["job_type"] == "query"]) + executions_post = metrics.execution_count with display_options.pandas_repr(bigframes.options.display): pandas_repr = pandas_df.head(10)._repr_html_() @@ -1003,14 +980,14 @@ def test_mimebundle_html_repr_w_all_rows(scalars_dfs, session): + f"[{len(pandas_df.index)} rows x {len(pandas_df.columns)} columns in total]" ) assert actual == expected - assert (queries_post - queries_pre) <= 2 + assert (executions_post - executions_pre) <= 3 def test_df_column_name_with_space(scalars_dfs): scalars_df, scalars_pandas_df = scalars_dfs col_name_dict = {"bool_col": "bool col"} df_pandas = scalars_df.rename(columns=col_name_dict).to_pandas() - bigframes.testing.utils.assert_index_equal( + pd.testing.assert_index_equal( df_pandas.columns, scalars_pandas_df.rename(columns=col_name_dict).columns ) @@ -1019,7 +996,7 @@ def test_df_column_name_duplicate(scalars_dfs): scalars_df, scalars_pandas_df = scalars_dfs col_name_dict = {"int64_too": "int64_col"} df_pandas = scalars_df.rename(columns=col_name_dict).to_pandas() - bigframes.testing.utils.assert_index_equal( + pd.testing.assert_index_equal( df_pandas.columns, scalars_pandas_df.rename(columns=col_name_dict).columns ) @@ -1030,7 +1007,7 @@ def test_get_df_column_name_duplicate(scalars_dfs): bf_result = scalars_df.rename(columns=col_name_dict)["int64_col"].to_pandas() pd_result = scalars_pandas_df.rename(columns=col_name_dict)["int64_col"] - bigframes.testing.utils.assert_index_equal(bf_result.columns, pd_result.columns) + pd.testing.assert_index_equal(bf_result.columns, pd_result.columns) @pytest.mark.parametrize( @@ -1147,7 +1124,7 @@ def test_assign_new_column_w_loc(scalars_dfs): # Convert default pandas dtypes `int64` to match BigQuery DataFrames dtypes. pd_result["new_col"] = pd_result["new_col"].astype("Int64") - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -1169,7 +1146,7 @@ def test_assign_new_column_w_setitem(scalars_dfs, scalar): # Convert default pandas dtypes `float64` to match BigQuery DataFrames dtypes. pd_result["new_col"] = pd_result["new_col"].astype("Float64") - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) def test_assign_new_column_w_setitem_dataframe(scalars_dfs): @@ -1182,7 +1159,7 @@ def test_assign_new_column_w_setitem_dataframe(scalars_dfs): # Convert default pandas dtypes `int64` to match BigQuery DataFrames dtypes. pd_df["int64_col"] = pd_df["int64_col"].astype("Int64") - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df) def test_assign_new_column_w_setitem_dataframe_error(scalars_dfs): @@ -1208,7 +1185,7 @@ def test_assign_new_column_w_setitem_list(scalars_dfs): # Convert default pandas dtypes `int64` to match BigQuery DataFrames dtypes. pd_result["new_col"] = pd_result["new_col"].astype("Int64") - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) def test_assign_new_column_w_setitem_list_repeated(scalars_dfs): @@ -1226,7 +1203,7 @@ def test_assign_new_column_w_setitem_list_repeated(scalars_dfs): pd_result["new_col"] = pd_result["new_col"].astype("Int64") pd_result["new_col_2"] = pd_result["new_col_2"].astype("Int64") - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) def test_assign_new_column_w_setitem_list_custom_index(scalars_dfs): @@ -1246,7 +1223,7 @@ def test_assign_new_column_w_setitem_list_custom_index(scalars_dfs): # Convert default pandas dtypes `int64` to match BigQuery DataFrames dtypes. pd_result["new_col"] = pd_result["new_col"].astype("Int64") - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) def test_assign_new_column_w_setitem_list_error(scalars_dfs): @@ -1288,9 +1265,7 @@ def test_setitem_multicolumn_with_literals(scalars_dfs, key, value): bf_result[key] = value pd_result[key] = value - bigframes.testing.utils.assert_frame_equal( - pd_result, bf_result.to_pandas(), check_dtype=False - ) + pd.testing.assert_frame_equal(pd_result, bf_result.to_pandas(), check_dtype=False) def test_setitem_multicolumn_with_literals_different_lengths_raise_error(scalars_dfs): @@ -1309,9 +1284,7 @@ def test_setitem_multicolumn_with_dataframes(scalars_dfs): bf_result[["int64_col", "int64_too"]] = bf_result[["int64_too", "int64_col"]] / 2 pd_result[["int64_col", "int64_too"]] = pd_result[["int64_too", "int64_col"]] / 2 - bigframes.testing.utils.assert_frame_equal( - pd_result, bf_result.to_pandas(), check_dtype=False - ) + pd.testing.assert_frame_equal(pd_result, bf_result.to_pandas(), check_dtype=False) def test_setitem_multicolumn_with_dataframes_series_on_rhs_raise_error(scalars_dfs): @@ -1466,7 +1439,7 @@ def test_assign_different_df_w_loc( # Convert default pandas dtypes `int64` to match BigQuery DataFrames dtypes. pd_result["int64_col"] = pd_result["int64_col"].astype("Int64") - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) def test_assign_different_df_w_setitem( @@ -1485,7 +1458,7 @@ def test_assign_different_df_w_setitem( # Convert default pandas dtypes `int64` to match BigQuery DataFrames dtypes. pd_result["int64_col"] = pd_result["int64_col"].astype("Int64") - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) def test_assign_callable_lambda(scalars_dfs): @@ -1555,7 +1528,7 @@ def test_df_dropna_by_thresh(scalars_dfs, axis, ignore_index, subset, thresh): bf_result = df_result.to_pandas() # Pandas uses int64 instead of Int64 (nullable) dtype. pd_result.index = pd_result.index.astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) def test_df_dropna_range_columns(scalars_dfs): @@ -1603,7 +1576,7 @@ def test_df_fillna(scalars_dfs, col, fill_value): bf_result = scalars_df[col].fillna(fill_value).to_pandas() pd_result = scalars_pandas_df[col].fillna(fill_value) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pd.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) def test_df_replace_scalar_scalar(scalars_dfs): @@ -1612,7 +1585,7 @@ def test_df_replace_scalar_scalar(scalars_dfs): pd_result = scalars_pandas_df.replace(555.555, 3) # pandas has narrower result types as they are determined dynamically - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_frame_equal(pd_result, bf_result, check_dtype=False) def test_df_replace_regex_scalar(scalars_dfs): @@ -1620,7 +1593,7 @@ def test_df_replace_regex_scalar(scalars_dfs): bf_result = scalars_df.replace("^H.l", "Howdy, Planet!", regex=True).to_pandas() pd_result = scalars_pandas_df.replace("^H.l", "Howdy, Planet!", regex=True) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, ) @@ -1632,7 +1605,7 @@ def test_df_replace_list_scalar(scalars_dfs): pd_result = scalars_pandas_df.replace([555.555, 3.2], 3) # pandas has narrower result types as they are determined dynamically - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, check_dtype=False, @@ -1644,7 +1617,7 @@ def test_df_replace_value_dict(scalars_dfs): bf_result = scalars_df.replace(1, {"int64_col": 100, "int64_too": 200}).to_pandas() pd_result = scalars_pandas_df.replace(1, {"int64_col": 100, "int64_too": 200}) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, ) @@ -1861,9 +1834,7 @@ def test_df_cross_merge(scalars_dfs): ), "cross", ) - bigframes.testing.utils.assert_frame_equal( - bf_result, pd_result, check_index_type=False - ) + pd.testing.assert_frame_equal(bf_result, pd_result, check_index_type=False) @pytest.mark.parametrize( @@ -2016,9 +1987,7 @@ def test_self_merge_self_w_on_args(): bf_result = bf_df1.merge( bf_df2, left_on=["A", "C"], right_on=["B", "C"], how="inner" ).to_pandas() - bigframes.testing.utils.assert_frame_equal( - bf_result, pd_result, check_index_type=False - ) + pd.testing.assert_frame_equal(bf_result, pd_result, check_index_type=False) @pytest.mark.parametrize( @@ -2059,7 +2028,7 @@ def test_get_dtypes(scalars_df_default_index): "timestamp_col": pd.ArrowDtype(pa.timestamp("us", tz="UTC")), "duration_col": pd.ArrowDtype(pa.duration("us")), } - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( dtypes, pd.Series(dtypes_dict), ) @@ -2075,7 +2044,7 @@ def test_get_dtypes_array_struct_query(session): ) dtypes = df.dtypes - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( dtypes, pd.Series( { @@ -2095,7 +2064,7 @@ def test_get_dtypes_array_struct_query(session): def test_get_dtypes_array_struct_table(nested_df): dtypes = nested_df.dtypes - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( dtypes, pd.Series( { @@ -2194,8 +2163,7 @@ def test_len(scalars_dfs): ) @pytest.mark.parametrize( "write_engine", - # TODO(b/502298527): Reenable bigquery_write test - ["bigquery_load", "bigquery_streaming"], + ["bigquery_load", "bigquery_streaming", "bigquery_write"], ) def test_df_len_local(session, n_rows, write_engine): assert ( @@ -2634,7 +2602,7 @@ def test_combine( ) # Some dtype inconsistency for all-NULL columns - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pd.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) @pytest.mark.parametrize( @@ -2672,7 +2640,7 @@ def test_df_update(overwrite, filter_func): bf_df1.update(bf_df2, overwrite=overwrite, filter_func=filter_func) pd_df1.update(pd_df2, overwrite=overwrite, filter_func=filter_func) - bigframes.testing.utils.assert_frame_equal(bf_df1.to_pandas(), pd_df1) + pd.testing.assert_frame_equal(bf_df1.to_pandas(), pd_df1) def test_df_idxmin(): @@ -2684,7 +2652,7 @@ def test_df_idxmin(): bf_result = bf_df.idxmin().to_pandas() pd_result = pd_df.idxmin() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, check_index_type=False, check_dtype=False ) @@ -2698,7 +2666,7 @@ def test_df_idxmax(): bf_result = bf_df.idxmax().to_pandas() pd_result = pd_df.idxmax() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, check_index_type=False, check_dtype=False ) @@ -2716,6 +2684,7 @@ def test_df_idxmax(): ], ) def test_df_align(join, axis): + index1: pandas.Index = pandas.Index([1, 2, 3, 4], dtype="Int64") index2: pandas.Index = pandas.Index([1, 2, 4, 5], dtype="Int64") @@ -2738,12 +2707,8 @@ def test_df_align(join, axis): assert isinstance(bf_result1, dataframe.DataFrame) and isinstance( bf_result2, dataframe.DataFrame ) - bigframes.testing.utils.assert_frame_equal( - bf_result1.to_pandas(), pd_result1, check_dtype=False - ) - bigframes.testing.utils.assert_frame_equal( - bf_result2.to_pandas(), pd_result2, check_dtype=False - ) + pd.testing.assert_frame_equal(bf_result1.to_pandas(), pd_result1, check_dtype=False) + pd.testing.assert_frame_equal(bf_result2.to_pandas(), pd_result2, check_dtype=False) def test_combine_first( @@ -2768,7 +2733,7 @@ def test_combine_first( pd_result = pd_df_a.combine_first(pd_df_b) # Some dtype inconsistency for all-NULL columns - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pd.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) @pytest.mark.parametrize( @@ -2795,9 +2760,9 @@ def test_df_corr_w_numeric_only(scalars_dfs_maybe_ordered, columns, numeric_only # BigFrames and Pandas differ in their data type handling: # - Column types: BigFrames uses Float64, Pandas uses float64. # - Index types: BigFrames uses strign, Pandas uses object. - bigframes.testing.utils.assert_index_equal(bf_result.columns, pd_result.columns) + pd.testing.assert_index_equal(bf_result.columns, pd_result.columns) # Only check row order in ordered mode. - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, @@ -2839,9 +2804,9 @@ def test_cov_w_numeric_only(scalars_dfs_maybe_ordered, columns, numeric_only): # BigFrames and Pandas differ in their data type handling: # - Column types: BigFrames uses Float64, Pandas uses float64. # - Index types: BigFrames uses strign, Pandas uses object. - bigframes.testing.utils.assert_index_equal(bf_result.columns, pd_result.columns) + pd.testing.assert_index_equal(bf_result.columns, pd_result.columns) # Only check row order in ordered mode. - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, @@ -2862,7 +2827,7 @@ def test_df_corrwith_df(scalars_dfs_maybe_ordered): # BigFrames and Pandas differ in their data type handling: # - Column types: BigFrames uses Float64, Pandas uses float64. # - Index types: BigFrames uses strign, Pandas uses object. - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -2883,7 +2848,7 @@ def test_df_corrwith_df_numeric_only(scalars_dfs): # BigFrames and Pandas differ in their data type handling: # - Column types: BigFrames uses Float64, Pandas uses float64. # - Index types: BigFrames uses strign, Pandas uses object. - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -2912,7 +2877,7 @@ def test_df_corrwith_series(scalars_dfs_maybe_ordered): # BigFrames and Pandas differ in their data type handling: # - Column types: BigFrames uses Float64, Pandas uses float64. # - Index types: BigFrames uses strign, Pandas uses object. - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -3108,23 +3073,18 @@ def test_binop_with_self_aggregate(scalars_dfs_maybe_ordered): df_columns = ["int64_col", "float64_col", "int64_too"] - history_before = scalars_df._session.execution_history().to_dataframe() - queries_before = ( - len(history_before[history_before["job_type"] == "query"]) - if "job_type" in history_before.columns - else 0 - ) - + # Ensure that this takes the optimized single-query path by counting executions + execution_count_before = scalars_df._session._metrics.execution_count bf_df = scalars_df[df_columns] bf_result = (bf_df - bf_df.mean()).to_pandas() - - history_after = scalars_df._session.execution_history().to_dataframe() - queries_after = len(history_after[history_after["job_type"] == "query"]) + execution_count_after = scalars_df._session._metrics.execution_count pd_df = scalars_pandas_df[df_columns] pd_result = pd_df - pd_df.mean() - assert (queries_after - queries_before) == 1 + executions = execution_count_after - execution_count_before + + assert executions == 1 assert_frame_equal(bf_result, pd_result, check_dtype=False) @@ -3133,23 +3093,18 @@ def test_binop_with_self_aggregate_w_index_reset(scalars_dfs_maybe_ordered): df_columns = ["int64_col", "float64_col", "int64_too"] - history_before = scalars_df._session.execution_history().to_dataframe() - queries_before = ( - len(history_before[history_before["job_type"] == "query"]) - if "job_type" in history_before.columns - else 0 - ) - + # Ensure that this takes the optimized single-query path by counting executions + execution_count_before = scalars_df._session._metrics.execution_count bf_df = scalars_df[df_columns].reset_index(drop=True) bf_result = (bf_df - bf_df.mean()).to_pandas() - - history_after = scalars_df._session.execution_history().to_dataframe() - queries_after = len(history_after[history_after["job_type"] == "query"]) + execution_count_after = scalars_df._session._metrics.execution_count pd_df = scalars_pandas_df[df_columns].reset_index(drop=True) pd_result = pd_df - pd_df.mean() - assert (queries_after - queries_before) == 1 + executions = execution_count_after - execution_count_before + + assert executions == 1 pd_result.index = pd_result.index.astype("Int64") assert_frame_equal(bf_result, pd_result, check_dtype=False, check_index_type=False) @@ -3193,7 +3148,7 @@ def test_binop_df_df_binary_op( pd_result = pd_df_a - pd_df_b # Some dtype inconsistency for all-NULL columns - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pd.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) # Differnt table will only work for explicit index, since default index orders are arbitrary. @@ -3303,11 +3258,9 @@ def test_join_different_table_with_duplicate_column_name( pd_result = pd_df_a.join(pd_df_b, how=how, lsuffix="_l", rsuffix="_r") # Ensure no inplace changes - bigframes.testing.utils.assert_index_equal(bf_df_a.columns, pd_df_a.columns) - bigframes.testing.utils.assert_index_equal(bf_df_b.index.to_pandas(), pd_df_b.index) - bigframes.testing.utils.assert_frame_equal( - bf_result, pd_result, check_index_type=False - ) + pd.testing.assert_index_equal(bf_df_a.columns, pd_df_a.columns) + pd.testing.assert_index_equal(bf_df_b.index.to_pandas(), pd_df_b.index) + pd.testing.assert_frame_equal(bf_result, pd_result, check_index_type=False) @all_joins @@ -3335,14 +3288,14 @@ def test_join_param_on_with_duplicate_column_name_not_on_col( pd_result = pd_df_a.join( pd_df_b, on="int64_too", how=how, lsuffix="_l", rsuffix="_r" ) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.sort_index(), pd_result.sort_index(), check_like=True, check_index_type=False, check_names=False, ) - bigframes.testing.utils.assert_index_equal(bf_result.columns, pd_result.columns) + pd.testing.assert_index_equal(bf_result.columns, pd_result.columns) @pytest.mark.skipif( @@ -3373,14 +3326,14 @@ def test_join_param_on_with_duplicate_column_name_on_col( pd_result = pd_df_a.join( pd_df_b, on="int64_too", how=how, lsuffix="_l", rsuffix="_r" ) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.sort_index(), pd_result.sort_index(), check_like=True, check_index_type=False, check_names=False, ) - bigframes.testing.utils.assert_index_equal(bf_result.columns, pd_result.columns) + pd.testing.assert_index_equal(bf_result.columns, pd_result.columns) @all_joins @@ -3525,7 +3478,7 @@ def test_dataframe_numeric_analytic_op( bf_series = operator(scalars_df_index[columns]) pd_series = operator(scalars_pandas_df_index[columns]) bf_result = bf_series.to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_series, bf_result, check_dtype=False) + pd.testing.assert_frame_equal(pd_series, bf_result, check_dtype=False) @pytest.mark.parametrize( @@ -3550,7 +3503,7 @@ def test_dataframe_general_analytic_op( bf_series = operator(scalars_df_index[col_names]) pd_series = operator(scalars_pandas_df_index[col_names]) bf_result = bf_series.to_pandas() - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_series, bf_result, ) @@ -3568,7 +3521,7 @@ def test_dataframe_diff(scalars_df_index, scalars_pandas_df_index, periods): col_names = ["int64_too", "float64_col", "int64_col"] bf_result = scalars_df_index[col_names].diff(periods=periods).to_pandas() pd_result = scalars_pandas_df_index[col_names].diff(periods=periods) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, ) @@ -3587,7 +3540,7 @@ def test_dataframe_pct_change(scalars_df_index, scalars_pandas_df_index, periods bf_result = scalars_df_index[col_names].pct_change(periods=periods).to_pandas() # pandas 3.0 does not automatically ffill anymore pd_result = scalars_pandas_df_index[col_names].ffill().pct_change(periods=periods) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, ) @@ -3601,7 +3554,7 @@ def test_dataframe_agg_single_string(scalars_dfs): pd_result = scalars_pandas_df[numeric_cols].agg("sum") assert bf_result.dtype == "Float64" - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, check_dtype=False, check_index_type=False ) @@ -3621,7 +3574,7 @@ def test_dataframe_agg_int_single_string(scalars_dfs, agg): pd_result = scalars_pandas_df[numeric_cols].agg(agg) assert bf_result.dtype == "Int64" - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, check_dtype=False, check_index_type=False ) @@ -3676,7 +3629,7 @@ def test_dataframe_agg_int_multi_string(scalars_dfs): # Pandas may produce narrower numeric types # Pandas has object index type - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, check_dtype=False, check_index_type=False ) @@ -3719,7 +3672,7 @@ def test_df_transpose_repeated_uses_cache(): bf_df = bf_df.transpose() + i pd_df = pd_df.transpose() + i - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_df, bf_df.to_pandas(), check_dtype=False, check_index_type=False ) @@ -3762,7 +3715,7 @@ def test_df_melt_default(scalars_dfs): pd_result = scalars_pandas_df[columns].melt() # Pandas produces int64 index, Bigframes produces Int64 (nullable) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_index_type=False, @@ -3791,7 +3744,7 @@ def test_df_melt_parameterized(scalars_dfs): ) # Pandas produces int64 index, Bigframes produces Int64 (nullable) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_index_type=False, check_dtype=False ) @@ -3844,7 +3797,7 @@ def test_df_pivot(scalars_dfs, values, index, columns): # Pandas produces NaN, where bq dataframes produces pd.NA bf_result = bf_result.fillna(float("nan")) pd_result = pd_result.fillna(float("nan")) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pd.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) @pytest.mark.parametrize( @@ -3865,7 +3818,7 @@ def test_df_pivot_hockey(hockey_df, hockey_pandas_df, values, index, columns): ) # Pandas produces NaN, where bq dataframes produces pd.NA - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pd.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) @pytest.mark.parametrize( @@ -3906,7 +3859,7 @@ def test_df_pivot_table( aggfunc=aggfunc, fill_value=fill_value, ) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_column_type=False ) @@ -3986,7 +3939,7 @@ def test__dir__with_rename(scalars_dfs): def test_loc_select_columns_w_repeats(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index[["int64_col", "int64_col", "int64_too"]].to_pandas() pd_result = scalars_pandas_df_index[["int64_col", "int64_col", "int64_too"]] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4010,7 +3963,7 @@ def test_loc_select_columns_w_repeats(scalars_df_index, scalars_pandas_df_index) def test_iloc_slice(scalars_df_index, scalars_pandas_df_index, start, stop, step): bf_result = scalars_df_index.iloc[start:stop:step].to_pandas() pd_result = scalars_pandas_df_index.iloc[start:stop:step] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4028,7 +3981,7 @@ def test_iloc_slice_after_cache( scalars_df_index.cache() bf_result = scalars_df_index.iloc[start:stop:step].to_pandas() pd_result = scalars_pandas_df_index.iloc[start:stop:step] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4061,7 +4014,7 @@ def test_iloc_single_integer(scalars_df_index, scalars_pandas_df_index, index): bf_result = scalars_df_index.iloc[index] pd_result = scalars_pandas_df_index.iloc[index] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -4086,14 +4039,14 @@ def test_iloc_tuple_multi_columns(scalars_df_index, scalars_pandas_df_index, ind bf_result = scalars_df_index.iloc[index].to_pandas() pd_result = scalars_pandas_df_index.iloc[index] - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) def test_iloc_tuple_multi_columns_single_row(scalars_df_index, scalars_pandas_df_index): index = (2, [2, 1, 3, -4]) bf_result = scalars_df_index.iloc[index] pd_result = scalars_pandas_df_index.iloc[index] - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -4147,7 +4100,7 @@ def test_loc_bool_series(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.loc[scalars_df_index.bool_col].to_pandas() pd_result = scalars_pandas_df_index.loc[scalars_pandas_df_index.bool_col] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4158,7 +4111,7 @@ def test_loc_list_select_rows_and_columns(scalars_df_index, scalars_pandas_df_in bf_result = scalars_df_index.loc[idx_list, ["bool_col", "int64_col"]].to_pandas() pd_result = scalars_pandas_df_index.loc[idx_list, ["bool_col", "int64_col"]] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4167,7 +4120,7 @@ def test_loc_list_select_rows_and_columns(scalars_df_index, scalars_pandas_df_in def test_loc_select_column(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.loc[:, "int64_col"].to_pandas() pd_result = scalars_pandas_df_index.loc[:, "int64_col"] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -4178,7 +4131,7 @@ def test_loc_select_with_column_condition(scalars_df_index, scalars_pandas_df_in pd_result = scalars_pandas_df_index.loc[ :, scalars_pandas_df_index.dtypes == "Int64" ] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4201,7 +4154,7 @@ def test_loc_select_with_column_condition_bf_series( pd_result = scalars_pandas_df_index.loc[ :, scalars_pandas_df_index.nunique() > size_half ] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4215,7 +4168,7 @@ def test_loc_single_index_with_duplicate(scalars_df_index, scalars_pandas_df_ind index = "Hello, World!" bf_result = scalars_df_index.loc[index] pd_result = scalars_pandas_df_index.loc[index] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) @@ -4227,7 +4180,7 @@ def test_loc_single_index_no_duplicate(scalars_df_index, scalars_pandas_df_index index = -2345 bf_result = scalars_df_index.loc[index] pd_result = scalars_pandas_df_index.loc[index] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -4241,7 +4194,7 @@ def test_at_with_duplicate(scalars_df_index, scalars_pandas_df_index): index = "Hello, World!" bf_result = scalars_df_index.at[index, "int64_too"] pd_result = scalars_pandas_df_index.at[index, "int64_too"] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4266,7 +4219,7 @@ def test_loc_setitem_bool_series_scalar_new_col(scalars_dfs): # pandas uses float64 instead pd_df["new_col"] = pd_df["new_col"].astype("Float64") - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_df.to_pandas(), pd_df, ) @@ -4290,7 +4243,7 @@ def test_loc_setitem_bool_series_scalar_existing_col(scalars_dfs, col, value): bf_df.loc[bf_df["int64_too"] == 1, col] = value pd_df.loc[pd_df["int64_too"] == 1, col] = value - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_df.to_pandas(), pd_df, ) @@ -4443,9 +4396,7 @@ def test_dataframe_aggregates_quantile_mono(scalars_df_index, scalars_pandas_df_ # Pandas may produce narrower numeric types, but bigframes always produces Float64 pd_result = pd_result.astype("Float64") - bigframes.testing.utils.assert_series_equal( - bf_result, pd_result, check_index_type=False - ) + pd.testing.assert_series_equal(bf_result, pd_result, check_index_type=False) def test_dataframe_aggregates_quantile_multi(scalars_df_index, scalars_pandas_df_index): @@ -4458,7 +4409,7 @@ def test_dataframe_aggregates_quantile_multi(scalars_df_index, scalars_pandas_df pd_result = pd_result.astype("Float64") pd_result.index = pd_result.index.astype("Float64") - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -4484,9 +4435,7 @@ def test_dataframe_bool_aggregates(scalars_df_index, scalars_pandas_df_index, op bf_result = bf_series.to_pandas() pd_series.index = pd_series.index.astype(bf_result.index.dtype) - bigframes.testing.utils.assert_series_equal( - pd_series, bf_result, check_index_type=False - ) + pd.testing.assert_series_equal(pd_series, bf_result, check_index_type=False) def test_dataframe_prod(scalars_df_index, scalars_pandas_df_index): @@ -4498,9 +4447,7 @@ def test_dataframe_prod(scalars_df_index, scalars_pandas_df_index): # Pandas may produce narrower numeric types, but bigframes always produces Float64 pd_series = pd_series.astype("Float64") # Pandas has object index type - bigframes.testing.utils.assert_series_equal( - pd_series, bf_result, check_index_type=False - ) + pd.testing.assert_series_equal(pd_series, bf_result, check_index_type=False) def test_df_skew_too_few_values(scalars_dfs): @@ -4512,9 +4459,7 @@ def test_df_skew_too_few_values(scalars_dfs): # Pandas may produce narrower numeric types, but bigframes always produces Float64 pd_result = pd_result.astype("Float64") - bigframes.testing.utils.assert_series_equal( - pd_result, bf_result, check_index_type=False - ) + pd.testing.assert_series_equal(pd_result, bf_result, check_index_type=False) @pytest.mark.parametrize( @@ -4547,9 +4492,7 @@ def test_df_kurt_too_few_values(scalars_dfs): # Pandas may produce narrower numeric types, but bigframes always produces Float64 pd_result = pd_result.astype("Float64") - bigframes.testing.utils.assert_series_equal( - pd_result, bf_result, check_index_type=False - ) + pd.testing.assert_series_equal(pd_result, bf_result, check_index_type=False) def test_df_kurt(scalars_dfs): @@ -4561,9 +4504,7 @@ def test_df_kurt(scalars_dfs): # Pandas may produce narrower numeric types, but bigframes always produces Float64 pd_result = pd_result.astype("Float64") - bigframes.testing.utils.assert_series_equal( - pd_result, bf_result, check_index_type=False - ) + pd.testing.assert_series_equal(pd_result, bf_result, check_index_type=False) @pytest.mark.parametrize( @@ -4647,7 +4588,7 @@ def test_df_add_prefix(scalars_df_index, scalars_pandas_df_index, axis): pd_result = scalars_pandas_df_index.add_prefix("prefix_", axis) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_index_type=False, @@ -4668,7 +4609,7 @@ def test_df_add_suffix(scalars_df_index, scalars_pandas_df_index, axis): pd_result = scalars_pandas_df_index.add_suffix("_suffix", axis) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_index_type=False, @@ -4688,7 +4629,7 @@ def test_df_columns_filter_items(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index.filter(items=["string_col", "int64_col"]) # Ignore column ordering as pandas order differently depending on version - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.sort_index(axis=1), pd_result.sort_index(axis=1), ) @@ -4699,7 +4640,7 @@ def test_df_columns_filter_like(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index.filter(like="64_col") - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4710,7 +4651,7 @@ def test_df_columns_filter_regex(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index.filter(regex="^[^_]+$") - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4742,7 +4683,7 @@ def test_df_rows_filter_like(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index.filter(like="ello", axis=0) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4756,7 +4697,7 @@ def test_df_rows_filter_regex(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index.filter(regex="^[GH].*", axis=0) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4787,7 +4728,7 @@ def test_df_reindex_rows_index(scalars_df_index, scalars_pandas_df_index): # Pandas uses int64 instead of Int64 (nullable) dtype. pd_result.index = pd_result.index.astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4812,7 +4753,7 @@ def test_df_reindex_columns(scalars_df_index, scalars_pandas_df_index): # Pandas uses float64 as default for newly created empty column, bf uses Float64 pd_result.not_a_col = pd_result.not_a_col.astype(pandas.Float64Dtype()) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4827,7 +4768,7 @@ def test_df_reindex_columns_with_same_order(scalars_df_index, scalars_pandas_df_ bf_result = bf.reindex(columns=columns).to_pandas() pd_result = pd_df.reindex(columns=columns) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4917,7 +4858,7 @@ def test_df_reindex_like(scalars_df_index, scalars_pandas_df_index): pd_result.index = pd_result.index.astype(pd.Int64Dtype()) # Pandas uses float64 as default for newly created empty column, bf uses Float64 pd_result.not_a_col = pd_result.not_a_col.astype(pandas.Float64Dtype()) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -4928,7 +4869,7 @@ def test_df_values(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index.values # Numpy isn't equipped to compare non-numeric objects, so convert back to dataframe - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd.DataFrame(bf_result), pd.DataFrame(pd_result), check_dtype=False ) @@ -4938,7 +4879,7 @@ def test_df_to_numpy(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index.to_numpy() # Numpy isn't equipped to compare non-numeric objects, so convert back to dataframe - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd.DataFrame(bf_result), pd.DataFrame(pd_result), check_dtype=False ) @@ -4948,7 +4889,7 @@ def test_df___array__(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index.__array__() # Numpy isn't equipped to compare non-numeric objects, so convert back to dataframe - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd.DataFrame(bf_result), pd.DataFrame(pd_result), check_dtype=False ) @@ -5040,7 +4981,7 @@ def test_loc_list_string_index(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.loc[index_list].to_pandas() pd_result = scalars_pandas_df_index.loc[index_list] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -5052,7 +4993,7 @@ def test_loc_list_integer_index(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.loc[index_list] pd_result = scalars_pandas_df_index.loc[index_list] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) @@ -5087,7 +5028,7 @@ def test_iloc_list(scalars_df_index, scalars_pandas_df_index, index_list): bf_result = scalars_df_index.iloc[index_list] pd_result = scalars_pandas_df_index.iloc[index_list] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) @@ -5107,7 +5048,7 @@ def test_iloc_list_partial_ordering( bf_result = scalars_df_partial_ordering.iloc[index_list] pd_result = scalars_pandas_df_index.iloc[index_list] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) @@ -5125,13 +5066,14 @@ def test_iloc_list_multiindex(scalars_dfs): bf_result = scalars_df.iloc[index_list] pd_result = scalars_pandas_df.iloc[index_list] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) def test_iloc_empty_list(scalars_df_index, scalars_pandas_df_index): + index_list: List[int] = [] bf_result = scalars_df_index.iloc[index_list] @@ -5145,7 +5087,7 @@ def test_rename_axis(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.rename_axis("newindexname") pd_result = scalars_pandas_df_index.rename_axis("newindexname") - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) @@ -5155,7 +5097,7 @@ def test_rename_axis_nonstring(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.rename_axis((4,)) pd_result = scalars_pandas_df_index.rename_axis((4,)) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) @@ -5171,7 +5113,7 @@ def test_loc_bf_series_string_index(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.loc[bf_string_series] pd_result = scalars_pandas_df_index.loc[pd_string_series] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) @@ -5189,7 +5131,7 @@ def test_loc_bf_series_multiindex(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_multiindex.loc[bf_string_series] pd_result = scalars_pandas_df_multiindex.loc[pd_string_series] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) @@ -5202,7 +5144,7 @@ def test_loc_bf_index_integer_index(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.loc[bf_index] pd_result = scalars_pandas_df_index.loc[pd_index] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) @@ -5222,7 +5164,7 @@ def test_loc_bf_index_integer_index_renamed_col( bf_result = scalars_df_index.loc[bf_index] pd_result = scalars_pandas_df_index.loc[pd_index] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, ) @@ -5248,7 +5190,7 @@ def test_df_drop_duplicates(scalars_df_index, scalars_pandas_df_index, keep, sub columns = ["bool_col", "int64_too", "int64_col"] bf_df = scalars_df_index[columns].drop_duplicates(subset, keep=keep).to_pandas() pd_df = scalars_pandas_df_index[columns].drop_duplicates(subset, keep=keep) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_df, bf_df, ) @@ -5275,7 +5217,7 @@ def test_df_drop_duplicates_w_json(json_df, keep): pd_df = json_pandas_df.drop_duplicates(keep=keep) pd_df["json_col"] = pd_df["json_col"].astype(dtypes.JSON_DTYPE) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_df, bf_df, ) @@ -5300,7 +5242,7 @@ def test_df_duplicated(scalars_df_index, scalars_pandas_df_index, keep, subset): columns = ["bool_col", "int64_too", "int64_col"] bf_series = scalars_df_index[columns].duplicated(subset, keep=keep).to_pandas() pd_series = scalars_pandas_df_index[columns].duplicated(subset, keep=keep) - bigframes.testing.utils.assert_series_equal(pd_series, bf_series, check_dtype=False) + pd.testing.assert_series_equal(pd_series, bf_series, check_dtype=False) def test_df_from_dict_columns_orient(): @@ -5353,10 +5295,7 @@ def test_df_to_dict(scalars_df_index, scalars_pandas_df_index): def test_df_to_excel(scalars_df_index, scalars_pandas_df_index): unsupported = ["timestamp_col"] - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.drop(columns=unsupported).to_excel(bf_result_file) scalars_pandas_df_index.drop(columns=unsupported).to_excel(pd_result_file) bf_result = bf_result_file.read() @@ -5374,12 +5313,9 @@ def test_df_to_latex(scalars_df_index, scalars_pandas_df_index): def test_df_to_json_local_str(scalars_df_index, scalars_pandas_df_index): - # pandas 3.0 bugged for serializing date col - bf_result = scalars_df_index.drop(columns="date_col").to_json() + bf_result = scalars_df_index.to_json() # default_handler for arrow types that have no default conversion - pd_result = scalars_pandas_df_index.drop(columns="date_col").to_json( - default_handler=str - ) + pd_result = scalars_pandas_df_index.to_json(default_handler=str) assert bf_result == pd_result @@ -5390,10 +5326,7 @@ def test_df_to_json_local_file(scalars_df_index, scalars_pandas_df_index): # duration not fully supported at pandas level scalars_df_index = scalars_df_index.drop(columns="duration_col") scalars_pandas_df_index = scalars_pandas_df_index.drop(columns="duration_col") - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.to_json(bf_result_file, orient="table") # default_handler for arrow types that have no default conversion scalars_pandas_df_index.to_json( @@ -5415,10 +5348,7 @@ def test_df_to_csv_local_str(scalars_df_index, scalars_pandas_df_index): def test_df_to_csv_local_file(scalars_df_index, scalars_pandas_df_index): - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.to_csv(bf_result_file) scalars_pandas_df_index.to_csv(pd_result_file) @@ -5442,10 +5372,7 @@ def test_df_to_parquet_local_bytes(scalars_df_index, scalars_pandas_df_index): def test_df_to_parquet_local_file(scalars_df_index, scalars_pandas_df_index): # GEOGRAPHY not supported in parquet export. unsupported = ["geography_col"] - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.drop(columns=unsupported).to_parquet(bf_result_file) scalars_pandas_df_index.drop(columns=unsupported).to_parquet(pd_result_file) @@ -5492,10 +5419,7 @@ def test_df_to_markdown(scalars_df_index, scalars_pandas_df_index): def test_df_to_pickle(scalars_df_index, scalars_pandas_df_index): - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.to_pickle(bf_result_file) scalars_pandas_df_index.to_pickle(pd_result_file) bf_result = bf_result_file.read() @@ -5544,7 +5468,7 @@ def test_df_eval(scalars_dfs, expr): bf_result = scalars_df.eval(expr).to_pandas() pd_result = scalars_pandas_df.eval(expr) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -5565,7 +5489,7 @@ def test_df_query(scalars_dfs, expr): bf_result = scalars_df.query(expr).to_pandas() pd_result = scalars_pandas_df.query(expr) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -5590,12 +5514,8 @@ def test_df_value_counts(scalars_dfs, subset, normalize, ascending, dropna): subset, normalize=normalize, ascending=ascending, dropna=dropna ) - bigframes.testing.utils.assert_series_equal( - bf_result, - pd_result, - check_dtype=False, - check_index_type=False, - ignore_order=True, # different pandas versions inconsistent for tie-handling + pd.testing.assert_series_equal( + bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -5642,7 +5562,7 @@ def test_df_rank_with_nulls( .astype(pd.Float64Dtype()) ) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -5739,7 +5659,7 @@ def test_df_dot_inline(session): pd_result[name] = pd_result[name].astype(pd.Int64Dtype()) pd_result.index = pd_result.index.astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -5756,7 +5676,7 @@ def test_df_dot( for name in pd_result.columns: pd_result[name] = pd_result[name].astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -5773,7 +5693,7 @@ def test_df_dot_operator( for name in pd_result.columns: pd_result[name] = pd_result[name].astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, ) @@ -5796,7 +5716,7 @@ def test_df_dot_series_inline(): pd_result = pd_result.astype(pd.Int64Dtype()) pd_result.index = pd_result.index.astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -5812,7 +5732,7 @@ def test_df_dot_series( # Pandas result is object instead of Int64 (nullable) dtype. pd_result = pd_result.astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -5828,15 +5748,22 @@ def test_df_dot_operator_series( # Pandas result is object instead of Int64 (nullable) dtype. pd_result = pd_result.astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) +# TODO(tswast): We may be able to re-enable this test after we break large +# queries up in https://github.com/googleapis/python-bigquery-dataframes/pull/427 +@pytest.mark.skipif( + sys.version_info >= (3, 12), + # See: https://github.com/python/cpython/issues/112282 + reason="setrecursionlimit has no effect on the Python C stack since Python 3.12.", +) def test_recursion_limit(scalars_df_index): scalars_df_index = scalars_df_index[["int64_too", "int64_col", "float64_col"]] - for i in range(250): + for i in range(400): scalars_df_index = scalars_df_index + 4 scalars_df_index.to_pandas() @@ -5941,45 +5868,6 @@ def test_to_gbq_table_labels(scalars_df_index): assert table.labels["test"] == "labels" -def test_to_gbq_obj_ref_persists(session): - # Test that saving and loading an Object Reference retains its dtype - import uuid - - import google.cloud.bigquery - - sql = """ - SELECT STRUCT('gs://cloud-samples-data/vision/ocr/sign.jpg' AS uri, CAST(NULL AS STRING) AS version, CAST(NULL AS STRING) AS authorizer, PARSE_JSON('{}') AS details) AS uris - """ - df_init = session.read_gbq(sql) - - tmp_table_id = f"bigframes-dev.bigframes_tests_sys.tmp_obj_ref_{uuid.uuid4().hex}" - df_init.to_gbq(tmp_table_id, if_exists="replace") - - client = session.bqclient - table = client.get_table(tmp_table_id) - schema = list(table.schema) - for i, field in enumerate(schema): - if field.name == "uris": - schema[i] = google.cloud.bigquery.SchemaField( - name=field.name, - field_type=field.field_type, - mode=field.mode, - description="bigframes_dtype: OBJ_REF_DTYPE", - fields=field.fields, - ) - break - table.schema = schema - client.update_table(table, ["schema"]) - - bdf = session.read_gbq(tmp_table_id) - - destination_table = "bigframes-dev.bigframes_tests_sys.test_obj_ref_persistence" - bdf.to_gbq(destination_table, if_exists="replace") - - loaded_df = session.read_gbq(destination_table) - assert loaded_df["uris"].dtype == dtypes.OBJ_REF_DTYPE - - @pytest.mark.parametrize( ("col_names", "ignore_index"), [ @@ -5998,24 +5886,18 @@ def test_dataframe_explode(col_names, ignore_index, session): "C": [["a", "b", "c"], np.nan, ["d", "e"]], } + metrics = session._metrics df = bpd.DataFrame(data, session=session) pd_df = df.to_pandas() pd_result = pd_df.explode(col_names, ignore_index=ignore_index) bf_result = df.explode(col_names, ignore_index=ignore_index) - history_pre = session.execution_history().to_dataframe() - queries_pre = ( - len(history_pre[history_pre["job_type"] == "query"]) - if "job_type" in history_pre.columns - else 0 - ) - + # Check that to_pandas() results in at most a single query execution + execs_pre = metrics.execution_count bf_materialized = bf_result.to_pandas() + execs_post = metrics.execution_count - history_post = session.execution_history().to_dataframe() - queries_post = len(history_post[history_post["job_type"] == "query"]) - - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_materialized, pd_result, check_index_type=False, @@ -6023,7 +5905,7 @@ def test_dataframe_explode(col_names, ignore_index, session): ) # we test this property on this method in particular as compilation # is non-deterministic and won't use the query cache as implemented - assert (queries_post - queries_pre) <= 1 + assert execs_post - execs_pre <= 1 @pytest.mark.parametrize( @@ -6046,7 +5928,7 @@ def test_dataframe_explode_reserve_order(ignore_index, ordered): pd_res = pd_df.explode(["a", "b"], ignore_index=ignore_index).astype( pd.Int64Dtype() ) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( res if ordered else res.sort_index(), pd_res, check_index_type=False, @@ -6082,12 +5964,7 @@ def test_resample_with_column( scalars_df_index, scalars_pandas_df_index, on, rule, origin ): # TODO: supply a reason why this isn't compatible with pandas 1.x - pytest.importorskip("pandas", minversion="2.2.0") - # TODO: supply a reason why this isn't compatible with pandas 1.x - if pandas.__version__.startswith("3"): - pytest.skip( - "pandas 3.0 behavior diverges for day offsets: https://github.com/pandas-dev/pandas/pull/61985" - ) + pytest.importorskip("pandas", minversion="2.0.0") bf_result = ( scalars_df_index.resample(rule=rule, on=on, origin=origin)[ ["int64_col", "int64_too"] @@ -6098,9 +5975,7 @@ def test_resample_with_column( pd_result = scalars_pandas_df_index.resample(rule=rule, on=on, origin=origin)[ ["int64_col", "int64_too"] ].max() - # TODO: (b/484364312) - pd_result.index.names = bf_result.index.names - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -6139,10 +6014,6 @@ def test_resample_with_index( label, ): # TODO: supply a reason why this isn't compatible with pandas 1.x - if rule == "100d" and pandas.__version__.startswith("3"): - pytest.skip( - "pandas 3.0 behavior diverges for day offsets: https://github.com/pandas-dev/pandas/pull/61985" - ) pytest.importorskip("pandas", minversion="2.0.0") scalars_df_index = scalars_df_index.set_index(index_col, append=index_append) scalars_pandas_df_index = scalars_pandas_df_index.set_index( @@ -6159,8 +6030,6 @@ def test_resample_with_index( .resample(rule=rule, level=level, closed=closed, origin=origin, label=label) .min() ) - # TODO: (b/484364312) - pd_result.index.names = bf_result.index.names assert_frame_equal(bf_result, pd_result) @@ -6214,9 +6083,7 @@ def test_resample_start_time(rule, origin, data): pd_result = scalars_pandas_df_index.resample(rule=rule, origin=origin).min() - # TODO: (b/484364312) - pd_result.index.names = bf_result.index.names - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -6241,9 +6108,7 @@ def test_df_astype(scalars_dfs, dtype): bf_result = bf_df.astype(dtype).to_pandas() pd_result = pd_df.astype(dtype) - bigframes.testing.utils.assert_frame_equal( - bf_result, pd_result, check_index_type=False - ) + pd.testing.assert_frame_equal(bf_result, pd_result, check_index_type=False) def test_df_astype_python_types(scalars_dfs): @@ -6257,9 +6122,7 @@ def test_df_astype_python_types(scalars_dfs): {"bool_col": "string[pyarrow]", "int64_col": pd.Float64Dtype()} ) - bigframes.testing.utils.assert_frame_equal( - bf_result, pd_result, check_index_type=False - ) + pd.testing.assert_frame_equal(bf_result, pd_result, check_index_type=False) def test_astype_invalid_type_fail(scalars_dfs): @@ -6279,16 +6142,11 @@ def test_agg_with_dict_lists_strings(scalars_dfs): bf_result = bf_df.agg(agg_funcs).to_pandas() pd_result = pd_df.agg(agg_funcs) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) -@pytest.mark.skipif( - pandas.__version__.startswith("3"), - # See: https://github.com/python/cpython/issues/112282 - reason="pandas 3.0 miscaculates variance", -) def test_agg_with_dict_lists_callables(scalars_dfs): bf_df, pd_df = scalars_dfs agg_funcs = { @@ -6299,7 +6157,7 @@ def test_agg_with_dict_lists_callables(scalars_dfs): bf_result = bf_df.agg(agg_funcs).to_pandas() pd_result = pd_df.agg(agg_funcs) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -6314,7 +6172,7 @@ def test_agg_with_dict_list_and_str(scalars_dfs): bf_result = bf_df.agg(agg_funcs).to_pandas() pd_result = pd_df.agg(agg_funcs) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -6331,7 +6189,7 @@ def test_agg_with_dict_strs(scalars_dfs): pd_result = pd_df.agg(agg_funcs) pd_result.index = pd_result.index.astype("string[pyarrow]") - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -6353,7 +6211,7 @@ def test_df_agg_with_builtins(scalars_dfs): .agg({"int64_col": [len, sum, min, max, list], "bool_col": [all, any, max]}) ) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -6367,20 +6225,3 @@ def test_agg_with_dict_containing_non_existing_col_raise_key_error(scalars_dfs): with pytest.raises(KeyError): bf_df.agg(agg_funcs) - - -def test_empty_agg_projection_succeeds(): - # Tests that the compiler generates a SELECT 1 fallback for empty aggregations, - # protecting against BigQuery syntax errors when both groups and metrics are empty. - import importlib - - bq = importlib.import_module( - "bigframes_vendored.ibis.backends.sql.compilers.bigquery" - ) - sg = importlib.import_module("bigframes_vendored.sqlglot") - - compiler = bq.BigQueryCompiler() - res = compiler.visit_Aggregate( - "op", parent=sg.table("parent_table"), groups=[], metrics=[] - ) - assert "SELECT 1" in res.sql() diff --git a/tests/system/small/test_dataframe_io.py b/tests/system/small/test_dataframe_io.py index ef21e929afa..02acb8d8f25 100644 --- a/tests/system/small/test_dataframe_io.py +++ b/tests/system/small/test_dataframe_io.py @@ -12,27 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -import typing from typing import Tuple import google.api_core.exceptions import numpy import numpy.testing import pandas as pd +import pandas.testing import pyarrow as pa import pytest + +import bigframes.dtypes as dtypes +from bigframes.testing import utils + +try: + import pandas_gbq # type: ignore +except ImportError: # pragma: NO COVER + # TODO(b/332758806): Run system tests without "extras" + pandas_gbq = None + +import typing + from google.cloud import bigquery import bigframes import bigframes.dataframe -import bigframes.dtypes as dtypes import bigframes.enums import bigframes.features import bigframes.pandas as bpd -import bigframes.testing -from bigframes.testing import utils - -pandas_gbq = pytest.importorskip("pandas_gbq") def test_sql_executes(scalars_df_default_index, bigquery_client): @@ -62,8 +69,7 @@ def test_sql_executes(scalars_df_default_index, bigquery_client): .sort_values("rowindex") .reset_index(drop=True) ) - bq_result["bytes_col"] = bq_result["bytes_col"].astype(dtypes.BYTES_DTYPE) - bigframes.testing.utils.assert_frame_equal(bf_result, bq_result, check_dtype=False) + pandas.testing.assert_frame_equal(bf_result, bq_result, check_dtype=False) def test_sql_executes_and_includes_named_index( @@ -94,8 +100,7 @@ def test_sql_executes_and_includes_named_index( .set_index("string_col") .sort_values("rowindex") ) - bq_result["bytes_col"] = bq_result["bytes_col"].astype(dtypes.BYTES_DTYPE) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, bq_result, check_dtype=False, check_index_type=False ) @@ -128,8 +133,7 @@ def test_sql_executes_and_includes_named_multiindex( .set_index(["string_col", "bool_col"]) .sort_values("rowindex") ) - bq_result["bytes_col"] = bq_result["bytes_col"].astype(dtypes.BYTES_DTYPE) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, bq_result, check_dtype=False, check_index_type=False ) @@ -253,6 +257,7 @@ def test_to_pandas_override_global_option(scalars_df_index): # Direct call to_pandas uses global default setting (allow_large_results=True), # table has 'bqdf' prefix. with bigframes.option_context("compute.allow_large_results", True): + scalars_df_index.to_pandas() table_id = scalars_df_index._query_job.destination.table_id assert table_id is not None @@ -323,6 +328,7 @@ def test_to_pandas_dry_run(session, scalars_pandas_df_multi_index): def test_to_arrow_override_global_option(scalars_df_index): # Direct call to_arrow uses global default setting (allow_large_results=True), with bigframes.option_context("compute.allow_large_results", True): + scalars_df_index.to_arrow() table_id = scalars_df_index._query_job.destination.table_id assert table_id is not None @@ -357,8 +363,8 @@ def test_to_pandas_batches_w_empty_dataframe(session): { "idx1": [], "idx2": [], - "col1": pd.Series([], dtype="string[pyarrow]"), - "col2": pd.Series([], dtype="Int64"), + "col1": pandas.Series([], dtype="string[pyarrow]"), + "col2": pandas.Series([], dtype="Int64"), }, session=session, ).set_index(["idx1", "idx2"], drop=True) @@ -367,7 +373,7 @@ def test_to_pandas_batches_w_empty_dataframe(session): assert len(results) == 1 assert list(results[0].index.names) == ["idx1", "idx2"] assert list(results[0].columns) == ["col1", "col2"] - bigframes.testing.utils.assert_series_equal(results[0].dtypes, empty.dtypes) + pandas.testing.assert_series_equal(results[0].dtypes, empty.dtypes) @pytest.mark.skipif( @@ -508,9 +514,8 @@ def test_to_csv_index( dtype = scalars_df.reset_index().dtypes.to_dict() dtype.pop("geography_col") dtype.pop("rowindex") - # read_csv will decode into bytes, numeric inproperly, convert_pandas_dtypes will encode properly from string + # read_csv will decode into bytes inproperly, convert_pandas_dtypes will encode properly from string dtype.pop("bytes_col") - dtype.pop("numeric_col") gcs_df = pd.read_csv( utils.get_first_file_from_wildcard(path), dtype=dtype, @@ -547,9 +552,8 @@ def test_to_csv_tabs( dtype = scalars_df.reset_index().dtypes.to_dict() dtype.pop("geography_col") dtype.pop("rowindex") - # read_csv will decode into bytes, numeric inproperly, convert_pandas_dtypes will encode properly from string + # read_csv will decode into bytes inproperly, convert_pandas_dtypes will encode properly from string dtype.pop("bytes_col") - dtype.pop("numeric_col") gcs_df = pd.read_csv( utils.get_first_file_from_wildcard(path), sep="\t", @@ -571,7 +575,7 @@ def test_to_csv_tabs( ("index"), [True, False], ) -@pytest.mark.skipif(pandas_gbq is None, reason="required by pandas_gbq.read_gbq") +@pytest.mark.skipif(pandas_gbq is None, reason="required by pd.read_gbq") def test_to_gbq_w_index(scalars_dfs, dataset_id, index): """Test the `to_gbq` API with the `index` parameter.""" scalars_df, scalars_pandas_df = scalars_dfs @@ -584,7 +588,7 @@ def test_to_gbq_w_index(scalars_dfs, dataset_id, index): index_col = None df_in.to_gbq(destination_table, if_exists="replace", index=index) - df_out = pandas_gbq.read_gbq(destination_table, index_col=index_col) + df_out = pd.read_gbq(destination_table, index_col=index_col) if index: df_out = df_out.sort_index() @@ -592,7 +596,7 @@ def test_to_gbq_w_index(scalars_dfs, dataset_id, index): df_out = df_out.sort_values("rowindex_2").reset_index(drop=True) utils.convert_pandas_dtypes(df_out, bytes_col=False) - # pandas_gbq.read_gbq interprets bytes_col as object, reconvert to pyarrow binary + # pd.read_gbq interprets bytes_col as object, reconvert to pyarrow binary df_out["bytes_col"] = df_out["bytes_col"].astype(pd.ArrowDtype(pa.binary())) expected = scalars_pandas_df.copy() expected.index.name = index_col @@ -604,7 +608,7 @@ def test_to_gbq_if_exists_is_fail(scalars_dfs, dataset_id): destination_table = f"{dataset_id}.test_to_gbq_if_exists_is_fails" scalars_df.to_gbq(destination_table) - gcs_df = pandas_gbq.read_gbq(destination_table, index_col="rowindex") + gcs_df = pd.read_gbq(destination_table, index_col="rowindex") assert len(gcs_df) == len(scalars_pandas_df) pd.testing.assert_index_equal(gcs_df.columns, scalars_pandas_df.columns) @@ -621,27 +625,20 @@ def test_to_gbq_if_exists_is_replace(scalars_dfs, dataset_id): destination_table = f"{dataset_id}.test_to_gbq_if_exists_is_replace" scalars_df.to_gbq(destination_table) - gcs_df = pandas_gbq.read_gbq(destination_table, index_col="rowindex") + gcs_df = pd.read_gbq(destination_table, index_col="rowindex") assert len(gcs_df) == len(scalars_pandas_df) pd.testing.assert_index_equal(gcs_df.columns, scalars_pandas_df.columns) # When replacing a table with same schema scalars_df.to_gbq(destination_table, if_exists="replace") - gcs_df = pandas_gbq.read_gbq(destination_table, index_col="rowindex") + gcs_df = pd.read_gbq(destination_table, index_col="rowindex") assert len(gcs_df) == len(scalars_pandas_df) pd.testing.assert_index_equal(gcs_df.columns, scalars_pandas_df.columns) - # When replacing a table with same schema but different column order - reordered_df = scalars_df[scalars_df.columns[::-1]] - reordered_df.to_gbq(destination_table, if_exists="replace") - gcs_df = pandas_gbq.read_gbq(destination_table, index_col="rowindex") - assert len(gcs_df) == len(scalars_pandas_df) - pd.testing.assert_index_equal(gcs_df.columns, reordered_df.columns) - # When replacing a table with different schema partitial_scalars_df = scalars_df.drop(columns=["string_col"]) partitial_scalars_df.to_gbq(destination_table, if_exists="replace") - gcs_df = pandas_gbq.read_gbq(destination_table, index_col="rowindex") + gcs_df = pd.read_gbq(destination_table, index_col="rowindex") assert len(gcs_df) == len(partitial_scalars_df) pd.testing.assert_index_equal(gcs_df.columns, partitial_scalars_df.columns) @@ -651,20 +648,20 @@ def test_to_gbq_if_exists_is_append(scalars_dfs, dataset_id): destination_table = f"{dataset_id}.test_to_gbq_if_exists_is_append" scalars_df.to_gbq(destination_table) - gcs_df = pandas_gbq.read_gbq(destination_table, index_col="rowindex") + gcs_df = pd.read_gbq(destination_table, index_col="rowindex") assert len(gcs_df) == len(scalars_pandas_df) pd.testing.assert_index_equal(gcs_df.columns, scalars_pandas_df.columns) # When appending to a table with same schema scalars_df.to_gbq(destination_table, if_exists="append") - gcs_df = pandas_gbq.read_gbq(destination_table, index_col="rowindex") + gcs_df = pd.read_gbq(destination_table, index_col="rowindex") assert len(gcs_df) == 2 * len(scalars_pandas_df) pd.testing.assert_index_equal(gcs_df.columns, scalars_pandas_df.columns) # When appending to a table with different schema partitial_scalars_df = scalars_df.drop(columns=["string_col"]) partitial_scalars_df.to_gbq(destination_table, if_exists="append") - gcs_df = pandas_gbq.read_gbq(destination_table, index_col="rowindex") + gcs_df = pd.read_gbq(destination_table, index_col="rowindex") assert len(gcs_df) == 3 * len(partitial_scalars_df) pd.testing.assert_index_equal(gcs_df.columns, scalars_df.columns) @@ -849,8 +846,6 @@ def test_to_gbq_w_None_column_names( """Test the `to_gbq` API with None as a column name.""" destination_table = f"{dataset_id}.test_to_gbq_w_none_column_names" - # pandas 3.0 str datatypes produces nan instead of None, so cast to object - # scalars_df_index.columns = scalars_df_index.columns.astype(object) scalars_df_index = scalars_df_index.rename(columns={"int64_too": None}) scalars_df_index.to_gbq(destination_table, if_exists="replace") @@ -1009,51 +1004,6 @@ def test_to_gbq_timedelta_tag_ignored_when_appending(bigquery_client, dataset_id assert table.schema[0].description is None -def test_to_gbq_obj_ref(session, dataset_id: str, bigquery_client): - import uuid - - import google.cloud.bigquery - - destination_table = f"{dataset_id}.test_to_gbq_obj_ref" - sql = """ - SELECT STRUCT('gs://cloud-samples-data/vision/ocr/sign.jpg' AS uri, CAST(NULL AS STRING) AS version, CAST(NULL AS STRING) AS authorizer, PARSE_JSON('{}') AS details) AS uri_col - """ - df_init = session.read_gbq(sql) - - tmp_table_id = f"{dataset_id}.tmp_obj_ref_{uuid.uuid4().hex}" - df_init.to_gbq(tmp_table_id, if_exists="replace") - - client = session.bqclient - table = client.get_table(tmp_table_id) - schema = list(table.schema) - for i, field in enumerate(schema): - if field.name == "uri_col": - schema[i] = google.cloud.bigquery.SchemaField( - name=field.name, - field_type=field.field_type, - mode=field.mode, - description="bigframes_dtype: OBJ_REF_DTYPE", - fields=field.fields, - ) - break - table.schema = schema - client.update_table(table, ["schema"]) - - df = session.read_gbq(tmp_table_id) - df = df.rename(columns={"uri_col": "obj_ref_col"}) - - df.to_gbq(destination_table, if_exists="replace") - - table = bigquery_client.get_table(destination_table) - obj_ref_field = next(f for f in table.schema if f.name == "obj_ref_col") - assert obj_ref_field.field_type == "RECORD" - assert obj_ref_field.description == "bigframes_dtype: OBJ_REF_DTYPE" - - reloaded_df = session.read_gbq(destination_table) - assert reloaded_df["obj_ref_col"].dtype == dtypes.OBJ_REF_DTYPE - assert len(reloaded_df) == 1 - - @pytest.mark.parametrize( ("index"), [True, False], diff --git a/tests/system/small/test_encryption.py b/tests/system/small/test_encryption.py index db87184371b..1f30df451d2 100644 --- a/tests/system/small/test_encryption.py +++ b/tests/system/small/test_encryption.py @@ -15,9 +15,9 @@ import random +from google.cloud import bigquery import pandas import pytest -from google.cloud import bigquery import bigframes import bigframes.ml.linear_model diff --git a/tests/system/small/test_groupby.py b/tests/system/small/test_groupby.py index 8dde3146434..579e7cd414d 100644 --- a/tests/system/small/test_groupby.py +++ b/tests/system/small/test_groupby.py @@ -17,7 +17,7 @@ import pytest import bigframes.pandas as bpd -import bigframes.testing.utils +from bigframes.testing.utils import assert_frame_equal # ================= # DataFrame.groupby @@ -51,16 +51,14 @@ def test_dataframe_groupby_numeric_aggregate( pd_result = operator(scalars_pandas_df_index[col_names].groupby("string_col")) bf_result_computed = bf_result.to_pandas() # Pandas std function produces float64, not matching Float64 from bigframes - bigframes.testing.utils.assert_frame_equal( - pd_result, bf_result_computed, check_dtype=False - ) + pd.testing.assert_frame_equal(pd_result, bf_result_computed, check_dtype=False) def test_dataframe_groupby_head(scalars_df_index, scalars_pandas_df_index): col_names = ["int64_too", "float64_col", "int64_col", "bool_col", "string_col"] bf_result = scalars_df_index[col_names].groupby("bool_col").head(2).to_pandas() pd_result = scalars_pandas_df_index[col_names].groupby("bool_col").head(2) - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_frame_equal(pd_result, bf_result, check_dtype=False) def test_dataframe_groupby_len(scalars_df_index, scalars_pandas_df_index): @@ -101,7 +99,7 @@ def test_dataframe_groupby_quantile(scalars_df_index, scalars_pandas_df_index, q scalars_df_index[col_names].groupby("string_col").quantile(q) ).to_pandas() pd_result = scalars_pandas_df_index[col_names].groupby("string_col").quantile(q) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, check_dtype=False, check_index_type=False ) @@ -125,7 +123,7 @@ def test_dataframe_groupby_rank( scalars_df_index, scalars_pandas_df_index, na_option, method, ascending, pct ): # TODO: supply a reason why this isn't compatible with pandas 1.x - pytest.importorskip("pandas", minversion="2.2.0") + pytest.importorskip("pandas", minversion="2.0.0") col_names = ["int64_too", "float64_col", "int64_col", "string_col"] bf_result = ( scalars_df_index[col_names] @@ -141,7 +139,7 @@ def test_dataframe_groupby_rank( .astype("float64") .astype("Float64") ) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, check_dtype=False, check_index_type=False ) @@ -169,9 +167,7 @@ def test_dataframe_groupby_aggregate( pd_result = operator(scalars_pandas_df_index[col_names].groupby("string_col")) bf_result_computed = bf_result.to_pandas() - bigframes.testing.utils.assert_frame_equal( - pd_result, bf_result_computed, check_dtype=False - ) + pd.testing.assert_frame_equal(pd_result, bf_result_computed, check_dtype=False) def test_dataframe_groupby_corr(scalars_df_index, scalars_pandas_df_index): @@ -179,7 +175,7 @@ def test_dataframe_groupby_corr(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index[col_names].groupby("bool_col").corr().to_pandas() pd_result = scalars_pandas_df_index[col_names].groupby("bool_col").corr() - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, check_dtype=False, check_index_type=False ) @@ -189,7 +185,7 @@ def test_dataframe_groupby_cov(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index[col_names].groupby("bool_col").cov().to_pandas() pd_result = scalars_pandas_df_index[col_names].groupby("bool_col").cov() - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, check_dtype=False, check_index_type=False ) @@ -209,7 +205,7 @@ def test_dataframe_groupby_agg_string( pd_result = scalars_pandas_df_index[col_names].groupby("string_col").agg("count") bf_result_computed = bf_result.to_pandas(ordered=ordered) - bigframes.testing.utils.assert_frame_equal( + assert_frame_equal( pd_result, bf_result_computed, check_dtype=False, ignore_order=not ordered ) @@ -219,9 +215,7 @@ def test_dataframe_groupby_agg_size_string(scalars_df_index, scalars_pandas_df_i bf_result = scalars_df_index[col_names].groupby("string_col").agg("size") pd_result = scalars_pandas_df_index[col_names].groupby("string_col").agg("size") - bigframes.testing.utils.assert_series_equal( - pd_result, bf_result.to_pandas(), check_dtype=False - ) + pd.testing.assert_series_equal(pd_result, bf_result.to_pandas(), check_dtype=False) def test_dataframe_groupby_agg_list(scalars_df_index, scalars_pandas_df_index): @@ -239,7 +233,7 @@ def test_dataframe_groupby_agg_list(scalars_df_index, scalars_pandas_df_index): # some inconsistency between versions, so normalize to bigframes behavior pd_result = pd_result.rename({"amin": "min"}, axis="columns") bf_result_computed = bf_result_computed.rename({"amin": "min"}, axis="columns") - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result_computed, check_dtype=False, check_index_type=False ) @@ -258,9 +252,7 @@ def test_dataframe_groupby_agg_list_w_column_multi_index( pd_result = pd_df.groupby(level=0).agg(["count", np.min, "size"]) bf_result_computed = bf_result.to_pandas() - bigframes.testing.utils.assert_frame_equal( - pd_result, bf_result_computed, check_dtype=False - ) + pd.testing.assert_frame_equal(pd_result, bf_result_computed, check_dtype=False) @pytest.mark.parametrize( @@ -290,7 +282,7 @@ def test_dataframe_groupby_agg_dict_with_list( ) bf_result_computed = bf_result.to_pandas() - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result_computed, check_dtype=False, check_index_type=False ) @@ -309,9 +301,7 @@ def test_dataframe_groupby_agg_dict_no_lists(scalars_df_index, scalars_pandas_df ) bf_result_computed = bf_result.to_pandas() - bigframes.testing.utils.assert_frame_equal( - pd_result, bf_result_computed, check_dtype=False - ) + pd.testing.assert_frame_equal(pd_result, bf_result_computed, check_dtype=False) def test_dataframe_groupby_agg_named(scalars_df_index, scalars_pandas_df_index): @@ -334,9 +324,7 @@ def test_dataframe_groupby_agg_named(scalars_df_index, scalars_pandas_df_index): ) bf_result_computed = bf_result.to_pandas() - bigframes.testing.utils.assert_frame_equal( - pd_result, bf_result_computed, check_dtype=False - ) + pd.testing.assert_frame_equal(pd_result, bf_result_computed, check_dtype=False) def test_dataframe_groupby_agg_kw_tuples(scalars_df_index, scalars_pandas_df_index): @@ -356,9 +344,7 @@ def test_dataframe_groupby_agg_kw_tuples(scalars_df_index, scalars_pandas_df_ind ) bf_result_computed = bf_result.to_pandas() - bigframes.testing.utils.assert_frame_equal( - pd_result, bf_result_computed, check_dtype=False - ) + pd.testing.assert_frame_equal(pd_result, bf_result_computed, check_dtype=False) @pytest.mark.parametrize( @@ -403,7 +389,7 @@ def test_dataframe_groupby_multi_sum( # BigQuery DataFrames default indices use nullable Int64 always pd_series.index = pd_series.index.astype("Int64") - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_series, bf_result, ) @@ -442,9 +428,7 @@ def test_dataframe_groupby_analytic( ) bf_result_computed = bf_result.to_pandas() - bigframes.testing.utils.assert_frame_equal( - pd_result, bf_result_computed, check_dtype=False - ) + pd.testing.assert_frame_equal(pd_result, bf_result_computed, check_dtype=False) @pytest.mark.parametrize( @@ -465,9 +449,7 @@ def test_dataframe_groupby_cumcount( ) bf_result_computed = bf_result.to_pandas() - bigframes.testing.utils.assert_series_equal( - pd_result, bf_result_computed, check_dtype=False - ) + pd.testing.assert_series_equal(pd_result, bf_result_computed, check_dtype=False) def test_dataframe_groupby_size_as_index_false( @@ -477,7 +459,7 @@ def test_dataframe_groupby_size_as_index_false( bf_result_computed = bf_result.to_pandas() pd_result = scalars_pandas_df_index.groupby("string_col", as_index=False).size() - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result_computed, check_dtype=False, check_index_type=False ) @@ -489,9 +471,7 @@ def test_dataframe_groupby_size_as_index_true( pd_result = scalars_pandas_df_index.groupby("string_col", as_index=True).size() bf_result_computed = bf_result.to_pandas() - bigframes.testing.utils.assert_series_equal( - pd_result, bf_result_computed, check_dtype=False - ) + pd.testing.assert_series_equal(pd_result, bf_result_computed, check_dtype=False) def test_dataframe_groupby_skew(scalars_df_index, scalars_pandas_df_index): @@ -499,20 +479,21 @@ def test_dataframe_groupby_skew(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index[col_names].groupby("bool_col").skew().to_pandas() pd_result = scalars_pandas_df_index[col_names].groupby("bool_col").skew() - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_frame_equal(pd_result, bf_result, check_dtype=False) -@pytest.mark.skipif( - not pd.__version__.startswith("3"), - reason="groupby.kurt not supported on legacy pandas versions", -) def test_dataframe_groupby_kurt(scalars_df_index, scalars_pandas_df_index): col_names = ["float64_col", "int64_col", "bool_col"] bf_result = scalars_df_index[col_names].groupby("bool_col").kurt().to_pandas() # Pandas doesn't have groupby.kurt yet: https://github.com/pandas-dev/pandas/issues/40139 - pd_result = scalars_pandas_df_index[col_names].groupby("bool_col").kurt() + pd_result = ( + scalars_pandas_df_index[col_names] + .groupby("bool_col") + .apply(pd.Series.kurt) + .drop("bool_col", axis=1) + ) - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_frame_equal(pd_result, bf_result, check_dtype=False) @pytest.mark.parametrize( @@ -528,7 +509,7 @@ def test_dataframe_groupby_diff(scalars_df_index, scalars_pandas_df_index, order pd_result = scalars_pandas_df_index[col_names].groupby("string_col").diff(-1) bf_result_computed = bf_result.to_pandas(ordered=ordered) - bigframes.testing.utils.assert_frame_equal( + assert_frame_equal( pd_result, bf_result_computed, check_dtype=False, ignore_order=not ordered ) @@ -545,7 +526,7 @@ def test_dataframe_groupby_getitem( scalars_pandas_df_index[col_names].groupby("string_col")["int64_col"].min() ) - bigframes.testing.utils.assert_series_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) def test_dataframe_groupby_getitem_error( @@ -576,7 +557,7 @@ def test_dataframe_groupby_getitem_list( scalars_pandas_df_index[col_names].groupby("string_col")[col_names].min() ) - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_frame_equal(pd_result, bf_result, check_dtype=False) def test_dataframe_groupby_getitem_list_error( @@ -609,15 +590,11 @@ def test_dataframe_groupby_nonnumeric_with_mean(): bf_result = bpd.DataFrame(df).groupby(["key1", "key2"]).mean().to_pandas() - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, check_index_type=False, check_dtype=False ) -@pytest.mark.skipif( - pd.__version__.startswith("3"), - reason="value_counts behavior change b/485962498", -) @pytest.mark.parametrize( ("subset", "normalize", "ascending", "dropna", "as_index"), [ @@ -654,14 +631,10 @@ def test_dataframe_groupby_value_counts( ) if as_index: - bigframes.testing.utils.assert_series_equal( - pd_result, bf_result, check_dtype=False - ) + pd.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) else: pd_result.index = pd_result.index.astype("Int64") - bigframes.testing.utils.assert_frame_equal( - pd_result, bf_result, check_dtype=False - ) + pd.testing.assert_frame_equal(pd_result, bf_result, check_dtype=False) @pytest.mark.parametrize( @@ -687,7 +660,7 @@ def test_dataframe_groupby_first( .groupby(scalars_pandas_df_index.int64_col % 2) .first(numeric_only=numeric_only, min_count=min_count) ) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, ) @@ -711,7 +684,7 @@ def test_dataframe_groupby_last( pd_result = scalars_pandas_df_index.groupby( scalars_pandas_df_index.int64_col % 2 ).last(numeric_only=numeric_only, min_count=min_count) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result, ) @@ -747,7 +720,7 @@ def test_series_groupby_agg_string(scalars_df_index, scalars_pandas_df_index, ag ) bf_result_computed = bf_result.to_pandas() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result_computed, check_dtype=False, check_names=False ) @@ -765,7 +738,7 @@ def test_series_groupby_agg_list(scalars_df_index, scalars_pandas_df_index): ) bf_result_computed = bf_result.to_pandas() - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result, bf_result_computed, check_dtype=False, check_names=False ) @@ -820,7 +793,7 @@ def test_series_groupby_rank( .astype("float64") .astype("Float64") ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, check_dtype=False, check_index_type=False ) @@ -835,7 +808,7 @@ def test_series_groupby_head(scalars_df_index, scalars_pandas_df_index, dropna): pd_result = scalars_pandas_df_index.groupby("bool_col", dropna=dropna)[ "int64_too" ].head(1) - bigframes.testing.utils.assert_series_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) def test_series_groupby_kurt(scalars_df_index, scalars_pandas_df_index): @@ -850,7 +823,7 @@ def test_series_groupby_kurt(scalars_df_index, scalars_pandas_df_index): pd.Series.kurt ) - bigframes.testing.utils.assert_series_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) def test_series_groupby_size(scalars_df_index, scalars_pandas_df_index): @@ -864,9 +837,7 @@ def test_series_groupby_size(scalars_df_index, scalars_pandas_df_index): ) bf_result_computed = bf_result.to_pandas() - bigframes.testing.utils.assert_series_equal( - pd_result, bf_result_computed, check_dtype=False - ) + pd.testing.assert_series_equal(pd_result, bf_result_computed, check_dtype=False) def test_series_groupby_skew(scalars_df_index, scalars_pandas_df_index): @@ -882,7 +853,7 @@ def test_series_groupby_skew(scalars_df_index, scalars_pandas_df_index): .skew() ) - bigframes.testing.utils.assert_series_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) @pytest.mark.parametrize( @@ -897,15 +868,11 @@ def test_series_groupby_quantile(scalars_df_index, scalars_pandas_df_index, q): scalars_df_index.groupby("string_col")["int64_col"].quantile(q) ).to_pandas() pd_result = scalars_pandas_df_index.groupby("string_col")["int64_col"].quantile(q) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, check_dtype=False, check_index_type=False ) -@pytest.mark.skipif( - pd.__version__.startswith("3"), - reason="Pandas 3 change value_counts behavior", -) @pytest.mark.parametrize( ("normalize", "ascending", "dropna"), [ @@ -938,7 +905,7 @@ def test_series_groupby_value_counts( pd_result = scalars_pandas_df_index.groupby("bool_col")["string_col"].value_counts( normalize=normalize, ascending=ascending, dropna=dropna ) - bigframes.testing.utils.assert_series_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) @pytest.mark.parametrize( @@ -959,7 +926,7 @@ def test_series_groupby_first( pd_result = scalars_pandas_df_index.groupby("string_col")["int64_col"].first( numeric_only=numeric_only, min_count=min_count ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, ) @@ -983,92 +950,4 @@ def test_series_groupby_last( pd_result = scalars_pandas_df_index.groupby("string_col")["int64_col"].last( numeric_only=numeric_only, min_count=min_count ) - bigframes.testing.utils.assert_series_equal(pd_result, bf_result) - - -def test_series_groupby_agg_transpile_system(scalars_df_index, scalars_pandas_df_index): - def custom_agg(s): - return s.sum() - s.mean() - - bf_df = scalars_df_index.dropna(subset=["int64_col", "bool_col"]) - pd_df = scalars_pandas_df_index.dropna(subset=["int64_col", "bool_col"]) - - with bpd.option_context("experiments.enable_python_transpiler", True): - bf_result = bf_df.groupby("bool_col")["int64_col"].agg(custom_agg).to_pandas() - pd_result = pd_df.groupby("bool_col")["int64_col"].agg(custom_agg) - - bigframes.testing.utils.assert_series_equal(pd_result, bf_result, check_dtype=False) - - -def test_dataframe_groupby_agg_transpile_system( - scalars_df_index, scalars_pandas_df_index -): - def custom_agg(s): - return (s.max() - s.min()) / s.count() - - bf_df = scalars_df_index.dropna(subset=["int64_col", "int64_too", "bool_col"]) - pd_df = scalars_pandas_df_index.dropna( - subset=["int64_col", "int64_too", "bool_col"] - ) - - with bpd.option_context("experiments.enable_python_transpiler", True): - bf_result = ( - bf_df[["int64_col", "int64_too", "bool_col"]] - .groupby("bool_col") - .agg(custom_agg) - .to_pandas() - ) - pd_result = ( - pd_df[["int64_col", "int64_too", "bool_col"]] - .groupby("bool_col") - .agg(custom_agg) - ) - - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result, check_dtype=False) - - -def test_series_groupby_transform_transpile_system( - scalars_df_index, scalars_pandas_df_index -): - def custom_transform(s): - return s - s.mean() - - bf_df = scalars_df_index.dropna(subset=["int64_col", "bool_col"]) - pd_df = scalars_pandas_df_index.dropna(subset=["int64_col", "bool_col"]) - - with bpd.option_context("experiments.enable_python_transpiler", True): - bf_result = ( - bf_df.groupby("bool_col")["int64_col"] - .transform(custom_transform) - .to_pandas() - ) - pd_result = pd_df.groupby("bool_col")["int64_col"].transform(custom_transform) - - bigframes.testing.utils.assert_series_equal(pd_result, bf_result, check_dtype=False) - - -def test_dataframe_groupby_transform_transpile_system( - scalars_df_index, scalars_pandas_df_index -): - def custom_transform(s): - return (s - s.min()) / (s.max() - s.min()) - - bf_df = scalars_df_index.dropna(subset=["int64_col", "int64_too", "bool_col"]) - pd_df = scalars_pandas_df_index.dropna( - subset=["int64_col", "int64_too", "bool_col"] - ) - - with bpd.option_context("experiments.enable_python_transpiler", True): - bf_result = ( - bf_df[["int64_col", "int64_too", "bool_col"]] - .groupby("bool_col") - .transform(custom_transform) - .to_pandas() - ) - pd_result = ( - pd_df[["int64_col", "int64_too", "bool_col"]] - .groupby("bool_col") - .transform(custom_transform) - ) - - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_series_equal(pd_result, bf_result) diff --git a/tests/system/small/test_index.py b/tests/system/small/test_index.py index 26ac609b3c6..0ec1fb61432 100644 --- a/tests/system/small/test_index.py +++ b/tests/system/small/test_index.py @@ -18,8 +18,8 @@ import pandas as pd import pytest -import bigframes.pandas as bpd from bigframes import dtypes +import bigframes.pandas as bpd from bigframes.testing.utils import assert_pandas_index_equal_ignore_index_type diff --git a/tests/system/small/test_index_io.py b/tests/system/small/test_index_io.py index b4d7c06da52..306b15e67a2 100644 --- a/tests/system/small/test_index_io.py +++ b/tests/system/small/test_index_io.py @@ -18,6 +18,7 @@ def test_to_pandas_override_global_option(scalars_df_index): with bigframes.option_context("compute.allow_large_results", True): + bf_index = scalars_df_index.index # Direct call to_pandas uses global default setting (allow_large_results=True), @@ -42,6 +43,7 @@ def test_to_pandas_dry_run(scalars_df_index): def test_to_numpy_override_global_option(scalars_df_index): with bigframes.option_context("compute.allow_large_results", True): + bf_index = scalars_df_index.index # Direct call to_numpy uses global default setting (allow_large_results=True), diff --git a/tests/system/small/test_magics.py b/tests/system/small/test_magics.py deleted file mode 100644 index eac0f233f98..00000000000 --- a/tests/system/small/test_magics.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import pandas as pd -import pytest - -import bigframes -import bigframes.pandas as bpd - -IPython = pytest.importorskip("IPython") - - -MAGIC_NAME = "bqsql" - - -@pytest.fixture(scope="module") -def ip(): - """Provides a persistent IPython shell instance for the test session.""" - from IPython.testing.globalipapp import get_ipython - - shell = get_ipython() - shell.extension_manager.load_extension("bigframes") - return shell - - -def test_magic_select_lit_to_var(ip): - bigframes.close_session() - - line = "dst_var" - cell_body = "SELECT 3" - - ip.run_cell_magic(MAGIC_NAME, line, cell_body) - - assert "dst_var" in ip.user_ns - result_df = ip.user_ns["dst_var"] - assert result_df.shape == (1, 1) - assert result_df.to_pandas().iloc[0, 0] == 3 - - -def test_magic_select_lit_dry_run(ip): - bigframes.close_session() - - line = "dst_var --dry_run" - cell_body = "SELECT 3" - - ip.run_cell_magic(MAGIC_NAME, line, cell_body) - - assert "dst_var" in ip.user_ns - result_df = ip.user_ns["dst_var"] - assert result_df.totalBytesProcessed == 0 - - -def test_magic_select_lit_display(ip): - from IPython.utils.capture import capture_output - - bigframes.close_session() - - cell_body = "SELECT 3" - - with capture_output() as io: - ip.run_cell_magic(MAGIC_NAME, "", cell_body) - assert len(io.outputs) > 0 - # Check that the output has data, regardless of the format (html, plain, etc) - available_formats = io.outputs[0].data.keys() - assert len(available_formats) > 0 - - -def test_magic_select_interpolate(ip): - bigframes.close_session() - df = bpd.read_pandas( - pd.DataFrame({"col_a": [1, 2, 3, 4, 5, 6], "col_b": [1, 2, 1, 3, 1, 2]}) - ) - const_val = 1 - - ip.push({"df": df, "const_val": const_val}) - - query = """ - SELECT - SUM(col_a) AS total - FROM - {df} - WHERE col_b={const_val} - """ - - ip.run_cell_magic(MAGIC_NAME, "dst_var", query) - - assert "dst_var" in ip.user_ns - result_df = ip.user_ns["dst_var"] - assert result_df.shape == (1, 1) - assert result_df.loc[0, "total"] == 9 diff --git a/tests/system/small/test_multiindex.py b/tests/system/small/test_multiindex.py index 18368fc5126..a28e02a54fa 100644 --- a/tests/system/small/test_multiindex.py +++ b/tests/system/small/test_multiindex.py @@ -17,7 +17,7 @@ import pytest import bigframes.pandas as bpd -import bigframes.testing.utils +from bigframes.testing.utils import assert_frame_equal # Sample MultiIndex for testing DataFrames where() method. _MULTI_INDEX = pandas.MultiIndex.from_tuples( @@ -58,7 +58,7 @@ def test_multi_index_from_arrays(): names=[" 1index 1", "_1index 2"], ) assert bf_idx.names == pd_idx.names - bigframes.testing.utils.assert_index_equal(bf_idx.to_pandas(), pd_idx) + pandas.testing.assert_index_equal(bf_idx.to_pandas(), pd_idx) def test_read_pandas_multi_index_axes(): @@ -90,7 +90,7 @@ def test_read_pandas_multi_index_axes(): bf_df = bpd.DataFrame(pandas_df) bf_df_computed = bf_df.to_pandas() - bigframes.testing.utils.assert_frame_equal(bf_df_computed, pandas_df) + pandas.testing.assert_frame_equal(bf_df_computed, pandas_df) # Row Multi-index tests @@ -98,7 +98,7 @@ def test_set_multi_index(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.set_index(["bool_col", "int64_too"]).to_pandas() pd_result = scalars_pandas_df_index.set_index(["bool_col", "int64_too"]) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -127,7 +127,7 @@ def test_df_reset_multi_index(scalars_df_index, scalars_pandas_df_index, level, if pd_result.index.dtype != bf_result.index.dtype: pd_result.index = pd_result.index.astype(bf_result.index.dtype) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -160,9 +160,9 @@ def test_series_reset_multi_index( pd_result.index = pd_result.index.astype(pandas.Int64Dtype()) if drop: - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pandas.testing.assert_series_equal(bf_result, pd_result) else: - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) def test_series_multi_index_idxmin(scalars_df_index, scalars_pandas_df_index): @@ -187,7 +187,7 @@ def test_binop_series_series_matching_multi_indices( bf_result = bf_left["int64_col"] + bf_right["int64_too"] pd_result = pd_left["int64_col"] + pd_right["int64_too"] - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( bf_result.sort_index().to_pandas(), pd_result.sort_index() ) @@ -203,7 +203,7 @@ def test_binop_df_series_matching_multi_indices( bf_result = bf_left[["int64_col", "int64_too"]].add(bf_right["int64_too"], axis=0) pd_result = pd_left[["int64_col", "int64_too"]].add(pd_right["int64_too"], axis=0) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result.sort_index().to_pandas(), pd_result.sort_index() ) @@ -217,7 +217,7 @@ def test_binop_multi_index_mono_index(scalars_df_index, scalars_pandas_df_index) bf_result = bf_left["int64_col"] + bf_right["int64_too"] pd_result = pd_left["int64_col"] + pd_right["int64_too"] - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pandas.testing.assert_series_equal(bf_result.to_pandas(), pd_result) def test_binop_overlapping_multi_indices(scalars_df_index, scalars_pandas_df_index): @@ -229,7 +229,7 @@ def test_binop_overlapping_multi_indices(scalars_df_index, scalars_pandas_df_ind bf_result = bf_left["int64_col"] + bf_right["int64_too"] pd_result = pd_left["int64_col"] + pd_right["int64_too"] - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( bf_result.sort_index().to_pandas(), pd_result.sort_index() ) @@ -245,7 +245,7 @@ def test_concat_compatible_multi_indices(scalars_df_index, scalars_pandas_df_ind bf_result = bpd.concat([bf_left, bf_right]) pd_result = pandas.concat([pd_left, pd_right]) - bigframes.testing.utils.assert_frame_equal(bf_result.to_pandas(), pd_result) + pandas.testing.assert_frame_equal(bf_result.to_pandas(), pd_result) def test_concat_multi_indices_ignore_index(scalars_df_index, scalars_pandas_df_index): @@ -260,7 +260,7 @@ def test_concat_multi_indices_ignore_index(scalars_df_index, scalars_pandas_df_i # Pandas uses int64 instead of Int64 (nullable) dtype. pd_result.index = pd_result.index.astype(pandas.Int64Dtype()) - bigframes.testing.utils.assert_frame_equal(bf_result.to_pandas(), pd_result) + pandas.testing.assert_frame_equal(bf_result.to_pandas(), pd_result) @pytest.mark.parametrize( @@ -277,7 +277,7 @@ def test_multi_index_loc_multi_row(scalars_df_index, scalars_pandas_df_index, ke ) pd_result = scalars_pandas_df_index.set_index(["int64_too", "string_col"]).loc[key] - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) def test_multi_index_loc_single_row(scalars_df_index, scalars_pandas_df_index): @@ -288,7 +288,7 @@ def test_multi_index_loc_single_row(scalars_df_index, scalars_pandas_df_index): (2, "capitalize, This ") ] - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pandas.testing.assert_series_equal(bf_result, pd_result) def test_multi_index_getitem_bool(scalars_df_index, scalars_pandas_df_index): @@ -298,7 +298,7 @@ def test_multi_index_getitem_bool(scalars_df_index, scalars_pandas_df_index): bf_result = bf_frame[bf_frame["int64_col"] > 0].to_pandas() pd_result = pd_frame[pd_frame["int64_col"] > 0] - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -318,7 +318,7 @@ def test_df_multi_index_droplevel(scalars_df_index, scalars_pandas_df_index, lev bf_result = bf_frame.droplevel(level).to_pandas() pd_result = pd_frame.droplevel(level) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -338,7 +338,7 @@ def test_series_multi_index_droplevel(scalars_df_index, scalars_pandas_df_index, bf_result = bf_frame["string_col"].droplevel(level).to_pandas() pd_result = pd_frame["string_col"].droplevel(level) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pandas.testing.assert_series_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -357,7 +357,7 @@ def test_multi_index_drop(scalars_df_index, scalars_pandas_df_index, labels, lev bf_result = bf_frame.drop(labels=labels, axis="index", level=level).to_pandas() pd_result = pd_frame.drop(labels=labels, axis="index", level=level) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -382,7 +382,7 @@ def test_df_multi_index_reorder_levels( bf_result = bf_frame.reorder_levels(order).to_pandas() pd_result = pd_frame.reorder_levels(order) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -407,7 +407,7 @@ def test_series_multi_index_reorder_levels( bf_result = bf_frame["string_col"].reorder_levels(order).to_pandas() pd_result = pd_frame["string_col"].reorder_levels(order) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pandas.testing.assert_series_equal(bf_result, pd_result) def test_df_multi_index_swaplevel(scalars_df_index, scalars_pandas_df_index): @@ -417,7 +417,7 @@ def test_df_multi_index_swaplevel(scalars_df_index, scalars_pandas_df_index): bf_result = bf_frame.swaplevel().to_pandas() pd_result = pd_frame.swaplevel() - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) def test_series_multi_index_swaplevel(scalars_df_index, scalars_pandas_df_index): @@ -427,7 +427,7 @@ def test_series_multi_index_swaplevel(scalars_df_index, scalars_pandas_df_index) bf_result = bf_frame["string_col"].swaplevel(0, 2).to_pandas() pd_result = pd_frame["string_col"].swaplevel(0, 2) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pandas.testing.assert_series_equal(bf_result, pd_result) def test_multi_index_series_groupby(scalars_df_index, scalars_pandas_df_index): @@ -443,7 +443,7 @@ def test_multi_index_series_groupby(scalars_df_index, scalars_pandas_df_index): pd_frame["float64_col"].groupby([pd_frame.int64_col % 2, "bool_col"]).mean() ) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pandas.testing.assert_series_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -470,7 +470,7 @@ def test_multi_index_series_groupby_level( .mean() ) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pandas.testing.assert_series_equal(bf_result, pd_result) def test_multi_index_dataframe_groupby(scalars_df_index, scalars_pandas_df_index): @@ -485,7 +485,7 @@ def test_multi_index_dataframe_groupby(scalars_df_index, scalars_pandas_df_index numeric_only=True ) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -514,16 +514,13 @@ def test_multi_index_dataframe_groupby_level_aggregate( ) # For as_index=False, pandas will drop index levels used as groupings # In the future, it will include this in the result, bigframes already does this behavior - if not pandas.__version__.startswith("3"): - if not as_index: - for col in index_cols: - if col in bf_result.columns: - bf_result = bf_result.drop(col, axis=1) + if not as_index: + for col in index_cols: + if col in bf_result.columns: + bf_result = bf_result.drop(col, axis=1) # Pandas will have int64 index, while bigquery will have Int64 when resetting - bigframes.testing.utils.assert_frame_equal( - bf_result, pd_result, check_index_type=False - ) + pandas.testing.assert_frame_equal(bf_result, pd_result, check_index_type=False) @pytest.mark.parametrize( @@ -556,7 +553,7 @@ def test_multi_index_dataframe_groupby_level_analytic( .cumsum(numeric_only=True) ) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) all_joins = pytest.mark.parametrize( @@ -586,7 +583,7 @@ def test_multi_index_dataframe_join(scalars_dfs, how): (["bool_col", "rowindex_2"]) )[["float64_col"]] pd_result = pd_df_a.join(pd_df_b, how=how) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, ignore_order=True) + assert_frame_equal(bf_result, pd_result, ignore_order=True) @all_joins @@ -607,7 +604,7 @@ def test_multi_index_dataframe_join_on(scalars_dfs, how): pd_df_a = pd_df_a.assign(rowindex_2=pd_df_a["rowindex_2"] + 2) pd_df_b = pd_df[["float64_col"]] pd_result = pd_df_a.join(pd_df_b, on="rowindex_2", how=how) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, ignore_order=True) + assert_frame_equal(bf_result, pd_result, ignore_order=True) def test_multi_index_dataframe_where_series_cond_none_other( @@ -635,7 +632,7 @@ def test_multi_index_dataframe_where_series_cond_none_other( bf_result = dataframe_bf.where(series_cond_bf).to_pandas() pd_result = dataframe_pd.where(series_cond_pd) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_index_type=False, @@ -671,7 +668,7 @@ def test_multi_index_dataframe_where_series_cond_dataframe_other( bf_result = dataframe_bf.where(series_cond_bf, dataframe_other_bf).to_pandas() pd_result = dataframe_pd.where(series_cond_pd, dataframe_other_pd) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_index_type=False, @@ -703,7 +700,7 @@ def test_multi_index_dataframe_where_dataframe_cond_constant_other( bf_result = dataframe_bf.where(dataframe_cond_bf, other).to_pandas() pd_result = dataframe_pd.where(dataframe_cond_pd, other) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_index_type=False, @@ -736,7 +733,7 @@ def test_multi_index_dataframe_where_dataframe_cond_dataframe_other( bf_result = dataframe_bf.where(dataframe_cond_bf, dataframe_other_bf).to_pandas() pd_result = dataframe_pd.where(dataframe_cond_pd, dataframe_other_pd) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_index_type=False, @@ -768,7 +765,7 @@ def test_multi_index_series_groupby_level_aggregate( .mean() ) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result, check_dtype=False) + pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) @pytest.mark.parametrize( @@ -795,7 +792,7 @@ def test_multi_index_series_groupby_level_analytic( .cumsum() ) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result, check_dtype=False) + pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) def test_multi_index_series_rename_dict_same_type( @@ -810,7 +807,7 @@ def test_multi_index_series_rename_dict_same_type( "string_col" ].rename({1: 100, 2: 200}) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -828,7 +825,7 @@ def test_multi_index_df_reindex(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index.set_index(["rowindex_2", "string_col"]).reindex( index=new_index ) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -846,15 +843,15 @@ def test_column_multi_index_getitem(scalars_df_index, scalars_pandas_df_index): bf_a = bf_df["a"].to_pandas() pd_a = pd_df["a"] - bigframes.testing.utils.assert_frame_equal(bf_a, pd_a) + pandas.testing.assert_frame_equal(bf_a, pd_a) bf_b = bf_df["b"].to_pandas() pd_b = pd_df["b"] - bigframes.testing.utils.assert_frame_equal(bf_b, pd_b) + pandas.testing.assert_frame_equal(bf_b, pd_b) bf_fullkey = bf_df[("a", "int64_too")].to_pandas() pd_fullkey = pd_df[("a", "int64_too")] - bigframes.testing.utils.assert_series_equal(bf_fullkey, pd_fullkey) + pandas.testing.assert_series_equal(bf_fullkey, pd_fullkey) def test_column_multi_index_concat(scalars_df_index, scalars_pandas_df_index): @@ -879,7 +876,7 @@ def test_column_multi_index_concat(scalars_df_index, scalars_pandas_df_index): bf_result = bpd.concat([bf_df1, bf_df2, bf_df1]).to_pandas() pd_result = pandas.concat([pd_df1, pd_df2, pd_df1]) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) def test_column_multi_index_drop(scalars_df_index, scalars_pandas_df_index): @@ -892,7 +889,7 @@ def test_column_multi_index_drop(scalars_df_index, scalars_pandas_df_index): bf_a = bf_df.drop(("a", "int64_too"), axis=1).to_pandas() pd_a = pd_df.drop(("a", "int64_too"), axis=1) - bigframes.testing.utils.assert_frame_equal(bf_a, pd_a) + pandas.testing.assert_frame_equal(bf_a, pd_a) @pytest.mark.parametrize( @@ -916,7 +913,7 @@ def test_column_multi_index_assign(scalars_df_index, scalars_pandas_df_index, ke pd_result = pd_df.assign(**kwargs) # Pandas assign results in non-nullable dtype - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) def test_column_multi_index_rename(scalars_df_index, scalars_pandas_df_index): @@ -930,7 +927,7 @@ def test_column_multi_index_rename(scalars_df_index, scalars_pandas_df_index): bf_result = bf_df.rename(columns={"b": "c"}).to_pandas() pd_result = pd_df.rename(columns={"b": "c"}) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -960,7 +957,7 @@ def test_column_multi_index_reset_index( # Pandas uses int64 instead of Int64 (nullable) dtype. pd_result.index = pd_result.index.astype(pandas.Int64Dtype()) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) def test_column_multi_index_binary_op(scalars_df_index, scalars_pandas_df_index): @@ -974,7 +971,7 @@ def test_column_multi_index_binary_op(scalars_df_index, scalars_pandas_df_index) bf_result = (bf_df[("a", "a")] + 3).to_pandas() pd_result = pd_df[("a", "a")] + 3 - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pandas.testing.assert_series_equal(bf_result, pd_result) def test_column_multi_index_any(): @@ -991,7 +988,7 @@ def test_column_multi_index_any(): pd_result = pd_df.isna().any() bf_result = bf_df.isna().any().to_pandas() - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result.reset_index(drop=False), pd_result.reset_index(drop=False), check_dtype=False, @@ -1011,9 +1008,7 @@ def test_column_multi_index_agg(scalars_df_index, scalars_pandas_df_index): # Pandas may produce narrower numeric types, but bigframes always produces Float64 pd_result = pd_result.astype("Float64") - bigframes.testing.utils.assert_frame_equal( - bf_result, pd_result, check_index_type=False - ) + pandas.testing.assert_frame_equal(bf_result, pd_result, check_index_type=False) def test_column_multi_index_prefix_suffix(scalars_df_index, scalars_pandas_df_index): @@ -1027,7 +1022,7 @@ def test_column_multi_index_prefix_suffix(scalars_df_index, scalars_pandas_df_in bf_result = bf_df.add_prefix("prefixed_").add_suffix("_suffixed").to_pandas() pd_result = pd_df.add_prefix("prefixed_").add_suffix("_suffixed") - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) def test_column_multi_index_cumsum(scalars_df_index, scalars_pandas_df_index): @@ -1043,7 +1038,7 @@ def test_column_multi_index_cumsum(scalars_df_index, scalars_pandas_df_index): bf_result = bf_df.cumsum().to_pandas() pd_result = pd_df.cumsum() - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) @pytest.mark.parametrize( @@ -1076,7 +1071,7 @@ def test_column_multi_index_stack(level): # Pandas produces NaN, where bq dataframes produces pd.NA # Column ordering seems to depend on pandas version assert isinstance(pd_result, pandas.DataFrame) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -1104,7 +1099,7 @@ def test_column_multi_index_melt(): pd_result = pd_df.melt() # BigFrames uses different string and int types, but values are identical - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_index_type=False, check_dtype=False ) @@ -1126,7 +1121,7 @@ def test_column_multi_index_unstack(scalars_df_index, scalars_pandas_df_index): # Pandas produces NaN, where bq dataframes produces pd.NA # Column ordering seems to depend on pandas version - bigframes.testing.utils.assert_series_equal(bf_result, pd_result, check_dtype=False) + pandas.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) def test_corr_w_multi_index(scalars_df_index, scalars_pandas_df_index): @@ -1147,7 +1142,7 @@ def test_corr_w_multi_index(scalars_df_index, scalars_pandas_df_index): # BigFrames and Pandas differ in their data type handling: # - Column types: BigFrames uses Float64, Pandas uses float64. # - Index types: BigFrames uses strign, Pandas uses object. - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -1170,7 +1165,7 @@ def test_cov_w_multi_index(scalars_df_index, scalars_pandas_df_index): # BigFrames and Pandas differ in their data type handling: # - Column types: BigFrames uses Float64, Pandas uses float64. # - Index types: BigFrames uses string, Pandas uses object. - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -1249,7 +1244,7 @@ def test_column_multi_index_droplevel(scalars_df_index, scalars_pandas_df_index) bf_result = bf_df.droplevel(1, axis=1).to_pandas() pd_result = pd_df.droplevel(1, axis=1) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) def test_df_column_multi_index_reindex(scalars_df_index, scalars_pandas_df_index): @@ -1271,7 +1266,7 @@ def test_df_column_multi_index_reindex(scalars_df_index, scalars_pandas_df_index # Pandas uses float64 as default for newly created empty column, bf uses Float64 pd_result[("z", "a")] = pd_result[("z", "a")].astype(pandas.Float64Dtype()) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, ) @@ -1290,7 +1285,7 @@ def test_column_multi_index_reorder_levels(scalars_df_index, scalars_pandas_df_i bf_result = bf_df.reorder_levels([-2, -1, 0], axis=1).to_pandas() pd_result = pd_df.reorder_levels([-2, -1, 0], axis=1) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -1307,7 +1302,7 @@ def test_df_multi_index_unstack(hockey_df, hockey_pandas_df, level): ["team_name", "position"], append=True ).unstack(level=level) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) @pytest.mark.parametrize( @@ -1324,7 +1319,7 @@ def test_series_multi_index_unstack(hockey_df, hockey_pandas_df, level): "number" ].unstack(level=level) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) def test_column_multi_index_swaplevel(scalars_df_index, scalars_pandas_df_index): @@ -1340,7 +1335,7 @@ def test_column_multi_index_swaplevel(scalars_df_index, scalars_pandas_df_index) bf_result = bf_df.swaplevel(-3, -1, axis=1).to_pandas() pd_result = pd_df.swaplevel(-3, -1, axis=1) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pandas.testing.assert_frame_equal(bf_result, pd_result) def test_df_multi_index_dot_not_supported(): @@ -1414,7 +1409,7 @@ def test_explode_w_column_multi_index(): assert isinstance(pd_df, pandas.DataFrame) assert isinstance(pd_df["col0"], pandas.DataFrame) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( df["col0"].explode("col00").to_pandas(), pd_df["col0"].explode("col00"), check_dtype=False, @@ -1432,7 +1427,7 @@ def test_explode_w_multi_index(): df = bpd.DataFrame(data, index=multi_index, columns=columns) pd_df = df.to_pandas() - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( df.explode("col00").to_pandas(), pd_df.explode("col00"), check_dtype=False, @@ -1456,7 +1451,7 @@ def test_column_multi_index_w_na_stack(scalars_df_index, scalars_pandas_df_index # Pandas produces pd.NA, where bq dataframes produces NaN pd_result["c"] = pd_result["c"].replace(pandas.NA, np.nan) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, check_dtype=False) + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) @pytest.mark.parametrize( @@ -1487,37 +1482,6 @@ def test_multiindex_eq_const(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.set_index(col_name).index == (2, False) pd_result = scalars_pandas_df_index.set_index(col_name).index == (2, False) - bigframes.testing.utils.assert_index_equal( + pandas.testing.assert_index_equal( pandas.Index(pd_result, dtype="boolean"), bf_result.to_pandas() ) - - -def test_count_empty_multiindex_columns(session): - df = pandas.DataFrame( - [], index=[1, 2], columns=pandas.MultiIndex.from_tuples([], names=["a", "b"]) - ) - bdf = session.read_pandas(df) - - # count() operation unpivots columns, triggering the empty MultiIndex bug internally - count_df = bdf.count() - - # The local fix ensures that empty unpivoted columns generate properly typed NULLs - # rather than failing syntax validation downstream in BigQuery. - # We compile to `.sql` to verify it succeeds locally without evaluating on BigQuery natively. - _ = count_df.to_frame().sql - - # Assert structural layout is correct - assert count_df.index.nlevels == 2 - assert list(count_df.index.names) == ["a", "b"] - - -def test_dataframe_melt_multiindex(session): - # Tests that `melt` operations via count do not cause MultiIndex drops in Arrow - df = pandas.DataFrame({"A": [1], "B": ["string"], "C": [3]}) - df.columns = pandas.MultiIndex.from_tuples( - [("Group1", "A"), ("Group2", "B"), ("Group1", "C")] - ) - bdf = session.read_pandas(df) - - count_df = bdf.count().to_pandas() - assert count_df.shape[0] == 3 diff --git a/tests/system/small/test_null_index.py b/tests/system/small/test_null_index.py index eb9dc114dde..4aa7ba8c77c 100644 --- a/tests/system/small/test_null_index.py +++ b/tests/system/small/test_null_index.py @@ -381,6 +381,7 @@ def test_null_index_df_concat(scalars_df_null_index, scalars_pandas_df_default_i def test_null_index_map_dict_input( scalars_df_null_index, scalars_pandas_df_default_index ): + local_map = dict() # construct a local map, incomplete to cover behavior for s in scalars_pandas_df_default_index.string_col[:-3]: diff --git a/tests/system/small/test_numpy.py b/tests/system/small/test_numpy.py index 774f72bef4a..490f9271142 100644 --- a/tests/system/small/test_numpy.py +++ b/tests/system/small/test_numpy.py @@ -16,8 +16,6 @@ import pandas as pd import pytest -import bigframes.testing.utils - @pytest.mark.parametrize( ("opname",), @@ -47,9 +45,7 @@ def test_series_ufuncs(floats_pd, floats_bf, opname): bf_result = getattr(np, opname)(floats_bf).to_pandas() pd_result = getattr(np, opname)(floats_pd) - bigframes.testing.utils.assert_series_equal( - bf_result, pd_result, nulls_are_nan=True - ) + pd.testing.assert_series_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -83,7 +79,7 @@ def test_df_ufuncs(scalars_dfs, opname): ): pd_result["int64_col"] = pd_result["int64_col"].astype(pd.Float64Dtype()) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, nulls_are_nan=True) + pd.testing.assert_frame_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -103,7 +99,7 @@ def test_df_binary_ufuncs(scalars_dfs, opname): bf_result = op(scalars_df[["float64_col", "int64_col"]], 5.1).to_pandas() pd_result = op(scalars_pandas_df[["float64_col", "int64_col"]], 5.1) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, nulls_are_nan=True) + pd.testing.assert_frame_equal(bf_result, pd_result) # Operations tested here don't work on full dataframe in numpy+pandas @@ -135,9 +131,7 @@ def test_series_binary_ufuncs(scalars_dfs, x, y, opname): bf_result = op(scalars_df[x], scalars_df[y]).to_pandas() pd_result = op(scalars_pandas_df[x], scalars_pandas_df[y]) - bigframes.testing.utils.assert_series_equal( - bf_result, pd_result, nulls_are_nan=True - ) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_binary_ufuncs_reverse(scalars_dfs): @@ -147,9 +141,7 @@ def test_series_binary_ufuncs_reverse(scalars_dfs): bf_result = np.subtract(5.1, scalars_df["int64_col"]).to_pandas() pd_result = np.subtract(5.1, scalars_pandas_df["int64_col"]) - bigframes.testing.utils.assert_series_equal( - bf_result, pd_result, nulls_are_nan=True - ) + pd.testing.assert_series_equal(bf_result, pd_result) def test_df_binary_ufuncs_reverse(scalars_dfs): @@ -162,4 +154,4 @@ def test_df_binary_ufuncs_reverse(scalars_dfs): scalars_pandas_df[["float64_col", "int64_col"]], ) - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result, nulls_are_nan=True) + pd.testing.assert_frame_equal(bf_result, pd_result) diff --git a/tests/system/small/test_pandas.py b/tests/system/small/test_pandas.py index 356e498021b..a1c0dc9851f 100644 --- a/tests/system/small/test_pandas.py +++ b/tests/system/small/test_pandas.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import typing from datetime import datetime +import typing import pandas as pd import pyarrow as pa @@ -21,8 +21,7 @@ import pytz import bigframes.pandas as bpd -import bigframes.testing -from bigframes.testing.utils import assert_frame_equal, assert_series_equal +from bigframes.testing.utils import assert_frame_equal @pytest.mark.parametrize( @@ -51,84 +50,6 @@ def test_concat_dataframe_w_struct_cols(nested_structs_df, nested_structs_pandas pd.testing.assert_frame_equal(bf_result, pd_result) -def test_nested_structs_dtypes_and_edge_cases(nested_structs_df): - """Explicitly verify dtypes and edge case values for all supported types.""" - import datetime as dt - import decimal - - import numpy as np - import pandas as pd - - import bigframes.dtypes as bfd - - # 1. Verify BigFrames dtypes - expected_bf_dtypes = { - "person": nested_structs_df["person"].dtype, - "bool_col": bfd.BOOL_DTYPE, - "int64_col": bfd.INT_DTYPE, - "float64_col": bfd.FLOAT_DTYPE, - "string_col": bfd.STRING_DTYPE, - "json_col": bfd.JSON_DTYPE, - "date_col": bfd.DATE_DTYPE, - "time_col": bfd.TIME_DTYPE, - "datetime_col": bfd.DATETIME_DTYPE, - "timestamp_col": bfd.TIMESTAMP_DTYPE, - "bytes_col": bfd.BYTES_DTYPE, - "numeric_col": bfd.NUMERIC_DTYPE, - "bignumeric_col": bfd.BIGNUMERIC_DTYPE, - "geography_col": bfd.GEO_DTYPE, - "duration_col": bfd.TIMEDELTA_DTYPE, - } - - for col_name, expected_dtype in expected_bf_dtypes.items(): - assert nested_structs_df[col_name].dtype == expected_dtype, ( - f"Dtype mismatch for {col_name}" - ) - - # 2. Convert to pandas for value assertions - pd_df = nested_structs_df.to_pandas() - - # Verify we have 6 rows - assert len(pd_df) == 6 - - # Row 1: Normal typical values - assert pd_df.loc[1, "bool_col"] == True - assert pd_df.loc[1, "int64_col"] == 123456789 - assert pd_df.loc[1, "float64_col"] == 1.25 - assert pd_df.loc[1, "string_col"] == "Hello World" - assert pd_df.loc[1, "json_col"] == '{"a":1,"b":[1,2]}' - assert pd_df.loc[1, "date_col"] == dt.date(2026, 6, 24) - - # Row 2: Min bounds / negative infinity - assert pd_df.loc[2, "int64_col"] == -9223372036854775808 - assert pd_df.loc[2, "float64_col"] == float("-inf") - assert pd_df.loc[2, "numeric_col"] == decimal.Decimal( - "-99999999999999999999999999999.999999999" - ) - - # Row 3: Max bounds / infinity - assert pd_df.loc[3, "int64_col"] == 9223372036854775807 - assert pd_df.loc[3, "float64_col"] == float("inf") - - # Row 4: SQL NULLs (omitted keys) - assert pd.isna(pd_df.loc[4, "bool_col"]) - assert pd.isna(pd_df.loc[4, "int64_col"]) - assert pd.isna(pd_df.loc[4, "float64_col"]) - assert pd.isna(pd_df.loc[4, "json_col"]) - assert pd.isna(pd_df.loc[4, "geography_col"]) - - # Row 5: Special edge cases (NaN, empty, multiline) - assert np.isnan(pd_df.loc[5, "float64_col"]) - assert pd_df.loc[5, "float64_col"] is not pd.NA - assert not pd_df["float64_col"].isna().loc[5] - assert pd_df.loc[5, "string_col"] == 'Line 1\nLine 2\n"Quotes"' - assert pd_df.loc[5, "bytes_col"] == b"\x00" - - # Row 6: JSON null literal - assert pd_df.loc[6, "json_col"] == "null" - assert not pd_df["json_col"].isna().loc[6] - - def test_concat_series(scalars_dfs): scalars_df, scalars_pandas_df = scalars_dfs bf_result = bpd.concat( @@ -143,7 +64,7 @@ def test_concat_series(scalars_dfs): ] ) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -180,7 +101,7 @@ def test_get_dummies_dataframe(scalars_dfs, kwargs): # dtype argument above is needed for pandas v1 only # adjust for expected dtype differences - for column_name, type_name in zip(pd_result.columns, pd_result.dtypes): + for (column_name, type_name) in zip(pd_result.columns, pd_result.dtypes): if type_name == "bool": pd_result[column_name] = pd_result[column_name].astype("boolean") @@ -194,30 +115,22 @@ def test_get_dummies_dataframe_duplicate_labels(scalars_dfs): scalars_df, scalars_pandas_df = scalars_dfs scalars_renamed_df = scalars_df.rename( - columns={ - "int64_too": "int64_col", - "float64_col": "dup_col", - "string_col": "dup_col", - } + columns={"int64_too": "int64_col", "float64_col": None, "string_col": None} ) scalars_renamed_pandas_df = scalars_pandas_df.rename( - columns={ - "int64_too": "int64_col", - "float64_col": "dup_col", - "string_col": "dup_col", - } + columns={"int64_too": "int64_col", "float64_col": None, "string_col": None} ) bf_result = bpd.get_dummies( - scalars_renamed_df, columns=["int64_col", "dup_col"], dtype=bool + scalars_renamed_df, columns=["int64_col", None], dtype=bool ) pd_result = pd.get_dummies( - scalars_renamed_pandas_df, columns=["int64_col", "dup_col"], dtype=bool + scalars_renamed_pandas_df, columns=["int64_col", None], dtype=bool ) # dtype argument above is needed for pandas v1 only # adjust for expected dtype differences - for column_name, type_name in zip(pd_result.columns, pd_result.dtypes): + for (column_name, type_name) in zip(pd_result.columns, pd_result.dtypes): if type_name == "bool": pd_result[column_name] = pd_result[column_name].astype("boolean") @@ -234,7 +147,7 @@ def test_get_dummies_series(scalars_dfs): # dtype argument above is needed for pandas v1 only # adjust for expected dtype differences - for column_name, type_name in zip(pd_result.columns, pd_result.dtypes): + for (column_name, type_name) in zip(pd_result.columns, pd_result.dtypes): if type_name == "bool": # pragma: NO COVER pd_result[column_name] = pd_result[column_name].astype("boolean") pd_result.columns = pd_result.columns.astype(object) @@ -255,7 +168,7 @@ def test_get_dummies_series_nameless(scalars_dfs): # dtype argument above is needed for pandas v1 only # adjust for expected dtype differences - for column_name, type_name in zip(pd_result.columns, pd_result.dtypes): + for (column_name, type_name) in zip(pd_result.columns, pd_result.dtypes): if type_name == "bool": # pragma: NO COVER pd_result[column_name] = pd_result[column_name].astype("boolean") pd_result.columns = pd_result.columns.astype(object) @@ -619,9 +532,7 @@ def _convert_pandas_category(pd_s: pd.Series): f"Input must be a pandas Series with categorical data: {pd_s.dtype}" ) - if pd.api.types.is_object_dtype( - pd_s.cat.categories.dtype - ) or pd.api.types.is_string_dtype(pd_s.cat.categories.dtype): + if pd.api.types.is_object_dtype(pd_s.cat.categories.dtype): return pd_s.astype(pd.StringDtype(storage="pyarrow")) if not isinstance(pd_s.cat.categories.dtype, pd.IntervalDtype): @@ -637,9 +548,9 @@ def _convert_pandas_category(pd_s: pd.Series): right_key = "right_inclusive" subtype = pd_s.cat.categories.dtype.subtype # type: ignore - if pd.api.types.is_float_dtype(subtype): # type: ignore + if pd.api.types.is_float_dtype(subtype): interval_dtype = pa.float64() - elif pd.api.types.is_integer_dtype(subtype): # type: ignore + elif pd.api.types.is_integer_dtype(subtype): interval_dtype = pa.int64() else: raise ValueError(f"Unknown category type: {subtype}") @@ -680,7 +591,7 @@ def test_cut_for_array(): bf_result = bpd.cut(sc, x) pd_result = _convert_pandas_category(pd_result) - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) @pytest.mark.parametrize( @@ -699,7 +610,7 @@ def test_cut_by_int_bins(scalars_dfs, labels, right): bf_result = bpd.cut(scalars_df["float64_col"], 5, labels=labels, right=right) pd_result = _convert_pandas_category(pd_result) - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) def test_cut_by_int_bins_w_labels(scalars_dfs): @@ -710,7 +621,7 @@ def test_cut_by_int_bins_w_labels(scalars_dfs): bf_result = bpd.cut(scalars_df["float64_col"], 5, labels=labels) pd_result = _convert_pandas_category(pd_result) - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) @pytest.mark.parametrize( @@ -753,7 +664,7 @@ def test_cut_by_numeric_breaks(scalars_dfs, breaks, right, labels): ).to_pandas() pd_result_converted = _convert_pandas_category(pd_result) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result_converted) + pd.testing.assert_series_equal(bf_result, pd_result_converted) def test_cut_by_numeric_breaks_w_labels(scalars_dfs): @@ -765,7 +676,7 @@ def test_cut_by_numeric_breaks_w_labels(scalars_dfs): bf_result = bpd.cut(scalars_df["float64_col"], bins, labels=labels) pd_result = _convert_pandas_category(pd_result) - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) @pytest.mark.parametrize( @@ -805,7 +716,7 @@ def test_cut_by_interval_bins(scalars_dfs, bins, right, labels): pd_result = pd.cut(scalars_pandas_df["int64_too"], bins, labels=labels, right=right) pd_result_converted = _convert_pandas_category(pd_result) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result_converted) + pd.testing.assert_series_equal(bf_result, pd_result_converted) def test_cut_by_interval_bins_w_labels(scalars_dfs): @@ -817,7 +728,7 @@ def test_cut_by_interval_bins_w_labels(scalars_dfs): bf_result = bpd.cut(scalars_df["float64_col"], bins, labels=labels) pd_result = _convert_pandas_category(pd_result) - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) @pytest.mark.parametrize( @@ -834,7 +745,7 @@ def test_cut_by_edge_cases_bins(scalars_dfs, bins, labels): pd_result = pd.cut(scalars_pandas_df["int64_too"], bins, labels=labels) pd_result_converted = _convert_pandas_category(pd_result) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result_converted) + pd.testing.assert_series_equal(bf_result, pd_result_converted) def test_cut_empty_array_raises_error(): @@ -863,7 +774,7 @@ def test_qcut(scalars_dfs, q): bf_result = bpd.qcut(scalars_df["float64_col"], q, labels=False, duplicates="drop") pd_result = pd_result.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) @pytest.mark.parametrize( @@ -905,12 +816,10 @@ def test_to_datetime_iterable(arg, utc, unit, format): .to_pandas() .astype("datetime64[ns, UTC]" if utc else "datetime64[ns]") ) - pd_result = ( - pd.Series(pd.to_datetime(arg, utc=utc, unit=unit, format=format)) - .dt.floor("us") - .astype("datetime64[ns, UTC]" if utc else "datetime64[ns]") - ) - bigframes.testing.utils.assert_series_equal( + pd_result = pd.Series( + pd.to_datetime(arg, utc=utc, unit=unit, format=format) + ).dt.floor("us") + pd.testing.assert_series_equal( bf_result, pd_result, check_index_type=False, check_names=False ) @@ -921,10 +830,8 @@ def test_to_datetime_series(scalars_dfs): bf_result = ( bpd.to_datetime(scalars_df[col], unit="s").to_pandas().astype("datetime64[s]") ) - pd_result = pd.Series(pd.to_datetime(scalars_pandas_df[col], unit="s")).astype( - "datetime64[s]" - ) - bigframes.testing.utils.assert_series_equal( + pd_result = pd.Series(pd.to_datetime(scalars_pandas_df[col], unit="s")) + pd.testing.assert_series_equal( bf_result, pd_result, check_index_type=False, check_names=False ) @@ -945,12 +852,8 @@ def test_to_datetime_series(scalars_dfs): ) def test_to_datetime_unit_param(arg, unit): bf_result = bpd.to_datetime(arg, unit=unit).to_pandas().astype("datetime64[ns]") - pd_result = ( - pd.Series(pd.to_datetime(arg, unit=unit)) - .dt.floor("us") - .astype("datetime64[ns]") - ) - bigframes.testing.utils.assert_series_equal( + pd_result = pd.Series(pd.to_datetime(arg, unit=unit)).dt.floor("us") + pd.testing.assert_series_equal( bf_result, pd_result, check_index_type=False, check_names=False ) @@ -970,12 +873,8 @@ def test_to_datetime_format_param(arg, utc, format): .to_pandas() .astype("datetime64[ns, UTC]" if utc else "datetime64[ns]") ) - pd_result = ( - pd.Series(pd.to_datetime(arg, utc=utc, format=format)) - .dt.floor("us") - .astype("datetime64[ns, UTC]" if utc else "datetime64[ns]") - ) - bigframes.testing.utils.assert_series_equal( + pd_result = pd.Series(pd.to_datetime(arg, utc=utc, format=format)).dt.floor("us") + pd.testing.assert_series_equal( bf_result, pd_result, check_index_type=False, check_names=False ) @@ -1022,18 +921,13 @@ def test_to_datetime_format_param(arg, utc, format): ], ) def test_to_datetime_string_inputs(arg, utc, output_in_utc, format): - normalized_type = "datetime64[ns, UTC]" if output_in_utc else "datetime64[ns]" - bf_result = ( - bpd.to_datetime(arg, utc=utc, format=format).to_pandas().astype(normalized_type) - ) - pd_result = ( - pd.Series(pd.to_datetime(arg, utc=utc, format=format)) - .dt.floor("us") - .astype(normalized_type) + bpd.to_datetime(arg, utc=utc, format=format) + .to_pandas() + .astype("datetime64[ns, UTC]" if output_in_utc else "datetime64[ns]") ) - - bigframes.testing.utils.assert_series_equal( + pd_result = pd.Series(pd.to_datetime(arg, utc=utc, format=format)).dt.floor("us") + pd.testing.assert_series_equal( bf_result, pd_result, check_index_type=False, check_names=False ) @@ -1070,14 +964,13 @@ def test_to_datetime_string_inputs(arg, utc, output_in_utc, format): ], ) def test_to_datetime_timestamp_inputs(arg, utc, output_in_utc): - normalized_type = "datetime64[ns, UTC]" if output_in_utc else "datetime64[ns]" - - bf_result = bpd.to_datetime(arg, utc=utc).to_pandas().astype(normalized_type) - pd_result = ( - pd.Series(pd.to_datetime(arg, utc=utc)).dt.floor("us").astype(normalized_type) + bf_result = ( + bpd.to_datetime(arg, utc=utc) + .to_pandas() + .astype("datetime64[ns, UTC]" if output_in_utc else "datetime64[ns]") ) - - bigframes.testing.utils.assert_series_equal( + pd_result = pd.Series(pd.to_datetime(arg, utc=utc)).dt.floor("us") + pd.testing.assert_series_equal( bf_result, pd_result, check_index_type=False, check_names=False ) @@ -1126,8 +1019,10 @@ def test_to_timedelta_with_bf_integer_series(session, unit): .astype("timedelta64[ns]") ) - expected_result = pd.to_timedelta(pd_series, unit).astype("timedelta64[ns]") - assert_series_equal(actual_result, expected_result, check_index_type=False) + expected_result = pd.to_timedelta(pd_series, unit) + pd.testing.assert_series_equal( + actual_result, expected_result, check_index_type=False + ) def test_to_timedelta_with_bf_float_series_value_rounded_down(session): @@ -1139,10 +1034,8 @@ def test_to_timedelta_with_bf_float_series_value_rounded_down(session): .astype("timedelta64[ns]") ) - expected_result = pd.Series([pd.Timedelta(1, "us"), pd.Timedelta(2, "us")]).astype( - "timedelta64[ns]" - ) - bigframes.testing.utils.assert_series_equal( + expected_result = pd.Series([pd.Timedelta(1, "us"), pd.Timedelta(2, "us")]) + pd.testing.assert_series_equal( actual_result, expected_result, check_index_type=False ) @@ -1162,8 +1055,8 @@ def test_to_timedelta_with_list_like_input(session, input): .astype("timedelta64[ns]") ) - expected_result = pd.Series(pd.to_timedelta(input, "s")).astype("timedelta64[ns]") - bigframes.testing.utils.assert_series_equal( + expected_result = pd.Series(pd.to_timedelta(input, "s")) + pd.testing.assert_series_equal( actual_result, expected_result, check_index_type=False ) @@ -1193,7 +1086,7 @@ def test_to_timedelta_on_timedelta_series__should_be_no_op(scalars_dfs): bpd.to_timedelta(bf_series, unit="s").to_pandas().astype("timedelta64[ns]") ) - expected_result = pd.to_timedelta(pd_series, unit="s").astype("timedelta64[ns]") - bigframes.testing.utils.assert_series_equal( + expected_result = pd.to_timedelta(pd_series, unit="s") + pd.testing.assert_series_equal( actual_result, expected_result, check_index_type=False ) diff --git a/tests/system/small/test_pandas_options.py b/tests/system/small/test_pandas_options.py index a9ec4355f07..7a750ddfd3c 100644 --- a/tests/system/small/test_pandas_options.py +++ b/tests/system/small/test_pandas_options.py @@ -14,8 +14,8 @@ import datetime import re -import warnings from unittest import mock +import warnings import google.api_core.exceptions import pandas.testing @@ -50,6 +50,7 @@ def test_read_gbq_start_sets_session_location( query_prefix, reset_default_session_and_location, ): + # Form query as a table name or a SQL depending on the test scenario query_tokyo = test_data_tables_tokyo["scalars"] query = test_data_tables["scalars"] @@ -316,9 +317,7 @@ def test_credentials_need_reauthentication( with warnings.catch_warnings(record=True) as warned: bpd.close_session() # CleanupFailedWarning: can't clean up - # The test forces a failure during cleanup and asserts that one or more warning is generated - # when/if multiple temp tables might have been left over. - assert len(warned) >= 1 + assert len(warned) == 1 assert warned[0].category == bigframes.exceptions.CleanupFailedWarning assert ( @@ -342,9 +341,8 @@ def test_max_rows_normal_execution_within_limit( expected = scalars_pandas_df_index.head(10) pandas.testing.assert_frame_equal(result, expected) - with ( - bpd.option_context("compute.maximum_result_rows", 10), - bpd.option_context("display.repr_mode", "head"), + with bpd.option_context("compute.maximum_result_rows", 10), bpd.option_context( + "display.repr_mode", "head" ): df = scalars_df_index.head(10) assert repr(df) is not None @@ -363,14 +361,12 @@ def test_max_rows_normal_execution_within_limit( def test_max_rows_exceeds_limit(scalars_df_index): """Test to_pandas() raises MaximumRowsDownloadedExceeded when the limit is exceeded.""" - with ( - bpd.option_context("compute.maximum_result_rows", 5), - pytest.raises(bigframes.exceptions.MaximumResultRowsExceeded, match="5"), + with bpd.option_context("compute.maximum_result_rows", 5), pytest.raises( + bigframes.exceptions.MaximumResultRowsExceeded, match="5" ): scalars_df_index.to_pandas() - with ( - bpd.option_context("compute.maximum_result_rows", 5), - pytest.raises(bigframes.exceptions.MaximumResultRowsExceeded, match="5"), + with bpd.option_context("compute.maximum_result_rows", 5), pytest.raises( + bigframes.exceptions.MaximumResultRowsExceeded, match="5" ): next(iter(scalars_df_index.to_pandas_batches())) diff --git a/tests/system/small/test_polars_execution.py b/tests/system/small/test_polars_execution.py index fad8d9dba2f..1b58dc9d12b 100644 --- a/tests/system/small/test_polars_execution.py +++ b/tests/system/small/test_polars_execution.py @@ -39,7 +39,7 @@ def test_polar_execution_sorted(session_w_polars, scalars_pandas_df_index): ] bf_result = bf_df.sort_index(ascending=False)[["int64_too", "bool_col"]].to_pandas() - assert session_w_polars._metrics.execution_count == execution_count_before + 1 + assert session_w_polars._metrics.execution_count == execution_count_before assert_frame_equal(bf_result, pd_result) @@ -56,7 +56,7 @@ def test_polar_execution_sorted_filtered(session_w_polars, scalars_pandas_df_ind .to_pandas() ) - assert session_w_polars._metrics.execution_count == execution_count_before + 1 + assert session_w_polars._metrics.execution_count == execution_count_before assert_frame_equal(bf_result, pd_result) @@ -70,28 +70,5 @@ def test_polar_execution_unsupported_sql_fallback( bf_result = bf_df.to_pandas() # geo fns not supported by polar engine yet, so falls back to bq execution - assert session_w_polars._metrics.execution_count == (execution_count_before + 2) + assert session_w_polars._metrics.execution_count == (execution_count_before + 1) assert math.isclose(bf_result.geo_area.sum(), 70.52332050, rel_tol=0.00001) - - -def test_polars_execution_history(session_w_polars): - import pandas as pd - - # Create a small local DataFrame - pdf = pd.DataFrame({"col_a": [1, 2, 3], "col_b": ["x", "y", "z"]}) - - # Read simple local data - df = session_w_polars.read_pandas(pdf) - - # Trigger execution - _ = df.to_pandas() - - # Verify the execution history captured the local job - history = session_w_polars.execution_history().to_dataframe() - - # Verify we have at least one job and logged as polars - assert len(history) > 0 - last_job = history.iloc[-1] - - assert last_job["job_type"] == "polars" - assert last_job["status"] == "DONE" diff --git a/tests/system/small/test_progress_bar.py b/tests/system/small/test_progress_bar.py index a179e18332a..d726bfde2c2 100644 --- a/tests/system/small/test_progress_bar.py +++ b/tests/system/small/test_progress_bar.py @@ -92,10 +92,9 @@ def test_progress_bar_load_jobs( while len(df) < MAX_INLINE_DF_BYTES: df = pd.DataFrame(np.repeat(df.values, 2, axis=0)) - with ( - bf.option_context("display.progress_bar", "terminal"), - tempfile.TemporaryDirectory() as dir, - ): + with bf.option_context( + "display.progress_bar", "terminal" + ), tempfile.TemporaryDirectory() as dir: path = dir + "/test_read_csv_progress_bar*.csv" df.to_csv(path, index=False) capsys.readouterr() # clear output @@ -104,23 +103,6 @@ def test_progress_bar_load_jobs( assert_loading_msg_exist(capsys.readouterr().out, pattern="Load") -def test_progress_bar_uniqueness_check(session: bf.Session, capsys): - # Ensure strictly_ordered is True (default) to trigger uniqueness check - assert session._strictly_ordered - - capsys.readouterr() # clear output - - with bf.option_context("display.progress_bar", "terminal"): - # Read a table and specify a non-unique index_col to trigger the check. - # We use a public table to make it a "real" test. - session.read_gbq_table( - "bigquery-public-data.ml_datasets.penguins", - index_col="island", - ) - - assert_loading_msg_exist(capsys.readouterr().out) - - def assert_loading_msg_exist(capstdout: str, pattern=job_load_message_regex): num_loading_msg = 0 lines = capstdout.split("\n") @@ -169,7 +151,7 @@ def test_query_job_dry_run_series(penguins_df_default_index: bf.dataframe.DataFr def test_repr_anywidget_dataframe(penguins_df_default_index: bf.dataframe.DataFrame): pytest.importorskip("anywidget") - with bf.option_context("display.render_mode", "anywidget"): + with bf.option_context("display.repr_mode", "anywidget"): actual_repr = repr(penguins_df_default_index) assert "species" in actual_repr assert "island" in actual_repr @@ -178,7 +160,7 @@ def test_repr_anywidget_dataframe(penguins_df_default_index: bf.dataframe.DataFr def test_repr_anywidget_index(penguins_df_default_index: bf.dataframe.DataFrame): pytest.importorskip("anywidget") - with bf.option_context("display.render_mode", "anywidget"): + with bf.option_context("display.repr_mode", "anywidget"): index = penguins_df_default_index.index actual_repr = repr(index) # In non-interactive environments, should still get a useful summary. diff --git a/tests/system/small/test_series.py b/tests/system/small/test_series.py index 2e80b75c0b4..a95c9623e52 100644 --- a/tests/system/small/test_series.py +++ b/tests/system/small/test_series.py @@ -22,18 +22,16 @@ import geopandas as gpd # type: ignore import google.api_core.exceptions import numpy +from packaging.version import Version import pandas as pd import pyarrow as pa # type: ignore import pytest import shapely.geometry # type: ignore -from packaging.version import Version import bigframes.dtypes as dtypes import bigframes.features import bigframes.pandas import bigframes.series as series -import bigframes.testing -import bigframes.testing.utils from bigframes.testing.utils import ( assert_frame_equal, assert_series_equal, @@ -49,7 +47,7 @@ def test_series_construct_copy(scalars_dfs): pd_result = pd.Series( scalars_pandas_df["int64_col"], name="test_series", dtype="Float64" ) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_construct_nullable_ints(): @@ -64,7 +62,7 @@ def test_series_construct_nullable_ints(): ) expected = pd.Series([1, 3, pd.NA], dtype=pd.Int64Dtype(), index=expected_index) - bigframes.testing.utils.assert_series_equal(bf_result, expected) + pd.testing.assert_series_equal(bf_result, expected) def test_series_construct_timestamps(): @@ -76,9 +74,7 @@ def test_series_construct_timestamps(): bf_result = series.Series(datetimes).to_pandas() pd_result = pd.Series(datetimes, dtype=pd.ArrowDtype(pa.timestamp("us"))) - bigframes.testing.utils.assert_series_equal( - bf_result, pd_result, check_index_type=False - ) + pd.testing.assert_series_equal(bf_result, pd_result, check_index_type=False) def test_series_construct_copy_with_index(scalars_dfs): @@ -95,7 +91,7 @@ def test_series_construct_copy_with_index(scalars_dfs): dtype="Float64", index=scalars_pandas_df["int64_too"], ) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_construct_copy_index(scalars_dfs): @@ -112,7 +108,7 @@ def test_series_construct_copy_index(scalars_dfs): dtype="Float64", index=scalars_pandas_df["int64_too"], ) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_construct_pandas(scalars_dfs): @@ -124,7 +120,7 @@ def test_series_construct_pandas(scalars_dfs): scalars_pandas_df["int64_col"], name="test_series", dtype="Float64" ) assert bf_result.shape == pd_result.shape - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) def test_series_construct_from_list(): @@ -134,7 +130,7 @@ def test_series_construct_from_list(): # BigQuery DataFrame default indices use nullable Int64 always pd_result.index = pd_result.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_construct_reindex(): @@ -145,7 +141,7 @@ def test_series_construct_reindex(): # BigQuery DataFrame default indices use nullable Int64 always pd_result.index = pd_result.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_construct_from_list_w_index(): @@ -159,7 +155,7 @@ def test_series_construct_from_list_w_index(): # BigQuery DataFrame default indices use nullable Int64 always pd_result.index = pd_result.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_construct_empty(session: bigframes.Session): @@ -180,7 +176,7 @@ def test_series_construct_scalar_no_index(): # BigQuery DataFrame default indices use nullable Int64 always pd_result.index = pd_result.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_construct_scalar_w_index(): @@ -192,7 +188,7 @@ def test_series_construct_scalar_w_index(): # BigQuery DataFrame default indices use nullable Int64 always pd_result.index = pd_result.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_construct_nan(): @@ -202,7 +198,7 @@ def test_series_construct_nan(): pd_result.index = pd_result.index.astype("Int64") pd_result = pd_result.astype("Float64") - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_construct_scalar_w_bf_index(): @@ -213,7 +209,7 @@ def test_series_construct_scalar_w_bf_index(): pd_result = pd_result.astype("string[pyarrow]") - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_construct_from_list_escaped_strings(): @@ -229,7 +225,7 @@ def test_series_construct_from_list_escaped_strings(): # BigQuery DataFrame default indices use nullable Int64 always pd_result.index = pd_result.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) def test_series_construct_geodata(): @@ -244,7 +240,7 @@ def test_series_construct_geodata(): series = bigframes.pandas.Series(pd_series) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_series, series.to_pandas(), check_index_type=False ) @@ -262,7 +258,7 @@ def test_series_construct_w_dtype(dtype): expected = pd.Series(data, dtype=dtype) expected.index = expected.index.astype("Int64") series = bigframes.pandas.Series(data, dtype=dtype) - bigframes.testing.utils.assert_series_equal(series.to_pandas(), expected) + pd.testing.assert_series_equal(series.to_pandas(), expected) def test_series_construct_w_dtype_for_struct(): @@ -279,7 +275,7 @@ def test_series_construct_w_dtype_for_struct(): series = bigframes.pandas.Series(data, dtype=dtype) expected = pd.Series(data, dtype=dtype) expected.index = expected.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(series.to_pandas(), expected) + pd.testing.assert_series_equal(series.to_pandas(), expected) def test_series_construct_w_dtype_for_array_string(): @@ -297,7 +293,7 @@ def test_series_construct_w_dtype_for_array_string(): else: check_dtype = False - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( series.to_pandas(), expected, check_dtype=check_dtype ) @@ -317,7 +313,7 @@ def test_series_construct_w_dtype_for_array_struct(): else: check_dtype = False - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( series.to_pandas(), expected, check_dtype=check_dtype ) @@ -327,7 +323,7 @@ def test_series_construct_local_unordered_has_sequential_index(unordered_session ["Sun", "Mon", "Tues", "Wed", "Thurs", "Fri", "Sat"], session=unordered_session ) expected: pd.Index = pd.Index([0, 1, 2, 3, 4, 5, 6], dtype=pd.Int64Dtype()) - bigframes.testing.utils.assert_index_equal(series.index.to_pandas(), expected) + pd.testing.assert_index_equal(series.index.to_pandas(), expected) @pytest.mark.parametrize( @@ -389,14 +385,14 @@ def test_series_construct_w_nested_json_dtype(): ), ) - bigframes.testing.utils.assert_series_equal(s.to_pandas(), s2.to_pandas()) + pd.testing.assert_series_equal(s.to_pandas(), s2.to_pandas()) def test_series_keys(scalars_dfs): scalars_df, scalars_pandas_df = scalars_dfs bf_result = scalars_df["int64_col"].keys().to_pandas() pd_result = scalars_pandas_df["int64_col"].keys() - bigframes.testing.utils.assert_index_equal(bf_result, pd_result) + pd.testing.assert_index_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -541,7 +537,7 @@ def test_series___getitem__(scalars_dfs, index_col, key): scalars_pandas_df = scalars_pandas_df.set_index(index_col, drop=False) bf_result = scalars_df[col_name][key] pd_result = scalars_pandas_df[col_name][key] - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) @pytest.mark.parametrize( @@ -554,8 +550,6 @@ def test_series___getitem__(scalars_dfs, index_col, key): ), ) def test_series___getitem___with_int_key(scalars_dfs, key): - if pd.__version__.startswith("3."): - pytest.skip("pandas 3.0 dropped getitem with int key") col_name = "int64_too" index_col = "string_col" scalars_df, scalars_pandas_df = scalars_dfs @@ -595,7 +589,7 @@ def test_series___setitem__(scalars_dfs, index_col, key, value): bf_series[key] = value pd_series[key] = value - bigframes.testing.utils.assert_series_equal(bf_series.to_pandas(), pd_series) + pd.testing.assert_series_equal(bf_series.to_pandas(), pd_series) @pytest.mark.parametrize( @@ -620,7 +614,7 @@ def test_series___setitem___with_int_key_numeric(scalars_dfs, key, value): bf_series[key] = value pd_series[key] = value - bigframes.testing.utils.assert_series_equal(bf_series.to_pandas(), pd_series) + pd.testing.assert_series_equal(bf_series.to_pandas(), pd_series) def test_series___setitem___with_default_index(scalars_dfs): @@ -717,7 +711,7 @@ def test_series_replace_scalar_scalar(scalars_dfs): ) pd_result = scalars_pandas_df[col_name].replace("Hello, World!", "Howdy, Planet!") - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, ) @@ -733,7 +727,7 @@ def test_series_replace_regex_scalar(scalars_dfs): "^H.l", "Howdy, Planet!", regex=True ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, ) @@ -751,7 +745,7 @@ def test_series_replace_list_scalar(scalars_dfs): ["Hello, World!", "T"], "Howdy, Planet!" ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, ) @@ -763,7 +757,7 @@ def test_series_replace_nans_with_pd_na(scalars_dfs): bf_result = scalars_df[col_name].replace({pd.NA: "UNKNOWN"}).to_pandas() pd_result = scalars_pandas_df[col_name].replace({pd.NA: "UNKNOWN"}) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, ) @@ -788,7 +782,7 @@ def test_series_replace_dict(scalars_dfs, replacement_dict): bf_result = scalars_df[col_name].replace(replacement_dict).to_pandas() pd_result = scalars_pandas_df[col_name].replace(replacement_dict) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, ) @@ -844,9 +838,7 @@ def test_series_dropna(scalars_dfs, ignore_index): col_name = "string_col" bf_result = scalars_df[col_name].dropna(ignore_index=ignore_index).to_pandas() pd_result = scalars_pandas_df[col_name].dropna(ignore_index=ignore_index) - bigframes.testing.utils.assert_series_equal( - pd_result, bf_result, check_index_type=False - ) + pd.testing.assert_series_equal(pd_result, bf_result, check_index_type=False) @pytest.mark.parametrize( @@ -882,9 +874,7 @@ def test_series_agg_multi_string(scalars_dfs): # Pandas may produce narrower numeric types, but bigframes always produces Float64 pd_result = pd_result.astype("Float64") - bigframes.testing.utils.assert_series_equal( - pd_result, bf_result, check_index_type=False - ) + pd.testing.assert_series_equal(pd_result, bf_result, check_index_type=False) @pytest.mark.parametrize( @@ -1001,7 +991,7 @@ def test_mode_stat(scalars_df_index, scalars_pandas_df_index, col_name): ## Mode implicitly resets index, and bigframes default indices use nullable Int64 pd_result.index = pd_result.index.astype("Int64") - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -1169,7 +1159,7 @@ def test_mods(scalars_dfs, col_x, col_y, method): else: bf_result = bf_series.astype("Float64").to_pandas() pd_result = getattr(scalars_pandas_df[col_x], method)(scalars_pandas_df[col_y]) - bigframes.testing.utils.assert_series_equal(pd_result, bf_result) + pd.testing.assert_series_equal(pd_result, bf_result) # We work around a pandas bug that doesn't handle correlating nullable dtypes by doing this @@ -1233,12 +1223,19 @@ def test_divmods_series(scalars_dfs, col_x, col_y, method): scalars_pandas_df[col_y] ) # BigQuery's mod functions return NUMERIC values for non-INT64 inputs. - bigframes.testing.utils.assert_series_equal( - pd_div_result, bf_div_result.to_pandas(), check_dtype=False - ) - bigframes.testing.utils.assert_series_equal( - pd_mod_result, bf_mod_result.to_pandas(), check_dtype=False - ) + if bf_div_result.dtype == pd.Int64Dtype(): + pd.testing.assert_series_equal(pd_div_result, bf_div_result.to_pandas()) + else: + pd.testing.assert_series_equal( + pd_div_result, bf_div_result.astype("Float64").to_pandas() + ) + + if bf_mod_result.dtype == pd.Int64Dtype(): + pd.testing.assert_series_equal(pd_mod_result, bf_mod_result.to_pandas()) + else: + pd.testing.assert_series_equal( + pd_mod_result, bf_mod_result.astype("Float64").to_pandas() + ) @pytest.mark.parametrize( @@ -1268,20 +1265,16 @@ def test_divmods_scalars(scalars_dfs, col_x, other, method): pd_div_result, pd_mod_result = getattr(scalars_pandas_df[col_x], method)(other) # BigQuery's mod functions return NUMERIC values for non-INT64 inputs. if bf_div_result.dtype == pd.Int64Dtype(): - bigframes.testing.utils.assert_series_equal( - pd_div_result, bf_div_result.to_pandas(), check_dtype=False - ) + pd.testing.assert_series_equal(pd_div_result, bf_div_result.to_pandas()) else: - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_div_result, bf_div_result.astype("Float64").to_pandas() ) if bf_mod_result.dtype == pd.Int64Dtype(): - bigframes.testing.utils.assert_series_equal( - pd_div_result, bf_div_result.to_pandas(), check_dtype=False - ) + pd.testing.assert_series_equal(pd_mod_result, bf_mod_result.to_pandas()) else: - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_mod_result, bf_mod_result.astype("Float64").to_pandas() ) @@ -1354,9 +1347,7 @@ def test_series_add_different_table_default_index( + scalars_df_2_default_index["float64_col"].to_pandas() ) # TODO(swast): Can remove sort_index() when there's default ordering. - bigframes.testing.utils.assert_series_equal( - bf_result.sort_index(), pd_result.sort_index() - ) + pd.testing.assert_series_equal(bf_result.sort_index(), pd_result.sort_index()) def test_series_add_different_table_with_index( @@ -1367,7 +1358,7 @@ def test_series_add_different_table_with_index( # When index values are unique, we can emulate with values from the same # DataFrame. pd_result = scalars_pandas_df["float64_col"] + scalars_pandas_df["int64_col"] - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) def test_reset_index_drop(scalars_df_index, scalars_pandas_df_index): @@ -1386,7 +1377,7 @@ def test_reset_index_drop(scalars_df_index, scalars_pandas_df_index): # BigQuery DataFrames default indices use nullable Int64 always pd_result.index = pd_result.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) def test_series_reset_index_allow_duplicates(scalars_df_index, scalars_pandas_df_index): @@ -1405,7 +1396,7 @@ def test_series_reset_index_allow_duplicates(scalars_df_index, scalars_pandas_df pd_result.index = pd_result.index.astype(pd.Int64Dtype()) # reset_index should maintain the original ordering. - bigframes.testing.utils.assert_frame_equal(bf_result, pd_result) + pd.testing.assert_frame_equal(bf_result, pd_result) def test_series_reset_index_duplicates_error(scalars_df_index): @@ -1424,7 +1415,7 @@ def test_series_reset_index_inplace(scalars_df_index, scalars_pandas_df_index): # BigQuery DataFrames default indices use nullable Int64 always pd_result.index = pd_result.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) @pytest.mark.parametrize( @@ -1451,7 +1442,7 @@ def test_reset_index_no_drop(scalars_df_index, scalars_pandas_df_index, name): # BigQuery DataFrames default indices use nullable Int64 always pd_result.index = pd_result.index.astype("Int64") - bigframes.testing.utils.assert_frame_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_frame_equal(bf_result.to_pandas(), pd_result) def test_copy(scalars_df_index, scalars_pandas_df_index): @@ -1468,7 +1459,7 @@ def test_copy(scalars_df_index, scalars_pandas_df_index): pd_series.loc[0] = 3.4 assert bf_copy.to_pandas().loc[0] != bf_series.to_pandas().loc[0] - bigframes.testing.utils.assert_series_equal(bf_copy.to_pandas(), pd_copy) + pd.testing.assert_series_equal(bf_copy.to_pandas(), pd_copy) def test_isin_raise_error(scalars_df_index, scalars_pandas_df_index): @@ -1509,7 +1500,7 @@ def test_isin(scalars_dfs, col_name, test_set): scalars_df, scalars_pandas_df = scalars_dfs bf_result = scalars_df[col_name].isin(test_set).to_pandas() pd_result = scalars_pandas_df[col_name].isin(test_set).astype("boolean") - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, ) @@ -1549,7 +1540,7 @@ def test_isin_bigframes_values(scalars_dfs, col_name, test_set, session): scalars_df[col_name].isin(series.Series(test_set, session=session)).to_pandas() ) pd_result = scalars_pandas_df[col_name].isin(test_set).astype("boolean") - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, ) @@ -1559,11 +1550,7 @@ def test_isin_bigframes_index(scalars_dfs, session): scalars_df, scalars_pandas_df = scalars_dfs bf_result = ( scalars_df["string_col"] - .isin( - bigframes.pandas.Index( - ["Hello, World!", "Hi", "こんにちは"], session=session - ) - ) + .isin(bigframes.pandas.Index(["Hello, World!", "Hi", "こんにちは"], session=session)) .to_pandas() ) pd_result = ( @@ -1571,7 +1558,7 @@ def test_isin_bigframes_index(scalars_dfs, session): .isin(pd.Index(["Hello, World!", "Hi", "こんにちは"])) .astype("boolean") ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result, ) @@ -1616,7 +1603,7 @@ def test_isin_bigframes_values_as_predicate( pd_predicate = scalars_pandas_df[col_name].isin(test_set) pd_result = scalars_pandas_df[pd_predicate] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( pd_result.reset_index(), bf_result.reset_index(), ) @@ -1717,10 +1704,10 @@ def test_loc_setitem_cell(scalars_df_index, scalars_pandas_df_index): pd_series.loc[2] = "This value isn't in the test data." bf_result = bf_series.to_pandas() pd_result = pd_series - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) # Per Copy-on-Write semantics, other references to the original DataFrame # should remain unchanged. - bigframes.testing.utils.assert_series_equal(bf_original.to_pandas(), pd_original) + pd.testing.assert_series_equal(bf_original.to_pandas(), pd_original) def test_at_setitem_row_label_scalar(scalars_dfs): @@ -1731,7 +1718,7 @@ def test_at_setitem_row_label_scalar(scalars_dfs): pd_series.at[1] = 1000 bf_result = bf_series.to_pandas() pd_result = pd_series.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_ne_obj_series(scalars_dfs): @@ -1903,10 +1890,6 @@ def test_series_binop_w_other_types(scalars_dfs, other): bf_result = (scalars_df["int64_col"].head(3) + other).to_pandas() pd_result = scalars_pandas_df["int64_col"].head(3) + other - if isinstance(other, pd.Series): - # pandas 3.0 preserves series name, bigframe, earlier pandas do not - pd_result.index.name = bf_result.index.name - assert_series_equal( bf_result, pd_result, @@ -2015,7 +1998,7 @@ def test_series_quantile(scalars_dfs): pd_result = pd_series.quantile([0.0, 0.4, 0.6, 1.0]) bf_result = bf_series.quantile([0.0, 0.4, 0.6, 1.0]) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result.to_pandas(), check_dtype=False, check_index_type=False ) @@ -2064,7 +2047,7 @@ def test_cumprod(scalars_dfs): col_name = "float64_col" bf_result = scalars_df[col_name].cumprod() pd_result = scalars_pandas_df[col_name].cumprod() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_result, bf_result.to_pandas(), ) @@ -2165,7 +2148,7 @@ def test_groupby_level_sum(scalars_dfs): bf_series = scalars_df[col_name].groupby(level=0).sum() pd_series = scalars_pandas_df[col_name].groupby(level=0).sum() # TODO(swast): Update groupby to use index based on group by key(s). - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_series.sort_index(), bf_series.to_pandas().sort_index(), ) @@ -2179,7 +2162,7 @@ def test_groupby_level_list_sum(scalars_dfs): bf_series = scalars_df[col_name].groupby(level=["rowindex"]).sum() pd_series = scalars_pandas_df[col_name].groupby(level=["rowindex"]).sum() # TODO(swast): Update groupby to use index based on group by key(s). - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_series.sort_index(), bf_series.to_pandas().sort_index(), ) @@ -2296,7 +2279,7 @@ def test_groupby_window_ops(scalars_df_index, scalars_pandas_df_index, operator) scalars_pandas_df_index[col_name].groupby(scalars_pandas_df_index[group_key]) ).astype(bf_series.dtype) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_series, bf_series, ) @@ -2312,7 +2295,7 @@ def test_groupby_window_ops(scalars_df_index, scalars_pandas_df_index, operator) def test_drop_label(scalars_df_index, scalars_pandas_df_index, label, col_name): bf_series = scalars_df_index[col_name].drop(label).to_pandas() pd_series = scalars_pandas_df_index[col_name].drop(label) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_series, bf_series, ) @@ -2322,7 +2305,7 @@ def test_drop_label_list(scalars_df_index, scalars_pandas_df_index): col_name = "int64_col" bf_series = scalars_df_index[col_name].drop([1, 3]).to_pandas() pd_series = scalars_pandas_df_index[col_name].drop([1, 3]) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_series, bf_series, ) @@ -2346,7 +2329,7 @@ def test_drop_label_list(scalars_df_index, scalars_pandas_df_index): def test_drop_duplicates(scalars_df_index, scalars_pandas_df_index, keep, col_name): bf_series = scalars_df_index[col_name].drop_duplicates(keep=keep).to_pandas() pd_series = scalars_pandas_df_index[col_name].drop_duplicates(keep=keep) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd_series, bf_series, ) @@ -2383,7 +2366,7 @@ def test_unique(scalars_df_index, scalars_pandas_df_index, col_name): def test_duplicated(scalars_df_index, scalars_pandas_df_index, keep, col_name): bf_series = scalars_df_index[col_name].duplicated(keep=keep).to_pandas() pd_series = scalars_pandas_df_index[col_name].duplicated(keep=keep) - bigframes.testing.utils.assert_series_equal(pd_series, bf_series, check_dtype=False) + pd.testing.assert_series_equal(pd_series, bf_series, check_dtype=False) def test_shape(scalars_dfs): @@ -2517,7 +2500,7 @@ def test_head_then_scalar_operation(scalars_dfs): bf_result = (scalars_df["float64_col"].head(1) + 4).to_pandas() pd_result = scalars_pandas_df["float64_col"].head(1) + 4 - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2533,7 +2516,7 @@ def test_head_then_series_operation(scalars_dfs): "float64_col" ].head(2) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2544,7 +2527,7 @@ def test_series_peek(scalars_dfs): peek_result = scalars_df["float64_col"].peek(n=3, force=False) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( peek_result, scalars_pandas_df["float64_col"].reindex_like(peek_result), ) @@ -2563,7 +2546,7 @@ def test_series_peek_with_large_results_not_allowed(scalars_dfs): # The metrics won't be fully updated when we call query_and_wait. print(session.slot_millis_sum - slot_millis_sum) assert session.slot_millis_sum - slot_millis_sum < 500 - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( peek_result, scalars_pandas_df["float64_col"].reindex_like(peek_result), ) @@ -2577,7 +2560,7 @@ def test_series_peek_multi_index(scalars_dfs): pd_series = scalars_pandas_df.set_index(["string_col", "bool_col"])["float64_col"] pd_series.name = ("2-part", "name") peek_result = bf_series.peek(n=3, force=False) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( peek_result, pd_series.reindex_like(peek_result), ) @@ -2589,7 +2572,7 @@ def test_series_peek_filtered(scalars_dfs): n=3, force=False ) pd_result = scalars_pandas_df[scalars_pandas_df.int64_col > 0]["float64_col"] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( peek_result, pd_result.reindex_like(peek_result), ) @@ -2605,7 +2588,7 @@ def test_series_peek_force(scalars_dfs): peek_result = df_filtered.peek(n=3, force=True) pd_cumsum_df = scalars_pandas_df[["int64_col", "int64_too"]].cumsum() pd_result = pd_cumsum_df[pd_cumsum_df.int64_col > 0]["int64_too"] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( peek_result, pd_result.reindex_like(peek_result), ) @@ -2621,7 +2604,7 @@ def test_series_peek_force_float(scalars_dfs): peek_result = df_filtered.peek(n=3, force=True) pd_cumsum_df = scalars_pandas_df[["int64_col", "float64_col"]].cumsum() pd_result = pd_cumsum_df[pd_cumsum_df.float64_col > 0]["float64_col"] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( peek_result, pd_result.reindex_like(peek_result), ) @@ -2633,7 +2616,7 @@ def test_shift(scalars_df_index, scalars_pandas_df_index): # cumsum does not behave well on nullable ints in pandas, produces object type and never ignores NA pd_result = scalars_pandas_df_index[col_name].shift().astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2644,7 +2627,7 @@ def test_series_ffill(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index[col_name].ffill(limit=1).to_pandas() pd_result = scalars_pandas_df_index[col_name].ffill(limit=1) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2655,7 +2638,7 @@ def test_series_bfill(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index[col_name].bfill(limit=2).to_pandas() pd_result = scalars_pandas_df_index[col_name].bfill(limit=2) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2670,7 +2653,7 @@ def test_cumsum_int(scalars_df_index, scalars_pandas_df_index): # cumsum does not behave well on nullable ints in pandas, produces object type and never ignores NA pd_result = scalars_pandas_df_index[col_name].cumsum().astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2691,7 +2674,7 @@ def test_cumsum_int_ordered(scalars_df_index, scalars_pandas_df_index): .astype(pd.Int64Dtype()) ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2710,7 +2693,7 @@ def test_series_nlargest(scalars_df_index, scalars_pandas_df_index, keep): bf_result = scalars_df_index[col_name].nlargest(4, keep=keep).to_pandas() pd_result = scalars_pandas_df_index[col_name].nlargest(4, keep=keep) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2733,7 +2716,7 @@ def test_diff(scalars_df_index, scalars_pandas_df_index, periods): .astype(pd.Int64Dtype()) ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2752,7 +2735,7 @@ def test_series_pct_change(scalars_df_index, scalars_pandas_df_index, periods): # cumsum does not behave well on nullable ints in pandas, produces object type and never ignores NA pd_result = scalars_pandas_df_index["int64_col"].ffill().pct_change(periods=periods) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2771,7 +2754,7 @@ def test_series_nsmallest(scalars_df_index, scalars_pandas_df_index, keep): bf_result = scalars_df_index[col_name].nsmallest(2, keep=keep).to_pandas() pd_result = scalars_pandas_df_index[col_name].nsmallest(2, keep=keep) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2820,7 +2803,7 @@ def test_series_rank( .astype(pd.Float64Dtype()) ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2832,7 +2815,7 @@ def test_cast_float_to_int(scalars_df_index, scalars_pandas_df_index): # cumsum does not behave well on nullable floats in pandas, produces object type and never ignores NA pd_result = scalars_pandas_df_index[col_name].astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2844,7 +2827,7 @@ def test_cast_float_to_bool(scalars_df_index, scalars_pandas_df_index): # cumsum does not behave well on nullable floats in pandas, produces object type and never ignores NA pd_result = scalars_pandas_df_index[col_name].astype(pd.BooleanDtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2862,7 +2845,7 @@ def test_cumsum_nested(scalars_df_index, scalars_pandas_df_index): .astype(pd.Float64Dtype()) ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2891,7 +2874,7 @@ def test_nested_analytic_ops_align(scalars_df_index, scalars_pandas_df_index): + pd_series.expanding().max() ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2907,7 +2890,7 @@ def test_cumsum_int_filtered(scalars_df_index, scalars_pandas_df_index): # cumsum does not behave well on nullable ints in pandas, produces object type and never ignores NA pd_result = pd_col[pd_col > -2].cumsum().astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2919,7 +2902,7 @@ def test_cumsum_float(scalars_df_index, scalars_pandas_df_index): # cumsum does not behave well on nullable floats in pandas, produces object type and never ignores NA pd_result = scalars_pandas_df_index[col_name].cumsum().astype(pd.Float64Dtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2930,7 +2913,7 @@ def test_cummin_int(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index[col_name].cummin().to_pandas() pd_result = scalars_pandas_df_index[col_name].cummin() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2941,7 +2924,7 @@ def test_cummax_int(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index[col_name].cummax().to_pandas() pd_result = scalars_pandas_df_index[col_name].cummax() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -2974,7 +2957,7 @@ def test_value_counts(scalars_dfs, kwargs): bf_result = s.value_counts(**kwargs).to_pandas() pd_result = pd_s.value_counts(**kwargs) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3012,17 +2995,18 @@ def test_value_counts_w_cut(scalars_dfs): pd_result = pd_cut.value_counts() pd_result.index = pd_result.index.astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result.astype(pd.Int64Dtype()), ) def test_iloc_nested(scalars_df_index, scalars_pandas_df_index): + bf_result = scalars_df_index["string_col"].iloc[1:].iloc[1:].to_pandas() pd_result = scalars_pandas_df_index["string_col"].iloc[1:].iloc[1:] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3051,7 +3035,7 @@ def test_iloc_nested(scalars_df_index, scalars_pandas_df_index): def test_series_iloc(scalars_df_index, scalars_pandas_df_index, start, stop, step): bf_result = scalars_df_index["string_col"].iloc[start:stop:step].to_pandas() pd_result = scalars_pandas_df_index["string_col"].iloc[start:stop:step] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3087,7 +3071,7 @@ def test_series_add_prefix(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index["int64_too"].add_prefix("prefix_") # Index will be object type in pandas, string type in bigframes, but same values - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, check_index_type=False, @@ -3100,7 +3084,7 @@ def test_series_add_suffix(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index["int64_too"].add_suffix("_suffix") # Index will be object type in pandas, string type in bigframes, but same values - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, check_index_type=False, @@ -3128,7 +3112,7 @@ def test_series_filter_like(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index["float64_col"].filter(like="ello") - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3142,7 +3126,7 @@ def test_series_filter_regex(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index["float64_col"].filter(regex="^[GH].*") - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3157,7 +3141,7 @@ def test_series_reindex(scalars_df_index, scalars_pandas_df_index): # Pandas uses int64 instead of Int64 (nullable) dtype. pd_result.index = pd_result.index.astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3184,7 +3168,7 @@ def test_series_reindex_like(scalars_df_index, scalars_pandas_df_index): # Pandas uses int64 instead of Int64 (nullable) dtype. pd_result.index = pd_result.index.astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3200,7 +3184,7 @@ def test_where_with_series(scalars_df_index, scalars_pandas_df_index): scalars_pandas_df_index["bool_col"], scalars_pandas_df_index["int64_too"] ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3225,7 +3209,7 @@ def test_where_with_different_indices(scalars_df_index, scalars_pandas_df_index) ) ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3239,7 +3223,7 @@ def test_where_with_default(scalars_df_index, scalars_pandas_df_index): scalars_pandas_df_index["bool_col"] ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3259,7 +3243,7 @@ def _is_positive(x): cond=_is_positive, other=lambda x: x * 10 ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3308,7 +3292,7 @@ def test_clip_filtered_two_sided(scalars_df_index, scalars_pandas_df_index): upper_pd = scalars_pandas_df_index["int64_too"].iloc[:5] + 1 pd_result = col_pd.clip(lower_pd, upper_pd) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3323,7 +3307,7 @@ def test_clip_filtered_one_sided(scalars_df_index, scalars_pandas_df_index): lower_pd = scalars_pandas_df_index["int64_too"].iloc[2:] - 1 pd_result = col_pd.clip(lower_pd, None) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3353,7 +3337,7 @@ def test_between(scalars_df_index, scalars_pandas_df_index, left, right, inclusi ) pd_result = scalars_pandas_df_index["int64_col"].between(left, right, inclusive) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result.astype(pd.BooleanDtype()), ) @@ -3391,7 +3375,7 @@ def test_series_case_when(scalars_dfs_maybe_ordered): bf_result = bf_series.case_when(bf_conditions).to_pandas() pd_result = pd_series.case_when(pd_conditions) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result.astype(pd.Int64Dtype()), ) @@ -3427,7 +3411,7 @@ def test_series_case_when_change_type(scalars_dfs_maybe_ordered): bf_result = bf_series.case_when(bf_conditions).to_pandas() pd_result = pd_series.case_when(pd_conditions) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result.astype("string[pyarrow]"), ) @@ -3456,7 +3440,7 @@ def test_to_json(gcs_folder, scalars_df_index, scalars_pandas_df_index): scalars_df_index["int64_col"].to_json(path, lines=True, orient="records") gcs_df = pd.read_json(get_first_file_from_wildcard(path), lines=True) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( gcs_df["int64_col"].astype(pd.Int64Dtype()), scalars_pandas_df_index["int64_col"], check_dtype=False, @@ -3469,7 +3453,7 @@ def test_to_csv(gcs_folder, scalars_df_index, scalars_pandas_df_index): scalars_df_index["int64_col"].to_csv(path) gcs_df = pd.read_csv(get_first_file_from_wildcard(path)) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( gcs_df["int64_col"].astype(pd.Int64Dtype()), scalars_pandas_df_index["int64_col"], check_dtype=False, @@ -3494,10 +3478,7 @@ def test_series_to_json_local_str(scalars_df_index, scalars_pandas_df_index): def test_series_to_json_local_file(scalars_df_index, scalars_pandas_df_index): # TODO: supply a reason why this isn't compatible with pandas 1.x pytest.importorskip("pandas", minversion="2.0.0") - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.int64_col.to_json(bf_result_file) scalars_pandas_df_index.int64_col.to_json(pd_result_file) @@ -3516,10 +3497,7 @@ def test_series_to_csv_local_str(scalars_df_index, scalars_pandas_df_index): def test_series_to_csv_local_file(scalars_df_index, scalars_pandas_df_index): - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.int64_col.to_csv(bf_result_file) scalars_pandas_df_index.int64_col.to_csv(pd_result_file) @@ -3604,7 +3582,7 @@ def test_series_values(scalars_df_index, scalars_pandas_df_index): pd_result = scalars_pandas_df_index["int64_too"].values # Numpy isn't equipped to compare non-numeric objects, so convert back to dataframe - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( pd.Series(bf_result), pd.Series(pd_result), check_dtype=False ) @@ -3637,7 +3615,7 @@ def test_sort_values(scalars_df_index, scalars_pandas_df_index, ascending, na_po ascending=ascending, na_position=na_position ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3650,7 +3628,7 @@ def test_series_sort_values_inplace(scalars_df_index, scalars_pandas_df_index): bf_result = bf_series.to_pandas() pd_result = scalars_pandas_df_index["int64_col"].sort_values(ascending=False) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3669,7 +3647,7 @@ def test_sort_index(scalars_df_index, scalars_pandas_df_index, ascending): ) pd_result = scalars_pandas_df_index["int64_too"].sort_index(ascending=ascending) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3681,7 +3659,7 @@ def test_series_sort_index_inplace(scalars_df_index, scalars_pandas_df_index): bf_result = bf_series.to_pandas() pd_result = scalars_pandas_df_index["int64_too"].sort_index(ascending=False) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3733,7 +3711,7 @@ def _ten_times(x): cond=lambda x: x > 0, other=_ten_times ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -3842,7 +3820,7 @@ def test_astype(scalars_df_index, scalars_pandas_df_index, column, to_type, erro pytest.importorskip("pandas", minversion="2.0.0") bf_result = scalars_df_index[column].astype(to_type, errors=errors).to_pandas() pd_result = scalars_pandas_df_index[column].astype(to_type) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_astype_python(session): @@ -3853,7 +3831,7 @@ def test_series_astype_python(session): index=pd.Index([0, 1, 2, 3], dtype="Int64"), ) result = session.read_pandas(input).astype(float, errors="null").to_pandas() - bigframes.testing.utils.assert_series_equal(result, exepcted) + pd.testing.assert_series_equal(result, exepcted) def test_astype_safe(session): @@ -3864,7 +3842,7 @@ def test_astype_safe(session): index=pd.Index([0, 1, 2, 3], dtype="Int64"), ) result = session.read_pandas(input).astype("Float64", errors="null").to_pandas() - bigframes.testing.utils.assert_series_equal(result, exepcted) + pd.testing.assert_series_equal(result, exepcted) def test_series_astype_w_invalid_error(session): @@ -3885,7 +3863,7 @@ def test_astype_numeric_to_int(scalars_df_index, scalars_pandas_df_index): .apply(lambda x: None if pd.isna(x) else math.trunc(x)) .astype(to_type) ) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) @pytest.mark.parametrize( @@ -3903,38 +3881,34 @@ def test_date_time_astype_int( pytest.importorskip("pandas", minversion="2.0.0") bf_result = scalars_df_index[column].astype(to_type).to_pandas() pd_result = scalars_pandas_df_index[column].astype(to_type) - bigframes.testing.utils.assert_series_equal(bf_result, pd_result, check_dtype=False) + pd.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) assert bf_result.dtype == "Int64" -def test_string_astype_int(session): - pd_series = pd.Series(["4", "-7", "0", "-03"]) - bf_series = series.Series(pd_series, session=session) +def test_string_astype_int(): + pd_series = pd.Series(["4", "-7", "0", " -03"]) + bf_series = series.Series(pd_series) pd_result = pd_series.astype("Int64") bf_result = bf_series.astype("Int64").to_pandas() - bigframes.testing.utils.assert_series_equal( - bf_result, pd_result, check_index_type=False - ) + pd.testing.assert_series_equal(bf_result, pd_result, check_index_type=False) -def test_string_astype_float(session): +def test_string_astype_float(): pd_series = pd.Series( - ["1", "-1", "-0", "000", "-03.235", "naN", "-inf", "INf", ".33", "7.235e-8"] + ["1", "-1", "-0", "000", " -03.235", "naN", "-inf", "INf", ".33", "7.235e-8"] ) - bf_series = series.Series(pd_series, session=session) + bf_series = series.Series(pd_series) pd_result = pd_series.astype("Float64") bf_result = bf_series.astype("Float64").to_pandas() - bigframes.testing.utils.assert_series_equal( - bf_result, pd_result, check_index_type=False - ) + pd.testing.assert_series_equal(bf_result, pd_result, check_index_type=False) -def test_string_astype_date(session): +def test_string_astype_date(): if int(pa.__version__.split(".")[0]) < 15: pytest.skip( "Avoid pyarrow.lib.ArrowNotImplementedError: " @@ -3945,33 +3919,29 @@ def test_string_astype_date(session): pd.ArrowDtype(pa.string()) ) - bf_series = series.Series(pd_series, session=session) + bf_series = series.Series(pd_series) # TODO(b/340885567): fix type error pd_result = pd_series.astype("date32[day][pyarrow]") # type: ignore bf_result = bf_series.astype("date32[day][pyarrow]").to_pandas() - bigframes.testing.utils.assert_series_equal( - bf_result, pd_result, check_index_type=False - ) + pd.testing.assert_series_equal(bf_result, pd_result, check_index_type=False) -def test_string_astype_datetime(session): +def test_string_astype_datetime(): pd_series = pd.Series( ["2014-08-15 08:15:12", "2015-08-15 08:15:12.654754", "2016-02-29 00:00:00"] ).astype(pd.ArrowDtype(pa.string())) - bf_series = series.Series(pd_series, session=session) + bf_series = series.Series(pd_series) pd_result = pd_series.astype(pd.ArrowDtype(pa.timestamp("us"))) bf_result = bf_series.astype(pd.ArrowDtype(pa.timestamp("us"))).to_pandas() - bigframes.testing.utils.assert_series_equal( - bf_result, pd_result, check_index_type=False - ) + pd.testing.assert_series_equal(bf_result, pd_result, check_index_type=False) -def test_string_astype_timestamp(session): +def test_string_astype_timestamp(): pd_series = pd.Series( [ "2014-08-15 08:15:12+00:00", @@ -3980,26 +3950,23 @@ def test_string_astype_timestamp(session): ] ).astype(pd.ArrowDtype(pa.string())) - bf_series = series.Series(pd_series, session=session) + bf_series = series.Series(pd_series) pd_result = pd_series.astype(pd.ArrowDtype(pa.timestamp("us", tz="UTC"))) bf_result = bf_series.astype( pd.ArrowDtype(pa.timestamp("us", tz="UTC")) ).to_pandas() - bigframes.testing.utils.assert_series_equal( - bf_result, pd_result, check_index_type=False - ) + pd.testing.assert_series_equal(bf_result, pd_result, check_index_type=False) -def test_timestamp_astype_string(session): +def test_timestamp_astype_string(): bf_series = series.Series( [ "2014-08-15 08:15:12+00:00", "2015-08-15 08:15:12.654754+05:00", "2016-02-29 00:00:00+08:00", - ], - session=session, + ] ).astype(pd.ArrowDtype(pa.timestamp("us", tz="UTC"))) expected_result = pd.Series( @@ -4011,71 +3978,68 @@ def test_timestamp_astype_string(session): ) bf_result = bf_series.astype(pa.string()).to_pandas() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, expected_result, check_index_type=False, check_dtype=False ) assert bf_result.dtype == "string[pyarrow]" @pytest.mark.parametrize("errors", ["raise", "null"]) -def test_float_astype_json(errors, session): - data = ["1.25", "2500000000.1", None, "-12323.24"] - bf_series = series.Series(data, dtype=dtypes.FLOAT_DTYPE, session=session) +def test_float_astype_json(errors): + data = ["1.25", "2500000000", None, "-12323.24"] + bf_series = series.Series(data, dtype=dtypes.FLOAT_DTYPE) bf_result = bf_series.astype(dtypes.JSON_DTYPE, errors=errors) assert bf_result.dtype == dtypes.JSON_DTYPE - bf_result_pandas = bf_result.to_pandas() - expected_data = [float(x) if x is not None else None for x in data] - expected_result = pd.Series(expected_data, dtype=dtypes.JSON_DTYPE) + expected_result = pd.Series(data, dtype=dtypes.JSON_DTYPE) expected_result.index = expected_result.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result_pandas, expected_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), expected_result) -def test_float_astype_json_str(session): - data = ["1.25", "2500000000.1", None, "-12323.24"] - bf_series = series.Series(data, dtype=dtypes.FLOAT_DTYPE, session=session) +def test_float_astype_json_str(): + data = ["1.25", "2500000000", None, "-12323.24"] + bf_series = series.Series(data, dtype=dtypes.FLOAT_DTYPE) bf_result = bf_series.astype("json") assert bf_result.dtype == dtypes.JSON_DTYPE - expected_data = [float(x) if x is not None else None for x in data] - expected_result = pd.Series(expected_data, dtype=dtypes.JSON_DTYPE) + expected_result = pd.Series(data, dtype=dtypes.JSON_DTYPE) expected_result.index = expected_result.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), expected_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), expected_result) @pytest.mark.parametrize("errors", ["raise", "null"]) -def test_string_astype_json(errors, session): +def test_string_astype_json(errors): data = [ "1", None, '["1","3","5"]', '{"a":1,"b":["x","y"],"c":{"x":[],"z":false}}', ] - bf_series = series.Series(data, dtype=dtypes.STRING_DTYPE, session=session) + bf_series = series.Series(data, dtype=dtypes.STRING_DTYPE) bf_result = bf_series.astype(dtypes.JSON_DTYPE, errors=errors) assert bf_result.dtype == dtypes.JSON_DTYPE pd_result = bf_series.to_pandas().astype(dtypes.JSON_DTYPE) - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), pd_result) + pd.testing.assert_series_equal(bf_result.to_pandas(), pd_result) -def test_string_astype_json_in_safe_mode(session): +def test_string_astype_json_in_safe_mode(): data = ["this is not a valid json string"] - bf_series = series.Series(data, dtype=dtypes.STRING_DTYPE, session=session) + bf_series = series.Series(data, dtype=dtypes.STRING_DTYPE) bf_result = bf_series.astype(dtypes.JSON_DTYPE, errors="null") assert bf_result.dtype == dtypes.JSON_DTYPE expected = pd.Series([None], dtype=dtypes.JSON_DTYPE) expected.index = expected.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), expected) + pd.testing.assert_series_equal(bf_result.to_pandas(), expected) -def test_string_astype_json_raise_error(session): +def test_string_astype_json_raise_error(): data = ["this is not a valid json string"] - bf_series = series.Series(data, dtype=dtypes.STRING_DTYPE, session=session) + bf_series = series.Series(data, dtype=dtypes.STRING_DTYPE) with pytest.raises( google.api_core.exceptions.BadRequest, match="syntax error while parsing value", @@ -4099,8 +4063,8 @@ def test_string_astype_json_raise_error(session): ), ], ) -def test_json_astype_others(data, to_type, errors, session): - bf_series = series.Series(data, dtype=dtypes.JSON_DTYPE, session=session) +def test_json_astype_others(data, to_type, errors): + bf_series = series.Series(data, dtype=dtypes.JSON_DTYPE) bf_result = bf_series.astype(to_type, errors=errors) assert bf_result.dtype == to_type @@ -4108,7 +4072,7 @@ def test_json_astype_others(data, to_type, errors, session): load_data = [json.loads(item) if item is not None else None for item in data] expected = pd.Series(load_data, dtype=to_type) expected.index = expected.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), expected) + pd.testing.assert_series_equal(bf_result.to_pandas(), expected) @pytest.mark.parametrize( @@ -4120,8 +4084,8 @@ def test_json_astype_others(data, to_type, errors, session): pytest.param(["true", None], dtypes.STRING_DTYPE, id="to_string"), ], ) -def test_json_astype_others_raise_error(data, to_type, session): - bf_series = series.Series(data, dtype=dtypes.JSON_DTYPE, session=session) +def test_json_astype_others_raise_error(data, to_type): + bf_series = series.Series(data, dtype=dtypes.JSON_DTYPE) with pytest.raises(google.api_core.exceptions.BadRequest): bf_series.astype(to_type, errors="raise").to_pandas() @@ -4135,14 +4099,14 @@ def test_json_astype_others_raise_error(data, to_type, session): pytest.param(["true", None], dtypes.STRING_DTYPE, id="to_string"), ], ) -def test_json_astype_others_in_safe_mode(data, to_type, session): - bf_series = series.Series(data, dtype=dtypes.JSON_DTYPE, session=session) +def test_json_astype_others_in_safe_mode(data, to_type): + bf_series = series.Series(data, dtype=dtypes.JSON_DTYPE) bf_result = bf_series.astype(to_type, errors="null") assert bf_result.dtype == to_type expected = pd.Series([None, None], dtype=to_type) expected.index = expected.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(bf_result.to_pandas(), expected) + pd.testing.assert_series_equal(bf_result.to_pandas(), expected) @pytest.mark.parametrize( @@ -4165,7 +4129,7 @@ def test_loc_bool_series_explicit_index(scalars_df_index, scalars_pandas_df_inde bf_result = scalars_df_index.string_col.loc[scalars_df_index.bool_col].to_pandas() pd_result = scalars_pandas_df_index.string_col.loc[scalars_pandas_df_index.bool_col] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result, pd_result, ) @@ -4226,7 +4190,7 @@ def test_rename(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.string_col.rename("newname") pd_result = scalars_pandas_df_index.string_col.rename("newname") - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4236,7 +4200,7 @@ def test_rename_nonstring(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.string_col.rename((4, 2)) pd_result = scalars_pandas_df_index.string_col.rename((4, 2)) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4248,7 +4212,7 @@ def test_rename_dict_same_type(scalars_df_index, scalars_pandas_df_index): pd_result.index = pd_result.index.astype("Int64") - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4258,7 +4222,7 @@ def test_rename_axis(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.string_col.rename_axis("newindexname") pd_result = scalars_pandas_df_index.string_col.rename_axis("newindexname") - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4275,7 +4239,7 @@ def test_loc_list_string_index(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.string_col.loc[index_list] pd_result = scalars_pandas_df_index.string_col.loc[index_list] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4287,7 +4251,7 @@ def test_loc_list_integer_index(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.bool_col.loc[index_list] pd_result = scalars_pandas_df_index.bool_col.loc[index_list] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4303,7 +4267,7 @@ def test_loc_list_multiindex(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_multiindex.int64_too.loc[index_list] pd_result = scalars_pandas_df_multiindex.int64_too.loc[index_list] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4315,7 +4279,7 @@ def test_iloc_list(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.string_col.iloc[index_list] pd_result = scalars_pandas_df_index.string_col.iloc[index_list] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4329,7 +4293,7 @@ def test_iloc_list_nameless(scalars_df_index, scalars_pandas_df_index): pd_series = scalars_pandas_df_index.string_col.rename(None) pd_result = pd_series.iloc[index_list] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4344,7 +4308,7 @@ def test_loc_list_nameless(scalars_df_index, scalars_pandas_df_index): pd_series = scalars_pandas_df_index.string_col.rename(None) pd_result = pd_series.loc[index_list] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4360,7 +4324,7 @@ def test_loc_bf_series_string_index(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.date_col.loc[bf_string_series] pd_result = scalars_pandas_df_index.date_col.loc[pd_string_series] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4378,7 +4342,7 @@ def test_loc_bf_series_multiindex(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_multiindex.int64_too.loc[bf_string_series] pd_result = scalars_pandas_df_multiindex.int64_too.loc[pd_string_series] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4391,7 +4355,7 @@ def test_loc_bf_index_integer_index(scalars_df_index, scalars_pandas_df_index): bf_result = scalars_df_index.date_col.loc[bf_index] pd_result = scalars_pandas_df_index.date_col.loc[pd_index] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4405,7 +4369,7 @@ def test_loc_single_index_with_duplicate(scalars_df_index, scalars_pandas_df_ind index = "Hello, World!" bf_result = scalars_df_index.date_col.loc[index] pd_result = scalars_pandas_df_index.date_col.loc[index] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4450,8 +4414,8 @@ def test_query_job_setters(scalars_dfs): ([1, 1, 1, 1, 1],), ], ) -def test_is_monotonic_increasing(series_input, session): - scalars_df = series.Series(series_input, dtype=pd.Int64Dtype(), session=session) +def test_is_monotonic_increasing(series_input): + scalars_df = series.Series(series_input, dtype=pd.Int64Dtype()) scalars_pandas_df = pd.Series(series_input, dtype=pd.Int64Dtype()) assert ( scalars_df.is_monotonic_increasing == scalars_pandas_df.is_monotonic_increasing @@ -4469,8 +4433,8 @@ def test_is_monotonic_increasing(series_input, session): ([1, 1, 1, 1, 1],), ], ) -def test_is_monotonic_decreasing(series_input, session): - scalars_df = series.Series(series_input, session=session) +def test_is_monotonic_decreasing(series_input): + scalars_df = series.Series(series_input) scalars_pandas_df = pd.Series(series_input) assert ( scalars_df.is_monotonic_decreasing == scalars_pandas_df.is_monotonic_decreasing @@ -4490,7 +4454,7 @@ def test_map_dict_input(scalars_dfs): pd_result = pd_result.astype("Int64") # pandas type differences bf_result = scalars_df.string_col.map(local_map) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4509,7 +4473,7 @@ def test_map_series_input(scalars_dfs): pd_result = scalars_pandas_df.int64_too.map(pd_map_series) bf_result = scalars_df.int64_too.map(bf_map_series) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_result.to_pandas(), pd_result, ) @@ -4596,7 +4560,7 @@ def test_apply_lambda(scalars_dfs, col, lambda_): bf_result = bf_col.apply(lambda_, by_row=False).to_pandas() pd_col = scalars_pandas_df[col] - if pd.__version__[:3] in ("2.2", "2.3", "3.0"): + if pd.__version__[:3] in ("2.2", "2.3"): pd_result = pd_col.apply(lambda_, by_row=False) else: pd_result = pd_col.apply(lambda_) @@ -4689,7 +4653,7 @@ def foo(x): pd_col = scalars_pandas_df["int64_col"] - if pd.__version__[:3] in ("2.2", "2.3", "3.0"): + if pd.__version__[:3] in ("2.2", "2.3"): pd_result = pd_col.apply(foo, by_row=False) else: pd_result = pd_col.apply(foo) @@ -4739,7 +4703,9 @@ def foo(x: int, y: int, df): ) pd_result = ( - scalars_pandas_df_index[column].pipe((foo, "df"), x=7, y=9).pipe(lambda x: x**2) + scalars_pandas_df_index[column] + .pipe((foo, "df"), x=7, y=9) + .pipe(lambda x: x**2) ) assert_series_equal(bf_result, pd_result) @@ -4768,7 +4734,7 @@ def foo(x: int, y: int, df): def test_series_explode(data): s = bigframes.pandas.Series(data) pd_s = s.to_pandas() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( s.explode().to_pandas(), pd_s.explode(), check_index_type=False, @@ -4814,7 +4780,7 @@ def test_series_explode_w_index(index, ignore_index): s = bigframes.pandas.Series(data, index=index) pd_s = pd.Series(data, index=index) # TODO(b/340885567): fix type error - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( s.explode(ignore_index=ignore_index).to_pandas(), # type: ignore pd_s.explode(ignore_index=ignore_index).astype(pd.Float64Dtype()), # type: ignore check_index_type=False, @@ -4839,7 +4805,7 @@ def test_series_explode_reserve_order(ignore_index, ordered): # TODO(b/340885567): fix type error pd_res = pd_s.explode(ignore_index=ignore_index).astype(pd.Int64Dtype()) # type: ignore pd_res.index = pd_res.index.astype(pd.Int64Dtype()) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( res if ordered else res.sort_index(), pd_res, ) @@ -4861,7 +4827,7 @@ def test_series_construct_empty_array(): dtype=pd.ArrowDtype(pa.list_(pa.float64())), index=pd.Index([0], dtype=pd.Int64Dtype()), ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( expected, s.to_pandas(), ) @@ -4878,7 +4844,7 @@ def test_series_construct_empty_array(): ) def test_series_explode_null(data): s = bigframes.pandas.Series(data) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( s.explode().to_pandas(), s.to_pandas().explode(), check_dtype=False, @@ -4903,9 +4869,7 @@ def test_resample(scalars_df_index, scalars_pandas_df_index, append, level, col, ] bf_result = scalars_df_index.resample(rule=rule, level=level).min().to_pandas() pd_result = scalars_pandas_df_index.resample(rule=rule, level=level).min() - # TODO: (b/484364312) - pd_result.index.names = bf_result.index.names - bigframes.testing.utils.assert_series_equal(bf_result, pd_result) + pd.testing.assert_series_equal(bf_result, pd_result) def test_series_struct_get_field_by_attribute( @@ -4917,13 +4881,13 @@ def test_series_struct_get_field_by_attribute( bf_series = nested_structs_df["person"] df_series = nested_structs_pandas_df["person"] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_series.address.city.to_pandas(), df_series.struct.field("address").struct.field("city"), check_dtype=False, check_index=False, ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( bf_series.address.country.to_pandas(), df_series.struct.field("address").struct.field("country"), check_dtype=False, @@ -4992,9 +4956,3 @@ def test_series_item_with_empty(session): with pytest.raises(ValueError, match=re.escape(expected_message)): bf_s_empty.item() - - -def test_series_sql(session): - s = bigframes.pandas.Series([], session=session) - - assert len(s.sql) > 0 diff --git a/tests/system/small/test_series_io.py b/tests/system/small/test_series_io.py index 83c2de70cae..426679d37d0 100644 --- a/tests/system/small/test_series_io.py +++ b/tests/system/small/test_series_io.py @@ -22,6 +22,7 @@ def test_to_pandas_override_global_option(scalars_df_index): with bigframes.option_context("compute.allow_large_results", True): + bf_series = scalars_df_index["int64_col"] # Direct call to_pandas uses global default setting (allow_large_results=True) @@ -30,23 +31,13 @@ def test_to_pandas_override_global_option(scalars_df_index): assert table_id is not None session = bf_series._block.session - - history_before = session.execution_history().to_dataframe() - queries_before = ( - len(history_before[history_before["job_type"] == "query"]) - if "job_type" in history_before.columns - else 0 - ) + execution_count = session._metrics.execution_count # When allow_large_results=False, a query_job object should not be created. # Therefore, the table_id should remain unchanged. bf_series.to_pandas(allow_large_results=False) assert bf_series._query_job.destination.table_id == table_id - - history_after = session.execution_history().to_dataframe() - queries_after = len(history_after[history_after["job_type"] == "query"]) - - assert (queries_after - queries_before) == 1 + assert session._metrics.execution_count - execution_count == 1 @pytest.mark.parametrize( diff --git a/tests/system/small/test_session.py b/tests/system/small/test_session.py index 76788da8a11..698f531d57b 100644 --- a/tests/system/small/test_session.py +++ b/tests/system/small/test_session.py @@ -19,8 +19,8 @@ import textwrap import time import typing -import warnings from typing import List, Optional, Sequence +import warnings import bigframes_vendored.pandas.io.gbq as vendored_pandas_gbq import db_dtypes # type:ignore @@ -37,7 +37,6 @@ import bigframes.dtypes import bigframes.ml.linear_model import bigframes.session.execution_spec -import bigframes.testing from bigframes.testing import utils all_write_engines = pytest.mark.parametrize( @@ -47,8 +46,7 @@ "bigquery_inline", "bigquery_load", "bigquery_streaming", - # TODO(b/502298527): Reenable bigquery_write test - # "bigquery_write", + "bigquery_write", ], ) @@ -114,10 +112,11 @@ def test_read_gbq_tokyo( df.sort_index(inplace=True) expected = scalars_pandas_df_index + # use_explicit_destination=True, otherwise might use path with no query_job exec_result = session_tokyo._executor.execute( df._block.expr, bigframes.session.execution_spec.ExecutionSpec( - destination_spec=bigframes.session.execution_spec.EphemeralTableSpec() + bigframes.session.execution_spec.CacheSpec(()), promise_under_10gb=False ), ) assert exec_result.query_job is not None @@ -327,7 +326,7 @@ def test_read_gbq_w_anonymous_query_results_table(session: bigframes.Session): df = session.read_gbq(destination, index_col="name") result = df.to_pandas() expected.index = expected.index.astype(result.index.dtype) - bigframes.testing.utils.assert_frame_equal(result, expected, check_dtype=False) + pd.testing.assert_frame_equal(result, expected, check_dtype=False) def test_read_gbq_w_primary_keys_table( @@ -350,10 +349,10 @@ def test_read_gbq_w_primary_keys_table( # Verify that the DataFrame is already sorted by primary keys. sorted_result = result.sort_values(primary_keys) - bigframes.testing.utils.assert_frame_equal(result, sorted_result) + pd.testing.assert_frame_equal(result, sorted_result) # Verify that we're working from a snapshot rather than a copy of the table. - assert "FOR SYSTEM_TIME AS OF" in df.sql + assert "FOR SYSTEM_TIME AS OF TIMESTAMP" in df.sql def test_read_gbq_w_primary_keys_table_and_filters( @@ -389,7 +388,7 @@ def test_read_gbq_w_primary_keys_table_and_filters( # Verify that the DataFrame is already sorted by primary keys. sorted_result = result.sort_values(primary_keys) - bigframes.testing.utils.assert_frame_equal(result, sorted_result) + pd.testing.assert_frame_equal(result, sorted_result) @pytest.mark.parametrize( @@ -534,9 +533,7 @@ def test_read_gbq_w_ambigous_name( .to_pandas() ) pd_df = pd.DataFrame({"x": [2, 1], "ambiguous_name": [20, 10]}) - bigframes.testing.utils.assert_frame_equal( - df, pd_df, check_dtype=False, check_index_type=False - ) + pd.testing.assert_frame_equal(df, pd_df, check_dtype=False, check_index_type=False) def test_read_gbq_table_clustered_with_filter(session: bigframes.Session): @@ -771,10 +768,8 @@ def test_read_gbq_w_json_and_compare_w_pandas_json(session): dtype=pd.ArrowDtype(db_dtypes.JSONArrowType()), ) pd_df.index = pd_df.index.astype("Int64") - bigframes.testing.utils.assert_series_equal(df.dtypes, pd_df.dtypes) - bigframes.testing.utils.assert_series_equal( - df["json_col"].to_pandas(), pd_df["json_col"] - ) + pd.testing.assert_series_equal(df.dtypes, pd_df.dtypes) + pd.testing.assert_series_equal(df["json_col"].to_pandas(), pd_df["json_col"]) def test_read_gbq_w_json_in_struct(session): @@ -872,22 +867,24 @@ def test_read_pandas(session, scalars_dfs): result = df.to_pandas() expected = scalars_pandas_df - bigframes.testing.utils.assert_frame_equal(result, expected) + pd.testing.assert_frame_equal(result, expected) def test_read_pandas_series(session): + idx: pd.Index = pd.Index([2, 7, 1, 2, 8], dtype=pd.Int64Dtype()) pd_series = pd.Series([3, 1, 4, 1, 5], dtype=pd.Int64Dtype(), index=idx) bf_series = session.read_pandas(pd_series) - bigframes.testing.utils.assert_series_equal(bf_series.to_pandas(), pd_series) + pd.testing.assert_series_equal(bf_series.to_pandas(), pd_series) def test_read_pandas_index(session): + pd_idx: pd.Index = pd.Index([2, 7, 1, 2, 8], dtype=pd.Int64Dtype()) bf_idx = session.read_pandas(pd_idx) - bigframes.testing.utils.assert_index_equal(bf_idx.to_pandas(), pd_idx) + pd.testing.assert_index_equal(bf_idx.to_pandas(), pd_idx) def test_read_pandas_w_unsupported_mixed_dtype(session): @@ -917,7 +914,7 @@ def test_read_pandas_col_label_w_space(session: bigframes.Session): ) result = session.read_pandas(expected).to_pandas() - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( result, expected, check_index_type=False, check_dtype=False ) @@ -925,7 +922,7 @@ def test_read_pandas_col_label_w_space(session: bigframes.Session): def test_read_pandas_multi_index(session, scalars_pandas_df_multi_index): df = session.read_pandas(scalars_pandas_df_multi_index) result = df.to_pandas() - bigframes.testing.utils.assert_frame_equal(result, scalars_pandas_df_multi_index) + pd.testing.assert_frame_equal(result, scalars_pandas_df_multi_index) def test_read_pandas_rowid_exists_adds_suffix(session, scalars_pandas_df_default_index): @@ -933,9 +930,7 @@ def test_read_pandas_rowid_exists_adds_suffix(session, scalars_pandas_df_default pandas_df["rowid"] = np.arange(pandas_df.shape[0]) df_roundtrip = session.read_pandas(pandas_df).to_pandas() - bigframes.testing.utils.assert_frame_equal( - df_roundtrip, pandas_df, check_dtype=False - ) + pd.testing.assert_frame_equal(df_roundtrip, pandas_df, check_dtype=False) def test_read_pandas_tokyo( @@ -950,7 +945,7 @@ def test_read_pandas_tokyo( result = session_tokyo._executor.execute( df._block.expr, bigframes.session.execution_spec.ExecutionSpec( - destination_spec=bigframes.session.execution_spec.EphemeralTableSpec() + bigframes.session.execution_spec.CacheSpec(()), promise_under_10gb=False ), ) assert result.query_job is not None @@ -974,14 +969,12 @@ def test_read_pandas_timedelta_dataframes(session, write_engine): expected_result = pandas_df.astype(bigframes.dtypes.TIMEDELTA_DTYPE) expected_result.index = expected_result.index.astype(bigframes.dtypes.INT_DTYPE) - bigframes.testing.utils.assert_frame_equal(actual_result, expected_result) + pd.testing.assert_frame_equal(actual_result, expected_result) @all_write_engines def test_read_pandas_timedelta_series(session, write_engine): - expected_series = pd.Series(pd.to_timedelta([1, 2, 3], unit="d")).astype( - "timedelta64[ns]" - ) + expected_series = pd.Series(pd.to_timedelta([1, 2, 3], unit="d")) actual_result = ( session.read_pandas(expected_series, write_engine=write_engine) @@ -989,15 +982,15 @@ def test_read_pandas_timedelta_series(session, write_engine): .astype("timedelta64[ns]") ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( actual_result, expected_series, check_index_type=False ) @all_write_engines def test_read_pandas_timedelta_index(session, write_engine): - expected_index = pd.to_timedelta([1, 2, 3], unit="d").astype( - "timedelta64[ns]" + expected_index = pd.to_timedelta( + [1, 2, 3], unit="d" ) # to_timedelta returns an index actual_result = ( @@ -1006,7 +999,7 @@ def test_read_pandas_timedelta_index(session, write_engine): .astype("timedelta64[ns]") ) - bigframes.testing.utils.assert_index_equal(actual_result, expected_index) + pd.testing.assert_index_equal(actual_result, expected_index) @all_write_engines @@ -1025,9 +1018,7 @@ def test_read_pandas_json_dataframes(session, write_engine): expected_df, write_engine=write_engine ).to_pandas() - bigframes.testing.utils.assert_frame_equal( - actual_result, expected_df, check_index_type=False - ) + pd.testing.assert_frame_equal(actual_result, expected_df, check_index_type=False) @all_write_engines @@ -1043,7 +1034,7 @@ def test_read_pandas_json_series(session, write_engine): actual_result = session.read_pandas( expected_series, write_engine=write_engine ).to_pandas() - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( actual_result, expected_series, check_index_type=False ) @@ -1071,7 +1062,7 @@ def test_read_pandas_json_index(session, write_engine): actual_result = session.read_pandas( expected_index, write_engine=write_engine ).to_pandas() - bigframes.testing.utils.assert_index_equal(actual_result, expected_index) + pd.testing.assert_index_equal(actual_result, expected_index) @pytest.mark.parametrize( @@ -1096,9 +1087,7 @@ def test_read_pandas_w_nested_json_fails(session, write_engine): pa.list_(pa.struct([("json_field", bigframes.dtypes.JSON_ARROW_TYPE)])) ), ) - with pytest.raises( - NotImplementedError, match="Nested JSON types are currently unsupported" - ): + with pytest.raises(NotImplementedError, match="Nested JSON types, found in column"): session.read_pandas(pd_s, write_engine=write_engine) @@ -1108,8 +1097,7 @@ def test_read_pandas_w_nested_json_fails(session, write_engine): pytest.param("default"), pytest.param("bigquery_inline"), pytest.param("bigquery_streaming"), - # TODO(b/502298527): Reenable bigquery_write test - # pytest.param("bigquery_write"), + pytest.param("bigquery_write"), ], ) def test_read_pandas_w_nested_json(session, write_engine): @@ -1133,7 +1121,7 @@ def test_read_pandas_w_nested_json(session, write_engine): .to_pandas() .reset_index(drop=True) ) - bigframes.testing.utils.assert_series_equal(bq_s, pd_s) + pd.testing.assert_series_equal(bq_s, pd_s) @pytest.mark.parametrize( @@ -1185,9 +1173,7 @@ def test_read_pandas_w_nested_json_index_fails(session, write_engine): pa.list_(pa.struct([("json_field", bigframes.dtypes.JSON_ARROW_TYPE)])) ), ) - with pytest.raises( - NotImplementedError, match="Nested JSON types are currently unsupported" - ): + with pytest.raises(NotImplementedError, match="Nested JSON types, found in"): session.read_pandas(pd_idx, write_engine=write_engine) @@ -1197,8 +1183,7 @@ def test_read_pandas_w_nested_json_index_fails(session, write_engine): pytest.param("default"), pytest.param("bigquery_inline"), pytest.param("bigquery_streaming"), - # TODO(b/502298527): Reenable bigquery_write test - # pytest.param("bigquery_write"), + pytest.param("bigquery_write"), ], ) def test_read_pandas_w_nested_json_index(session, write_engine): @@ -1218,7 +1203,7 @@ def test_read_pandas_w_nested_json_index(session, write_engine): ), ) bq_idx = session.read_pandas(pd_idx, write_engine=write_engine).to_pandas() - bigframes.testing.utils.assert_index_equal(bq_idx, pd_idx) + pd.testing.assert_index_equal(bq_idx, pd_idx) @all_write_engines @@ -1232,15 +1217,13 @@ def test_read_csv_for_gcs_file_w_write_engine(session, df_and_gcs_csv, write_eng write_engine=write_engine, dtype=scalars_df.dtypes.to_dict(), ) - bigframes.testing.utils.assert_frame_equal( - pd_df.to_pandas(), scalars_df.to_pandas() - ) + pd.testing.assert_frame_equal(pd_df.to_pandas(), scalars_df.to_pandas()) if write_engine in ("default", "bigquery_load"): bf_df = session.read_csv( path, engine="bigquery", index_col="rowindex", write_engine=write_engine ) - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) @pytest.mark.parametrize( @@ -1268,10 +1251,8 @@ def test_read_csv_for_local_file_w_sep(session, df_and_local_csv, sep): pd_df = session.read_csv( buffer, index_col="rowindex", sep=sep, dtype=scalars_df.dtypes.to_dict() ) - bigframes.testing.utils.assert_frame_equal( - bf_df.to_pandas(), scalars_df.to_pandas() - ) - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), scalars_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) @pytest.mark.parametrize( @@ -1303,7 +1284,7 @@ def test_read_csv_for_index_col_w_false(session, df_and_local_csv, index_col): # (b/280889935) or guarantee row ordering. bf_df = bf_df.set_index("rowindex").sort_index() pd_df = pd_df.set_index("rowindex") - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) @pytest.mark.parametrize( @@ -1326,7 +1307,7 @@ def test_read_csv_for_index_col(session, df_and_gcs_csv, index_col): ) assert bf_df.shape == pd_df.shape - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) @pytest.mark.parametrize( @@ -1379,7 +1360,7 @@ def test_read_csv_for_gcs_wildcard_path(session, df_and_gcs_csv): assert bf_df.shape == pd_df.shape assert bf_df.columns.tolist() == pd_df.columns.tolist() - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) def test_read_csv_for_names(session, df_and_gcs_csv_for_two_columns): @@ -1398,7 +1379,7 @@ def test_read_csv_for_names(session, df_and_gcs_csv_for_two_columns): # (b/280889935) or guarantee row ordering. bf_df = bf_df.set_index(names[0]).sort_index() pd_df = pd_df.set_index(names[0]) - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) def test_read_csv_for_names_more_than_columns_can_raise_error( @@ -1427,7 +1408,7 @@ def test_read_csv_for_names_less_than_columns(session, df_and_gcs_csv_for_two_co # Pandas's index name is None, while BigFrames's index name is "rowindex". pd_df.index.name = "rowindex" - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) def test_read_csv_for_names_less_than_columns_raise_error_when_index_col_set( @@ -1465,7 +1446,7 @@ def test_read_csv_for_names_and_index_col( assert bf_df.shape == pd_df.shape assert bf_df.columns.tolist() == pd_df.columns.tolist() - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_df.to_pandas(), pd_df.to_pandas(), check_index_type=False ) @@ -1497,7 +1478,7 @@ def test_read_csv_for_names_and_usecols( # (b/280889935) or guarantee row ordering. bf_df = bf_df.set_index(names[0]).sort_index() pd_df = pd_df.set_index(names[0]) - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) def test_read_csv_for_names_and_invalid_usecols( @@ -1544,7 +1525,7 @@ def test_read_csv_for_names_and_usecols_and_indexcol( assert bf_df.shape == pd_df.shape assert bf_df.columns.tolist() == pd_df.columns.tolist() - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) def test_read_csv_for_names_less_than_columns_and_same_usecols( @@ -1567,7 +1548,7 @@ def test_read_csv_for_names_less_than_columns_and_same_usecols( # (b/280889935) or guarantee row ordering. bf_df = bf_df.set_index(names[0]).sort_index() pd_df = pd_df.set_index(names[0]) - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) def test_read_csv_for_names_less_than_columns_and_mismatched_usecols( @@ -1612,7 +1593,7 @@ def test_read_csv_for_dtype(session, df_and_gcs_csv_for_two_columns): # (b/280889935) or guarantee row ordering. bf_df = bf_df.set_index("rowindex").sort_index() pd_df = pd_df.set_index("rowindex") - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) def test_read_csv_for_dtype_w_names(session, df_and_gcs_csv_for_two_columns): @@ -1632,7 +1613,7 @@ def test_read_csv_for_dtype_w_names(session, df_and_gcs_csv_for_two_columns): # (b/280889935) or guarantee row ordering. bf_df = bf_df.set_index("a").sort_index() pd_df = pd_df.set_index("a") - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) @pytest.mark.parametrize( @@ -1699,10 +1680,8 @@ def test_read_csv_for_gcs_file_w_header(session, df_and_gcs_csv, header): # (b/280889935) or guarantee row ordering. bf_df = bf_df.set_index("rowindex").sort_index() pd_df = pd_df.set_index("rowindex") - bigframes.testing.utils.assert_frame_equal( - bf_df.to_pandas(), scalars_df.to_pandas() - ) - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), scalars_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) def test_read_csv_w_usecols(session, df_and_local_csv): @@ -1730,7 +1709,7 @@ def test_read_csv_w_usecols(session, df_and_local_csv): # (b/280889935) or guarantee row ordering. bf_df = bf_df.set_index("rowindex").sort_index() pd_df = pd_df.set_index("rowindex") - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) def test_read_csv_w_usecols_and_indexcol(session, df_and_local_csv): @@ -1756,7 +1735,7 @@ def test_read_csv_w_usecols_and_indexcol(session, df_and_local_csv): assert bf_df.shape == pd_df.shape assert bf_df.columns.tolist() == pd_df.columns.tolist() - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) def test_read_csv_w_indexcol_not_in_usecols(session, df_and_local_csv): @@ -1811,10 +1790,10 @@ def test_read_csv_local_w_encoding(session, penguins_pandas_df_default_index): bf_df = session.read_csv( path, engine="bigquery", index_col="rowindex", encoding="ISO-8859-1" ) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( bf_df.to_pandas(), penguins_pandas_df_default_index ) - bigframes.testing.utils.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) + pd.testing.assert_frame_equal(bf_df.to_pandas(), pd_df.to_pandas()) def test_read_pickle_local(session, penguins_pandas_df_default_index, tmp_path): @@ -1823,9 +1802,7 @@ def test_read_pickle_local(session, penguins_pandas_df_default_index, tmp_path): penguins_pandas_df_default_index.to_pickle(path) df = session.read_pickle(path) - bigframes.testing.utils.assert_frame_equal( - penguins_pandas_df_default_index, df.to_pandas() - ) + pd.testing.assert_frame_equal(penguins_pandas_df_default_index, df.to_pandas()) def test_read_pickle_buffer(session, penguins_pandas_df_default_index): @@ -1834,9 +1811,7 @@ def test_read_pickle_buffer(session, penguins_pandas_df_default_index): buffer.seek(0) df = session.read_pickle(buffer) - bigframes.testing.utils.assert_frame_equal( - penguins_pandas_df_default_index, df.to_pandas() - ) + pd.testing.assert_frame_equal(penguins_pandas_df_default_index, df.to_pandas()) def test_read_pickle_series_buffer(session): @@ -1855,9 +1830,7 @@ def test_read_pickle_gcs(session, penguins_pandas_df_default_index, gcs_folder): penguins_pandas_df_default_index.to_pickle(path) df = session.read_pickle(path) - bigframes.testing.utils.assert_frame_equal( - penguins_pandas_df_default_index, df.to_pandas() - ) + pd.testing.assert_frame_equal(penguins_pandas_df_default_index, df.to_pandas()) @pytest.mark.parametrize( @@ -1913,8 +1886,7 @@ def test_read_parquet_gcs( df_out = ( session.read_parquet(read_path, engine=engine) # Restore order. - .set_index(df_write.index.name) - .sort_index() + .set_index(df_write.index.name).sort_index() # Restore index. .set_index(typing.cast(str, df_in.index.name)) ) @@ -1931,132 +1903,7 @@ def test_read_parquet_gcs( assert df_out.size != 0 pd_df_in = df_in.to_pandas() pd_df_out = df_out.to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_df_in, pd_df_out) - - -@pytest.mark.parametrize( - ("engine", "filename"), - ( - pytest.param( - "bigquery", - "000000000000.orc", - id="bigquery", - ), - pytest.param( - "auto", - "000000000000.orc", - id="auto", - ), - pytest.param( - "pyarrow", - "000000000000.orc", - id="pyarrow", - ), - pytest.param( - "bigquery", - "*.orc", - id="bigquery_wildcard", - ), - pytest.param( - "auto", - "*.orc", - id="auto_wildcard", - marks=pytest.mark.xfail( - raises=ValueError, - ), - ), - ), -) -def test_read_orc_gcs( - session: bigframes.Session, scalars_dfs, gcs_folder, engine, filename -): - pytest.importorskip( - "pandas", - minversion="2.0.0", - reason="pandas<2 does not handle nullable int columns well", - ) - scalars_df, _ = scalars_dfs - write_path = gcs_folder + test_read_orc_gcs.__name__ + "000000000000.orc" - read_path = gcs_folder + test_read_orc_gcs.__name__ + filename - - df_in: bigframes.dataframe.DataFrame = scalars_df.copy() - df_in = df_in.drop( - columns=[ - "geography_col", - "time_col", - "datetime_col", - "duration_col", - "timestamp_col", - ] - ) - df_write = df_in.reset_index(drop=False) - df_write.index.name = f"ordering_id_{random.randrange(1_000_000)}" - df_write.to_orc(write_path) - - df_out = ( - session.read_orc(read_path, engine=engine) - .set_index(df_write.index.name) - .sort_index() - .set_index(typing.cast(str, df_in.index.name)) - ) - - assert df_out.size != 0 - pd_df_in = df_in.to_pandas() - pd_df_out = df_out.to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_df_in, pd_df_out) - - -@pytest.mark.parametrize( - ("engine", "filename"), - ( - pytest.param( - "bigquery", - "000000000000.avro", - id="bigquery", - ), - pytest.param( - "bigquery", - "*.avro", - id="bigquery_wildcard", - ), - ), -) -def test_read_avro_gcs( - session: bigframes.Session, scalars_dfs, gcs_folder, engine, filename -): - scalars_df, _ = scalars_dfs - write_uri = gcs_folder + test_read_avro_gcs.__name__ + "*.avro" - read_uri = gcs_folder + test_read_avro_gcs.__name__ + filename - - df_in: bigframes.dataframe.DataFrame = scalars_df.copy() - # datetime round-trips back as str in avro - df_in = df_in.drop(columns=["geography_col", "duration_col", "datetime_col"]) - df_write = df_in.reset_index(drop=False) - index_name = f"ordering_id_{random.randrange(1_000_000)}" - df_write.index.name = index_name - - # Create a BigQuery table - table_id = df_write.to_gbq() - - # Extract to GCS as Avro - client = session.bqclient - extract_job_config = bigquery.ExtractJobConfig() - extract_job_config.destination_format = "AVRO" - extract_job_config.use_avro_logical_types = True - - client.extract_table(table_id, write_uri, job_config=extract_job_config).result() - - df_out = ( - session.read_avro(read_uri, engine=engine) - .set_index(index_name) - .sort_index() - .set_index(typing.cast(str, df_in.index.name)) - ) - - assert df_out.size != 0 - pd_df_in = df_in.to_pandas() - pd_df_out = df_out.to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_df_in, pd_df_out) + pd.testing.assert_frame_equal(pd_df_in, pd_df_out) @pytest.mark.parametrize( @@ -2090,8 +1937,7 @@ def test_read_parquet_gcs_compressed( df_out = ( session.read_parquet(path, engine="bigquery") # Restore order. - .set_index(df_write.index.name) - .sort_index() + .set_index(df_write.index.name).sort_index() # Restore index. .set_index(typing.cast(str, df_in.index.name)) ) @@ -2107,7 +1953,7 @@ def test_read_parquet_gcs_compressed( assert df_out.size != 0 pd_df_in = df_in.to_pandas() pd_df_out = df_out.to_pandas() - bigframes.testing.utils.assert_frame_equal(pd_df_in, pd_df_out) + pd.testing.assert_frame_equal(pd_df_in, pd_df_out) @pytest.mark.parametrize( @@ -2152,7 +1998,7 @@ def test_read_json_gcs_bq_engine(session, scalars_dfs, gcs_folder): df = session.read_json(read_path, lines=True, orient="records", engine="bigquery") # The auto detects of BigQuery load job does not preserve any ordering of columns for json. - bigframes.testing.utils.assert_index_equal( + pd.testing.assert_index_equal( df.columns.sort_values(), scalars_df.columns.sort_values() ) @@ -2177,7 +2023,7 @@ def test_read_json_gcs_bq_engine(session, scalars_dfs, gcs_folder): ] ) assert df.shape[0] == scalars_df.shape[0] - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( df.dtypes.sort_index(), scalars_df.dtypes.sort_index() ) @@ -2203,7 +2049,7 @@ def test_read_json_gcs_default_engine(session, scalars_dfs, gcs_folder): orient="records", ) - bigframes.testing.utils.assert_index_equal(df.columns, scalars_df.columns) + pd.testing.assert_index_equal(df.columns, scalars_df.columns) # The auto detects of BigQuery load job have restrictions to detect the bytes, # numeric and geometry types, so they're skipped here. @@ -2217,7 +2063,7 @@ def test_read_json_gcs_default_engine(session, scalars_dfs, gcs_folder): scalars_df = scalars_df.drop(columns=["date_col", "datetime_col", "time_col"]) assert df.shape[0] == scalars_df.shape[0] - bigframes.testing.utils.assert_series_equal(df.dtypes, scalars_df.dtypes) + pd.testing.assert_series_equal(df.dtypes, scalars_df.dtypes) @pytest.mark.parametrize( @@ -2365,7 +2211,7 @@ def _assert_query_dry_run_stats_are_valid(result: pd.Series): ] ) - bigframes.testing.utils.assert_index_equal(result.index, expected_index) + pd.testing.assert_index_equal(result.index, expected_index) assert result["columnCount"] + result["indexLevel"] > 0 @@ -2385,5 +2231,5 @@ def _assert_table_dry_run_stats_are_valid(result: pd.Series): ] ) - bigframes.testing.utils.assert_index_equal(result.index, expected_index) + pd.testing.assert_index_equal(result.index, expected_index) assert result["columnCount"] == len(result["columnDtypes"]) diff --git a/tests/system/small/test_unordered.py b/tests/system/small/test_unordered.py index c8db041fec2..c7ff0ca1dd4 100644 --- a/tests/system/small/test_unordered.py +++ b/tests/system/small/test_unordered.py @@ -261,9 +261,7 @@ def test_resample_with_index(unordered_session, rule, origin, data): assert isinstance(bf_result.index, bpd.DatetimeIndex) assert isinstance(pd_result.index, pd.DatetimeIndex) - # TODO: (b/484364312) - pd_result.index.name = bf_result.index.name - assert_frame_equal( + pd.testing.assert_frame_equal( bf_result.to_pandas(), pd_result, check_index_type=False, @@ -292,4 +290,4 @@ def test_unordered_df_pivot( # Pandas produces NaN, where bq dataframes produces pd.NA bf_result = bf_result.fillna(float("nan")) pd_result = pd_result.fillna(float("nan")) - assert_frame_equal(bf_result, pd_result, check_dtype=False) + pd.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False) diff --git a/tests/system/small/test_window.py b/tests/system/small/test_window.py index a70a676e84d..29ab581f76f 100644 --- a/tests/system/small/test_window.py +++ b/tests/system/small/test_window.py @@ -18,7 +18,6 @@ import pandas as pd import pytest -import bigframes.testing.utils from bigframes import dtypes @@ -62,9 +61,7 @@ def test_dataframe_rolling_closed_param(rows_rolling_dfs, closed): actual_result = bf_df.rolling(window=3, closed=closed).sum().to_pandas() expected_result = pd_df.rolling(window=3, closed=closed).sum() - bigframes.testing.utils.assert_frame_equal( - actual_result, expected_result, check_dtype=False - ) + pd.testing.assert_frame_equal(actual_result, expected_result, check_dtype=False) @pytest.mark.parametrize("closed", ["left", "right", "both", "neither"]) @@ -83,7 +80,7 @@ def test_dataframe_groupby_rolling_closed_param(rows_rolling_dfs, closed): expected_result = ( pd_df.groupby(pd_df["int64_too"] % 2).rolling(window=3, closed=closed).sum() ) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( actual_result[check_columns], expected_result, check_dtype=False ) @@ -94,9 +91,7 @@ def test_dataframe_rolling_on(rows_rolling_dfs): actual_result = bf_df.rolling(window=3, on="int64_too").sum().to_pandas() expected_result = pd_df.rolling(window=3, on="int64_too").sum() - bigframes.testing.utils.assert_frame_equal( - actual_result, expected_result, check_dtype=False - ) + pd.testing.assert_frame_equal(actual_result, expected_result, check_dtype=False) def test_dataframe_rolling_on_invalid_column_raise_error(rows_rolling_dfs): @@ -121,7 +116,7 @@ def test_dataframe_groupby_rolling_on(rows_rolling_dfs): expected_result = ( pd_df.groupby(pd_df["int64_too"] % 2).rolling(window=3, on="float64_col").sum() ) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( actual_result[check_columns], expected_result, check_dtype=False ) @@ -140,9 +135,7 @@ def test_series_rolling_closed_param(rows_rolling_series, closed): actual_result = bf_series.rolling(window=3, closed=closed).sum().to_pandas() expected_result = df_series.rolling(window=3, closed=closed).sum() - bigframes.testing.utils.assert_series_equal( - actual_result, expected_result, check_dtype=False - ) + pd.testing.assert_series_equal(actual_result, expected_result, check_dtype=False) @pytest.mark.parametrize("closed", ["left", "right", "both", "neither"]) @@ -159,9 +152,7 @@ def test_series_groupby_rolling_closed_param(rows_rolling_series, closed): expected_result = ( df_series.groupby(df_series % 2).rolling(window=3, closed=closed).sum() ) - bigframes.testing.utils.assert_series_equal( - actual_result, expected_result, check_dtype=False - ) + pd.testing.assert_series_equal(actual_result, expected_result, check_dtype=False) @pytest.mark.parametrize( @@ -195,9 +186,7 @@ def test_series_window_agg_ops(rows_rolling_series, windowing, agg_op): actual_result = agg_op(windowing(bf_series)).to_pandas() expected_result = agg_op(windowing(pd_series)) - bigframes.testing.utils.assert_series_equal( - expected_result, actual_result, check_dtype=False - ) + pd.testing.assert_series_equal(expected_result, actual_result, check_dtype=False) @pytest.mark.parametrize( @@ -236,7 +225,7 @@ def test_dataframe_window_agg_ops(scalars_dfs, windowing, agg_op): bf_result = agg_op(windowing(bf_df)).to_pandas() pd_result = agg_op(windowing(pd_df)) - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_frame_equal(pd_result, bf_result, check_dtype=False) @pytest.mark.parametrize( @@ -269,10 +258,6 @@ def test_dataframe_window_agg_ops(scalars_dfs, windowing, agg_op): ], ) def test_dataframe_window_agg_func(scalars_dfs, windowing, func): - if pd.__version__.startswith("3"): - pytest.skip( - "pandas 3.0 bugged for this case 'Length of values (8) does not match length of index (9)'" - ) bf_df, pd_df = scalars_dfs target_columns = ["int64_too", "float64_col", "bool_col", "int64_col"] index_column = "bool_col" @@ -283,7 +268,7 @@ def test_dataframe_window_agg_func(scalars_dfs, windowing, func): pd_result = windowing(pd_df).agg(func) - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_frame_equal(pd_result, bf_result, check_dtype=False) def test_series_window_agg_single_func(scalars_dfs): @@ -296,7 +281,7 @@ def test_series_window_agg_single_func(scalars_dfs): pd_result = pd_series.expanding().agg("sum") - bigframes.testing.utils.assert_series_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_series_equal(pd_result, bf_result, check_dtype=False) def test_series_window_agg_multi_func(scalars_dfs): @@ -309,7 +294,7 @@ def test_series_window_agg_multi_func(scalars_dfs): pd_result = pd_series.expanding().agg(["sum", np.mean]) - bigframes.testing.utils.assert_frame_equal(pd_result, bf_result, check_dtype=False) + pd.testing.assert_frame_equal(pd_result, bf_result, check_dtype=False) @pytest.mark.parametrize("closed", ["left", "right", "both", "neither"]) @@ -335,7 +320,7 @@ def test_series_range_rolling(range_rolling_dfs, window, closed, ascending): .rolling(window=window, closed=closed) .min() ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( actual_result, expected_result, check_dtype=False, check_index=False ) @@ -356,7 +341,7 @@ def test_series_groupby_range_rolling(range_rolling_dfs): expected_result = ( pd_series.sort_index().groupby(pd_series % 2 == 0).rolling(window="3s").min() ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( actual_result, expected_result, check_dtype=False, check_index=False ) @@ -387,7 +372,7 @@ def test_dataframe_range_rolling(range_rolling_dfs, window, closed, ascending): # Need to cast Pandas index type. Otherwise it uses DatetimeIndex that # does not exist in BigFrame expected_result.index = expected_result.index.astype(dtypes.TIMESTAMP_DTYPE) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( actual_result, expected_result, check_dtype=False, @@ -404,7 +389,7 @@ def test_dataframe_range_rolling_on(range_rolling_dfs): # Need to specify the column order because Pandas (seemingly) # re-arranges columns alphabetically cols = ["ts_col", "int_col", "float_col"] - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( actual_result[cols], expected_result[cols], check_dtype=False, @@ -428,7 +413,7 @@ def test_dataframe_groupby_range_rolling(range_rolling_dfs): pd_df.sort_values(on).groupby("int_col").rolling(window="3s", on=on).min() ) expected_result.index = expected_result.index.set_names("index", level=1) - bigframes.testing.utils.assert_frame_equal( + pd.testing.assert_frame_equal( actual_result, expected_result, check_dtype=False, @@ -455,7 +440,7 @@ def test_range_rolling_order_info_lookup(range_rolling_dfs): .rolling(window="3s") .count() ) - bigframes.testing.utils.assert_series_equal( + pd.testing.assert_series_equal( actual_result, expected_result, check_dtype=False, check_index=False ) diff --git a/tests/unit/_config/test_bigquery_options.py b/tests/unit/_config/test_bigquery_options.py index 0c51abfd95c..57486125b78 100644 --- a/tests/unit/_config/test_bigquery_options.py +++ b/tests/unit/_config/test_bigquery_options.py @@ -13,8 +13,8 @@ # limitations under the License. import re -import warnings from unittest import mock +import warnings import google.auth.credentials import pytest diff --git a/tests/unit/_config/test_experiment_options.py b/tests/unit/_config/test_experiment_options.py index 0d66b2156ab..deeee2e46a7 100644 --- a/tests/unit/_config/test_experiment_options.py +++ b/tests/unit/_config/test_experiment_options.py @@ -15,18 +15,34 @@ import pytest import bigframes._config.experiment_options as experiment_options +import bigframes.exceptions as bfe -def test_sql_compiler_default_stable(): +def test_semantic_operators_default_false(): options = experiment_options.ExperimentOptions() - assert options.sql_compiler == "stable" + assert options.semantic_operators is False -def test_sql_compiler_set_experimental_shows_warning(): +def test_semantic_operators_set_true_shows_warning(): options = experiment_options.ExperimentOptions() with pytest.warns(FutureWarning): - options.sql_compiler = "experimental" + options.semantic_operators = True - assert options.sql_compiler == "experimental" + assert options.semantic_operators is True + + +def test_ai_operators_default_false(): + options = experiment_options.ExperimentOptions() + + assert options.ai_operators is False + + +def test_ai_operators_set_true_shows_warning(): + options = experiment_options.ExperimentOptions() + + with pytest.warns(bfe.PreviewWarning): + options.ai_operators = True + + assert options.ai_operators is True diff --git a/tests/unit/bigquery/generated/__init__.py b/tests/unit/bigquery/generated/__init__.py deleted file mode 100644 index 58d482ea386..00000000000 --- a/tests/unit/bigquery/generated/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/unit/bigquery/generated/global_namespace/__init__.py b/tests/unit/bigquery/generated/global_namespace/__init__.py deleted file mode 100644 index 58d482ea386..00000000000 --- a/tests/unit/bigquery/generated/global_namespace/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/unit/bigquery/generated/global_namespace/test_aead_encryption.py b/tests/unit/bigquery/generated/global_namespace/test_aead_encryption.py deleted file mode 100644 index 818151952ff..00000000000 --- a/tests/unit/bigquery/generated/global_namespace/test_aead_encryption.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: scripts/data/sql-functions/global_namespace/aead_encryption.yaml -# by the script: scripts/generate_bigframes_bigquery.py - -import bigframes.bigquery as bbq -import bigframes.core.col -import bigframes.core.expression as ex -import bigframes.operations.googlesql.global_namespace.aead_encryption as aead_encryption_op -import bigframes.pandas as bpd - - -def test_deterministic_decrypt_bytes_expression(): - # Call the function with col() expressions - result = bbq.deterministic_decrypt_bytes( - bpd.col("keyset"), - bpd.col("ciphertext"), - bpd.col("additional_data"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == aead_encryption_op._DETERMINISTIC_DECRYPT_BYTES_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "keyset" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "ciphertext" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "additional_data" - - -def test_deterministic_decrypt_string_expression(): - # Call the function with col() expressions - result = bbq.deterministic_decrypt_string( - bpd.col("keyset"), - bpd.col("ciphertext"), - bpd.col("additional_data"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == aead_encryption_op._DETERMINISTIC_DECRYPT_STRING_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "keyset" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "ciphertext" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "additional_data" - - -def test_deterministic_encrypt_expression(): - # Call the function with col() expressions - result = bbq.deterministic_encrypt( - bpd.col("keyset"), - bpd.col("plaintext"), - bpd.col("additional_data"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == aead_encryption_op._DETERMINISTIC_ENCRYPT_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "keyset" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "plaintext" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "additional_data" diff --git a/tests/unit/bigquery/generated/global_namespace/test_array.py b/tests/unit/bigquery/generated/global_namespace/test_array.py deleted file mode 100644 index 56b85386902..00000000000 --- a/tests/unit/bigquery/generated/global_namespace/test_array.py +++ /dev/null @@ -1,339 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: scripts/data/sql-functions/global_namespace/array.yaml -# by the script: scripts/generate_bigframes_bigquery.py - -import bigframes.bigquery as bbq -import bigframes.core.col -import bigframes.core.expression as ex -import bigframes.operations.googlesql.global_namespace.array as array_op -import bigframes.pandas as bpd - - -def test_array_concat_expression(): - # Call the function with col() expressions - result = bbq.array_concat( - bpd.col("array_expression_1"), - bpd.col("array_expression_2"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._ARRAY_CONCAT_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "array_expression_1" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "array_expression_2" - - -def test_array_first_expression(): - # Call the function with col() expressions - result = bbq.array_first( - bpd.col("array_expression"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._ARRAY_FIRST_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "array_expression" - - -def test_array_first_n_expression(): - # Call the function with col() expressions - result = bbq.array_first_n( - bpd.col("input_array"), - bpd.col("n"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._ARRAY_FIRST_N_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "input_array" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "n" - - -def test_array_includes_expression(): - # Call the function with col() expressions - result = bbq.array_includes( - bpd.col("array_to_search"), - bpd.col("search_value"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._ARRAY_INCLUDES_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "array_to_search" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "search_value" - - -def test_array_includes_all_expression(): - # Call the function with col() expressions - result = bbq.array_includes_all( - bpd.col("array_to_search"), - bpd.col("search_values"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._ARRAY_INCLUDES_ALL_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "array_to_search" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "search_values" - - -def test_array_includes_any_expression(): - # Call the function with col() expressions - result = bbq.array_includes_any( - bpd.col("array_to_search"), - bpd.col("search_values"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._ARRAY_INCLUDES_ANY_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "array_to_search" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "search_values" - - -def test_array_is_distinct_expression(): - # Call the function with col() expressions - result = bbq.array_is_distinct( - bpd.col("array_expression"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._ARRAY_IS_DISTINCT_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "array_expression" - - -def test_array_last_expression(): - # Call the function with col() expressions - result = bbq.array_last( - bpd.col("array_expression"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._ARRAY_LAST_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "array_expression" - - -def test_array_length_expression(): - # Call the function with col() expressions - result = bbq.array_length( - bpd.col("series"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._ARRAY_LENGTH_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "series" - - -def test_array_reverse_expression(): - # Call the function with col() expressions - result = bbq.array_reverse( - bpd.col("value"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._ARRAY_REVERSE_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "value" - - -def test_array_slice_expression(): - # Call the function with col() expressions - result = bbq.array_slice( - bpd.col("array_to_slice"), - bpd.col("start_offset"), - bpd.col("end_offset"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._ARRAY_SLICE_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "array_to_slice" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "start_offset" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "end_offset" - - -def test_array_to_string_expression(): - # Call the function with col() expressions - result = bbq.array_to_string( - bpd.col("series"), - bpd.col("delimiter"), - bpd.col("null_text"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._ARRAY_TO_STRING_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "series" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "delimiter" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "null_text" - - -def test_flatten_expression(): - # Call the function with col() expressions - result = bbq.flatten( - bpd.col("array_to_flatten"), - bpd.col("depth"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._FLATTEN_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "array_to_flatten" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "depth" - - -def test_generate_array_expression(): - # Call the function with col() expressions - result = bbq.generate_array( - bpd.col("start_expression"), - bpd.col("end_expression"), - bpd.col("step_expression"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == array_op._GENERATE_ARRAY_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "start_expression" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "end_expression" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "step_expression" diff --git a/tests/unit/bigquery/generated/global_namespace/test_bit.py b/tests/unit/bigquery/generated/global_namespace/test_bit.py deleted file mode 100644 index 2cccafc0643..00000000000 --- a/tests/unit/bigquery/generated/global_namespace/test_bit.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: scripts/data/sql-functions/global_namespace/bit.yaml -# by the script: scripts/generate_bigframes_bigquery.py - -import bigframes.bigquery as bbq -import bigframes.core.col -import bigframes.core.expression as ex -import bigframes.operations.googlesql.global_namespace.bit as bit_op -import bigframes.pandas as bpd - - -def test_bit_count_expression(): - # Call the function with col() expressions - result = bbq.bit_count( - bpd.col("expression"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == bit_op._BIT_COUNT_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "expression" diff --git a/tests/unit/bigquery/generated/global_namespace/test_conversion.py b/tests/unit/bigquery/generated/global_namespace/test_conversion.py deleted file mode 100644 index 84dfc02465c..00000000000 --- a/tests/unit/bigquery/generated/global_namespace/test_conversion.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: scripts/data/sql-functions/global_namespace/conversion.yaml -# by the script: scripts/generate_bigframes_bigquery.py - -import bigframes.bigquery as bbq -import bigframes.core.col -import bigframes.core.expression as ex -import bigframes.operations.googlesql.global_namespace.conversion as conversion_op -import bigframes.pandas as bpd - - -def test_bool__expression(): - # Call the function with col() expressions - result = bbq.bool_( - bpd.col("json_string_expression"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == conversion_op._BOOL_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "json_string_expression" - - -def test_double_expression(): - # Call the function with col() expressions - result = bbq.double( - bpd.col("json_string_expression"), - bpd.col("wide_number_mode"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == conversion_op._DOUBLE_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "json_string_expression" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "wide_number_mode" - - -def test_float64_expression(): - # Call the function with col() expressions - result = bbq.float64( - bpd.col("json_string_expression"), - bpd.col("wide_number_mode"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == conversion_op._FLOAT64_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "json_string_expression" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "wide_number_mode" - - -def test_int64_expression(): - # Call the function with col() expressions - result = bbq.int64( - bpd.col("json_string_expression"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == conversion_op._INT64_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "json_string_expression" - - -def test_parse_bignumeric_expression(): - # Call the function with col() expressions - result = bbq.parse_bignumeric( - bpd.col("string_expression"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == conversion_op._PARSE_BIGNUMERIC_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "string_expression" - - -def test_parse_numeric_expression(): - # Call the function with col() expressions - result = bbq.parse_numeric( - bpd.col("string_expression"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == conversion_op._PARSE_NUMERIC_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "string_expression" - - -def test_string_expression(): - # Call the function with col() expressions - result = bbq.string( - bpd.col("expression"), - bpd.col("timezone"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == conversion_op._STRING_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "expression" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "timezone" diff --git a/tests/unit/bigquery/generated/global_namespace/test_date.py b/tests/unit/bigquery/generated/global_namespace/test_date.py deleted file mode 100644 index 6484208584f..00000000000 --- a/tests/unit/bigquery/generated/global_namespace/test_date.py +++ /dev/null @@ -1,340 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: scripts/data/sql-functions/global_namespace/date.yaml -# by the script: scripts/generate_bigframes_bigquery.py - -import bigframes.bigquery as bbq -import bigframes.core.col -import bigframes.core.expression as ex -import bigframes.operations.googlesql.global_namespace.date as date_op -import bigframes.pandas as bpd - - -def test_current_date_expression(): - # Call the function with col() expressions - result = bbq.current_date( - bpd.col("time_zone_expression"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._CURRENT_DATE_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "time_zone_expression" - - -def test_date_expression(): - # Call the function with col() expressions - result = bbq.date( - bpd.col("expression"), - bpd.col("time_zone_expression"), - bpd.col("year"), - bpd.col("month"), - bpd.col("day"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._DATE_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 5 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "expression" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "time_zone_expression" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "year" - assert isinstance(expr.inputs[3], ex.UnboundVariableExpression) - assert expr.inputs[3].id == "month" - assert isinstance(expr.inputs[4], ex.UnboundVariableExpression) - assert expr.inputs[4].id == "day" - - -def test_date_add_expression(): - # Call the function with col() expressions - result = bbq.date_add( - bpd.col("date_expression"), - bpd.col("int64_expression"), - bpd.col("date_part"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._DATE_ADD_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "date_expression" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "int64_expression" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "date_part" - - -def test_date_diff_expression(): - # Call the function with col() expressions - result = bbq.date_diff( - bpd.col("end_date"), - bpd.col("start_date"), - bpd.col("granularity"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._DATE_DIFF_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "end_date" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "start_date" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "granularity" - - -def test_date_from_unix_date_expression(): - # Call the function with col() expressions - result = bbq.date_from_unix_date( - bpd.col("int64_expression"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._DATE_FROM_UNIX_DATE_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "int64_expression" - - -def test_date_sub_expression(): - # Call the function with col() expressions - result = bbq.date_sub( - bpd.col("date_expression"), - bpd.col("int64_expression"), - bpd.col("date_part"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._DATE_SUB_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "date_expression" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "int64_expression" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "date_part" - - -def test_date_trunc_expression(): - # Call the function with col() expressions - result = bbq.date_trunc( - bpd.col("date_value"), - bpd.col("granularity"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._DATE_TRUNC_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "date_value" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "granularity" - - -def test_extract_expression(): - # Call the function with col() expressions - result = bbq.extract( - bpd.col("date_expression"), - bpd.col("part"), - bpd.col("time_zone"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._EXTRACT_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "date_expression" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "part" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "time_zone" - - -def test_format_date_expression(): - # Call the function with col() expressions - result = bbq.format_date( - bpd.col("format_string"), - bpd.col("date_expr"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._FORMAT_DATE_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "format_string" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "date_expr" - - -def test_generate_date_array_expression(): - # Call the function with col() expressions - result = bbq.generate_date_array( - bpd.col("start_date"), - bpd.col("end_date"), - bpd.col("int64_expression"), - bpd.col("date_part"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._GENERATE_DATE_ARRAY_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 4 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "start_date" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "end_date" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "int64_expression" - assert isinstance(expr.inputs[3], ex.UnboundVariableExpression) - assert expr.inputs[3].id == "date_part" - - -def test_last_day_expression(): - # Call the function with col() expressions - result = bbq.last_day( - bpd.col("date_expression"), - bpd.col("date_part"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._LAST_DAY_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "date_expression" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "date_part" - - -def test_parse_date_expression(): - # Call the function with col() expressions - result = bbq.parse_date( - bpd.col("format_string"), - bpd.col("date_string"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._PARSE_DATE_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 2 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "format_string" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "date_string" - - -def test_unix_date_expression(): - # Call the function with col() expressions - result = bbq.unix_date( - bpd.col("date_expression"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == date_op._UNIX_DATE_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 1 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "date_expression" diff --git a/tests/unit/bigquery/generated/test_aead.py b/tests/unit/bigquery/generated/test_aead.py deleted file mode 100644 index ce728b41899..00000000000 --- a/tests/unit/bigquery/generated/test_aead.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DO NOT MODIFY THIS FILE DIRECTLY. -# This file was generated from: scripts/data/sql-functions/aead.yaml -# by the script: scripts/generate_bigframes_bigquery.py - -import bigframes.bigquery as bbq -import bigframes.core.col -import bigframes.core.expression as ex -import bigframes.operations.googlesql.aead as aead_op -import bigframes.pandas as bpd - - -def test_decrypt_bytes_expression(): - # Call the function with col() expressions - result = bbq.aead.decrypt_bytes( - bpd.col("keyset"), - bpd.col("ciphertext"), - bpd.col("additional_data"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == aead_op._DECRYPT_BYTES_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "keyset" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "ciphertext" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "additional_data" - - -def test_decrypt_string_expression(): - # Call the function with col() expressions - result = bbq.aead.decrypt_string( - bpd.col("keyset"), - bpd.col("ciphertext"), - bpd.col("additional_data"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == aead_op._DECRYPT_STRING_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "keyset" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "ciphertext" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "additional_data" - - -def test_encrypt_expression(): - # Call the function with col() expressions - result = bbq.aead.encrypt( - bpd.col("keyset"), - bpd.col("plaintext"), - bpd.col("additional_data"), - ) - - # Verify result is a col Expression - assert isinstance(result, bigframes.core.col.Expression) - - # Verify the internal expression structure - expr = result._value - assert isinstance(expr, ex.OpExpression) - assert expr.op == aead_op._ENCRYPT_OP - - # Verify arguments are free variables matching the names - assert len(expr.inputs) == 3 - assert isinstance(expr.inputs[0], ex.UnboundVariableExpression) - assert expr.inputs[0].id == "keyset" - assert isinstance(expr.inputs[1], ex.UnboundVariableExpression) - assert expr.inputs[1].id == "plaintext" - assert isinstance(expr.inputs[2], ex.UnboundVariableExpression) - assert expr.inputs[2].id == "additional_data" diff --git a/tests/unit/bigquery/test_ai.py b/tests/unit/bigquery/test_ai.py deleted file mode 100644 index 2cb876d39a5..00000000000 --- a/tests/unit/bigquery/test_ai.py +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from unittest import mock - -import pandas as pd -import pytest - -import bigframes.bigquery as bbq -import bigframes.dataframe -import bigframes.series -import bigframes.session - - -@pytest.fixture -def mock_session(): - return mock.create_autospec(spec=bigframes.session.Session) - - -@pytest.fixture -def mock_dataframe(mock_session): - df = mock.create_autospec(spec=bigframes.dataframe.DataFrame) - df._session = mock_session - df.sql = "SELECT * FROM my_table" - df._to_sql_query.return_value = ("SELECT * FROM my_table", None, None) - return df - - -@pytest.fixture -def mock_embedding_series(mock_session): - series = mock.create_autospec(spec=bigframes.series.Series) - series._session = mock_session - # Mock to_frame to return a mock dataframe - df = mock.create_autospec(spec=bigframes.dataframe.DataFrame) - df._session = mock_session - df.sql = "SELECT my_col AS content FROM my_table" - df._to_sql_query.return_value = ( - "SELECT my_col AS content FROM my_table", - None, - None, - ) - series.copy.return_value = series - series.to_frame.return_value = df - return series - - -@pytest.fixture -def mock_text_series(mock_session): - series = mock.create_autospec(spec=bigframes.series.Series) - series._session = mock_session - # Mock to_frame to return a mock dataframe - df = mock.create_autospec(spec=bigframes.dataframe.DataFrame) - df._session = mock_session - df.sql = "SELECT my_col AS prompt FROM my_table" - df._to_sql_query.return_value = ( - "SELECT my_col AS prompt FROM my_table", - None, - None, - ) - series.copy.return_value = series - series.to_frame.return_value = df - return series - - -def test_generate_embedding_with_dataframe(mock_dataframe, mock_session): - model_name = "project.dataset.model" - - bbq.ai.generate_embedding( - model_name, - mock_dataframe, - output_dimensionality=256, - ) - - mock_session.read_gbq_query.assert_called_once() - query = mock_session.read_gbq_query.call_args[0][0] - - # Normalize whitespace for comparison - query = " ".join(query.split()) - - expected_part_1 = "SELECT * FROM AI.GENERATE_EMBEDDING(" - expected_part_2 = f"MODEL `{model_name}`," - expected_part_3 = "(SELECT * FROM my_table)," - expected_part_4 = "STRUCT(256 AS `OUTPUT_DIMENSIONALITY`)" - - assert expected_part_1 in query - assert expected_part_2 in query - assert expected_part_3 in query - assert expected_part_4 in query - - -def test_generate_embedding_with_series(mock_embedding_series, mock_session): - model_name = "project.dataset.model" - - bbq.ai.generate_embedding( - model_name, - mock_embedding_series, - start_second=0.0, - end_second=10.0, - interval_seconds=5.0, - ) - - mock_session.read_gbq_query.assert_called_once() - query = mock_session.read_gbq_query.call_args[0][0] - query = " ".join(query.split()) - - assert f"MODEL `{model_name}`" in query - assert "(SELECT my_col AS content FROM my_table)" in query - assert ( - "STRUCT(0.0 AS `START_SECOND`, 10.0 AS `END_SECOND`, 5.0 AS `INTERVAL_SECONDS`)" - in query - ) - - -def test_generate_embedding_defaults(mock_dataframe, mock_session): - model_name = "project.dataset.model" - - bbq.ai.generate_embedding( - model_name, - mock_dataframe, - ) - - mock_session.read_gbq_query.assert_called_once() - query = mock_session.read_gbq_query.call_args[0][0] - query = " ".join(query.split()) - - assert f"MODEL `{model_name}`" in query - assert "STRUCT()" in query - - -@mock.patch("bigframes.pandas.read_pandas") -def test_generate_embedding_with_pandas_dataframe( - read_pandas_mock, mock_dataframe, mock_session -): - # This tests that pandas input path works and calls read_pandas - model_name = "project.dataset.model" - - # Mock return value of read_pandas to be a BigFrames DataFrame - read_pandas_mock.return_value = mock_dataframe - - pandas_df = pd.DataFrame({"content": ["test"]}) - - bbq.ai.generate_embedding( - model_name, - pandas_df, - ) - - read_pandas_mock.assert_called_once() - # Check that read_pandas was called with something (the pandas df) - assert read_pandas_mock.call_args[0][0] is pandas_df - - mock_session.read_gbq_query.assert_called_once() - - -def test_generate_text_with_dataframe(mock_dataframe, mock_session): - model_name = "project.dataset.model" - - bbq.ai.generate_text( - model_name, - mock_dataframe, - max_output_tokens=256, - ) - - mock_session.read_gbq_query.assert_called_once() - query = mock_session.read_gbq_query.call_args[0][0] - - # Normalize whitespace for comparison - query = " ".join(query.split()) - - expected_part_1 = "SELECT * FROM AI.GENERATE_TEXT(" - expected_part_2 = f"MODEL `{model_name}`," - expected_part_3 = "(SELECT * FROM my_table)," - expected_part_4 = "STRUCT(256 AS `MAX_OUTPUT_TOKENS`)" - - assert expected_part_1 in query - assert expected_part_2 in query - assert expected_part_3 in query - assert expected_part_4 in query - - -def test_generate_text_with_series(mock_text_series, mock_session): - model_name = "project.dataset.model" - - bbq.ai.generate_text( - model_name, - mock_text_series, - ) - - mock_session.read_gbq_query.assert_called_once() - query = mock_session.read_gbq_query.call_args[0][0] - query = " ".join(query.split()) - - assert f"MODEL `{model_name}`" in query - assert "(SELECT my_col AS prompt FROM my_table)" in query - - -def test_generate_text_defaults(mock_dataframe, mock_session): - model_name = "project.dataset.model" - - bbq.ai.generate_text( - model_name, - mock_dataframe, - ) - - mock_session.read_gbq_query.assert_called_once() - query = mock_session.read_gbq_query.call_args[0][0] - query = " ".join(query.split()) - - assert f"MODEL `{model_name}`" in query - assert "STRUCT()" in query - - -def test_generate_table_with_dataframe(mock_dataframe, mock_session): - model_name = "project.dataset.model" - - bbq.ai.generate_table( - model_name, - mock_dataframe, - output_schema="col1 STRING, col2 INT64", - ) - - mock_session.read_gbq_query.assert_called_once() - query = mock_session.read_gbq_query.call_args[0][0] - - # Normalize whitespace for comparison - query = " ".join(query.split()) - - expected_part_1 = "SELECT * FROM AI.GENERATE_TABLE(" - expected_part_2 = f"MODEL `{model_name}`," - expected_part_3 = "(SELECT * FROM my_table)," - expected_part_4 = "STRUCT('col1 STRING, col2 INT64' AS `output_schema`)" - - assert expected_part_1 in query - assert expected_part_2 in query - assert expected_part_3 in query - assert expected_part_4 in query - - -def test_generate_table_with_options(mock_dataframe, mock_session): - model_name = "project.dataset.model" - - bbq.ai.generate_table( - model_name, - mock_dataframe, - output_schema="col1 STRING", - temperature=0.5, - max_output_tokens=100, - ) - - mock_session.read_gbq_query.assert_called_once() - query = mock_session.read_gbq_query.call_args[0][0] - query = " ".join(query.split()) - - assert f"MODEL `{model_name}`" in query - assert "(SELECT * FROM my_table)" in query - assert ( - "STRUCT('col1 STRING' AS `output_schema`, 0.5 AS `temperature`, 100 AS `max_output_tokens`)" - in query - ) - - -def test_generate_table_with_mapping_schema(mock_dataframe, mock_session): - model_name = "project.dataset.model" - - bbq.ai.generate_table( - model_name, - mock_dataframe, - output_schema={"col1": "STRING", "col2": "INT64"}, - ) - - mock_session.read_gbq_query.assert_called_once() - query = mock_session.read_gbq_query.call_args[0][0] - - # Normalize whitespace for comparison - query = " ".join(query.split()) - - expected_part_1 = "SELECT * FROM AI.GENERATE_TABLE(" - expected_part_2 = f"MODEL `{model_name}`," - expected_part_3 = "(SELECT * FROM my_table)," - expected_part_4 = "STRUCT('col1 STRING, col2 INT64' AS `output_schema`)" - - assert expected_part_1 in query - assert expected_part_2 in query - assert expected_part_3 in query - assert expected_part_4 in query - - -@mock.patch("bigframes.pandas.read_pandas") -def test_generate_text_with_pandas_dataframe( - read_pandas_mock, mock_dataframe, mock_session -): - # This tests that pandas input path works and calls read_pandas - model_name = "project.dataset.model" - - # Mock return value of read_pandas to be a BigFrames DataFrame - read_pandas_mock.return_value = mock_dataframe - - pandas_df = pd.DataFrame({"content": ["test"]}) - - bbq.ai.generate_text( - model_name, - pandas_df, - ) - - read_pandas_mock.assert_called_once() - # Check that read_pandas was called with something (the pandas df) - assert read_pandas_mock.call_args[0][0] is pandas_df - - mock_session.read_gbq_query.assert_called_once() diff --git a/tests/unit/bigquery/test_mathematical.py b/tests/unit/bigquery/test_mathematical.py deleted file mode 100644 index f0cb16ae145..00000000000 --- a/tests/unit/bigquery/test_mathematical.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import bigframes.bigquery as bbq -import bigframes.core.col as col -import bigframes.core.expression as ex -import bigframes.dtypes as dtypes -import bigframes.operations as ops - - -def test_rand_returns_expression(): - expr = bbq.rand() - - assert isinstance(expr, col.Expression) - node = expr._value - assert isinstance(node, ex.OpExpression) - op = node.op - assert isinstance(op, ops.GoogleSqlScalarOp) - assert op.sql_name == "RAND" - assert op.output_type() == dtypes.FLOAT_DTYPE - assert not op.is_deterministic - assert len(node.inputs) == 0 diff --git a/tests/unit/bigquery/test_ml.py b/tests/unit/bigquery/test_ml.py index a68133225d4..063ddafccae 100644 --- a/tests/unit/bigquery/test_ml.py +++ b/tests/unit/bigquery/test_ml.py @@ -40,6 +40,31 @@ def mock_session(): MODEL_NAME = "test-project.test-dataset.test-model" +def test_get_model_name_and_session_with_pandas_series_model_input(): + model_name, _ = ml_ops._get_model_name_and_session(MODEL_SERIES) + assert model_name == MODEL_NAME + + +def test_get_model_name_and_session_with_pandas_series_model_input_missing_model_reference(): + model_series = pd.Series({"some_other_key": "value"}) + with pytest.raises( + ValueError, match="modelReference must be present in the pandas Series" + ): + ml_ops._get_model_name_and_session(model_series) + + +@mock.patch("bigframes.pandas.read_pandas") +def test_to_sql_with_pandas_dataframe(read_pandas_mock): + df = pd.DataFrame({"col1": [1, 2, 3]}) + read_pandas_mock.return_value._to_sql_query.return_value = ( + "SELECT * FROM `pandas_df`", + [], + [], + ) + ml_ops._to_sql(df) + read_pandas_mock.assert_called_once() + + @mock.patch("bigframes.bigquery._operations.ml._get_model_metadata") @mock.patch("bigframes.pandas.read_pandas") def test_create_model_with_pandas_dataframe( @@ -120,96 +145,3 @@ def test_global_explain_with_pandas_series_model(read_gbq_query_mock): generated_sql = read_gbq_query_mock.call_args[0][0] assert "ML.GLOBAL_EXPLAIN" in generated_sql assert f"MODEL `{MODEL_NAME}`" in generated_sql - - -@mock.patch("bigframes.pandas.read_gbq_query") -@mock.patch("bigframes.pandas.read_pandas") -def test_transform_with_pandas_dataframe(read_pandas_mock, read_gbq_query_mock): - df = pd.DataFrame({"col1": [1, 2, 3]}) - read_pandas_mock.return_value._to_sql_query.return_value = ( - "SELECT * FROM `pandas_df`", - [], - [], - ) - ml_ops.transform(MODEL_SERIES, input_=df) - read_pandas_mock.assert_called_once() - read_gbq_query_mock.assert_called_once() - generated_sql = read_gbq_query_mock.call_args[0][0] - assert "ML.TRANSFORM" in generated_sql - assert f"MODEL `{MODEL_NAME}`" in generated_sql - assert "(SELECT * FROM `pandas_df`)" in generated_sql - - -@mock.patch("bigframes.pandas.read_gbq_query") -@mock.patch("bigframes.pandas.read_pandas") -def test_generate_text_with_pandas_dataframe(read_pandas_mock, read_gbq_query_mock): - df = pd.DataFrame({"col1": [1, 2, 3]}) - read_pandas_mock.return_value._to_sql_query.return_value = ( - "SELECT * FROM `pandas_df`", - [], - [], - ) - ml_ops.generate_text( - MODEL_SERIES, - input_=df, - temperature=0.5, - max_output_tokens=128, - top_k=20, - top_p=0.9, - flatten_json_output=True, - stop_sequences=["a", "b"], - ground_with_google_search=True, - request_type="TYPE", - ) - read_pandas_mock.assert_called_once() - read_gbq_query_mock.assert_called_once() - generated_sql = read_gbq_query_mock.call_args[0][0] - assert "ML.GENERATE_TEXT" in generated_sql - assert f"MODEL `{MODEL_NAME}`" in generated_sql - assert "(SELECT * FROM `pandas_df`)" in generated_sql - assert "STRUCT(\n 0.5 AS `temperature`" in generated_sql - assert "128 AS `max_output_tokens`" in generated_sql - assert "20 AS `top_k`" in generated_sql - assert "0.9 AS `top_p`" in generated_sql - assert "TRUE AS `flatten_json_output`" in generated_sql - assert "['a', 'b'] AS `stop_sequences`" in generated_sql - assert "TRUE AS `ground_with_google_search`" in generated_sql - assert "'TYPE' AS `request_type`" in generated_sql - - -@mock.patch("bigframes.pandas.read_gbq_query") -def test_get_insights(read_gbq_query_mock): - ml_ops.get_insights(MODEL_SERIES) - read_gbq_query_mock.assert_called_once() - generated_sql = read_gbq_query_mock.call_args[0][0] - assert "ML.GET_INSIGHTS" in generated_sql - assert f"MODEL `{MODEL_NAME}`" in generated_sql - - -@mock.patch("bigframes.pandas.read_gbq_query") -@mock.patch("bigframes.pandas.read_pandas") -def test_generate_embedding_with_pandas_dataframe( - read_pandas_mock, read_gbq_query_mock -): - df = pd.DataFrame({"col1": [1, 2, 3]}) - read_pandas_mock.return_value._to_sql_query.return_value = ( - "SELECT * FROM `pandas_df`", - [], - [], - ) - ml_ops.generate_embedding( - MODEL_SERIES, - input_=df, - flatten_json_output=True, - task_type="RETRIEVAL_DOCUMENT", - output_dimensionality=256, - ) - read_pandas_mock.assert_called_once() - read_gbq_query_mock.assert_called_once() - generated_sql = read_gbq_query_mock.call_args[0][0] - assert "ML.GENERATE_EMBEDDING" in generated_sql - assert f"MODEL `{MODEL_NAME}`" in generated_sql - assert "(SELECT * FROM `pandas_df`)" in generated_sql - assert "STRUCT(\n TRUE AS `flatten_json_output`" in generated_sql - assert "'RETRIEVAL_DOCUMENT' AS `task_type`" in generated_sql - assert "256 AS `output_dimensionality`" in generated_sql diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index d880fe54242..a9b26afeef2 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -12,25 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import datetime -import json -import pathlib -import typing - -import numpy as np -import pandas as pd -import pyarrow as pa import pytest -from google.cloud import bigquery - -import bigframes.core as core -import bigframes.pandas as bpd -import bigframes.testing.mocks as mocks -import bigframes.testing.utils -from bigframes import dtypes - -CURRENT_DIR = pathlib.Path(__file__).parent -DATA_DIR = CURRENT_DIR.parent / "data" @pytest.fixture(scope="session") @@ -40,277 +22,3 @@ def polars_session(): from bigframes.testing import polars_session return polars_session.TestSession() - - -def _create_compiler_session(table_name, table_schema): - """Helper function to create a compiler session.""" - from bigframes.testing import compiler_session - - anonymous_dataset = bigquery.DatasetReference.from_string( - "bigframes-dev.sqlglot_test" - ) - session = mocks.create_bigquery_session( - table_name=table_name, - table_schema=table_schema, - anonymous_dataset=anonymous_dataset, - ) - session._executor = compiler_session.SQLCompilerExecutor() - return session - - -@pytest.fixture(scope="session") -def compiler_session(scalar_types_table_schema): - """Compiler session for scalar types.""" - return _create_compiler_session("scalar_types", scalar_types_table_schema) - - -@pytest.fixture(scope="session") -def compiler_session_w_repeated_types(repeated_types_table_schema): - """Compiler session for repeated data types.""" - return _create_compiler_session("repeated_types", repeated_types_table_schema) - - -@pytest.fixture(scope="session") -def compiler_session_w_nested_structs_types(nested_structs_types_table_schema): - """Compiler session for nested STRUCT data types.""" - return _create_compiler_session( - "nested_structs_types", nested_structs_types_table_schema - ) - - -@pytest.fixture(scope="session") -def compiler_session_w_json_types(json_types_table_schema): - """Compiler session for JSON data types.""" - return _create_compiler_session("json_types", json_types_table_schema) - - -@pytest.fixture(scope="session") -def scalar_types_table_schema() -> typing.Sequence[bigquery.SchemaField]: - return [ - bigquery.SchemaField("bool_col", "BOOLEAN"), - bigquery.SchemaField("bytes_col", "BYTES"), - bigquery.SchemaField("date_col", "DATE"), - bigquery.SchemaField("datetime_col", "DATETIME"), - bigquery.SchemaField("geography_col", "GEOGRAPHY"), - bigquery.SchemaField("int64_col", "INTEGER"), - bigquery.SchemaField("int64_too", "INTEGER"), - bigquery.SchemaField("numeric_col", "NUMERIC"), - bigquery.SchemaField("float64_col", "FLOAT"), - bigquery.SchemaField("rowindex", "INTEGER"), - bigquery.SchemaField("rowindex_2", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("string_col", "STRING"), - bigquery.SchemaField("time_col", "TIME"), - bigquery.SchemaField("timestamp_col", "TIMESTAMP"), - bigquery.SchemaField("duration_col", "INTEGER"), - ] - - -@pytest.fixture(scope="session") -def scalar_types_df(compiler_session) -> bpd.DataFrame: - """Returns a BigFrames DataFrame containing all scalar types and using the `rowindex` - column as the index.""" - bf_df = compiler_session._loader.read_gbq_table( - "bigframes-dev.sqlglot_test.scalar_types", - enable_snapshot=False, - ) - bf_df = bf_df.set_index("rowindex", drop=False) - return bf_df - - -@pytest.fixture(scope="session") -def scalar_types_pandas_df() -> pd.DataFrame: - """Returns a pandas DataFrame containing all scalar types and using the `rowindex` - column as the index.""" - # TODO: add tests for empty dataframes - df = pd.read_json( - DATA_DIR / "scalars.jsonl", - lines=True, - ) - bigframes.testing.utils.convert_pandas_dtypes(df, bytes_col=True) - - df = df.set_index("rowindex", drop=False) - return df - - -@pytest.fixture(scope="module") -def scalar_types_array_value( - scalar_types_pandas_df: pd.DataFrame, compiler_session: bigframes.Session -) -> core.ArrayValue: - managed_data_source = core.local_data.ManagedArrowTable.from_pandas( - scalar_types_pandas_df - ) - return core.ArrayValue.from_managed(managed_data_source, compiler_session) - - -@pytest.fixture(scope="session") -def nested_structs_types_table_schema() -> typing.Sequence[bigquery.SchemaField]: - return [ - bigquery.SchemaField("id", "INTEGER"), - bigquery.SchemaField( - "people", - "RECORD", - fields=[ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("age", "INTEGER"), - bigquery.SchemaField( - "address", - "RECORD", - fields=[ - bigquery.SchemaField("city", "STRING"), - bigquery.SchemaField("country", "STRING"), - ], - ), - ], - ), - ] - - -@pytest.fixture(scope="session") -def nested_structs_types_df(compiler_session_w_nested_structs_types) -> bpd.DataFrame: - """Returns a BigFrames DataFrame containing all scalar types and using the `rowindex` - column as the index.""" - bf_df = compiler_session_w_nested_structs_types._loader.read_gbq_table( - "bigframes-dev.sqlglot_test.nested_structs_types", - enable_snapshot=False, - ) - bf_df = bf_df.set_index("id", drop=False) - return bf_df - - -@pytest.fixture(scope="session") -def nested_structs_pandas_df() -> pd.DataFrame: - """Returns a pandas DataFrame containing STRUCT types and using the `id` - column as the index.""" - - df = pd.read_json( - DATA_DIR / "nested_structs.jsonl", - lines=True, - ) - df = df.set_index("id") - - address_struct_schema = pa.struct( - [pa.field("city", pa.string()), pa.field("country", pa.string())] - ) - person_struct_schema = pa.struct( - [ - pa.field("name", pa.string()), - pa.field("age", pa.int64()), - pa.field("address", address_struct_schema), - ] - ) - df["person"] = df["person"].astype(pd.ArrowDtype(person_struct_schema)) - - def to_json_str(val): - if val is None or (isinstance(val, float) and np.isnan(val)): - return None - return json.dumps(val) - - df["json_col"] = df["json_col"].apply(to_json_str).astype(dtypes.JSON_DTYPE) - - # timestamp_col - def parse_timestamp(val): - if pd.isna(val): - return None - if isinstance(val, str): - return datetime.datetime.fromisoformat(val.replace("Z", "+00:00")) - if hasattr(val, "to_pydatetime"): - return val.to_pydatetime() - return val - - timestamp_vals = [parse_timestamp(x) for x in df["timestamp_col"]] - timestamp_arr = pa.array(timestamp_vals, type=dtypes.TIMESTAMP_DTYPE.pyarrow_dtype) - df["timestamp_col"] = pd.Series( - timestamp_arr, index=df.index, dtype=dtypes.TIMESTAMP_DTYPE - ) - - return df - - -@pytest.fixture(scope="session") -def repeated_types_table_schema() -> typing.Sequence[bigquery.SchemaField]: - return [ - bigquery.SchemaField("rowindex", "INTEGER"), - bigquery.SchemaField("int_list_col", "INTEGER", "REPEATED"), - bigquery.SchemaField("bool_list_col", "BOOLEAN", "REPEATED"), - bigquery.SchemaField("float_list_col", "FLOAT", "REPEATED"), - bigquery.SchemaField("date_list_col", "DATE", "REPEATED"), - bigquery.SchemaField("date_time_list_col", "DATETIME", "REPEATED"), - bigquery.SchemaField("numeric_list_col", "NUMERIC", "REPEATED"), - bigquery.SchemaField("string_list_col", "STRING", "REPEATED"), - ] - - -@pytest.fixture(scope="session") -def repeated_types_df(compiler_session_w_repeated_types) -> bpd.DataFrame: - """Returns a BigFrames DataFrame containing all scalar types and using the `rowindex` - column as the index.""" - bf_df = compiler_session_w_repeated_types._loader.read_gbq_table( - "bigframes-dev.sqlglot_test.repeated_types", - enable_snapshot=False, - ) - bf_df = bf_df.set_index("rowindex", drop=False) - return bf_df - - -@pytest.fixture(scope="session") -def repeated_types_pandas_df() -> pd.DataFrame: - """Returns a pandas DataFrame containing LIST types and using the `rowindex` - column as the index.""" - - df = pd.read_json( - DATA_DIR / "repeated.jsonl", - lines=True, - ) - # TODO: add dtype conversion here if needed. - df = df.set_index("rowindex") - return df - - -@pytest.fixture(scope="session") -def json_types_table_schema() -> typing.Sequence[bigquery.SchemaField]: - return [ - bigquery.SchemaField("rowindex", "INTEGER"), - bigquery.SchemaField("json_col", "JSON"), - ] - - -@pytest.fixture(scope="session") -def json_types_df(compiler_session_w_json_types) -> bpd.DataFrame: - """Returns a BigFrames DataFrame containing JSON types and using the `rowindex` - column as the index.""" - bf_df = compiler_session_w_json_types._loader.read_gbq_table( - "bigframes-dev.sqlglot_test.json_types", - enable_snapshot=False, - ) - # TODO(b/427305807): Why `drop=False` will produce two "rowindex" columns? - bf_df = bf_df.set_index("rowindex", drop=True) - return bf_df - - -@pytest.fixture(scope="session") -def json_pandas_df() -> pd.DataFrame: - """Returns a pandas DataFrame containing JSON types and using the `rowindex` - column as the index.""" - json_data = [ - "null", - "true", - "100", - "0.98", - '"a string"', - "[]", - "[1, 2, 3]", - '[{"a": 1}, {"a": 2}, {"a": null}, {}]', - '"100"', - '{"date": "2024-07-16"}', - '{"int_value": 2, "null_filed": null}', - '{"list_data": [10, 20, 30]}', - ] - df = pd.DataFrame( - { - "rowindex": pd.Series(range(len(json_data)), dtype=dtypes.INT_DTYPE), - "json_col": pd.Series(json_data, dtype=dtypes.JSON_DTYPE), - }, - ) - # TODO(b/427305807): Why `drop=False` will produce two "rowindex" columns? - df = df.set_index("rowindex", drop=True) - return df diff --git a/bigframes/operations/googlesql/global_namespace/__init__.py b/tests/unit/core/compile/googlesql/__init__.py similarity index 95% rename from bigframes/operations/googlesql/global_namespace/__init__.py rename to tests/unit/core/compile/googlesql/__init__.py index 58d482ea386..6d5e14bcf4a 100644 --- a/bigframes/operations/googlesql/global_namespace/__init__.py +++ b/tests/unit/core/compile/googlesql/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/unit/core/compile/googlesql/test_expression.py b/tests/unit/core/compile/googlesql/test_expression.py new file mode 100644 index 00000000000..e72598b1760 --- /dev/null +++ b/tests/unit/core/compile/googlesql/test_expression.py @@ -0,0 +1,37 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import bigframes.core.compile.googlesql as sql + + +@pytest.mark.parametrize( + ("table_id", "dataset_id", "project_id", "expected"), + [ + pytest.param("a", None, None, "`a`"), + pytest.param("a", "b", None, "`b`.`a`"), + pytest.param("a", "b", "c", "`c`.`b`.`a`"), + pytest.param("a", None, "c", None, marks=pytest.mark.xfail(raises=ValueError)), + ], +) +def test_table_expression(table_id, dataset_id, project_id, expected): + expr = sql.TableExpression( + table_id=table_id, dataset_id=dataset_id, project_id=project_id + ) + assert expr.sql() == expected + + +def test_escape_chars(): + assert sql._escape_chars("\a\b\f\n\r\t\v\\?'\"`") == r"\a\b\f\n\r\t\v\\\?\'\"\`" diff --git a/tests/unit/core/compile/googlesql/test_function.py b/tests/unit/core/compile/googlesql/test_function.py new file mode 100644 index 00000000000..4edfda6f345 --- /dev/null +++ b/tests/unit/core/compile/googlesql/test_function.py @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import bigframes.core.compile.googlesql as sql + + +def test_cast(): + col = sql.ColumnExpression("col") + assert sql.Cast(col, sql.DataType.STRING).sql() == "CAST (`col` AS STRING)" + assert sql.Cast(col, sql.DataType.FLOAT64).sql() == "CAST (`col` AS FLOAT64)" diff --git a/tests/unit/core/compile/googlesql/test_query.py b/tests/unit/core/compile/googlesql/test_query.py new file mode 100644 index 00000000000..b8d1d024e2b --- /dev/null +++ b/tests/unit/core/compile/googlesql/test_query.py @@ -0,0 +1,223 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import MagicMock + +import google.cloud.bigquery as bigquery +import pytest + +import bigframes.core.compile.googlesql as sql + + +@pytest.mark.parametrize( + ("table_id", "dataset_id", "project_id", "expected"), + [ + pytest.param("a", None, None, "`a`"), + pytest.param("a", "b", None, "`b`.`a`"), + pytest.param("a", "b", "c", "`c`.`b`.`a`"), + pytest.param("a", None, "c", None, marks=pytest.mark.xfail(raises=ValueError)), + ], +) +def test_table_expression(table_id, dataset_id, project_id, expected): + expr = sql.TableExpression( + table_id=table_id, dataset_id=dataset_id, project_id=project_id + ) + assert expr.sql() == expected + + +@pytest.mark.parametrize( + ("table_name", "alias", "expected"), + [ + pytest.param("a", None, "`a`"), + pytest.param("a", "aa", "`a` AS `aa`"), + ], +) +def test_from_item_w_table_name(table_name, alias, expected): + expr = sql.FromItem( + sql.TableExpression(table_id=table_name), + as_alias=None + if alias is None + else sql.AsAlias(sql.AliasExpression(alias=alias)), + ) + assert expr.sql() == expected + + +def test_from_item_w_query_expr(): + from_clause = sql.FromClause( + sql.FromItem(expression=sql.TableExpression(table_id="table_a")) + ) + select = sql.Select( + select_list=[sql.SelectAll(sql.StarExpression())], + from_clause_list=[from_clause], + ) + query_expr = sql.QueryExpr(select=select) + expected = "SELECT\n*\nFROM\n`table_a`" + + # A QueryExpr object + expr = sql.FromItem(expression=query_expr) + assert expr.sql() == f"({expected})" + + # A str object + expr = sql.FromItem(expression=expected) + assert expr.sql() == f"({expected})" + + +def test_from_item_w_cte(): + expr = sql.FromItem(expression=sql.CTEExpression("test")) + assert expr.sql() == "`test`" + + +def test_from_item_w_table_ref(): + mock_table_ref = MagicMock(spec=bigquery.TableReference) + mock_table_ref.table_id = "mock_table" + mock_table_ref.dataset_id = "mock_dataset" + mock_table_ref.project = "mock_project" + + from_item = sql.FromItem.from_source(mock_table_ref) + + assert from_item.sql() == "`mock_project`.`mock_dataset`.`mock_table`" + + +@pytest.mark.parametrize( + ("col_name", "alias", "expected"), + [ + pytest.param("a", None, "`a`"), + pytest.param("a", "aa", "`a` AS `aa`"), + ], +) +def test_select_expression(col_name, alias, expected): + expr = sql.SelectExpression( + expression=sql.ColumnExpression(col_name), + alias=None if alias is None else sql.AliasExpression(alias=alias), + ) + assert expr.sql() == expected + + +def test_select(): + select_1 = sql.SelectExpression(expression=sql.ColumnExpression("a")) + select_2 = sql.SelectExpression( + expression=sql.ColumnExpression("b"), alias=sql.AliasExpression(alias="bb") + ) + from_1 = sql.FromItem(expression=sql.TableExpression(table_id="table_a")) + from_2 = sql.FromItem( + expression="SELECT * FROM project.table_b", + as_alias=sql.AsAlias(sql.AliasExpression(alias="table_b")), + ) + expr = sql.Select( + select_list=[select_1, select_2], + from_clause_list=[sql.FromClause(from_1), sql.FromClause(from_2)], + ) + expected = "SELECT\n`a`,\n`b` AS `bb`\nFROM\n`table_a`,\n(SELECT * FROM project.table_b) AS `table_b`" + + assert expr.sql() == expected + + +@pytest.mark.parametrize( + "columns, source, expected", + [ + ( + ["a", "b", "c"], + "select * from test", + "SELECT\nDISTINCT\n`a`,\n`b`,\n`c`\nFROM\n(select * from test)", + ), + ( + "a", + "select * from test", + "SELECT\nDISTINCT\n`a`\nFROM\n(select * from test)", + ), + ], +) +def test_select_from_str(columns, source, expected): + expr = sql.Select().from_(source).select(columns, distinct=True) + assert expr.sql() == expected + + +@pytest.mark.parametrize( + ("columns", "distinct", "expected"), + [ + pytest.param( + ["a", "b", "c"], + True, + "SELECT\nDISTINCT\n`a`,\n`b`,\n`c`\nFROM\n`mock_project`.`mock_dataset`.`mock_table`", + ), + pytest.param( + None, + True, + "SELECT\nDISTINCT\n*\nFROM\n`mock_project`.`mock_dataset`.`mock_table`", + ), + pytest.param( + None, False, "SELECT\n*\nFROM\n`mock_project`.`mock_dataset`.`mock_table`" + ), + ], +) +def test_select_from_table_ref(columns, distinct, expected): + mock_table_ref = MagicMock(spec=bigquery.TableReference) + mock_table_ref.table_id = "mock_table" + mock_table_ref.dataset_id = "mock_dataset" + mock_table_ref.project = "mock_project" + + expr = sql.Select().from_(mock_table_ref).select(columns, distinct=distinct) + assert expr.sql() == expected + + +def test_query_expr_w_cte(): + # Test a simple SELECT query. + from_clause1 = sql.FromClause( + sql.FromItem(expression=sql.TableExpression(table_id="table_a")) + ) + select1 = sql.Select( + select_list=[sql.SelectAll(sql.StarExpression())], + from_clause_list=[from_clause1], + ) + query1 = sql.QueryExpr(select=select1) + query1_sql = "SELECT\n*\nFROM\n`table_a`" + assert query1.sql() == query1_sql + + # Test a query with CTE statements. + cte1 = sql.NonRecursiveCTE(cte_name=sql.CTEExpression("a"), query_expr=query1) + cte2 = sql.NonRecursiveCTE(cte_name=sql.CTEExpression("b"), query_expr=query1) + + cte1_sql = f"`a` AS (\n{query1_sql}\n)" + cte2_sql = f"`b` AS (\n{query1_sql}\n)" + assert cte1.sql() == cte1_sql + assert cte2.sql() == cte2_sql + + with_cte_list = [cte1, cte2] + select2 = sql.Select( + select_list=[ + sql.SelectExpression( + sql.ColumnExpression(parent=cte1.cte_name, name="column_x") + ), + sql.SelectAll(sql.StarExpression(parent=cte2.cte_name)), + ], + from_clause_list=[ + sql.FromClause(sql.FromItem(expression=cte1.cte_name)), + sql.FromClause(sql.FromItem(expression=cte2.cte_name)), + ], + distinct=True, + ) + select2_sql = "SELECT\nDISTINCT\n`a`.`column_x`,\n`b`.*\nFROM\n`a`,\n`b`" + assert select2.sql() == select2_sql + + query2 = sql.QueryExpr(select=select2, with_cte_list=with_cte_list) + query2_sql = f"WITH {cte1_sql},\n{cte2_sql}\n{select2_sql}" + assert query2.sql() == query2_sql + + +def test_identifier(): + assert sql.identifier("\aa") == r"`\aa`" + + +def test_escape_chars(): + assert sql._escape_chars("\a\b\f\n\r\t\v\\?'\"`") == r"\a\b\f\n\r\t\v\\\?\'\"\`" diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_binary_compiler/test_corr/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_binary_compiler/test_corr/out.sql index fb930323dbd..5c838f48827 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_binary_compiler/test_corr/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_binary_compiler/test_corr/out.sql @@ -1,8 +1,8 @@ WITH `bfcte_0` AS ( SELECT - `int64_col`, - `float64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT CORR(`int64_col`, `float64_col`) AS `bfcol_2` diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_binary_compiler/test_cov/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_binary_compiler/test_cov/out.sql index 92b8ea4d3ab..eda082250a6 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_binary_compiler/test_cov/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_binary_compiler/test_cov/out.sql @@ -1,8 +1,8 @@ WITH `bfcte_0` AS ( SELECT - `int64_col`, - `float64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT COVAR_SAMP(`int64_col`, `float64_col`) AS `bfcol_2` diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_row_number/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_row_number/out.sql index 7056c8b0af3..f1197465f0d 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_row_number/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_row_number/out.sql @@ -1,3 +1,27 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `bytes_col`, + `date_col`, + `datetime_col`, + `duration_col`, + `float64_col`, + `geography_col`, + `int64_col`, + `int64_too`, + `numeric_col`, + `rowindex`, + `rowindex_2`, + `string_col`, + `time_col`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ROW_NUMBER() OVER () - 1 AS `bfcol_32` + FROM `bfcte_0` +) SELECT - ROW_NUMBER() OVER () - 1 AS `row_number` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_32` AS `row_number` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_row_number_with_window/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_row_number_with_window/out.sql index 8efea4b51bc..bfa67b8a747 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_row_number_with_window/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_row_number_with_window/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ROW_NUMBER() OVER (ORDER BY `int64_col` ASC NULLS LAST) - 1 AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ROW_NUMBER() OVER (ORDER BY `int64_col` ASC NULLS LAST) - 1 AS `row_number` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `row_number` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_size/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_size/out.sql index 4d67203ecc6..ed8e0c7619d 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_size/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_size/out.sql @@ -1,7 +1,21 @@ WITH `bfcte_0` AS ( SELECT - * - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + `bool_col`, + `bytes_col`, + `date_col`, + `datetime_col`, + `duration_col`, + `float64_col`, + `geography_col`, + `int64_col`, + `int64_too`, + `numeric_col`, + `rowindex`, + `rowindex_2`, + `string_col`, + `time_col`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT COUNT(1) AS `bfcol_32` diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_ordered_unary_compiler/test_array_agg/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_ordered_unary_compiler/test_array_agg/out.sql index f929970a227..eafbc39daf8 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_ordered_unary_compiler/test_array_agg/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_ordered_unary_compiler/test_array_agg/out.sql @@ -1,7 +1,7 @@ WITH `bfcte_0` AS ( SELECT `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT ARRAY_AGG(`int64_col` IGNORE NULLS ORDER BY `int64_col` IS NULL ASC, `int64_col` ASC) AS `bfcol_1` diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_ordered_unary_compiler/test_string_agg/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_ordered_unary_compiler/test_string_agg/out.sql index 7e697719b36..321341d4a0a 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_ordered_unary_compiler/test_string_agg/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_ordered_unary_compiler/test_string_agg/out.sql @@ -1,7 +1,7 @@ WITH `bfcte_0` AS ( SELECT `string_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT COALESCE( diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_all/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_all/out.sql index dc1f6fb4f79..0be2fea80b2 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_all/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_all/out.sql @@ -2,7 +2,7 @@ WITH `bfcte_0` AS ( SELECT `bool_col`, `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT COALESCE(LOGICAL_AND(`bool_col`), TRUE) AS `bfcol_2`, diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_all_w_window/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_all_w_window/out.sql index 7e4c9d6c3c9..829e5a88361 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_all_w_window/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_all_w_window/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + COALESCE(LOGICAL_AND(`bool_col`) OVER (), TRUE) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - COALESCE(LOGICAL_AND(`bool_col`) OVER (), TRUE) AS `agg_bool` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_bool` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any/out.sql index 8ae589fb09f..ae62e22e36d 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any/out.sql @@ -2,7 +2,7 @@ WITH `bfcte_0` AS ( SELECT `bool_col`, `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT COALESCE(LOGICAL_OR(`bool_col`), FALSE) AS `bfcol_2`, diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_value/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_value/out.sql index e8556018852..4a13901f1c4 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_value/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_value/out.sql @@ -1,7 +1,7 @@ WITH `bfcte_0` AS ( SELECT `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT ANY_VALUE(`int64_col`) AS `bfcol_1` diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_value/window_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_value/window_out.sql index 020d7603b98..ea15243d90a 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_value/window_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_value/window_out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ANY_VALUE(`int64_col`) OVER () AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ANY_VALUE(`int64_col`) OVER () AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_value/window_partition_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_value/window_partition_out.sql index 577c5929b91..e722318fbce 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_value/window_partition_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_value/window_partition_out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ANY_VALUE(`int64_col`) OVER (PARTITION BY `string_col`) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - ANY_VALUE(`int64_col`) OVER (PARTITION BY `string_col`) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_w_window/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_w_window/out.sql index 33045c4b70d..337f0ff9638 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_w_window/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_any_w_window/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + COALESCE(LOGICAL_OR(`bool_col`) OVER (), FALSE) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - COALESCE(LOGICAL_OR(`bool_col`) OVER (), FALSE) AS `agg_bool` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_bool` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_approx_quartiles/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_approx_quartiles/out.sql index e2a119499f2..9eabb2d88a7 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_approx_quartiles/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_approx_quartiles/out.sql @@ -1,7 +1,7 @@ WITH `bfcte_0` AS ( SELECT `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT APPROX_QUANTILES(`int64_col`, 4)[OFFSET(1)] AS `bfcol_1`, diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_approx_top_count/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_approx_top_count/out.sql index 1c391c6691f..b5e6275381b 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_approx_top_count/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_approx_top_count/out.sql @@ -1,7 +1,7 @@ WITH `bfcte_0` AS ( SELECT `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT APPROX_TOP_COUNT(`int64_col`, 10) AS `bfcol_1` diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_count/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_count/out.sql index 61f073b7dc8..9d18367cf61 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_count/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_count/out.sql @@ -1,7 +1,7 @@ WITH `bfcte_0` AS ( SELECT `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT COUNT(`int64_col`) AS `bfcol_1` diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_count/window_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_count/window_out.sql index e46b49e7e48..0baac953118 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_count/window_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_count/window_out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + COUNT(`int64_col`) OVER () AS `bfcol_1` + FROM `bfcte_0` +) SELECT - COUNT(`int64_col`) OVER () AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_count/window_partition_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_count/window_partition_out.sql index 98088d97dfc..6d3f8564599 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_count/window_partition_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_count/window_partition_out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + COUNT(`int64_col`) OVER (PARTITION BY `string_col`) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - COUNT(`int64_col`) OVER (PARTITION BY `string_col`) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/int_bins.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/int_bins.sql index ac5525fe63f..015ac327998 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/int_bins.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/int_bins.sql @@ -1,47 +1,55 @@ -SELECT - CASE - WHEN `int64_col` <= MIN(`int64_col`) OVER () + ( - 1 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) - ) - THEN STRUCT( - ( - MIN(`int64_col`) OVER () + ( - 0 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) - ) - ) - ( - ( - MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER () - ) * 0.001 - ) AS `left_exclusive`, - MIN(`int64_col`) OVER () + ( +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN `int64_col` <= MIN(`int64_col`) OVER () + ( 1 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) - ) + 0 AS `right_inclusive` - ) - WHEN `int64_col` <= MIN(`int64_col`) OVER () + ( - 2 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) - ) - THEN STRUCT( - ( + ) + THEN STRUCT( + ( + MIN(`int64_col`) OVER () + ( + 0 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) + ) + ) - ( + ( + MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER () + ) * 0.001 + ) AS `left_exclusive`, MIN(`int64_col`) OVER () + ( 1 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) - ) - ) - 0 AS `left_exclusive`, - MIN(`int64_col`) OVER () + ( + ) + 0 AS `right_inclusive` + ) + WHEN `int64_col` <= MIN(`int64_col`) OVER () + ( 2 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) - ) + 0 AS `right_inclusive` - ) - WHEN ( - `int64_col` - ) IS NOT NULL - THEN STRUCT( - ( + ) + THEN STRUCT( + ( + MIN(`int64_col`) OVER () + ( + 1 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) + ) + ) - 0 AS `left_exclusive`, MIN(`int64_col`) OVER () + ( 2 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) - ) - ) - 0 AS `left_exclusive`, - MIN(`int64_col`) OVER () + ( - 3 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) - ) + 0 AS `right_inclusive` - ) - END AS `int_bins` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + ) + 0 AS `right_inclusive` + ) + WHEN `int64_col` IS NOT NULL + THEN STRUCT( + ( + MIN(`int64_col`) OVER () + ( + 2 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) + ) + ) - 0 AS `left_exclusive`, + MIN(`int64_col`) OVER () + ( + 3 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) + ) + 0 AS `right_inclusive` + ) + END AS `bfcol_1` + FROM `bfcte_0` +) +SELECT + `bfcol_1` AS `int_bins` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/int_bins_labels.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/int_bins_labels.sql index 94e9f57b28e..c98682f2b83 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/int_bins_labels.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/int_bins_labels.sql @@ -1,16 +1,24 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN `int64_col` < MIN(`int64_col`) OVER () + ( + 1 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) + ) + THEN 'a' + WHEN `int64_col` < MIN(`int64_col`) OVER () + ( + 2 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) + ) + THEN 'b' + WHEN `int64_col` IS NOT NULL + THEN 'c' + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN `int64_col` < MIN(`int64_col`) OVER () + ( - 1 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) - ) - THEN 'a' - WHEN `int64_col` < MIN(`int64_col`) OVER () + ( - 2 * IEEE_DIVIDE(MAX(`int64_col`) OVER () - MIN(`int64_col`) OVER (), 3) - ) - THEN 'b' - WHEN ( - `int64_col` - ) IS NOT NULL - THEN 'c' - END AS `int_bins_labels` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `int_bins_labels` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/interval_bins.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/interval_bins.sql index 10f9778f55e..a3e689b11ec 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/interval_bins.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/interval_bins.sql @@ -1,8 +1,18 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN `int64_col` > 0 AND `int64_col` <= 1 + THEN STRUCT(0 AS `left_exclusive`, 1 AS `right_inclusive`) + WHEN `int64_col` > 1 AND `int64_col` <= 2 + THEN STRUCT(1 AS `left_exclusive`, 2 AS `right_inclusive`) + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN `int64_col` > 0 AND `int64_col` <= 1 - THEN STRUCT(0 AS `left_exclusive`, 1 AS `right_inclusive`) - WHEN `int64_col` > 1 AND `int64_col` <= 2 - THEN STRUCT(1 AS `left_exclusive`, 2 AS `right_inclusive`) - END AS `interval_bins` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `interval_bins` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/interval_bins_labels.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/interval_bins_labels.sql index 247c71a6349..1a8a92e38ee 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/interval_bins_labels.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_cut/interval_bins_labels.sql @@ -1,8 +1,18 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN `int64_col` > 0 AND `int64_col` <= 1 + THEN 0 + WHEN `int64_col` > 1 AND `int64_col` <= 2 + THEN 1 + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN `int64_col` > 0 AND `int64_col` <= 1 - THEN 0 - WHEN `int64_col` > 1 AND `int64_col` <= 2 - THEN 1 - END AS `interval_bins_labels` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `interval_bins_labels` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_dense_rank/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_dense_rank/out.sql index 95f53752c34..76b455a65c9 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_dense_rank/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_dense_rank/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + DENSE_RANK() OVER (ORDER BY `int64_col` DESC) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - DENSE_RANK() OVER (ORDER BY `int64_col` DESC) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_bool/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_bool/out.sql index 592f3e240a4..96d23c4747d 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_bool/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_bool/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `bool_col` <> LAG(`bool_col`, 1) OVER (ORDER BY `bool_col` DESC) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - `bool_col` <> LAG(`bool_col`, 1) OVER (ORDER BY `bool_col` DESC) AS `diff_bool` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `diff_bool` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_date/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_date/out.sql index 4b41355d948..4f1729d2e28 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_date/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_date/out.sql @@ -1,5 +1,15 @@ +WITH `bfcte_0` AS ( + SELECT + `date_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(FLOOR( + DATE_DIFF(`date_col`, LAG(`date_col`, 1) OVER (ORDER BY `date_col` ASC NULLS LAST), DAY) * 86400000000 + ) AS INT64) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CAST(FLOOR( - DATE_DIFF(`date_col`, LAG(`date_col`, 1) OVER (ORDER BY `date_col` ASC NULLS LAST), DAY) * 86400000000 - ) AS INT64) AS `diff_date` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `diff_date` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_datetime/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_datetime/out.sql index 866f49b1ed4..9c279a479d5 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_datetime/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_datetime/out.sql @@ -1,7 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `datetime_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + DATETIME_DIFF( + `datetime_col`, + LAG(`datetime_col`, 1) OVER (ORDER BY `datetime_col` ASC NULLS LAST), + MICROSECOND + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - DATETIME_DIFF( - `datetime_col`, - LAG(`datetime_col`, 1) OVER (ORDER BY `datetime_col` ASC NULLS LAST), - MICROSECOND - ) AS `diff_datetime` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `diff_datetime` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_int/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_int/out.sql index 4c8a0880f3b..95d786b951e 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_int/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_int/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `int64_col` - LAG(`int64_col`, 1) OVER (ORDER BY `int64_col` ASC NULLS LAST) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - `int64_col` - LAG(`int64_col`, 1) OVER (ORDER BY `int64_col` ASC NULLS LAST) AS `diff_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `diff_int` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_timestamp/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_timestamp/out.sql index 364f6b69d84..1f8b8227b4a 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_timestamp/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_diff_w_timestamp/out.sql @@ -1,7 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + TIMESTAMP_DIFF( + `timestamp_col`, + LAG(`timestamp_col`, 1) OVER (ORDER BY `timestamp_col` DESC), + MICROSECOND + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - TIMESTAMP_DIFF( - `timestamp_col`, - LAG(`timestamp_col`, 1) OVER (ORDER BY `timestamp_col` DESC), - MICROSECOND - ) AS `diff_timestamp` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `diff_timestamp` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_first/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_first/out.sql index 86aedff91d1..b053178f584 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_first/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_first/out.sql @@ -1,6 +1,16 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + FIRST_VALUE(`int64_col`) OVER ( + ORDER BY `int64_col` DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - FIRST_VALUE(`int64_col`) OVER ( - ORDER BY `int64_col` DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_first_non_null/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_first_non_null/out.sql index b7851a350ed..2ef7b7151e2 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_first_non_null/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_first_non_null/out.sql @@ -1,6 +1,16 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + FIRST_VALUE(`int64_col` IGNORE NULLS) OVER ( + ORDER BY `int64_col` ASC NULLS LAST + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - FIRST_VALUE(`int64_col` IGNORE NULLS) OVER ( - ORDER BY `int64_col` ASC NULLS LAST - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_last/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_last/out.sql index d0bb802c333..61e90ee612e 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_last/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_last/out.sql @@ -1,6 +1,16 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + LAST_VALUE(`int64_col`) OVER ( + ORDER BY `int64_col` DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - LAST_VALUE(`int64_col`) OVER ( - ORDER BY `int64_col` DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_last_non_null/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_last_non_null/out.sql index 39d063a3c99..c626c263ace 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_last_non_null/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_last_non_null/out.sql @@ -1,6 +1,16 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + LAST_VALUE(`int64_col` IGNORE NULLS) OVER ( + ORDER BY `int64_col` ASC NULLS LAST + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - LAST_VALUE(`int64_col` IGNORE NULLS) OVER ( - ORDER BY `int64_col` ASC NULLS LAST - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_max/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_max/out.sql index 7e01c2c7187..1537d735ead 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_max/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_max/out.sql @@ -1,7 +1,7 @@ WITH `bfcte_0` AS ( SELECT `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT MAX(`int64_col`) AS `bfcol_1` diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_max/window_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_max/window_out.sql index d6dec51cdb4..f55201418a9 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_max/window_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_max/window_out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + MAX(`int64_col`) OVER () AS `bfcol_1` + FROM `bfcte_0` +) SELECT - MAX(`int64_col`) OVER () AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_max/window_partition_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_max/window_partition_out.sql index a35a64a8e5f..ac9b2df84e1 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_max/window_partition_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_max/window_partition_out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + MAX(`int64_col`) OVER (PARTITION BY `string_col`) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - MAX(`int64_col`) OVER (PARTITION BY `string_col`) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_mean/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_mean/out.sql index 94287fc432b..0b33d0b1d0a 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_mean/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_mean/out.sql @@ -1,23 +1,27 @@ WITH `bfcte_0` AS ( SELECT `bool_col`, - `int64_col`, `duration_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, `int64_col` AS `bfcol_6`, `bool_col` AS `bfcol_7`, `duration_col` AS `bfcol_8` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( + FROM `bfcte_0` +), `bfcte_2` AS ( SELECT AVG(`bfcol_6`) AS `bfcol_12`, AVG(CAST(`bfcol_7` AS INT64)) AS `bfcol_13`, CAST(FLOOR(AVG(`bfcol_8`)) AS INT64) AS `bfcol_14`, CAST(FLOOR(AVG(`bfcol_6`)) AS INT64) AS `bfcol_15` - FROM `bfcte_0` + FROM `bfcte_1` ) SELECT `bfcol_12` AS `int64_col`, `bfcol_13` AS `bool_col`, `bfcol_14` AS `duration_col`, `bfcol_15` AS `int64_col_w_floor` -FROM `bfcte_1` \ No newline at end of file +FROM `bfcte_2` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_mean/window_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_mean/window_out.sql index 3443cd2a680..fdb59809c31 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_mean/window_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_mean/window_out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AVG(`int64_col`) OVER () AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AVG(`int64_col`) OVER () AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_mean/window_partition_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_mean/window_partition_out.sql index b94b84ddb81..d96121e54da 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_mean/window_partition_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_mean/window_partition_out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AVG(`int64_col`) OVER (PARTITION BY `string_col`) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - AVG(`int64_col`) OVER (PARTITION BY `string_col`) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_median/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_median/out.sql index 7d1215163f8..bfe94622b31 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_median/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_median/out.sql @@ -3,7 +3,7 @@ WITH `bfcte_0` AS ( `date_col`, `int64_col`, `string_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT APPROX_QUANTILES(`int64_col`, 2)[OFFSET(1)] AS `bfcol_3`, diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_min/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_min/out.sql index 144c07d7010..0848313456e 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_min/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_min/out.sql @@ -1,7 +1,7 @@ WITH `bfcte_0` AS ( SELECT `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT MIN(`int64_col`) AS `bfcol_1` diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_min/window_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_min/window_out.sql index 031c19eff16..cbda2b7d581 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_min/window_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_min/window_out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + MIN(`int64_col`) OVER () AS `bfcol_1` + FROM `bfcte_0` +) SELECT - MIN(`int64_col`) OVER () AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_min/window_partition_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_min/window_partition_out.sql index 2de5bd5f717..d601832950e 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_min/window_partition_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_min/window_partition_out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + MIN(`int64_col`) OVER (PARTITION BY `string_col`) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - MIN(`int64_col`) OVER (PARTITION BY `string_col`) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_nunique/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_nunique/out.sql index e0cc1a2eac5..f0b54934b45 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_nunique/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_nunique/out.sql @@ -1,7 +1,7 @@ WITH `bfcte_0` AS ( SELECT `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT COUNT(DISTINCT `int64_col`) AS `bfcol_1` diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_pop_var/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_pop_var/out.sql index b855c791182..2d38311f45a 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_pop_var/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_pop_var/out.sql @@ -2,7 +2,7 @@ WITH `bfcte_0` AS ( SELECT `bool_col`, `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT VAR_POP(`int64_col`) AS `bfcol_4`, diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_pop_var/window_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_pop_var/window_out.sql index 3bfaedd3953..430da33e3c3 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_pop_var/window_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_pop_var/window_out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + VAR_POP(`int64_col`) OVER () AS `bfcol_1` + FROM `bfcte_0` +) SELECT - VAR_POP(`int64_col`) OVER () AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_product/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_product/out.sql index 33204f2ff56..94ca21988e9 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_product/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_product/out.sql @@ -1,7 +1,7 @@ WITH `bfcte_0` AS ( SELECT `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT CASE diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_product/window_partition_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_product/window_partition_out.sql index 532349d3599..c5f12f70093 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_product/window_partition_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_product/window_partition_out.sql @@ -1,16 +1,27 @@ -SELECT - CASE - WHEN LOGICAL_OR(`int64_col` = 0) OVER (PARTITION BY `string_col`) - THEN 0 - ELSE POWER( - 2, - SUM(IF(`int64_col` = 0, 0, LOG(ABS(`int64_col`), 2))) OVER (PARTITION BY `string_col`) - ) * POWER( - -1, - MOD( - SUM(CASE WHEN SIGN(`int64_col`) = -1 THEN 1 ELSE 0 END) OVER (PARTITION BY `string_col`), - 2 +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN LOGICAL_OR(`int64_col` = 0) OVER (PARTITION BY `string_col`) + THEN 0 + ELSE POWER( + 2, + SUM(IF(`int64_col` = 0, 0, LOG(ABS(`int64_col`), 2))) OVER (PARTITION BY `string_col`) + ) * POWER( + -1, + MOD( + SUM(CASE WHEN SIGN(`int64_col`) = -1 THEN 1 ELSE 0 END) OVER (PARTITION BY `string_col`), + 2 + ) ) - ) - END AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + END AS `bfcol_2` + FROM `bfcte_0` +) +SELECT + `bfcol_2` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_qcut/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_qcut/out.sql index cb1541d083b..1aa2e436caa 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_qcut/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_qcut/out.sql @@ -1,51 +1,61 @@ -SELECT - `rowindex`, - `int64_col`, - IF( - ( - `int64_col` - ) IS NOT NULL, +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + NOT `int64_col` IS NULL AS `bfcol_4` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, IF( `int64_col` IS NULL, NULL, CAST(GREATEST( - CEIL( - PERCENT_RANK() OVER (PARTITION BY ( - `int64_col` - ) IS NOT NULL ORDER BY `int64_col` ASC) * 4 - ) - 1, + CEIL(PERCENT_RANK() OVER (PARTITION BY `bfcol_4` ORDER BY `int64_col` ASC) * 4) - 1, 0 ) AS INT64) - ), - NULL - ) AS `qcut_w_int`, - IF( - ( - `int64_col` - ) IS NOT NULL, + ) AS `bfcol_5` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + IF(`bfcol_4`, `bfcol_5`, NULL) AS `bfcol_6` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + NOT `int64_col` IS NULL AS `bfcol_10` + FROM `bfcte_3` +), `bfcte_5` AS ( + SELECT + *, CASE - WHEN PERCENT_RANK() OVER (PARTITION BY ( - `int64_col` - ) IS NOT NULL ORDER BY `int64_col` ASC) < 0 + WHEN PERCENT_RANK() OVER (PARTITION BY `bfcol_10` ORDER BY `int64_col` ASC) < 0 THEN NULL - WHEN PERCENT_RANK() OVER (PARTITION BY ( - `int64_col` - ) IS NOT NULL ORDER BY `int64_col` ASC) <= 0.25 + WHEN PERCENT_RANK() OVER (PARTITION BY `bfcol_10` ORDER BY `int64_col` ASC) <= 0.25 THEN 0 - WHEN PERCENT_RANK() OVER (PARTITION BY ( - `int64_col` - ) IS NOT NULL ORDER BY `int64_col` ASC) <= 0.5 + WHEN PERCENT_RANK() OVER (PARTITION BY `bfcol_10` ORDER BY `int64_col` ASC) <= 0.5 THEN 1 - WHEN PERCENT_RANK() OVER (PARTITION BY ( - `int64_col` - ) IS NOT NULL ORDER BY `int64_col` ASC) <= 0.75 + WHEN PERCENT_RANK() OVER (PARTITION BY `bfcol_10` ORDER BY `int64_col` ASC) <= 0.75 THEN 2 - WHEN PERCENT_RANK() OVER (PARTITION BY ( - `int64_col` - ) IS NOT NULL ORDER BY `int64_col` ASC) <= 1 + WHEN PERCENT_RANK() OVER (PARTITION BY `bfcol_10` ORDER BY `int64_col` ASC) <= 1 THEN 3 ELSE NULL - END, - NULL - ) AS `qcut_w_list` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + END AS `bfcol_11` + FROM `bfcte_4` +), `bfcte_6` AS ( + SELECT + *, + IF(`bfcol_10`, `bfcol_11`, NULL) AS `bfcol_12` + FROM `bfcte_5` +) +SELECT + `rowindex`, + `int64_col`, + `bfcol_6` AS `qcut_w_int`, + `bfcol_12` AS `qcut_w_list` +FROM `bfcte_6` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_quantile/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_quantile/out.sql index 656d01ea2e5..e337356d965 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_quantile/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_quantile/out.sql @@ -2,7 +2,7 @@ WITH `bfcte_0` AS ( SELECT `bool_col`, `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT PERCENTILE_CONT(`int64_col`, 0.5) OVER () AS `bfcol_4`, diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_rank/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_rank/out.sql index 2170d6cdcf7..96b121bde49 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_rank/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_rank/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + RANK() OVER (ORDER BY `int64_col` DESC NULLS FIRST) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - RANK() OVER (ORDER BY `int64_col` DESC NULLS FIRST) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/lag.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/lag.sql index 2bea343497f..7d1d62f1ae4 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/lag.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/lag.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + LAG(`int64_col`, 1) OVER (ORDER BY `int64_col` ASC) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - LAG(`int64_col`, 1) OVER (ORDER BY `int64_col` ASC) AS `lag` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `lag` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/lead.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/lead.sql index 5055f443718..67b40c99db0 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/lead.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/lead.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + LEAD(`int64_col`, 1) OVER (ORDER BY `int64_col` ASC) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - LEAD(`int64_col`, 1) OVER (ORDER BY `int64_col` ASC) AS `lead` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `lead` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/noop.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/noop.sql index 65af6af7c79..0202cf5c214 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/noop.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_shift/noop.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `int64_col` AS `bfcol_1` + FROM `bfcte_0` +) SELECT - `int64_col` AS `noop` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `noop` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_std/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_std/out.sql index e3c3d7b5253..36a50302a66 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_std/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_std/out.sql @@ -1,23 +1,27 @@ WITH `bfcte_0` AS ( SELECT `bool_col`, - `int64_col`, `duration_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, `int64_col` AS `bfcol_6`, `bool_col` AS `bfcol_7`, `duration_col` AS `bfcol_8` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( + FROM `bfcte_0` +), `bfcte_2` AS ( SELECT STDDEV(`bfcol_6`) AS `bfcol_12`, STDDEV(CAST(`bfcol_7` AS INT64)) AS `bfcol_13`, CAST(FLOOR(STDDEV(`bfcol_8`)) AS INT64) AS `bfcol_14`, CAST(FLOOR(STDDEV(`bfcol_6`)) AS INT64) AS `bfcol_15` - FROM `bfcte_0` + FROM `bfcte_1` ) SELECT `bfcol_12` AS `int64_col`, `bfcol_13` AS `bool_col`, `bfcol_14` AS `duration_col`, `bfcol_15` AS `int64_col_w_floor` -FROM `bfcte_1` \ No newline at end of file +FROM `bfcte_2` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_std/window_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_std/window_out.sql index 225dd5acf66..80e0cf5bc62 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_std/window_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_std/window_out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + STDDEV(`int64_col`) OVER () AS `bfcol_1` + FROM `bfcte_0` +) SELECT - STDDEV(`int64_col`) OVER () AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_sum/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_sum/out.sql index c67eef9da34..2bf6c26cd4b 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_sum/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_sum/out.sql @@ -2,7 +2,7 @@ WITH `bfcte_0` AS ( SELECT `bool_col`, `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT COALESCE(SUM(`int64_col`), 0) AS `bfcol_4`, diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_sum/window_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_sum/window_out.sql index ea5a12edfb5..47426abcbd0 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_sum/window_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_sum/window_out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + COALESCE(SUM(`int64_col`) OVER (), 0) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - COALESCE(SUM(`int64_col`) OVER (), 0) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_sum/window_partition_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_sum/window_partition_out.sql index ec6083b1a9d..fd1bd4f630d 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_sum/window_partition_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_sum/window_partition_out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + COALESCE(SUM(`int64_col`) OVER (PARTITION BY `string_col`), 0) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - COALESCE(SUM(`int64_col`) OVER (PARTITION BY `string_col`), 0) AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_var/out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_var/out.sql index b35d67c1ce1..733a22438ce 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_var/out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_var/out.sql @@ -2,7 +2,7 @@ WITH `bfcte_0` AS ( SELECT `bool_col`, `int64_col` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_1` AS ( SELECT VARIANCE(`int64_col`) AS `bfcol_4`, diff --git a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_var/window_out.sql b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_var/window_out.sql index e33797d02fb..e9d6c1cb932 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_var/window_out.sql +++ b/tests/unit/core/compile/sqlglot/aggregations/snapshots/test_unary_compiler/test_var/window_out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + VARIANCE(`int64_col`) OVER () AS `bfcol_1` + FROM `bfcte_0` +) SELECT - VARIANCE(`int64_col`) OVER () AS `agg_int64` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `agg_int64` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/aggregations/test_binary_compiler.py b/tests/unit/core/compile/sqlglot/aggregations/test_binary_compiler.py index 11f5cd6bad8..0897b535bee 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/test_binary_compiler.py +++ b/tests/unit/core/compile/sqlglot/aggregations/test_binary_compiler.py @@ -16,10 +16,10 @@ import pytest -import bigframes.pandas as bpd from bigframes.core import agg_expressions as agg_exprs from bigframes.core import array_value, identifiers, nodes from bigframes.operations import aggregations as agg_ops +import bigframes.pandas as bpd pytest.importorskip("pytest_snapshot") diff --git a/tests/unit/core/compile/sqlglot/aggregations/test_nullary_compiler.py b/tests/unit/core/compile/sqlglot/aggregations/test_nullary_compiler.py index 0ce8437b904..f9ddf3e0c08 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/test_nullary_compiler.py +++ b/tests/unit/core/compile/sqlglot/aggregations/test_nullary_compiler.py @@ -16,10 +16,10 @@ import pytest -import bigframes.pandas as bpd from bigframes.core import agg_expressions as agg_exprs from bigframes.core import array_value, identifiers, nodes, ordering, window_spec from bigframes.operations import aggregations as agg_ops +import bigframes.pandas as bpd pytest.importorskip("pytest_snapshot") diff --git a/tests/unit/core/compile/sqlglot/aggregations/test_op_registration.py b/tests/unit/core/compile/sqlglot/aggregations/test_op_registration.py index 9306bbf6559..c6c1c211510 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/test_op_registration.py +++ b/tests/unit/core/compile/sqlglot/aggregations/test_op_registration.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest from bigframes_vendored.sqlglot import expressions as sge +import pytest from bigframes.core.compile.sqlglot.aggregations import op_registration from bigframes.operations import aggregations as agg_ops @@ -42,23 +42,3 @@ def test_func(input: sge.Expression) -> sge.Expression: ValueError, match=r".*first parameter must be a window operator.*" ): test_func(sge.to_identifier("A")) - - -def test_register_already_registered_raise_error(): - reg = op_registration.OpRegistration() - - @reg.register(agg_ops.SizeOp) - def test_func1(op, input): - return input - - with pytest.raises(ValueError, match=r".*is already registered.*"): - - @reg.register(agg_ops.SizeOp) - def test_func2(op, input): - return input - - -def test_getitem_not_registered_raise_error(): - reg = op_registration.OpRegistration() - with pytest.raises(ValueError, match=r".*is not registered.*"): - _ = reg[agg_ops.SizeOp()] diff --git a/tests/unit/core/compile/sqlglot/aggregations/test_ordered_unary_compiler.py b/tests/unit/core/compile/sqlglot/aggregations/test_ordered_unary_compiler.py index dd8912a452b..d3a36866f0a 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/test_ordered_unary_compiler.py +++ b/tests/unit/core/compile/sqlglot/aggregations/test_ordered_unary_compiler.py @@ -16,10 +16,10 @@ import pytest -import bigframes.pandas as bpd from bigframes.core import agg_expressions as agg_exprs from bigframes.core import array_value, identifiers, nodes, ordering from bigframes.operations import aggregations as agg_ops +import bigframes.pandas as bpd pytest.importorskip("pytest_snapshot") diff --git a/tests/unit/core/compile/sqlglot/aggregations/test_unary_compiler.py b/tests/unit/core/compile/sqlglot/aggregations/test_unary_compiler.py index 7c827cd6dc1..d9bfb1f5f3d 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/test_unary_compiler.py +++ b/tests/unit/core/compile/sqlglot/aggregations/test_unary_compiler.py @@ -16,7 +16,6 @@ import pytest -import bigframes.pandas as bpd from bigframes.core import agg_expressions as agg_exprs from bigframes.core import ( array_value, @@ -27,6 +26,7 @@ window_spec, ) from bigframes.operations import aggregations as agg_ops +import bigframes.pandas as bpd pytest.importorskip("pytest_snapshot") diff --git a/tests/unit/core/compile/sqlglot/aggregations/test_windows.py b/tests/unit/core/compile/sqlglot/aggregations/test_windows.py index 98d0452c9a7..d1204c60104 100644 --- a/tests/unit/core/compile/sqlglot/aggregations/test_windows.py +++ b/tests/unit/core/compile/sqlglot/aggregations/test_windows.py @@ -18,15 +18,15 @@ import pandas as pd import pytest -import bigframes.core.expression as ex -import bigframes.core.identifiers as ids -import bigframes.core.ordering as ordering from bigframes import dtypes from bigframes.core import window_spec from bigframes.core.compile.sqlglot.aggregations.windows import ( apply_window_if_present, get_window_order_by, ) +import bigframes.core.expression as ex +import bigframes.core.identifiers as ids +import bigframes.core.ordering as ordering class WindowsTest(unittest.TestCase): diff --git a/tests/unit/core/compile/sqlglot/conftest.py b/tests/unit/core/compile/sqlglot/conftest.py new file mode 100644 index 00000000000..cb5a14b690e --- /dev/null +++ b/tests/unit/core/compile/sqlglot/conftest.py @@ -0,0 +1,280 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pathlib +import typing + +from google.cloud import bigquery +import pandas as pd +import pyarrow as pa +import pytest + +from bigframes import dtypes +import bigframes.core as core +import bigframes.pandas as bpd +import bigframes.testing.mocks as mocks +import bigframes.testing.utils + +CURRENT_DIR = pathlib.Path(__file__).parent +DATA_DIR = CURRENT_DIR.parent.parent.parent.parent / "data" + + +def _create_compiler_session(table_name, table_schema): + """Helper function to create a compiler session.""" + from bigframes.testing import compiler_session + + anonymous_dataset = bigquery.DatasetReference.from_string( + "bigframes-dev.sqlglot_test" + ) + session = mocks.create_bigquery_session( + table_name=table_name, + table_schema=table_schema, + anonymous_dataset=anonymous_dataset, + ) + session._executor = compiler_session.SQLCompilerExecutor() + return session + + +@pytest.fixture(scope="session") +def compiler_session(scalar_types_table_schema): + """Compiler session for scalar types.""" + return _create_compiler_session("scalar_types", scalar_types_table_schema) + + +@pytest.fixture(scope="session") +def compiler_session_w_repeated_types(repeated_types_table_schema): + """Compiler session for repeated data types.""" + return _create_compiler_session("repeated_types", repeated_types_table_schema) + + +@pytest.fixture(scope="session") +def compiler_session_w_nested_structs_types(nested_structs_types_table_schema): + """Compiler session for nested STRUCT data types.""" + return _create_compiler_session( + "nested_structs_types", nested_structs_types_table_schema + ) + + +@pytest.fixture(scope="session") +def compiler_session_w_json_types(json_types_table_schema): + """Compiler session for JSON data types.""" + return _create_compiler_session("json_types", json_types_table_schema) + + +@pytest.fixture(scope="session") +def scalar_types_table_schema() -> typing.Sequence[bigquery.SchemaField]: + return [ + bigquery.SchemaField("bool_col", "BOOLEAN"), + bigquery.SchemaField("bytes_col", "BYTES"), + bigquery.SchemaField("date_col", "DATE"), + bigquery.SchemaField("datetime_col", "DATETIME"), + bigquery.SchemaField("geography_col", "GEOGRAPHY"), + bigquery.SchemaField("int64_col", "INTEGER"), + bigquery.SchemaField("int64_too", "INTEGER"), + bigquery.SchemaField("numeric_col", "NUMERIC"), + bigquery.SchemaField("float64_col", "FLOAT"), + bigquery.SchemaField("rowindex", "INTEGER"), + bigquery.SchemaField("rowindex_2", "INTEGER", mode="REQUIRED"), + bigquery.SchemaField("string_col", "STRING"), + bigquery.SchemaField("time_col", "TIME"), + bigquery.SchemaField("timestamp_col", "TIMESTAMP"), + bigquery.SchemaField("duration_col", "INTEGER"), + ] + + +@pytest.fixture(scope="session") +def scalar_types_df(compiler_session) -> bpd.DataFrame: + """Returns a BigFrames DataFrame containing all scalar types and using the `rowindex` + column as the index.""" + bf_df = compiler_session._loader.read_gbq_table( + "bigframes-dev.sqlglot_test.scalar_types", + enable_snapshot=False, + ) + bf_df = bf_df.set_index("rowindex", drop=False) + return bf_df + + +@pytest.fixture(scope="session") +def scalar_types_pandas_df() -> pd.DataFrame: + """Returns a pandas DataFrame containing all scalar types and using the `rowindex` + column as the index.""" + # TODO: add tests for empty dataframes + df = pd.read_json( + DATA_DIR / "scalars.jsonl", + lines=True, + ) + bigframes.testing.utils.convert_pandas_dtypes(df, bytes_col=True) + + df = df.set_index("rowindex", drop=False) + return df + + +@pytest.fixture(scope="module") +def scalar_types_array_value( + scalar_types_pandas_df: pd.DataFrame, compiler_session: bigframes.Session +) -> core.ArrayValue: + managed_data_source = core.local_data.ManagedArrowTable.from_pandas( + scalar_types_pandas_df + ) + return core.ArrayValue.from_managed(managed_data_source, compiler_session) + + +@pytest.fixture(scope="session") +def nested_structs_types_table_schema() -> typing.Sequence[bigquery.SchemaField]: + return [ + bigquery.SchemaField("id", "INTEGER"), + bigquery.SchemaField( + "people", + "RECORD", + fields=[ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("age", "INTEGER"), + bigquery.SchemaField( + "address", + "RECORD", + fields=[ + bigquery.SchemaField("city", "STRING"), + bigquery.SchemaField("country", "STRING"), + ], + ), + ], + ), + ] + + +@pytest.fixture(scope="session") +def nested_structs_types_df(compiler_session_w_nested_structs_types) -> bpd.DataFrame: + """Returns a BigFrames DataFrame containing all scalar types and using the `rowindex` + column as the index.""" + bf_df = compiler_session_w_nested_structs_types._loader.read_gbq_table( + "bigframes-dev.sqlglot_test.nested_structs_types", + enable_snapshot=False, + ) + bf_df = bf_df.set_index("id", drop=False) + return bf_df + + +@pytest.fixture(scope="session") +def nested_structs_pandas_df() -> pd.DataFrame: + """Returns a pandas DataFrame containing STRUCT types and using the `id` + column as the index.""" + + df = pd.read_json( + DATA_DIR / "nested_structs.jsonl", + lines=True, + ) + df = df.set_index("id") + + address_struct_schema = pa.struct( + [pa.field("city", pa.string()), pa.field("country", pa.string())] + ) + person_struct_schema = pa.struct( + [ + pa.field("name", pa.string()), + pa.field("age", pa.int64()), + pa.field("address", address_struct_schema), + ] + ) + df["person"] = df["person"].astype(pd.ArrowDtype(person_struct_schema)) + return df + + +@pytest.fixture(scope="session") +def repeated_types_table_schema() -> typing.Sequence[bigquery.SchemaField]: + return [ + bigquery.SchemaField("rowindex", "INTEGER"), + bigquery.SchemaField("int_list_col", "INTEGER", "REPEATED"), + bigquery.SchemaField("bool_list_col", "BOOLEAN", "REPEATED"), + bigquery.SchemaField("float_list_col", "FLOAT", "REPEATED"), + bigquery.SchemaField("date_list_col", "DATE", "REPEATED"), + bigquery.SchemaField("date_time_list_col", "DATETIME", "REPEATED"), + bigquery.SchemaField("numeric_list_col", "NUMERIC", "REPEATED"), + bigquery.SchemaField("string_list_col", "STRING", "REPEATED"), + ] + + +@pytest.fixture(scope="session") +def repeated_types_df(compiler_session_w_repeated_types) -> bpd.DataFrame: + """Returns a BigFrames DataFrame containing all scalar types and using the `rowindex` + column as the index.""" + bf_df = compiler_session_w_repeated_types._loader.read_gbq_table( + "bigframes-dev.sqlglot_test.repeated_types", + enable_snapshot=False, + ) + bf_df = bf_df.set_index("rowindex", drop=False) + return bf_df + + +@pytest.fixture(scope="session") +def repeated_types_pandas_df() -> pd.DataFrame: + """Returns a pandas DataFrame containing LIST types and using the `rowindex` + column as the index.""" + + df = pd.read_json( + DATA_DIR / "repeated.jsonl", + lines=True, + ) + # TODO: add dtype conversion here if needed. + df = df.set_index("rowindex") + return df + + +@pytest.fixture(scope="session") +def json_types_table_schema() -> typing.Sequence[bigquery.SchemaField]: + return [ + bigquery.SchemaField("rowindex", "INTEGER"), + bigquery.SchemaField("json_col", "JSON"), + ] + + +@pytest.fixture(scope="session") +def json_types_df(compiler_session_w_json_types) -> bpd.DataFrame: + """Returns a BigFrames DataFrame containing JSON types and using the `rowindex` + column as the index.""" + bf_df = compiler_session_w_json_types._loader.read_gbq_table( + "bigframes-dev.sqlglot_test.json_types", + enable_snapshot=False, + ) + # TODO(b/427305807): Why `drop=False` will produce two "rowindex" columns? + bf_df = bf_df.set_index("rowindex", drop=True) + return bf_df + + +@pytest.fixture(scope="session") +def json_pandas_df() -> pd.DataFrame: + """Returns a pandas DataFrame containing JSON types and using the `rowindex` + column as the index.""" + json_data = [ + "null", + "true", + "100", + "0.98", + '"a string"', + "[]", + "[1, 2, 3]", + '[{"a": 1}, {"a": 2}, {"a": null}, {}]', + '"100"', + '{"date": "2024-07-16"}', + '{"int_value": 2, "null_filed": null}', + '{"list_data": [10, 20, 30]}', + ] + df = pd.DataFrame( + { + "rowindex": pd.Series(range(len(json_data)), dtype=dtypes.INT_DTYPE), + "json_col": pd.Series(json_data, dtype=dtypes.JSON_DTYPE), + }, + ) + # TODO(b/427305807): Why `drop=False` will produce two "rowindex" columns? + df = df.set_index("rowindex", drop=True) + return df diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify/None/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify/None/out.sql deleted file mode 100644 index fc29d96cc1a..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify/None/out.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT - AI.CLASSIFY(input => STRUCT(`string_col`), categories => ['greeting', 'rejection']) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify/bigframes-dev.us.bigframes-default-connection/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify/bigframes-dev.us.bigframes-default-connection/out.sql deleted file mode 100644 index 969b946725b..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify/bigframes-dev.us.bigframes-default-connection/out.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT - AI.CLASSIFY( - input => STRUCT(`string_col`), - categories => ['greeting', 'rejection'], - connection_id => 'bigframes-dev.us.bigframes-default-connection' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify/out.sql new file mode 100644 index 00000000000..a40784a3ca5 --- /dev/null +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify/out.sql @@ -0,0 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.CLASSIFY( + input => (`string_col`), + categories => ['greeting', 'rejection'], + connection_id => 'bigframes-dev.us.bigframes-default-connection' + ) AS `bfcol_1` + FROM `bfcte_0` +) +SELECT + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify_multi_with_list_examples/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify_multi_with_list_examples/out.sql deleted file mode 100644 index 74078e98606..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify_multi_with_list_examples/out.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - AI.CLASSIFY( - input => STRUCT(`string_col`), - categories => ['greeting', 'rejection'], - examples => [('hi', ['greeting', 'positive']), ('bye', ['rejection', 'negative'])], - output_mode => 'multi' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify_with_output_mode/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify_with_output_mode/out.sql deleted file mode 100644 index 08d7476d77f..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify_with_output_mode/out.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT - AI.CLASSIFY( - input => STRUCT(`string_col`), - categories => ['greeting', 'rejection'], - output_mode => 'multi' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify_with_params/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify_with_params/out.sql deleted file mode 100644 index 30542740a2d..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_classify_with_params/out.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - AI.CLASSIFY( - input => STRUCT(`string_col`), - categories => ['greeting', 'rejection'], - examples => [('hi', 'greeting'), ('bye', 'rejection')], - endpoint => 'gemini-2.5-flash', - max_error_ratio => 0.1 - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed/out.sql deleted file mode 100644 index 9c18a7cd532..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed/out.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT - AI.EMBED(`string_col`, endpoint => 'text-embedding-005') AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed_with_connection_id/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed_with_connection_id/out.sql deleted file mode 100644 index 0968a101b22..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed_with_connection_id/out.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT - AI.EMBED( - `string_col`, - endpoint => 'text-embedding-005', - connection_id => 'bigframes-dev.us.bigframes-default-connection' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed_with_model/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed_with_model/out.sql deleted file mode 100644 index 4c3c76f87b6..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed_with_model/out.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT - AI.EMBED(`string_col`, model => 'embeddinggemma-300m') AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed_with_model_param_and_title/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed_with_model_param_and_title/out.sql deleted file mode 100644 index 873db838682..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed_with_model_param_and_title/out.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - AI.EMBED( - `string_col`, - endpoint => 'text-embedding-005', - task_type => 'retrieval_document', - title => 'My Document', - model_params => JSON '{"outputDimensionality": 256}' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed_with_task_type_and_title/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed_with_task_type_and_title/out.sql deleted file mode 100644 index 9e4db995871..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_embed_with_task_type_and_title/out.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - AI.EMBED( - `string_col`, - endpoint => 'text-embedding-005', - task_type => 'RETRIEVAL_DOCUMENT', - title => 'My Document', - model_params => JSON '{"outputDimensionality": 256}' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate/out.sql index 622782fa7d6..ec3515e7ed7 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate/out.sql @@ -1,7 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE( + prompt => (`string_col`, ' is the same as ', `string_col`), + endpoint => 'gemini-2.5-flash', + request_type => 'SHARED' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - endpoint => 'gemini-2.5-flash', - request_type => 'SHARED' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_bool/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_bool/out.sql index a71bce037a5..3a09da7c3a2 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_bool/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_bool/out.sql @@ -1,6 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE_BOOL( + prompt => (`string_col`, ' is the same as ', `string_col`), + endpoint => 'gemini-2.5-flash', + request_type => 'SHARED' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE_BOOL( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - endpoint => 'gemini-2.5-flash' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_bool_with_connection_id/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_bool_with_connection_id/out.sql index db1ec378aaf..f844ed16918 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_bool_with_connection_id/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_bool_with_connection_id/out.sql @@ -1,7 +1,18 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE_BOOL( + prompt => (`string_col`, ' is the same as ', `string_col`), + connection_id => 'bigframes-dev.us.bigframes-default-connection', + endpoint => 'gemini-2.5-flash', + request_type => 'SHARED' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE_BOOL( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - connection_id => 'bigframes-dev.us.bigframes-default-connection', - endpoint => 'gemini-2.5-flash' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_bool_with_model_param/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_bool_with_model_param/out.sql index 76af8833e63..2a81ced7823 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_bool_with_model_param/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_bool_with_model_param/out.sql @@ -1,6 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE_BOOL( + prompt => (`string_col`, ' is the same as ', `string_col`), + request_type => 'SHARED', + model_params => JSON '{}' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE_BOOL( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - model_params => JSON '{}' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_double/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_double/out.sql index 1cef7568798..3b894296210 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_double/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_double/out.sql @@ -1,6 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE_DOUBLE( + prompt => (`string_col`, ' is the same as ', `string_col`), + endpoint => 'gemini-2.5-flash', + request_type => 'SHARED' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE_DOUBLE( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - endpoint => 'gemini-2.5-flash' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_double_with_connection_id/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_double_with_connection_id/out.sql index d0088721e38..fae92515cbe 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_double_with_connection_id/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_double_with_connection_id/out.sql @@ -1,7 +1,18 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE_DOUBLE( + prompt => (`string_col`, ' is the same as ', `string_col`), + connection_id => 'bigframes-dev.us.bigframes-default-connection', + endpoint => 'gemini-2.5-flash', + request_type => 'SHARED' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE_DOUBLE( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - connection_id => 'bigframes-dev.us.bigframes-default-connection', - endpoint => 'gemini-2.5-flash' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_double_with_model_param/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_double_with_model_param/out.sql index 2b50e05b7fe..480ee09ef65 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_double_with_model_param/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_double_with_model_param/out.sql @@ -1,6 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE_DOUBLE( + prompt => (`string_col`, ' is the same as ', `string_col`), + request_type => 'SHARED', + model_params => JSON '{}' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE_DOUBLE( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - model_params => JSON '{}' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_int/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_int/out.sql index 9ef143c8b9e..f33af547c7f 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_int/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_int/out.sql @@ -1,6 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE_INT( + prompt => (`string_col`, ' is the same as ', `string_col`), + endpoint => 'gemini-2.5-flash', + request_type => 'SHARED' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE_INT( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - endpoint => 'gemini-2.5-flash' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_int_with_connection_id/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_int_with_connection_id/out.sql index 3fa3e8cc05e..a0c92c959c2 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_int_with_connection_id/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_int_with_connection_id/out.sql @@ -1,7 +1,18 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE_INT( + prompt => (`string_col`, ' is the same as ', `string_col`), + connection_id => 'bigframes-dev.us.bigframes-default-connection', + endpoint => 'gemini-2.5-flash', + request_type => 'SHARED' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE_INT( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - connection_id => 'bigframes-dev.us.bigframes-default-connection', - endpoint => 'gemini-2.5-flash' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_int_with_model_param/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_int_with_model_param/out.sql index 18adea8a062..2929e57ba0c 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_int_with_model_param/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_int_with_model_param/out.sql @@ -1,6 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE_INT( + prompt => (`string_col`, ' is the same as ', `string_col`), + request_type => 'SHARED', + model_params => JSON '{}' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE_INT( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - model_params => JSON '{}' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_with_connection_id/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_with_connection_id/out.sql index 14604cfc8df..19f85b181b2 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_with_connection_id/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_with_connection_id/out.sql @@ -1,7 +1,18 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE( + prompt => (`string_col`, ' is the same as ', `string_col`), + connection_id => 'bigframes-dev.us.bigframes-default-connection', + endpoint => 'gemini-2.5-flash', + request_type => 'SHARED' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - connection_id => 'bigframes-dev.us.bigframes-default-connection', - endpoint => 'gemini-2.5-flash' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_with_model_param/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_with_model_param/out.sql index 090a42d889f..745243db3a0 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_with_model_param/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_with_model_param/out.sql @@ -1,6 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE( + prompt => (`string_col`, ' is the same as ', `string_col`), + request_type => 'SHARED', + model_params => JSON '{}' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - model_params => JSON '{}' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_with_output_schema/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_with_output_schema/out.sql index 31c179e7b01..4f7867a0f20 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_with_output_schema/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_generate_with_output_schema/out.sql @@ -1,7 +1,18 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.GENERATE( + prompt => (`string_col`, ' is the same as ', `string_col`), + endpoint => 'gemini-2.5-flash', + request_type => 'SHARED', + output_schema => 'x INT64, y FLOAT64' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - AI.GENERATE( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - endpoint => 'gemini-2.5-flash', - output_schema => 'x INT64, y FLOAT64' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_if/None/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_if/None/out.sql deleted file mode 100644 index 59cf1c02a35..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_if/None/out.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT - AI.IF( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - optimization_mode => 'MINIMIZE_COST', - max_error_ratio => 0.5 - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_if/bigframes-dev.us.bigframes-default-connection/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_if/bigframes-dev.us.bigframes-default-connection/out.sql deleted file mode 100644 index 0f26ab3c6ea..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_if/bigframes-dev.us.bigframes-default-connection/out.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - AI.IF( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - connection_id => 'bigframes-dev.us.bigframes-default-connection', - optimization_mode => 'MINIMIZE_COST', - max_error_ratio => 0.5 - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_if/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_if/out.sql new file mode 100644 index 00000000000..275ba8d4239 --- /dev/null +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_if/out.sql @@ -0,0 +1,16 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.IF( + prompt => (`string_col`, ' is the same as ', `string_col`), + connection_id => 'bigframes-dev.us.bigframes-default-connection' + ) AS `bfcol_1` + FROM `bfcte_0` +) +SELECT + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_if_with_endpoint/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_if_with_endpoint/out.sql deleted file mode 100644 index 4dd910528a4..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_if_with_endpoint/out.sql +++ /dev/null @@ -1,6 +0,0 @@ -SELECT - AI.IF( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - endpoint => 'gemini-2.5-flash' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_score/None/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_score/None/out.sql deleted file mode 100644 index 37590eec4f0..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_score/None/out.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT - AI.SCORE(prompt => STRUCT(`string_col`, ' is the same as ', `string_col`)) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_score/bigframes-dev.us.bigframes-default-connection/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_score/bigframes-dev.us.bigframes-default-connection/out.sql deleted file mode 100644 index 696c7e9f318..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_score/bigframes-dev.us.bigframes-default-connection/out.sql +++ /dev/null @@ -1,6 +0,0 @@ -SELECT - AI.SCORE( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - connection_id => 'bigframes-dev.us.bigframes-default-connection' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_score/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_score/out.sql new file mode 100644 index 00000000000..01c71065b92 --- /dev/null +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_score/out.sql @@ -0,0 +1,16 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + AI.SCORE( + prompt => (`string_col`, ' is the same as ', `string_col`), + connection_id => 'bigframes-dev.us.bigframes-default-connection' + ) AS `bfcol_1` + FROM `bfcte_0` +) +SELECT + `bfcol_1` AS `result` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_score_with_endpoint_and_max_error_ratio/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_score_with_endpoint_and_max_error_ratio/out.sql deleted file mode 100644 index a802e5a396b..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_score_with_endpoint_and_max_error_ratio/out.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT - AI.SCORE( - prompt => STRUCT(`string_col`, ' is the same as ', `string_col`), - endpoint => 'gemini-2.5-flash', - max_error_ratio => 0.5 - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_similarity/None/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_similarity/None/out.sql deleted file mode 100644 index 1df70aaf18e..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_similarity/None/out.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT - AI.SIMILARITY(content1 => `string_col`, content2 => `string_col`, endpoint => 'text-embedding-005') AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_similarity/bigframes-dev.us.bigframes-default-connection/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_similarity/bigframes-dev.us.bigframes-default-connection/out.sql deleted file mode 100644 index db57188ffa0..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_similarity/bigframes-dev.us.bigframes-default-connection/out.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - AI.SIMILARITY( - content1 => `string_col`, - content2 => `string_col`, - endpoint => 'text-embedding-005', - connection_id => 'bigframes-dev.us.bigframes-default-connection' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_similarity_with_model/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_similarity_with_model/out.sql deleted file mode 100644 index 704f9f94491..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_similarity_with_model/out.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT - AI.SIMILARITY(content1 => `string_col`, content2 => `string_col`, model => 'embeddinggemma-300m') AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_similarity_with_model_param/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_similarity_with_model_param/out.sql deleted file mode 100644 index 5173ac43bd9..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_ai_ops/test_ai_similarity_with_model_param/out.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - AI.SIMILARITY( - content1 => `string_col`, - content2 => `string_col`, - endpoint => 'text-embedding-005', - model_params => JSON '{"outputDimensionality": 256}' - ) AS `result` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_index/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_index/out.sql index a1f089424a1..d8e223d5f85 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_index/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_index/out.sql @@ -1,4 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_list_col` + FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` +), `bfcte_1` AS ( + SELECT + *, + `string_list_col`[SAFE_OFFSET(1)] AS `bfcol_1` + FROM `bfcte_0` +) SELECT - IF(SUBSTRING(`string_col`, 2, 1) <> '', SUBSTRING(`string_col`, 2, 1), NULL) AS `string_index`, - [`int64_col`, `int64_too`][SAFE_OFFSET(1)] AS `array_index` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_list_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_reduce_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_reduce_op/out.sql index 1053ec1c2c6..b9f87bfd1ed 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_reduce_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_reduce_op/out.sql @@ -1,27 +1,37 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_list_col`, + `float_list_col`, + `string_list_col` + FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` +), `bfcte_1` AS ( + SELECT + *, + ( + SELECT + COALESCE(SUM(bf_arr_reduce_uid), 0) + FROM UNNEST(`float_list_col`) AS bf_arr_reduce_uid + ) AS `bfcol_3`, + ( + SELECT + STDDEV(bf_arr_reduce_uid) + FROM UNNEST(`float_list_col`) AS bf_arr_reduce_uid + ) AS `bfcol_4`, + ( + SELECT + COUNT(bf_arr_reduce_uid) + FROM UNNEST(`string_list_col`) AS bf_arr_reduce_uid + ) AS `bfcol_5`, + ( + SELECT + COALESCE(LOGICAL_OR(bf_arr_reduce_uid), FALSE) + FROM UNNEST(`bool_list_col`) AS bf_arr_reduce_uid + ) AS `bfcol_6` + FROM `bfcte_0` +) SELECT - ( - SELECT - COALESCE(SUM(bf_arr_reduce_uid), 0) - FROM UNNEST(`float_list_col`) AS bf_arr_reduce_uid - ) AS `sum_float`, - ( - SELECT - STDDEV(bf_arr_reduce_uid) - FROM UNNEST(`float_list_col`) AS bf_arr_reduce_uid - ) AS `std_float`, - ( - SELECT - COUNT(bf_arr_reduce_uid) - FROM UNNEST(`string_list_col`) AS bf_arr_reduce_uid - ) AS `count_str`, - ( - SELECT - COALESCE(LOGICAL_OR(bf_arr_reduce_uid), FALSE) - FROM UNNEST(`bool_list_col`) AS bf_arr_reduce_uid - ) AS `any_bool`, - ( - SELECT - ARRAY_AGG(bf_arr_reduce_uid IGNORE NULLS) - FROM UNNEST(`string_list_col`) AS bf_arr_reduce_uid - ) AS `array_agg_str` -FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` AS `bft_0` \ No newline at end of file + `bfcol_3` AS `sum_float`, + `bfcol_4` AS `std_float`, + `bfcol_5` AS `count_str`, + `bfcol_6` AS `any_bool` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_slice/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_slice/out.sql deleted file mode 100644 index ffec3b8e934..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_slice/out.sql +++ /dev/null @@ -1,17 +0,0 @@ -SELECT - SUBSTRING(`string_col`, 2, 4) AS `string_slice`, - ARRAY( - SELECT - el - FROM UNNEST([`int64_col`, `int64_too`]) AS el WITH OFFSET AS slice_idx - WHERE - slice_idx >= 1 - ) AS `slice_only_start`, - ARRAY( - SELECT - el - FROM UNNEST([`int64_col`, `int64_too`]) AS el WITH OFFSET AS slice_idx - WHERE - slice_idx >= 1 AND slice_idx < 5 - ) AS `slice_start_stop` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_slice_with_only_start/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_slice_with_only_start/out.sql new file mode 100644 index 00000000000..0034ffd69cd --- /dev/null +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_slice_with_only_start/out.sql @@ -0,0 +1,19 @@ +WITH `bfcte_0` AS ( + SELECT + `string_list_col` + FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` +), `bfcte_1` AS ( + SELECT + *, + ARRAY( + SELECT + el + FROM UNNEST(`string_list_col`) AS el WITH OFFSET AS slice_idx + WHERE + slice_idx >= 1 + ) AS `bfcol_1` + FROM `bfcte_0` +) +SELECT + `bfcol_1` AS `string_list_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_slice_with_start_and_stop/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_slice_with_start_and_stop/out.sql new file mode 100644 index 00000000000..f0638fa3afc --- /dev/null +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_slice_with_start_and_stop/out.sql @@ -0,0 +1,19 @@ +WITH `bfcte_0` AS ( + SELECT + `string_list_col` + FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` +), `bfcte_1` AS ( + SELECT + *, + ARRAY( + SELECT + el + FROM UNNEST(`string_list_col`) AS el WITH OFFSET AS slice_idx + WHERE + slice_idx >= 1 AND slice_idx < 5 + ) AS `bfcol_1` + FROM `bfcte_0` +) +SELECT + `bfcol_1` AS `string_list_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_to_string/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_to_string/out.sql index 27587771506..09446bb8f51 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_to_string/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_array_to_string/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_list_col` + FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` +), `bfcte_1` AS ( + SELECT + *, + ARRAY_TO_STRING(`string_list_col`, '.') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ARRAY_TO_STRING(`string_list_col`, '.') AS `string_list_col` -FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_list_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_to_array_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_to_array_op/out.sql index f7d8d748b4a..3e297016584 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_to_array_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_to_array_op/out.sql @@ -1,10 +1,26 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `float64_col`, + `int64_col`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + [COALESCE(`bool_col`, FALSE)] AS `bfcol_8`, + [COALESCE(`int64_col`, 0)] AS `bfcol_9`, + [COALESCE(`string_col`, ''), COALESCE(`string_col`, '')] AS `bfcol_10`, + [ + COALESCE(`int64_col`, 0), + CAST(COALESCE(`bool_col`, FALSE) AS INT64), + COALESCE(`float64_col`, 0.0) + ] AS `bfcol_11` + FROM `bfcte_0` +) SELECT - [COALESCE(`bool_col`, FALSE)] AS `bool_col`, - [COALESCE(`int64_col`, 0)] AS `int64_col`, - [COALESCE(`string_col`, ''), COALESCE(`string_col`, '')] AS `strs_col`, - [ - COALESCE(`int64_col`, 0), - CAST(COALESCE(`bool_col`, FALSE) AS INT64), - COALESCE(`float64_col`, 0.0) - ] AS `numeric_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_8` AS `bool_col`, + `bfcol_9` AS `int64_col`, + `bfcol_10` AS `strs_col`, + `bfcol_11` AS `numeric_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_to_array_with_subquery_expression/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_to_array_with_subquery_expression/out.sql deleted file mode 100644 index 63dfcec026b..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_array_ops/test_to_array_with_subquery_expression/out.sql +++ /dev/null @@ -1,12 +0,0 @@ -SELECT - [ - COALESCE( - ( - SELECT - COALESCE(SUM(bf_arr_reduce_uid), 0) - FROM UNNEST(`float_list_col`) AS bf_arr_reduce_uid - ), - 0.0 - ) - ] AS `arr_subquery_coalesce` -FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_blob_ops/test_obj_fetch_metadata/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_blob_ops/test_obj_fetch_metadata/out.sql new file mode 100644 index 00000000000..bd99b860648 --- /dev/null +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_blob_ops/test_obj_fetch_metadata/out.sql @@ -0,0 +1,25 @@ +WITH `bfcte_0` AS ( + SELECT + `rowindex`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + OBJ.MAKE_REF(`string_col`, 'bigframes-dev.test-region.bigframes-default-connection') AS `bfcol_4` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + OBJ.FETCH_METADATA(`bfcol_4`) AS `bfcol_7` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_7`.`version` AS `bfcol_10` + FROM `bfcte_2` +) +SELECT + `rowindex`, + `bfcol_10` AS `version` +FROM `bfcte_3` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_blob_ops/test_obj_get_access_url/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_blob_ops/test_obj_get_access_url/out.sql new file mode 100644 index 00000000000..28c2f2ce181 --- /dev/null +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_blob_ops/test_obj_get_access_url/out.sql @@ -0,0 +1,25 @@ +WITH `bfcte_0` AS ( + SELECT + `rowindex`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + OBJ.MAKE_REF(`string_col`, 'bigframes-dev.test-region.bigframes-default-connection') AS `bfcol_4` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + OBJ.GET_ACCESS_URL(`bfcol_4`, 'R') AS `bfcol_7` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + JSON_VALUE(`bfcol_7`, '$.access_urls.read_url') AS `bfcol_10` + FROM `bfcte_2` +) +SELECT + `rowindex`, + `bfcol_10` AS `string_col` +FROM `bfcte_3` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_blob_ops/test_obj_make_ref/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_blob_ops/test_obj_make_ref/out.sql new file mode 100644 index 00000000000..d74449c986e --- /dev/null +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_blob_ops/test_obj_make_ref/out.sql @@ -0,0 +1,15 @@ +WITH `bfcte_0` AS ( + SELECT + `rowindex`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + OBJ.MAKE_REF(`string_col`, 'bigframes-dev.test-region.bigframes-default-connection') AS `bfcol_4` + FROM `bfcte_0` +) +SELECT + `rowindex`, + `bfcol_4` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_bool_ops/test_and_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_bool_ops/test_and_op/out.sql index d6f6587ead9..7e46e10708d 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_bool_ops/test_and_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_bool_ops/test_and_op/out.sql @@ -1,9 +1,42 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `bool_col` AS `bfcol_7`, + `int64_col` AS `bfcol_8`, + `int64_col` & `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_7` AND `bfcol_7` AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + IF(`bfcol_15` = FALSE, `bfcol_15`, NULL) AS `bfcol_29` + FROM `bfcte_2` +) SELECT - `rowindex`, - `bool_col`, - `int64_col`, - `int64_col` & `int64_col` AS `int_and_int`, - `bool_col` AND `bool_col` AS `bool_and_bool`, - IF(`bool_col` = FALSE, `bool_col`, NULL) AS `bool_and_null`, - IF(`bool_col` = FALSE, `bool_col`, NULL) AS `null_and_bool` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_24` AS `rowindex`, + `bfcol_25` AS `bool_col`, + `bfcol_26` AS `int64_col`, + `bfcol_27` AS `int_and_int`, + `bfcol_28` AS `bool_and_bool`, + `bfcol_29` AS `bool_and_null` +FROM `bfcte_3` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_bool_ops/test_or_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_bool_ops/test_or_op/out.sql index dad4cee9d0b..c8e9cf65a91 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_bool_ops/test_or_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_bool_ops/test_or_op/out.sql @@ -1,9 +1,42 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `bool_col` AS `bfcol_7`, + `int64_col` AS `bfcol_8`, + `int64_col` | `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_7` OR `bfcol_7` AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + IF(`bfcol_15` = TRUE, `bfcol_15`, NULL) AS `bfcol_29` + FROM `bfcte_2` +) SELECT - `rowindex`, - `bool_col`, - `int64_col`, - `int64_col` | `int64_col` AS `int_and_int`, - `bool_col` OR `bool_col` AS `bool_and_bool`, - IF(`bool_col` = TRUE, `bool_col`, NULL) AS `bool_and_null`, - IF(`bool_col` = TRUE, `bool_col`, NULL) AS `null_and_bool` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_24` AS `rowindex`, + `bfcol_25` AS `bool_col`, + `bfcol_26` AS `int64_col`, + `bfcol_27` AS `int_and_int`, + `bfcol_28` AS `bool_and_bool`, + `bfcol_29` AS `bool_and_null` +FROM `bfcte_3` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_bool_ops/test_xor_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_bool_ops/test_xor_op/out.sql index 4be3b9f94ad..d6a081cbbde 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_bool_ops/test_xor_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_bool_ops/test_xor_op/out.sql @@ -1,23 +1,51 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `bool_col` AS `bfcol_7`, + `int64_col` AS `bfcol_8`, + `int64_col` ^ `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + ( + `bfcol_7` AND NOT `bfcol_7` + ) OR ( + NOT `bfcol_7` AND `bfcol_7` + ) AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + ( + `bfcol_15` AND NOT CAST(NULL AS BOOLEAN) + ) + OR ( + NOT `bfcol_15` AND CAST(NULL AS BOOLEAN) + ) AS `bfcol_29` + FROM `bfcte_2` +) SELECT - `rowindex`, - `bool_col`, - `int64_col`, - `int64_col` ^ `int64_col` AS `int_and_int`, - ( - `bool_col` AND NOT `bool_col` - ) OR ( - NOT `bool_col` AND `bool_col` - ) AS `bool_and_bool`, - ( - `bool_col` AND NOT CAST(NULL AS BOOLEAN) - ) - OR ( - NOT `bool_col` AND CAST(NULL AS BOOLEAN) - ) AS `bool_and_null`, - ( - `bool_col` AND NOT CAST(NULL AS BOOLEAN) - ) - OR ( - NOT `bool_col` AND CAST(NULL AS BOOLEAN) - ) AS `null_and_bool` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_24` AS `rowindex`, + `bfcol_25` AS `bool_col`, + `bfcol_26` AS `int64_col`, + `bfcol_27` AS `int_and_int`, + `bfcol_28` AS `bool_and_bool`, + `bfcol_29` AS `bool_and_null` +FROM `bfcte_3` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_eq_null_match/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_eq_null_match/out.sql index 3d23b8576ec..57af99a52bd 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_eq_null_match/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_eq_null_match/out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + COALESCE(CAST(`int64_col` AS STRING), '$NULL_SENTINEL$') = COALESCE(CAST(CAST(`bool_col` AS INT64) AS STRING), '$NULL_SENTINEL$') AS `bfcol_4` + FROM `bfcte_0` +) SELECT - COALESCE(CAST(`int64_col` AS STRING), '$NULL_SENTINEL$') = COALESCE(CAST(CAST(`bool_col` AS INT64) AS STRING), '$NULL_SENTINEL$') AS `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_4` AS `int64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_eq_numeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_eq_numeric/out.sql index 7827731881e..a21e0089416 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_eq_numeric/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_eq_numeric/out.sql @@ -1,11 +1,67 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `bool_col` AS `bfcol_8`, + `int64_col` = `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_7` = 1 AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + `bfcol_15` IS NULL AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + `bfcol_25` = CAST(`bfcol_26` AS INT64) AS `bfcol_42` + FROM `bfcte_3` +), `bfcte_5` AS ( + SELECT + *, + `bfcol_36` AS `bfcol_50`, + `bfcol_37` AS `bfcol_51`, + `bfcol_38` AS `bfcol_52`, + `bfcol_39` AS `bfcol_53`, + `bfcol_40` AS `bfcol_54`, + `bfcol_41` AS `bfcol_55`, + `bfcol_42` AS `bfcol_56`, + CAST(`bfcol_38` AS INT64) = `bfcol_37` AS `bfcol_57` + FROM `bfcte_4` +) SELECT - `rowindex`, - `int64_col`, - `bool_col`, - `int64_col` = `int64_col` AS `int_eq_int`, - `int64_col` = 1 AS `int_eq_1`, - `int64_col` IS NULL AS `int_eq_null`, - `int64_col` IS NULL AS `null_eq_int`, - `int64_col` = CAST(`bool_col` AS INT64) AS `int_eq_bool`, - CAST(`bool_col` AS INT64) = `int64_col` AS `bool_eq_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_50` AS `rowindex`, + `bfcol_51` AS `int64_col`, + `bfcol_52` AS `bool_col`, + `bfcol_53` AS `int_eq_int`, + `bfcol_54` AS `int_eq_1`, + `bfcol_55` AS `int_eq_null`, + `bfcol_56` AS `int_eq_bool`, + `bfcol_57` AS `bool_eq_int` +FROM `bfcte_5` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_ge_numeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_ge_numeric/out.sql index 5903cf03699..e99fe49c8e0 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_ge_numeric/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_ge_numeric/out.sql @@ -1,10 +1,54 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `bool_col` AS `bfcol_8`, + `int64_col` >= `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_7` >= 1 AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + `bfcol_15` >= CAST(`bfcol_16` AS INT64) AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + CAST(`bfcol_26` AS INT64) >= `bfcol_25` AS `bfcol_42` + FROM `bfcte_3` +) SELECT - `rowindex`, - `int64_col`, - `bool_col`, - `int64_col` >= `int64_col` AS `int_ge_int`, - `int64_col` >= 1 AS `int_ge_1`, - NULL AS `null_ge_int`, - `int64_col` >= CAST(`bool_col` AS INT64) AS `int_ge_bool`, - CAST(`bool_col` AS INT64) >= `int64_col` AS `bool_ge_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_36` AS `rowindex`, + `bfcol_37` AS `int64_col`, + `bfcol_38` AS `bool_col`, + `bfcol_39` AS `int_ge_int`, + `bfcol_40` AS `int_ge_1`, + `bfcol_41` AS `int_ge_bool`, + `bfcol_42` AS `bool_ge_int` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_gt_numeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_gt_numeric/out.sql index 42bf029240f..4e5aba3d31e 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_gt_numeric/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_gt_numeric/out.sql @@ -1,10 +1,54 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `bool_col` AS `bfcol_8`, + `int64_col` > `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_7` > 1 AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + `bfcol_15` > CAST(`bfcol_16` AS INT64) AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + CAST(`bfcol_26` AS INT64) > `bfcol_25` AS `bfcol_42` + FROM `bfcte_3` +) SELECT - `rowindex`, - `int64_col`, - `bool_col`, - `int64_col` > `int64_col` AS `int_gt_int`, - `int64_col` > 1 AS `int_gt_1`, - NULL AS `null_gt_int`, - `int64_col` > CAST(`bool_col` AS INT64) AS `int_gt_bool`, - CAST(`bool_col` AS INT64) > `int64_col` AS `bool_gt_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_36` AS `rowindex`, + `bfcol_37` AS `int64_col`, + `bfcol_38` AS `bool_col`, + `bfcol_39` AS `int_gt_int`, + `bfcol_40` AS `int_gt_1`, + `bfcol_41` AS `int_gt_bool`, + `bfcol_42` AS `bool_gt_int` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_is_in/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_is_in/out.sql index 308e6f9cbd7..ec85f060dac 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_is_in/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_is_in/out.sql @@ -1,19 +1,35 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + COALESCE(`bool_col` IN (TRUE, FALSE), FALSE) AS `bfcol_3`, + COALESCE(`int64_col` IN (1, 2, 3), FALSE) AS `bfcol_4`, + `int64_col` IS NULL AS `bfcol_5`, + COALESCE(`int64_col` IN (1.0, 2.0, 3.0), FALSE) AS `bfcol_6`, + FALSE AS `bfcol_7`, + COALESCE(`int64_col` IN (2.5, 3), FALSE) AS `bfcol_8`, + FALSE AS `bfcol_9`, + FALSE AS `bfcol_10`, + COALESCE(`int64_col` IN (123456), FALSE) AS `bfcol_11`, + ( + `float64_col` IS NULL + ) OR `float64_col` IN (1, 2, 3) AS `bfcol_12` + FROM `bfcte_0` +) SELECT - COALESCE(`bool_col` IN (TRUE, FALSE), FALSE) AS `bools`, - COALESCE(`int64_col` IN (1, 2, 3), FALSE) AS `ints`, - `int64_col` IS NULL AS `ints_w_null`, - COALESCE(`int64_col` IN (1.0, 2.0, 3.0), FALSE) AS `floats`, - FALSE AS `strings`, - COALESCE(`int64_col` IN (2.5, 3, 1e-10, CAST('Infinity' AS FLOAT64), NULL, 0), FALSE) AS `mixed`, - FALSE AS `empty`, - FALSE AS `empty_wo_match_nulls`, - COALESCE(`int64_col` IN (123456), FALSE) AS `ints_wo_match_nulls`, - ( - `float64_col` IS NULL - ) OR `float64_col` IN (1, 2, 3) AS `float_in_ints`, - ( - `int64_col` IS NULL - ) OR `int64_col` IN (2) AS `mixed_with_null`, - COALESCE(CAST(`bool_col` AS INT64) IN (1, 2.5), FALSE) AS `bool_in_mixed`, - `int64_col` IS NULL AS `only_null_match` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_3` AS `bools`, + `bfcol_4` AS `ints`, + `bfcol_5` AS `ints_w_null`, + `bfcol_6` AS `floats`, + `bfcol_7` AS `strings`, + `bfcol_8` AS `mixed`, + `bfcol_9` AS `empty`, + `bfcol_10` AS `empty_wo_match_nulls`, + `bfcol_11` AS `ints_wo_match_nulls`, + `bfcol_12` AS `float_in_ints` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_le_numeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_le_numeric/out.sql index c6c86510102..97a00d1c88b 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_le_numeric/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_le_numeric/out.sql @@ -1,10 +1,54 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `bool_col` AS `bfcol_8`, + `int64_col` <= `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_7` <= 1 AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + `bfcol_15` <= CAST(`bfcol_16` AS INT64) AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + CAST(`bfcol_26` AS INT64) <= `bfcol_25` AS `bfcol_42` + FROM `bfcte_3` +) SELECT - `rowindex`, - `int64_col`, - `bool_col`, - `int64_col` <= `int64_col` AS `int_le_int`, - `int64_col` <= 1 AS `int_le_1`, - NULL AS `null_le_int`, - `int64_col` <= CAST(`bool_col` AS INT64) AS `int_le_bool`, - CAST(`bool_col` AS INT64) <= `int64_col` AS `bool_le_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_36` AS `rowindex`, + `bfcol_37` AS `int64_col`, + `bfcol_38` AS `bool_col`, + `bfcol_39` AS `int_le_int`, + `bfcol_40` AS `int_le_1`, + `bfcol_41` AS `int_le_bool`, + `bfcol_42` AS `bool_le_int` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_lt_numeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_lt_numeric/out.sql index ec5c317a8e5..addebd3187c 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_lt_numeric/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_lt_numeric/out.sql @@ -1,10 +1,54 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `bool_col` AS `bfcol_8`, + `int64_col` < `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_7` < 1 AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + `bfcol_15` < CAST(`bfcol_16` AS INT64) AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + CAST(`bfcol_26` AS INT64) < `bfcol_25` AS `bfcol_42` + FROM `bfcte_3` +) SELECT - `rowindex`, - `int64_col`, - `bool_col`, - `int64_col` < `int64_col` AS `int_lt_int`, - `int64_col` < 1 AS `int_lt_1`, - NULL AS `null_lt_int`, - `int64_col` < CAST(`bool_col` AS INT64) AS `int_lt_bool`, - CAST(`bool_col` AS INT64) < `int64_col` AS `bool_lt_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_36` AS `rowindex`, + `bfcol_37` AS `int64_col`, + `bfcol_38` AS `bool_col`, + `bfcol_39` AS `int_lt_int`, + `bfcol_40` AS `int_lt_1`, + `bfcol_41` AS `int_lt_bool`, + `bfcol_42` AS `bool_lt_int` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_maximum_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_maximum_op/out.sql index a469fa47cf1..bbef2127070 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_maximum_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_maximum_op/out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + GREATEST(`int64_col`, `float64_col`) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - GREATEST(`int64_col`, `float64_col`) AS `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `int64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_minimum_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_minimum_op/out.sql index ea82af979a3..1f00f5892ef 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_minimum_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_minimum_op/out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + LEAST(`int64_col`, `float64_col`) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - LEAST(`int64_col`, `float64_col`) AS `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `int64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_ne_numeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_ne_numeric/out.sql index 448a6146294..1a1ff6e44d2 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_ne_numeric/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_comparison_ops/test_ne_numeric/out.sql @@ -1,15 +1,69 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `bool_col` AS `bfcol_8`, + `int64_col` <> `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_7` <> 1 AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + ( + `bfcol_15` + ) IS NOT NULL AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + `bfcol_25` <> CAST(`bfcol_26` AS INT64) AS `bfcol_42` + FROM `bfcte_3` +), `bfcte_5` AS ( + SELECT + *, + `bfcol_36` AS `bfcol_50`, + `bfcol_37` AS `bfcol_51`, + `bfcol_38` AS `bfcol_52`, + `bfcol_39` AS `bfcol_53`, + `bfcol_40` AS `bfcol_54`, + `bfcol_41` AS `bfcol_55`, + `bfcol_42` AS `bfcol_56`, + CAST(`bfcol_38` AS INT64) <> `bfcol_37` AS `bfcol_57` + FROM `bfcte_4` +) SELECT - `rowindex`, - `int64_col`, - `bool_col`, - `int64_col` <> `int64_col` AS `int_ne_int`, - `int64_col` <> 1 AS `int_ne_1`, - ( - `int64_col` - ) IS NOT NULL AS `int_ne_null`, - ( - `int64_col` - ) IS NOT NULL AS `null_ne_int`, - `int64_col` <> CAST(`bool_col` AS INT64) AS `int_ne_bool`, - CAST(`bool_col` AS INT64) <> `int64_col` AS `bool_ne_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_50` AS `rowindex`, + `bfcol_51` AS `int64_col`, + `bfcol_52` AS `bool_col`, + `bfcol_53` AS `int_ne_int`, + `bfcol_54` AS `int_ne_1`, + `bfcol_55` AS `int_ne_null`, + `bfcol_56` AS `int_ne_bool`, + `bfcol_57` AS `bool_ne_int` +FROM `bfcte_5` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_add_timedelta/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_add_timedelta/out.sql index b1ccf096cfa..2fef18eeb8a 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_add_timedelta/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_add_timedelta/out.sql @@ -1,10 +1,60 @@ +WITH `bfcte_0` AS ( + SELECT + `date_col`, + `rowindex`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `timestamp_col` AS `bfcol_7`, + `date_col` AS `bfcol_8`, + TIMESTAMP_ADD(CAST(`date_col` AS DATETIME), INTERVAL 86400000000 MICROSECOND) AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + TIMESTAMP_ADD(`bfcol_7`, INTERVAL 86400000000 MICROSECOND) AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + TIMESTAMP_ADD(CAST(`bfcol_16` AS DATETIME), INTERVAL 86400000000 MICROSECOND) AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + TIMESTAMP_ADD(`bfcol_25`, INTERVAL 86400000000 MICROSECOND) AS `bfcol_42` + FROM `bfcte_3` +), `bfcte_5` AS ( + SELECT + *, + 172800000000 AS `bfcol_50` + FROM `bfcte_4` +) SELECT - `rowindex`, - `timestamp_col`, - `date_col`, - TIMESTAMP_ADD(CAST(`date_col` AS DATETIME), INTERVAL 86400000000 MICROSECOND) AS `date_add_timedelta`, - TIMESTAMP_ADD(`timestamp_col`, INTERVAL 86400000000 MICROSECOND) AS `timestamp_add_timedelta`, - TIMESTAMP_ADD(CAST(`date_col` AS DATETIME), INTERVAL 86400000000 MICROSECOND) AS `timedelta_add_date`, - TIMESTAMP_ADD(`timestamp_col`, INTERVAL 86400000000 MICROSECOND) AS `timedelta_add_timestamp`, - 172800000000 AS `timedelta_add_timedelta` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_36` AS `rowindex`, + `bfcol_37` AS `timestamp_col`, + `bfcol_38` AS `date_col`, + `bfcol_39` AS `date_add_timedelta`, + `bfcol_40` AS `timestamp_add_timedelta`, + `bfcol_41` AS `timedelta_add_date`, + `bfcol_42` AS `timedelta_add_timestamp`, + `bfcol_50` AS `timedelta_add_timedelta` +FROM `bfcte_5` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_date/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_date/out.sql index eb0d2f11049..b8f46ceafef 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_date/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_date/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + DATE(`timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - DATE(`timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_datetime_to_integer_label/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_datetime_to_integer_label/out.sql index 4b0696386c1..5260dd680a3 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_datetime_to_integer_label/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_datetime_to_integer_label/out.sql @@ -1,76 +1,38 @@ -SELECT - CAST(FLOOR( - IEEE_DIVIDE( - UNIX_MICROS(CAST(`datetime_col` AS TIMESTAMP)) - UNIX_MICROS(CAST(`timestamp_col` AS TIMESTAMP)), - 86400000000 - ) - ) AS INT64) AS `fixed_freq`, - CAST(FLOOR(IEEE_DIVIDE(UNIX_MICROS(CAST(`datetime_col` AS TIMESTAMP)) - 0, 86400000000)) AS INT64) AS `origin_epoch`, - CAST(FLOOR( - IEEE_DIVIDE( - UNIX_MICROS(CAST(`datetime_col` AS TIMESTAMP)) - UNIX_MICROS(CAST(CAST(`timestamp_col` AS DATE) AS TIMESTAMP)), - 86400000000 - ) - ) AS INT64) AS `origin_start_day`, - CASE - WHEN UNIX_MICROS( - CAST(TIMESTAMP_TRUNC(`datetime_col`, WEEK(MONDAY)) + INTERVAL 6 DAY AS TIMESTAMP) - ) = UNIX_MICROS( - CAST(TIMESTAMP_TRUNC(`timestamp_col`, WEEK(MONDAY)) + INTERVAL 6 DAY AS TIMESTAMP) - ) - THEN 0 - ELSE CAST(FLOOR( +WITH `bfcte_0` AS ( + SELECT + `datetime_col`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(FLOOR( IEEE_DIVIDE( - UNIX_MICROS( - CAST(TIMESTAMP_TRUNC(`datetime_col`, WEEK(MONDAY)) + INTERVAL 6 DAY AS TIMESTAMP) - ) - UNIX_MICROS( - CAST(TIMESTAMP_TRUNC(`timestamp_col`, WEEK(MONDAY)) + INTERVAL 6 DAY AS TIMESTAMP) - ) - 1, - 604800000000 + UNIX_MICROS(CAST(`datetime_col` AS TIMESTAMP)) - UNIX_MICROS(CAST(`timestamp_col` AS TIMESTAMP)), + 86400000000 ) - ) AS INT64) + 1 - END AS `non_fixed_freq_weekly`, - CASE - WHEN ( - EXTRACT(YEAR FROM `datetime_col`) * 12 + EXTRACT(MONTH FROM `datetime_col`) - 1 - ) = ( - EXTRACT(YEAR FROM `timestamp_col`) * 12 + EXTRACT(MONTH FROM `timestamp_col`) - 1 - ) - THEN 0 - ELSE CAST(FLOOR( - IEEE_DIVIDE( - ( - EXTRACT(YEAR FROM `datetime_col`) * 12 + EXTRACT(MONTH FROM `datetime_col`) - 1 - ) - ( - EXTRACT(YEAR FROM `timestamp_col`) * 12 + EXTRACT(MONTH FROM `timestamp_col`) - 1 - ) - 1, - 1 + ) AS INT64) AS `bfcol_2`, + CASE + WHEN UNIX_MICROS( + CAST(TIMESTAMP_TRUNC(`datetime_col`, WEEK(MONDAY)) + INTERVAL 6 DAY AS TIMESTAMP) + ) = UNIX_MICROS( + CAST(TIMESTAMP_TRUNC(`timestamp_col`, WEEK(MONDAY)) + INTERVAL 6 DAY AS TIMESTAMP) ) - ) AS INT64) + 1 - END AS `non_fixed_freq_monthly`, - CASE - WHEN ( - EXTRACT(YEAR FROM `datetime_col`) * 4 + EXTRACT(QUARTER FROM `datetime_col`) - 1 - ) = ( - EXTRACT(YEAR FROM `timestamp_col`) * 4 + EXTRACT(QUARTER FROM `timestamp_col`) - 1 - ) - THEN 0 - ELSE CAST(FLOOR( - IEEE_DIVIDE( - ( - EXTRACT(YEAR FROM `datetime_col`) * 4 + EXTRACT(QUARTER FROM `datetime_col`) - 1 - ) - ( - EXTRACT(YEAR FROM `timestamp_col`) * 4 + EXTRACT(QUARTER FROM `timestamp_col`) - 1 - ) - 1, - 1 - ) - ) AS INT64) + 1 - END AS `non_fixed_freq_quarterly`, - CASE - WHEN EXTRACT(YEAR FROM `datetime_col`) = EXTRACT(YEAR FROM `timestamp_col`) - THEN 0 - ELSE CAST(FLOOR( - IEEE_DIVIDE(EXTRACT(YEAR FROM `datetime_col`) - EXTRACT(YEAR FROM `timestamp_col`) - 1, 1) - ) AS INT64) + 1 - END AS `non_fixed_freq_yearly` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + THEN 0 + ELSE CAST(FLOOR( + IEEE_DIVIDE( + UNIX_MICROS( + CAST(TIMESTAMP_TRUNC(`datetime_col`, WEEK(MONDAY)) + INTERVAL 6 DAY AS TIMESTAMP) + ) - UNIX_MICROS( + CAST(TIMESTAMP_TRUNC(`timestamp_col`, WEEK(MONDAY)) + INTERVAL 6 DAY AS TIMESTAMP) + ) - 1, + 604800000000 + ) + ) AS INT64) + 1 + END AS `bfcol_3` + FROM `bfcte_0` +) +SELECT + `bfcol_2` AS `fixed_freq`, + `bfcol_3` AS `non_fixed_freq_weekly` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_day/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_day/out.sql index b9c030cb53e..52d80fd2a61 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_day/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_day/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + EXTRACT(DAY FROM `timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - EXTRACT(DAY FROM `timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_dayofweek/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_dayofweek/out.sql index a25d520d804..0119bbb4e9f 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_dayofweek/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_dayofweek/out.sql @@ -1,5 +1,19 @@ +WITH `bfcte_0` AS ( + SELECT + `date_col`, + `datetime_col`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(MOD(EXTRACT(DAYOFWEEK FROM `datetime_col`) + 5, 7) AS INT64) AS `bfcol_6`, + CAST(MOD(EXTRACT(DAYOFWEEK FROM `timestamp_col`) + 5, 7) AS INT64) AS `bfcol_7`, + CAST(MOD(EXTRACT(DAYOFWEEK FROM `date_col`) + 5, 7) AS INT64) AS `bfcol_8` + FROM `bfcte_0` +) SELECT - CAST(MOD(EXTRACT(DAYOFWEEK FROM `datetime_col`) + 5, 7) AS INT64) AS `datetime_col`, - CAST(MOD(EXTRACT(DAYOFWEEK FROM `timestamp_col`) + 5, 7) AS INT64) AS `timestamp_col`, - CAST(MOD(EXTRACT(DAYOFWEEK FROM `date_col`) + 5, 7) AS INT64) AS `date_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_6` AS `datetime_col`, + `bfcol_7` AS `timestamp_col`, + `bfcol_8` AS `date_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_dayofyear/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_dayofyear/out.sql index 87a410911b5..521419757ab 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_dayofyear/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_dayofyear/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + EXTRACT(DAYOFYEAR FROM `timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - EXTRACT(DAYOFYEAR FROM `timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_floor_dt/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_floor_dt/out.sql index 49fb8fe5749..fe76efb609b 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_floor_dt/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_floor_dt/out.sql @@ -1,14 +1,36 @@ +WITH `bfcte_0` AS ( + SELECT + `datetime_col`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + TIMESTAMP_TRUNC(`timestamp_col`, MICROSECOND) AS `bfcol_2`, + TIMESTAMP_TRUNC(`timestamp_col`, MILLISECOND) AS `bfcol_3`, + TIMESTAMP_TRUNC(`timestamp_col`, SECOND) AS `bfcol_4`, + TIMESTAMP_TRUNC(`timestamp_col`, MINUTE) AS `bfcol_5`, + TIMESTAMP_TRUNC(`timestamp_col`, HOUR) AS `bfcol_6`, + TIMESTAMP_TRUNC(`timestamp_col`, DAY) AS `bfcol_7`, + TIMESTAMP_TRUNC(`timestamp_col`, WEEK(MONDAY)) AS `bfcol_8`, + TIMESTAMP_TRUNC(`timestamp_col`, MONTH) AS `bfcol_9`, + TIMESTAMP_TRUNC(`timestamp_col`, QUARTER) AS `bfcol_10`, + TIMESTAMP_TRUNC(`timestamp_col`, YEAR) AS `bfcol_11`, + TIMESTAMP_TRUNC(`datetime_col`, MICROSECOND) AS `bfcol_12`, + TIMESTAMP_TRUNC(`datetime_col`, MICROSECOND) AS `bfcol_13` + FROM `bfcte_0` +) SELECT - TIMESTAMP_TRUNC(`timestamp_col`, MICROSECOND) AS `timestamp_col_us`, - TIMESTAMP_TRUNC(`timestamp_col`, MILLISECOND) AS `timestamp_col_ms`, - TIMESTAMP_TRUNC(`timestamp_col`, SECOND) AS `timestamp_col_s`, - TIMESTAMP_TRUNC(`timestamp_col`, MINUTE) AS `timestamp_col_min`, - TIMESTAMP_TRUNC(`timestamp_col`, HOUR) AS `timestamp_col_h`, - TIMESTAMP_TRUNC(`timestamp_col`, DAY) AS `timestamp_col_D`, - TIMESTAMP_TRUNC(`timestamp_col`, WEEK(MONDAY)) AS `timestamp_col_W`, - TIMESTAMP_TRUNC(`timestamp_col`, MONTH) AS `timestamp_col_M`, - TIMESTAMP_TRUNC(`timestamp_col`, QUARTER) AS `timestamp_col_Q`, - TIMESTAMP_TRUNC(`timestamp_col`, YEAR) AS `timestamp_col_Y`, - TIMESTAMP_TRUNC(`datetime_col`, MICROSECOND) AS `datetime_col_q`, - TIMESTAMP_TRUNC(`datetime_col`, MICROSECOND) AS `datetime_col_us` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `timestamp_col_us`, + `bfcol_3` AS `timestamp_col_ms`, + `bfcol_4` AS `timestamp_col_s`, + `bfcol_5` AS `timestamp_col_min`, + `bfcol_6` AS `timestamp_col_h`, + `bfcol_7` AS `timestamp_col_D`, + `bfcol_8` AS `timestamp_col_W`, + `bfcol_9` AS `timestamp_col_M`, + `bfcol_10` AS `timestamp_col_Q`, + `bfcol_11` AS `timestamp_col_Y`, + `bfcol_12` AS `datetime_col_q`, + `bfcol_13` AS `datetime_col_us` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_hour/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_hour/out.sql index e971057f527..5fc6621a7ca 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_hour/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_hour/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + EXTRACT(HOUR FROM `timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - EXTRACT(HOUR FROM `timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_fixed/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_fixed/out.sql index 244bd88deb7..8a759e85f98 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_fixed/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_fixed/out.sql @@ -1,5 +1,16 @@ +WITH `bfcte_0` AS ( + SELECT + `rowindex`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(TIMESTAMP_MICROS( + CAST(CAST(`rowindex` AS BIGNUMERIC) * 86400000000 + CAST(UNIX_MICROS(CAST(`timestamp_col` AS TIMESTAMP)) AS BIGNUMERIC) AS INT64) + ) AS TIMESTAMP) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - CAST(TIMESTAMP_MICROS( - CAST(CAST(`rowindex` AS BIGNUMERIC) * 86400000000 + CAST(UNIX_MICROS(CAST(`timestamp_col` AS TIMESTAMP)) AS BIGNUMERIC) AS INT64) - ) AS TIMESTAMP) AS `fixed_freq` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `fixed_freq` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_month/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_month/out.sql index 1ece688b91f..a9e64fead63 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_month/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_month/out.sql @@ -1,39 +1,50 @@ -SELECT - CAST(TIMESTAMP( - DATETIME( - CASE - WHEN MOD( - `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 12 + EXTRACT(MONTH FROM `timestamp_col`) - 1, - 12 - ) + 1 = 12 - THEN CAST(FLOOR( - IEEE_DIVIDE( +WITH `bfcte_0` AS ( + SELECT + `rowindex`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(TIMESTAMP( + DATETIME( + CASE + WHEN MOD( + `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 12 + EXTRACT(MONTH FROM `timestamp_col`) - 1, + 12 + ) + 1 = 12 + THEN CAST(FLOOR( + IEEE_DIVIDE( + `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 12 + EXTRACT(MONTH FROM `timestamp_col`) - 1, + 12 + ) + ) AS INT64) + 1 + ELSE CAST(FLOOR( + IEEE_DIVIDE( + `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 12 + EXTRACT(MONTH FROM `timestamp_col`) - 1, + 12 + ) + ) AS INT64) + END, + CASE + WHEN MOD( `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 12 + EXTRACT(MONTH FROM `timestamp_col`) - 1, 12 - ) - ) AS INT64) + 1 - ELSE CAST(FLOOR( - IEEE_DIVIDE( + ) + 1 = 12 + THEN 1 + ELSE MOD( `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 12 + EXTRACT(MONTH FROM `timestamp_col`) - 1, 12 - ) - ) AS INT64) - END, - CASE - WHEN MOD( - `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 12 + EXTRACT(MONTH FROM `timestamp_col`) - 1, - 12 - ) + 1 = 12 - THEN 1 - ELSE MOD( - `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 12 + EXTRACT(MONTH FROM `timestamp_col`) - 1, - 12 - ) + 1 + 1 - END, - 1, - 0, - 0, - 0 - ) - ) - INTERVAL 1 DAY AS TIMESTAMP) AS `non_fixed_freq_monthly` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + ) + 1 + 1 + END, + 1, + 0, + 0, + 0 + ) + ) - INTERVAL 1 DAY AS TIMESTAMP) AS `bfcol_2` + FROM `bfcte_0` +) +SELECT + `bfcol_2` AS `non_fixed_freq_monthly` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_quarter/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_quarter/out.sql index 683b26be91b..58064855a9e 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_quarter/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_quarter/out.sql @@ -1,43 +1,54 @@ +WITH `bfcte_0` AS ( + SELECT + `rowindex`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(DATETIME( + CASE + WHEN ( + MOD( + `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 4 + EXTRACT(QUARTER FROM `timestamp_col`) - 1, + 4 + ) + 1 + ) * 3 = 12 + THEN CAST(FLOOR( + IEEE_DIVIDE( + `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 4 + EXTRACT(QUARTER FROM `timestamp_col`) - 1, + 4 + ) + ) AS INT64) + 1 + ELSE CAST(FLOOR( + IEEE_DIVIDE( + `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 4 + EXTRACT(QUARTER FROM `timestamp_col`) - 1, + 4 + ) + ) AS INT64) + END, + CASE + WHEN ( + MOD( + `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 4 + EXTRACT(QUARTER FROM `timestamp_col`) - 1, + 4 + ) + 1 + ) * 3 = 12 + THEN 1 + ELSE ( + MOD( + `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 4 + EXTRACT(QUARTER FROM `timestamp_col`) - 1, + 4 + ) + 1 + ) * 3 + 1 + END, + 1, + 0, + 0, + 0 + ) - INTERVAL 1 DAY AS TIMESTAMP) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - CAST(DATETIME( - CASE - WHEN ( - MOD( - `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 4 + EXTRACT(QUARTER FROM `timestamp_col`) - 1, - 4 - ) + 1 - ) * 3 = 12 - THEN CAST(FLOOR( - IEEE_DIVIDE( - `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 4 + EXTRACT(QUARTER FROM `timestamp_col`) - 1, - 4 - ) - ) AS INT64) + 1 - ELSE CAST(FLOOR( - IEEE_DIVIDE( - `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 4 + EXTRACT(QUARTER FROM `timestamp_col`) - 1, - 4 - ) - ) AS INT64) - END, - CASE - WHEN ( - MOD( - `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 4 + EXTRACT(QUARTER FROM `timestamp_col`) - 1, - 4 - ) + 1 - ) * 3 = 12 - THEN 1 - ELSE ( - MOD( - `rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) * 4 + EXTRACT(QUARTER FROM `timestamp_col`) - 1, - 4 - ) + 1 - ) * 3 + 1 - END, - 1, - 0, - 0, - 0 - ) - INTERVAL 1 DAY AS TIMESTAMP) AS `non_fixed_freq` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `non_fixed_freq` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_week/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_week/out.sql index 6196e6976b0..142f8561f48 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_week/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_week/out.sql @@ -1,7 +1,18 @@ +WITH `bfcte_0` AS ( + SELECT + `rowindex`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(TIMESTAMP_MICROS( + CAST(CAST(`rowindex` AS BIGNUMERIC) * 604800000000 + CAST(UNIX_MICROS( + TIMESTAMP_TRUNC(CAST(`timestamp_col` AS TIMESTAMP), WEEK(MONDAY)) + INTERVAL 6 DAY + ) AS BIGNUMERIC) AS INT64) + ) AS TIMESTAMP) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - CAST(TIMESTAMP_MICROS( - CAST(CAST(`rowindex` AS BIGNUMERIC) * 604800000000 + CAST(UNIX_MICROS( - TIMESTAMP_TRUNC(CAST(`timestamp_col` AS TIMESTAMP), WEEK(MONDAY)) + INTERVAL 6 DAY - ) AS BIGNUMERIC) AS INT64) - ) AS TIMESTAMP) AS `non_fixed_freq_weekly` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `non_fixed_freq_weekly` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_year/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_year/out.sql index e0d05ec5b4b..ab77a9d1906 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_year/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_integer_label_to_datetime_year/out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `rowindex`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(TIMESTAMP(DATETIME(`rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) + 1, 1, 1, 0, 0, 0)) - INTERVAL 1 DAY AS TIMESTAMP) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - CAST(TIMESTAMP(DATETIME(`rowindex` * 1 + EXTRACT(YEAR FROM `timestamp_col`) + 1, 1, 1, 0, 0, 0)) - INTERVAL 1 DAY AS TIMESTAMP) AS `non_fixed_freq_yearly` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `non_fixed_freq_yearly` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_iso_day/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_iso_day/out.sql index bf7dfea7378..9422844b34f 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_iso_day/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_iso_day/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(MOD(EXTRACT(DAYOFWEEK FROM `timestamp_col`) + 5, 7) AS INT64) + 1 AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CAST(MOD(EXTRACT(DAYOFWEEK FROM `timestamp_col`) + 5, 7) AS INT64) + 1 AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_iso_week/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_iso_week/out.sql index ce231592164..4db49fb10fa 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_iso_week/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_iso_week/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + EXTRACT(ISOWEEK FROM `timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - EXTRACT(ISOWEEK FROM `timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_iso_year/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_iso_year/out.sql index aea4bec4371..8d49933202c 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_iso_year/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_iso_year/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + EXTRACT(ISOYEAR FROM `timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - EXTRACT(ISOYEAR FROM `timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_minute/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_minute/out.sql index ed1ffcee104..e089a77af51 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_minute/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_minute/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + EXTRACT(MINUTE FROM `timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - EXTRACT(MINUTE FROM `timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_month/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_month/out.sql index 8defb0312e9..53d135903ba 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_month/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_month/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + EXTRACT(MONTH FROM `timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - EXTRACT(MONTH FROM `timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_normalize/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_normalize/out.sql index 0ae08c77ad0..b542dfea72a 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_normalize/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_normalize/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + TIMESTAMP_TRUNC(`timestamp_col`, DAY) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - TIMESTAMP_TRUNC(`timestamp_col`, DAY) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_quarter/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_quarter/out.sql index 9426f685855..4a232cb5a30 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_quarter/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_quarter/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + EXTRACT(QUARTER FROM `timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - EXTRACT(QUARTER FROM `timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_second/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_second/out.sql index 953a0ff762a..e86d830b737 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_second/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_second/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + EXTRACT(SECOND FROM `timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - EXTRACT(SECOND FROM `timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_strftime/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_strftime/out.sql index 308c040640d..1d8f62f948a 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_strftime/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_strftime/out.sql @@ -1,6 +1,22 @@ +WITH `bfcte_0` AS ( + SELECT + `date_col`, + `datetime_col`, + `time_col`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + FORMAT_DATE('%Y-%m-%d', `date_col`) AS `bfcol_8`, + FORMAT_DATETIME('%Y-%m-%d', `datetime_col`) AS `bfcol_9`, + FORMAT_TIME('%Y-%m-%d', `time_col`) AS `bfcol_10`, + FORMAT_TIMESTAMP('%Y-%m-%d', `timestamp_col`) AS `bfcol_11` + FROM `bfcte_0` +) SELECT - FORMAT_DATE('%Y-%m-%d', `date_col`) AS `date_col`, - FORMAT_DATETIME('%Y-%m-%d', `datetime_col`) AS `datetime_col`, - FORMAT_TIME('%Y-%m-%d', `time_col`) AS `time_col`, - FORMAT_TIMESTAMP('%Y-%m-%d', `timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_8` AS `date_col`, + `bfcol_9` AS `datetime_col`, + `bfcol_10` AS `time_col`, + `bfcol_11` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_sub_timedelta/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_sub_timedelta/out.sql index 5c8b130d59d..ebcffd67f61 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_sub_timedelta/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_sub_timedelta/out.sql @@ -1,11 +1,82 @@ +WITH `bfcte_0` AS ( + SELECT + `date_col`, + `duration_col`, + `rowindex`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_8`, + `timestamp_col` AS `bfcol_9`, + `date_col` AS `bfcol_10`, + `duration_col` AS `bfcol_11` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_11` AS `bfcol_18`, + `bfcol_10` AS `bfcol_19`, + TIMESTAMP_SUB(CAST(`bfcol_10` AS DATETIME), INTERVAL `bfcol_11` MICROSECOND) AS `bfcol_20` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + `bfcol_19` AS `bfcol_29`, + `bfcol_20` AS `bfcol_30`, + TIMESTAMP_SUB(`bfcol_17`, INTERVAL `bfcol_18` MICROSECOND) AS `bfcol_31` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + `bfcol_30` AS `bfcol_42`, + `bfcol_31` AS `bfcol_43`, + TIMESTAMP_DIFF(CAST(`bfcol_29` AS DATETIME), CAST(`bfcol_29` AS DATETIME), MICROSECOND) AS `bfcol_44` + FROM `bfcte_3` +), `bfcte_5` AS ( + SELECT + *, + `bfcol_38` AS `bfcol_52`, + `bfcol_39` AS `bfcol_53`, + `bfcol_40` AS `bfcol_54`, + `bfcol_41` AS `bfcol_55`, + `bfcol_42` AS `bfcol_56`, + `bfcol_43` AS `bfcol_57`, + `bfcol_44` AS `bfcol_58`, + TIMESTAMP_DIFF(`bfcol_39`, `bfcol_39`, MICROSECOND) AS `bfcol_59` + FROM `bfcte_4` +), `bfcte_6` AS ( + SELECT + *, + `bfcol_52` AS `bfcol_68`, + `bfcol_53` AS `bfcol_69`, + `bfcol_54` AS `bfcol_70`, + `bfcol_55` AS `bfcol_71`, + `bfcol_56` AS `bfcol_72`, + `bfcol_57` AS `bfcol_73`, + `bfcol_58` AS `bfcol_74`, + `bfcol_59` AS `bfcol_75`, + `bfcol_54` - `bfcol_54` AS `bfcol_76` + FROM `bfcte_5` +) SELECT - `rowindex`, - `timestamp_col`, - `duration_col`, - `date_col`, - TIMESTAMP_SUB(CAST(`date_col` AS DATETIME), INTERVAL `duration_col` MICROSECOND) AS `date_sub_timedelta`, - TIMESTAMP_SUB(`timestamp_col`, INTERVAL `duration_col` MICROSECOND) AS `timestamp_sub_timedelta`, - TIMESTAMP_DIFF(CAST(`date_col` AS DATETIME), CAST(`date_col` AS DATETIME), MICROSECOND) AS `timestamp_sub_date`, - TIMESTAMP_DIFF(`timestamp_col`, `timestamp_col`, MICROSECOND) AS `date_sub_timestamp`, - `duration_col` - `duration_col` AS `timedelta_sub_timedelta` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_68` AS `rowindex`, + `bfcol_69` AS `timestamp_col`, + `bfcol_70` AS `duration_col`, + `bfcol_71` AS `date_col`, + `bfcol_72` AS `date_sub_timedelta`, + `bfcol_73` AS `timestamp_sub_timedelta`, + `bfcol_74` AS `timestamp_sub_date`, + `bfcol_75` AS `date_sub_timestamp`, + `bfcol_76` AS `timedelta_sub_timedelta` +FROM `bfcte_6` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_time/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_time/out.sql index e46ca373909..5a8ab600bac 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_time/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_time/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + TIME(`timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - TIME(`timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_to_datetime/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_to_datetime/out.sql index 50142f20ba5..a8d40a84867 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_to_datetime/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_to_datetime/out.sql @@ -1,7 +1,19 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col`, + `int64_col`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(TIMESTAMP_MICROS(CAST(TRUNC(`int64_col` * 0.001) AS INT64)) AS DATETIME) AS `bfcol_6`, + SAFE_CAST(`string_col` AS DATETIME) AS `bfcol_7`, + CAST(TIMESTAMP_MICROS(CAST(TRUNC(`float64_col` * 0.001) AS INT64)) AS DATETIME) AS `bfcol_8` + FROM `bfcte_0` +) SELECT - DATETIME(TIMESTAMP_MICROS(CAST(TRUNC(`int64_col` * 0.001) AS INT64)), 'UTC') AS `int64_col`, - SAFE_CAST(`string_col` AS DATETIME) AS `string_col`, - DATETIME(TIMESTAMP_MICROS(CAST(TRUNC(`float64_col` * 0.001) AS INT64)), 'UTC') AS `float64_col`, - DATETIME(`timestamp_col`, 'UTC') AS `timestamp_col`, - SAFE_CAST(`string_col` AS DATETIME) AS `string_col_fmt` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + `bfcol_6` AS `int64_col`, + `bfcol_7` AS `string_col`, + `bfcol_8` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_to_timestamp/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_to_timestamp/out.sql index e0fb530cc6d..a5f9ee1112b 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_to_timestamp/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_to_timestamp/out.sql @@ -1,10 +1,24 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(TIMESTAMP_MICROS(CAST(TRUNC(`int64_col` * 0.001) AS INT64)) AS TIMESTAMP) AS `bfcol_2`, + CAST(TIMESTAMP_MICROS(CAST(TRUNC(`float64_col` * 0.001) AS INT64)) AS TIMESTAMP) AS `bfcol_3`, + CAST(TIMESTAMP_MICROS(CAST(TRUNC(`int64_col` * 1000000) AS INT64)) AS TIMESTAMP) AS `bfcol_4`, + CAST(TIMESTAMP_MICROS(CAST(TRUNC(`int64_col` * 1000) AS INT64)) AS TIMESTAMP) AS `bfcol_5`, + CAST(TIMESTAMP_MICROS(CAST(TRUNC(`int64_col`) AS INT64)) AS TIMESTAMP) AS `bfcol_6`, + CAST(TIMESTAMP_MICROS(CAST(TRUNC(`int64_col` * 0.001) AS INT64)) AS TIMESTAMP) AS `bfcol_7` + FROM `bfcte_0` +) SELECT - CAST(TIMESTAMP_MICROS(CAST(TRUNC(`int64_col` * 0.001) AS INT64)) AS TIMESTAMP) AS `int64_col`, - CAST(TIMESTAMP_MICROS(CAST(TRUNC(`float64_col` * 0.001) AS INT64)) AS TIMESTAMP) AS `float64_col`, - CAST(TIMESTAMP_MICROS(CAST(TRUNC(`int64_col` * 1000000) AS INT64)) AS TIMESTAMP) AS `int64_col_s`, - CAST(TIMESTAMP_MICROS(CAST(TRUNC(`int64_col` * 1000) AS INT64)) AS TIMESTAMP) AS `int64_col_ms`, - CAST(TIMESTAMP_MICROS(CAST(TRUNC(`int64_col`) AS INT64)) AS TIMESTAMP) AS `int64_col_us`, - CAST(TIMESTAMP_MICROS(CAST(TRUNC(`int64_col` * 0.001) AS INT64)) AS TIMESTAMP) AS `int64_col_ns`, - TIMESTAMP(`datetime_col`) AS `datetime_col`, - PARSE_TIMESTAMP('%Y-%m-%d', `string_col`, 'UTC') AS `string_col_fmt` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `int64_col`, + `bfcol_3` AS `float64_col`, + `bfcol_4` AS `int64_col_s`, + `bfcol_5` AS `int64_col_ms`, + `bfcol_6` AS `int64_col_us`, + `bfcol_7` AS `int64_col_ns` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_unix_micros/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_unix_micros/out.sql index a212164e6ce..e6515017f25 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_unix_micros/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_unix_micros/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + UNIX_MICROS(`timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - UNIX_MICROS(`timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_unix_millis/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_unix_millis/out.sql index 8df5ad956a3..caec5effe0a 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_unix_millis/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_unix_millis/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + UNIX_MILLIS(`timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - UNIX_MILLIS(`timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_unix_seconds/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_unix_seconds/out.sql index 7344ca82949..6dc0ea2a02a 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_unix_seconds/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_unix_seconds/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + UNIX_SECONDS(`timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - UNIX_SECONDS(`timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_year/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_year/out.sql index f1a1d7085ef..1ceb674137c 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_year/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_datetime_ops/test_year/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + EXTRACT(YEAR FROM `timestamp_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - EXTRACT(YEAR FROM `timestamp_col`) AS `timestamp_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `timestamp_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_bool/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_bool/out.sql index 2f75cf4cf7f..1f90accd0bb 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_bool/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_bool/out.sql @@ -1,5 +1,18 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `bool_col` AS `bfcol_2`, + `float64_col` <> 0 AS `bfcol_3`, + `float64_col` <> 0 AS `bfcol_4` + FROM `bfcte_0` +) SELECT - `bool_col`, - `float64_col` <> 0 AS `float64_col`, - `float64_col` <> 0 AS `float64_w_safe` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `bool_col`, + `bfcol_3` AS `float64_col`, + `bfcol_4` AS `float64_w_safe` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_float/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_float/out.sql index 7f7bd86084e..32c8da56fa4 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_float/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_float/out.sql @@ -1,5 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(CAST(`bool_col` AS INT64) AS FLOAT64) AS `bfcol_1`, + CAST('1.34235e4' AS FLOAT64) AS `bfcol_2`, + SAFE_CAST(SAFE_CAST(`bool_col` AS INT64) AS FLOAT64) AS `bfcol_3` + FROM `bfcte_0` +) SELECT - CAST(CAST(`bool_col` AS INT64) AS FLOAT64) AS `bool_col`, - CAST('1.34235e4' AS FLOAT64) AS `str_const`, - SAFE_CAST(SAFE_CAST(`bool_col` AS INT64) AS FLOAT64) AS `bool_w_safe` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + `bfcol_1` AS `bool_col`, + `bfcol_2` AS `str_const`, + `bfcol_3` AS `bool_w_safe` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_from_json/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_from_json/out.sql index c9450a92800..d1577c0664d 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_from_json/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_from_json/out.sql @@ -1,7 +1,21 @@ +WITH `bfcte_0` AS ( + SELECT + `json_col` + FROM `bigframes-dev`.`sqlglot_test`.`json_types` +), `bfcte_1` AS ( + SELECT + *, + INT64(`json_col`) AS `bfcol_1`, + FLOAT64(`json_col`) AS `bfcol_2`, + BOOL(`json_col`) AS `bfcol_3`, + STRING(`json_col`) AS `bfcol_4`, + SAFE.INT64(`json_col`) AS `bfcol_5` + FROM `bfcte_0` +) SELECT - SAFE.INT64(`json_col`) AS `int64_col`, - SAFE.FLOAT64(`json_col`) AS `float64_col`, - SAFE.BOOL(`json_col`) AS `bool_col`, - SAFE.STRING(`json_col`) AS `string_col`, - SAFE.INT64(`json_col`) AS `int64_w_safe` -FROM `bigframes-dev`.`sqlglot_test`.`json_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `int64_col`, + `bfcol_2` AS `float64_col`, + `bfcol_3` AS `bool_col`, + `bfcol_4` AS `string_col`, + `bfcol_5` AS `int64_w_safe` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_int/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_int/out.sql index 8d44c674dc9..e0fe2af9a9d 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_int/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_int/out.sql @@ -1,11 +1,33 @@ +WITH `bfcte_0` AS ( + SELECT + `datetime_col`, + `float64_col`, + `numeric_col`, + `time_col`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + UNIX_MICROS(CAST(`datetime_col` AS TIMESTAMP)) AS `bfcol_5`, + UNIX_MICROS(SAFE_CAST(`datetime_col` AS TIMESTAMP)) AS `bfcol_6`, + TIME_DIFF(CAST(`time_col` AS TIME), '00:00:00', MICROSECOND) AS `bfcol_7`, + TIME_DIFF(SAFE_CAST(`time_col` AS TIME), '00:00:00', MICROSECOND) AS `bfcol_8`, + UNIX_MICROS(`timestamp_col`) AS `bfcol_9`, + CAST(TRUNC(`numeric_col`) AS INT64) AS `bfcol_10`, + CAST(TRUNC(`float64_col`) AS INT64) AS `bfcol_11`, + SAFE_CAST(TRUNC(`float64_col`) AS INT64) AS `bfcol_12`, + CAST('100' AS INT64) AS `bfcol_13` + FROM `bfcte_0` +) SELECT - UNIX_MICROS(CAST(`datetime_col` AS TIMESTAMP)) AS `datetime_col`, - UNIX_MICROS(SAFE_CAST(`datetime_col` AS TIMESTAMP)) AS `datetime_w_safe`, - TIME_DIFF(CAST(`time_col` AS TIME), '00:00:00', MICROSECOND) AS `time_col`, - TIME_DIFF(SAFE_CAST(`time_col` AS TIME), '00:00:00', MICROSECOND) AS `time_w_safe`, - UNIX_MICROS(`timestamp_col`) AS `timestamp_col`, - CAST(TRUNC(`numeric_col`) AS INT64) AS `numeric_col`, - CAST(TRUNC(`float64_col`) AS INT64) AS `float64_col`, - SAFE_CAST(TRUNC(`float64_col`) AS INT64) AS `float64_w_safe`, - CAST('100' AS INT64) AS `str_const` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_5` AS `datetime_col`, + `bfcol_6` AS `datetime_w_safe`, + `bfcol_7` AS `time_col`, + `bfcol_8` AS `time_w_safe`, + `bfcol_9` AS `timestamp_col`, + `bfcol_10` AS `numeric_col`, + `bfcol_11` AS `float64_col`, + `bfcol_12` AS `float64_w_safe`, + `bfcol_13` AS `str_const` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_json/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_json/out.sql index b62cee83a91..2defc2e72b0 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_json/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_json/out.sql @@ -1,8 +1,26 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `float64_col`, + `int64_col`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + PARSE_JSON(CAST(`int64_col` AS STRING)) AS `bfcol_4`, + PARSE_JSON(CAST(`float64_col` AS STRING)) AS `bfcol_5`, + PARSE_JSON(CAST(`bool_col` AS STRING)) AS `bfcol_6`, + PARSE_JSON(`string_col`) AS `bfcol_7`, + PARSE_JSON(CAST(`bool_col` AS STRING)) AS `bfcol_8`, + PARSE_JSON_IN_SAFE(`string_col`) AS `bfcol_9` + FROM `bfcte_0` +) SELECT - PARSE_JSON(CAST(`int64_col` AS STRING)) AS `int64_col`, - PARSE_JSON(CAST(`float64_col` AS STRING)) AS `float64_col`, - PARSE_JSON(CAST(`bool_col` AS STRING)) AS `bool_col`, - PARSE_JSON(`string_col`) AS `string_col`, - PARSE_JSON(CAST(`bool_col` AS STRING)) AS `bool_w_safe`, - SAFE.PARSE_JSON(`string_col`) AS `string_w_safe` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_4` AS `int64_col`, + `bfcol_5` AS `float64_col`, + `bfcol_6` AS `bool_col`, + `bfcol_7` AS `string_col`, + `bfcol_8` AS `bool_w_safe`, + `bfcol_9` AS `string_w_safe` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_string/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_string/out.sql index 174f18d9823..da6eb6ce187 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_string/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_string/out.sql @@ -1,5 +1,18 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(`int64_col` AS STRING) AS `bfcol_2`, + INITCAP(CAST(`bool_col` AS STRING)) AS `bfcol_3`, + INITCAP(SAFE_CAST(`bool_col` AS STRING)) AS `bfcol_4` + FROM `bfcte_0` +) SELECT - CAST(`int64_col` AS STRING) AS `int64_col`, - INITCAP(CAST(`bool_col` AS STRING)) AS `bool_col`, - INITCAP(SAFE_CAST(`bool_col` AS STRING)) AS `bool_w_safe` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + `bfcol_2` AS `int64_col`, + `bfcol_3` AS `bool_col`, + `bfcol_4` AS `bool_w_safe` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_time_like/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_time_like/out.sql index f50505592bb..6523d8376cc 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_time_like/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_astype_time_like/out.sql @@ -1,6 +1,19 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(TIMESTAMP_MICROS(`int64_col`) AS DATETIME) AS `bfcol_1`, + CAST(TIMESTAMP_MICROS(`int64_col`) AS TIME) AS `bfcol_2`, + CAST(TIMESTAMP_MICROS(`int64_col`) AS TIMESTAMP) AS `bfcol_3`, + SAFE_CAST(TIMESTAMP_MICROS(`int64_col`) AS TIME) AS `bfcol_4` + FROM `bfcte_0` +) SELECT - CAST(TIMESTAMP_MICROS(`int64_col`) AS DATETIME) AS `int64_to_datetime`, - CAST(TIMESTAMP_MICROS(`int64_col`) AS TIME) AS `int64_to_time`, - CAST(TIMESTAMP_MICROS(`int64_col`) AS TIMESTAMP) AS `int64_to_timestamp`, - SAFE_CAST(TIMESTAMP_MICROS(`int64_col`) AS TIME) AS `int64_to_time_safe` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `int64_to_datetime`, + `bfcol_2` AS `int64_to_time`, + `bfcol_3` AS `int64_to_timestamp`, + `bfcol_4` AS `int64_to_time_safe` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_binary_remote_function_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_binary_remote_function_op/out.sql index 29f9d69cb25..7272a3a5be1 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_binary_remote_function_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_binary_remote_function_op/out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `my_project`.`my_dataset`.`my_routine`(`int64_col`, `float64_col`) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - `my_project`.`my_dataset`.`my_routine`(`int64_col`, `float64_col`) AS `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `int64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_case_when_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_case_when_op/out.sql index 58e901fecc0..08a489e2401 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_case_when_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_case_when_op/out.sql @@ -1,13 +1,29 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `float64_col`, + `int64_col`, + `int64_too` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE WHEN `bool_col` THEN `int64_col` END AS `bfcol_4`, + CASE WHEN `bool_col` THEN `int64_col` WHEN `bool_col` THEN `int64_too` END AS `bfcol_5`, + CASE WHEN `bool_col` THEN `bool_col` WHEN `bool_col` THEN `bool_col` END AS `bfcol_6`, + CASE + WHEN `bool_col` + THEN `int64_col` + WHEN `bool_col` + THEN CAST(`bool_col` AS INT64) + WHEN `bool_col` + THEN `float64_col` + END AS `bfcol_7` + FROM `bfcte_0` +) SELECT - CASE WHEN `bool_col` THEN `int64_col` END AS `single_case`, - CASE WHEN `bool_col` THEN `int64_col` WHEN `bool_col` THEN `int64_too` END AS `double_case`, - CASE WHEN `bool_col` THEN `bool_col` WHEN `bool_col` THEN `bool_col` END AS `bool_types_case`, - CASE - WHEN `bool_col` - THEN `int64_col` - WHEN `bool_col` - THEN CAST(`bool_col` AS INT64) - WHEN `bool_col` - THEN `float64_col` - END AS `mixed_types_cast` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_4` AS `single_case`, + `bfcol_5` AS `double_case`, + `bfcol_6` AS `bool_types_case`, + `bfcol_7` AS `mixed_types_cast` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_clip/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_clip/out.sql index bbfeb304181..b1625931478 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_clip/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_clip/out.sql @@ -1,3 +1,15 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `int64_too`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + GREATEST(LEAST(`rowindex`, `int64_too`), `int64_col`) AS `bfcol_3` + FROM `bfcte_0` +) SELECT - GREATEST(LEAST(`rowindex`, `int64_too`), `int64_col`) AS `result_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_3` AS `result_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_coalesce/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_coalesce/out.sql index 4f88ec71d88..451de48b642 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_coalesce/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_coalesce/out.sql @@ -1,4 +1,16 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `int64_too` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `int64_col` AS `bfcol_2`, + COALESCE(`int64_too`, `int64_col`) AS `bfcol_3` + FROM `bfcte_0` +) SELECT - `int64_col`, - COALESCE(`int64_too`, `int64_col`) AS `int64_too` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `int64_col`, + `bfcol_3` AS `int64_too` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_fillna/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_fillna/out.sql index ae6f975da5a..07f2877e740 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_fillna/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_fillna/out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + COALESCE(`int64_col`, `float64_col`) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - COALESCE(`int64_col`, `float64_col`) AS `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `int64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_hash/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_hash/out.sql index b1afe9db39b..19fce600910 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_hash/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_hash/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + FARM_FINGERPRINT(`string_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - FARM_FINGERPRINT(`string_col`) AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_invert/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_invert/out.sql index 5cd1b15a776..1bd2eb7426c 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_invert/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_invert/out.sql @@ -1,11 +1,25 @@ -SELECT - ~( +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `bytes_col`, `int64_col` - ) AS `int64_col`, - ~( - `bytes_col` - ) AS `bytes_col`, - NOT ( - `bool_col` - ) AS `bool_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ~( + `int64_col` + ) AS `bfcol_6`, + ~( + `bytes_col` + ) AS `bfcol_7`, + NOT ( + `bool_col` + ) AS `bfcol_8` + FROM `bfcte_0` +) +SELECT + `bfcol_6` AS `int64_col`, + `bfcol_7` AS `bytes_col`, + `bfcol_8` AS `bool_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_isnull/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_isnull/out.sql index cfe38ae3600..0a549bdd442 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_isnull/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_isnull/out.sql @@ -1,5 +1,13 @@ -SELECT - ( +WITH `bfcte_0` AS ( + SELECT `float64_col` - ) IS NULL AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `float64_col` IS NULL AS `bfcol_1` + FROM `bfcte_0` +) +SELECT + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_map/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_map/out.sql index 3b1d0446b3b..49eada22301 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_map/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_map/out.sql @@ -1,9 +1,19 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN `string_col` = 'value1' + THEN 'mapped1' + WHEN `string_col` IS NULL + THEN 'UNKNOWN' + ELSE `string_col` + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN `string_col` = 'value1' - THEN 'mapped1' - WHEN `string_col` IS NULL - THEN 'UNKNOWN' - ELSE `string_col` - END AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_nary_remote_function_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_nary_remote_function_op/out.sql index a1977d809f7..a6641b13db6 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_nary_remote_function_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_nary_remote_function_op/out.sql @@ -1,3 +1,15 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col`, + `int64_col`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `my_project`.`my_dataset`.`my_routine`(`int64_col`, `float64_col`, `string_col`) AS `bfcol_3` + FROM `bfcte_0` +) SELECT - `my_project`.`my_dataset`.`my_routine`(`int64_col`, `float64_col`, `string_col`) AS `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_3` AS `int64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_notnull/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_notnull/out.sql index 97b9f54f429..bf3425fe6de 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_notnull/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_notnull/out.sql @@ -1,5 +1,13 @@ -SELECT - ( +WITH `bfcte_0` AS ( + SELECT `float64_col` - ) IS NOT NULL AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + NOT `float64_col` IS NULL AS `bfcol_1` + FROM `bfcte_0` +) +SELECT + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_remote_function_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_remote_function_op/out.sql index a1977d809f7..dee0d35355b 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_remote_function_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_remote_function_op/out.sql @@ -1,3 +1,19 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `my_project`.`my_dataset`.`my_routine`(`int64_col`) AS `bfcol_1`, + IF( + `int64_col` IS NULL, + `int64_col`, + `my_project`.`my_dataset`.`my_routine`(`int64_col`) + ) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - `my_project`.`my_dataset`.`my_routine`(`int64_col`, `float64_col`, `string_col`) AS `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `apply_on_null_true`, + `bfcol_2` AS `apply_on_null_false` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_row_key/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_row_key/out.sql index f5bf9b3b6ee..13b27c2e146 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_row_key/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_row_key/out.sql @@ -1,46 +1,70 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `bytes_col`, + `date_col`, + `datetime_col`, + `duration_col`, + `float64_col`, + `geography_col`, + `int64_col`, + `int64_too`, + `numeric_col`, + `rowindex`, + `rowindex_2`, + `string_col`, + `time_col`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CONCAT( + CAST(FARM_FINGERPRINT( + CONCAT( + CONCAT('\\', REPLACE(COALESCE(CAST(`rowindex` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`bool_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`bytes_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`date_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`datetime_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(ST_ASTEXT(`geography_col`), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`int64_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`int64_too` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`numeric_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`float64_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`rowindex` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`rowindex_2` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(`string_col`, ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`time_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`timestamp_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`duration_col` AS STRING), ''), '\\', '\\\\')) + ) + ) AS STRING), + CAST(FARM_FINGERPRINT( + CONCAT( + CONCAT('\\', REPLACE(COALESCE(CAST(`rowindex` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`bool_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`bytes_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`date_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`datetime_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(ST_ASTEXT(`geography_col`), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`int64_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`int64_too` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`numeric_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`float64_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`rowindex` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`rowindex_2` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(`string_col`, ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`time_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`timestamp_col` AS STRING), ''), '\\', '\\\\')), + CONCAT('\\', REPLACE(COALESCE(CAST(`duration_col` AS STRING), ''), '\\', '\\\\')), + '_' + ) + ) AS STRING), + CAST(RAND() AS STRING) + ) AS `bfcol_31` + FROM `bfcte_0` +) SELECT - CONCAT( - CAST(FARM_FINGERPRINT( - CONCAT( - CONCAT('\\', REPLACE(COALESCE(CAST(`rowindex` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`bool_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`bytes_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`date_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`datetime_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(ST_ASTEXT(`geography_col`), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`int64_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`int64_too` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`numeric_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`float64_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`rowindex` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`rowindex_2` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(`string_col`, ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`time_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`timestamp_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`duration_col` AS STRING), ''), '\\', '\\\\')) - ) - ) AS STRING), - CAST(FARM_FINGERPRINT( - CONCAT( - CONCAT('\\', REPLACE(COALESCE(CAST(`rowindex` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`bool_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`bytes_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`date_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`datetime_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(ST_ASTEXT(`geography_col`), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`int64_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`int64_too` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`numeric_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`float64_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`rowindex` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`rowindex_2` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(`string_col`, ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`time_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`timestamp_col` AS STRING), ''), '\\', '\\\\')), - CONCAT('\\', REPLACE(COALESCE(CAST(`duration_col` AS STRING), ''), '\\', '\\\\')), - '_' - ) - ) AS STRING), - CAST(RAND() AS STRING) - ) AS `row_key` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_31` AS `row_key` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_sql_scalar_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_sql_scalar_op/out.sql index 8f50ff28ca4..611cbf4e7e8 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_sql_scalar_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_sql_scalar_op/out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `bytes_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CAST(`bool_col` AS INT64) + BYTE_LENGTH(`bytes_col`) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - CAST(`bool_col` AS INT64) + BYTE_LENGTH(`bytes_col`) AS `bool_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `bool_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_to_json/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_to_json/out.sql deleted file mode 100644 index 86d6f0e9fbb..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_to_json/out.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - IF(`int64_col` IS NULL, NULL, TO_JSON(`int64_col`)) AS `int64_col`, - IF(`float64_col` IS NULL, NULL, TO_JSON(`float64_col`)) AS `float64_col`, - IF(`bool_col` IS NULL, NULL, TO_JSON(`bool_col`)) AS `bool_col`, - SAFE.PARSE_JSON(`string_col`) AS `string_col`, - IF(`bool_col` IS NULL, NULL, TO_JSON(`bool_col`)) AS `bool_w_safe`, - SAFE.PARSE_JSON(`string_col`) AS `string_w_safe` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_where/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_where/out.sql index 1ca3b009898..872c7943335 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_where/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_where/out.sql @@ -1,3 +1,15 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + IF(`bool_col`, `int64_col`, `float64_col`) AS `bfcol_3` + FROM `bfcte_0` +) SELECT - IF(`bool_col`, `int64_col`, `float64_col`) AS `result_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_3` AS `result_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_area/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_area/out.sql index 78c786b036e..105b5f1665d 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_area/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_area/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_AREA(`geography_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ST_AREA(`geography_col`) AS `geography_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `geography_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_astext/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_astext/out.sql index 526c0c37d7e..c338baeb5f1 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_astext/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_astext/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_ASTEXT(`geography_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ST_ASTEXT(`geography_col`) AS `geography_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `geography_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_boundary/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_boundary/out.sql index 4bf43469cf5..2d4ac2e9609 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_boundary/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_boundary/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_BOUNDARY(`geography_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ST_BOUNDARY(`geography_col`) AS `geography_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `geography_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_buffer/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_buffer/out.sql index 40669569fbb..84b3ab1600e 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_buffer/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_buffer/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_BUFFER(`geography_col`, 1.0, 8.0, FALSE) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ST_BUFFER(`geography_col`, 1.0, 8.0, FALSE) AS `geography_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `geography_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_centroid/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_centroid/out.sql index accd33bd627..733f1e9495b 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_centroid/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_centroid/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_CENTROID(`geography_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ST_CENTROID(`geography_col`) AS `geography_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `geography_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_convexhull/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_convexhull/out.sql index e4a718d42a9..11b3b7f6917 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_convexhull/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_convexhull/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_CONVEXHULL(`geography_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ST_CONVEXHULL(`geography_col`) AS `geography_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `geography_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_difference/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_difference/out.sql index 2a17ef1c7c8..4e18216ddac 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_difference/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_difference/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_DIFFERENCE(`geography_col`, `geography_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ST_DIFFERENCE(`geography_col`, `geography_col`) AS `geography_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `geography_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_distance/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_distance/out.sql index 4c55ddd0824..e98a581de72 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_distance/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_distance/out.sql @@ -1,4 +1,15 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_DISTANCE(`geography_col`, `geography_col`, TRUE) AS `bfcol_1`, + ST_DISTANCE(`geography_col`, `geography_col`, FALSE) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - ST_DISTANCE(`geography_col`, `geography_col`, TRUE) AS `spheroid`, - ST_DISTANCE(`geography_col`, `geography_col`, FALSE) AS `no_spheroid` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `spheroid`, + `bfcol_2` AS `no_spheroid` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_geogfromtext/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_geogfromtext/out.sql index db62766d4c9..1bbb1143493 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_geogfromtext/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_geogfromtext/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + SAFE.ST_GEOGFROMTEXT(`string_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - SAFE.ST_GEOGFROMTEXT(`string_col`) AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_geogpoint/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_geogpoint/out.sql index 3299ef0bd2d..f6c953d161a 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_geogpoint/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_geogpoint/out.sql @@ -1,3 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `rowindex`, + `rowindex_2` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_GEOGPOINT(`rowindex`, `rowindex_2`) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - ST_GEOGPOINT(`rowindex`, `rowindex_2`) AS `rowindex` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `rowindex` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_intersection/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_intersection/out.sql index a615ddf042e..f9290fe01a6 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_intersection/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_intersection/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_INTERSECTION(`geography_col`, `geography_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ST_INTERSECTION(`geography_col`, `geography_col`) AS `geography_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `geography_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_isclosed/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_isclosed/out.sql index 4f04e70b569..516f175c13b 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_isclosed/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_isclosed/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_ISCLOSED(`geography_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ST_ISCLOSED(`geography_col`) AS `geography_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `geography_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_length/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_length/out.sql index ee64b20ca46..80eef1c906e 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_length/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_st_length/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_LENGTH(`geography_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ST_LENGTH(`geography_col`) AS `geography_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `geography_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_x/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_x/out.sql index c1ab623d9a1..826eb9f209d 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_x/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_x/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_X(`geography_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ST_X(`geography_col`) AS `geography_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `geography_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_y/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_y/out.sql index e7575606e6e..dd411820b28 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_y/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_geo_ops/test_geo_y/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `geography_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ST_Y(`geography_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ST_Y(`geography_col`) AS `geography_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `geography_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_extract/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_extract/out.sql index 7a7ad2f394d..435ee96df15 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_extract/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_extract/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `json_col` + FROM `bigframes-dev`.`sqlglot_test`.`json_types` +), `bfcte_1` AS ( + SELECT + *, + JSON_EXTRACT(`json_col`, '$') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - JSON_EXTRACT(`json_col`, '$') AS `json_col` -FROM `bigframes-dev`.`sqlglot_test`.`json_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `json_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_extract_array/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_extract_array/out.sql index f2c4cd72985..6c9c02594d9 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_extract_array/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_extract_array/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `json_col` + FROM `bigframes-dev`.`sqlglot_test`.`json_types` +), `bfcte_1` AS ( + SELECT + *, + JSON_EXTRACT_ARRAY(`json_col`, '$') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - JSON_EXTRACT_ARRAY(`json_col`, '$') AS `json_col` -FROM `bigframes-dev`.`sqlglot_test`.`json_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `json_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_extract_string_array/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_extract_string_array/out.sql index 61e8bae8a32..a3a51be3781 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_extract_string_array/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_extract_string_array/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `json_col` + FROM `bigframes-dev`.`sqlglot_test`.`json_types` +), `bfcte_1` AS ( + SELECT + *, + JSON_EXTRACT_STRING_ARRAY(`json_col`, '$') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - JSON_EXTRACT_STRING_ARRAY(`json_col`, '$') AS `json_col` -FROM `bigframes-dev`.`sqlglot_test`.`json_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `json_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_keys/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_keys/out.sql index 78004c1180c..640f933bb2b 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_keys/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_keys/out.sql @@ -1,4 +1,15 @@ +WITH `bfcte_0` AS ( + SELECT + `json_col` + FROM `bigframes-dev`.`sqlglot_test`.`json_types` +), `bfcte_1` AS ( + SELECT + *, + JSON_KEYS(`json_col`, NULL) AS `bfcol_1`, + JSON_KEYS(`json_col`, 2) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - JSON_KEYS(`json_col`, NULL) AS `json_keys`, - JSON_KEYS(`json_col`, 2) AS `json_keys_w_max_depth` -FROM `bigframes-dev`.`sqlglot_test`.`json_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `json_keys`, + `bfcol_2` AS `json_keys_w_max_depth` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_query/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_query/out.sql index 8aa312e9d75..164fe2e4267 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_query/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_query/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `json_col` + FROM `bigframes-dev`.`sqlglot_test`.`json_types` +), `bfcte_1` AS ( + SELECT + *, + JSON_QUERY(`json_col`, '$') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - JSON_QUERY(`json_col`, '$') AS `json_col` -FROM `bigframes-dev`.`sqlglot_test`.`json_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `json_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_query_array/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_query_array/out.sql index 898068fe595..4c3fa8e7e9b 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_query_array/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_query_array/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `json_col` + FROM `bigframes-dev`.`sqlglot_test`.`json_types` +), `bfcte_1` AS ( + SELECT + *, + JSON_QUERY_ARRAY(`json_col`, '$') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - JSON_QUERY_ARRAY(`json_col`, '$') AS `json_col` -FROM `bigframes-dev`.`sqlglot_test`.`json_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `json_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_set/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_set/out.sql index e515d5fdc3b..f41979ea2e8 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_set/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_set/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `json_col` + FROM `bigframes-dev`.`sqlglot_test`.`json_types` +), `bfcte_1` AS ( + SELECT + *, + JSON_SET(`json_col`, '$.a', 100) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - JSON_SET(`json_col`, '$.a', 100) AS `json_col` -FROM `bigframes-dev`.`sqlglot_test`.`json_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `json_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_value/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_value/out.sql index c9a73ae1942..72f72372409 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_value/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_value/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `json_col` + FROM `bigframes-dev`.`sqlglot_test`.`json_types` +), `bfcte_1` AS ( + SELECT + *, + JSON_VALUE(`json_col`, '$') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - JSON_VALUE(`json_col`, '$') AS `json_col` -FROM `bigframes-dev`.`sqlglot_test`.`json_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `json_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_value_array/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_value_array/out.sql deleted file mode 100644 index 8250c02934e..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_json_value_array/out.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT - JSON_VALUE_ARRAY(`json_col`, '$') AS `json_col` -FROM `bigframes-dev`.`sqlglot_test`.`json_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_parse_json/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_parse_json/out.sql index 55a195edf20..5f80187ba0c 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_parse_json/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_parse_json/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + PARSE_JSON(`string_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - PARSE_JSON(`string_col`) AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_to_json/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_to_json/out.sql index 0545577e27f..ebca0c51c52 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_to_json/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_to_json/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + TO_JSON(`string_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - SAFE.PARSE_JSON(`string_col`) AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_to_json_string/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_to_json_string/out.sql index 62886c26ed9..e282c89c80e 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_to_json_string/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_json_ops/test_to_json_string/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `json_col` + FROM `bigframes-dev`.`sqlglot_test`.`json_types` +), `bfcte_1` AS ( + SELECT + *, + TO_JSON_STRING(`json_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - TO_JSON_STRING(`json_col`) AS `json_col` -FROM `bigframes-dev`.`sqlglot_test`.`json_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `json_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_literals/test_float_literals/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_literals/test_float_literals/out.sql deleted file mode 100644 index 030e733edd7..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_literals/test_float_literals/out.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - CAST('Infinity' AS FLOAT64) AS `inf`, - CAST('-Infinity' AS FLOAT64) AS `ninf`, - NULL AS `nan`, - -0.0 AS `neg_zero`, - 1e-05 AS `0.00001`, - 1e-10 AS `1E-10` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_abs/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_abs/out.sql index bc53c60895a..0fb9589387a 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_abs/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_abs/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ABS(`float64_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ABS(`float64_col`) AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_add_numeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_add_numeric/out.sql index 3aa06fe16e3..1707aad8c1f 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_add_numeric/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_add_numeric/out.sql @@ -1,10 +1,54 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `bool_col` AS `bfcol_8`, + `int64_col` + `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_7` + 1 AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + `bfcol_15` + CAST(`bfcol_16` AS INT64) AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + CAST(`bfcol_26` AS INT64) + `bfcol_25` AS `bfcol_42` + FROM `bfcte_3` +) SELECT - `rowindex`, - `int64_col`, - `bool_col`, - `int64_col` + `int64_col` AS `int_add_int`, - `int64_col` + 1 AS `int_add_1`, - NULL AS `int_add_null`, - `int64_col` + CAST(`bool_col` AS INT64) AS `int_add_bool`, - CAST(`bool_col` AS INT64) + `int64_col` AS `bool_add_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_36` AS `rowindex`, + `bfcol_37` AS `int64_col`, + `bfcol_38` AS `bool_col`, + `bfcol_39` AS `int_add_int`, + `bfcol_40` AS `int_add_1`, + `bfcol_41` AS `int_add_bool`, + `bfcol_42` AS `bool_add_int` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_add_string/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_add_string/out.sql index cf4051464b7..cb674787ff1 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_add_string/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_add_string/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CONCAT(`string_col`, 'a') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CONCAT(`string_col`, 'a') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_add_timedelta/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_add_timedelta/out.sql index b1ccf096cfa..2fef18eeb8a 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_add_timedelta/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_add_timedelta/out.sql @@ -1,10 +1,60 @@ +WITH `bfcte_0` AS ( + SELECT + `date_col`, + `rowindex`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `timestamp_col` AS `bfcol_7`, + `date_col` AS `bfcol_8`, + TIMESTAMP_ADD(CAST(`date_col` AS DATETIME), INTERVAL 86400000000 MICROSECOND) AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + TIMESTAMP_ADD(`bfcol_7`, INTERVAL 86400000000 MICROSECOND) AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + TIMESTAMP_ADD(CAST(`bfcol_16` AS DATETIME), INTERVAL 86400000000 MICROSECOND) AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + TIMESTAMP_ADD(`bfcol_25`, INTERVAL 86400000000 MICROSECOND) AS `bfcol_42` + FROM `bfcte_3` +), `bfcte_5` AS ( + SELECT + *, + 172800000000 AS `bfcol_50` + FROM `bfcte_4` +) SELECT - `rowindex`, - `timestamp_col`, - `date_col`, - TIMESTAMP_ADD(CAST(`date_col` AS DATETIME), INTERVAL 86400000000 MICROSECOND) AS `date_add_timedelta`, - TIMESTAMP_ADD(`timestamp_col`, INTERVAL 86400000000 MICROSECOND) AS `timestamp_add_timedelta`, - TIMESTAMP_ADD(CAST(`date_col` AS DATETIME), INTERVAL 86400000000 MICROSECOND) AS `timedelta_add_date`, - TIMESTAMP_ADD(`timestamp_col`, INTERVAL 86400000000 MICROSECOND) AS `timedelta_add_timestamp`, - 172800000000 AS `timedelta_add_timedelta` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_36` AS `rowindex`, + `bfcol_37` AS `timestamp_col`, + `bfcol_38` AS `date_col`, + `bfcol_39` AS `date_add_timedelta`, + `bfcol_40` AS `timestamp_add_timedelta`, + `bfcol_41` AS `timedelta_add_date`, + `bfcol_42` AS `timedelta_add_timestamp`, + `bfcol_50` AS `timedelta_add_timedelta` +FROM `bfcte_5` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arccos/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arccos/out.sql index d00086dfde8..bb1766adf35 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arccos/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arccos/out.sql @@ -1,7 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN ABS(`float64_col`) > 1 + THEN CAST('NaN' AS FLOAT64) + ELSE ACOS(`float64_col`) + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN ABS(`float64_col`) > 1 - THEN CAST('NaN' AS FLOAT64) - ELSE ACOS(`float64_col`) - END AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arccosh/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arccosh/out.sql index f1a04757a0a..af556b9c3a3 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arccosh/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arccosh/out.sql @@ -1,7 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN `float64_col` < 1 + THEN CAST('NaN' AS FLOAT64) + ELSE ACOSH(`float64_col`) + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN `float64_col` < 1 - THEN CAST('NaN' AS FLOAT64) - ELSE ACOSH(`float64_col`) - END AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arcsin/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arcsin/out.sql index eff8f1f5007..8243232e0b5 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arcsin/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arcsin/out.sql @@ -1,7 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN ABS(`float64_col`) > 1 + THEN CAST('NaN' AS FLOAT64) + ELSE ASIN(`float64_col`) + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN ABS(`float64_col`) > 1 - THEN CAST('NaN' AS FLOAT64) - ELSE ASIN(`float64_col`) - END AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arcsinh/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arcsinh/out.sql index 557407f09ee..e6bf3b339c0 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arcsinh/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arcsinh/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ASINH(`float64_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ASINH(`float64_col`) AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arctan/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arctan/out.sql index d99b62f2cdb..a85ff6403cb 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arctan/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arctan/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ATAN(`float64_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ATAN(`float64_col`) AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arctan2/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arctan2/out.sql index 463896e981f..28fc8c869d7 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arctan2/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arctan2/out.sql @@ -1,4 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ATAN2(`int64_col`, `float64_col`) AS `bfcol_6`, + ATAN2(CAST(`bool_col` AS INT64), `float64_col`) AS `bfcol_7` + FROM `bfcte_0` +) SELECT - ATAN2(`int64_col`, `float64_col`) AS `int64_col`, - ATAN2(CAST(`bool_col` AS INT64), `float64_col`) AS `bool_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_6` AS `int64_col`, + `bfcol_7` AS `bool_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arctanh/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arctanh/out.sql index 9b016071480..dc6de62e7bc 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arctanh/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_arctanh/out.sql @@ -1,9 +1,19 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN ABS(`float64_col`) < 1 + THEN ATANH(`float64_col`) + WHEN ABS(`float64_col`) > 1 + THEN CAST('NaN' AS FLOAT64) + ELSE CAST('Infinity' AS FLOAT64) * `float64_col` + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN ABS(`float64_col`) < 1 - THEN ATANH(`float64_col`) - WHEN ABS(`float64_col`) > 1 - THEN CAST('NaN' AS FLOAT64) - ELSE CAST('Infinity' AS FLOAT64) * `float64_col` - END AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_ceil/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_ceil/out.sql index f69ae7f2760..922fe5c5508 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_ceil/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_ceil/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CEIL(`float64_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CEIL(`float64_col`) AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_cos/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_cos/out.sql index 427dfbb9a93..0acb2bfa944 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_cos/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_cos/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + COS(`float64_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - COS(`float64_col`) AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_cosh/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_cosh/out.sql index 0f119c254f0..8c84a250475 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_cosh/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_cosh/out.sql @@ -1,7 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN ABS(`float64_col`) > 709.78 + THEN CAST('Infinity' AS FLOAT64) + ELSE COSH(`float64_col`) + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN ABS(`float64_col`) > 709.78 - THEN CAST('Infinity' AS FLOAT64) - ELSE COSH(`float64_col`) - END AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_cosine_distance/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_cosine_distance/out.sql index 1c482fc8a78..ba6b6bfa9fa 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_cosine_distance/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_cosine_distance/out.sql @@ -1,4 +1,16 @@ +WITH `bfcte_0` AS ( + SELECT + `float_list_col`, + `int_list_col` + FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` +), `bfcte_1` AS ( + SELECT + *, + ML.DISTANCE(`int_list_col`, `int_list_col`, 'COSINE') AS `bfcol_2`, + ML.DISTANCE(`float_list_col`, `float_list_col`, 'COSINE') AS `bfcol_3` + FROM `bfcte_0` +) SELECT - ML.DISTANCE(`int_list_col`, `int_list_col`, 'COSINE') AS `int_list_col`, - ML.DISTANCE(`float_list_col`, `float_list_col`, 'COSINE') AS `float_list_col` -FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `int_list_col`, + `bfcol_3` AS `float_list_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_div_numeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_div_numeric/out.sql index e2ccf96410a..db11f1529fa 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_div_numeric/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_div_numeric/out.sql @@ -1,15 +1,122 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `float64_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_8`, + `int64_col` AS `bfcol_9`, + `bool_col` AS `bfcol_10`, + `float64_col` AS `bfcol_11`, + IEEE_DIVIDE(`int64_col`, `int64_col`) AS `bfcol_12` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_8` AS `bfcol_18`, + `bfcol_9` AS `bfcol_19`, + `bfcol_10` AS `bfcol_20`, + `bfcol_11` AS `bfcol_21`, + `bfcol_12` AS `bfcol_22`, + IEEE_DIVIDE(`bfcol_9`, 1) AS `bfcol_23` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_18` AS `bfcol_30`, + `bfcol_19` AS `bfcol_31`, + `bfcol_20` AS `bfcol_32`, + `bfcol_21` AS `bfcol_33`, + `bfcol_22` AS `bfcol_34`, + `bfcol_23` AS `bfcol_35`, + IEEE_DIVIDE(`bfcol_19`, 0.0) AS `bfcol_36` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_30` AS `bfcol_44`, + `bfcol_31` AS `bfcol_45`, + `bfcol_32` AS `bfcol_46`, + `bfcol_33` AS `bfcol_47`, + `bfcol_34` AS `bfcol_48`, + `bfcol_35` AS `bfcol_49`, + `bfcol_36` AS `bfcol_50`, + IEEE_DIVIDE(`bfcol_31`, `bfcol_33`) AS `bfcol_51` + FROM `bfcte_3` +), `bfcte_5` AS ( + SELECT + *, + `bfcol_44` AS `bfcol_60`, + `bfcol_45` AS `bfcol_61`, + `bfcol_46` AS `bfcol_62`, + `bfcol_47` AS `bfcol_63`, + `bfcol_48` AS `bfcol_64`, + `bfcol_49` AS `bfcol_65`, + `bfcol_50` AS `bfcol_66`, + `bfcol_51` AS `bfcol_67`, + IEEE_DIVIDE(`bfcol_47`, `bfcol_45`) AS `bfcol_68` + FROM `bfcte_4` +), `bfcte_6` AS ( + SELECT + *, + `bfcol_60` AS `bfcol_78`, + `bfcol_61` AS `bfcol_79`, + `bfcol_62` AS `bfcol_80`, + `bfcol_63` AS `bfcol_81`, + `bfcol_64` AS `bfcol_82`, + `bfcol_65` AS `bfcol_83`, + `bfcol_66` AS `bfcol_84`, + `bfcol_67` AS `bfcol_85`, + `bfcol_68` AS `bfcol_86`, + IEEE_DIVIDE(`bfcol_63`, 0.0) AS `bfcol_87` + FROM `bfcte_5` +), `bfcte_7` AS ( + SELECT + *, + `bfcol_78` AS `bfcol_98`, + `bfcol_79` AS `bfcol_99`, + `bfcol_80` AS `bfcol_100`, + `bfcol_81` AS `bfcol_101`, + `bfcol_82` AS `bfcol_102`, + `bfcol_83` AS `bfcol_103`, + `bfcol_84` AS `bfcol_104`, + `bfcol_85` AS `bfcol_105`, + `bfcol_86` AS `bfcol_106`, + `bfcol_87` AS `bfcol_107`, + IEEE_DIVIDE(`bfcol_79`, CAST(`bfcol_80` AS INT64)) AS `bfcol_108` + FROM `bfcte_6` +), `bfcte_8` AS ( + SELECT + *, + `bfcol_98` AS `bfcol_120`, + `bfcol_99` AS `bfcol_121`, + `bfcol_100` AS `bfcol_122`, + `bfcol_101` AS `bfcol_123`, + `bfcol_102` AS `bfcol_124`, + `bfcol_103` AS `bfcol_125`, + `bfcol_104` AS `bfcol_126`, + `bfcol_105` AS `bfcol_127`, + `bfcol_106` AS `bfcol_128`, + `bfcol_107` AS `bfcol_129`, + `bfcol_108` AS `bfcol_130`, + IEEE_DIVIDE(CAST(`bfcol_100` AS INT64), `bfcol_99`) AS `bfcol_131` + FROM `bfcte_7` +) SELECT - `rowindex`, - `int64_col`, - `bool_col`, - `float64_col`, - IEEE_DIVIDE(`int64_col`, `int64_col`) AS `int_div_int`, - IEEE_DIVIDE(`int64_col`, 1) AS `int_div_1`, - IEEE_DIVIDE(`int64_col`, 0.0) AS `int_div_0`, - NULL AS `int_div_null`, - IEEE_DIVIDE(`int64_col`, `float64_col`) AS `int_div_float`, - IEEE_DIVIDE(`float64_col`, `int64_col`) AS `float_div_int`, - IEEE_DIVIDE(`float64_col`, 0.0) AS `float_div_0`, - IEEE_DIVIDE(`int64_col`, CAST(`bool_col` AS INT64)) AS `int_div_bool`, - IEEE_DIVIDE(CAST(`bool_col` AS INT64), `int64_col`) AS `bool_div_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_120` AS `rowindex`, + `bfcol_121` AS `int64_col`, + `bfcol_122` AS `bool_col`, + `bfcol_123` AS `float64_col`, + `bfcol_124` AS `int_div_int`, + `bfcol_125` AS `int_div_1`, + `bfcol_126` AS `int_div_0`, + `bfcol_127` AS `int_div_float`, + `bfcol_128` AS `float_div_int`, + `bfcol_129` AS `float_div_0`, + `bfcol_130` AS `int_div_bool`, + `bfcol_131` AS `bool_div_int` +FROM `bfcte_8` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_div_timedelta/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_div_timedelta/out.sql index a733ed81278..1a82a67368c 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_div_timedelta/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_div_timedelta/out.sql @@ -1,10 +1,21 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `rowindex`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `timestamp_col` AS `bfcol_7`, + `int64_col` AS `bfcol_8`, + CAST(FLOOR(IEEE_DIVIDE(86400000000, `int64_col`)) AS INT64) AS `bfcol_9` + FROM `bfcte_0` +) SELECT - `rowindex`, - `timestamp_col`, - `int64_col`, - CAST(IF( - IEEE_DIVIDE(86400000000, `int64_col`) > 0, - FLOOR(IEEE_DIVIDE(86400000000, `int64_col`)), - CEIL(IEEE_DIVIDE(86400000000, `int64_col`)) - ) AS INT64) AS `timedelta_div_numeric` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_6` AS `rowindex`, + `bfcol_7` AS `timestamp_col`, + `bfcol_8` AS `int64_col`, + `bfcol_9` AS `timedelta_div_numeric` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_euclidean_distance/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_euclidean_distance/out.sql index 349d78584a9..3327a99f4b6 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_euclidean_distance/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_euclidean_distance/out.sql @@ -1,4 +1,16 @@ +WITH `bfcte_0` AS ( + SELECT + `int_list_col`, + `numeric_list_col` + FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` +), `bfcte_1` AS ( + SELECT + *, + ML.DISTANCE(`int_list_col`, `int_list_col`, 'EUCLIDEAN') AS `bfcol_2`, + ML.DISTANCE(`numeric_list_col`, `numeric_list_col`, 'EUCLIDEAN') AS `bfcol_3` + FROM `bfcte_0` +) SELECT - ML.DISTANCE(`int_list_col`, `int_list_col`, 'EUCLIDEAN') AS `int_list_col`, - ML.DISTANCE(`numeric_list_col`, `numeric_list_col`, 'EUCLIDEAN') AS `numeric_list_col` -FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `int_list_col`, + `bfcol_3` AS `numeric_list_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_exp/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_exp/out.sql index 178282ca087..610b96cda70 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_exp/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_exp/out.sql @@ -1,7 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN `float64_col` > 709.78 + THEN CAST('Infinity' AS FLOAT64) + ELSE EXP(`float64_col`) + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN `float64_col` > 709.78 - THEN CAST('Infinity' AS FLOAT64) - ELSE EXP(`float64_col`) - END AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_expm1/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_expm1/out.sql index 6c896448f24..13038bf8e85 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_expm1/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_expm1/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + IF(`float64_col` > 709.78, CAST('Infinity' AS FLOAT64), EXP(`float64_col`) - 1) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - IF(`float64_col` > 709.78, CAST('Infinity' AS FLOAT64), EXP(`float64_col`) - 1) AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_floor/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_floor/out.sql index 31b715623cb..e0c2e1072e8 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_floor/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_floor/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + FLOOR(`float64_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - FLOOR(`float64_col`) AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_floordiv_numeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_floordiv_numeric/out.sql deleted file mode 100644 index 8307b1b8ada..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_floordiv_numeric/out.sql +++ /dev/null @@ -1,48 +0,0 @@ -SELECT - `rowindex`, - `int64_col`, - `bool_col`, - `float64_col`, - CASE - WHEN `int64_col` = CAST(0 AS INT64) - THEN CAST(0 AS INT64) * `int64_col` - ELSE CAST(FLOOR(IEEE_DIVIDE(`int64_col`, `int64_col`)) AS INT64) - END AS `int_div_int`, - CASE - WHEN 1 = CAST(0 AS INT64) - THEN CAST(0 AS INT64) * `int64_col` - ELSE CAST(FLOOR(IEEE_DIVIDE(`int64_col`, 1)) AS INT64) - END AS `int_div_1`, - CASE - WHEN 0.0 = CAST(0 AS INT64) - THEN CAST('Infinity' AS FLOAT64) * `int64_col` - ELSE CAST(FLOOR(IEEE_DIVIDE(`int64_col`, 0.0)) AS INT64) - END AS `int_div_0`, - NULL AS `int_div_null`, - CASE - WHEN `float64_col` = CAST(0 AS INT64) - THEN CAST('Infinity' AS FLOAT64) * `int64_col` - ELSE CAST(FLOOR(IEEE_DIVIDE(`int64_col`, `float64_col`)) AS INT64) - END AS `int_div_float`, - CASE - WHEN `int64_col` = CAST(0 AS INT64) - THEN CAST('Infinity' AS FLOAT64) * `float64_col` - ELSE CAST(FLOOR(IEEE_DIVIDE(`float64_col`, `int64_col`)) AS INT64) - END AS `float_div_int`, - CASE - WHEN 0.0 = CAST(0 AS INT64) - THEN CAST('Infinity' AS FLOAT64) * `float64_col` - ELSE CAST(FLOOR(IEEE_DIVIDE(`float64_col`, 0.0)) AS INT64) - END AS `float_div_0`, - NULL AS `float_div_null`, - CASE - WHEN CAST(`bool_col` AS INT64) = CAST(0 AS INT64) - THEN CAST(0 AS INT64) * `int64_col` - ELSE CAST(FLOOR(IEEE_DIVIDE(`int64_col`, CAST(`bool_col` AS INT64))) AS INT64) - END AS `int_div_bool`, - CASE - WHEN `int64_col` = CAST(0 AS INT64) - THEN CAST(0 AS INT64) * CAST(`bool_col` AS INT64) - ELSE CAST(FLOOR(IEEE_DIVIDE(CAST(`bool_col` AS INT64), `int64_col`)) AS INT64) - END AS `bool_div_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_floordiv_timedelta/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_floordiv_timedelta/out.sql index 4d978991eb5..2fe20fb6188 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_floordiv_timedelta/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_floordiv_timedelta/out.sql @@ -1,6 +1,18 @@ +WITH `bfcte_0` AS ( + SELECT + `date_col`, + `rowindex`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + 43200000000 AS `bfcol_6` + FROM `bfcte_0` +) SELECT `rowindex`, `timestamp_col`, `date_col`, - 43200000000 AS `timedelta_div_numeric` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_6` AS `timedelta_div_numeric` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_isfinite/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_isfinite/out.sql deleted file mode 100644 index 54cbe2dd689..00000000000 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_isfinite/out.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT - NOT IS_INF(`float64_col`) OR IS_NAN(`float64_col`) AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_ln/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_ln/out.sql index 53ab88b7fc3..bd4cfa7c9a3 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_ln/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_ln/out.sql @@ -1,11 +1,21 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN `float64_col` IS NULL + THEN NULL + WHEN `float64_col` > 0 + THEN LN(`float64_col`) + WHEN `float64_col` < 0 + THEN CAST('NaN' AS FLOAT64) + ELSE CAST('-Infinity' AS FLOAT64) + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN `float64_col` IS NULL - THEN NULL - WHEN `float64_col` > 0 - THEN LN(`float64_col`) - WHEN `float64_col` < 0 - THEN CAST('NaN' AS FLOAT64) - ELSE CAST('-Infinity' AS FLOAT64) - END AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_log10/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_log10/out.sql index 2037649332f..c5bbff0e624 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_log10/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_log10/out.sql @@ -1,11 +1,21 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN `float64_col` IS NULL + THEN NULL + WHEN `float64_col` > 0 + THEN LOG(`float64_col`, 10) + WHEN `float64_col` < 0 + THEN CAST('NaN' AS FLOAT64) + ELSE CAST('-Infinity' AS FLOAT64) + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN `float64_col` IS NULL - THEN NULL - WHEN `float64_col` > 0 - THEN LOG(`float64_col`, 10) - WHEN `float64_col` < 0 - THEN CAST('NaN' AS FLOAT64) - ELSE CAST('-Infinity' AS FLOAT64) - END AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_log1p/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_log1p/out.sql index f7ddf4c223f..22e67e24eed 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_log1p/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_log1p/out.sql @@ -1,11 +1,21 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN `float64_col` IS NULL + THEN NULL + WHEN `float64_col` > -1 + THEN LN(1 + `float64_col`) + WHEN `float64_col` < -1 + THEN CAST('NaN' AS FLOAT64) + ELSE CAST('-Infinity' AS FLOAT64) + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN `float64_col` IS NULL - THEN NULL - WHEN `float64_col` > -1 - THEN LN(1 + `float64_col`) - WHEN `float64_col` < -1 - THEN CAST('NaN' AS FLOAT64) - ELSE CAST('-Infinity' AS FLOAT64) - END AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_manhattan_distance/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_manhattan_distance/out.sql index b6132a9fd6e..185bb7b277c 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_manhattan_distance/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_manhattan_distance/out.sql @@ -1,4 +1,16 @@ +WITH `bfcte_0` AS ( + SELECT + `float_list_col`, + `numeric_list_col` + FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` +), `bfcte_1` AS ( + SELECT + *, + ML.DISTANCE(`float_list_col`, `float_list_col`, 'MANHATTAN') AS `bfcol_2`, + ML.DISTANCE(`numeric_list_col`, `numeric_list_col`, 'MANHATTAN') AS `bfcol_3` + FROM `bfcte_0` +) SELECT - ML.DISTANCE(`float_list_col`, `float_list_col`, 'MANHATTAN') AS `float_list_col`, - ML.DISTANCE(`numeric_list_col`, `numeric_list_col`, 'MANHATTAN') AS `numeric_list_col` -FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` AS `bft_0` \ No newline at end of file + `bfcol_2` AS `float_list_col`, + `bfcol_3` AS `numeric_list_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_mod_numeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_mod_numeric/out.sql index 78107415b43..241ffa0b5ea 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_mod_numeric/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_mod_numeric/out.sql @@ -1,194 +1,292 @@ -SELECT - `rowindex`, - `int64_col`, - `float64_col`, - CASE - WHEN `int64_col` = CAST(0 AS INT64) - THEN CAST(0 AS INT64) * `int64_col` - WHEN `int64_col` < CAST(0 AS INT64) - AND ( - MOD(`int64_col`, `int64_col`) - ) > CAST(0 AS INT64) - THEN `int64_col` + ( - MOD(`int64_col`, `int64_col`) - ) - WHEN `int64_col` > CAST(0 AS INT64) - AND ( - MOD(`int64_col`, `int64_col`) - ) < CAST(0 AS INT64) - THEN `int64_col` + ( - MOD(`int64_col`, `int64_col`) - ) - ELSE MOD(`int64_col`, `int64_col`) - END AS `int_mod_int`, - CASE - WHEN -( - `int64_col` - ) = CAST(0 AS INT64) - THEN CAST(0 AS INT64) * `int64_col` - WHEN -( - `int64_col` - ) < CAST(0 AS INT64) - AND ( - MOD(`int64_col`, -( - `int64_col` - )) - ) > CAST(0 AS INT64) - THEN -( - `int64_col` - ) + ( - MOD(`int64_col`, -( - `int64_col` - )) - ) - WHEN -( - `int64_col` - ) > CAST(0 AS INT64) - AND ( - MOD(`int64_col`, -( - `int64_col` - )) - ) < CAST(0 AS INT64) - THEN -( - `int64_col` - ) + ( - MOD(`int64_col`, -( - `int64_col` +WITH `bfcte_0` AS ( + SELECT + `float64_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `float64_col` AS `bfcol_8`, + CASE + WHEN `int64_col` = CAST(0 AS INT64) + THEN CAST(0 AS INT64) * `int64_col` + WHEN `int64_col` < CAST(0 AS INT64) + AND ( + MOD(`int64_col`, `int64_col`) + ) > CAST(0 AS INT64) + THEN `int64_col` + ( + MOD(`int64_col`, `int64_col`) + ) + WHEN `int64_col` > CAST(0 AS INT64) + AND ( + MOD(`int64_col`, `int64_col`) + ) < CAST(0 AS INT64) + THEN `int64_col` + ( + MOD(`int64_col`, `int64_col`) + ) + ELSE MOD(`int64_col`, `int64_col`) + END AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + CASE + WHEN -( + `bfcol_7` + ) = CAST(0 AS INT64) + THEN CAST(0 AS INT64) * `bfcol_7` + WHEN -( + `bfcol_7` + ) < CAST(0 AS INT64) + AND ( + MOD(`bfcol_7`, -( + `bfcol_7` + )) + ) > CAST(0 AS INT64) + THEN -( + `bfcol_7` + ) + ( + MOD(`bfcol_7`, -( + `bfcol_7` + )) + ) + WHEN -( + `bfcol_7` + ) > CAST(0 AS INT64) + AND ( + MOD(`bfcol_7`, -( + `bfcol_7` + )) + ) < CAST(0 AS INT64) + THEN -( + `bfcol_7` + ) + ( + MOD(`bfcol_7`, -( + `bfcol_7` + )) + ) + ELSE MOD(`bfcol_7`, -( + `bfcol_7` )) - ) - ELSE MOD(`int64_col`, -( - `int64_col` - )) - END AS `int_mod_int_neg`, - CASE - WHEN 1 = CAST(0 AS INT64) - THEN CAST(0 AS INT64) * `int64_col` - WHEN 1 < CAST(0 AS INT64) AND ( - MOD(`int64_col`, 1) - ) > CAST(0 AS INT64) - THEN 1 + ( - MOD(`int64_col`, 1) - ) - WHEN 1 > CAST(0 AS INT64) AND ( - MOD(`int64_col`, 1) - ) < CAST(0 AS INT64) - THEN 1 + ( - MOD(`int64_col`, 1) - ) - ELSE MOD(`int64_col`, 1) - END AS `int_mod_1`, - CASE - WHEN 0 = CAST(0 AS INT64) - THEN CAST(0 AS INT64) * `int64_col` - WHEN 0 < CAST(0 AS INT64) AND ( - MOD(`int64_col`, 0) - ) > CAST(0 AS INT64) - THEN 0 + ( - MOD(`int64_col`, 0) - ) - WHEN 0 > CAST(0 AS INT64) AND ( - MOD(`int64_col`, 0) - ) < CAST(0 AS INT64) - THEN 0 + ( - MOD(`int64_col`, 0) - ) - ELSE MOD(`int64_col`, 0) - END AS `int_mod_0`, - CASE - WHEN CAST(`float64_col` AS BIGNUMERIC) = CAST(0 AS INT64) - THEN CAST('NaN' AS FLOAT64) * CAST(`float64_col` AS BIGNUMERIC) - WHEN CAST(`float64_col` AS BIGNUMERIC) < CAST(0 AS INT64) - AND ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(`float64_col` AS BIGNUMERIC)) - ) > CAST(0 AS INT64) - THEN CAST(`float64_col` AS BIGNUMERIC) + ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(`float64_col` AS BIGNUMERIC)) - ) - WHEN CAST(`float64_col` AS BIGNUMERIC) > CAST(0 AS INT64) - AND ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(`float64_col` AS BIGNUMERIC)) - ) < CAST(0 AS INT64) - THEN CAST(`float64_col` AS BIGNUMERIC) + ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(`float64_col` AS BIGNUMERIC)) - ) - ELSE MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(`float64_col` AS BIGNUMERIC)) - END AS `float_mod_float`, - CASE - WHEN CAST(-( - `float64_col` - ) AS BIGNUMERIC) = CAST(0 AS INT64) - THEN CAST('NaN' AS FLOAT64) * CAST(`float64_col` AS BIGNUMERIC) - WHEN CAST(-( - `float64_col` - ) AS BIGNUMERIC) < CAST(0 AS INT64) - AND ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(-( - `float64_col` + END AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + CASE + WHEN 1 = CAST(0 AS INT64) + THEN CAST(0 AS INT64) * `bfcol_15` + WHEN 1 < CAST(0 AS INT64) AND ( + MOD(`bfcol_15`, 1) + ) > CAST(0 AS INT64) + THEN 1 + ( + MOD(`bfcol_15`, 1) + ) + WHEN 1 > CAST(0 AS INT64) AND ( + MOD(`bfcol_15`, 1) + ) < CAST(0 AS INT64) + THEN 1 + ( + MOD(`bfcol_15`, 1) + ) + ELSE MOD(`bfcol_15`, 1) + END AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + CASE + WHEN 0 = CAST(0 AS INT64) + THEN CAST(0 AS INT64) * `bfcol_25` + WHEN 0 < CAST(0 AS INT64) AND ( + MOD(`bfcol_25`, 0) + ) > CAST(0 AS INT64) + THEN 0 + ( + MOD(`bfcol_25`, 0) + ) + WHEN 0 > CAST(0 AS INT64) AND ( + MOD(`bfcol_25`, 0) + ) < CAST(0 AS INT64) + THEN 0 + ( + MOD(`bfcol_25`, 0) + ) + ELSE MOD(`bfcol_25`, 0) + END AS `bfcol_42` + FROM `bfcte_3` +), `bfcte_5` AS ( + SELECT + *, + `bfcol_36` AS `bfcol_50`, + `bfcol_37` AS `bfcol_51`, + `bfcol_38` AS `bfcol_52`, + `bfcol_39` AS `bfcol_53`, + `bfcol_40` AS `bfcol_54`, + `bfcol_41` AS `bfcol_55`, + `bfcol_42` AS `bfcol_56`, + CASE + WHEN CAST(`bfcol_38` AS BIGNUMERIC) = CAST(0 AS INT64) + THEN CAST('NaN' AS FLOAT64) * CAST(`bfcol_38` AS BIGNUMERIC) + WHEN CAST(`bfcol_38` AS BIGNUMERIC) < CAST(0 AS INT64) + AND ( + MOD(CAST(`bfcol_38` AS BIGNUMERIC), CAST(`bfcol_38` AS BIGNUMERIC)) + ) > CAST(0 AS INT64) + THEN CAST(`bfcol_38` AS BIGNUMERIC) + ( + MOD(CAST(`bfcol_38` AS BIGNUMERIC), CAST(`bfcol_38` AS BIGNUMERIC)) + ) + WHEN CAST(`bfcol_38` AS BIGNUMERIC) > CAST(0 AS INT64) + AND ( + MOD(CAST(`bfcol_38` AS BIGNUMERIC), CAST(`bfcol_38` AS BIGNUMERIC)) + ) < CAST(0 AS INT64) + THEN CAST(`bfcol_38` AS BIGNUMERIC) + ( + MOD(CAST(`bfcol_38` AS BIGNUMERIC), CAST(`bfcol_38` AS BIGNUMERIC)) + ) + ELSE MOD(CAST(`bfcol_38` AS BIGNUMERIC), CAST(`bfcol_38` AS BIGNUMERIC)) + END AS `bfcol_57` + FROM `bfcte_4` +), `bfcte_6` AS ( + SELECT + *, + `bfcol_50` AS `bfcol_66`, + `bfcol_51` AS `bfcol_67`, + `bfcol_52` AS `bfcol_68`, + `bfcol_53` AS `bfcol_69`, + `bfcol_54` AS `bfcol_70`, + `bfcol_55` AS `bfcol_71`, + `bfcol_56` AS `bfcol_72`, + `bfcol_57` AS `bfcol_73`, + CASE + WHEN CAST(-( + `bfcol_52` + ) AS BIGNUMERIC) = CAST(0 AS INT64) + THEN CAST('NaN' AS FLOAT64) * CAST(`bfcol_52` AS BIGNUMERIC) + WHEN CAST(-( + `bfcol_52` + ) AS BIGNUMERIC) < CAST(0 AS INT64) + AND ( + MOD(CAST(`bfcol_52` AS BIGNUMERIC), CAST(-( + `bfcol_52` + ) AS BIGNUMERIC)) + ) > CAST(0 AS INT64) + THEN CAST(-( + `bfcol_52` + ) AS BIGNUMERIC) + ( + MOD(CAST(`bfcol_52` AS BIGNUMERIC), CAST(-( + `bfcol_52` + ) AS BIGNUMERIC)) + ) + WHEN CAST(-( + `bfcol_52` + ) AS BIGNUMERIC) > CAST(0 AS INT64) + AND ( + MOD(CAST(`bfcol_52` AS BIGNUMERIC), CAST(-( + `bfcol_52` + ) AS BIGNUMERIC)) + ) < CAST(0 AS INT64) + THEN CAST(-( + `bfcol_52` + ) AS BIGNUMERIC) + ( + MOD(CAST(`bfcol_52` AS BIGNUMERIC), CAST(-( + `bfcol_52` + ) AS BIGNUMERIC)) + ) + ELSE MOD(CAST(`bfcol_52` AS BIGNUMERIC), CAST(-( + `bfcol_52` ) AS BIGNUMERIC)) - ) > CAST(0 AS INT64) - THEN CAST(-( - `float64_col` - ) AS BIGNUMERIC) + ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(-( - `float64_col` - ) AS BIGNUMERIC)) - ) - WHEN CAST(-( - `float64_col` - ) AS BIGNUMERIC) > CAST(0 AS INT64) - AND ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(-( - `float64_col` - ) AS BIGNUMERIC)) - ) < CAST(0 AS INT64) - THEN CAST(-( - `float64_col` - ) AS BIGNUMERIC) + ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(-( - `float64_col` - ) AS BIGNUMERIC)) - ) - ELSE MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(-( - `float64_col` - ) AS BIGNUMERIC)) - END AS `float_mod_float_neg`, - CASE - WHEN CAST(1 AS BIGNUMERIC) = CAST(0 AS INT64) - THEN CAST('NaN' AS FLOAT64) * CAST(`float64_col` AS BIGNUMERIC) - WHEN CAST(1 AS BIGNUMERIC) < CAST(0 AS INT64) - AND ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(1 AS BIGNUMERIC)) - ) > CAST(0 AS INT64) - THEN CAST(1 AS BIGNUMERIC) + ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(1 AS BIGNUMERIC)) - ) - WHEN CAST(1 AS BIGNUMERIC) > CAST(0 AS INT64) - AND ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(1 AS BIGNUMERIC)) - ) < CAST(0 AS INT64) - THEN CAST(1 AS BIGNUMERIC) + ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(1 AS BIGNUMERIC)) - ) - ELSE MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(1 AS BIGNUMERIC)) - END AS `float_mod_1`, - CASE - WHEN CAST(0 AS BIGNUMERIC) = CAST(0 AS INT64) - THEN CAST('NaN' AS FLOAT64) * CAST(`float64_col` AS BIGNUMERIC) - WHEN CAST(0 AS BIGNUMERIC) < CAST(0 AS INT64) - AND ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(0 AS BIGNUMERIC)) - ) > CAST(0 AS INT64) - THEN CAST(0 AS BIGNUMERIC) + ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(0 AS BIGNUMERIC)) - ) - WHEN CAST(0 AS BIGNUMERIC) > CAST(0 AS INT64) - AND ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(0 AS BIGNUMERIC)) - ) < CAST(0 AS INT64) - THEN CAST(0 AS BIGNUMERIC) + ( - MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(0 AS BIGNUMERIC)) - ) - ELSE MOD(CAST(`float64_col` AS BIGNUMERIC), CAST(0 AS BIGNUMERIC)) - END AS `float_mod_0`, - NULL AS `float_mod_null` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + END AS `bfcol_74` + FROM `bfcte_5` +), `bfcte_7` AS ( + SELECT + *, + `bfcol_66` AS `bfcol_84`, + `bfcol_67` AS `bfcol_85`, + `bfcol_68` AS `bfcol_86`, + `bfcol_69` AS `bfcol_87`, + `bfcol_70` AS `bfcol_88`, + `bfcol_71` AS `bfcol_89`, + `bfcol_72` AS `bfcol_90`, + `bfcol_73` AS `bfcol_91`, + `bfcol_74` AS `bfcol_92`, + CASE + WHEN CAST(1 AS BIGNUMERIC) = CAST(0 AS INT64) + THEN CAST('NaN' AS FLOAT64) * CAST(`bfcol_68` AS BIGNUMERIC) + WHEN CAST(1 AS BIGNUMERIC) < CAST(0 AS INT64) + AND ( + MOD(CAST(`bfcol_68` AS BIGNUMERIC), CAST(1 AS BIGNUMERIC)) + ) > CAST(0 AS INT64) + THEN CAST(1 AS BIGNUMERIC) + ( + MOD(CAST(`bfcol_68` AS BIGNUMERIC), CAST(1 AS BIGNUMERIC)) + ) + WHEN CAST(1 AS BIGNUMERIC) > CAST(0 AS INT64) + AND ( + MOD(CAST(`bfcol_68` AS BIGNUMERIC), CAST(1 AS BIGNUMERIC)) + ) < CAST(0 AS INT64) + THEN CAST(1 AS BIGNUMERIC) + ( + MOD(CAST(`bfcol_68` AS BIGNUMERIC), CAST(1 AS BIGNUMERIC)) + ) + ELSE MOD(CAST(`bfcol_68` AS BIGNUMERIC), CAST(1 AS BIGNUMERIC)) + END AS `bfcol_93` + FROM `bfcte_6` +), `bfcte_8` AS ( + SELECT + *, + `bfcol_84` AS `bfcol_104`, + `bfcol_85` AS `bfcol_105`, + `bfcol_86` AS `bfcol_106`, + `bfcol_87` AS `bfcol_107`, + `bfcol_88` AS `bfcol_108`, + `bfcol_89` AS `bfcol_109`, + `bfcol_90` AS `bfcol_110`, + `bfcol_91` AS `bfcol_111`, + `bfcol_92` AS `bfcol_112`, + `bfcol_93` AS `bfcol_113`, + CASE + WHEN CAST(0 AS BIGNUMERIC) = CAST(0 AS INT64) + THEN CAST('NaN' AS FLOAT64) * CAST(`bfcol_86` AS BIGNUMERIC) + WHEN CAST(0 AS BIGNUMERIC) < CAST(0 AS INT64) + AND ( + MOD(CAST(`bfcol_86` AS BIGNUMERIC), CAST(0 AS BIGNUMERIC)) + ) > CAST(0 AS INT64) + THEN CAST(0 AS BIGNUMERIC) + ( + MOD(CAST(`bfcol_86` AS BIGNUMERIC), CAST(0 AS BIGNUMERIC)) + ) + WHEN CAST(0 AS BIGNUMERIC) > CAST(0 AS INT64) + AND ( + MOD(CAST(`bfcol_86` AS BIGNUMERIC), CAST(0 AS BIGNUMERIC)) + ) < CAST(0 AS INT64) + THEN CAST(0 AS BIGNUMERIC) + ( + MOD(CAST(`bfcol_86` AS BIGNUMERIC), CAST(0 AS BIGNUMERIC)) + ) + ELSE MOD(CAST(`bfcol_86` AS BIGNUMERIC), CAST(0 AS BIGNUMERIC)) + END AS `bfcol_114` + FROM `bfcte_7` +) +SELECT + `bfcol_104` AS `rowindex`, + `bfcol_105` AS `int64_col`, + `bfcol_106` AS `float64_col`, + `bfcol_107` AS `int_mod_int`, + `bfcol_108` AS `int_mod_int_neg`, + `bfcol_109` AS `int_mod_1`, + `bfcol_110` AS `int_mod_0`, + `bfcol_111` AS `float_mod_float`, + `bfcol_112` AS `float_mod_float_neg`, + `bfcol_113` AS `float_mod_1`, + `bfcol_114` AS `float_mod_0` +FROM `bfcte_8` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_mul_numeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_mul_numeric/out.sql index ebe8d571d65..d0c537e4820 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_mul_numeric/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_mul_numeric/out.sql @@ -1,10 +1,54 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `bool_col` AS `bfcol_8`, + `int64_col` * `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_7` * 1 AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + `bfcol_15` * CAST(`bfcol_16` AS INT64) AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + CAST(`bfcol_26` AS INT64) * `bfcol_25` AS `bfcol_42` + FROM `bfcte_3` +) SELECT - `rowindex`, - `int64_col`, - `bool_col`, - `int64_col` * `int64_col` AS `int_mul_int`, - `int64_col` * 1 AS `int_mul_1`, - NULL AS `int_mul_null`, - `int64_col` * CAST(`bool_col` AS INT64) AS `int_mul_bool`, - CAST(`bool_col` AS INT64) * `int64_col` AS `bool_mul_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_36` AS `rowindex`, + `bfcol_37` AS `int64_col`, + `bfcol_38` AS `bool_col`, + `bfcol_39` AS `int_mul_int`, + `bfcol_40` AS `int_mul_1`, + `bfcol_41` AS `int_mul_bool`, + `bfcol_42` AS `bool_mul_int` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_mul_timedelta/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_mul_timedelta/out.sql index 8285d1e7d4a..ebdf296b2b2 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_mul_timedelta/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_mul_timedelta/out.sql @@ -1,16 +1,43 @@ +WITH `bfcte_0` AS ( + SELECT + `duration_col`, + `int64_col`, + `rowindex`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_8`, + `timestamp_col` AS `bfcol_9`, + `int64_col` AS `bfcol_10`, + `duration_col` AS `bfcol_11` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_10` AS `bfcol_18`, + `bfcol_11` AS `bfcol_19`, + CAST(FLOOR(`bfcol_11` * `bfcol_10`) AS INT64) AS `bfcol_20` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + `bfcol_19` AS `bfcol_29`, + `bfcol_20` AS `bfcol_30`, + CAST(FLOOR(`bfcol_18` * `bfcol_19`) AS INT64) AS `bfcol_31` + FROM `bfcte_2` +) SELECT - `rowindex`, - `timestamp_col`, - `int64_col`, - `duration_col`, - CAST(IF( - `duration_col` * `int64_col` > 0, - FLOOR(`duration_col` * `int64_col`), - CEIL(`duration_col` * `int64_col`) - ) AS INT64) AS `timedelta_mul_numeric`, - CAST(IF( - `int64_col` * `duration_col` > 0, - FLOOR(`int64_col` * `duration_col`), - CEIL(`int64_col` * `duration_col`) - ) AS INT64) AS `numeric_mul_timedelta` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_26` AS `rowindex`, + `bfcol_27` AS `timestamp_col`, + `bfcol_28` AS `int64_col`, + `bfcol_29` AS `duration_col`, + `bfcol_30` AS `timedelta_mul_numeric`, + `bfcol_31` AS `numeric_mul_timedelta` +FROM `bfcte_3` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_neg/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_neg/out.sql index 13a9f3f6734..4374af349b7 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_neg/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_neg/out.sql @@ -1,5 +1,15 @@ -SELECT - -( +WITH `bfcte_0` AS ( + SELECT `float64_col` - ) AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + -( + `float64_col` + ) AS `bfcol_1` + FROM `bfcte_0` +) +SELECT + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_pos/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_pos/out.sql index 1890218bdd9..1ed016029a2 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_pos/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_pos/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `float64_col` AS `bfcol_1` + FROM `bfcte_0` +) SELECT - `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_pow/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_pow/out.sql index 7202903ebe3..05fbaa12c92 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_pow/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_pow/out.sql @@ -1,247 +1,329 @@ -SELECT - `rowindex`, - `int64_col`, - `float64_col`, - CASE - WHEN `int64_col` <> 0 AND `int64_col` * LN(ABS(`int64_col`)) > 43.66827237527655 - THEN NULL - ELSE CAST(POWER(CAST(`int64_col` AS NUMERIC), `int64_col`) AS INT64) - END AS `int_pow_int`, - CASE - WHEN `float64_col` = CAST(0 AS INT64) - THEN 1 - WHEN `int64_col` = 1 - THEN 1 - WHEN `int64_col` = CAST(0 AS INT64) AND `float64_col` < CAST(0 AS INT64) - THEN CAST('Infinity' AS FLOAT64) - WHEN ABS(`int64_col`) = CAST('Infinity' AS FLOAT64) - THEN POWER( - `int64_col`, - CASE - WHEN ABS(`float64_col`) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(`float64_col`) - ELSE `float64_col` - END - ) - WHEN ABS(`float64_col`) > 9007199254740992 - THEN POWER( - `int64_col`, - CASE - WHEN ABS(`float64_col`) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(`float64_col`) - ELSE `float64_col` - END - ) - WHEN `int64_col` < CAST(0 AS INT64) - AND NOT ( - CAST(`float64_col` AS INT64) = `float64_col` - ) - THEN CAST('NaN' AS FLOAT64) - WHEN `int64_col` <> CAST(0 AS INT64) AND `float64_col` * LN(ABS(`int64_col`)) > 709.78 - THEN CAST('Infinity' AS FLOAT64) * CASE - WHEN `int64_col` < CAST(0 AS INT64) AND MOD(CAST(`float64_col` AS INT64), 2) = 1 - THEN -1 - ELSE 1 - END - ELSE POWER( - `int64_col`, - CASE - WHEN ABS(`float64_col`) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(`float64_col`) - ELSE `float64_col` - END - ) - END AS `int_pow_float`, - CASE - WHEN `int64_col` = CAST(0 AS INT64) - THEN 1 - WHEN `float64_col` = 1 - THEN 1 - WHEN `float64_col` = CAST(0 AS INT64) AND `int64_col` < CAST(0 AS INT64) - THEN CAST('Infinity' AS FLOAT64) - WHEN ABS(`float64_col`) = CAST('Infinity' AS FLOAT64) - THEN POWER( - `float64_col`, - CASE - WHEN ABS(`int64_col`) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(`int64_col`) - ELSE `int64_col` - END - ) - WHEN ABS(`int64_col`) > 9007199254740992 - THEN POWER( - `float64_col`, - CASE - WHEN ABS(`int64_col`) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(`int64_col`) - ELSE `int64_col` - END - ) - WHEN `float64_col` < CAST(0 AS INT64) - AND NOT ( - CAST(`int64_col` AS INT64) = `int64_col` - ) - THEN CAST('NaN' AS FLOAT64) - WHEN `float64_col` <> CAST(0 AS INT64) - AND `int64_col` * LN(ABS(`float64_col`)) > 709.78 - THEN CAST('Infinity' AS FLOAT64) * CASE - WHEN `float64_col` < CAST(0 AS INT64) AND MOD(CAST(`int64_col` AS INT64), 2) = 1 - THEN -1 - ELSE 1 - END - ELSE POWER( - `float64_col`, - CASE - WHEN ABS(`int64_col`) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(`int64_col`) - ELSE `int64_col` - END - ) - END AS `float_pow_int`, - CASE - WHEN `float64_col` = CAST(0 AS INT64) - THEN 1 - WHEN `float64_col` = 1 - THEN 1 - WHEN `float64_col` = CAST(0 AS INT64) AND `float64_col` < CAST(0 AS INT64) - THEN CAST('Infinity' AS FLOAT64) - WHEN ABS(`float64_col`) = CAST('Infinity' AS FLOAT64) - THEN POWER( - `float64_col`, - CASE - WHEN ABS(`float64_col`) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(`float64_col`) - ELSE `float64_col` - END - ) - WHEN ABS(`float64_col`) > 9007199254740992 - THEN POWER( - `float64_col`, - CASE - WHEN ABS(`float64_col`) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(`float64_col`) - ELSE `float64_col` - END - ) - WHEN `float64_col` < CAST(0 AS INT64) - AND NOT ( - CAST(`float64_col` AS INT64) = `float64_col` - ) - THEN CAST('NaN' AS FLOAT64) - WHEN `float64_col` <> CAST(0 AS INT64) - AND `float64_col` * LN(ABS(`float64_col`)) > 709.78 - THEN CAST('Infinity' AS FLOAT64) * CASE - WHEN `float64_col` < CAST(0 AS INT64) AND MOD(CAST(`float64_col` AS INT64), 2) = 1 - THEN -1 - ELSE 1 - END - ELSE POWER( - `float64_col`, - CASE - WHEN ABS(`float64_col`) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(`float64_col`) - ELSE `float64_col` - END - ) - END AS `float_pow_float`, - CASE - WHEN `int64_col` <> 0 AND 0 * LN(ABS(`int64_col`)) > 43.66827237527655 - THEN NULL - ELSE CAST(POWER(CAST(`int64_col` AS NUMERIC), 0) AS INT64) - END AS `int_pow_0`, - CASE - WHEN 0 = CAST(0 AS INT64) - THEN 1 - WHEN `float64_col` = 1 - THEN 1 - WHEN `float64_col` = CAST(0 AS INT64) AND 0 < CAST(0 AS INT64) - THEN CAST('Infinity' AS FLOAT64) - WHEN ABS(`float64_col`) = CAST('Infinity' AS FLOAT64) - THEN POWER( - `float64_col`, - CASE - WHEN ABS(0) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(0) - ELSE 0 - END - ) - WHEN ABS(0) > 9007199254740992 - THEN POWER( - `float64_col`, - CASE - WHEN ABS(0) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(0) - ELSE 0 +WITH `bfcte_0` AS ( + SELECT + `float64_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `float64_col` AS `bfcol_8`, + CASE + WHEN `int64_col` <> 0 AND `int64_col` * LN(ABS(`int64_col`)) > 43.66827237527655 + THEN NULL + ELSE CAST(POWER(CAST(`int64_col` AS NUMERIC), `int64_col`) AS INT64) + END AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + CASE + WHEN `bfcol_8` = CAST(0 AS INT64) + THEN 1 + WHEN `bfcol_7` = 1 + THEN 1 + WHEN `bfcol_7` = CAST(0 AS INT64) AND `bfcol_8` < CAST(0 AS INT64) + THEN CAST('Infinity' AS FLOAT64) + WHEN ABS(`bfcol_7`) = CAST('Infinity' AS FLOAT64) + THEN POWER( + `bfcol_7`, + CASE + WHEN ABS(`bfcol_8`) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(`bfcol_8`) + ELSE `bfcol_8` + END + ) + WHEN ABS(`bfcol_8`) > 9007199254740992 + THEN POWER( + `bfcol_7`, + CASE + WHEN ABS(`bfcol_8`) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(`bfcol_8`) + ELSE `bfcol_8` + END + ) + WHEN `bfcol_7` < CAST(0 AS INT64) AND NOT CAST(`bfcol_8` AS INT64) = `bfcol_8` + THEN CAST('NaN' AS FLOAT64) + WHEN `bfcol_7` <> CAST(0 AS INT64) AND `bfcol_8` * LN(ABS(`bfcol_7`)) > 709.78 + THEN CAST('Infinity' AS FLOAT64) * CASE + WHEN `bfcol_7` < CAST(0 AS INT64) AND MOD(CAST(`bfcol_8` AS INT64), 2) = 1 + THEN -1 + ELSE 1 END - ) - WHEN `float64_col` < CAST(0 AS INT64) AND NOT ( - CAST(0 AS INT64) = 0 - ) - THEN CAST('NaN' AS FLOAT64) - WHEN `float64_col` <> CAST(0 AS INT64) AND 0 * LN(ABS(`float64_col`)) > 709.78 - THEN CAST('Infinity' AS FLOAT64) * CASE - WHEN `float64_col` < CAST(0 AS INT64) AND MOD(CAST(0 AS INT64), 2) = 1 - THEN -1 - ELSE 1 - END - ELSE POWER( - `float64_col`, - CASE - WHEN ABS(0) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(0) - ELSE 0 + ELSE POWER( + `bfcol_7`, + CASE + WHEN ABS(`bfcol_8`) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(`bfcol_8`) + ELSE `bfcol_8` + END + ) + END AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + CASE + WHEN `bfcol_15` = CAST(0 AS INT64) + THEN 1 + WHEN `bfcol_16` = 1 + THEN 1 + WHEN `bfcol_16` = CAST(0 AS INT64) AND `bfcol_15` < CAST(0 AS INT64) + THEN CAST('Infinity' AS FLOAT64) + WHEN ABS(`bfcol_16`) = CAST('Infinity' AS FLOAT64) + THEN POWER( + `bfcol_16`, + CASE + WHEN ABS(`bfcol_15`) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(`bfcol_15`) + ELSE `bfcol_15` + END + ) + WHEN ABS(`bfcol_15`) > 9007199254740992 + THEN POWER( + `bfcol_16`, + CASE + WHEN ABS(`bfcol_15`) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(`bfcol_15`) + ELSE `bfcol_15` + END + ) + WHEN `bfcol_16` < CAST(0 AS INT64) AND NOT CAST(`bfcol_15` AS INT64) = `bfcol_15` + THEN CAST('NaN' AS FLOAT64) + WHEN `bfcol_16` <> CAST(0 AS INT64) AND `bfcol_15` * LN(ABS(`bfcol_16`)) > 709.78 + THEN CAST('Infinity' AS FLOAT64) * CASE + WHEN `bfcol_16` < CAST(0 AS INT64) AND MOD(CAST(`bfcol_15` AS INT64), 2) = 1 + THEN -1 + ELSE 1 END - ) - END AS `float_pow_0`, - CASE - WHEN `int64_col` <> 0 AND 1 * LN(ABS(`int64_col`)) > 43.66827237527655 - THEN NULL - ELSE CAST(POWER(CAST(`int64_col` AS NUMERIC), 1) AS INT64) - END AS `int_pow_1`, - CASE - WHEN 1 = CAST(0 AS INT64) - THEN 1 - WHEN `float64_col` = 1 - THEN 1 - WHEN `float64_col` = CAST(0 AS INT64) AND 1 < CAST(0 AS INT64) - THEN CAST('Infinity' AS FLOAT64) - WHEN ABS(`float64_col`) = CAST('Infinity' AS FLOAT64) - THEN POWER( - `float64_col`, - CASE - WHEN ABS(1) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(1) + ELSE POWER( + `bfcol_16`, + CASE + WHEN ABS(`bfcol_15`) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(`bfcol_15`) + ELSE `bfcol_15` + END + ) + END AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + CASE + WHEN `bfcol_26` = CAST(0 AS INT64) + THEN 1 + WHEN `bfcol_26` = 1 + THEN 1 + WHEN `bfcol_26` = CAST(0 AS INT64) AND `bfcol_26` < CAST(0 AS INT64) + THEN CAST('Infinity' AS FLOAT64) + WHEN ABS(`bfcol_26`) = CAST('Infinity' AS FLOAT64) + THEN POWER( + `bfcol_26`, + CASE + WHEN ABS(`bfcol_26`) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(`bfcol_26`) + ELSE `bfcol_26` + END + ) + WHEN ABS(`bfcol_26`) > 9007199254740992 + THEN POWER( + `bfcol_26`, + CASE + WHEN ABS(`bfcol_26`) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(`bfcol_26`) + ELSE `bfcol_26` + END + ) + WHEN `bfcol_26` < CAST(0 AS INT64) AND NOT CAST(`bfcol_26` AS INT64) = `bfcol_26` + THEN CAST('NaN' AS FLOAT64) + WHEN `bfcol_26` <> CAST(0 AS INT64) AND `bfcol_26` * LN(ABS(`bfcol_26`)) > 709.78 + THEN CAST('Infinity' AS FLOAT64) * CASE + WHEN `bfcol_26` < CAST(0 AS INT64) AND MOD(CAST(`bfcol_26` AS INT64), 2) = 1 + THEN -1 ELSE 1 END - ) - WHEN ABS(1) > 9007199254740992 - THEN POWER( - `float64_col`, - CASE - WHEN ABS(1) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(1) + ELSE POWER( + `bfcol_26`, + CASE + WHEN ABS(`bfcol_26`) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(`bfcol_26`) + ELSE `bfcol_26` + END + ) + END AS `bfcol_42` + FROM `bfcte_3` +), `bfcte_5` AS ( + SELECT + *, + `bfcol_36` AS `bfcol_50`, + `bfcol_37` AS `bfcol_51`, + `bfcol_38` AS `bfcol_52`, + `bfcol_39` AS `bfcol_53`, + `bfcol_40` AS `bfcol_54`, + `bfcol_41` AS `bfcol_55`, + `bfcol_42` AS `bfcol_56`, + CASE + WHEN `bfcol_37` <> 0 AND 0 * LN(ABS(`bfcol_37`)) > 43.66827237527655 + THEN NULL + ELSE CAST(POWER(CAST(`bfcol_37` AS NUMERIC), 0) AS INT64) + END AS `bfcol_57` + FROM `bfcte_4` +), `bfcte_6` AS ( + SELECT + *, + `bfcol_50` AS `bfcol_66`, + `bfcol_51` AS `bfcol_67`, + `bfcol_52` AS `bfcol_68`, + `bfcol_53` AS `bfcol_69`, + `bfcol_54` AS `bfcol_70`, + `bfcol_55` AS `bfcol_71`, + `bfcol_56` AS `bfcol_72`, + `bfcol_57` AS `bfcol_73`, + CASE + WHEN 0 = CAST(0 AS INT64) + THEN 1 + WHEN `bfcol_52` = 1 + THEN 1 + WHEN `bfcol_52` = CAST(0 AS INT64) AND 0 < CAST(0 AS INT64) + THEN CAST('Infinity' AS FLOAT64) + WHEN ABS(`bfcol_52`) = CAST('Infinity' AS FLOAT64) + THEN POWER( + `bfcol_52`, + CASE + WHEN ABS(0) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(0) + ELSE 0 + END + ) + WHEN ABS(0) > 9007199254740992 + THEN POWER( + `bfcol_52`, + CASE + WHEN ABS(0) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(0) + ELSE 0 + END + ) + WHEN `bfcol_52` < CAST(0 AS INT64) AND NOT CAST(0 AS INT64) = 0 + THEN CAST('NaN' AS FLOAT64) + WHEN `bfcol_52` <> CAST(0 AS INT64) AND 0 * LN(ABS(`bfcol_52`)) > 709.78 + THEN CAST('Infinity' AS FLOAT64) * CASE + WHEN `bfcol_52` < CAST(0 AS INT64) AND MOD(CAST(0 AS INT64), 2) = 1 + THEN -1 ELSE 1 END - ) - WHEN `float64_col` < CAST(0 AS INT64) AND NOT ( - CAST(1 AS INT64) = 1 - ) - THEN CAST('NaN' AS FLOAT64) - WHEN `float64_col` <> CAST(0 AS INT64) AND 1 * LN(ABS(`float64_col`)) > 709.78 - THEN CAST('Infinity' AS FLOAT64) * CASE - WHEN `float64_col` < CAST(0 AS INT64) AND MOD(CAST(1 AS INT64), 2) = 1 - THEN -1 - ELSE 1 - END - ELSE POWER( - `float64_col`, - CASE - WHEN ABS(1) > 9007199254740992 - THEN CAST('Infinity' AS FLOAT64) * SIGN(1) + ELSE POWER( + `bfcol_52`, + CASE + WHEN ABS(0) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(0) + ELSE 0 + END + ) + END AS `bfcol_74` + FROM `bfcte_5` +), `bfcte_7` AS ( + SELECT + *, + `bfcol_66` AS `bfcol_84`, + `bfcol_67` AS `bfcol_85`, + `bfcol_68` AS `bfcol_86`, + `bfcol_69` AS `bfcol_87`, + `bfcol_70` AS `bfcol_88`, + `bfcol_71` AS `bfcol_89`, + `bfcol_72` AS `bfcol_90`, + `bfcol_73` AS `bfcol_91`, + `bfcol_74` AS `bfcol_92`, + CASE + WHEN `bfcol_67` <> 0 AND 1 * LN(ABS(`bfcol_67`)) > 43.66827237527655 + THEN NULL + ELSE CAST(POWER(CAST(`bfcol_67` AS NUMERIC), 1) AS INT64) + END AS `bfcol_93` + FROM `bfcte_6` +), `bfcte_8` AS ( + SELECT + *, + `bfcol_84` AS `bfcol_104`, + `bfcol_85` AS `bfcol_105`, + `bfcol_86` AS `bfcol_106`, + `bfcol_87` AS `bfcol_107`, + `bfcol_88` AS `bfcol_108`, + `bfcol_89` AS `bfcol_109`, + `bfcol_90` AS `bfcol_110`, + `bfcol_91` AS `bfcol_111`, + `bfcol_92` AS `bfcol_112`, + `bfcol_93` AS `bfcol_113`, + CASE + WHEN 1 = CAST(0 AS INT64) + THEN 1 + WHEN `bfcol_86` = 1 + THEN 1 + WHEN `bfcol_86` = CAST(0 AS INT64) AND 1 < CAST(0 AS INT64) + THEN CAST('Infinity' AS FLOAT64) + WHEN ABS(`bfcol_86`) = CAST('Infinity' AS FLOAT64) + THEN POWER( + `bfcol_86`, + CASE + WHEN ABS(1) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(1) + ELSE 1 + END + ) + WHEN ABS(1) > 9007199254740992 + THEN POWER( + `bfcol_86`, + CASE + WHEN ABS(1) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(1) + ELSE 1 + END + ) + WHEN `bfcol_86` < CAST(0 AS INT64) AND NOT CAST(1 AS INT64) = 1 + THEN CAST('NaN' AS FLOAT64) + WHEN `bfcol_86` <> CAST(0 AS INT64) AND 1 * LN(ABS(`bfcol_86`)) > 709.78 + THEN CAST('Infinity' AS FLOAT64) * CASE + WHEN `bfcol_86` < CAST(0 AS INT64) AND MOD(CAST(1 AS INT64), 2) = 1 + THEN -1 ELSE 1 END - ) - END AS `float_pow_1`, - NULL AS `float_pow_null`, - NULL AS `null_pow_float` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + ELSE POWER( + `bfcol_86`, + CASE + WHEN ABS(1) > 9007199254740992 + THEN CAST('Infinity' AS FLOAT64) * SIGN(1) + ELSE 1 + END + ) + END AS `bfcol_114` + FROM `bfcte_7` +) +SELECT + `bfcol_104` AS `rowindex`, + `bfcol_105` AS `int64_col`, + `bfcol_106` AS `float64_col`, + `bfcol_107` AS `int_pow_int`, + `bfcol_108` AS `int_pow_float`, + `bfcol_109` AS `float_pow_int`, + `bfcol_110` AS `float_pow_float`, + `bfcol_111` AS `int_pow_0`, + `bfcol_112` AS `float_pow_0`, + `bfcol_113` AS `int_pow_1`, + `bfcol_114` AS `float_pow_1` +FROM `bfcte_8` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_round/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_round/out.sql index 9ac8e1065b5..9ce76f7c63f 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_round/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_round/out.sql @@ -1,11 +1,81 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `float64_col` AS `bfcol_8`, + CAST(ROUND(`int64_col`, 0) AS INT64) AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + CAST(ROUND(`bfcol_7`, 1) AS INT64) AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + CAST(ROUND(`bfcol_15`, -1) AS INT64) AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + ROUND(`bfcol_26`, 0) AS `bfcol_42` + FROM `bfcte_3` +), `bfcte_5` AS ( + SELECT + *, + `bfcol_36` AS `bfcol_50`, + `bfcol_37` AS `bfcol_51`, + `bfcol_38` AS `bfcol_52`, + `bfcol_39` AS `bfcol_53`, + `bfcol_40` AS `bfcol_54`, + `bfcol_41` AS `bfcol_55`, + `bfcol_42` AS `bfcol_56`, + ROUND(`bfcol_38`, 1) AS `bfcol_57` + FROM `bfcte_4` +), `bfcte_6` AS ( + SELECT + *, + `bfcol_50` AS `bfcol_66`, + `bfcol_51` AS `bfcol_67`, + `bfcol_52` AS `bfcol_68`, + `bfcol_53` AS `bfcol_69`, + `bfcol_54` AS `bfcol_70`, + `bfcol_55` AS `bfcol_71`, + `bfcol_56` AS `bfcol_72`, + `bfcol_57` AS `bfcol_73`, + ROUND(`bfcol_52`, -1) AS `bfcol_74` + FROM `bfcte_5` +) SELECT - `rowindex`, - `int64_col`, - `float64_col`, - CAST(ROUND(`int64_col`, 0) AS INT64) AS `int_round_0`, - CAST(ROUND(`int64_col`, 1) AS INT64) AS `int_round_1`, - CAST(ROUND(`int64_col`, -1) AS INT64) AS `int_round_m1`, - ROUND(`float64_col`, 0) AS `float_round_0`, - ROUND(`float64_col`, 1) AS `float_round_1`, - ROUND(`float64_col`, -1) AS `float_round_m1` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_66` AS `rowindex`, + `bfcol_67` AS `int64_col`, + `bfcol_68` AS `float64_col`, + `bfcol_69` AS `int_round_0`, + `bfcol_70` AS `int_round_1`, + `bfcol_71` AS `int_round_m1`, + `bfcol_72` AS `float_round_0`, + `bfcol_73` AS `float_round_1`, + `bfcol_74` AS `float_round_m1` +FROM `bfcte_6` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sin/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sin/out.sql index ddc7cfab6ce..1699b6d8df8 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sin/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sin/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + SIN(`float64_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - SIN(`float64_col`) AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sinh/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sinh/out.sql index a1d71a7a065..c1ea003e2d3 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sinh/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sinh/out.sql @@ -1,7 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN ABS(`float64_col`) > 709.78 + THEN SIGN(`float64_col`) * CAST('Infinity' AS FLOAT64) + ELSE SINH(`float64_col`) + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN ABS(`float64_col`) > 709.78 - THEN SIGN(`float64_col`) * CAST('Infinity' AS FLOAT64) - ELSE SINH(`float64_col`) - END AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sqrt/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sqrt/out.sql index 6162a69d571..152545d5505 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sqrt/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sqrt/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE WHEN `float64_col` < 0 THEN CAST('NaN' AS FLOAT64) ELSE SQRT(`float64_col`) END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE WHEN `float64_col` < 0 THEN CAST('NaN' AS FLOAT64) ELSE SQRT(`float64_col`) END AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sub_numeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sub_numeric/out.sql index c1d0350a664..7e0f07af7b7 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sub_numeric/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sub_numeric/out.sql @@ -1,10 +1,54 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `bool_col` AS `bfcol_8`, + `int64_col` - `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_7` - 1 AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + `bfcol_15` - CAST(`bfcol_16` AS INT64) AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + CAST(`bfcol_26` AS INT64) - `bfcol_25` AS `bfcol_42` + FROM `bfcte_3` +) SELECT - `rowindex`, - `int64_col`, - `bool_col`, - `int64_col` - `int64_col` AS `int_sub_int`, - `int64_col` - 1 AS `int_sub_1`, - NULL AS `int_sub_null`, - `int64_col` - CAST(`bool_col` AS INT64) AS `int_sub_bool`, - CAST(`bool_col` AS INT64) - `int64_col` AS `bool_sub_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_36` AS `rowindex`, + `bfcol_37` AS `int64_col`, + `bfcol_38` AS `bool_col`, + `bfcol_39` AS `int_add_int`, + `bfcol_40` AS `int_add_1`, + `bfcol_41` AS `int_add_bool`, + `bfcol_42` AS `bool_add_int` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sub_timedelta/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sub_timedelta/out.sql index 5c8b130d59d..ebcffd67f61 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sub_timedelta/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_sub_timedelta/out.sql @@ -1,11 +1,82 @@ +WITH `bfcte_0` AS ( + SELECT + `date_col`, + `duration_col`, + `rowindex`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_8`, + `timestamp_col` AS `bfcol_9`, + `date_col` AS `bfcol_10`, + `duration_col` AS `bfcol_11` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + `bfcol_11` AS `bfcol_18`, + `bfcol_10` AS `bfcol_19`, + TIMESTAMP_SUB(CAST(`bfcol_10` AS DATETIME), INTERVAL `bfcol_11` MICROSECOND) AS `bfcol_20` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + `bfcol_19` AS `bfcol_29`, + `bfcol_20` AS `bfcol_30`, + TIMESTAMP_SUB(`bfcol_17`, INTERVAL `bfcol_18` MICROSECOND) AS `bfcol_31` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + `bfcol_30` AS `bfcol_42`, + `bfcol_31` AS `bfcol_43`, + TIMESTAMP_DIFF(CAST(`bfcol_29` AS DATETIME), CAST(`bfcol_29` AS DATETIME), MICROSECOND) AS `bfcol_44` + FROM `bfcte_3` +), `bfcte_5` AS ( + SELECT + *, + `bfcol_38` AS `bfcol_52`, + `bfcol_39` AS `bfcol_53`, + `bfcol_40` AS `bfcol_54`, + `bfcol_41` AS `bfcol_55`, + `bfcol_42` AS `bfcol_56`, + `bfcol_43` AS `bfcol_57`, + `bfcol_44` AS `bfcol_58`, + TIMESTAMP_DIFF(`bfcol_39`, `bfcol_39`, MICROSECOND) AS `bfcol_59` + FROM `bfcte_4` +), `bfcte_6` AS ( + SELECT + *, + `bfcol_52` AS `bfcol_68`, + `bfcol_53` AS `bfcol_69`, + `bfcol_54` AS `bfcol_70`, + `bfcol_55` AS `bfcol_71`, + `bfcol_56` AS `bfcol_72`, + `bfcol_57` AS `bfcol_73`, + `bfcol_58` AS `bfcol_74`, + `bfcol_59` AS `bfcol_75`, + `bfcol_54` - `bfcol_54` AS `bfcol_76` + FROM `bfcte_5` +) SELECT - `rowindex`, - `timestamp_col`, - `duration_col`, - `date_col`, - TIMESTAMP_SUB(CAST(`date_col` AS DATETIME), INTERVAL `duration_col` MICROSECOND) AS `date_sub_timedelta`, - TIMESTAMP_SUB(`timestamp_col`, INTERVAL `duration_col` MICROSECOND) AS `timestamp_sub_timedelta`, - TIMESTAMP_DIFF(CAST(`date_col` AS DATETIME), CAST(`date_col` AS DATETIME), MICROSECOND) AS `timestamp_sub_date`, - TIMESTAMP_DIFF(`timestamp_col`, `timestamp_col`, MICROSECOND) AS `date_sub_timestamp`, - `duration_col` - `duration_col` AS `timedelta_sub_timedelta` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_68` AS `rowindex`, + `bfcol_69` AS `timestamp_col`, + `bfcol_70` AS `duration_col`, + `bfcol_71` AS `date_col`, + `bfcol_72` AS `date_sub_timedelta`, + `bfcol_73` AS `timestamp_sub_timedelta`, + `bfcol_74` AS `timestamp_sub_date`, + `bfcol_75` AS `date_sub_timestamp`, + `bfcol_76` AS `timedelta_sub_timedelta` +FROM `bfcte_6` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_tan/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_tan/out.sql index 138b5f84a3b..f09d26a188a 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_tan/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_tan/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + TAN(`float64_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - TAN(`float64_col`) AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_tanh/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_tanh/out.sql index c5db31c9579..a5e5a87fbc4 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_tanh/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_tanh/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + TANH(`float64_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - TANH(`float64_col`) AS `float64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `float64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_unsafe_pow_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_unsafe_pow_op/out.sql index 0795b64a209..9957a346654 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_unsafe_pow_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_unsafe_pow_op/out.sql @@ -1,14 +1,43 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `bool_col` AS `bfcol_3`, + `int64_col` AS `bfcol_4`, + `float64_col` AS `bfcol_5`, + ( + `int64_col` >= 0 + ) AND ( + `int64_col` <= 10 + ) AS `bfcol_6` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + * + FROM `bfcte_1` + WHERE + `bfcol_6` +), `bfcte_3` AS ( + SELECT + *, + POWER(`bfcol_4`, `bfcol_4`) AS `bfcol_14`, + POWER(`bfcol_4`, `bfcol_5`) AS `bfcol_15`, + POWER(`bfcol_5`, `bfcol_4`) AS `bfcol_16`, + POWER(`bfcol_5`, `bfcol_5`) AS `bfcol_17`, + POWER(`bfcol_4`, CAST(`bfcol_3` AS INT64)) AS `bfcol_18`, + POWER(CAST(`bfcol_3` AS INT64), `bfcol_4`) AS `bfcol_19` + FROM `bfcte_2` +) SELECT - POWER(`int64_col`, `int64_col`) AS `int_pow_int`, - POWER(`int64_col`, `float64_col`) AS `int_pow_float`, - POWER(`float64_col`, `int64_col`) AS `float_pow_int`, - POWER(`float64_col`, `float64_col`) AS `float_pow_float`, - POWER(`int64_col`, CAST(`bool_col` AS INT64)) AS `int_pow_bool`, - POWER(CAST(`bool_col` AS INT64), `int64_col`) AS `bool_pow_int` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -WHERE - ( - `int64_col` >= 0 - ) AND ( - `int64_col` <= 10 - ) \ No newline at end of file + `bfcol_14` AS `int_pow_int`, + `bfcol_15` AS `int_pow_float`, + `bfcol_16` AS `float_pow_int`, + `bfcol_17` AS `float_pow_float`, + `bfcol_18` AS `int_pow_bool`, + `bfcol_19` AS `bool_pow_int` +FROM `bfcte_3` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_add_string/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_add_string/out.sql index cf4051464b7..cb674787ff1 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_add_string/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_add_string/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CONCAT(`string_col`, 'a') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CONCAT(`string_col`, 'a') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_capitalize/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_capitalize/out.sql index d11ce9b9934..dd1f1473f41 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_capitalize/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_capitalize/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + INITCAP(`string_col`, '') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - INITCAP(`string_col`, '') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_endswith/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_endswith/out.sql index 0295af27992..eeb25740946 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_endswith/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_endswith/out.sql @@ -1,5 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + ENDS_WITH(`string_col`, 'ab') AS `bfcol_1`, + ENDS_WITH(`string_col`, 'ab') OR ENDS_WITH(`string_col`, 'cd') AS `bfcol_2`, + FALSE AS `bfcol_3` + FROM `bfcte_0` +) SELECT - ENDS_WITH(`string_col`, 'ab') AS `single`, - ENDS_WITH(`string_col`, 'ab') OR ENDS_WITH(`string_col`, 'cd') AS `double`, - FALSE AS `empty` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `single`, + `bfcol_2` AS `double`, + `bfcol_3` AS `empty` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isalnum/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isalnum/out.sql index 7654299c79e..61c2643f161 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isalnum/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isalnum/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + REGEXP_CONTAINS(`string_col`, '^(\\p{N}|\\p{L})+$') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - REGEXP_CONTAINS(`string_col`, '^(\\p{N}|\\p{L})+$') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isalpha/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isalpha/out.sql index 33a08ff054d..2b086f3e3d9 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isalpha/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isalpha/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + REGEXP_CONTAINS(`string_col`, '^\\p{L}+$') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - REGEXP_CONTAINS(`string_col`, '^\\p{L}+$') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isdecimal/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isdecimal/out.sql index 7f266cbf604..d4dddc348f0 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isdecimal/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isdecimal/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + REGEXP_CONTAINS(`string_col`, '^(\\p{Nd})+$') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - REGEXP_CONTAINS(`string_col`, '^(\\p{Nd})+$') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isdigit/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isdigit/out.sql index 9134d035153..eba0e51ed09 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isdigit/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isdigit/out.sql @@ -1,6 +1,16 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + REGEXP_CONTAINS( + `string_col`, + '^[\\p{Nd}\\x{00B9}\\x{00B2}\\x{00B3}\\x{2070}\\x{2074}-\\x{2079}\\x{2080}-\\x{2089}]+$' + ) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - REGEXP_CONTAINS( - `string_col`, - '^[\\p{Nd}\\x{00B9}\\x{00B2}\\x{00B3}\\x{2070}\\x{2074}-\\x{2079}\\x{2080}-\\x{2089}]+$' - ) AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_islower/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_islower/out.sql index bce92035e5c..b6ff57797c6 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_islower/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_islower/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + LOWER(`string_col`) = `string_col` AND UPPER(`string_col`) <> `string_col` AS `bfcol_1` + FROM `bfcte_0` +) SELECT - LOWER(`string_col`) = `string_col` AND UPPER(`string_col`) <> `string_col` AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isnumeric/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isnumeric/out.sql index 82baa081f54..6143b3685a2 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isnumeric/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isnumeric/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + REGEXP_CONTAINS(`string_col`, '^\\pN+$') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - REGEXP_CONTAINS(`string_col`, '^\\pN+$') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isspace/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isspace/out.sql index 2b44a592d93..47ccd642d40 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isspace/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isspace/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + REGEXP_CONTAINS(`string_col`, '^\\s+$') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - REGEXP_CONTAINS(`string_col`, '^\\s+$') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isupper/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isupper/out.sql index 17ac14ac53f..54f7b55ce3d 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isupper/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_isupper/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + UPPER(`string_col`) = `string_col` AND LOWER(`string_col`) <> `string_col` AS `bfcol_1` + FROM `bfcte_0` +) SELECT - UPPER(`string_col`) = `string_col` AND LOWER(`string_col`) <> `string_col` AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_len/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_len/out.sql index cff109d09dc..63e8e160bfc 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_len/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_len/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + LENGTH(`string_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - LENGTH(`string_col`) AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_len_w_array/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_len_w_array/out.sql index 1862deb6015..609c4131e65 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_len_w_array/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_len_w_array/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int_list_col` + FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` +), `bfcte_1` AS ( + SELECT + *, + ARRAY_LENGTH(`int_list_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - ARRAY_LENGTH(`int_list_col`) AS `int_list_col` -FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `int_list_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_lower/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_lower/out.sql index 851de35ecb0..0a9623162aa 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_lower/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_lower/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + LOWER(`string_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - LOWER(`string_col`) AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_lstrip/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_lstrip/out.sql index 4023605f8c9..1b73ee32585 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_lstrip/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_lstrip/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + LTRIM(`string_col`, ' ') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - LTRIM(`string_col`, ' ') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_regex_replace_str/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_regex_replace_str/out.sql index 4728c961ab3..2fd3365a803 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_regex_replace_str/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_regex_replace_str/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + REGEXP_REPLACE(`string_col`, 'e', 'a') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - REGEXP_REPLACE(`string_col`, 'e', 'a') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_replace_str/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_replace_str/out.sql index 154af44b500..61b2e2f432d 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_replace_str/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_replace_str/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + REPLACE(`string_col`, 'e', 'a') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - REPLACE(`string_col`, 'e', 'a') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_reverse/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_reverse/out.sql index 97bf57f79e1..f9d287a5917 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_reverse/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_reverse/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + REVERSE(`string_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - REVERSE(`string_col`) AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_rstrip/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_rstrip/out.sql index c3d25b56e24..72bdbba29f1 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_rstrip/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_rstrip/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + RTRIM(`string_col`, ' ') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - RTRIM(`string_col`, ' ') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_startswith/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_startswith/out.sql index 760d3db7745..54c8adb7b86 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_startswith/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_startswith/out.sql @@ -1,5 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + STARTS_WITH(`string_col`, 'ab') AS `bfcol_1`, + STARTS_WITH(`string_col`, 'ab') OR STARTS_WITH(`string_col`, 'cd') AS `bfcol_2`, + FALSE AS `bfcol_3` + FROM `bfcte_0` +) SELECT - STARTS_WITH(`string_col`, 'ab') AS `single`, - STARTS_WITH(`string_col`, 'ab') OR STARTS_WITH(`string_col`, 'cd') AS `double`, - FALSE AS `empty` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `single`, + `bfcol_2` AS `double`, + `bfcol_3` AS `empty` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_contains/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_contains/out.sql index 9071a252bc1..e973a97136b 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_contains/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_contains/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `string_col` LIKE '%e%' AS `bfcol_1` + FROM `bfcte_0` +) SELECT - `string_col` LIKE '%e%' AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_contains_regex/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_contains_regex/out.sql index 958f9af6f39..510e52e254c 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_contains_regex/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_contains_regex/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + REGEXP_CONTAINS(`string_col`, 'e') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - REGEXP_CONTAINS(`string_col`, 'e') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_extract/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_extract/out.sql index a87f5d9836d..3a6d48a18c5 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_extract/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_extract/out.sql @@ -1,12 +1,23 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + IF( + REGEXP_CONTAINS(`string_col`, '([a-z]*)'), + REGEXP_REPLACE(`string_col`, CONCAT('.*?(', '([a-z]*)', ').*'), '\\1'), + NULL + ) AS `bfcol_1`, + IF( + REGEXP_CONTAINS(`string_col`, '([a-z]*)'), + REGEXP_REPLACE(`string_col`, CONCAT('.*?', '([a-z]*)', '.*'), '\\1'), + NULL + ) AS `bfcol_2` + FROM `bfcte_0` +) SELECT - IF( - REGEXP_CONTAINS(`string_col`, '([a-z]*)'), - REGEXP_REPLACE(`string_col`, CONCAT('.*?(', '([a-z]*)', ').*'), '\\1'), - NULL - ) AS `zero`, - IF( - REGEXP_CONTAINS(`string_col`, '([a-z]*)'), - REGEXP_REPLACE(`string_col`, CONCAT('.*?', '([a-z]*)', '.*'), '\\1'), - NULL - ) AS `one` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `zero`, + `bfcol_2` AS `one` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_find/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_find/out.sql index cf21fb8234a..82847d5e22c 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_find/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_find/out.sql @@ -1,6 +1,19 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + INSTR(`string_col`, 'e', 1) - 1 AS `bfcol_1`, + INSTR(`string_col`, 'e', 3) - 1 AS `bfcol_2`, + INSTR(SUBSTRING(`string_col`, 1, 5), 'e') - 1 AS `bfcol_3`, + INSTR(SUBSTRING(`string_col`, 3, 3), 'e') - 1 AS `bfcol_4` + FROM `bfcte_0` +) SELECT - INSTR(`string_col`, 'e', 1) - 1 AS `none_none`, - INSTR(`string_col`, 'e', 3) - 1 AS `start_none`, - INSTR(SUBSTRING(`string_col`, 1, 5), 'e') - 1 AS `none_end`, - INSTR(SUBSTRING(`string_col`, 3, 3), 'e') - 1 AS `start_end` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `none_none`, + `bfcol_2` AS `start_none`, + `bfcol_3` AS `none_end`, + `bfcol_4` AS `start_end` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_get/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_get/out.sql index b4c7c504fc9..f868b730327 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_get/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_get/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + IF(SUBSTRING(`string_col`, 2, 1) <> '', SUBSTRING(`string_col`, 2, 1), NULL) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - IF(SUBSTRING(`string_col`, 2, 1) <> '', SUBSTRING(`string_col`, 2, 1), NULL) AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_pad/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_pad/out.sql index 29766cca6c1..2bb6042fe99 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_pad/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_pad/out.sql @@ -1,13 +1,25 @@ -SELECT - LPAD(`string_col`, GREATEST(LENGTH(`string_col`), 10), '-') AS `left`, - RPAD(`string_col`, GREATEST(LENGTH(`string_col`), 10), '-') AS `right`, - RPAD( - LPAD( - `string_col`, - CAST(FLOOR(SAFE_DIVIDE(GREATEST(LENGTH(`string_col`), 10) - LENGTH(`string_col`), 2)) AS INT64) + LENGTH(`string_col`), +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + LPAD(`string_col`, GREATEST(LENGTH(`string_col`), 10), '-') AS `bfcol_1`, + RPAD(`string_col`, GREATEST(LENGTH(`string_col`), 10), '-') AS `bfcol_2`, + RPAD( + LPAD( + `string_col`, + CAST(FLOOR(SAFE_DIVIDE(GREATEST(LENGTH(`string_col`), 10) - LENGTH(`string_col`), 2)) AS INT64) + LENGTH(`string_col`), + '-' + ), + GREATEST(LENGTH(`string_col`), 10), '-' - ), - GREATEST(LENGTH(`string_col`), 10), - '-' - ) AS `both` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + ) AS `bfcol_3` + FROM `bfcte_0` +) +SELECT + `bfcol_1` AS `left`, + `bfcol_2` AS `right`, + `bfcol_3` AS `both` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_repeat/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_repeat/out.sql index ed3d06ed35a..90a52a40b14 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_repeat/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_repeat/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + REPEAT(`string_col`, 2) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - REPEAT(`string_col`, 2) AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_slice/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_slice/out.sql index f011480ad30..8bd2a5f7feb 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_slice/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_str_slice/out.sql @@ -1,18 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + SUBSTRING(`string_col`, 2, 2) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - SUBSTRING(`string_col`, 2, 2) AS `1_3`, - SUBSTRING(`string_col`, 1, 3) AS `none_3`, - SUBSTRING(`string_col`, 2) AS `1_none`, - SUBSTRING(`string_col`, -3) AS `m3_none`, - SUBSTRING(`string_col`, 1, GREATEST(0, LENGTH(`string_col`) + -3)) AS `none_m3`, - SUBSTRING( - `string_col`, - GREATEST(1, LENGTH(`string_col`) + -4), - GREATEST(0, LENGTH(`string_col`) + -3) - GREATEST(0, LENGTH(`string_col`) + -5) - ) AS `m5_m3`, - SUBSTRING(`string_col`, 2, GREATEST(0, LENGTH(`string_col`) + -4)) AS `1_m3`, - SUBSTRING( - `string_col`, - GREATEST(1, LENGTH(`string_col`) + -2), - GREATEST(0, 5 - GREATEST(0, LENGTH(`string_col`) + -3)) - ) AS `m3_5` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_strconcat/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_strconcat/out.sql index cf4051464b7..cb674787ff1 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_strconcat/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_strconcat/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CONCAT(`string_col`, 'a') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CONCAT(`string_col`, 'a') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_string_split/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_string_split/out.sql index 5145d6686a8..37b15a0cf91 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_string_split/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_string_split/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + SPLIT(`string_col`, ',') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - SPLIT(`string_col`, ',') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_strip/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_strip/out.sql index e07185292bb..ebe4c39bbf5 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_strip/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_strip/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + TRIM(`string_col`, ' ') AS `bfcol_1` + FROM `bfcte_0` +) SELECT - TRIM(`string_col`, ' ') AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_upper/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_upper/out.sql index 88bd78bd095..aa14c5f05d8 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_upper/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_upper/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + UPPER(`string_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - UPPER(`string_col`) AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_zfill/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_zfill/out.sql index 818d2907add..79c4f695aaf 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_zfill/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_string_ops/test_zfill/out.sql @@ -1,7 +1,17 @@ +WITH `bfcte_0` AS ( + SELECT + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN STARTS_WITH(`string_col`, '-') + THEN CONCAT('-', LPAD(SUBSTRING(`string_col`, 2), GREATEST(LENGTH(`string_col`), 10) - 1, '0')) + ELSE LPAD(`string_col`, GREATEST(LENGTH(`string_col`), 10), '0') + END AS `bfcol_1` + FROM `bfcte_0` +) SELECT - CASE - WHEN STARTS_WITH(`string_col`, '-') - THEN CONCAT('-', LPAD(SUBSTRING(`string_col`, 2), GREATEST(LENGTH(`string_col`), 10) - 1, '0')) - ELSE LPAD(`string_col`, GREATEST(LENGTH(`string_col`), 10), '0') - END AS `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_struct_ops/test_struct_field/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_struct_ops/test_struct_field/out.sql index 6c3760aa36e..b85e88a90a5 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_struct_ops/test_struct_field/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_struct_ops/test_struct_field/out.sql @@ -1,4 +1,15 @@ +WITH `bfcte_0` AS ( + SELECT + `people` + FROM `bigframes-dev`.`sqlglot_test`.`nested_structs_types` +), `bfcte_1` AS ( + SELECT + *, + `people`.`name` AS `bfcol_1`, + `people`.`name` AS `bfcol_2` + FROM `bfcte_0` +) SELECT - `people`.`name` AS `string`, - `people`.`name` AS `int` -FROM `bigframes-dev`.`sqlglot_test`.`nested_structs_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `string`, + `bfcol_2` AS `int` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_struct_ops/test_struct_op/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_struct_ops/test_struct_op/out.sql index 3549149609a..575a1620806 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_struct_ops/test_struct_op/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_struct_ops/test_struct_op/out.sql @@ -1,8 +1,21 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `float64_col`, + `int64_col`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + STRUCT( + `bool_col` AS bool_col, + `int64_col` AS int64_col, + `float64_col` AS float64_col, + `string_col` AS string_col + ) AS `bfcol_4` + FROM `bfcte_0` +) SELECT - STRUCT( - `bool_col` AS bool_col, - `int64_col` AS int64_col, - `float64_col` AS float64_col, - `string_col` AS string_col - ) AS `result_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_4` AS `result_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_timedelta_ops/test_timedelta_floor/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_timedelta_ops/test_timedelta_floor/out.sql index 6eb6f8e989d..432aefd7f69 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_timedelta_ops/test_timedelta_floor/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_timedelta_ops/test_timedelta_floor/out.sql @@ -1,3 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + FLOOR(`int64_col`) AS `bfcol_1` + FROM `bfcte_0` +) SELECT - FLOOR(`int64_col`) AS `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_1` AS `int64_col` +FROM `bfcte_1` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_timedelta_ops/test_to_timedelta/out.sql b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_timedelta_ops/test_to_timedelta/out.sql index 59aa3d9b0b3..ed7dbc7c8a9 100644 --- a/tests/unit/core/compile/sqlglot/expressions/snapshots/test_timedelta_ops/test_to_timedelta/out.sql +++ b/tests/unit/core/compile/sqlglot/expressions/snapshots/test_timedelta_ops/test_to_timedelta/out.sql @@ -1,9 +1,54 @@ +WITH `bfcte_0` AS ( + SELECT + `float64_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `float64_col` AS `bfcol_8`, + `int64_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_6` AS `bfcol_14`, + `bfcol_7` AS `bfcol_15`, + `bfcol_8` AS `bfcol_16`, + `bfcol_9` AS `bfcol_17`, + CAST(FLOOR(`bfcol_8` * 1000000) AS INT64) AS `bfcol_18` + FROM `bfcte_1` +), `bfcte_3` AS ( + SELECT + *, + `bfcol_14` AS `bfcol_24`, + `bfcol_15` AS `bfcol_25`, + `bfcol_16` AS `bfcol_26`, + `bfcol_17` AS `bfcol_27`, + `bfcol_18` AS `bfcol_28`, + `bfcol_15` * 3600000000 AS `bfcol_29` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + `bfcol_24` AS `bfcol_36`, + `bfcol_25` AS `bfcol_37`, + `bfcol_26` AS `bfcol_38`, + `bfcol_27` AS `bfcol_39`, + `bfcol_28` AS `bfcol_40`, + `bfcol_29` AS `bfcol_41`, + `bfcol_27` AS `bfcol_42` + FROM `bfcte_3` +) SELECT - `rowindex`, - `int64_col`, - `float64_col`, - `int64_col` AS `duration_us`, - CAST(FLOOR(`float64_col` * 1000000) AS INT64) AS `duration_s`, - `int64_col` * 3600000000 AS `duration_w`, - `int64_col` AS `duration_on_duration` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file + `bfcol_36` AS `rowindex`, + `bfcol_37` AS `int64_col`, + `bfcol_38` AS `float64_col`, + `bfcol_39` AS `duration_us`, + `bfcol_40` AS `duration_s`, + `bfcol_41` AS `duration_w`, + `bfcol_42` AS `duration_on_duration` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/expressions/test_ai_ops.py b/tests/unit/core/compile/sqlglot/expressions/test_ai_ops.py index 57c52490860..c0cbece9054 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_ai_ops.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_ai_ops.py @@ -30,8 +30,11 @@ def test_ai_generate(scalar_types_df: dataframe.DataFrame, snapshot): op = ops.AIGenerate( prompt_context=(None, " is the same as ", None), + connection_id=None, endpoint="gemini-2.5-flash", - request_type="SHARED", + request_type="shared", + model_params=None, + output_schema=None, ) sql = utils._apply_ops_to_sql( @@ -48,6 +51,9 @@ def test_ai_generate_with_connection_id(scalar_types_df: dataframe.DataFrame, sn prompt_context=(None, " is the same as ", None), connection_id=CONNECTION_ID, endpoint="gemini-2.5-flash", + request_type="shared", + model_params=None, + output_schema=None, ) sql = utils._apply_ops_to_sql( @@ -64,6 +70,8 @@ def test_ai_generate_with_output_schema(scalar_types_df: dataframe.DataFrame, sn prompt_context=(None, " is the same as ", None), connection_id=None, endpoint="gemini-2.5-flash", + request_type="shared", + model_params=None, output_schema="x INT64, y FLOAT64", ) @@ -79,7 +87,11 @@ def test_ai_generate_with_model_param(scalar_types_df: dataframe.DataFrame, snap op = ops.AIGenerate( prompt_context=(None, " is the same as ", None), + connection_id=None, + endpoint=None, + request_type="shared", model_params=json.dumps(dict()), + output_schema=None, ) sql = utils._apply_ops_to_sql( @@ -94,7 +106,10 @@ def test_ai_generate_bool(scalar_types_df: dataframe.DataFrame, snapshot): op = ops.AIGenerateBool( prompt_context=(None, " is the same as ", None), + connection_id=None, endpoint="gemini-2.5-flash", + request_type="shared", + model_params=None, ) sql = utils._apply_ops_to_sql( @@ -113,6 +128,8 @@ def test_ai_generate_bool_with_connection_id( prompt_context=(None, " is the same as ", None), connection_id=CONNECTION_ID, endpoint="gemini-2.5-flash", + request_type="shared", + model_params=None, ) sql = utils._apply_ops_to_sql( @@ -129,6 +146,9 @@ def test_ai_generate_bool_with_model_param( op = ops.AIGenerateBool( prompt_context=(None, " is the same as ", None), + connection_id=None, + endpoint=None, + request_type="shared", model_params=json.dumps(dict()), ) @@ -145,7 +165,10 @@ def test_ai_generate_int(scalar_types_df: dataframe.DataFrame, snapshot): op = ops.AIGenerateInt( # The prompt does not make semantic sense but we only care about syntax correctness. prompt_context=(None, " is the same as ", None), + connection_id=None, endpoint="gemini-2.5-flash", + request_type="shared", + model_params=None, ) sql = utils._apply_ops_to_sql( @@ -165,6 +188,8 @@ def test_ai_generate_int_with_connection_id( prompt_context=(None, " is the same as ", None), connection_id=CONNECTION_ID, endpoint="gemini-2.5-flash", + request_type="shared", + model_params=None, ) sql = utils._apply_ops_to_sql( @@ -182,6 +207,9 @@ def test_ai_generate_int_with_model_param( op = ops.AIGenerateInt( # The prompt does not make semantic sense but we only care about syntax correctness. prompt_context=(None, " is the same as ", None), + connection_id=None, + endpoint=None, + request_type="shared", model_params=json.dumps(dict()), ) @@ -198,7 +226,10 @@ def test_ai_generate_double(scalar_types_df: dataframe.DataFrame, snapshot): op = ops.AIGenerateDouble( # The prompt does not make semantic sense but we only care about syntax correctness. prompt_context=(None, " is the same as ", None), + connection_id=None, endpoint="gemini-2.5-flash", + request_type="shared", + model_params=None, ) sql = utils._apply_ops_to_sql( @@ -218,6 +249,8 @@ def test_ai_generate_double_with_connection_id( prompt_context=(None, " is the same as ", None), connection_id=CONNECTION_ID, endpoint="gemini-2.5-flash", + request_type="shared", + model_params=None, ) sql = utils._apply_ops_to_sql( @@ -235,6 +268,9 @@ def test_ai_generate_double_with_model_param( op = ops.AIGenerateDouble( # The prompt does not make semantic sense but we only care about syntax correctness. prompt_context=(None, " is the same as ", None), + connection_id=None, + endpoint=None, + request_type="shared", model_params=json.dumps(dict()), ) @@ -245,84 +281,12 @@ def test_ai_generate_double_with_model_param( snapshot.assert_match(sql, "out.sql") -def test_ai_embed(scalar_types_df: dataframe.DataFrame, snapshot): - col_name = "string_col" - - op = ops.AIEmbed( - endpoint="text-embedding-005", - ) - - sql = utils._apply_ops_to_sql(scalar_types_df, [op.as_expr(col_name)], ["result"]) - - snapshot.assert_match(sql, "out.sql") - - -def test_ai_embed_with_connection_id(scalar_types_df: dataframe.DataFrame, snapshot): - col_name = "string_col" - - op = ops.AIEmbed( - endpoint="text-embedding-005", - connection_id=CONNECTION_ID, - ) - - sql = utils._apply_ops_to_sql(scalar_types_df, [op.as_expr(col_name)], ["result"]) - - snapshot.assert_match(sql, "out.sql") - - -def test_ai_embed_with_model(scalar_types_df: dataframe.DataFrame, snapshot): - col_name = "string_col" - - op = ops.AIEmbed( - model="embeddinggemma-300m", - ) - - sql = utils._apply_ops_to_sql(scalar_types_df, [op.as_expr(col_name)], ["result"]) - - snapshot.assert_match(sql, "out.sql") - - -def test_ai_embed_with_task_type_and_title( - scalar_types_df: dataframe.DataFrame, snapshot -): - col_name = "string_col" - - op = ops.AIEmbed( - endpoint="text-embedding-005", - task_type="RETRIEVAL_DOCUMENT", - title="My Document", - model_params=json.dumps({"outputDimensionality": 256}), - ) - - sql = utils._apply_ops_to_sql(scalar_types_df, [op.as_expr(col_name)], ["result"]) - - snapshot.assert_match(sql, "out.sql") - - -@pytest.mark.parametrize("connection_id", [None, CONNECTION_ID]) -def test_ai_if(scalar_types_df: dataframe.DataFrame, snapshot, connection_id): - col_name = "string_col" - - op = ops.AIIf( - prompt_context=(None, " is the same as ", None), - connection_id=connection_id, - optimization_mode="MINIMIZE_COST", - max_error_ratio=0.5, - ) - - sql = utils._apply_ops_to_sql( - scalar_types_df, [op.as_expr(col_name, col_name)], ["result"] - ) - - snapshot.assert_match(sql, "out.sql") - - -def test_ai_if_with_endpoint(scalar_types_df: dataframe.DataFrame, snapshot): +def test_ai_if(scalar_types_df: dataframe.DataFrame, snapshot): col_name = "string_col" op = ops.AIIf( prompt_context=(None, " is the same as ", None), - endpoint="gemini-2.5-flash", + connection_id=CONNECTION_ID, ) sql = utils._apply_ops_to_sql( @@ -332,65 +296,13 @@ def test_ai_if_with_endpoint(scalar_types_df: dataframe.DataFrame, snapshot): snapshot.assert_match(sql, "out.sql") -@pytest.mark.parametrize("connection_id", [None, CONNECTION_ID]) -def test_ai_classify(scalar_types_df: dataframe.DataFrame, snapshot, connection_id): +def test_ai_classify(scalar_types_df: dataframe.DataFrame, snapshot): col_name = "string_col" op = ops.AIClassify( prompt_context=(None,), categories=("greeting", "rejection"), - connection_id=connection_id, - ) - - sql = utils._apply_ops_to_sql(scalar_types_df, [op.as_expr(col_name)], ["result"]) - - snapshot.assert_match(sql, "out.sql") - - -def test_ai_classify_with_params(scalar_types_df: dataframe.DataFrame, snapshot): - col_name = "string_col" - - op = ops.AIClassify( - prompt_context=(None,), - categories=("greeting", "rejection"), - examples=(("hi", "greeting"), ("bye", "rejection")), - endpoint="gemini-2.5-flash", - max_error_ratio=0.1, - ) - - sql = utils._apply_ops_to_sql(scalar_types_df, [op.as_expr(col_name)], ["result"]) - - snapshot.assert_match(sql, "out.sql") - - -def test_ai_classify_with_output_mode(scalar_types_df: dataframe.DataFrame, snapshot): - col_name = "string_col" - - op = ops.AIClassify( - prompt_context=(None,), - categories=("greeting", "rejection"), - output_mode="multi", - ) - - sql = utils._apply_ops_to_sql(scalar_types_df, [op.as_expr(col_name)], ["result"]) - - snapshot.assert_match(sql, "out.sql") - - -def test_ai_classify_multi_with_list_examples( - scalar_types_df: dataframe.DataFrame, snapshot -): - col_name = "string_col" - - examples = ( - ("hi", ("greeting", "positive")), - ("bye", ("rejection", "negative")), - ) - op = ops.AIClassify( - prompt_context=(None,), - categories=("greeting", "rejection"), - examples=examples, - output_mode="multi", + connection_id=CONNECTION_ID, ) sql = utils._apply_ops_to_sql(scalar_types_df, [op.as_expr(col_name)], ["result"]) @@ -398,76 +310,12 @@ def test_ai_classify_multi_with_list_examples( snapshot.assert_match(sql, "out.sql") -@pytest.mark.parametrize("connection_id", [None, CONNECTION_ID]) -def test_ai_score(scalar_types_df: dataframe.DataFrame, snapshot, connection_id): - col_name = "string_col" - - op = ops.AIScore( - prompt_context=(None, " is the same as ", None), - connection_id=connection_id, - ) - - sql = utils._apply_ops_to_sql( - scalar_types_df, [op.as_expr(col_name, col_name)], ["result"] - ) - - snapshot.assert_match(sql, "out.sql") - - -def test_ai_score_with_endpoint_and_max_error_ratio( - scalar_types_df: dataframe.DataFrame, snapshot -): +def test_ai_score(scalar_types_df: dataframe.DataFrame, snapshot): col_name = "string_col" op = ops.AIScore( prompt_context=(None, " is the same as ", None), - endpoint="gemini-2.5-flash", - max_error_ratio=0.5, - ) - - sql = utils._apply_ops_to_sql( - scalar_types_df, [op.as_expr(col_name, col_name)], ["result"] - ) - - snapshot.assert_match(sql, "out.sql") - - -@pytest.mark.parametrize("connection_id", [None, CONNECTION_ID]) -def test_ai_similarity(scalar_types_df: dataframe.DataFrame, snapshot, connection_id): - col_name = "string_col" - - op = ops.AISimilarity( - endpoint="text-embedding-005", - connection_id=connection_id, - ) - - sql = utils._apply_ops_to_sql( - scalar_types_df, [op.as_expr(col_name, col_name)], ["result"] - ) - - snapshot.assert_match(sql, "out.sql") - - -def test_ai_similarity_with_model(scalar_types_df: dataframe.DataFrame, snapshot): - col_name = "string_col" - - op = ops.AISimilarity( - model="embeddinggemma-300m", - ) - - sql = utils._apply_ops_to_sql( - scalar_types_df, [op.as_expr(col_name, col_name)], ["result"] - ) - - snapshot.assert_match(sql, "out.sql") - - -def test_ai_similarity_with_model_param(scalar_types_df: dataframe.DataFrame, snapshot): - col_name = "string_col" - - op = ops.AISimilarity( - endpoint="text-embedding-005", - model_params=json.dumps({"outputDimensionality": 256}), + connection_id=CONNECTION_ID, ) sql = utils._apply_ops_to_sql( diff --git a/tests/unit/core/compile/sqlglot/expressions/test_array_ops.py b/tests/unit/core/compile/sqlglot/expressions/test_array_ops.py index 1b358b3a3b1..67c8bb0e5ca 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_array_ops.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_array_ops.py @@ -1,23 +1,24 @@ # Copyright 2025 Google LLC # -# Licensed under the Apache License, Version 2.0 (the \"License\"); +# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, +# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import pytest -import bigframes.operations.aggregations as agg_ops -import bigframes.pandas as bpd from bigframes import operations as ops from bigframes.core import expression +from bigframes.operations._op_converters import convert_index, convert_slice +import bigframes.operations.aggregations as agg_ops +import bigframes.pandas as bpd from bigframes.testing import utils pytest.importorskip("pytest_snapshot") @@ -33,18 +34,13 @@ def test_array_to_string(repeated_types_df: bpd.DataFrame, snapshot): snapshot.assert_match(sql, "out.sql") -def test_array_index(scalar_types_df: bpd.DataFrame, snapshot): - ops_map = { - "string_index": ops.GetItemOp(key=1).as_expr("string_col"), - "array_index": expression.OpExpression( - ops.GetItemOp(key=1), - (ops.ToArrayOp().as_expr("int64_col", "int64_too"),), - ), - } - +def test_array_index(repeated_types_df: bpd.DataFrame, snapshot): + col_name = "string_list_col" + bf_df = repeated_types_df[[col_name]] sql = utils._apply_ops_to_sql( - scalar_types_df, list(ops_map.values()), list(ops_map.keys()) + bf_df, [convert_index(1).as_expr(col_name)], [col_name] ) + snapshot.assert_match(sql, "out.sql") @@ -54,9 +50,6 @@ def test_array_reduce_op(repeated_types_df: bpd.DataFrame, snapshot): "std_float": ops.ArrayReduceOp(agg_ops.StdOp()).as_expr("float_list_col"), "count_str": ops.ArrayReduceOp(agg_ops.CountOp()).as_expr("string_list_col"), "any_bool": ops.ArrayReduceOp(agg_ops.AnyOp()).as_expr("bool_list_col"), - "array_agg_str": ops.ArrayReduceOp(agg_ops.ArrayAggOp()).as_expr( - "string_list_col" - ), } sql = utils._apply_ops_to_sql( @@ -65,23 +58,23 @@ def test_array_reduce_op(repeated_types_df: bpd.DataFrame, snapshot): snapshot.assert_match(sql, "out.sql") -def test_array_slice(scalar_types_df: bpd.DataFrame, snapshot): - array_expr = ops.ToArrayOp().as_expr("int64_col", "int64_too") - ops_map = { - "string_slice": ops.ArraySliceOp(start=1, stop=5).as_expr("string_col"), - "slice_only_start": expression.OpExpression( - ops.ArraySliceOp(start=1, stop=None), - (array_expr,), - ), - "slice_start_stop": expression.OpExpression( - ops.ArraySliceOp(start=1, stop=5), - (array_expr,), - ), - } +def test_array_slice_with_only_start(repeated_types_df: bpd.DataFrame, snapshot): + col_name = "string_list_col" + bf_df = repeated_types_df[[col_name]] + sql = utils._apply_ops_to_sql( + bf_df, [convert_slice(slice(1, None)).as_expr(col_name)], [col_name] + ) + snapshot.assert_match(sql, "out.sql") + + +def test_array_slice_with_start_and_stop(repeated_types_df: bpd.DataFrame, snapshot): + col_name = "string_list_col" + bf_df = repeated_types_df[[col_name]] sql = utils._apply_ops_to_sql( - scalar_types_df, list(ops_map.values()), list(ops_map.keys()) + bf_df, [convert_slice(slice(1, 5)).as_expr(col_name)], [col_name] ) + snapshot.assert_match(sql, "out.sql") @@ -104,14 +97,3 @@ def test_to_array_op(scalar_types_df: bpd.DataFrame, snapshot): sql = utils._apply_ops_to_sql(bf_df, list(ops_map.values()), list(ops_map.keys())) snapshot.assert_match(sql, "out.sql") - - -def test_to_array_with_subquery_expression(repeated_types_df: bpd.DataFrame, snapshot): - reduced = ops.ArrayReduceOp(agg_ops.SumOp()).as_expr("float_list_col") - coalesced_reduced = ops.coalesce_op.as_expr(reduced, expression.const(0.0)) - array_expr = ops.ToArrayOp().as_expr(coalesced_reduced) - - sql = utils._apply_ops_to_sql( - repeated_types_df, [array_expr], ["arr_subquery_coalesce"] - ) - snapshot.assert_match(sql, "out.sql") diff --git a/tests/unit/core/compile/sqlglot/expressions/test_literals.py b/tests/unit/core/compile/sqlglot/expressions/test_blob_ops.py similarity index 54% rename from tests/unit/core/compile/sqlglot/expressions/test_literals.py rename to tests/unit/core/compile/sqlglot/expressions/test_blob_ops.py index aa0d7a1e5b1..80aa22aaac9 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_literals.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_blob_ops.py @@ -14,22 +14,23 @@ import pytest -import bigframes.core.expression as ex import bigframes.pandas as bpd -from bigframes.testing import utils pytest.importorskip("pytest_snapshot") -def test_float_literals(scalar_types_df: bpd.DataFrame, snapshot): - bf_df = scalar_types_df[["float64_col"]] - ops_map = { - "inf": ex.const(float("inf")), - "ninf": ex.const(float("-inf")), - "nan": ex.const(float("nan")), - "neg_zero": ex.const(-0.0), - "0.00001": ex.const(0.00001), - "1E-10": ex.const(1e-10), - } - sql = utils._apply_ops_to_sql(bf_df, list(ops_map.values()), list(ops_map.keys())) +def test_obj_fetch_metadata(scalar_types_df: bpd.DataFrame, snapshot): + blob_s = scalar_types_df["string_col"].str.to_blob() + sql = blob_s.blob.version().to_frame().sql snapshot.assert_match(sql, "out.sql") + + +def test_obj_get_access_url(scalar_types_df: bpd.DataFrame, snapshot): + blob_s = scalar_types_df["string_col"].str.to_blob() + sql = blob_s.blob.read_url().to_frame().sql + snapshot.assert_match(sql, "out.sql") + + +def test_obj_make_ref(scalar_types_df: bpd.DataFrame, snapshot): + blob_df = scalar_types_df["string_col"].str.to_blob() + snapshot.assert_match(blob_df.to_frame().sql, "out.sql") diff --git a/tests/unit/core/compile/sqlglot/expressions/test_bool_ops.py b/tests/unit/core/compile/sqlglot/expressions/test_bool_ops.py index bd51ea905a2..601fd86e4e9 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_bool_ops.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_bool_ops.py @@ -26,7 +26,6 @@ def test_and_op(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_and_int"] = bf_df["int64_col"] & bf_df["int64_col"] bf_df["bool_and_bool"] = bf_df["bool_col"] & bf_df["bool_col"] bf_df["bool_and_null"] = bf_df["bool_col"] & pd.NA # type: ignore - bf_df["null_and_bool"] = pd.NA & bf_df["bool_col"] # type: ignore snapshot.assert_match(bf_df.sql, "out.sql") @@ -36,7 +35,6 @@ def test_or_op(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_and_int"] = bf_df["int64_col"] | bf_df["int64_col"] bf_df["bool_and_bool"] = bf_df["bool_col"] | bf_df["bool_col"] bf_df["bool_and_null"] = bf_df["bool_col"] | pd.NA # type: ignore - bf_df["null_and_bool"] = pd.NA | bf_df["bool_col"] # type: ignore snapshot.assert_match(bf_df.sql, "out.sql") @@ -46,5 +44,4 @@ def test_xor_op(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_and_int"] = bf_df["int64_col"] ^ bf_df["int64_col"] bf_df["bool_and_bool"] = bf_df["bool_col"] ^ bf_df["bool_col"] bf_df["bool_and_null"] = bf_df["bool_col"] ^ pd.NA # type: ignore - bf_df["null_and_bool"] = pd.NA ^ bf_df["bool_col"] # type: ignore snapshot.assert_match(bf_df.sql, "out.sql") diff --git a/tests/unit/core/compile/sqlglot/expressions/test_comparison_ops.py b/tests/unit/core/compile/sqlglot/expressions/test_comparison_ops.py index 73aceaedeeb..3c13bc798bc 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_comparison_ops.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_comparison_ops.py @@ -15,8 +15,8 @@ import pandas as pd import pytest -import bigframes.pandas as bpd from bigframes import operations as ops +import bigframes.pandas as bpd from bigframes.testing import utils pytest.importorskip("pytest_snapshot") @@ -35,17 +35,7 @@ def test_is_in(scalar_types_df: bpd.DataFrame, snapshot): int_col ), "strings": ops.IsInOp(values=("1.0", "2.0")).as_expr(int_col), - "mixed": ops.IsInOp( - values=( - "1.0", - 2.5, - 3, - 1e-10, - float("inf"), - float("nan"), - 0, - ) - ).as_expr(int_col), + "mixed": ops.IsInOp(values=("1.0", 2.5, 3)).as_expr(int_col), "empty": ops.IsInOp(values=()).as_expr(int_col), "empty_wo_match_nulls": ops.IsInOp(values=(), match_nulls=False).as_expr( int_col @@ -54,13 +44,6 @@ def test_is_in(scalar_types_df: bpd.DataFrame, snapshot): values=(None, 123456), match_nulls=False ).as_expr(int_col), "float_in_ints": ops.IsInOp(values=(1, 2, 3, None)).as_expr(float_col), - "mixed_with_null": ops.IsInOp( - values=("1.0", 2, None), match_nulls=True - ).as_expr(int_col), - "bool_in_mixed": ops.IsInOp(values=(1, 2.5)).as_expr(bool_col), - "only_null_match": ops.IsInOp(values=(None,), match_nulls=True).as_expr( - int_col - ), } sql = utils._apply_ops_to_sql(bf_df, list(ops_map.values()), list(ops_map.keys())) @@ -79,7 +62,6 @@ def test_eq_numeric(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_eq_int"] = bf_df["int64_col"] == bf_df["int64_col"] bf_df["int_eq_1"] = bf_df["int64_col"] == 1 bf_df["int_eq_null"] = bf_df["int64_col"] == pd.NA - bf_df["null_eq_int"] = pd.NA == bf_df["int64_col"] bf_df["int_eq_bool"] = bf_df["int64_col"] == bf_df["bool_col"] bf_df["bool_eq_int"] = bf_df["bool_col"] == bf_df["int64_col"] @@ -92,7 +74,6 @@ def test_gt_numeric(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_gt_int"] = bf_df["int64_col"] > bf_df["int64_col"] bf_df["int_gt_1"] = bf_df["int64_col"] > 1 - bf_df["null_gt_int"] = pd.NA > bf_df["int64_col"] bf_df["int_gt_bool"] = bf_df["int64_col"] > bf_df["bool_col"] bf_df["bool_gt_int"] = bf_df["bool_col"] > bf_df["int64_col"] @@ -105,7 +86,6 @@ def test_ge_numeric(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_ge_int"] = bf_df["int64_col"] >= bf_df["int64_col"] bf_df["int_ge_1"] = bf_df["int64_col"] >= 1 - bf_df["null_ge_int"] = pd.NA >= bf_df["int64_col"] bf_df["int_ge_bool"] = bf_df["int64_col"] >= bf_df["bool_col"] bf_df["bool_ge_int"] = bf_df["bool_col"] >= bf_df["int64_col"] @@ -118,7 +98,6 @@ def test_lt_numeric(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_lt_int"] = bf_df["int64_col"] < bf_df["int64_col"] bf_df["int_lt_1"] = bf_df["int64_col"] < 1 - bf_df["null_lt_int"] = pd.NA < bf_df["int64_col"] bf_df["int_lt_bool"] = bf_df["int64_col"] < bf_df["bool_col"] bf_df["bool_lt_int"] = bf_df["bool_col"] < bf_df["int64_col"] @@ -131,7 +110,6 @@ def test_le_numeric(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_le_int"] = bf_df["int64_col"] <= bf_df["int64_col"] bf_df["int_le_1"] = bf_df["int64_col"] <= 1 - bf_df["null_le_int"] = pd.NA <= bf_df["int64_col"] bf_df["int_le_bool"] = bf_df["int64_col"] <= bf_df["bool_col"] bf_df["bool_le_int"] = bf_df["bool_col"] <= bf_df["int64_col"] @@ -159,7 +137,6 @@ def test_ne_numeric(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_ne_int"] = bf_df["int64_col"] != bf_df["int64_col"] bf_df["int_ne_1"] = bf_df["int64_col"] != 1 bf_df["int_ne_null"] = bf_df["int64_col"] != pd.NA - bf_df["null_ne_int"] = pd.NA != bf_df["int64_col"] bf_df["int_ne_bool"] = bf_df["int64_col"] != bf_df["bool_col"] bf_df["bool_ne_int"] = bf_df["bool_col"] != bf_df["int64_col"] diff --git a/tests/unit/core/compile/sqlglot/expressions/test_datetime_ops.py b/tests/unit/core/compile/sqlglot/expressions/test_datetime_ops.py index e86059b160a..95156748e96 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_datetime_ops.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_datetime_ops.py @@ -15,8 +15,8 @@ import pandas as pd import pytest -import bigframes.pandas as bpd from bigframes import operations as ops +import bigframes.pandas as bpd from bigframes.testing import utils pytest.importorskip("pytest_snapshot") @@ -62,39 +62,10 @@ def test_datetime_to_integer_label(scalar_types_df: bpd.DataFrame, snapshot): bf_df = scalar_types_df[col_names] ops_map = { "fixed_freq": ops.DatetimeToIntegerLabelOp( - freq=pd.tseries.offsets.Day(), # type: ignore[arg-type] - origin="start", - closed="left", # type: ignore - ).as_expr("datetime_col", "timestamp_col"), - "origin_epoch": ops.DatetimeToIntegerLabelOp( - freq=pd.tseries.offsets.Day(), # type: ignore[arg-type] - origin="epoch", - closed="left", # type: ignore - ).as_expr("datetime_col", "timestamp_col"), - "origin_start_day": ops.DatetimeToIntegerLabelOp( - freq=pd.tseries.offsets.Day(), # type: ignore[arg-type] - origin="start_day", - closed="left", # type: ignore + freq=pd.tseries.offsets.Day(), origin="start", closed="left" # type: ignore ).as_expr("datetime_col", "timestamp_col"), "non_fixed_freq_weekly": ops.DatetimeToIntegerLabelOp( - freq=pd.tseries.offsets.Week(weekday=6), # type: ignore[arg-type] - origin="start", - closed="left", # type: ignore - ).as_expr("datetime_col", "timestamp_col"), - "non_fixed_freq_monthly": ops.DatetimeToIntegerLabelOp( - freq=pd.tseries.offsets.MonthEnd(), # type: ignore[arg-type] - origin="start", - closed="left", # type: ignore - ).as_expr("datetime_col", "timestamp_col"), - "non_fixed_freq_quarterly": ops.DatetimeToIntegerLabelOp( - freq=pd.tseries.offsets.QuarterEnd(startingMonth=12), # type: ignore[arg-type] - origin="start", - closed="left", # type: ignore - ).as_expr("datetime_col", "timestamp_col"), - "non_fixed_freq_yearly": ops.DatetimeToIntegerLabelOp( - freq=pd.tseries.offsets.YearEnd(), # type: ignore[arg-type] - origin="start", - closed="left", # type: ignore + freq=pd.tseries.offsets.Week(weekday=6), origin="start", closed="left" # type: ignore ).as_expr("datetime_col", "timestamp_col"), } @@ -209,19 +180,16 @@ def test_time(scalar_types_df: bpd.DataFrame, snapshot): def test_to_datetime(scalar_types_df: bpd.DataFrame, snapshot): - col_names = ["int64_col", "string_col", "float64_col", "timestamp_col"] + col_names = ["int64_col", "string_col", "float64_col"] bf_df = scalar_types_df[col_names] ops_map = {col_name: ops.ToDatetimeOp().as_expr(col_name) for col_name in col_names} - ops_map["string_col_fmt"] = ops.ToDatetimeOp(format="%Y-%m-%d").as_expr( - "string_col" - ) sql = utils._apply_ops_to_sql(bf_df, list(ops_map.values()), list(ops_map.keys())) - snapshot.assert_match(sql + "\n", "out.sql") + snapshot.assert_match(sql, "out.sql") def test_to_timestamp(scalar_types_df: bpd.DataFrame, snapshot): - bf_df = scalar_types_df[["int64_col", "string_col", "float64_col", "datetime_col"]] + bf_df = scalar_types_df[["int64_col", "string_col", "float64_col"]] ops_map = { "int64_col": ops.ToTimestampOp().as_expr("int64_col"), "float64_col": ops.ToTimestampOp().as_expr("float64_col"), @@ -229,8 +197,6 @@ def test_to_timestamp(scalar_types_df: bpd.DataFrame, snapshot): "int64_col_ms": ops.ToTimestampOp(unit="ms").as_expr("int64_col"), "int64_col_us": ops.ToTimestampOp(unit="us").as_expr("int64_col"), "int64_col_ns": ops.ToTimestampOp(unit="ns").as_expr("int64_col"), - "datetime_col": ops.ToTimestampOp().as_expr("datetime_col"), - "string_col_fmt": ops.ToTimestampOp(format="%Y-%m-%d").as_expr("string_col"), } sql = utils._apply_ops_to_sql(bf_df, list(ops_map.values()), list(ops_map.keys())) @@ -334,9 +300,7 @@ def test_integer_label_to_datetime_fixed(scalar_types_df: bpd.DataFrame, snapsho bf_df = scalar_types_df[col_names] ops_map = { "fixed_freq": ops.IntegerLabelToDatetimeOp( - freq=pd.tseries.offsets.Day(), # type: ignore[arg-type] - origin="start", - label="left", # type: ignore + freq=pd.tseries.offsets.Day(), origin="start", label="left" # type: ignore ).as_expr("rowindex", "timestamp_col"), } @@ -349,9 +313,7 @@ def test_integer_label_to_datetime_week(scalar_types_df: bpd.DataFrame, snapshot bf_df = scalar_types_df[col_names] ops_map = { "non_fixed_freq_weekly": ops.IntegerLabelToDatetimeOp( - freq=pd.tseries.offsets.Week(weekday=6), # type: ignore[arg-type] - origin="start", - label="left", # type: ignore + freq=pd.tseries.offsets.Week(weekday=6), origin="start", label="left" # type: ignore ).as_expr("rowindex", "timestamp_col"), } diff --git a/tests/unit/core/compile/sqlglot/expressions/test_generic_ops.py b/tests/unit/core/compile/sqlglot/expressions/test_generic_ops.py index e3669e1b0ed..2667e482c88 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_generic_ops.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_generic_ops.py @@ -12,15 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from google.cloud import bigquery import pandas as pd import pytest -from google.cloud import bigquery -import bigframes.pandas as bpd from bigframes import dtypes from bigframes import operations as ops from bigframes.core import expression as ex from bigframes.functions import udf_def +import bigframes.pandas as bpd from bigframes.testing import utils pytest.importorskip("pytest_snapshot") @@ -60,7 +60,7 @@ def test_astype_float(scalar_types_df: bpd.DataFrame, snapshot): "bool_w_safe": ops.AsTypeOp(to_type=to_type, safe=True).as_expr("bool_col"), } sql = utils._apply_ops_to_sql(bf_df, list(ops_map.values()), list(ops_map.keys())) - snapshot.assert_match(sql + "\n", "out.sql") + snapshot.assert_match(sql, "out.sql") def test_astype_bool(scalar_types_df: bpd.DataFrame, snapshot): @@ -107,19 +107,23 @@ def test_astype_string(scalar_types_df: bpd.DataFrame, snapshot): "bool_w_safe": ops.AsTypeOp(to_type=to_type, safe=True).as_expr("bool_col"), } sql = utils._apply_ops_to_sql(bf_df, list(ops_map.values()), list(ops_map.keys())) - snapshot.assert_match(sql + "\n", "out.sql") + snapshot.assert_match(sql, "out.sql") -def test_to_json(scalar_types_df: bpd.DataFrame, snapshot): +def test_astype_json(scalar_types_df: bpd.DataFrame, snapshot): bf_df = scalar_types_df ops_map = { - "int64_col": ops.ToJSON().as_expr("int64_col"), - "float64_col": ops.ToJSON().as_expr("float64_col"), - "bool_col": ops.ToJSON().as_expr("bool_col"), - "string_col": ops.ToJSON().as_expr("string_col"), - "bool_w_safe": ops.ToJSON(safe=True).as_expr("bool_col"), - "string_w_safe": ops.ToJSON(safe=True).as_expr("string_col"), + "int64_col": ops.AsTypeOp(to_type=dtypes.JSON_DTYPE).as_expr("int64_col"), + "float64_col": ops.AsTypeOp(to_type=dtypes.JSON_DTYPE).as_expr("float64_col"), + "bool_col": ops.AsTypeOp(to_type=dtypes.JSON_DTYPE).as_expr("bool_col"), + "string_col": ops.AsTypeOp(to_type=dtypes.JSON_DTYPE).as_expr("string_col"), + "bool_w_safe": ops.AsTypeOp(to_type=dtypes.JSON_DTYPE, safe=True).as_expr( + "bool_col" + ), + "string_w_safe": ops.AsTypeOp(to_type=dtypes.JSON_DTYPE, safe=True).as_expr( + "string_col" + ), } sql = utils._apply_ops_to_sql(bf_df, list(ops_map.values()), list(ops_map.keys())) snapshot.assert_match(sql, "out.sql") @@ -129,11 +133,11 @@ def test_astype_from_json(json_types_df: bpd.DataFrame, snapshot): bf_df = json_types_df ops_map = { - "int64_col": ops.JSONDecode(to_type=dtypes.INT_DTYPE).as_expr("json_col"), - "float64_col": ops.JSONDecode(to_type=dtypes.FLOAT_DTYPE).as_expr("json_col"), - "bool_col": ops.JSONDecode(to_type=dtypes.BOOL_DTYPE).as_expr("json_col"), - "string_col": ops.JSONDecode(to_type=dtypes.STRING_DTYPE).as_expr("json_col"), - "int64_w_safe": ops.JSONDecode(to_type=dtypes.INT_DTYPE, safe=True).as_expr( + "int64_col": ops.AsTypeOp(to_type=dtypes.INT_DTYPE).as_expr("json_col"), + "float64_col": ops.AsTypeOp(to_type=dtypes.FLOAT_DTYPE).as_expr("json_col"), + "bool_col": ops.AsTypeOp(to_type=dtypes.BOOL_DTYPE).as_expr("json_col"), + "string_col": ops.AsTypeOp(to_type=dtypes.STRING_DTYPE).as_expr("json_col"), + "int64_w_safe": ops.AsTypeOp(to_type=dtypes.INT_DTYPE, safe=True).as_expr( "json_col" ), } @@ -141,20 +145,24 @@ def test_astype_from_json(json_types_df: bpd.DataFrame, snapshot): snapshot.assert_match(sql, "out.sql") -def test_tojson_invalid(scalar_types_df: bpd.DataFrame, json_types_df: bpd.DataFrame): +def test_astype_json_invalid( + scalar_types_df: bpd.DataFrame, json_types_df: bpd.DataFrame +): # Test invalid cast to JSON - with pytest.raises(TypeError): + with pytest.raises(TypeError, match="Cannot cast timestamp.* to .*json.*"): ops_map_to = { - "datetime_to_json": ops.ToJSON().as_expr("datetime_col"), + "datetime_to_json": ops.AsTypeOp(to_type=dtypes.JSON_DTYPE).as_expr( + "datetime_col" + ), } utils._apply_ops_to_sql( scalar_types_df, list(ops_map_to.values()), list(ops_map_to.keys()) ) # Test invalid cast from JSON - with pytest.raises(TypeError): + with pytest.raises(TypeError, match="Cannot cast .*json.* to timestamp.*"): ops_map_from = { - "json_to_datetime": ops.JSONDecode(to_type=dtypes.DATETIME_DTYPE).as_expr( + "json_to_datetime": ops.AsTypeOp(to_type=dtypes.DATETIME_DTYPE).as_expr( "json_col" ), } @@ -164,28 +172,101 @@ def test_tojson_invalid(scalar_types_df: bpd.DataFrame, json_types_df: bpd.DataF def test_remote_function_op(scalar_types_df: bpd.DataFrame, snapshot): + bf_df = scalar_types_df[["int64_col"]] + function_def = udf_def.BigqueryUdf( + routine_ref=bigquery.RoutineReference.from_string( + "my_project.my_dataset.my_routine" + ), + signature=udf_def.UdfSignature( + input_types=( + udf_def.UdfField( + "x", + bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 + ), + ), + ), + output_bq_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.FLOAT64 + ), + ), + ) + ops_map = { + "apply_on_null_true": ops.RemoteFunctionOp( + function_def=function_def, apply_on_null=True + ).as_expr("int64_col"), + "apply_on_null_false": ops.RemoteFunctionOp( + function_def=function_def, apply_on_null=False + ).as_expr("int64_col"), + } + sql = utils._apply_ops_to_sql(bf_df, list(ops_map.values()), list(ops_map.keys())) + snapshot.assert_match(sql, "out.sql") + + +def test_binary_remote_function_op(scalar_types_df: bpd.DataFrame, snapshot): + bf_df = scalar_types_df[["int64_col", "float64_col"]] + op = ops.BinaryRemoteFunctionOp( + function_def=udf_def.BigqueryUdf( + routine_ref=bigquery.RoutineReference.from_string( + "my_project.my_dataset.my_routine" + ), + signature=udf_def.UdfSignature( + input_types=( + udf_def.UdfField( + "x", + bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 + ), + ), + udf_def.UdfField( + "y", + bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.FLOAT64 + ), + ), + ), + output_bq_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.FLOAT64 + ), + ), + ) + ) + sql = utils._apply_binary_op(bf_df, op, "int64_col", "float64_col") + + snapshot.assert_match(sql, "out.sql") + + +def test_nary_remote_function_op(scalar_types_df: bpd.DataFrame, snapshot): bf_df = scalar_types_df[["int64_col", "float64_col", "string_col"]] - op = ops.RemoteFunctionOp( + op = ops.NaryRemoteFunctionOp( function_def=udf_def.BigqueryUdf( routine_ref=bigquery.RoutineReference.from_string( "my_project.my_dataset.my_routine" ), signature=udf_def.UdfSignature( - inputs=( - udf_def.UdfArg( + input_types=( + udf_def.UdfField( "x", - udf_def.DirectScalarType(int), + bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 + ), ), - udf_def.UdfArg( + udf_def.UdfField( "y", - udf_def.DirectScalarType(float), + bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.FLOAT64 + ), ), - udf_def.UdfArg( + udf_def.UdfField( "z", - udf_def.DirectScalarType(str), + bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.STRING + ), ), ), - output=udf_def.DirectScalarType(float), + output_bq_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.FLOAT64 + ), ), ) ) diff --git a/tests/unit/core/compile/sqlglot/expressions/test_geo_ops.py b/tests/unit/core/compile/sqlglot/expressions/test_geo_ops.py index 85e374c76db..9047ce4d04c 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_geo_ops.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_geo_ops.py @@ -14,13 +14,23 @@ import pytest -import bigframes.pandas as bpd from bigframes import operations as ops +import bigframes.pandas as bpd from bigframes.testing import utils pytest.importorskip("pytest_snapshot") +def test_geo_area(scalar_types_df: bpd.DataFrame, snapshot): + col_name = "geography_col" + bf_df = scalar_types_df[[col_name]] + sql = utils._apply_ops_to_sql( + bf_df, [ops.geo_area_op.as_expr(col_name)], [col_name] + ) + + snapshot.assert_match(sql, "out.sql") + + def test_geo_st_astext(scalar_types_df: bpd.DataFrame, snapshot): col_name = "geography_col" bf_df = scalar_types_df[[col_name]] @@ -51,6 +61,16 @@ def test_geo_st_buffer(scalar_types_df: bpd.DataFrame, snapshot): snapshot.assert_match(sql, "out.sql") +def test_geo_st_centroid(scalar_types_df: bpd.DataFrame, snapshot): + col_name = "geography_col" + bf_df = scalar_types_df[[col_name]] + sql = utils._apply_ops_to_sql( + bf_df, [ops.geo_st_centroid_op.as_expr(col_name)], [col_name] + ) + + snapshot.assert_match(sql, "out.sql") + + def test_geo_st_convexhull(scalar_types_df: bpd.DataFrame, snapshot): col_name = "geography_col" bf_df = scalar_types_df[[col_name]] diff --git a/tests/unit/core/compile/sqlglot/expressions/test_json_ops.py b/tests/unit/core/compile/sqlglot/expressions/test_json_ops.py index 69eb8681abc..1c5894fc966 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_json_ops.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_json_ops.py @@ -14,9 +14,9 @@ import pytest +from bigframes import operations as ops import bigframes.core.expression as ex import bigframes.pandas as bpd -from bigframes import operations as ops from bigframes.testing import utils pytest.importorskip("pytest_snapshot") @@ -95,16 +95,6 @@ def test_json_value(json_types_df: bpd.DataFrame, snapshot): snapshot.assert_match(sql, "out.sql") -def test_json_value_array(json_types_df: bpd.DataFrame, snapshot): - col_name = "json_col" - bf_df = json_types_df[[col_name]] - sql = utils._apply_ops_to_sql( - bf_df, [ops.JSONValueArray(json_path="$").as_expr(col_name)], [col_name] - ) - - snapshot.assert_match(sql, "out.sql") - - def test_parse_json(scalar_types_df: bpd.DataFrame, snapshot): col_name = "string_col" bf_df = scalar_types_df[[col_name]] diff --git a/tests/unit/core/compile/sqlglot/expressions/test_numeric_ops.py b/tests/unit/core/compile/sqlglot/expressions/test_numeric_ops.py index b0442f6992e..1a08a80eb1d 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_numeric_ops.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_numeric_ops.py @@ -15,10 +15,9 @@ import pandas as pd import pytest +from bigframes import operations as ops import bigframes.core.expression as ex import bigframes.pandas as bpd -from bigframes import operations as ops -from bigframes.operations import numeric_ops from bigframes.testing import utils pytest.importorskip("pytest_snapshot") @@ -157,16 +156,6 @@ def test_floor(scalar_types_df: bpd.DataFrame, snapshot): snapshot.assert_match(sql, "out.sql") -def test_isfinite(scalar_types_df: bpd.DataFrame, snapshot): - col_name = "float64_col" - bf_df = scalar_types_df[[col_name]] - sql = utils._apply_ops_to_sql( - bf_df, [numeric_ops.isfinite_op.as_expr(col_name)], [col_name] - ) - - snapshot.assert_match(sql, "out.sql") - - def test_ln(scalar_types_df: bpd.DataFrame, snapshot): col_name = "float64_col" bf_df = scalar_types_df[[col_name]] @@ -220,9 +209,6 @@ def test_pow(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_pow_1"] = bf_df["int64_col"] ** 1 bf_df["float_pow_1"] = bf_df["float64_col"] ** 1 - bf_df["float_pow_null"] = bf_df["float64_col"] ** pd.NA - bf_df["null_pow_float"] = pd.NA ** bf_df["float64_col"] - snapshot.assert_match(bf_df.sql, "out.sql") @@ -285,7 +271,6 @@ def test_add_numeric(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_add_int"] = bf_df["int64_col"] + bf_df["int64_col"] bf_df["int_add_1"] = bf_df["int64_col"] + 1 - bf_df["int_add_null"] = bf_df["int64_col"] + pd.NA bf_df["int_add_bool"] = bf_df["int64_col"] + bf_df["bool_col"] bf_df["bool_add_int"] = bf_df["bool_col"] + bf_df["int64_col"] @@ -327,7 +312,6 @@ def test_div_numeric(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_div_int"] = bf_df["int64_col"] / bf_df["int64_col"] bf_df["int_div_1"] = bf_df["int64_col"] / 1 bf_df["int_div_0"] = bf_df["int64_col"] / 0.0 - bf_df["int_div_null"] = bf_df["int64_col"] / pd.NA bf_df["int_div_float"] = bf_df["int64_col"] / bf_df["float64_col"] bf_df["float_div_int"] = bf_df["float64_col"] / bf_df["int64_col"] @@ -368,18 +352,14 @@ def test_floordiv_numeric(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_div_int"] = bf_df["int64_col"] // bf_df["int64_col"] bf_df["int_div_1"] = bf_df["int64_col"] // 1 bf_df["int_div_0"] = bf_df["int64_col"] // 0.0 - bf_df["int_div_null"] = bf_df["int64_col"] // pd.NA bf_df["int_div_float"] = bf_df["int64_col"] // bf_df["float64_col"] bf_df["float_div_int"] = bf_df["float64_col"] // bf_df["int64_col"] bf_df["float_div_0"] = bf_df["float64_col"] // 0.0 - bf_df["float_div_null"] = bf_df["float64_col"] // pd.NA bf_df["int_div_bool"] = bf_df["int64_col"] // bf_df["bool_col"] bf_df["bool_div_int"] = bf_df["bool_col"] // bf_df["int64_col"] - snapshot.assert_match(bf_df.sql, "out.sql") - def test_floordiv_timedelta(scalar_types_df: bpd.DataFrame, snapshot): bf_df = scalar_types_df[["timestamp_col", "date_col"]] @@ -410,7 +390,6 @@ def test_mul_numeric(scalar_types_df: bpd.DataFrame, snapshot): bf_df["int_mul_int"] = bf_df["int64_col"] * bf_df["int64_col"] bf_df["int_mul_1"] = bf_df["int64_col"] * 1 - bf_df["int_mul_null"] = bf_df["int64_col"] * pd.NA bf_df["int_mul_bool"] = bf_df["int64_col"] * bf_df["bool_col"] bf_df["bool_mul_int"] = bf_df["bool_col"] * bf_df["int64_col"] @@ -441,20 +420,17 @@ def test_mod_numeric(scalar_types_df: bpd.DataFrame, snapshot): bf_df["float_mod_1"] = bf_df["float64_col"] % 1 bf_df["float_mod_0"] = bf_df["float64_col"] % 0 - bf_df["float_mod_null"] = bf_df["float64_col"] % pd.NA - snapshot.assert_match(bf_df.sql, "out.sql") def test_sub_numeric(scalar_types_df: bpd.DataFrame, snapshot): bf_df = scalar_types_df[["int64_col", "bool_col"]] - bf_df["int_sub_int"] = bf_df["int64_col"] - bf_df["int64_col"] - bf_df["int_sub_1"] = bf_df["int64_col"] - 1 - bf_df["int_sub_null"] = bf_df["int64_col"] - pd.NA + bf_df["int_add_int"] = bf_df["int64_col"] - bf_df["int64_col"] + bf_df["int_add_1"] = bf_df["int64_col"] - 1 - bf_df["int_sub_bool"] = bf_df["int64_col"] - bf_df["bool_col"] - bf_df["bool_sub_int"] = bf_df["bool_col"] - bf_df["int64_col"] + bf_df["int_add_bool"] = bf_df["int64_col"] - bf_df["bool_col"] + bf_df["bool_add_int"] = bf_df["bool_col"] - bf_df["int64_col"] snapshot.assert_match(bf_df.sql, "out.sql") diff --git a/tests/unit/core/compile/sqlglot/expressions/test_string_ops.py b/tests/unit/core/compile/sqlglot/expressions/test_string_ops.py index 67efcfb08ca..b1fbbb0fc9b 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_string_ops.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_string_ops.py @@ -14,9 +14,9 @@ import pytest +from bigframes import operations as ops import bigframes.core.expression as ex import bigframes.pandas as bpd -from bigframes import operations as ops from bigframes.testing import utils pytest.importorskip("pytest_snapshot") @@ -183,6 +183,7 @@ def test_rstrip(scalar_types_df: bpd.DataFrame, snapshot): def test_startswith(scalar_types_df: bpd.DataFrame, snapshot): + col_name = "string_col" bf_df = scalar_types_df[[col_name]] ops_map = { @@ -198,7 +199,7 @@ def test_str_get(scalar_types_df: bpd.DataFrame, snapshot): col_name = "string_col" bf_df = scalar_types_df[[col_name]] sql = utils._apply_ops_to_sql( - bf_df, [ops.GetItemOp(1).as_expr(col_name)], [col_name] + bf_df, [ops.StrGetOp(1).as_expr(col_name)], [col_name] ) snapshot.assert_match(sql, "out.sql") @@ -219,17 +220,9 @@ def test_str_pad(scalar_types_df: bpd.DataFrame, snapshot): def test_str_slice(scalar_types_df: bpd.DataFrame, snapshot): col_name = "string_col" bf_df = scalar_types_df[[col_name]] - ops_map = { - "1_3": ops.StrSliceOp(1, 3).as_expr(col_name), - "none_3": ops.StrSliceOp(None, 3).as_expr(col_name), - "1_none": ops.StrSliceOp(1, None).as_expr(col_name), - "m3_none": ops.StrSliceOp(-3, None).as_expr(col_name), - "none_m3": ops.StrSliceOp(None, -3).as_expr(col_name), - "m5_m3": ops.StrSliceOp(-5, -3).as_expr(col_name), - "1_m3": ops.StrSliceOp(1, -3).as_expr(col_name), - "m3_5": ops.StrSliceOp(-3, 5).as_expr(col_name), - } - sql = utils._apply_ops_to_sql(bf_df, list(ops_map.values()), list(ops_map.keys())) + sql = utils._apply_ops_to_sql( + bf_df, [ops.StrSliceOp(1, 3).as_expr(col_name)], [col_name] + ) snapshot.assert_match(sql, "out.sql") diff --git a/tests/unit/core/compile/sqlglot/expressions/test_struct_ops.py b/tests/unit/core/compile/sqlglot/expressions/test_struct_ops.py index 5e1f3d505cb..0e24426fe8e 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_struct_ops.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_struct_ops.py @@ -16,9 +16,9 @@ import pytest -import bigframes.pandas as bpd from bigframes import operations as ops from bigframes.core import expression as ex +import bigframes.pandas as bpd from bigframes.testing import utils pytest.importorskip("pytest_snapshot") diff --git a/tests/unit/core/compile/sqlglot/expressions/test_timedelta_ops.py b/tests/unit/core/compile/sqlglot/expressions/test_timedelta_ops.py index ae1f6d017c4..164c11aab51 100644 --- a/tests/unit/core/compile/sqlglot/expressions/test_timedelta_ops.py +++ b/tests/unit/core/compile/sqlglot/expressions/test_timedelta_ops.py @@ -14,8 +14,8 @@ import pytest -import bigframes.pandas as bpd from bigframes import operations as ops +import bigframes.pandas as bpd from bigframes.testing import utils pytest.importorskip("pytest_snapshot") diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_aggregate/test_compile_aggregate/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_aggregate/test_compile_aggregate/out.sql index cfd9c7c87f0..949ed82574d 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_aggregate/test_compile_aggregate/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_aggregate/test_compile_aggregate/out.sql @@ -1,15 +1,19 @@ WITH `bfcte_0` AS ( SELECT `bool_col`, - `int64_too`, + `int64_too` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, `int64_too` AS `bfcol_2`, `bool_col` AS `bfcol_3` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( + FROM `bfcte_0` +), `bfcte_2` AS ( SELECT `bfcol_3`, COALESCE(SUM(`bfcol_2`), 0) AS `bfcol_6` - FROM `bfcte_0` + FROM `bfcte_1` WHERE NOT `bfcol_3` IS NULL GROUP BY @@ -18,6 +22,6 @@ WITH `bfcte_0` AS ( SELECT `bfcol_3` AS `bool_col`, `bfcol_6` AS `int64_too` -FROM `bfcte_1` +FROM `bfcte_2` ORDER BY `bfcol_3` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_aggregate/test_compile_aggregate_wo_dropna/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_aggregate/test_compile_aggregate_wo_dropna/out.sql index e71099d82e6..3c09250858d 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_aggregate/test_compile_aggregate_wo_dropna/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_aggregate/test_compile_aggregate_wo_dropna/out.sql @@ -1,21 +1,25 @@ WITH `bfcte_0` AS ( SELECT `bool_col`, - `int64_too`, + `int64_too` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, `int64_too` AS `bfcol_2`, `bool_col` AS `bfcol_3` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( + FROM `bfcte_0` +), `bfcte_2` AS ( SELECT `bfcol_3`, COALESCE(SUM(`bfcol_2`), 0) AS `bfcol_6` - FROM `bfcte_0` + FROM `bfcte_1` GROUP BY `bfcol_3` ) SELECT `bfcol_3` AS `bool_col`, `bfcol_6` AS `int64_too` -FROM `bfcte_1` +FROM `bfcte_2` ORDER BY `bfcol_3` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_concat/test_compile_concat/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_concat/test_compile_concat/out.sql index 48614357865..a0d7db2b1a2 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_concat/test_compile_concat/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_concat/test_compile_concat/out.sql @@ -1,48 +1,82 @@ -WITH `bfcte_0` AS ( - SELECT - `rowindex` AS `bfcol_3`, - `rowindex` AS `bfcol_4`, - `int64_col` AS `bfcol_5`, - `string_col` AS `bfcol_6` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( - SELECT - `bfcol_17` AS `bfcol_23`, - `bfcol_18` AS `bfcol_24`, - `bfcol_19` AS `bfcol_25`, - `bfcol_20` AS `bfcol_26`, - `bfcol_21` AS `bfcol_27`, - `bfcol_22` AS `bfcol_28` +WITH `bfcte_1` AS ( + SELECT + `int64_col`, + `rowindex`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_3` AS ( + SELECT + *, + ROW_NUMBER() OVER () - 1 AS `bfcol_7` + FROM `bfcte_1` +), `bfcte_5` AS ( + SELECT + *, + 0 AS `bfcol_8` + FROM `bfcte_3` +), `bfcte_6` AS ( + SELECT + `rowindex` AS `bfcol_9`, + `rowindex` AS `bfcol_10`, + `int64_col` AS `bfcol_11`, + `string_col` AS `bfcol_12`, + `bfcol_8` AS `bfcol_13`, + `bfcol_7` AS `bfcol_14` + FROM `bfcte_5` +), `bfcte_0` AS ( + SELECT + `int64_col`, + `rowindex`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_2` AS ( + SELECT + *, + ROW_NUMBER() OVER () - 1 AS `bfcol_22` + FROM `bfcte_0` +), `bfcte_4` AS ( + SELECT + *, + 1 AS `bfcol_23` + FROM `bfcte_2` +), `bfcte_7` AS ( + SELECT + `rowindex` AS `bfcol_24`, + `rowindex` AS `bfcol_25`, + `int64_col` AS `bfcol_26`, + `string_col` AS `bfcol_27`, + `bfcol_23` AS `bfcol_28`, + `bfcol_22` AS `bfcol_29` + FROM `bfcte_4` +), `bfcte_8` AS ( + SELECT + * FROM ( - ( - SELECT - `bfcol_3` AS `bfcol_17`, - `bfcol_4` AS `bfcol_18`, - `bfcol_5` AS `bfcol_19`, - `bfcol_6` AS `bfcol_20`, - 0 AS `bfcol_21`, - ROW_NUMBER() OVER () - 1 AS `bfcol_22` - FROM `bfcte_0` - ) + SELECT + `bfcol_9` AS `bfcol_30`, + `bfcol_10` AS `bfcol_31`, + `bfcol_11` AS `bfcol_32`, + `bfcol_12` AS `bfcol_33`, + `bfcol_13` AS `bfcol_34`, + `bfcol_14` AS `bfcol_35` + FROM `bfcte_6` UNION ALL - ( - SELECT - `bfcol_3` AS `bfcol_11`, - `bfcol_4` AS `bfcol_12`, - `bfcol_5` AS `bfcol_13`, - `bfcol_6` AS `bfcol_14`, - 1 AS `bfcol_15`, - ROW_NUMBER() OVER () - 1 AS `bfcol_16` - FROM `bfcte_0` - ) + SELECT + `bfcol_24` AS `bfcol_30`, + `bfcol_25` AS `bfcol_31`, + `bfcol_26` AS `bfcol_32`, + `bfcol_27` AS `bfcol_33`, + `bfcol_28` AS `bfcol_34`, + `bfcol_29` AS `bfcol_35` + FROM `bfcte_7` ) ) SELECT - `bfcol_23` AS `rowindex`, - `bfcol_24` AS `rowindex_1`, - `bfcol_25` AS `int64_col`, - `bfcol_26` AS `string_col` -FROM `bfcte_1` + `bfcol_30` AS `rowindex`, + `bfcol_31` AS `rowindex_1`, + `bfcol_32` AS `int64_col`, + `bfcol_33` AS `string_col` +FROM `bfcte_8` ORDER BY - `bfcol_27` ASC NULLS LAST, - `bfcol_28` ASC NULLS LAST \ No newline at end of file + `bfcol_34` ASC NULLS LAST, + `bfcol_35` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_concat/test_compile_concat_filter_sorted/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_concat/test_compile_concat_filter_sorted/out.sql index 477a47036ae..8e65381fef1 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_concat/test_compile_concat_filter_sorted/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_concat/test_compile_concat_filter_sorted/out.sql @@ -1,63 +1,142 @@ -WITH `bfcte_0` AS ( +WITH `bfcte_2` AS ( SELECT - `float64_col` AS `bfcol_7`, - `int64_too` AS `bfcol_8` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_6` AS ( + SELECT + *, + ROW_NUMBER() OVER (ORDER BY `int64_col` ASC NULLS LAST) - 1 AS `bfcol_4` + FROM `bfcte_2` +), `bfcte_10` AS ( + SELECT + *, + 0 AS `bfcol_5` + FROM `bfcte_6` +), `bfcte_13` AS ( + SELECT + `float64_col` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `bfcol_5` AS `bfcol_8`, + `bfcol_4` AS `bfcol_9` + FROM `bfcte_10` +), `bfcte_0` AS ( + SELECT + `bool_col`, + `float64_col`, + `int64_too` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_4` AS ( + SELECT + * + FROM `bfcte_0` WHERE `bool_col` +), `bfcte_8` AS ( + SELECT + *, + ROW_NUMBER() OVER () - 1 AS `bfcol_15` + FROM `bfcte_4` +), `bfcte_12` AS ( + SELECT + *, + 1 AS `bfcol_16` + FROM `bfcte_8` +), `bfcte_14` AS ( + SELECT + `float64_col` AS `bfcol_17`, + `int64_too` AS `bfcol_18`, + `bfcol_16` AS `bfcol_19`, + `bfcol_15` AS `bfcol_20` + FROM `bfcte_12` ), `bfcte_1` AS ( SELECT - `float64_col` AS `bfcol_5`, - `int64_col` AS `bfcol_6` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_2` AS ( + `float64_col`, + `int64_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_5` AS ( + SELECT + *, + ROW_NUMBER() OVER (ORDER BY `int64_col` ASC NULLS LAST) - 1 AS `bfcol_25` + FROM `bfcte_1` +), `bfcte_9` AS ( + SELECT + *, + 2 AS `bfcol_26` + FROM `bfcte_5` +), `bfcte_15` AS ( + SELECT + `float64_col` AS `bfcol_27`, + `int64_col` AS `bfcol_28`, + `bfcol_26` AS `bfcol_29`, + `bfcol_25` AS `bfcol_30` + FROM `bfcte_9` +), `bfcte_0` AS ( + SELECT + `bool_col`, + `float64_col`, + `int64_too` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_3` AS ( + SELECT + * + FROM `bfcte_0` + WHERE + `bool_col` +), `bfcte_7` AS ( + SELECT + *, + ROW_NUMBER() OVER () - 1 AS `bfcol_36` + FROM `bfcte_3` +), `bfcte_11` AS ( + SELECT + *, + 3 AS `bfcol_37` + FROM `bfcte_7` +), `bfcte_16` AS ( + SELECT + `float64_col` AS `bfcol_38`, + `int64_too` AS `bfcol_39`, + `bfcol_37` AS `bfcol_40`, + `bfcol_36` AS `bfcol_41` + FROM `bfcte_11` +), `bfcte_17` AS ( SELECT - `bfcol_21` AS `bfcol_33`, - `bfcol_22` AS `bfcol_34`, - `bfcol_23` AS `bfcol_35`, - `bfcol_24` AS `bfcol_36` + * FROM ( - ( - SELECT - `bfcol_5` AS `bfcol_21`, - `bfcol_6` AS `bfcol_22`, - 0 AS `bfcol_23`, - ROW_NUMBER() OVER (ORDER BY `bfcol_6` ASC NULLS LAST) - 1 AS `bfcol_24` - FROM `bfcte_1` - ) + SELECT + `bfcol_6` AS `bfcol_42`, + `bfcol_7` AS `bfcol_43`, + `bfcol_8` AS `bfcol_44`, + `bfcol_9` AS `bfcol_45` + FROM `bfcte_13` UNION ALL - ( - SELECT - `bfcol_7` AS `bfcol_29`, - `bfcol_8` AS `bfcol_30`, - 1 AS `bfcol_31`, - ROW_NUMBER() OVER () - 1 AS `bfcol_32` - FROM `bfcte_0` - ) + SELECT + `bfcol_17` AS `bfcol_42`, + `bfcol_18` AS `bfcol_43`, + `bfcol_19` AS `bfcol_44`, + `bfcol_20` AS `bfcol_45` + FROM `bfcte_14` UNION ALL - ( - SELECT - `bfcol_5` AS `bfcol_17`, - `bfcol_6` AS `bfcol_18`, - 2 AS `bfcol_19`, - ROW_NUMBER() OVER (ORDER BY `bfcol_6` ASC NULLS LAST) - 1 AS `bfcol_20` - FROM `bfcte_1` - ) + SELECT + `bfcol_27` AS `bfcol_42`, + `bfcol_28` AS `bfcol_43`, + `bfcol_29` AS `bfcol_44`, + `bfcol_30` AS `bfcol_45` + FROM `bfcte_15` UNION ALL - ( - SELECT - `bfcol_7` AS `bfcol_25`, - `bfcol_8` AS `bfcol_26`, - 3 AS `bfcol_27`, - ROW_NUMBER() OVER () - 1 AS `bfcol_28` - FROM `bfcte_0` - ) + SELECT + `bfcol_38` AS `bfcol_42`, + `bfcol_39` AS `bfcol_43`, + `bfcol_40` AS `bfcol_44`, + `bfcol_41` AS `bfcol_45` + FROM `bfcte_16` ) ) SELECT - `bfcol_33` AS `float64_col`, - `bfcol_34` AS `int64_col` -FROM `bfcte_2` + `bfcol_42` AS `float64_col`, + `bfcol_43` AS `int64_col` +FROM `bfcte_17` ORDER BY - `bfcol_35` ASC NULLS LAST, - `bfcol_36` ASC NULLS LAST \ No newline at end of file + `bfcol_44` ASC NULLS LAST, + `bfcol_45` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_explode/test_compile_explode_dataframe/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_explode/test_compile_explode_dataframe/out.sql index e2a80e201bb..5d9019439f2 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_explode/test_compile_explode_dataframe/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_explode/test_compile_explode_dataframe/out.sql @@ -1,9 +1,9 @@ WITH `bfcte_0` AS ( SELECT - `rowindex`, `int_list_col`, + `rowindex`, `string_list_col` - FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` AS `bft_0` + FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` ), `bfcte_1` AS ( SELECT * diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_explode/test_compile_explode_series/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_explode/test_compile_explode_series/out.sql index 03ac4d0e03a..8ba4559da83 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_explode/test_compile_explode_series/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_explode/test_compile_explode_series/out.sql @@ -1,8 +1,8 @@ WITH `bfcte_0` AS ( SELECT - `rowindex`, - `int_list_col` - FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` AS `bft_0` + `int_list_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` ), `bfcte_1` AS ( SELECT * diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_filter/test_compile_filter/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_filter/test_compile_filter/out.sql index 3e367c1e1e2..f5fff16f602 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_filter/test_compile_filter/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_filter/test_compile_filter/out.sql @@ -1,7 +1,25 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_5`, + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7`, + `rowindex` >= 1 AS `bfcol_8` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + * + FROM `bfcte_1` + WHERE + `bfcol_8` +) SELECT - `rowindex`, - `rowindex` AS `rowindex_1`, - `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -WHERE - `rowindex` >= 1 \ No newline at end of file + `bfcol_5` AS `rowindex`, + `bfcol_6` AS `rowindex_1`, + `bfcol_7` AS `int64_col` +FROM `bfcte_2` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_fromrange/test_compile_fromrange/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_fromrange/test_compile_fromrange/out.sql deleted file mode 100644 index 4f4e2496498..00000000000 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_fromrange/test_compile_fromrange/out.sql +++ /dev/null @@ -1,75 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - * - FROM UNNEST(ARRAY>[STRUCT(CAST('2021-01-01T13:00:00' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:01' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:02' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:03' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:04' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:05' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:06' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:07' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:08' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:09' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:10' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:11' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:12' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:13' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:14' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:15' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:16' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:17' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:18' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:19' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:20' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:21' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:22' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:23' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:24' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:25' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:26' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:27' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:28' AS DATETIME)), STRUCT(CAST('2021-01-01T13:00:29' AS DATETIME))]) -), `bfcte_1` AS ( - SELECT - * - FROM UNNEST(ARRAY>[STRUCT(CAST('2021-01-01T13:00:00' AS DATETIME), 0, 10), STRUCT(CAST('2021-01-01T13:00:01' AS DATETIME), 1, 11), STRUCT(CAST('2021-01-01T13:00:02' AS DATETIME), 2, 12), STRUCT(CAST('2021-01-01T13:00:03' AS DATETIME), 3, 13), STRUCT(CAST('2021-01-01T13:00:04' AS DATETIME), 4, 14), STRUCT(CAST('2021-01-01T13:00:05' AS DATETIME), 5, 15), STRUCT(CAST('2021-01-01T13:00:06' AS DATETIME), 6, 16), STRUCT(CAST('2021-01-01T13:00:07' AS DATETIME), 7, 17), STRUCT(CAST('2021-01-01T13:00:08' AS DATETIME), 8, 18), STRUCT(CAST('2021-01-01T13:00:09' AS DATETIME), 9, 19), STRUCT(CAST('2021-01-01T13:00:10' AS DATETIME), 10, 20), STRUCT(CAST('2021-01-01T13:00:11' AS DATETIME), 11, 21), STRUCT(CAST('2021-01-01T13:00:12' AS DATETIME), 12, 22), STRUCT(CAST('2021-01-01T13:00:13' AS DATETIME), 13, 23), STRUCT(CAST('2021-01-01T13:00:14' AS DATETIME), 14, 24), STRUCT(CAST('2021-01-01T13:00:15' AS DATETIME), 15, 25), STRUCT(CAST('2021-01-01T13:00:16' AS DATETIME), 16, 26), STRUCT(CAST('2021-01-01T13:00:17' AS DATETIME), 17, 27), STRUCT(CAST('2021-01-01T13:00:18' AS DATETIME), 18, 28), STRUCT(CAST('2021-01-01T13:00:19' AS DATETIME), 19, 29), STRUCT(CAST('2021-01-01T13:00:20' AS DATETIME), 20, 30), STRUCT(CAST('2021-01-01T13:00:21' AS DATETIME), 21, 31), STRUCT(CAST('2021-01-01T13:00:22' AS DATETIME), 22, 32), STRUCT(CAST('2021-01-01T13:00:23' AS DATETIME), 23, 33), STRUCT(CAST('2021-01-01T13:00:24' AS DATETIME), 24, 34), STRUCT(CAST('2021-01-01T13:00:25' AS DATETIME), 25, 35), STRUCT(CAST('2021-01-01T13:00:26' AS DATETIME), 26, 36), STRUCT(CAST('2021-01-01T13:00:27' AS DATETIME), 27, 37), STRUCT(CAST('2021-01-01T13:00:28' AS DATETIME), 28, 38), STRUCT(CAST('2021-01-01T13:00:29' AS DATETIME), 29, 39)]) -), `bfcte_2` AS ( - SELECT - `bfcol_0` AS `bfcol_4` - FROM `bfcte_0` -), `bfcte_3` AS ( - SELECT - `bfcol_1` AS `bfcol_5`, - `bfcol_2` AS `bfcol_6`, - `bfcol_3` AS `bfcol_7` - FROM `bfcte_1` -), `bfcte_4` AS ( - SELECT - MIN(`bfcol_4`) AS `bfcol_8` - FROM `bfcte_2` -), `bfcte_5` AS ( - SELECT - `bfcol_6` AS `bfcol_11`, - `bfcol_7` AS `bfcol_12`, - CAST(FLOOR( - IEEE_DIVIDE( - UNIX_MICROS(CAST(`bfcol_5` AS TIMESTAMP)) - UNIX_MICROS(CAST(CAST(`bfcol_8` AS DATE) AS TIMESTAMP)), - 7000000 - ) - ) AS INT64) AS `bfcol_13` - FROM `bfcte_3` - CROSS JOIN `bfcte_4` -), `bfcte_6` AS ( - SELECT - CAST(FLOOR( - IEEE_DIVIDE( - UNIX_MICROS(CAST(`bfcol_4` AS TIMESTAMP)) - UNIX_MICROS(CAST(CAST(`bfcol_8` AS DATE) AS TIMESTAMP)), - 7000000 - ) - ) AS INT64) AS `bfcol_14` - FROM `bfcte_2` - CROSS JOIN `bfcte_4` -), `bfcte_7` AS ( - SELECT - MAX(`bfcol_14`) AS `bfcol_15` - FROM `bfcte_6` -), `bfcte_8` AS ( - SELECT - MIN(`bfcol_14`) AS `bfcol_16` - FROM `bfcte_6` -), `bfcte_9` AS ( - SELECT - `bfcol_27` AS `bfcol_17` - FROM `bfcte_8` - CROSS JOIN `bfcte_7` - CROSS JOIN UNNEST(GENERATE_ARRAY(`bfcol_16`, `bfcol_15`, 1)) AS `bfcol_27` -) -SELECT - CAST(TIMESTAMP_MICROS( - CAST(CAST(`bfcol_17` AS BIGNUMERIC) * 7000000 + CAST(UNIX_MICROS(CAST(CAST(`bfcol_8` AS DATE) AS TIMESTAMP)) AS BIGNUMERIC) AS INT64) - ) AS DATETIME) AS `timestamp_col`, - `bfcol_11` AS `int64_col`, - `bfcol_12` AS `int64_too` -FROM ( - SELECT - * - FROM `bfcte_9` - CROSS JOIN `bfcte_4` -) -LEFT JOIN `bfcte_5` - ON `bfcol_17` = `bfcol_13` -ORDER BY - `bfcol_17` ASC NULLS LAST diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_geo/test_st_regionstats/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_geo/test_st_regionstats/out.sql index 457436e98c4..63076077cf5 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_geo/test_st_regionstats/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_geo/test_st_regionstats/out.sql @@ -2,50 +2,35 @@ WITH `bfcte_0` AS ( SELECT * FROM UNNEST(ARRAY>[STRUCT('POINT(1 1)', 0)]) +), `bfcte_1` AS ( + SELECT + *, + ST_REGIONSTATS( + `bfcol_0`, + 'ee://some/raster/uri', + band => 'band1', + include => 'some equation', + options => JSON '{"scale": 100}' + ) AS `bfcol_2` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_2`.`min` AS `bfcol_5`, + `bfcol_2`.`max` AS `bfcol_6`, + `bfcol_2`.`sum` AS `bfcol_7`, + `bfcol_2`.`count` AS `bfcol_8`, + `bfcol_2`.`mean` AS `bfcol_9`, + `bfcol_2`.`area` AS `bfcol_10` + FROM `bfcte_1` ) SELECT - ST_REGIONSTATS( - `bfcol_0`, - 'ee://some/raster/uri', - band => 'band1', - include => 'some equation', - options => JSON '{"scale": 100}' - ).`min`, - ST_REGIONSTATS( - `bfcol_0`, - 'ee://some/raster/uri', - band => 'band1', - include => 'some equation', - options => JSON '{"scale": 100}' - ).`max`, - ST_REGIONSTATS( - `bfcol_0`, - 'ee://some/raster/uri', - band => 'band1', - include => 'some equation', - options => JSON '{"scale": 100}' - ).`sum`, - ST_REGIONSTATS( - `bfcol_0`, - 'ee://some/raster/uri', - band => 'band1', - include => 'some equation', - options => JSON '{"scale": 100}' - ).`count`, - ST_REGIONSTATS( - `bfcol_0`, - 'ee://some/raster/uri', - band => 'band1', - include => 'some equation', - options => JSON '{"scale": 100}' - ).`mean`, - ST_REGIONSTATS( - `bfcol_0`, - 'ee://some/raster/uri', - band => 'band1', - include => 'some equation', - options => JSON '{"scale": 100}' - ).`area` -FROM `bfcte_0` + `bfcol_5` AS `min`, + `bfcol_6` AS `max`, + `bfcol_7` AS `sum`, + `bfcol_8` AS `count`, + `bfcol_9` AS `mean`, + `bfcol_10` AS `area` +FROM `bfcte_2` ORDER BY `bfcol_1` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_geo/test_st_regionstats_without_optional_args/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_geo/test_st_regionstats_without_optional_args/out.sql index 410909d80c5..f7947119611 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_geo/test_st_regionstats_without_optional_args/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_geo/test_st_regionstats_without_optional_args/out.sql @@ -2,14 +2,29 @@ WITH `bfcte_0` AS ( SELECT * FROM UNNEST(ARRAY>[STRUCT('POINT(1 1)', 0)]) +), `bfcte_1` AS ( + SELECT + *, + ST_REGIONSTATS(`bfcol_0`, 'ee://some/raster/uri') AS `bfcol_2` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + *, + `bfcol_2`.`min` AS `bfcol_5`, + `bfcol_2`.`max` AS `bfcol_6`, + `bfcol_2`.`sum` AS `bfcol_7`, + `bfcol_2`.`count` AS `bfcol_8`, + `bfcol_2`.`mean` AS `bfcol_9`, + `bfcol_2`.`area` AS `bfcol_10` + FROM `bfcte_1` ) SELECT - ST_REGIONSTATS(`bfcol_0`, 'ee://some/raster/uri').`min`, - ST_REGIONSTATS(`bfcol_0`, 'ee://some/raster/uri').`max`, - ST_REGIONSTATS(`bfcol_0`, 'ee://some/raster/uri').`sum`, - ST_REGIONSTATS(`bfcol_0`, 'ee://some/raster/uri').`count`, - ST_REGIONSTATS(`bfcol_0`, 'ee://some/raster/uri').`mean`, - ST_REGIONSTATS(`bfcol_0`, 'ee://some/raster/uri').`area` -FROM `bfcte_0` + `bfcol_5` AS `min`, + `bfcol_6` AS `max`, + `bfcol_7` AS `sum`, + `bfcol_8` AS `count`, + `bfcol_9` AS `mean`, + `bfcol_10` AS `area` +FROM `bfcte_2` ORDER BY `bfcol_1` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_geo/test_st_simplify/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_geo/test_st_simplify/out.sql index 177cb5292b3..b8dd1587a86 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_geo/test_st_simplify/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_geo/test_st_simplify/out.sql @@ -1,10 +1,15 @@ WITH `bfcte_0` AS ( SELECT * - FROM UNNEST(ARRAY>[STRUCT(ST_GEOGFROMTEXT('LINESTRING(0 0, 1 1, 2 0)'), 0)]) + FROM UNNEST(ARRAY>[STRUCT('POINT(1 1)', 0)]) +), `bfcte_1` AS ( + SELECT + *, + ST_SIMPLIFY(`bfcol_0`, 123.125) AS `bfcol_2` + FROM `bfcte_0` ) SELECT - ST_SIMPLIFY(`bfcol_0`, 123.125) AS `0` -FROM `bfcte_0` + `bfcol_2` AS `0` +FROM `bfcte_1` ORDER BY `bfcol_1` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_isin/test_compile_isin/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_isin/test_compile_isin/out.sql index a062ec30e44..77aef6ad8bb 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_isin/test_compile_isin/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_isin/test_compile_isin/out.sql @@ -1,36 +1,41 @@ -WITH `bfcte_0` AS ( +WITH `bfcte_1` AS ( SELECT - `int64_too` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_3` AS ( + SELECT + `rowindex` AS `bfcol_2`, + `int64_col` AS `bfcol_3` + FROM `bfcte_1` +), `bfcte_0` AS ( SELECT - `rowindex` AS `bfcol_3`, - `int64_col` AS `bfcol_4` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + `int64_too` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_2` AS ( SELECT `int64_too` FROM `bfcte_0` GROUP BY `int64_too` -), `bfcte_3` AS ( - SELECT - `int64_too` AS `bfcol_0` - FROM `bfcte_2` ), `bfcte_4` AS ( SELECT - *, + `bfcte_3`.*, EXISTS( SELECT 1 - FROM `bfcte_3` + FROM ( + SELECT + `int64_too` AS `bfcol_4` + FROM `bfcte_2` + ) AS `bft_0` WHERE - COALESCE(`bfcol_4`, 0) = COALESCE(`bfcol_0`, 0) - AND COALESCE(`bfcol_4`, 1) = COALESCE(`bfcol_0`, 1) + COALESCE(`bfcte_3`.`bfcol_3`, 0) = COALESCE(`bft_0`.`bfcol_4`, 0) + AND COALESCE(`bfcte_3`.`bfcol_3`, 1) = COALESCE(`bft_0`.`bfcol_4`, 1) ) AS `bfcol_5` - FROM `bfcte_1` + FROM `bfcte_3` ) SELECT - `bfcol_3` AS `rowindex`, + `bfcol_2` AS `rowindex`, `bfcol_5` AS `int64_col` FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_isin/test_compile_isin_not_nullable/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_isin/test_compile_isin_not_nullable/out.sql index 81c83dee6c9..8089c5b462b 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_isin/test_compile_isin_not_nullable/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_isin/test_compile_isin_not_nullable/out.sql @@ -1,33 +1,34 @@ -WITH `bfcte_0` AS ( +WITH `bfcte_1` AS ( SELECT + `rowindex`, `rowindex_2` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_3` AS ( + SELECT + `rowindex` AS `bfcol_2`, + `rowindex_2` AS `bfcol_3` + FROM `bfcte_1` +), `bfcte_0` AS ( SELECT - `rowindex` AS `bfcol_3`, - `rowindex_2` AS `bfcol_4` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + `rowindex_2` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_2` AS ( SELECT `rowindex_2` FROM `bfcte_0` GROUP BY `rowindex_2` -), `bfcte_3` AS ( - SELECT - `rowindex_2` AS `bfcol_0` - FROM `bfcte_2` ), `bfcte_4` AS ( SELECT - *, - COALESCE(`bfcol_4` IN (( + `bfcte_3`.*, + `bfcte_3`.`bfcol_3` IN (( SELECT - * - FROM `bfcte_3` - )), FALSE) AS `bfcol_5` - FROM `bfcte_1` + `rowindex_2` AS `bfcol_4` + FROM `bfcte_2` + )) AS `bfcol_5` + FROM `bfcte_3` ) SELECT - `bfcol_3` AS `rowindex`, + `bfcol_2` AS `rowindex`, `bfcol_5` AS `rowindex_2` FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join/out.sql index cac57d0c8c8..3a7ff60d3ee 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join/out.sql @@ -1,18 +1,32 @@ -WITH `bfcte_0` AS ( +WITH `bfcte_1` AS ( SELECT - `int64_col` AS `bfcol_4`, - `int64_too` AS `bfcol_5` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_2` AS ( SELECT - `rowindex` AS `bfcol_6`, - `int64_col` AS `bfcol_7` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + `rowindex` AS `bfcol_2`, + `int64_col` AS `bfcol_3` + FROM `bfcte_1` +), `bfcte_0` AS ( + SELECT + `int64_col`, + `int64_too` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_3` AS ( + SELECT + `int64_col` AS `bfcol_6`, + `int64_too` AS `bfcol_7` + FROM `bfcte_0` +), `bfcte_4` AS ( + SELECT + * + FROM `bfcte_2` + LEFT JOIN `bfcte_3` + ON COALESCE(`bfcol_2`, 0) = COALESCE(`bfcol_6`, 0) + AND COALESCE(`bfcol_2`, 1) = COALESCE(`bfcol_6`, 1) ) SELECT - `bfcol_7` AS `int64_col`, - `bfcol_5` AS `int64_too` -FROM `bfcte_1` -LEFT JOIN `bfcte_0` - ON COALESCE(`bfcol_6`, 0) = COALESCE(`bfcol_4`, 0) - AND COALESCE(`bfcol_6`, 1) = COALESCE(`bfcol_4`, 1) \ No newline at end of file + `bfcol_3` AS `int64_col`, + `bfcol_7` AS `int64_too` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/bool_col/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/bool_col/out.sql index 5042f91cd95..30f363e900e 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/bool_col/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/bool_col/out.sql @@ -1,24 +1,33 @@ -WITH `bfcte_0` AS ( +WITH `bfcte_1` AS ( SELECT - `bool_col` AS `bfcol_0`, - `rowindex` AS `bfcol_1` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( - SELECT - `bfcol_1` AS `bfcol_2`, - `bfcol_0` AS `bfcol_3` - FROM `bfcte_0` + `bool_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_2` AS ( SELECT - `bfcol_1` AS `bfcol_4`, - `bfcol_0` AS `bfcol_5` + `rowindex` AS `bfcol_2`, + `bool_col` AS `bfcol_3` + FROM `bfcte_1` +), `bfcte_0` AS ( + SELECT + `bool_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_3` AS ( + SELECT + `rowindex` AS `bfcol_6`, + `bool_col` AS `bfcol_7` FROM `bfcte_0` +), `bfcte_4` AS ( + SELECT + * + FROM `bfcte_2` + INNER JOIN `bfcte_3` + ON COALESCE(CAST(`bfcol_3` AS STRING), '0') = COALESCE(CAST(`bfcol_7` AS STRING), '0') + AND COALESCE(CAST(`bfcol_3` AS STRING), '1') = COALESCE(CAST(`bfcol_7` AS STRING), '1') ) SELECT - `bfcol_4` AS `rowindex_x`, - `bfcol_5` AS `bool_col`, - `bfcol_2` AS `rowindex_y` -FROM `bfcte_2` -INNER JOIN `bfcte_1` - ON COALESCE(CAST(`bfcol_5` AS STRING), '0') = COALESCE(CAST(`bfcol_3` AS STRING), '0') - AND COALESCE(CAST(`bfcol_5` AS STRING), '1') = COALESCE(CAST(`bfcol_3` AS STRING), '1') \ No newline at end of file + `bfcol_2` AS `rowindex_x`, + `bfcol_3` AS `bool_col`, + `bfcol_6` AS `rowindex_y` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/float64_col/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/float64_col/out.sql index 544fedadc5b..9fa7673fb31 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/float64_col/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/float64_col/out.sql @@ -1,24 +1,33 @@ -WITH `bfcte_0` AS ( +WITH `bfcte_1` AS ( SELECT - `float64_col` AS `bfcol_0`, - `rowindex` AS `bfcol_1` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( - SELECT - `bfcol_1` AS `bfcol_2`, - `bfcol_0` AS `bfcol_3` - FROM `bfcte_0` + `float64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_2` AS ( SELECT - `bfcol_1` AS `bfcol_4`, - `bfcol_0` AS `bfcol_5` + `rowindex` AS `bfcol_2`, + `float64_col` AS `bfcol_3` + FROM `bfcte_1` +), `bfcte_0` AS ( + SELECT + `float64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_3` AS ( + SELECT + `rowindex` AS `bfcol_6`, + `float64_col` AS `bfcol_7` FROM `bfcte_0` +), `bfcte_4` AS ( + SELECT + * + FROM `bfcte_2` + INNER JOIN `bfcte_3` + ON IF(IS_NAN(`bfcol_3`), 2, COALESCE(`bfcol_3`, 0)) = IF(IS_NAN(`bfcol_7`), 2, COALESCE(`bfcol_7`, 0)) + AND IF(IS_NAN(`bfcol_3`), 3, COALESCE(`bfcol_3`, 1)) = IF(IS_NAN(`bfcol_7`), 3, COALESCE(`bfcol_7`, 1)) ) SELECT - `bfcol_4` AS `rowindex_x`, - `bfcol_5` AS `float64_col`, - `bfcol_2` AS `rowindex_y` -FROM `bfcte_2` -INNER JOIN `bfcte_1` - ON IF(IS_NAN(`bfcol_5`), 2.0, COALESCE(`bfcol_5`, 0.0)) = IF(IS_NAN(`bfcol_3`), 2.0, COALESCE(`bfcol_3`, 0.0)) - AND IF(IS_NAN(`bfcol_5`), 3, COALESCE(`bfcol_5`, 1.0)) = IF(IS_NAN(`bfcol_3`), 3, COALESCE(`bfcol_3`, 1.0)) \ No newline at end of file + `bfcol_2` AS `rowindex_x`, + `bfcol_3` AS `float64_col`, + `bfcol_6` AS `rowindex_y` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/int64_col/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/int64_col/out.sql index 05b9ceec4de..c9fca069d6a 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/int64_col/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/int64_col/out.sql @@ -1,24 +1,33 @@ -WITH `bfcte_0` AS ( +WITH `bfcte_1` AS ( SELECT - `int64_col` AS `bfcol_0`, - `rowindex` AS `bfcol_1` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( - SELECT - `bfcol_1` AS `bfcol_2`, - `bfcol_0` AS `bfcol_3` - FROM `bfcte_0` + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_2` AS ( SELECT - `bfcol_1` AS `bfcol_4`, - `bfcol_0` AS `bfcol_5` + `rowindex` AS `bfcol_2`, + `int64_col` AS `bfcol_3` + FROM `bfcte_1` +), `bfcte_0` AS ( + SELECT + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_3` AS ( + SELECT + `rowindex` AS `bfcol_6`, + `int64_col` AS `bfcol_7` FROM `bfcte_0` +), `bfcte_4` AS ( + SELECT + * + FROM `bfcte_2` + INNER JOIN `bfcte_3` + ON COALESCE(`bfcol_3`, 0) = COALESCE(`bfcol_7`, 0) + AND COALESCE(`bfcol_3`, 1) = COALESCE(`bfcol_7`, 1) ) SELECT - `bfcol_4` AS `rowindex_x`, - `bfcol_5` AS `int64_col`, - `bfcol_2` AS `rowindex_y` -FROM `bfcte_2` -INNER JOIN `bfcte_1` - ON COALESCE(`bfcol_5`, 0) = COALESCE(`bfcol_3`, 0) - AND COALESCE(`bfcol_5`, 1) = COALESCE(`bfcol_3`, 1) \ No newline at end of file + `bfcol_2` AS `rowindex_x`, + `bfcol_3` AS `int64_col`, + `bfcol_6` AS `rowindex_y` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/numeric_col/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/numeric_col/out.sql index 2e0114593e7..88649c65188 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/numeric_col/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/numeric_col/out.sql @@ -1,24 +1,33 @@ -WITH `bfcte_0` AS ( +WITH `bfcte_1` AS ( SELECT - `numeric_col` AS `bfcol_0`, - `rowindex` AS `bfcol_1` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( - SELECT - `bfcol_1` AS `bfcol_2`, - `bfcol_0` AS `bfcol_3` - FROM `bfcte_0` + `numeric_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` ), `bfcte_2` AS ( SELECT - `bfcol_1` AS `bfcol_4`, - `bfcol_0` AS `bfcol_5` + `rowindex` AS `bfcol_2`, + `numeric_col` AS `bfcol_3` + FROM `bfcte_1` +), `bfcte_0` AS ( + SELECT + `numeric_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_3` AS ( + SELECT + `rowindex` AS `bfcol_6`, + `numeric_col` AS `bfcol_7` FROM `bfcte_0` +), `bfcte_4` AS ( + SELECT + * + FROM `bfcte_2` + INNER JOIN `bfcte_3` + ON COALESCE(`bfcol_3`, CAST(0 AS NUMERIC)) = COALESCE(`bfcol_7`, CAST(0 AS NUMERIC)) + AND COALESCE(`bfcol_3`, CAST(1 AS NUMERIC)) = COALESCE(`bfcol_7`, CAST(1 AS NUMERIC)) ) SELECT - `bfcol_4` AS `rowindex_x`, - `bfcol_5` AS `numeric_col`, - `bfcol_2` AS `rowindex_y` -FROM `bfcte_2` -INNER JOIN `bfcte_1` - ON COALESCE(`bfcol_5`, CAST(0 AS NUMERIC)) = COALESCE(`bfcol_3`, CAST(0 AS NUMERIC)) - AND COALESCE(`bfcol_5`, CAST(1 AS NUMERIC)) = COALESCE(`bfcol_3`, CAST(1 AS NUMERIC)) \ No newline at end of file + `bfcol_2` AS `rowindex_x`, + `bfcol_3` AS `numeric_col`, + `bfcol_6` AS `rowindex_y` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/string_col/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/string_col/out.sql index 36aad503435..8758ec8340e 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/string_col/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/string_col/out.sql @@ -1,19 +1,33 @@ -WITH `bfcte_0` AS ( +WITH `bfcte_1` AS ( + SELECT + `rowindex`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_2` AS ( SELECT `rowindex` AS `bfcol_0`, `string_col` AS `bfcol_1` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( + FROM `bfcte_1` +), `bfcte_0` AS ( + SELECT + `rowindex`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_3` AS ( SELECT - `bfcol_0` AS `bfcol_2`, - `bfcol_1` AS `bfcol_3` + `rowindex` AS `bfcol_4`, + `string_col` AS `bfcol_5` FROM `bfcte_0` +), `bfcte_4` AS ( + SELECT + * + FROM `bfcte_2` + INNER JOIN `bfcte_3` + ON COALESCE(CAST(`bfcol_1` AS STRING), '0') = COALESCE(CAST(`bfcol_5` AS STRING), '0') + AND COALESCE(CAST(`bfcol_1` AS STRING), '1') = COALESCE(CAST(`bfcol_5` AS STRING), '1') ) SELECT `bfcol_0` AS `rowindex_x`, `bfcol_1` AS `string_col`, - `bfcol_2` AS `rowindex_y` -FROM `bfcte_0` -INNER JOIN `bfcte_1` - ON COALESCE(CAST(`bfcol_1` AS STRING), '0') = COALESCE(CAST(`bfcol_3` AS STRING), '0') - AND COALESCE(CAST(`bfcol_1` AS STRING), '1') = COALESCE(CAST(`bfcol_3` AS STRING), '1') \ No newline at end of file + `bfcol_4` AS `rowindex_y` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/time_col/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/time_col/out.sql index b945a1cbf38..42fc15cd1d4 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/time_col/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_join/test_compile_join_w_on/time_col/out.sql @@ -1,19 +1,33 @@ -WITH `bfcte_0` AS ( +WITH `bfcte_1` AS ( + SELECT + `rowindex`, + `time_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_2` AS ( SELECT `rowindex` AS `bfcol_0`, `time_col` AS `bfcol_1` - FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -), `bfcte_1` AS ( + FROM `bfcte_1` +), `bfcte_0` AS ( + SELECT + `rowindex`, + `time_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_3` AS ( SELECT - `bfcol_0` AS `bfcol_2`, - `bfcol_1` AS `bfcol_3` + `rowindex` AS `bfcol_4`, + `time_col` AS `bfcol_5` FROM `bfcte_0` +), `bfcte_4` AS ( + SELECT + * + FROM `bfcte_2` + INNER JOIN `bfcte_3` + ON COALESCE(CAST(`bfcol_1` AS STRING), '0') = COALESCE(CAST(`bfcol_5` AS STRING), '0') + AND COALESCE(CAST(`bfcol_1` AS STRING), '1') = COALESCE(CAST(`bfcol_5` AS STRING), '1') ) SELECT `bfcol_0` AS `rowindex_x`, `bfcol_1` AS `time_col`, - `bfcol_2` AS `rowindex_y` -FROM `bfcte_0` -INNER JOIN `bfcte_1` - ON COALESCE(CAST(`bfcol_1` AS STRING), '0') = COALESCE(CAST(`bfcol_3` AS STRING), '0') - AND COALESCE(CAST(`bfcol_1` AS STRING), '1') = COALESCE(CAST(`bfcol_3` AS STRING), '1') \ No newline at end of file + `bfcol_4` AS `rowindex_y` +FROM `bfcte_4` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_random_sample/test_compile_random_sample/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_random_sample/test_compile_random_sample/out.sql index 73879aa65df..aae34716d86 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_random_sample/test_compile_random_sample/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_random_sample/test_compile_random_sample/out.sql @@ -1,6 +1,7 @@ WITH `bfcte_0` AS ( SELECT - * + *, + RAND() AS `bfcol_16` FROM UNNEST(ARRAY>[STRUCT( TRUE, CAST(b'Hello, World!' AS BYTES), @@ -155,6 +156,12 @@ WITH `bfcte_0` AS ( 432000000000, 8 )]) +), `bfcte_1` AS ( + SELECT + * + FROM `bfcte_0` + WHERE + `bfcol_16` < 0.1 ) SELECT `bfcol_0` AS `bool_col`, @@ -172,12 +179,6 @@ SELECT `bfcol_12` AS `time_col`, `bfcol_13` AS `timestamp_col`, `bfcol_14` AS `duration_col` -FROM ( - SELECT - * - FROM `bfcte_0` - WHERE - RAND() < 0.1 -) +FROM `bfcte_1` ORDER BY `bfcol_15` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readlocal/test_compile_readlocal_w_structs_df/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readlocal/test_compile_readlocal_w_structs_df/out.sql index 58a01635b7d..7ded9cf5fff 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readlocal/test_compile_readlocal_w_structs_df/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readlocal/test_compile_readlocal_w_structs_df/out.sql @@ -1,145 +1,27 @@ WITH `bfcte_0` AS ( SELECT * - FROM UNNEST(ARRAY>, `bfcol_2` FLOAT64, `bfcol_3` FLOAT64, `bfcol_4` FLOAT64, `bfcol_5` STRING, `bfcol_6` JSON, `bfcol_7` STRING, `bfcol_8` STRING, `bfcol_9` STRING, `bfcol_10` TIMESTAMP, `bfcol_11` STRING, `bfcol_12` FLOAT64, `bfcol_13` FLOAT64, `bfcol_14` STRING, `bfcol_15` FLOAT64, `bfcol_16` INT64>>[STRUCT( + FROM UNNEST(ARRAY>, `bfcol_2` INT64>>[STRUCT( 1, STRUCT( 'Alice' AS `name`, 30 AS `age`, STRUCT('New York' AS `city`, 'USA' AS `country`) AS `address` ), - 1.0, - 123456789.0, - 1.25, - 'Hello World', - PARSE_JSON('{"a":1,"b":[1,2]}'), - '2026-06-24', - '12:34:56.789012', - '2026-06-24 12:34:56.789012', - CAST('2026-06-24T12:34:56.789012+00:00' AS TIMESTAMP), - 'SGVsbG8=', - 123456.789, - 123456.78901234567, - 'POINT(30 10)', - 1000.0, 0 ), STRUCT( 2, - STRUCT('' AS `name`, -1 AS `age`, STRUCT('' AS `city`, '' AS `country`) AS `address`), - 0.0, - -9.223372036854776e+18, - CAST('-Infinity' AS FLOAT64), - '', - PARSE_JSON('{}'), - '0001-01-01', - '00:00:00', - '0001-01-02 00:00:00', - CAST('0001-01-02T00:00:00+00:00' AS TIMESTAMP), - '', - -1e+29, - -1e+38, - 'POINT(0 0)', - -9223372036854776.0, - 1 - ), STRUCT( - 3, - STRUCT( - 'Very Long Name...' AS `name`, - 150 AS `age`, - STRUCT('City' AS `city`, 'Country' AS `country`) AS `address` - ), - 1.0, - 9.223372036854776e+18, - CAST('Infinity' AS FLOAT64), - 'Unicode: 🚀 Spark ✨', - PARSE_JSON('{"max":true,"nested":{"val":999}}'), - '9999-12-31', - '23:59:59.999999', - '9999-12-31 23:59:59.999999', - CAST('9999-12-31T23:59:59.999999+00:00' AS TIMESTAMP), - 'dmVyeSBsb25nIGJ5dGVzIHZhbHVl', - 1e+29, - 1e+38, - 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))', - 9223372036854776.0, - 2 - ), STRUCT( - 4, - CAST(NULL AS STRUCT>), - CAST(NULL AS FLOAT64), - CAST(NULL AS FLOAT64), - CAST(NULL AS FLOAT64), - CAST(NULL AS STRING), - CAST(NULL AS JSON), - CAST(NULL AS STRING), - CAST(NULL AS STRING), - CAST(NULL AS STRING), - CAST(NULL AS TIMESTAMP), - CAST(NULL AS STRING), - CAST(NULL AS FLOAT64), - CAST(NULL AS FLOAT64), - CAST(NULL AS STRING), - CAST(NULL AS FLOAT64), - 3 - ), STRUCT( - 5, STRUCT( 'Bob' AS `name`, - 0 AS `age`, - CAST(NULL AS STRUCT) AS `address` + 25 AS `age`, + STRUCT('London' AS `city`, 'UK' AS `country`) AS `address` ), - 0.0, - 0.0, - CAST(NULL AS FLOAT64), - 'Line 1\nLine 2\n"Quotes"', - PARSE_JSON('[1,"two",null]'), - '1970-01-01', - '12:00:00', - '1970-01-01 12:00:00', - CAST('1970-01-01T12:00:00+00:00' AS TIMESTAMP), - 'AA==', - 0.0, - 0.0, - 'LINESTRING(0 0, 1 1, 2 2)', - 0.0, - 4 - ), STRUCT( - 6, - CAST(NULL AS STRUCT>), - CAST(NULL AS FLOAT64), - CAST(NULL AS FLOAT64), - CAST(NULL AS FLOAT64), - CAST(NULL AS STRING), - CAST(NULL AS JSON), - CAST(NULL AS STRING), - CAST(NULL AS STRING), - CAST(NULL AS STRING), - CAST(NULL AS TIMESTAMP), - CAST(NULL AS STRING), - CAST(NULL AS FLOAT64), - CAST(NULL AS FLOAT64), - CAST(NULL AS STRING), - CAST(NULL AS FLOAT64), - 5 + 1 )]) ) SELECT `bfcol_0` AS `id`, - `bfcol_1` AS `person`, - `bfcol_2` AS `bool_col`, - `bfcol_3` AS `int64_col`, - `bfcol_4` AS `float64_col`, - `bfcol_5` AS `string_col`, - `bfcol_6` AS `json_col`, - `bfcol_7` AS `date_col`, - `bfcol_8` AS `time_col`, - `bfcol_9` AS `datetime_col`, - `bfcol_10` AS `timestamp_col`, - `bfcol_11` AS `bytes_col`, - `bfcol_12` AS `numeric_col`, - `bfcol_13` AS `bignumeric_col`, - `bfcol_14` AS `geography_col`, - `bfcol_15` AS `duration_col` + `bfcol_1` AS `person` FROM `bfcte_0` ORDER BY - `bfcol_16` ASC NULLS LAST \ No newline at end of file + `bfcol_2` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_astype_aliases/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_astype_aliases/out.sql deleted file mode 100644 index cd056c650fd..00000000000 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_astype_aliases/out.sql +++ /dev/null @@ -1,5 +0,0 @@ -SELECT - `rowindex`, - CAST(`timestamp_col` AS STRING) AS `timestamp_col`, - CAST(`int64_col` AS FLOAT64) AS `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable/out.sql index 626ef80d518..959a31a2a35 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable/out.sql @@ -1,3 +1,22 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `bytes_col`, + `date_col`, + `datetime_col`, + `duration_col`, + `float64_col`, + `geography_col`, + `int64_col`, + `int64_too`, + `numeric_col`, + `rowindex`, + `rowindex_2`, + `string_col`, + `time_col`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +) SELECT `rowindex`, `bool_col`, @@ -15,4 +34,4 @@ SELECT `time_col`, `timestamp_col`, `duration_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` \ No newline at end of file +FROM `bfcte_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_columns_filters/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_columns_filters/out.sql index 4d1b822245c..0d8a10c9566 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_columns_filters/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_columns_filters/out.sql @@ -1,7 +1,14 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `rowindex`, + `string_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` + WHERE + `rowindex` > 0 AND `string_col` IN ('Hello, World!') +) SELECT `rowindex`, `int64_col`, `string_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -WHERE - `rowindex` > 0 AND `string_col` IN ('Hello, World!') \ No newline at end of file +FROM `bfcte_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_json_types/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_json_types/out.sql index 054e850fd36..4b5750d7aaf 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_json_types/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_json_types/out.sql @@ -1,3 +1,10 @@ +WITH `bfcte_0` AS ( + SELECT + `json_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`json_types` +) SELECT - * -FROM `bigframes-dev`.`sqlglot_test`.`json_types` AS `bft_0` \ No newline at end of file + `rowindex`, + `json_col` +FROM `bfcte_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_limit/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_limit/out.sql index ff4f0656b12..856c7061dac 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_limit/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_limit/out.sql @@ -1,7 +1,13 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +) SELECT `rowindex`, `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` +FROM `bfcte_0` ORDER BY `rowindex` ASC NULLS LAST LIMIT 10 \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_nested_structs_types/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_nested_structs_types/out.sql index f75fa6f722c..79ae1ac9072 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_nested_structs_types/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_nested_structs_types/out.sql @@ -1,5 +1,11 @@ +WITH `bfcte_0` AS ( + SELECT + `id`, + `people` + FROM `bigframes-dev`.`sqlglot_test`.`nested_structs_types` +) SELECT `id`, `id` AS `id_1`, `people` -FROM `bigframes-dev`.`sqlglot_test`.`nested_structs_types` AS `bft_0` \ No newline at end of file +FROM `bfcte_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_ordering/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_ordering/out.sql index 7e6ddfd568f..edb8d7fbf4b 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_ordering/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_ordering/out.sql @@ -1,6 +1,12 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +) SELECT `rowindex`, `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` +FROM `bfcte_0` ORDER BY `int64_col` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_repeated_types/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_repeated_types/out.sql index 34b02b5209b..a22c845ef1c 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_repeated_types/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_repeated_types/out.sql @@ -1,3 +1,15 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_list_col`, + `date_list_col`, + `date_time_list_col`, + `float_list_col`, + `int_list_col`, + `numeric_list_col`, + `rowindex`, + `string_list_col` + FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` +) SELECT `rowindex`, `rowindex` AS `rowindex_1`, @@ -8,4 +20,4 @@ SELECT `date_time_list_col`, `numeric_list_col`, `string_list_col` -FROM `bigframes-dev`.`sqlglot_test`.`repeated_types` AS `bft_0` \ No newline at end of file +FROM `bfcte_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_system_time/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_system_time/out.sql index dcd40d78485..59c36870803 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_system_time/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_readtable/test_compile_readtable_w_system_time/out.sql @@ -1,3 +1,36 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `bytes_col`, + `date_col`, + `datetime_col`, + `duration_col`, + `float64_col`, + `geography_col`, + `int64_col`, + `int64_too`, + `numeric_col`, + `rowindex`, + `rowindex_2`, + `string_col`, + `time_col`, + `timestamp_col` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` FOR SYSTEM_TIME AS OF '2025-11-09T03:04:05.678901+00:00' +) SELECT - * -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` FOR SYSTEM_TIME AS OF '2025-11-09T03:04:05.678901+00:00' \ No newline at end of file + `bool_col`, + `bytes_col`, + `date_col`, + `datetime_col`, + `geography_col`, + `int64_col`, + `int64_too`, + `numeric_col`, + `float64_col`, + `rowindex`, + `rowindex_2`, + `string_col`, + `time_col`, + `timestamp_col`, + `duration_col` +FROM `bfcte_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_w_groupby_rolling/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_w_groupby_rolling/out.sql index 1051a0fb4c1..0dca6d9d49e 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_w_groupby_rolling/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_w_groupby_rolling/out.sql @@ -1,55 +1,76 @@ +WITH `bfcte_0` AS ( + SELECT + `bool_col`, + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + `rowindex` AS `bfcol_6`, + `bool_col` AS `bfcol_7`, + `int64_col` AS `bfcol_8`, + `bool_col` AS `bfcol_9` + FROM `bfcte_0` +), `bfcte_2` AS ( + SELECT + * + FROM `bfcte_1` + WHERE + NOT `bfcol_9` IS NULL +), `bfcte_3` AS ( + SELECT + *, + CASE + WHEN COALESCE( + SUM(CAST(NOT `bfcol_7` IS NULL AS INT64)) OVER ( + PARTITION BY `bfcol_9` + ORDER BY `bfcol_9` ASC NULLS LAST, `rowindex` ASC NULLS LAST + ROWS BETWEEN 3 PRECEDING AND CURRENT ROW + ), + 0 + ) < 3 + THEN NULL + ELSE COALESCE( + SUM(CAST(`bfcol_7` AS INT64)) OVER ( + PARTITION BY `bfcol_9` + ORDER BY `bfcol_9` ASC NULLS LAST, `rowindex` ASC NULLS LAST + ROWS BETWEEN 3 PRECEDING AND CURRENT ROW + ), + 0 + ) + END AS `bfcol_15` + FROM `bfcte_2` +), `bfcte_4` AS ( + SELECT + *, + CASE + WHEN COALESCE( + SUM(CAST(NOT `bfcol_8` IS NULL AS INT64)) OVER ( + PARTITION BY `bfcol_9` + ORDER BY `bfcol_9` ASC NULLS LAST, `rowindex` ASC NULLS LAST + ROWS BETWEEN 3 PRECEDING AND CURRENT ROW + ), + 0 + ) < 3 + THEN NULL + ELSE COALESCE( + SUM(`bfcol_8`) OVER ( + PARTITION BY `bfcol_9` + ORDER BY `bfcol_9` ASC NULLS LAST, `rowindex` ASC NULLS LAST + ROWS BETWEEN 3 PRECEDING AND CURRENT ROW + ), + 0 + ) + END AS `bfcol_16` + FROM `bfcte_3` +) SELECT - `bool_col`, - `rowindex`, - CASE - WHEN COALESCE( - SUM(CAST(( - `bool_col` - ) IS NOT NULL AS INT64)) OVER ( - PARTITION BY `bool_col` - ORDER BY `bool_col` ASC NULLS LAST, `rowindex` ASC NULLS LAST - ROWS BETWEEN 3 PRECEDING AND CURRENT ROW - ), - 0 - ) < 3 - THEN NULL - WHEN TRUE - THEN COALESCE( - SUM(CAST(`bool_col` AS INT64)) OVER ( - PARTITION BY `bool_col` - ORDER BY `bool_col` ASC NULLS LAST, `rowindex` ASC NULLS LAST - ROWS BETWEEN 3 PRECEDING AND CURRENT ROW - ), - 0 - ) - END AS `bool_col_1`, - CASE - WHEN COALESCE( - SUM(CAST(( - `int64_col` - ) IS NOT NULL AS INT64)) OVER ( - PARTITION BY `bool_col` - ORDER BY `bool_col` ASC NULLS LAST, `rowindex` ASC NULLS LAST - ROWS BETWEEN 3 PRECEDING AND CURRENT ROW - ), - 0 - ) < 3 - THEN NULL - WHEN TRUE - THEN COALESCE( - SUM(`int64_col`) OVER ( - PARTITION BY `bool_col` - ORDER BY `bool_col` ASC NULLS LAST, `rowindex` ASC NULLS LAST - ROWS BETWEEN 3 PRECEDING AND CURRENT ROW - ), - 0 - ) - END AS `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` -WHERE - ( - `bool_col` - ) IS NOT NULL + `bfcol_9` AS `bool_col`, + `bfcol_6` AS `rowindex`, + `bfcol_15` AS `bool_col_1`, + `bfcol_16` AS `int64_col` +FROM `bfcte_4` ORDER BY - `bool_col` ASC NULLS LAST, + `bfcol_9` ASC NULLS LAST, `rowindex` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_w_range_rolling/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_w_range_rolling/out.sql index 887e7e9212d..fe4cea08cb2 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_w_range_rolling/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_w_range_rolling/out.sql @@ -2,30 +2,32 @@ WITH `bfcte_0` AS ( SELECT * FROM UNNEST(ARRAY>[STRUCT(CAST('2025-01-01T00:00:00+00:00' AS TIMESTAMP), 0, 0), STRUCT(CAST('2025-01-01T00:00:01+00:00' AS TIMESTAMP), 1, 1), STRUCT(CAST('2025-01-01T00:00:02+00:00' AS TIMESTAMP), 2, 2), STRUCT(CAST('2025-01-01T00:00:03+00:00' AS TIMESTAMP), 3, 3), STRUCT(CAST('2025-01-01T00:00:04+00:00' AS TIMESTAMP), 0, 4), STRUCT(CAST('2025-01-01T00:00:05+00:00' AS TIMESTAMP), 1, 5), STRUCT(CAST('2025-01-01T00:00:06+00:00' AS TIMESTAMP), 2, 6), STRUCT(CAST('2025-01-01T00:00:07+00:00' AS TIMESTAMP), 3, 7), STRUCT(CAST('2025-01-01T00:00:08+00:00' AS TIMESTAMP), 0, 8), STRUCT(CAST('2025-01-01T00:00:09+00:00' AS TIMESTAMP), 1, 9), STRUCT(CAST('2025-01-01T00:00:10+00:00' AS TIMESTAMP), 2, 10), STRUCT(CAST('2025-01-01T00:00:11+00:00' AS TIMESTAMP), 3, 11), STRUCT(CAST('2025-01-01T00:00:12+00:00' AS TIMESTAMP), 0, 12), STRUCT(CAST('2025-01-01T00:00:13+00:00' AS TIMESTAMP), 1, 13), STRUCT(CAST('2025-01-01T00:00:14+00:00' AS TIMESTAMP), 2, 14), STRUCT(CAST('2025-01-01T00:00:15+00:00' AS TIMESTAMP), 3, 15), STRUCT(CAST('2025-01-01T00:00:16+00:00' AS TIMESTAMP), 0, 16), STRUCT(CAST('2025-01-01T00:00:17+00:00' AS TIMESTAMP), 1, 17), STRUCT(CAST('2025-01-01T00:00:18+00:00' AS TIMESTAMP), 2, 18), STRUCT(CAST('2025-01-01T00:00:19+00:00' AS TIMESTAMP), 3, 19)]) +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN COALESCE( + SUM(CAST(NOT `bfcol_1` IS NULL AS INT64)) OVER ( + ORDER BY UNIX_MICROS(`bfcol_0`) ASC + RANGE BETWEEN 2999999 PRECEDING AND CURRENT ROW + ), + 0 + ) < 1 + THEN NULL + ELSE COALESCE( + SUM(`bfcol_1`) OVER ( + ORDER BY UNIX_MICROS(`bfcol_0`) ASC + RANGE BETWEEN 2999999 PRECEDING AND CURRENT ROW + ), + 0 + ) + END AS `bfcol_6` + FROM `bfcte_0` ) SELECT `bfcol_0` AS `ts_col`, - CASE - WHEN COALESCE( - SUM(CAST(( - `bfcol_1` - ) IS NOT NULL AS INT64)) OVER ( - ORDER BY UNIX_MICROS(`bfcol_0`) ASC - RANGE BETWEEN 2999999 PRECEDING AND CURRENT ROW - ), - 0 - ) < 1 - THEN NULL - WHEN TRUE - THEN COALESCE( - SUM(`bfcol_1`) OVER ( - ORDER BY UNIX_MICROS(`bfcol_0`) ASC - RANGE BETWEEN 2999999 PRECEDING AND CURRENT ROW - ), - 0 - ) - END AS `int_col` -FROM `bfcte_0` + `bfcol_6` AS `int_col` +FROM `bfcte_1` ORDER BY `bfcol_0` ASC NULLS LAST, `bfcol_2` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_w_skips_nulls_op/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_w_skips_nulls_op/out.sql index 21bb8d5f088..bf1e76c55c7 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_w_skips_nulls_op/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_w_skips_nulls_op/out.sql @@ -1,19 +1,27 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN COALESCE( + SUM(CAST(NOT `int64_col` IS NULL AS INT64)) OVER (ORDER BY `rowindex` ASC NULLS LAST ROWS BETWEEN 2 PRECEDING AND CURRENT ROW), + 0 + ) < 3 + THEN NULL + ELSE COALESCE( + SUM(`int64_col`) OVER (ORDER BY `rowindex` ASC NULLS LAST ROWS BETWEEN 2 PRECEDING AND CURRENT ROW), + 0 + ) + END AS `bfcol_4` + FROM `bfcte_0` +) SELECT `rowindex`, - CASE - WHEN COALESCE( - SUM(CAST(( - `int64_col` - ) IS NOT NULL AS INT64)) OVER (ORDER BY `rowindex` ASC NULLS LAST ROWS BETWEEN 2 PRECEDING AND CURRENT ROW), - 0 - ) < 3 - THEN NULL - WHEN TRUE - THEN COALESCE( - SUM(`int64_col`) OVER (ORDER BY `rowindex` ASC NULLS LAST ROWS BETWEEN 2 PRECEDING AND CURRENT ROW), - 0 - ) - END AS `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + `bfcol_4` AS `int64_col` +FROM `bfcte_1` ORDER BY `rowindex` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_wo_skips_nulls_op/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_wo_skips_nulls_op/out.sql index 6ae1fffab7a..5ad435ddbb7 100644 --- a/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_wo_skips_nulls_op/out.sql +++ b/tests/unit/core/compile/sqlglot/snapshots/test_compile_window/test_compile_window_wo_skips_nulls_op/out.sql @@ -1,13 +1,21 @@ +WITH `bfcte_0` AS ( + SELECT + `int64_col`, + `rowindex` + FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` +), `bfcte_1` AS ( + SELECT + *, + CASE + WHEN COUNT(CAST(NOT `int64_col` IS NULL AS INT64)) OVER (ORDER BY `rowindex` ASC NULLS LAST ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) < 5 + THEN NULL + ELSE COUNT(`int64_col`) OVER (ORDER BY `rowindex` ASC NULLS LAST ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) + END AS `bfcol_4` + FROM `bfcte_0` +) SELECT `rowindex`, - CASE - WHEN COUNT(( - `int64_col` - ) IS NOT NULL) OVER (ORDER BY `rowindex` ASC NULLS LAST ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) < 5 - THEN NULL - WHEN TRUE - THEN COUNT(`int64_col`) OVER (ORDER BY `rowindex` ASC NULLS LAST ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) - END AS `int64_col` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` + `bfcol_4` AS `int64_col` +FROM `bfcte_1` ORDER BY `rowindex` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_dataframe_accessor/test_bigframes_sql_scalar/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_dataframe_accessor/test_bigframes_sql_scalar/out.sql deleted file mode 100644 index 80b3137b0b5..00000000000 --- a/tests/unit/core/compile/sqlglot/snapshots/test_dataframe_accessor/test_bigframes_sql_scalar/out.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT - `rowindex`, - ROUND(`int64_col` + `int64_too`) AS `0` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` diff --git a/tests/unit/core/compile/sqlglot/snapshots/test_dataframe_accessor/test_sql_scalar/out.sql b/tests/unit/core/compile/sqlglot/snapshots/test_dataframe_accessor/test_sql_scalar/out.sql deleted file mode 100644 index 80b3137b0b5..00000000000 --- a/tests/unit/core/compile/sqlglot/snapshots/test_dataframe_accessor/test_sql_scalar/out.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT - `rowindex`, - ROUND(`int64_col` + `int64_too`) AS `0` -FROM `bigframes-dev`.`sqlglot_test`.`scalar_types` AS `bft_0` diff --git a/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_create_external_table/out.sql b/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_create_external_table/out.sql deleted file mode 100644 index 867282de0e7..00000000000 --- a/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_create_external_table/out.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE EXTERNAL TABLE `my-project.my_dataset.my_table` ( - `col1` INT64, - `col2` STRING -) OPTIONS ( - format='CSV', - uris=['gs://bucket/path*'] -) \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_create_external_table_all_options/out.sql b/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_create_external_table_all_options/out.sql deleted file mode 100644 index a08ddf5ee5d..00000000000 --- a/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_create_external_table_all_options/out.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE OR REPLACE EXTERNAL TABLE `my-project.my_dataset.my_table` ( - `col1` INT64, - `col2` STRING -) WITH CONNECTION `my-connection` WITH PARTITION COLUMNS ( - `part1` DATE, - `part2` STRING -) OPTIONS ( - format='CSV', - uris=['gs://bucket/path*'] -) \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_create_external_table_if_not_exists/out.sql b/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_create_external_table_if_not_exists/out.sql deleted file mode 100644 index e05a553317b..00000000000 --- a/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_create_external_table_if_not_exists/out.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE EXTERNAL TABLE IF NOT EXISTS `my-project.my_dataset.my_table` ( - `col1` INT64 -) OPTIONS ( - format='CSV', - uris=['gs://bucket/path*'] -) \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_load_data_all_options/out.sql b/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_load_data_all_options/out.sql deleted file mode 100644 index 781019a0680..00000000000 --- a/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_load_data_all_options/out.sql +++ /dev/null @@ -1,10 +0,0 @@ -LOAD DATA OVERWRITE INTO `my-project.my_dataset.my_table` ( - `col1` INT64, - `col2` STRING -) PARTITION BY `date_col` CLUSTER BY - `cluster_col` OPTIONS ( - description='my table' -) FROM FILES (format='CSV', uris=['gs://bucket/path*']) WITH PARTITION COLUMNS ( - `part1` DATE, - `part2` STRING -) WITH CONNECTION `my-connection` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_load_data_minimal/out.sql b/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_load_data_minimal/out.sql deleted file mode 100644 index c5f66003257..00000000000 --- a/tests/unit/core/compile/sqlglot/sql/snapshots/test_ddl/test_load_data_minimal/out.sql +++ /dev/null @@ -1 +0,0 @@ -LOAD DATA INTO `my-project.my_dataset.my_table` FROM FILES (format='CSV', uris=['gs://bucket/path*']) \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/sql/snapshots/test_dml/test_insert_from_select/out.sql b/tests/unit/core/compile/sqlglot/sql/snapshots/test_dml/test_insert_from_select/out.sql deleted file mode 100644 index e2e9225c9f7..00000000000 --- a/tests/unit/core/compile/sqlglot/sql/snapshots/test_dml/test_insert_from_select/out.sql +++ /dev/null @@ -1,6 +0,0 @@ -INSERT INTO `bigframes-dev`.`sqlglot_test`.`dest_table` -( - SELECT - * - FROM `source_table` -) \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/sql/snapshots/test_dml/test_insert_from_table/out.sql b/tests/unit/core/compile/sqlglot/sql/snapshots/test_dml/test_insert_from_table/out.sql deleted file mode 100644 index 2486d8d0a3b..00000000000 --- a/tests/unit/core/compile/sqlglot/sql/snapshots/test_dml/test_insert_from_table/out.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT INTO `bigframes-dev`.`sqlglot_test`.`dest_table` -`source_table` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/sql/snapshots/test_dml/test_replace_from_select/out.sql b/tests/unit/core/compile/sqlglot/sql/snapshots/test_dml/test_replace_from_select/out.sql deleted file mode 100644 index c4f43f390ed..00000000000 --- a/tests/unit/core/compile/sqlglot/sql/snapshots/test_dml/test_replace_from_select/out.sql +++ /dev/null @@ -1,9 +0,0 @@ -MERGE INTO `bigframes-dev`.`sqlglot_test`.`dest_table` -USING ( - SELECT - * - FROM `source_table` -) -ON FALSE -WHEN NOT MATCHED BY SOURCE THEN DELETE -WHEN NOT MATCHED THEN INSERT ROW \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/sql/snapshots/test_dml/test_replace_from_table/out.sql b/tests/unit/core/compile/sqlglot/sql/snapshots/test_dml/test_replace_from_table/out.sql deleted file mode 100644 index bfc1532ca2d..00000000000 --- a/tests/unit/core/compile/sqlglot/sql/snapshots/test_dml/test_replace_from_table/out.sql +++ /dev/null @@ -1,5 +0,0 @@ -MERGE INTO `bigframes-dev`.`sqlglot_test`.`dest_table` -USING `source_table` -ON FALSE -WHEN NOT MATCHED BY SOURCE THEN DELETE -WHEN NOT MATCHED THEN INSERT ROW \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/sql/test_base.py b/tests/unit/core/compile/sqlglot/sql/test_base.py deleted file mode 100644 index 617f3636d40..00000000000 --- a/tests/unit/core/compile/sqlglot/sql/test_base.py +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import decimal -import re - -import numpy as np -import pandas as pd -import pyarrow as pa -import pytest -import shapely.geometry # type: ignore - -import bigframes.core.compile.sqlglot.sql.base as sql - - -@pytest.mark.parametrize( - ("value", "expected_pattern"), - ( - pytest.param(None, "NULL", id="null"), - pytest.param(True, "TRUE", id="true"), - pytest.param(False, "FALSE", id="false"), - pytest.param(123, "123", id="int"), - pytest.param(123.75, "123.75", id="float"), - pytest.param("abc", "'abc'", id="string"), - pytest.param( - b"\x01\x02\x03ABC", "CAST(b'\\x01\\x02\\x03ABC' AS BYTES)", id="bytes" - ), - pytest.param( - decimal.Decimal("123.75"), "CAST(123.75 AS NUMERIC)", id="decimal" - ), - pytest.param( - datetime.date(2025, 1, 1), "CAST('2025-01-01' AS DATE)", id="date" - ), - pytest.param( - datetime.datetime(2025, 1, 2, 3, 45, 6, 789123), - "CAST('2025-01-02T03:45:06.789123' AS DATETIME)", - id="datetime", - ), - pytest.param( - datetime.time(12, 34, 56, 789123), - "CAST('12:34:56.789123' AS TIME)", - id="time", - ), - pytest.param( - datetime.datetime( - 2025, 1, 2, 3, 45, 6, 789123, tzinfo=datetime.timezone.utc - ), - "CAST('2025-01-02T03:45:06.789123+00:00' AS TIMESTAMP)", - id="timestamp", - ), - pytest.param(np.int64(123), "123", id="np_int64"), - pytest.param(np.float64(123.75), "123.75", id="np_float64"), - pytest.param(float("inf"), "CAST('Infinity' AS FLOAT64)", id="inf"), - pytest.param(float("-inf"), "CAST('-Infinity' AS FLOAT64)", id="neg_inf"), - pytest.param(float("nan"), "NULL", id="nan"), - pytest.param(pd.NA, "NULL", id="pd_na"), - pytest.param(datetime.timedelta(seconds=1), "1000000", id="timedelta"), - pytest.param("POINT (0 1)", "'POINT (0 1)'", id="string_geo"), - ), -) -def test_literal(value, expected_pattern): - got = sql.to_sql(sql.literal(value)) - assert got == expected_pattern - - -def test_literal_for_geo(): - value = shapely.geometry.Point(0, 1) - expected_pattern = r"ST_GEOGFROMTEXT\('POINT \(0[.]?0* 1[.]?0*\)'\)" - got = sql.to_sql(sql.literal(value)) - assert re.match(expected_pattern, got) is not None - - -@pytest.mark.parametrize( - ("value", "dtype", "expected"), - ( - pytest.param( - decimal.Decimal("1.23"), - sql.dtypes.BIGNUMERIC_DTYPE, - "CAST(1.23 AS BIGNUMERIC)", - id="bignumeric", - ), - pytest.param( - [], - pd.ArrowDtype(pa.list_(pa.int64())), - "ARRAY[]", - id="empty_array", - ), - pytest.param( - {"a": 1, "b": "hello"}, - pd.ArrowDtype(pa.struct([("a", pa.int64()), ("b", pa.string())])), - "STRUCT(1 AS `a`, 'hello' AS `b`)", - id="struct", - ), - pytest.param( - float("nan"), - sql.dtypes.FLOAT_DTYPE, - "CAST('NaN' AS FLOAT64)", - id="explicit_nan", - ), - pytest.param( - pa.scalar(123, type=pa.int64()), - None, - "123", - id="pa_scalar_int", - ), - pytest.param( - pa.scalar(None, type=pa.int64()), - None, - "CAST(NULL AS INT64)", - id="pa_scalar_null", - ), - pytest.param( - {"a": 10}, - sql.dtypes.JSON_DTYPE, - "PARSE_JSON('{\\'a\\': 10}')", - id="json", - ), - ), -) -def test_literal_explicit_dtype(value, dtype, expected): - got = sql.to_sql(sql.literal(value, dtype=dtype)) - assert got == expected - - -@pytest.mark.parametrize( - ("value", "expected"), - ( - pytest.param([True, False], "[TRUE, FALSE]", id="bool"), - pytest.param([123, 456], "[123, 456]", id="int"), - pytest.param( - [123.75, 456.78, float("nan"), float("inf"), float("-inf")], - "[\n 123.75,\n 456.78,\n CAST('NaN' AS FLOAT64),\n CAST('Infinity' AS FLOAT64),\n CAST('-Infinity' AS FLOAT64)\n]", - id="float", - ), - pytest.param( - [b"\x01\x02\x03ABC", b"\x01\x02\x03ABC"], - "[CAST(b'\\x01\\x02\\x03ABC' AS BYTES), CAST(b'\\x01\\x02\\x03ABC' AS BYTES)]", - id="bytes", - ), - pytest.param( - [datetime.date(2025, 1, 1), datetime.date(2025, 1, 1)], - "[CAST('2025-01-01' AS DATE), CAST('2025-01-01' AS DATE)]", - id="date", - ), - ), -) -def test_literal_for_list(value: list, expected: str): - got = sql.to_sql(sql.literal(value)) - assert got == expected - - -def test_literal_null_type(): - import unittest.mock as mock - - mock_dtype = mock.Mock() - with mock.patch( - "bigframes.core.compile.sqlglot.sql.base.sgt.from_bigframes_dtype", - return_value="NULL", - ): - got = sql.to_sql(sql.literal(None, dtype=mock_dtype)) - assert got == "NULL" diff --git a/tests/unit/core/compile/sqlglot/sql/test_ddl.py b/tests/unit/core/compile/sqlglot/sql/test_ddl.py deleted file mode 100644 index 48080cd6b9c..00000000000 --- a/tests/unit/core/compile/sqlglot/sql/test_ddl.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from unittest import mock - -import pytest - -import bigframes.bigquery -import bigframes.core.compile.sqlglot.sql as sql -import bigframes.session - -pytest.importorskip("pytest_snapshot") - - -@pytest.fixture -def mock_session(): - return mock.create_autospec(spec=bigframes.session.Session) - - -def test_load_data_minimal(snapshot): - expr = sql.load_data( - "my-project.my_dataset.my_table", - from_files_options={"format": "CSV", "uris": ["gs://bucket/path*"]}, - ) - snapshot.assert_match(sql.to_sql(expr), "out.sql") - - -def test_load_data_all_options(snapshot): - expr = sql.load_data( - "my-project.my_dataset.my_table", - write_disposition="OVERWRITE", - columns={"col1": "INT64", "col2": "STRING"}, - partition_by=["date_col"], - cluster_by=["cluster_col"], - table_options={"description": "my table"}, - from_files_options={"format": "CSV", "uris": ["gs://bucket/path*"]}, - with_partition_columns={"part1": "DATE", "part2": "STRING"}, - connection_name="my-connection", - ) - snapshot.assert_match(sql.to_sql(expr), "out.sql") - - -@mock.patch("bigframes.bigquery._operations.table._get_table_metadata") -def test_create_external_table(get_table_metadata_mock, mock_session, snapshot): - bigframes.bigquery.create_external_table( - "my-project.my_dataset.my_table", - columns={"col1": "INT64", "col2": "STRING"}, - options={"format": "CSV", "uris": ["gs://bucket/path*"]}, - session=mock_session, - ) - mock_session.read_gbq_query.assert_called_once() - generated_sql = mock_session.read_gbq_query.call_args[0][0] - snapshot.assert_match(generated_sql, "out.sql") - get_table_metadata_mock.assert_called_once() - - -def test_create_external_table_all_options(snapshot): - expr = sql.create_external_table( - "my-project.my_dataset.my_table", - replace=True, - columns={"col1": "INT64", "col2": "STRING"}, - partition_columns={"part1": "DATE", "part2": "STRING"}, - connection_name="my-connection", - options={"format": "CSV", "uris": ["gs://bucket/path*"]}, - ) - snapshot.assert_match(sql.to_sql(expr), "out.sql") - - -def test_create_external_table_if_not_exists(snapshot): - expr = sql.create_external_table( - "my-project.my_dataset.my_table", - if_not_exists=True, - columns={"col1": "INT64"}, - options={"format": "CSV", "uris": ["gs://bucket/path*"]}, - ) - snapshot.assert_match(sql.to_sql(expr), "out.sql") diff --git a/tests/unit/core/compile/sqlglot/sql/test_dml.py b/tests/unit/core/compile/sqlglot/sql/test_dml.py deleted file mode 100644 index 99f10892d90..00000000000 --- a/tests/unit/core/compile/sqlglot/sql/test_dml.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import bigframes_vendored.sqlglot.expressions as sge -import pytest -from google.cloud import bigquery - -from bigframes.core.compile.sqlglot.sql import base, dml - -pytest.importorskip("pytest_snapshot") - - -def test_insert_from_select(snapshot): - query = sge.select("*").from_( - sge.Table(this=sge.Identifier(this="source_table", quoted=True)) - ) - destination = bigquery.TableReference.from_string( - "bigframes-dev.sqlglot_test.dest_table" - ) - - expr = dml.insert(query, destination) - sql = base.to_sql(expr) - - snapshot.assert_match(sql, "out.sql") - - -def test_insert_from_table(snapshot): - query = sge.Table(this=sge.Identifier(this="source_table", quoted=True)) - destination = bigquery.TableReference.from_string( - "bigframes-dev.sqlglot_test.dest_table" - ) - - expr = dml.insert(query, destination) - sql = base.to_sql(expr) - - snapshot.assert_match(sql, "out.sql") - - -def test_replace_from_select(snapshot): - query = sge.select("*").from_( - sge.Table(this=sge.Identifier(this="source_table", quoted=True)) - ) - destination = bigquery.TableReference.from_string( - "bigframes-dev.sqlglot_test.dest_table" - ) - - expr = dml.replace(query, destination) - sql = base.to_sql(expr) - - snapshot.assert_match(sql, "out.sql") - - -def test_replace_from_table(snapshot): - query = sge.Table(this=sge.Identifier(this="source_table", quoted=True)) - destination = bigquery.TableReference.from_string( - "bigframes-dev.sqlglot_test.dest_table" - ) - - expr = dml.replace(query, destination) - sql = base.to_sql(expr) - - snapshot.assert_match(sql, "out.sql") diff --git a/tests/unit/core/compile/sqlglot/test_compile_concat.py b/tests/unit/core/compile/sqlglot/test_compile_concat.py index d13da8ec570..c176b2e1164 100644 --- a/tests/unit/core/compile/sqlglot/test_compile_concat.py +++ b/tests/unit/core/compile/sqlglot/test_compile_concat.py @@ -14,8 +14,8 @@ import pytest -import bigframes.pandas as bpd from bigframes.core import ordering +import bigframes.pandas as bpd pytest.importorskip("pytest_snapshot") @@ -28,6 +28,7 @@ def test_compile_concat(scalar_types_df: bpd.DataFrame, snapshot): def test_compile_concat_filter_sorted(scalar_types_df: bpd.DataFrame, snapshot): + scalars_array_value = scalar_types_df._block.expr input_1 = scalars_array_value.select_columns(["float64_col", "int64_col"]).order_by( [ordering.ascending_over("int64_col")] diff --git a/tests/unit/core/compile/sqlglot/test_compile_fromrange.py b/tests/unit/core/compile/sqlglot/test_compile_fromrange.py deleted file mode 100644 index 8c25ca0310c..00000000000 --- a/tests/unit/core/compile/sqlglot/test_compile_fromrange.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pandas as pd -import pytest - -import bigframes.pandas as bpd - -pytest.importorskip("pytest_snapshot") - - -def test_compile_fromrange(compiler_session, snapshot): - data = { - "timestamp_col": pd.date_range( - start="2021-01-01 13:00:00", periods=30, freq="1s" - ), - "int64_col": range(30), - "int64_too": range(10, 40), - } - df = bpd.DataFrame(data, session=compiler_session).set_index("timestamp_col") - sql, _, _ = df.resample(rule="7s")._block.to_sql_query( - include_index=True, enable_cache=False - ) - snapshot.assert_match(sql.strip() + "\n", "out.sql") diff --git a/tests/unit/core/compile/sqlglot/test_compile_geo.py b/tests/unit/core/compile/sqlglot/test_compile_geo.py index 4aad2dfa315..50de1488e6c 100644 --- a/tests/unit/core/compile/sqlglot/test_compile_geo.py +++ b/tests/unit/core/compile/sqlglot/test_compile_geo.py @@ -13,7 +13,6 @@ # limitations under the License. import pytest -from shapely.geometry import LineString # type: ignore import bigframes.bigquery as bbq import bigframes.geopandas as gpd @@ -45,9 +44,7 @@ def test_st_regionstats_without_optional_args(compiler_session, snapshot): def test_st_simplify(compiler_session, snapshot): - geos = gpd.GeoSeries( - [LineString([(0, 0), (1, 1), (2, 0)])], session=compiler_session - ) + geos = gpd.GeoSeries(["POINT(1 1)"], session=compiler_session) result = bbq.st_simplify( geos, tolerance_meters=123.125, diff --git a/tests/unit/core/compile/sqlglot/test_compile_random_sample.py b/tests/unit/core/compile/sqlglot/test_compile_random_sample.py index 6aec633238c..486d994f877 100644 --- a/tests/unit/core/compile/sqlglot/test_compile_random_sample.py +++ b/tests/unit/core/compile/sqlglot/test_compile_random_sample.py @@ -14,9 +14,9 @@ import pytest +from bigframes.core import nodes import bigframes.core as core import bigframes.core.compile as compile -from bigframes.core import nodes pytest.importorskip("pytest_snapshot") diff --git a/tests/unit/core/compile/sqlglot/test_compile_readlocal.py b/tests/unit/core/compile/sqlglot/test_compile_readlocal.py index 03a8b39d9a0..c5fabd99e6f 100644 --- a/tests/unit/core/compile/sqlglot/test_compile_readlocal.py +++ b/tests/unit/core/compile/sqlglot/test_compile_readlocal.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys + import numpy as np import pandas as pd import pytest @@ -34,6 +36,7 @@ def test_compile_readlocal_w_structs_df( compiler_session_w_nested_structs_types: bigframes.Session, snapshot, ): + # TODO(b/427306734): Check why the output is different from the expected output. bf_df = bpd.DataFrame( nested_structs_pandas_df, session=compiler_session_w_nested_structs_types ) @@ -63,6 +66,8 @@ def test_compile_readlocal_w_json_df( def test_compile_readlocal_w_special_values( compiler_session: bigframes.Session, snapshot ): + if sys.version_info < (3, 12): + pytest.skip("Skipping test due to inconsistent SQL formatting") df = pd.DataFrame( { "col_none": [None, 1, 2], diff --git a/tests/unit/core/compile/sqlglot/test_compile_readtable.py b/tests/unit/core/compile/sqlglot/test_compile_readtable.py index 0f2058f21f6..dd776d9a8f1 100644 --- a/tests/unit/core/compile/sqlglot/test_compile_readtable.py +++ b/tests/unit/core/compile/sqlglot/test_compile_readtable.py @@ -18,7 +18,6 @@ import pytest import bigframes.pandas as bpd -from bigframes.core import bq_data pytest.importorskip("pytest_snapshot") @@ -64,7 +63,7 @@ def test_compile_readtable_w_system_time( table._properties["location"] = compiler_session._location compiler_session._loader._df_snapshot[str(table_ref)] = ( datetime.datetime(2025, 11, 9, 3, 4, 5, 678901, tzinfo=datetime.timezone.utc), - bq_data.GbqNativeTable.from_table(table), + table, ) bf_df = compiler_session.read_gbq_table(str(table_ref)) snapshot.assert_match(bf_df.sql, "out.sql") @@ -80,15 +79,3 @@ def test_compile_readtable_w_columns_filters(compiler_session, snapshot): filters=filters, ) snapshot.assert_match(bf_df.sql, "out.sql") - - -def test_compile_astype_aliases(scalar_types_df: bpd.DataFrame, snapshot): - # Test case for issue #17394 (CAST columns lose their aliases) - bf_df = scalar_types_df[["timestamp_col", "int64_col"]] - result = bf_df.astype( - { - "timestamp_col": "string[pyarrow]", - "int64_col": "Float64", - } - ) - snapshot.assert_match(result.sql + "\n", "out.sql") diff --git a/tests/unit/core/compile/sqlglot/test_dataframe_accessor.py b/tests/unit/core/compile/sqlglot/test_dataframe_accessor.py deleted file mode 100644 index e430f566497..00000000000 --- a/tests/unit/core/compile/sqlglot/test_dataframe_accessor.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest.mock as mock - -import pandas as pd -import pytest - -import bigframes.pandas as bpd -import bigframes.session - -pytest.importorskip("pytest_snapshot") - -# Only test on the latest pandas since column naming behavior is slightly -# different across versions, e.g. unnamed vs 0 for unnamed Series. -pytest.importorskip("pandas", minversion="3.0.0") - - -def test_sql_scalar(scalar_types_df: bpd.DataFrame, snapshot, monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - session.read_pandas.return_value = scalar_types_df - - def to_pandas(series, *, ordered): - assert ordered is True - sql, _, _ = series.to_frame()._to_sql_query(include_index=True) - return sql - - monkeypatch.setattr(bpd.Series, "to_pandas", to_pandas) - - df = pd.DataFrame({"int64_col": [1, 2], "int64_too": [3, 4]}) - result = df.bigquery.sql_scalar( - "ROUND({int64_col} + {int64_too})", - output_dtype=pd.Int64Dtype(), - session=session, - ) - - session.read_pandas.assert_called_once() - snapshot.assert_match(result.strip() + "\n", "out.sql") - - -def test_bigframes_sql_scalar(scalar_types_df: bpd.DataFrame, snapshot): - session = mock.create_autospec(bigframes.session.Session) - - result = scalar_types_df.bigquery.sql_scalar( - "ROUND({int64_col} + {int64_too})", - output_dtype=pd.Int64Dtype(), - session=session, - ) - - session.read_pandas.assert_not_called() - # Bigframes implementation returns a bigframes.series.Series - sql, _, _ = result.to_frame()._to_sql_query(include_index=True) - snapshot.assert_match(sql.strip() + "\n", "out.sql") diff --git a/tests/unit/core/compile/sqlglot/test_scalar_compiler.py b/tests/unit/core/compile/sqlglot/test_scalar_compiler.py index d8a59420452..3469d15d74b 100644 --- a/tests/unit/core/compile/sqlglot/test_scalar_compiler.py +++ b/tests/unit/core/compile/sqlglot/test_scalar_compiler.py @@ -17,13 +17,13 @@ import bigframes_vendored.sqlglot.expressions as sge import pytest -import bigframes.core.compile.sqlglot.expression_compiler as expression_compiler -import bigframes.operations as ops from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr +import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler +import bigframes.operations as ops def test_register_unary_op(): - compiler = expression_compiler.ExpressionCompiler() + compiler = scalar_compiler.ScalarOpCompiler() class MockUnaryOp(ops.UnaryOp): name = "mock_unary_op" @@ -43,7 +43,7 @@ def _(expr: TypedExpr) -> sge.Expression: def test_register_unary_op_pass_op(): - compiler = expression_compiler.ExpressionCompiler() + compiler = scalar_compiler.ScalarOpCompiler() class MockUnaryOp(ops.UnaryOp): name = "mock_unary_op_pass_op" @@ -63,7 +63,7 @@ def _(expr: TypedExpr, op: ops.UnaryOp) -> sge.Expression: def test_register_binary_op(): - compiler = expression_compiler.ExpressionCompiler() + compiler = scalar_compiler.ScalarOpCompiler() class MockBinaryOp(ops.BinaryOp): name = "mock_binary_op" @@ -84,7 +84,7 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: def test_register_binary_op_pass_on(): - compiler = expression_compiler.ExpressionCompiler() + compiler = scalar_compiler.ScalarOpCompiler() class MockBinaryOp(ops.BinaryOp): name = "mock_binary_op_pass_op" @@ -105,7 +105,7 @@ def _(left: TypedExpr, right: TypedExpr, op: ops.BinaryOp) -> sge.Expression: def test_register_ternary_op(): - compiler = expression_compiler.ExpressionCompiler() + compiler = scalar_compiler.ScalarOpCompiler() class MockTernaryOp(ops.TernaryOp): name = "mock_ternary_op" @@ -127,7 +127,7 @@ def _(arg1: TypedExpr, arg2: TypedExpr, arg3: TypedExpr) -> sge.Expression: def test_register_nary_op(): - compiler = expression_compiler.ExpressionCompiler() + compiler = scalar_compiler.ScalarOpCompiler() class MockNaryOp(ops.NaryOp): name = "mock_nary_op" @@ -148,7 +148,7 @@ def _(*args: TypedExpr) -> sge.Expression: def test_register_nary_op_pass_on(): - compiler = expression_compiler.ExpressionCompiler() + compiler = scalar_compiler.ScalarOpCompiler() class MockNaryOp(ops.NaryOp): name = "mock_nary_op_pass_op" @@ -171,7 +171,7 @@ def _(*args: TypedExpr, op: ops.NaryOp) -> sge.Expression: def test_binary_op_parentheses(): - compiler = expression_compiler.ExpressionCompiler() + compiler = scalar_compiler.ScalarOpCompiler() class MockAddOp(ops.BinaryOp): name = "mock_add_op" @@ -208,7 +208,7 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression: def test_register_duplicate_op_raises(): - compiler = expression_compiler.ExpressionCompiler() + compiler = scalar_compiler.ScalarOpCompiler() class MockUnaryOp(ops.UnaryOp): name = "mock_unary_op_duplicate" diff --git a/tests/unit/core/compile/sqlglot/tpch/conftest.py b/tests/unit/core/compile/sqlglot/tpch/conftest.py deleted file mode 100644 index b351b6988eb..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/conftest.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import functools -import unittest.mock as mock - -import pytest -from google.cloud import bigquery - -import bigframes.testing.mocks as mocks -from bigframes.testing import compiler_session - -freezegun = pytest.importorskip("freezegun") - -PROJECT_NAME = "bigframes-dev-perf" -DATASET_NAME = "tpch_0001t" -LOCATION_NAME = "test-region" - -TPCH_SCHEMAS = { - "LINEITEM": [ - bigquery.SchemaField("L_ORDERKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("L_PARTKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("L_SUPPKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("L_LINENUMBER", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("L_QUANTITY", "FLOAT", mode="REQUIRED"), - bigquery.SchemaField("L_EXTENDEDPRICE", "FLOAT", mode="REQUIRED"), - bigquery.SchemaField("L_DISCOUNT", "FLOAT", mode="REQUIRED"), - bigquery.SchemaField("L_TAX", "FLOAT", mode="REQUIRED"), - bigquery.SchemaField("L_RETURNFLAG", "STRING", mode="REQUIRED"), - bigquery.SchemaField("L_LINESTATUS", "STRING", mode="REQUIRED"), - bigquery.SchemaField("L_SHIPDATE", "DATE", mode="REQUIRED"), - bigquery.SchemaField("L_COMMITDATE", "DATE", mode="REQUIRED"), - bigquery.SchemaField("L_RECEIPTDATE", "DATE", mode="REQUIRED"), - bigquery.SchemaField("L_SHIPINSTRUCT", "STRING", mode="REQUIRED"), - bigquery.SchemaField("L_SHIPMODE", "STRING", mode="REQUIRED"), - bigquery.SchemaField("L_COMMENT", "STRING"), - ], - "ORDERS": [ - bigquery.SchemaField("O_ORDERKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("O_CUSTKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("O_ORDERSTATUS", "STRING", mode="REQUIRED"), - bigquery.SchemaField("O_TOTALPRICE", "FLOAT", mode="REQUIRED"), - bigquery.SchemaField("O_ORDERDATE", "DATE", mode="REQUIRED"), - bigquery.SchemaField("O_ORDERPRIORITY", "STRING", mode="REQUIRED"), - bigquery.SchemaField("O_CLERK", "STRING", mode="REQUIRED"), - bigquery.SchemaField("O_SHIPPRIORITY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("O_COMMENT", "STRING"), - ], - "PART": [ - bigquery.SchemaField("P_PARTKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("P_NAME", "STRING", mode="REQUIRED"), - bigquery.SchemaField("P_MFGR", "STRING", mode="REQUIRED"), - bigquery.SchemaField("P_BRAND", "STRING", mode="REQUIRED"), - bigquery.SchemaField("P_TYPE", "STRING", mode="REQUIRED"), - bigquery.SchemaField("P_SIZE", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("P_CONTAINER", "STRING", mode="REQUIRED"), - bigquery.SchemaField("P_RETAILPRICE", "FLOAT", mode="REQUIRED"), - bigquery.SchemaField("P_COMMENT", "STRING"), - ], - "SUPPLIER": [ - bigquery.SchemaField("S_SUPPKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("S_NAME", "STRING", mode="REQUIRED"), - bigquery.SchemaField("S_ADDRESS", "STRING", mode="REQUIRED"), - bigquery.SchemaField("S_NATIONKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("S_PHONE", "STRING", mode="REQUIRED"), - bigquery.SchemaField("S_ACCTBAL", "FLOAT", mode="REQUIRED"), - bigquery.SchemaField("S_COMMENT", "STRING"), - ], - "PARTSUPP": [ - bigquery.SchemaField("PS_PARTKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("PS_SUPPKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("PS_AVAILQTY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("PS_SUPPLYCOST", "FLOAT", mode="REQUIRED"), - bigquery.SchemaField("PS_COMMENT", "STRING"), - ], - "CUSTOMER": [ - bigquery.SchemaField("C_CUSTKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("C_NAME", "STRING", mode="REQUIRED"), - bigquery.SchemaField("C_ADDRESS", "STRING", mode="REQUIRED"), - bigquery.SchemaField("C_NATIONKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("C_PHONE", "STRING", mode="REQUIRED"), - bigquery.SchemaField("C_ACCTBAL", "FLOAT", mode="REQUIRED"), - bigquery.SchemaField("C_MKTSEGMENT", "STRING", mode="REQUIRED"), - bigquery.SchemaField("C_COMMENT", "STRING"), - ], - "NATION": [ - bigquery.SchemaField("N_NATIONKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("N_NAME", "STRING", mode="REQUIRED"), - bigquery.SchemaField("N_REGIONKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("N_COMMENT", "STRING"), - ], - "REGION": [ - bigquery.SchemaField("R_REGIONKEY", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("R_NAME", "STRING", mode="REQUIRED"), - bigquery.SchemaField("R_COMMENT", "STRING"), - ], -} - - -def _create_mock_bqclient(): - """Helper function to create a compiler session.""" - - bqclient = mock.create_autospec(bigquery.Client, instance=True) - bqclient.project = DATASET_NAME - bqclient.location = LOCATION_NAME - table_create_time = datetime.datetime.now() - - def get_table_mock(table_ref): - if isinstance(table_ref, str): - table_ref = bigquery.TableReference.from_string(table_ref) - - table_id = table_ref.table_id - schema = TPCH_SCHEMAS.get(table_id, []) - - table = mock.create_autospec(bigquery.Table, instance=True) - table._properties = {} - type(table).created = mock.PropertyMock(return_value=table_create_time) - type(table).location = mock.PropertyMock(return_value=LOCATION_NAME) - type(table).schema = mock.PropertyMock(return_value=schema) - type(table).project = table_ref.project - type(table).dataset_id = table_ref.dataset_id - type(table).table_id = table_id - type(table).num_rows = mock.PropertyMock(return_value=1000000000) - return table - - bqclient.get_table.side_effect = get_table_mock - return bqclient - - -@pytest.fixture(scope="session") -def tpch_session(): - anonymous_dataset = bigquery.DatasetReference.from_string( - f"{PROJECT_NAME}.{DATASET_NAME}" - ) - session = mocks.create_bigquery_session( - bqclient=_create_mock_bqclient(), - anonymous_dataset=anonymous_dataset, - ) - - # Disable snapshotting for TPC-H tests to keep snapshots clean - original_read_gbq_table = session._loader.read_gbq_table - - @functools.wraps(original_read_gbq_table) - def read_gbq_table_no_snapshot(*args, **kwargs): - kwargs["enable_snapshot"] = False - return original_read_gbq_table(*args, **kwargs) - - session._executor = compiler_session.SQLCompilerExecutor() - - with mock.patch.object( - session._loader, "read_gbq_table", new=read_gbq_table_no_snapshot - ): - yield session diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/1/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/1/out.sql deleted file mode 100644 index 84ed65ec174..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/1/out.sql +++ /dev/null @@ -1,75 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `L_QUANTITY`, - `L_EXTENDEDPRICE`, - `L_DISCOUNT`, - `L_TAX`, - `L_RETURNFLAG`, - `L_LINESTATUS`, - `L_SHIPDATE`, - `L_QUANTITY` AS `bfcol_7`, - `L_EXTENDEDPRICE` AS `bfcol_8`, - `L_DISCOUNT` AS `bfcol_9`, - `L_TAX` AS `bfcol_10`, - `L_RETURNFLAG` AS `bfcol_11`, - `L_LINESTATUS` AS `bfcol_12`, - `L_SHIPDATE` <= CAST('1998-09-02' AS DATE) AS `bfcol_13`, - `L_QUANTITY` AS `bfcol_27`, - `L_EXTENDEDPRICE` AS `bfcol_28`, - `L_DISCOUNT` AS `bfcol_29`, - `L_TAX` AS `bfcol_30`, - `L_RETURNFLAG` AS `bfcol_31`, - `L_LINESTATUS` AS `bfcol_32`, - `L_EXTENDEDPRICE` * ( - 1.0 - `L_DISCOUNT` - ) AS `bfcol_33`, - `L_QUANTITY` AS `bfcol_41`, - `L_EXTENDEDPRICE` AS `bfcol_42`, - `L_DISCOUNT` AS `bfcol_43`, - `L_RETURNFLAG` AS `bfcol_44`, - `L_LINESTATUS` AS `bfcol_45`, - `L_EXTENDEDPRICE` * ( - 1.0 - `L_DISCOUNT` - ) AS `bfcol_46`, - ( - `L_EXTENDEDPRICE` * ( - 1.0 - `L_DISCOUNT` - ) - ) * ( - 1.0 + `L_TAX` - ) AS `bfcol_47` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_0` - WHERE - `L_SHIPDATE` <= CAST('1998-09-02' AS DATE) -), `bfcte_1` AS ( - SELECT - `bfcol_44`, - `bfcol_45`, - COALESCE(SUM(`bfcol_41`), 0) AS `bfcol_55`, - COALESCE(SUM(`bfcol_42`), 0) AS `bfcol_56`, - COALESCE(SUM(`bfcol_46`), 0) AS `bfcol_57`, - COALESCE(SUM(`bfcol_47`), 0) AS `bfcol_58`, - AVG(`bfcol_41`) AS `bfcol_59`, - AVG(`bfcol_42`) AS `bfcol_60`, - AVG(`bfcol_43`) AS `bfcol_61`, - COUNT(`bfcol_41`) AS `bfcol_62` - FROM `bfcte_0` - GROUP BY - `bfcol_44`, - `bfcol_45` -) -SELECT - `bfcol_44` AS `L_RETURNFLAG`, - `bfcol_45` AS `L_LINESTATUS`, - `bfcol_55` AS `SUM_QTY`, - `bfcol_56` AS `SUM_BASE_PRICE`, - `bfcol_57` AS `SUM_DISC_PRICE`, - `bfcol_58` AS `SUM_CHARGE`, - `bfcol_59` AS `AVG_QTY`, - `bfcol_60` AS `AVG_PRICE`, - `bfcol_61` AS `AVG_DISC`, - `bfcol_62` AS `COUNT_ORDER` -FROM `bfcte_1` -ORDER BY - `bfcol_44` ASC NULLS LAST, - `bfcol_45` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/10/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/10/out.sql deleted file mode 100644 index 39dc2484342..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/10/out.sql +++ /dev/null @@ -1,162 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `N_NATIONKEY` AS `bfcol_0`, - `N_NAME` AS `bfcol_1` - FROM `bigframes-dev-perf`.`tpch_0001t`.`NATION` AS `bft_3` -), `bfcte_1` AS ( - SELECT - `L_ORDERKEY` AS `bfcol_2`, - `L_EXTENDEDPRICE` AS `bfcol_3`, - `L_DISCOUNT` AS `bfcol_4`, - `L_RETURNFLAG` AS `bfcol_5` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_2` -), `bfcte_2` AS ( - SELECT - `O_ORDERKEY` AS `bfcol_6`, - `O_CUSTKEY` AS `bfcol_7`, - `O_ORDERDATE` AS `bfcol_8` - FROM `bigframes-dev-perf`.`tpch_0001t`.`ORDERS` AS `bft_1` -), `bfcte_3` AS ( - SELECT - `C_CUSTKEY` AS `bfcol_9`, - `C_NAME` AS `bfcol_10`, - `C_ADDRESS` AS `bfcol_11`, - `C_NATIONKEY` AS `bfcol_12`, - `C_PHONE` AS `bfcol_13`, - `C_ACCTBAL` AS `bfcol_14`, - `C_COMMENT` AS `bfcol_15` - FROM `bigframes-dev-perf`.`tpch_0001t`.`CUSTOMER` AS `bft_0` -), `bfcte_4` AS ( - SELECT - `bfcol_9` AS `bfcol_16`, - `bfcol_10` AS `bfcol_17`, - `bfcol_11` AS `bfcol_18`, - `bfcol_12` AS `bfcol_19`, - `bfcol_13` AS `bfcol_20`, - `bfcol_14` AS `bfcol_21`, - `bfcol_15` AS `bfcol_22`, - `bfcol_6` AS `bfcol_23`, - `bfcol_8` AS `bfcol_24` - FROM `bfcte_3` - INNER JOIN `bfcte_2` - ON `bfcol_9` = `bfcol_7` -), `bfcte_5` AS ( - SELECT - `bfcol_16` AS `bfcol_25`, - `bfcol_17` AS `bfcol_26`, - `bfcol_18` AS `bfcol_27`, - `bfcol_19` AS `bfcol_28`, - `bfcol_20` AS `bfcol_29`, - `bfcol_21` AS `bfcol_30`, - `bfcol_22` AS `bfcol_31`, - `bfcol_24` AS `bfcol_32`, - `bfcol_3` AS `bfcol_33`, - `bfcol_4` AS `bfcol_34`, - `bfcol_5` AS `bfcol_35` - FROM `bfcte_4` - INNER JOIN `bfcte_1` - ON `bfcol_23` = `bfcol_2` -), `bfcte_6` AS ( - SELECT - `bfcol_25`, - `bfcol_26`, - `bfcol_27`, - `bfcol_28`, - `bfcol_29`, - `bfcol_30`, - `bfcol_31`, - `bfcol_32`, - `bfcol_33`, - `bfcol_34`, - `bfcol_35`, - `bfcol_0`, - `bfcol_1`, - `bfcol_25` AS `bfcol_47`, - `bfcol_26` AS `bfcol_48`, - `bfcol_27` AS `bfcol_49`, - `bfcol_29` AS `bfcol_50`, - `bfcol_30` AS `bfcol_51`, - `bfcol_31` AS `bfcol_52`, - `bfcol_33` AS `bfcol_53`, - `bfcol_34` AS `bfcol_54`, - `bfcol_1` AS `bfcol_55`, - ( - ( - `bfcol_32` >= CAST('1993-10-01' AS DATE) - ) - AND ( - `bfcol_32` < CAST('1994-01-01' AS DATE) - ) - ) - AND ( - `bfcol_35` = 'R' - ) AS `bfcol_56`, - `bfcol_25` AS `bfcol_76`, - `bfcol_26` AS `bfcol_77`, - `bfcol_27` AS `bfcol_78`, - `bfcol_29` AS `bfcol_79`, - `bfcol_30` AS `bfcol_80`, - `bfcol_31` AS `bfcol_81`, - `bfcol_1` AS `bfcol_82`, - ROUND(( - `bfcol_33` * ( - 1 - `bfcol_34` - ) - ), 2) AS `bfcol_83` - FROM `bfcte_5` - INNER JOIN `bfcte_0` - ON `bfcol_28` = `bfcol_0` - WHERE - ( - ( - `bfcol_32` >= CAST('1993-10-01' AS DATE) - ) - AND ( - `bfcol_32` < CAST('1994-01-01' AS DATE) - ) - ) - AND ( - `bfcol_35` = 'R' - ) -), `bfcte_7` AS ( - SELECT - `bfcol_76`, - `bfcol_77`, - `bfcol_80`, - `bfcol_79`, - `bfcol_82`, - `bfcol_78`, - `bfcol_81`, - COALESCE(SUM(`bfcol_83`), 0) AS `bfcol_92` - FROM `bfcte_6` - WHERE - NOT `bfcol_81` IS NULL - GROUP BY - `bfcol_76`, - `bfcol_77`, - `bfcol_80`, - `bfcol_79`, - `bfcol_82`, - `bfcol_78`, - `bfcol_81` -) -SELECT - `bfcol_76` AS `C_CUSTKEY`, - `bfcol_77` AS `C_NAME`, - `bfcol_92` AS `REVENUE`, - `bfcol_80` AS `C_ACCTBAL`, - `bfcol_82` AS `N_NAME`, - `bfcol_78` AS `C_ADDRESS`, - `bfcol_79` AS `C_PHONE`, - `bfcol_81` AS `C_COMMENT` -FROM `bfcte_7` -ORDER BY - `bfcol_92` DESC, - `bfcol_76` ASC NULLS LAST, - `bfcol_77` ASC NULLS LAST, - `bfcol_80` ASC NULLS LAST, - `bfcol_79` ASC NULLS LAST, - `bfcol_82` ASC NULLS LAST, - `bfcol_78` ASC NULLS LAST, - `bfcol_81` ASC NULLS LAST -LIMIT 20 \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/11/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/11/out.sql deleted file mode 100644 index 31a357be7c0..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/11/out.sql +++ /dev/null @@ -1,115 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - * - FROM UNNEST(ARRAY>[STRUCT(0.0, 0, 0)]) -), `bfcte_1` AS ( - SELECT - `PS_SUPPKEY` AS `bfcol_0`, - `PS_AVAILQTY` AS `bfcol_1`, - `PS_SUPPLYCOST` AS `bfcol_2` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PARTSUPP` AS `bft_2` -), `bfcte_2` AS ( - SELECT - `PS_PARTKEY` AS `bfcol_10`, - `PS_SUPPKEY` AS `bfcol_11`, - `PS_AVAILQTY` AS `bfcol_12`, - `PS_SUPPLYCOST` AS `bfcol_13` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PARTSUPP` AS `bft_2` -), `bfcte_3` AS ( - SELECT - `S_SUPPKEY` AS `bfcol_3`, - `S_NATIONKEY` AS `bfcol_4` - FROM `bigframes-dev-perf`.`tpch_0001t`.`SUPPLIER` AS `bft_1` -), `bfcte_4` AS ( - SELECT - `N_NATIONKEY` AS `bfcol_18` - FROM `bigframes-dev-perf`.`tpch_0001t`.`NATION` AS `bft_0` - WHERE - `N_NAME` = 'GERMANY' -), `bfcte_5` AS ( - SELECT - `bfcol_3` AS `bfcol_19` - FROM `bfcte_4` - INNER JOIN `bfcte_3` - ON `bfcol_18` = `bfcol_4` -), `bfcte_6` AS ( - SELECT - `bfcol_19`, - `bfcol_0`, - `bfcol_1`, - `bfcol_2`, - `bfcol_1` AS `bfcol_25`, - `bfcol_2` AS `bfcol_26`, - `bfcol_2` AS `bfcol_33`, - `bfcol_1` AS `bfcol_34`, - `bfcol_2` * `bfcol_1` AS `bfcol_40` - FROM `bfcte_5` - INNER JOIN `bfcte_1` - ON `bfcol_19` = `bfcol_0` -), `bfcte_7` AS ( - SELECT - `bfcol_19`, - `bfcol_10`, - `bfcol_11`, - `bfcol_12`, - `bfcol_13`, - `bfcol_10` AS `bfcol_27`, - `bfcol_13` * `bfcol_12` AS `bfcol_28` - FROM `bfcte_5` - INNER JOIN `bfcte_2` - ON `bfcol_19` = `bfcol_11` -), `bfcte_8` AS ( - SELECT - COALESCE(SUM(`bfcol_40`), 0) AS `bfcol_44` - FROM `bfcte_6` -), `bfcte_9` AS ( - SELECT - `bfcol_27`, - COALESCE(SUM(`bfcol_28`), 0) AS `bfcol_35` - FROM `bfcte_7` - GROUP BY - `bfcol_27` -), `bfcte_10` AS ( - SELECT - `bfcol_44`, - 0 AS `bfcol_45` - FROM `bfcte_8` -), `bfcte_11` AS ( - SELECT - `bfcol_27` AS `bfcol_41`, - ROUND(`bfcol_35`, 2) AS `bfcol_42` - FROM `bfcte_9` -), `bfcte_12` AS ( - SELECT - `bfcol_7`, - `bfcol_8`, - `bfcol_9`, - `bfcol_44`, - `bfcol_45`, - CASE WHEN `bfcol_9` = 0 THEN `bfcol_44` END AS `bfcol_46`, - IF(`bfcol_45` = 0, CASE WHEN `bfcol_9` = 0 THEN `bfcol_44` END, NULL) AS `bfcol_51` - FROM `bfcte_0` - CROSS JOIN `bfcte_10` -), `bfcte_13` AS ( - SELECT - `bfcol_7`, - `bfcol_8`, - ANY_VALUE(`bfcol_51`) AS `bfcol_55` - FROM `bfcte_12` - GROUP BY - `bfcol_7`, - `bfcol_8` -), `bfcte_14` AS ( - SELECT - `bfcol_55` * 0.0001 AS `bfcol_58` - FROM `bfcte_13` -) -SELECT - `bfcol_41` AS `PS_PARTKEY`, - `bfcol_42` AS `VALUE` -FROM `bfcte_11` -CROSS JOIN `bfcte_14` -WHERE - `bfcol_42` > `bfcol_58` -ORDER BY - `bfcol_42` DESC \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/12/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/12/out.sql deleted file mode 100644 index d5ab954a20b..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/12/out.sql +++ /dev/null @@ -1,90 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `L_ORDERKEY` AS `bfcol_0`, - `L_SHIPDATE` AS `bfcol_1`, - `L_COMMITDATE` AS `bfcol_2`, - `L_RECEIPTDATE` AS `bfcol_3`, - `L_SHIPMODE` AS `bfcol_4` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_1` -), `bfcte_1` AS ( - SELECT - `O_ORDERKEY` AS `bfcol_5`, - `O_ORDERPRIORITY` AS `bfcol_6` - FROM `bigframes-dev-perf`.`tpch_0001t`.`ORDERS` AS `bft_0` -), `bfcte_2` AS ( - SELECT - `bfcol_5`, - `bfcol_6`, - `bfcol_0`, - `bfcol_1`, - `bfcol_2`, - `bfcol_3`, - `bfcol_4`, - `bfcol_6` AS `bfcol_12`, - `bfcol_4` AS `bfcol_13`, - ( - ( - ( - COALESCE(COALESCE(`bfcol_4` IN ('MAIL', 'SHIP'), FALSE), FALSE) - AND ( - `bfcol_2` < `bfcol_3` - ) - ) - AND ( - `bfcol_1` < `bfcol_2` - ) - ) - AND ( - `bfcol_3` >= CAST('1994-01-01' AS DATE) - ) - ) - AND ( - `bfcol_3` < CAST('1995-01-01' AS DATE) - ) AS `bfcol_14`, - `bfcol_6` AS `bfcol_20`, - `bfcol_4` AS `bfcol_21`, - CAST(COALESCE(COALESCE(`bfcol_6` IN ('1-URGENT', '2-HIGH'), FALSE), FALSE) AS INT64) AS `bfcol_22`, - `bfcol_4` AS `bfcol_26`, - CAST(COALESCE(COALESCE(`bfcol_6` IN ('1-URGENT', '2-HIGH'), FALSE), FALSE) AS INT64) AS `bfcol_27`, - CAST(NOT ( - COALESCE(COALESCE(`bfcol_6` IN ('1-URGENT', '2-HIGH'), FALSE), FALSE) - ) AS INT64) AS `bfcol_28` - FROM `bfcte_1` - INNER JOIN `bfcte_0` - ON `bfcol_5` = `bfcol_0` - WHERE - ( - ( - ( - COALESCE(COALESCE(`bfcol_4` IN ('MAIL', 'SHIP'), FALSE), FALSE) - AND ( - `bfcol_2` < `bfcol_3` - ) - ) - AND ( - `bfcol_1` < `bfcol_2` - ) - ) - AND ( - `bfcol_3` >= CAST('1994-01-01' AS DATE) - ) - ) - AND ( - `bfcol_3` < CAST('1995-01-01' AS DATE) - ) -), `bfcte_3` AS ( - SELECT - `bfcol_26`, - COALESCE(SUM(`bfcol_27`), 0) AS `bfcol_32`, - COALESCE(SUM(`bfcol_28`), 0) AS `bfcol_33` - FROM `bfcte_2` - GROUP BY - `bfcol_26` -) -SELECT - `bfcol_26` AS `L_SHIPMODE`, - `bfcol_32` AS `HIGH_LINE_COUNT`, - `bfcol_33` AS `LOW_LINE_COUNT` -FROM `bfcte_3` -ORDER BY - `bfcol_26` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/13/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/13/out.sql deleted file mode 100644 index 6aab2b4fec7..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/13/out.sql +++ /dev/null @@ -1,39 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `O_ORDERKEY` AS `bfcol_10`, - `O_CUSTKEY` AS `bfcol_11` - FROM `bigframes-dev-perf`.`tpch_0001t`.`ORDERS` AS `bft_1` - WHERE - NOT ( - REGEXP_CONTAINS(`O_COMMENT`, 'special.*requests') - ) -), `bfcte_1` AS ( - SELECT - `C_CUSTKEY` AS `bfcol_3` - FROM `bigframes-dev-perf`.`tpch_0001t`.`CUSTOMER` AS `bft_0` -), `bfcte_2` AS ( - SELECT - `bfcol_3`, - COUNT(`bfcol_10`) AS `bfcol_14` - FROM `bfcte_1` - LEFT JOIN `bfcte_0` - ON `bfcol_3` = `bfcol_11` - GROUP BY - `bfcol_3` -), `bfcte_3` AS ( - SELECT - `bfcol_14`, - COUNT(1) AS `bfcol_16` - FROM `bfcte_2` - WHERE - NOT `bfcol_14` IS NULL - GROUP BY - `bfcol_14` -) -SELECT - `bfcol_14` AS `C_COUNT`, - `bfcol_16` AS `CUSTDIST` -FROM `bfcte_3` -ORDER BY - `bfcol_16` DESC, - `bfcol_14` DESC \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/14/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/14/out.sql deleted file mode 100644 index bde638a0f4c..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/14/out.sql +++ /dev/null @@ -1,164 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - * - FROM UNNEST(ARRAY>[STRUCT('TEMP', 0, 0)]) -), `bfcte_1` AS ( - SELECT - * - FROM UNNEST(ARRAY>[STRUCT('TEMP', 0, 0)]) -), `bfcte_2` AS ( - SELECT - `L_PARTKEY` AS `bfcol_0`, - `L_EXTENDEDPRICE` AS `bfcol_1`, - `L_DISCOUNT` AS `bfcol_2`, - `L_SHIPDATE` AS `bfcol_3` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_1` -), `bfcte_3` AS ( - SELECT - `P_PARTKEY` AS `bfcol_4` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PART` AS `bft_0` -), `bfcte_4` AS ( - SELECT - `P_PARTKEY` AS `bfcol_8`, - `P_TYPE` AS `bfcol_9` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PART` AS `bft_0` -), `bfcte_5` AS ( - SELECT - `bfcol_4`, - `bfcol_0`, - `bfcol_1`, - `bfcol_2`, - `bfcol_3`, - `bfcol_1` AS `bfcol_20`, - `bfcol_2` AS `bfcol_21`, - ( - `bfcol_3` >= CAST('1995-09-01' AS DATE) - ) - AND ( - `bfcol_3` < CAST('1995-10-01' AS DATE) - ) AS `bfcol_22`, - `bfcol_1` AS `bfcol_39`, - `bfcol_2` AS `bfcol_40`, - `bfcol_1` AS `bfcol_45`, - 1 - `bfcol_2` AS `bfcol_46`, - `bfcol_1` * ( - 1 - `bfcol_2` - ) AS `bfcol_51` - FROM `bfcte_3` - INNER JOIN `bfcte_2` - ON `bfcol_4` = `bfcol_0` - WHERE - ( - `bfcol_3` >= CAST('1995-09-01' AS DATE) - ) - AND ( - `bfcol_3` < CAST('1995-10-01' AS DATE) - ) -), `bfcte_6` AS ( - SELECT - `bfcol_8`, - `bfcol_9`, - `bfcol_0`, - `bfcol_1`, - `bfcol_2`, - `bfcol_3`, - `bfcol_9` AS `bfcol_23`, - `bfcol_1` AS `bfcol_24`, - `bfcol_2` AS `bfcol_25`, - ( - `bfcol_3` >= CAST('1995-09-01' AS DATE) - ) - AND ( - `bfcol_3` < CAST('1995-10-01' AS DATE) - ) AS `bfcol_26`, - ( - `bfcol_1` * ( - 1 - `bfcol_2` - ) - ) * CAST(REGEXP_CONTAINS(`bfcol_9`, 'PROMO') AS INT64) AS `bfcol_41` - FROM `bfcte_4` - INNER JOIN `bfcte_2` - ON `bfcol_8` = `bfcol_0` - WHERE - ( - `bfcol_3` >= CAST('1995-09-01' AS DATE) - ) - AND ( - `bfcol_3` < CAST('1995-10-01' AS DATE) - ) -), `bfcte_7` AS ( - SELECT - COALESCE(SUM(`bfcol_51`), 0) AS `bfcol_54` - FROM `bfcte_5` -), `bfcte_8` AS ( - SELECT - COALESCE(SUM(`bfcol_41`), 0) AS `bfcol_47` - FROM `bfcte_6` -), `bfcte_9` AS ( - SELECT - `bfcol_54`, - 0 AS `bfcol_59` - FROM `bfcte_7` -), `bfcte_10` AS ( - SELECT - `bfcol_47`, - 0 AS `bfcol_50` - FROM `bfcte_8` -), `bfcte_11` AS ( - SELECT - `bfcol_5`, - `bfcol_6`, - `bfcol_7`, - `bfcol_54`, - `bfcol_59`, - CASE WHEN `bfcol_7` = 0 THEN `bfcol_54` END AS `bfcol_64`, - IF(`bfcol_59` = 0, CASE WHEN `bfcol_7` = 0 THEN `bfcol_54` END, NULL) AS `bfcol_72` - FROM `bfcte_0` - CROSS JOIN `bfcte_9` -), `bfcte_12` AS ( - SELECT - `bfcol_10`, - `bfcol_11`, - `bfcol_12`, - `bfcol_47`, - `bfcol_50`, - CASE WHEN `bfcol_12` = 0 THEN `bfcol_47` END AS `bfcol_53`, - IF(`bfcol_50` = 0, CASE WHEN `bfcol_12` = 0 THEN `bfcol_47` END, NULL) AS `bfcol_60` - FROM `bfcte_1` - CROSS JOIN `bfcte_10` -), `bfcte_13` AS ( - SELECT - `bfcol_5`, - `bfcol_6`, - ANY_VALUE(`bfcol_72`) AS `bfcol_79` - FROM `bfcte_11` - GROUP BY - `bfcol_5`, - `bfcol_6` -), `bfcte_14` AS ( - SELECT - `bfcol_10`, - `bfcol_11`, - ANY_VALUE(`bfcol_60`) AS `bfcol_65` - FROM `bfcte_12` - GROUP BY - `bfcol_10`, - `bfcol_11` -), `bfcte_15` AS ( - SELECT - `bfcol_5` AS `bfcol_80`, - `bfcol_79` AS `bfcol_81` - FROM `bfcte_13` -), `bfcte_16` AS ( - SELECT - `bfcol_10` AS `bfcol_77`, - 100.0 * `bfcol_65` AS `bfcol_78` - FROM `bfcte_14` -) -SELECT - ROUND(IEEE_DIVIDE(`bfcol_78`, `bfcol_81`), 2) AS `PROMO_REVENUE` -FROM `bfcte_16` -FULL OUTER JOIN `bfcte_15` - ON `bfcol_77` = `bfcol_80` -ORDER BY - COALESCE(`bfcol_77`, `bfcol_80`) ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/15/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/15/out.sql deleted file mode 100644 index e3cc2bd9743..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/15/out.sql +++ /dev/null @@ -1,112 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - * - FROM UNNEST(ARRAY>[STRUCT('TOTAL_REVENUE', 0, 0)]) -), `bfcte_1` AS ( - SELECT - `L_SUPPKEY`, - `L_EXTENDEDPRICE`, - `L_DISCOUNT`, - `L_SHIPDATE`, - `L_SUPPKEY` AS `bfcol_12`, - `L_EXTENDEDPRICE` AS `bfcol_13`, - `L_DISCOUNT` AS `bfcol_14`, - ( - `L_SHIPDATE` >= CAST('1996-01-01' AS DATE) - ) - AND ( - `L_SHIPDATE` < CAST('1996-04-01' AS DATE) - ) AS `bfcol_15`, - `L_SUPPKEY` AS `bfcol_23`, - `L_EXTENDEDPRICE` * ( - 1 - `L_DISCOUNT` - ) AS `bfcol_24` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_1` - WHERE - ( - `L_SHIPDATE` >= CAST('1996-01-01' AS DATE) - ) - AND ( - `L_SHIPDATE` < CAST('1996-04-01' AS DATE) - ) -), `bfcte_2` AS ( - SELECT - `S_SUPPKEY` AS `bfcol_4` - FROM `bigframes-dev-perf`.`tpch_0001t`.`SUPPLIER` AS `bft_0` -), `bfcte_3` AS ( - SELECT - `S_SUPPKEY` AS `bfcol_8`, - `S_NAME` AS `bfcol_9`, - `S_ADDRESS` AS `bfcol_10`, - `S_PHONE` AS `bfcol_11` - FROM `bigframes-dev-perf`.`tpch_0001t`.`SUPPLIER` AS `bft_0` -), `bfcte_4` AS ( - SELECT - `bfcol_23`, - COALESCE(SUM(`bfcol_24`), 0) AS `bfcol_27` - FROM `bfcte_1` - GROUP BY - `bfcol_23` -), `bfcte_5` AS ( - SELECT - `bfcol_23` AS `bfcol_30`, - ROUND(`bfcol_27`, 2) AS `bfcol_31` - FROM `bfcte_4` -), `bfcte_6` AS ( - SELECT - MAX(`bfcol_31`) AS `bfcol_38` - FROM `bfcte_2` - INNER JOIN `bfcte_5` - ON `bfcol_4` = `bfcol_30` -), `bfcte_7` AS ( - SELECT - `bfcol_8` AS `bfcol_33`, - `bfcol_9` AS `bfcol_34`, - `bfcol_10` AS `bfcol_35`, - `bfcol_11` AS `bfcol_36`, - `bfcol_31` AS `bfcol_37` - FROM `bfcte_3` - INNER JOIN `bfcte_5` - ON `bfcol_8` = `bfcol_30` -), `bfcte_8` AS ( - SELECT - `bfcol_38`, - 0 AS `bfcol_39` - FROM `bfcte_6` -), `bfcte_9` AS ( - SELECT - `bfcol_5`, - `bfcol_6`, - `bfcol_7`, - `bfcol_38`, - `bfcol_39`, - CASE WHEN `bfcol_7` = 0 THEN `bfcol_38` END AS `bfcol_40`, - IF(`bfcol_39` = 0, CASE WHEN `bfcol_7` = 0 THEN `bfcol_38` END, NULL) AS `bfcol_45` - FROM `bfcte_0` - CROSS JOIN `bfcte_8` -), `bfcte_10` AS ( - SELECT - `bfcol_5`, - `bfcol_6`, - ANY_VALUE(`bfcol_45`) AS `bfcol_49` - FROM `bfcte_9` - GROUP BY - `bfcol_5`, - `bfcol_6` -), `bfcte_11` AS ( - SELECT - `bfcol_49` AS `bfcol_50` - FROM `bfcte_10` -) -SELECT - `bfcol_33` AS `S_SUPPKEY`, - `bfcol_34` AS `S_NAME`, - `bfcol_35` AS `S_ADDRESS`, - `bfcol_36` AS `S_PHONE`, - `bfcol_37` AS `TOTAL_REVENUE` -FROM `bfcte_7` -CROSS JOIN `bfcte_11` -WHERE - `bfcol_37` = `bfcol_50` -ORDER BY - `bfcol_33` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/16/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/16/out.sql deleted file mode 100644 index 228d51a76c7..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/16/out.sql +++ /dev/null @@ -1,88 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `S_SUPPKEY`, - `S_COMMENT`, - `S_SUPPKEY` AS `bfcol_8`, - NOT ( - REGEXP_CONTAINS(`S_COMMENT`, 'Customer.*Complaints') - ) AS `bfcol_9` - FROM `bigframes-dev-perf`.`tpch_0001t`.`SUPPLIER` AS `bft_2` - WHERE - NOT ( - REGEXP_CONTAINS(`S_COMMENT`, 'Customer.*Complaints') - ) -), `bfcte_1` AS ( - SELECT - `PS_PARTKEY` AS `bfcol_2`, - `PS_SUPPKEY` AS `bfcol_3` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PARTSUPP` AS `bft_1` -), `bfcte_2` AS ( - SELECT - `P_PARTKEY` AS `bfcol_4`, - `P_BRAND` AS `bfcol_5`, - `P_TYPE` AS `bfcol_6`, - `P_SIZE` AS `bfcol_7` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PART` AS `bft_0` -), `bfcte_3` AS ( - SELECT - `bfcol_8` - FROM `bfcte_0` - GROUP BY - `bfcol_8` -), `bfcte_4` AS ( - SELECT - `bfcol_5` AS `bfcol_55`, - `bfcol_6` AS `bfcol_56`, - `bfcol_7` AS `bfcol_57`, - `bfcol_3` AS `bfcol_58` - FROM `bfcte_2` - INNER JOIN `bfcte_1` - ON `bfcol_4` = `bfcol_2` - WHERE - `bfcol_5` <> 'Brand#45' - AND NOT ( - REGEXP_CONTAINS(`bfcol_6`, 'MEDIUM POLISHED') - ) - AND COALESCE(COALESCE(`bfcol_7` IN (49, 14, 23, 45, 19, 3, 36, 9), FALSE), FALSE) -), `bfcte_5` AS ( - SELECT - `bfcol_8` AS `bfcol_21` - FROM `bfcte_3` -), `bfcte_6` AS ( - SELECT - *, - COALESCE(`bfcol_58` IN (( - SELECT - * - FROM `bfcte_5` - )), FALSE) AS `bfcol_59` - FROM `bfcte_4` -), `bfcte_7` AS ( - SELECT - * - FROM `bfcte_6` - WHERE - `bfcol_59` -), `bfcte_8` AS ( - SELECT - `bfcol_55`, - `bfcol_56`, - `bfcol_57`, - COUNT(DISTINCT `bfcol_58`) AS `bfcol_69` - FROM `bfcte_7` - GROUP BY - `bfcol_55`, - `bfcol_56`, - `bfcol_57` -) -SELECT - `bfcol_55` AS `P_BRAND`, - `bfcol_56` AS `P_TYPE`, - `bfcol_57` AS `P_SIZE`, - `bfcol_69` AS `SUPPLIER_CNT` -FROM `bfcte_8` -ORDER BY - `bfcol_69` DESC, - `bfcol_55` ASC NULLS LAST, - `bfcol_56` ASC NULLS LAST, - `bfcol_57` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/17/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/17/out.sql deleted file mode 100644 index 40aacf917f1..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/17/out.sql +++ /dev/null @@ -1,97 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - * - FROM UNNEST(ARRAY>[STRUCT('L_EXTENDEDPRICE', 0, 0)]) -), `bfcte_1` AS ( - SELECT - `P_PARTKEY` AS `bfcol_15` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PART` AS `bft_1` - WHERE - ( - `P_BRAND` = 'Brand#23' - ) AND ( - `P_CONTAINER` = 'MED BOX' - ) -), `bfcte_2` AS ( - SELECT - `L_PARTKEY` AS `bfcol_3`, - `L_QUANTITY` AS `bfcol_4`, - `L_EXTENDEDPRICE` AS `bfcol_5` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_0` -), `bfcte_3` AS ( - SELECT - `L_PARTKEY` AS `bfcol_6`, - `L_QUANTITY` AS `bfcol_7` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_0` -), `bfcte_4` AS ( - SELECT - `bfcol_4` AS `bfcol_16`, - `bfcol_5` AS `bfcol_17`, - `bfcol_15` AS `bfcol_18` - FROM `bfcte_2` - RIGHT JOIN `bfcte_1` - ON `bfcol_3` = `bfcol_15` -), `bfcte_5` AS ( - SELECT - `bfcol_15`, - AVG(`bfcol_7`) AS `bfcol_21` - FROM `bfcte_3` - RIGHT JOIN `bfcte_1` - ON `bfcol_6` = `bfcol_15` - GROUP BY - `bfcol_15` -), `bfcte_6` AS ( - SELECT - `bfcol_15` AS `bfcol_24`, - `bfcol_21` * 0.2 AS `bfcol_25` - FROM `bfcte_5` -), `bfcte_7` AS ( - SELECT - `bfcol_24`, - `bfcol_25`, - `bfcol_16`, - `bfcol_17`, - `bfcol_18`, - `bfcol_17` AS `bfcol_29`, - `bfcol_16` < `bfcol_25` AS `bfcol_30` - FROM `bfcte_6` - INNER JOIN `bfcte_4` - ON `bfcol_24` = `bfcol_18` - WHERE - `bfcol_16` < `bfcol_25` -), `bfcte_8` AS ( - SELECT - COALESCE(SUM(`bfcol_29`), 0) AS `bfcol_34` - FROM `bfcte_7` -), `bfcte_9` AS ( - SELECT - `bfcol_34`, - 0 AS `bfcol_35` - FROM `bfcte_8` -), `bfcte_10` AS ( - SELECT - `bfcol_8`, - `bfcol_9`, - `bfcol_10`, - `bfcol_34`, - `bfcol_35`, - CASE WHEN `bfcol_10` = 0 THEN `bfcol_34` END AS `bfcol_36`, - IF(`bfcol_35` = 0, CASE WHEN `bfcol_10` = 0 THEN `bfcol_34` END, NULL) AS `bfcol_41` - FROM `bfcte_0` - CROSS JOIN `bfcte_9` -), `bfcte_11` AS ( - SELECT - `bfcol_8`, - `bfcol_9`, - ANY_VALUE(`bfcol_41`) AS `bfcol_45` - FROM `bfcte_10` - GROUP BY - `bfcol_8`, - `bfcol_9` -) -SELECT - ROUND(IEEE_DIVIDE(`bfcol_45`, 7.0), 2) AS `AVG_YEARLY` -FROM `bfcte_11` -ORDER BY - `bfcol_9` ASC NULLS LAST, - `bfcol_8` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/18/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/18/out.sql deleted file mode 100644 index 6fcdb343940..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/18/out.sql +++ /dev/null @@ -1,104 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `C_CUSTKEY` AS `bfcol_0`, - `C_NAME` AS `bfcol_1` - FROM `bigframes-dev-perf`.`tpch_0001t`.`CUSTOMER` AS `bft_2` -), `bfcte_1` AS ( - SELECT - `L_ORDERKEY` AS `bfcol_2`, - `L_QUANTITY` AS `bfcol_3` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_1` -), `bfcte_2` AS ( - SELECT - `O_ORDERKEY` AS `bfcol_4`, - `O_CUSTKEY` AS `bfcol_5`, - `O_TOTALPRICE` AS `bfcol_6`, - `O_ORDERDATE` AS `bfcol_7` - FROM `bigframes-dev-perf`.`tpch_0001t`.`ORDERS` AS `bft_0` -), `bfcte_3` AS ( - SELECT - `bfcol_2`, - COALESCE(SUM(`bfcol_3`), 0) AS `bfcol_8` - FROM `bfcte_1` - GROUP BY - `bfcol_2` -), `bfcte_4` AS ( - SELECT - `bfcol_2`, - `bfcol_8`, - `bfcol_2` AS `bfcol_9`, - `bfcol_8` > 300 AS `bfcol_10` - FROM `bfcte_3` - WHERE - `bfcol_8` > 300 -), `bfcte_5` AS ( - SELECT - `bfcol_9` - FROM `bfcte_4` - GROUP BY - `bfcol_9` -), `bfcte_6` AS ( - SELECT - `bfcol_9` AS `bfcol_13` - FROM `bfcte_5` -), `bfcte_7` AS ( - SELECT - *, - COALESCE(`bfcol_4` IN (( - SELECT - * - FROM `bfcte_6` - )), FALSE) AS `bfcol_14` - FROM `bfcte_2` -), `bfcte_8` AS ( - SELECT - `bfcol_4` AS `bfcol_20`, - `bfcol_5` AS `bfcol_21`, - `bfcol_6` AS `bfcol_22`, - `bfcol_7` AS `bfcol_23` - FROM `bfcte_7` - WHERE - `bfcol_14` -), `bfcte_9` AS ( - SELECT - `bfcol_20` AS `bfcol_24`, - `bfcol_21` AS `bfcol_25`, - `bfcol_22` AS `bfcol_26`, - `bfcol_23` AS `bfcol_27`, - `bfcol_3` AS `bfcol_28` - FROM `bfcte_8` - INNER JOIN `bfcte_1` - ON `bfcol_20` = `bfcol_2` -), `bfcte_10` AS ( - SELECT - `bfcol_1`, - `bfcol_0`, - `bfcol_24`, - `bfcol_27`, - `bfcol_26`, - COALESCE(SUM(`bfcol_28`), 0) AS `bfcol_35` - FROM `bfcte_9` - INNER JOIN `bfcte_0` - ON `bfcol_25` = `bfcol_0` - GROUP BY - `bfcol_1`, - `bfcol_0`, - `bfcol_24`, - `bfcol_27`, - `bfcol_26` -) -SELECT - `bfcol_1` AS `C_NAME`, - `bfcol_0` AS `C_CUSTKEY`, - `bfcol_24` AS `O_ORDERKEY`, - `bfcol_27` AS `O_ORDERDAT`, - `bfcol_26` AS `O_TOTALPRICE`, - `bfcol_35` AS `COL6` -FROM `bfcte_10` -ORDER BY - `bfcol_26` DESC, - `bfcol_27` ASC NULLS LAST, - `bfcol_1` ASC NULLS LAST, - `bfcol_0` ASC NULLS LAST, - `bfcol_24` ASC NULLS LAST -LIMIT 100 \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/19/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/19/out.sql deleted file mode 100644 index e7b817ecfd9..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/19/out.sql +++ /dev/null @@ -1,226 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - * - FROM UNNEST(ARRAY>[STRUCT(0)]) -), `bfcte_1` AS ( - SELECT - `L_PARTKEY` AS `bfcol_1`, - `L_QUANTITY` AS `bfcol_2`, - `L_EXTENDEDPRICE` AS `bfcol_3`, - `L_DISCOUNT` AS `bfcol_4`, - `L_SHIPINSTRUCT` AS `bfcol_5`, - `L_SHIPMODE` AS `bfcol_6` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_1` -), `bfcte_2` AS ( - SELECT - `P_PARTKEY` AS `bfcol_7`, - `P_BRAND` AS `bfcol_8`, - `P_SIZE` AS `bfcol_9`, - `P_CONTAINER` AS `bfcol_10` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PART` AS `bft_0` -), `bfcte_3` AS ( - SELECT - `bfcol_7`, - `bfcol_8`, - `bfcol_9`, - `bfcol_10`, - `bfcol_1`, - `bfcol_2`, - `bfcol_3`, - `bfcol_4`, - `bfcol_5`, - `bfcol_6`, - `bfcol_3` AS `bfcol_19`, - `bfcol_4` AS `bfcol_20`, - ( - COALESCE(COALESCE(`bfcol_6` IN ('AIR', 'AIR REG'), FALSE), FALSE) - AND ( - `bfcol_5` = 'DELIVER IN PERSON' - ) - ) - AND ( - ( - ( - ( - ( - ( - `bfcol_8` = 'Brand#12' - ) - AND COALESCE(COALESCE(`bfcol_10` IN ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG'), FALSE), FALSE) - ) - AND ( - ( - `bfcol_2` >= 1 - ) AND ( - `bfcol_2` <= 11 - ) - ) - ) - AND ( - ( - `bfcol_9` >= 1 - ) AND ( - `bfcol_9` <= 5 - ) - ) - ) - OR ( - ( - ( - ( - `bfcol_8` = 'Brand#23' - ) - AND COALESCE( - COALESCE(`bfcol_10` IN ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK'), FALSE), - FALSE - ) - ) - AND ( - ( - `bfcol_2` >= 10 - ) AND ( - `bfcol_2` <= 20 - ) - ) - ) - AND ( - ( - `bfcol_9` >= 1 - ) AND ( - `bfcol_9` <= 10 - ) - ) - ) - ) - OR ( - ( - ( - ( - `bfcol_8` = 'Brand#34' - ) - AND COALESCE(COALESCE(`bfcol_10` IN ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG'), FALSE), FALSE) - ) - AND ( - ( - `bfcol_2` >= 20 - ) AND ( - `bfcol_2` <= 30 - ) - ) - ) - AND ( - ( - `bfcol_9` >= 1 - ) AND ( - `bfcol_9` <= 15 - ) - ) - ) - ) AS `bfcol_21`, - `bfcol_3` AS `bfcol_27`, - 1 - `bfcol_4` AS `bfcol_28`, - `bfcol_3` * ( - 1 - `bfcol_4` - ) AS `bfcol_31` - FROM `bfcte_2` - INNER JOIN `bfcte_1` - ON `bfcol_7` = `bfcol_1` - WHERE - ( - COALESCE(COALESCE(`bfcol_6` IN ('AIR', 'AIR REG'), FALSE), FALSE) - AND ( - `bfcol_5` = 'DELIVER IN PERSON' - ) - ) - AND ( - ( - ( - ( - ( - ( - `bfcol_8` = 'Brand#12' - ) - AND COALESCE(COALESCE(`bfcol_10` IN ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG'), FALSE), FALSE) - ) - AND ( - ( - `bfcol_2` >= 1 - ) AND ( - `bfcol_2` <= 11 - ) - ) - ) - AND ( - ( - `bfcol_9` >= 1 - ) AND ( - `bfcol_9` <= 5 - ) - ) - ) - OR ( - ( - ( - ( - `bfcol_8` = 'Brand#23' - ) - AND COALESCE( - COALESCE(`bfcol_10` IN ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK'), FALSE), - FALSE - ) - ) - AND ( - ( - `bfcol_2` >= 10 - ) AND ( - `bfcol_2` <= 20 - ) - ) - ) - AND ( - ( - `bfcol_9` >= 1 - ) AND ( - `bfcol_9` <= 10 - ) - ) - ) - ) - OR ( - ( - ( - ( - `bfcol_8` = 'Brand#34' - ) - AND COALESCE(COALESCE(`bfcol_10` IN ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG'), FALSE), FALSE) - ) - AND ( - ( - `bfcol_2` >= 20 - ) AND ( - `bfcol_2` <= 30 - ) - ) - ) - AND ( - ( - `bfcol_9` >= 1 - ) AND ( - `bfcol_9` <= 15 - ) - ) - ) - ) -), `bfcte_4` AS ( - SELECT - COALESCE(SUM(`bfcol_31`), 0) AS `bfcol_33` - FROM `bfcte_3` -), `bfcte_5` AS ( - SELECT - * - FROM `bfcte_4` -) -SELECT - CASE WHEN `bfcol_0` = 0 THEN `bfcol_33` END AS `REVENUE` -FROM `bfcte_5` -CROSS JOIN `bfcte_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/2/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/2/out.sql deleted file mode 100644 index ae7be6a71da..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/2/out.sql +++ /dev/null @@ -1,197 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `R_REGIONKEY` AS `bfcol_0`, - `R_NAME` AS `bfcol_1` - FROM `bigframes-dev-perf`.`tpch_0001t`.`REGION` AS `bft_4` -), `bfcte_1` AS ( - SELECT - `N_NATIONKEY` AS `bfcol_2`, - `N_NAME` AS `bfcol_3`, - `N_REGIONKEY` AS `bfcol_4` - FROM `bigframes-dev-perf`.`tpch_0001t`.`NATION` AS `bft_3` -), `bfcte_2` AS ( - SELECT - `N_NATIONKEY` AS `bfcol_19`, - `N_REGIONKEY` AS `bfcol_20` - FROM `bigframes-dev-perf`.`tpch_0001t`.`NATION` AS `bft_3` -), `bfcte_3` AS ( - SELECT - `S_SUPPKEY` AS `bfcol_5`, - `S_NAME` AS `bfcol_6`, - `S_ADDRESS` AS `bfcol_7`, - `S_NATIONKEY` AS `bfcol_8`, - `S_PHONE` AS `bfcol_9`, - `S_ACCTBAL` AS `bfcol_10`, - `S_COMMENT` AS `bfcol_11` - FROM `bigframes-dev-perf`.`tpch_0001t`.`SUPPLIER` AS `bft_2` -), `bfcte_4` AS ( - SELECT - `S_SUPPKEY` AS `bfcol_21`, - `S_NATIONKEY` AS `bfcol_22` - FROM `bigframes-dev-perf`.`tpch_0001t`.`SUPPLIER` AS `bft_2` -), `bfcte_5` AS ( - SELECT - `PS_PARTKEY` AS `bfcol_12`, - `PS_SUPPKEY` AS `bfcol_13`, - `PS_SUPPLYCOST` AS `bfcol_14` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PARTSUPP` AS `bft_1` -), `bfcte_6` AS ( - SELECT - `P_PARTKEY` AS `bfcol_15`, - `P_MFGR` AS `bfcol_16`, - `P_TYPE` AS `bfcol_17`, - `P_SIZE` AS `bfcol_18` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PART` AS `bft_0` -), `bfcte_7` AS ( - SELECT - `P_PARTKEY` AS `bfcol_23`, - `P_TYPE` AS `bfcol_24`, - `P_SIZE` AS `bfcol_25` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PART` AS `bft_0` -), `bfcte_8` AS ( - SELECT - `bfcol_15` AS `bfcol_26`, - `bfcol_16` AS `bfcol_27`, - `bfcol_17` AS `bfcol_28`, - `bfcol_18` AS `bfcol_29`, - `bfcol_13` AS `bfcol_30`, - `bfcol_14` AS `bfcol_31` - FROM `bfcte_6` - INNER JOIN `bfcte_5` - ON `bfcol_15` = `bfcol_12` -), `bfcte_9` AS ( - SELECT - `bfcol_23` AS `bfcol_32`, - `bfcol_24` AS `bfcol_33`, - `bfcol_25` AS `bfcol_34`, - `bfcol_13` AS `bfcol_35`, - `bfcol_14` AS `bfcol_36` - FROM `bfcte_7` - INNER JOIN `bfcte_5` - ON `bfcol_23` = `bfcol_12` -), `bfcte_10` AS ( - SELECT - `bfcol_26` AS `bfcol_37`, - `bfcol_27` AS `bfcol_38`, - `bfcol_28` AS `bfcol_39`, - `bfcol_29` AS `bfcol_40`, - `bfcol_31` AS `bfcol_41`, - `bfcol_6` AS `bfcol_42`, - `bfcol_7` AS `bfcol_43`, - `bfcol_8` AS `bfcol_44`, - `bfcol_9` AS `bfcol_45`, - `bfcol_10` AS `bfcol_46`, - `bfcol_11` AS `bfcol_47` - FROM `bfcte_8` - INNER JOIN `bfcte_3` - ON `bfcol_30` = `bfcol_5` -), `bfcte_11` AS ( - SELECT - `bfcol_32` AS `bfcol_48`, - `bfcol_33` AS `bfcol_49`, - `bfcol_34` AS `bfcol_50`, - `bfcol_36` AS `bfcol_51`, - `bfcol_22` AS `bfcol_52` - FROM `bfcte_9` - INNER JOIN `bfcte_4` - ON `bfcol_35` = `bfcol_21` -), `bfcte_12` AS ( - SELECT - `bfcol_37` AS `bfcol_53`, - `bfcol_38` AS `bfcol_54`, - `bfcol_39` AS `bfcol_55`, - `bfcol_40` AS `bfcol_56`, - `bfcol_41` AS `bfcol_57`, - `bfcol_42` AS `bfcol_58`, - `bfcol_43` AS `bfcol_59`, - `bfcol_45` AS `bfcol_60`, - `bfcol_46` AS `bfcol_61`, - `bfcol_47` AS `bfcol_62`, - `bfcol_3` AS `bfcol_63`, - `bfcol_4` AS `bfcol_64` - FROM `bfcte_10` - INNER JOIN `bfcte_1` - ON `bfcol_44` = `bfcol_2` -), `bfcte_13` AS ( - SELECT - `bfcol_48` AS `bfcol_65`, - `bfcol_49` AS `bfcol_66`, - `bfcol_50` AS `bfcol_67`, - `bfcol_51` AS `bfcol_68`, - `bfcol_20` AS `bfcol_69` - FROM `bfcte_11` - INNER JOIN `bfcte_2` - ON `bfcol_52` = `bfcol_19` -), `bfcte_14` AS ( - SELECT - `bfcol_53` AS `bfcol_205`, - `bfcol_54` AS `bfcol_206`, - `bfcol_57` AS `bfcol_207`, - `bfcol_58` AS `bfcol_208`, - `bfcol_59` AS `bfcol_209`, - `bfcol_60` AS `bfcol_210`, - `bfcol_61` AS `bfcol_211`, - `bfcol_62` AS `bfcol_212`, - `bfcol_63` AS `bfcol_213` - FROM `bfcte_12` - INNER JOIN `bfcte_0` - ON `bfcol_64` = `bfcol_0` - WHERE - `bfcol_56` = 15 AND ENDS_WITH(`bfcol_55`, 'BRASS') AND `bfcol_1` = 'EUROPE' -), `bfcte_15` AS ( - SELECT - `bfcol_65`, - `bfcol_66`, - `bfcol_67`, - `bfcol_68`, - `bfcol_69`, - `bfcol_0`, - `bfcol_1`, - `bfcol_65` AS `bfcol_99`, - `bfcol_66` AS `bfcol_100`, - `bfcol_68` AS `bfcol_101`, - `bfcol_1` AS `bfcol_102`, - `bfcol_67` = 15 AS `bfcol_103`, - `bfcol_65` AS `bfcol_147`, - `bfcol_68` AS `bfcol_148`, - `bfcol_1` AS `bfcol_149`, - ENDS_WITH(`bfcol_66`, 'BRASS') AS `bfcol_150`, - `bfcol_65` AS `bfcol_189`, - `bfcol_68` AS `bfcol_190`, - `bfcol_1` = 'EUROPE' AS `bfcol_191` - FROM `bfcte_13` - INNER JOIN `bfcte_0` - ON `bfcol_69` = `bfcol_0` - WHERE - `bfcol_67` = 15 AND ENDS_WITH(`bfcol_66`, 'BRASS') AND `bfcol_1` = 'EUROPE' -), `bfcte_16` AS ( - SELECT - `bfcol_189`, - MIN(`bfcol_190`) AS `bfcol_216` - FROM `bfcte_15` - GROUP BY - `bfcol_189` -), `bfcte_17` AS ( - SELECT - `bfcol_189` AS `bfcol_214`, - `bfcol_216` - FROM `bfcte_16` -) -SELECT - `bfcol_211` AS `S_ACCTBAL`, - `bfcol_208` AS `S_NAME`, - `bfcol_213` AS `N_NAME`, - `bfcol_214` AS `P_PARTKEY`, - `bfcol_206` AS `P_MFGR`, - `bfcol_209` AS `S_ADDRESS`, - `bfcol_210` AS `S_PHONE`, - `bfcol_212` AS `S_COMMENT` -FROM `bfcte_17` -INNER JOIN `bfcte_14` - ON `bfcol_214` = `bfcol_205` AND `bfcol_216` = `bfcol_207` -ORDER BY - `bfcol_211` DESC, - `bfcol_213` ASC NULLS LAST, - `bfcol_208` ASC NULLS LAST, - `bfcol_214` ASC NULLS LAST -LIMIT 100 \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/20/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/20/out.sql deleted file mode 100644 index 197588f5c84..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/20/out.sql +++ /dev/null @@ -1,144 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `P_PARTKEY`, - `P_NAME`, - `P_PARTKEY` AS `bfcol_15`, - STARTS_WITH(`P_NAME`, 'forest') AS `bfcol_16` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PART` AS `bft_4` - WHERE - STARTS_WITH(`P_NAME`, 'forest') -), `bfcte_1` AS ( - SELECT - `PS_PARTKEY` AS `bfcol_2`, - `PS_SUPPKEY` AS `bfcol_3`, - `PS_AVAILQTY` AS `bfcol_4` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PARTSUPP` AS `bft_3` -), `bfcte_2` AS ( - SELECT - `L_PARTKEY`, - `L_SUPPKEY`, - `L_QUANTITY`, - `L_SHIPDATE`, - `L_PARTKEY` AS `bfcol_17`, - `L_SUPPKEY` AS `bfcol_18`, - `L_QUANTITY` AS `bfcol_19`, - ( - `L_SHIPDATE` >= CAST('1994-01-01' AS DATE) - ) - AND ( - `L_SHIPDATE` < CAST('1995-01-01' AS DATE) - ) AS `bfcol_20` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_2` - WHERE - ( - `L_SHIPDATE` >= CAST('1994-01-01' AS DATE) - ) - AND ( - `L_SHIPDATE` < CAST('1995-01-01' AS DATE) - ) -), `bfcte_3` AS ( - SELECT - `N_NATIONKEY` AS `bfcol_35` - FROM `bigframes-dev-perf`.`tpch_0001t`.`NATION` AS `bft_1` - WHERE - `N_NAME` = 'CANADA' -), `bfcte_4` AS ( - SELECT - `S_SUPPKEY` AS `bfcol_11`, - `S_NAME` AS `bfcol_12`, - `S_ADDRESS` AS `bfcol_13`, - `S_NATIONKEY` AS `bfcol_14` - FROM `bigframes-dev-perf`.`tpch_0001t`.`SUPPLIER` AS `bft_0` -), `bfcte_5` AS ( - SELECT - `bfcol_15` - FROM `bfcte_0` - GROUP BY - `bfcol_15` -), `bfcte_6` AS ( - SELECT - `bfcol_17`, - `bfcol_18`, - COALESCE(SUM(`bfcol_19`), 0) AS `bfcol_36` - FROM `bfcte_2` - GROUP BY - `bfcol_17`, - `bfcol_18` -), `bfcte_7` AS ( - SELECT - `bfcol_11` AS `bfcol_41`, - `bfcol_12` AS `bfcol_42`, - `bfcol_13` AS `bfcol_43` - FROM `bfcte_4` - INNER JOIN `bfcte_3` - ON `bfcol_14` = `bfcol_35` -), `bfcte_8` AS ( - SELECT - `bfcol_15` AS `bfcol_31` - FROM `bfcte_5` -), `bfcte_9` AS ( - SELECT - `bfcol_17` AS `bfcol_48`, - `bfcol_18` AS `bfcol_49`, - `bfcol_36` * 0.5 AS `bfcol_50` - FROM `bfcte_6` -), `bfcte_10` AS ( - SELECT - *, - COALESCE(`bfcol_2` IN (( - SELECT - * - FROM `bfcte_8` - )), FALSE) AS `bfcol_37` - FROM `bfcte_1` -), `bfcte_11` AS ( - SELECT - `bfcol_2` AS `bfcol_51`, - `bfcol_3` AS `bfcol_52`, - `bfcol_4` AS `bfcol_53` - FROM `bfcte_10` - WHERE - `bfcol_37` -), `bfcte_12` AS ( - SELECT - `bfcol_48`, - `bfcol_49`, - `bfcol_50`, - `bfcol_51`, - `bfcol_52`, - `bfcol_53`, - `bfcol_52` AS `bfcol_57`, - `bfcol_53` > `bfcol_50` AS `bfcol_58` - FROM `bfcte_9` - INNER JOIN `bfcte_11` - ON `bfcol_49` = `bfcol_52` AND `bfcol_48` = `bfcol_51` - WHERE - `bfcol_53` > `bfcol_50` -), `bfcte_13` AS ( - SELECT - `bfcol_57` - FROM `bfcte_12` - GROUP BY - `bfcol_57` -), `bfcte_14` AS ( - SELECT - `bfcol_57` AS `bfcol_61` - FROM `bfcte_13` -), `bfcte_15` AS ( - SELECT - *, - COALESCE(`bfcol_41` IN (( - SELECT - * - FROM `bfcte_14` - )), FALSE) AS `bfcol_62` - FROM `bfcte_7` -) -SELECT - `bfcol_42` AS `S_NAME`, - `bfcol_43` AS `S_ADDRESS` -FROM `bfcte_15` -WHERE - `bfcol_62` -ORDER BY - `bfcol_42` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/21/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/21/out.sql deleted file mode 100644 index 0caf29ca617..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/21/out.sql +++ /dev/null @@ -1,142 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `O_ORDERKEY` AS `bfcol_0`, - `O_ORDERSTATUS` AS `bfcol_1` - FROM `bigframes-dev-perf`.`tpch_0001t`.`ORDERS` AS `bft_3` -), `bfcte_1` AS ( - SELECT - `N_NATIONKEY` AS `bfcol_2`, - `N_NAME` AS `bfcol_3` - FROM `bigframes-dev-perf`.`tpch_0001t`.`NATION` AS `bft_2` -), `bfcte_2` AS ( - SELECT - `S_SUPPKEY` AS `bfcol_4`, - `S_NAME` AS `bfcol_5`, - `S_NATIONKEY` AS `bfcol_6` - FROM `bigframes-dev-perf`.`tpch_0001t`.`SUPPLIER` AS `bft_1` -), `bfcte_3` AS ( - SELECT - `L_ORDERKEY` AS `bfcol_30`, - `L_SUPPKEY` AS `bfcol_31` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_0` - WHERE - `L_RECEIPTDATE` > `L_COMMITDATE` -), `bfcte_4` AS ( - SELECT - `L_ORDERKEY` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_0` -), `bfcte_5` AS ( - SELECT - `L_ORDERKEY` AS `bfcol_32` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_0` - WHERE - `L_RECEIPTDATE` > `L_COMMITDATE` -), `bfcte_6` AS ( - SELECT - `L_ORDERKEY`, - COUNT(1) AS `bfcol_18` - FROM `bfcte_4` - GROUP BY - `L_ORDERKEY` -), `bfcte_7` AS ( - SELECT - `L_ORDERKEY` AS `bfcol_33` - FROM `bfcte_6` - WHERE - `bfcol_18` > 1 -), `bfcte_8` AS ( - SELECT - `bfcol_33` AS `bfcol_34`, - `bfcol_31` AS `bfcol_35` - FROM `bfcte_7` - INNER JOIN `bfcte_3` - ON `bfcol_33` = `bfcol_30` -), `bfcte_9` AS ( - SELECT - `bfcol_33`, - COUNT(1) AS `bfcol_37` - FROM `bfcte_7` - INNER JOIN `bfcte_5` - ON `bfcol_33` = `bfcol_32` - GROUP BY - `bfcol_33` -), `bfcte_10` AS ( - SELECT - `bfcol_33` AS `bfcol_36`, - `bfcol_37` - FROM `bfcte_9` -), `bfcte_11` AS ( - SELECT - `bfcol_36` AS `bfcol_38`, - `bfcol_37` AS `bfcol_39`, - `bfcol_35` AS `bfcol_40` - FROM `bfcte_10` - INNER JOIN `bfcte_8` - ON `bfcol_36` = `bfcol_34` -), `bfcte_12` AS ( - SELECT - `bfcol_38` AS `bfcol_41`, - `bfcol_39` AS `bfcol_42`, - `bfcol_5` AS `bfcol_43`, - `bfcol_6` AS `bfcol_44` - FROM `bfcte_11` - INNER JOIN `bfcte_2` - ON `bfcol_40` = `bfcol_4` -), `bfcte_13` AS ( - SELECT - `bfcol_41` AS `bfcol_45`, - `bfcol_42` AS `bfcol_46`, - `bfcol_43` AS `bfcol_47`, - `bfcol_3` AS `bfcol_48` - FROM `bfcte_12` - INNER JOIN `bfcte_1` - ON `bfcol_44` = `bfcol_2` -), `bfcte_14` AS ( - SELECT - `bfcol_45`, - `bfcol_46`, - `bfcol_47`, - `bfcol_48`, - `bfcol_0`, - `bfcol_1`, - `bfcol_47` AS `bfcol_53`, - ( - ( - `bfcol_46` = 1 - ) AND ( - `bfcol_48` = 'SAUDI ARABIA' - ) - ) - AND ( - `bfcol_1` = 'F' - ) AS `bfcol_54` - FROM `bfcte_13` - INNER JOIN `bfcte_0` - ON `bfcol_45` = `bfcol_0` - WHERE - ( - ( - `bfcol_46` = 1 - ) AND ( - `bfcol_48` = 'SAUDI ARABIA' - ) - ) - AND ( - `bfcol_1` = 'F' - ) -), `bfcte_15` AS ( - SELECT - `bfcol_53`, - COUNT(1) AS `bfcol_58` - FROM `bfcte_14` - GROUP BY - `bfcol_53` -) -SELECT - `bfcol_53` AS `S_NAME`, - `bfcol_58` AS `NUMWAIT` -FROM `bfcte_15` -ORDER BY - `bfcol_58` DESC, - `bfcol_53` ASC NULLS LAST -LIMIT 100 \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/22/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/22/out.sql deleted file mode 100644 index 5ab22d3cdaf..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/22/out.sql +++ /dev/null @@ -1,132 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - * - FROM UNNEST(ARRAY>[STRUCT('C_ACCTBAL', 0, 0)]) -), `bfcte_1` AS ( - SELECT - `O_CUSTKEY` - FROM `bigframes-dev-perf`.`tpch_0001t`.`ORDERS` AS `bft_1` -), `bfcte_2` AS ( - SELECT - `C_PHONE`, - `C_ACCTBAL`, - `C_ACCTBAL` AS `bfcol_9`, - SUBSTRING(`C_PHONE`, 1, 2) AS `bfcol_10`, - `C_ACCTBAL` AS `bfcol_19`, - COALESCE( - COALESCE(SUBSTRING(`C_PHONE`, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'), FALSE), - FALSE - ) AS `bfcol_20`, - `C_ACCTBAL` AS `bfcol_35`, - `C_ACCTBAL` > 0.0 AS `bfcol_36` - FROM `bigframes-dev-perf`.`tpch_0001t`.`CUSTOMER` AS `bft_0` - WHERE - COALESCE( - COALESCE(SUBSTRING(`C_PHONE`, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'), FALSE), - FALSE - ) - AND `C_ACCTBAL` > 0.0 -), `bfcte_3` AS ( - SELECT - `C_CUSTKEY` AS `bfcol_32`, - `C_ACCTBAL` AS `bfcol_33`, - SUBSTRING(`C_PHONE`, 1, 2) AS `bfcol_34` - FROM `bigframes-dev-perf`.`tpch_0001t`.`CUSTOMER` AS `bft_0` - WHERE - COALESCE( - COALESCE(SUBSTRING(`C_PHONE`, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'), FALSE), - FALSE - ) -), `bfcte_4` AS ( - SELECT - `O_CUSTKEY` - FROM `bfcte_1` - GROUP BY - `O_CUSTKEY` -), `bfcte_5` AS ( - SELECT - AVG(`bfcol_35`) AS `bfcol_40` - FROM `bfcte_2` -), `bfcte_6` AS ( - SELECT - `O_CUSTKEY` AS `bfcol_0` - FROM `bfcte_4` -), `bfcte_7` AS ( - SELECT - `bfcol_40`, - 0 AS `bfcol_41` - FROM `bfcte_5` -), `bfcte_8` AS ( - SELECT - `bfcol_3`, - `bfcol_4`, - `bfcol_5`, - `bfcol_40`, - `bfcol_41`, - CASE WHEN `bfcol_5` = 0 THEN `bfcol_40` END AS `bfcol_42`, - IF(`bfcol_41` = 0, CASE WHEN `bfcol_5` = 0 THEN `bfcol_40` END, NULL) AS `bfcol_47` - FROM `bfcte_0` - CROSS JOIN `bfcte_7` -), `bfcte_9` AS ( - SELECT - `bfcol_3`, - `bfcol_4`, - ANY_VALUE(`bfcol_47`) AS `bfcol_51` - FROM `bfcte_8` - GROUP BY - `bfcol_3`, - `bfcol_4` -), `bfcte_10` AS ( - SELECT - `bfcol_51` AS `bfcol_52` - FROM `bfcte_9` -), `bfcte_11` AS ( - SELECT - `bfcol_32` AS `bfcol_61`, - `bfcol_33` AS `bfcol_62`, - `bfcol_34` AS `bfcol_63` - FROM `bfcte_3` - CROSS JOIN `bfcte_10` - WHERE - `bfcol_33` > `bfcol_52` -), `bfcte_12` AS ( - SELECT - *, - COALESCE(`bfcol_61` IN (( - SELECT - * - FROM `bfcte_6` - )), FALSE) AS `bfcol_64` - FROM `bfcte_11` -), `bfcte_13` AS ( - SELECT - `bfcol_61`, - `bfcol_62`, - `bfcol_63`, - `bfcol_64`, - NOT ( - `bfcol_64` - ) AS `bfcol_65` - FROM `bfcte_12` - WHERE - NOT ( - `bfcol_64` - ) -), `bfcte_14` AS ( - SELECT - `bfcol_63`, - COUNT(`bfcol_61`) AS `bfcol_73`, - COALESCE(SUM(`bfcol_62`), 0) AS `bfcol_74` - FROM `bfcte_13` - WHERE - NOT `bfcol_63` IS NULL - GROUP BY - `bfcol_63` -) -SELECT - `bfcol_63` AS `CNTRYCODE`, - `bfcol_73` AS `NUMCUST`, - `bfcol_74` AS `TOTACCTBAL` -FROM `bfcte_14` -ORDER BY - `bfcol_63` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/3/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/3/out.sql deleted file mode 100644 index 71779e1adf9..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/3/out.sql +++ /dev/null @@ -1,76 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `O_ORDERKEY` AS `bfcol_32`, - `O_CUSTKEY` AS `bfcol_33`, - `O_ORDERDATE` AS `bfcol_34`, - `O_SHIPPRIORITY` AS `bfcol_35` - FROM `bigframes-dev-perf`.`tpch_0001t`.`ORDERS` AS `bft_2` - WHERE - `O_ORDERDATE` < CAST('1995-03-15' AS DATE) -), `bfcte_1` AS ( - SELECT - `L_ORDERKEY` AS `bfcol_36`, - `L_EXTENDEDPRICE` AS `bfcol_37`, - `L_DISCOUNT` AS `bfcol_38` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_1` - WHERE - `L_SHIPDATE` > CAST('1995-03-15' AS DATE) -), `bfcte_2` AS ( - SELECT - `C_CUSTKEY` AS `bfcol_39` - FROM `bigframes-dev-perf`.`tpch_0001t`.`CUSTOMER` AS `bft_0` - WHERE - `C_MKTSEGMENT` = 'BUILDING' -), `bfcte_3` AS ( - SELECT - `bfcol_37` AS `bfcol_40`, - `bfcol_38` AS `bfcol_41`, - `bfcol_32` AS `bfcol_42`, - `bfcol_33` AS `bfcol_43`, - `bfcol_34` AS `bfcol_44`, - `bfcol_35` AS `bfcol_45` - FROM `bfcte_1` - INNER JOIN `bfcte_0` - ON `bfcol_36` = `bfcol_32` -), `bfcte_4` AS ( - SELECT - `bfcol_39`, - `bfcol_40`, - `bfcol_41`, - `bfcol_42`, - `bfcol_43`, - `bfcol_44`, - `bfcol_45`, - `bfcol_42` AS `bfcol_51`, - `bfcol_44` AS `bfcol_52`, - `bfcol_45` AS `bfcol_53`, - `bfcol_40` * ( - 1 - `bfcol_41` - ) AS `bfcol_54` - FROM `bfcte_2` - INNER JOIN `bfcte_3` - ON `bfcol_39` = `bfcol_43` -), `bfcte_5` AS ( - SELECT - `bfcol_51`, - `bfcol_52`, - `bfcol_53`, - COALESCE(SUM(`bfcol_54`), 0) AS `bfcol_59` - FROM `bfcte_4` - GROUP BY - `bfcol_51`, - `bfcol_52`, - `bfcol_53` -) -SELECT - `bfcol_51` AS `L_ORDERKEY`, - `bfcol_59` AS `REVENUE`, - `bfcol_52` AS `O_ORDERDATE`, - `bfcol_53` AS `O_SHIPPRIORITY` -FROM `bfcte_5` -ORDER BY - `bfcol_59` DESC, - `bfcol_52` ASC NULLS LAST, - `bfcol_51` ASC NULLS LAST, - `bfcol_53` ASC NULLS LAST -LIMIT 10 \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/4/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/4/out.sql deleted file mode 100644 index 3235239710e..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/4/out.sql +++ /dev/null @@ -1,67 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `O_ORDERKEY` AS `bfcol_0`, - `O_ORDERDATE` AS `bfcol_1`, - `O_ORDERPRIORITY` AS `bfcol_2` - FROM `bigframes-dev-perf`.`tpch_0001t`.`ORDERS` AS `bft_1` -), `bfcte_1` AS ( - SELECT - `L_ORDERKEY` AS `bfcol_3`, - `L_COMMITDATE` AS `bfcol_4`, - `L_RECEIPTDATE` AS `bfcol_5` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_0` -), `bfcte_2` AS ( - SELECT - `bfcol_3`, - `bfcol_4`, - `bfcol_5`, - `bfcol_0`, - `bfcol_1`, - `bfcol_2`, - `bfcol_3` AS `bfcol_11`, - `bfcol_4` AS `bfcol_12`, - `bfcol_5` AS `bfcol_13`, - `bfcol_2` AS `bfcol_14`, - ( - `bfcol_1` >= CAST('1993-07-01' AS DATE) - ) - AND ( - `bfcol_1` < CAST('1993-10-01' AS DATE) - ) AS `bfcol_15`, - `bfcol_3` AS `bfcol_25`, - `bfcol_2` AS `bfcol_26`, - `bfcol_4` < `bfcol_5` AS `bfcol_27` - FROM `bfcte_1` - INNER JOIN `bfcte_0` - ON `bfcol_3` = `bfcol_0` - WHERE - ( - `bfcol_1` >= CAST('1993-07-01' AS DATE) - ) - AND ( - `bfcol_1` < CAST('1993-10-01' AS DATE) - ) - AND `bfcol_4` < `bfcol_5` -), `bfcte_3` AS ( - SELECT - `bfcol_26`, - `bfcol_25`, - COUNT(1) AS `bfcol_33` - FROM `bfcte_2` - GROUP BY - `bfcol_26`, - `bfcol_25` -), `bfcte_4` AS ( - SELECT - `bfcol_26`, - COUNT(`bfcol_25`) AS `bfcol_36` - FROM `bfcte_3` - GROUP BY - `bfcol_26` -) -SELECT - `bfcol_26` AS `O_ORDERPRIORITY`, - `bfcol_36` AS `ORDER_COUNT` -FROM `bfcte_4` -ORDER BY - `bfcol_26` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/5/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/5/out.sql deleted file mode 100644 index 5b707ce59ac..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/5/out.sql +++ /dev/null @@ -1,91 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `S_SUPPKEY` AS `bfcol_0`, - `S_NATIONKEY` AS `bfcol_1` - FROM `bigframes-dev-perf`.`tpch_0001t`.`SUPPLIER` AS `bft_5` -), `bfcte_1` AS ( - SELECT - `C_CUSTKEY` AS `bfcol_2`, - `C_NATIONKEY` AS `bfcol_3` - FROM `bigframes-dev-perf`.`tpch_0001t`.`CUSTOMER` AS `bft_4` -), `bfcte_2` AS ( - SELECT - `N_NATIONKEY` AS `bfcol_4`, - `N_NAME` AS `bfcol_5`, - `N_REGIONKEY` AS `bfcol_6` - FROM `bigframes-dev-perf`.`tpch_0001t`.`NATION` AS `bft_3` -), `bfcte_3` AS ( - SELECT - `R_REGIONKEY` AS `bfcol_32` - FROM `bigframes-dev-perf`.`tpch_0001t`.`REGION` AS `bft_2` - WHERE - `R_NAME` = 'ASIA' -), `bfcte_4` AS ( - SELECT - `O_ORDERKEY` AS `bfcol_33`, - `O_CUSTKEY` AS `bfcol_34` - FROM `bigframes-dev-perf`.`tpch_0001t`.`ORDERS` AS `bft_1` - WHERE - ( - `O_ORDERDATE` >= CAST('1994-01-01' AS DATE) - ) - AND ( - `O_ORDERDATE` < CAST('1995-01-01' AS DATE) - ) -), `bfcte_5` AS ( - SELECT - `L_ORDERKEY` AS `bfcol_29`, - `L_SUPPKEY` AS `bfcol_30`, - `L_EXTENDEDPRICE` * ( - 1.0 - `L_DISCOUNT` - ) AS `bfcol_31` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_0` -), `bfcte_6` AS ( - SELECT - `bfcol_4` AS `bfcol_35`, - `bfcol_5` AS `bfcol_36` - FROM `bfcte_3` - INNER JOIN `bfcte_2` - ON `bfcol_32` = `bfcol_6` -), `bfcte_7` AS ( - SELECT - `bfcol_35` AS `bfcol_37`, - `bfcol_36` AS `bfcol_38`, - `bfcol_2` AS `bfcol_39` - FROM `bfcte_6` - INNER JOIN `bfcte_1` - ON `bfcol_35` = `bfcol_3` -), `bfcte_8` AS ( - SELECT - `bfcol_33` AS `bfcol_40`, - `bfcol_37` AS `bfcol_41`, - `bfcol_38` AS `bfcol_42` - FROM `bfcte_4` - INNER JOIN `bfcte_7` - ON `bfcol_34` = `bfcol_39` -), `bfcte_9` AS ( - SELECT - `bfcol_30` AS `bfcol_43`, - `bfcol_31` AS `bfcol_44`, - `bfcol_41` AS `bfcol_45`, - `bfcol_42` AS `bfcol_46` - FROM `bfcte_5` - INNER JOIN `bfcte_8` - ON `bfcol_29` = `bfcol_40` -), `bfcte_10` AS ( - SELECT - `bfcol_46`, - COALESCE(SUM(`bfcol_44`), 0) AS `bfcol_49` - FROM `bfcte_9` - INNER JOIN `bfcte_0` - ON `bfcol_43` = `bfcol_0` AND `bfcol_45` = `bfcol_1` - GROUP BY - `bfcol_46` -) -SELECT - `bfcol_46` AS `N_NAME`, - `bfcol_49` AS `REVENUE` -FROM `bfcte_10` -ORDER BY - `bfcol_49` DESC, - `bfcol_46` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/6/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/6/out.sql deleted file mode 100644 index 3544fd18e48..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/6/out.sql +++ /dev/null @@ -1,61 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - * - FROM UNNEST(ARRAY>[STRUCT(0)]) -), `bfcte_1` AS ( - SELECT - `L_QUANTITY`, - `L_EXTENDEDPRICE`, - `L_DISCOUNT`, - `L_SHIPDATE`, - `L_QUANTITY` AS `bfcol_5`, - `L_EXTENDEDPRICE` AS `bfcol_6`, - `L_DISCOUNT` AS `bfcol_7`, - ( - `L_SHIPDATE` >= CAST('1994-01-01' AS DATE) - ) - AND ( - `L_SHIPDATE` < CAST('1995-01-01' AS DATE) - ) AS `bfcol_8`, - `L_QUANTITY` AS `bfcol_16`, - `L_EXTENDEDPRICE` AS `bfcol_17`, - `L_DISCOUNT` AS `bfcol_18`, - ( - `L_DISCOUNT` >= 0.05 - ) AND ( - `L_DISCOUNT` <= 0.07 - ) AS `bfcol_19`, - `L_EXTENDEDPRICE` AS `bfcol_27`, - `L_DISCOUNT` AS `bfcol_28`, - `L_QUANTITY` < 24 AS `bfcol_29`, - `L_EXTENDEDPRICE` AS `bfcol_35`, - `L_DISCOUNT` AS `bfcol_36`, - `L_EXTENDEDPRICE` * `L_DISCOUNT` AS `bfcol_39` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_0` - WHERE - ( - `L_SHIPDATE` >= CAST('1994-01-01' AS DATE) - ) - AND ( - `L_SHIPDATE` < CAST('1995-01-01' AS DATE) - ) - AND ( - `L_DISCOUNT` >= 0.05 - ) - AND ( - `L_DISCOUNT` <= 0.07 - ) - AND `L_QUANTITY` < 24 -), `bfcte_2` AS ( - SELECT - COALESCE(SUM(`bfcol_39`), 0) AS `bfcol_41` - FROM `bfcte_1` -), `bfcte_3` AS ( - SELECT - * - FROM `bfcte_2` -) -SELECT - CASE WHEN `bfcol_0` = 0 THEN `bfcol_41` END AS `REVENUE` -FROM `bfcte_3` -CROSS JOIN `bfcte_0` \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/7/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/7/out.sql deleted file mode 100644 index f180ca1b3ea..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/7/out.sql +++ /dev/null @@ -1,138 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `N_NATIONKEY` AS `bfcol_22`, - `N_NAME` AS `bfcol_23`, - COALESCE(COALESCE(`N_NAME` IN ('FRANCE', 'GERMANY'), FALSE), FALSE) AS `bfcol_24` - FROM `bigframes-dev-perf`.`tpch_0001t`.`NATION` AS `bft_4` - WHERE - COALESCE(COALESCE(`N_NAME` IN ('FRANCE', 'GERMANY'), FALSE), FALSE) -), `bfcte_1` AS ( - SELECT - `S_SUPPKEY` AS `bfcol_2`, - `S_NATIONKEY` AS `bfcol_3` - FROM `bigframes-dev-perf`.`tpch_0001t`.`SUPPLIER` AS `bft_3` -), `bfcte_2` AS ( - SELECT - `L_ORDERKEY` AS `bfcol_31`, - `L_SUPPKEY` AS `bfcol_32`, - `L_EXTENDEDPRICE` AS `bfcol_33`, - `L_DISCOUNT` AS `bfcol_34`, - `L_SHIPDATE` AS `bfcol_35` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_2` - WHERE - ( - `L_SHIPDATE` >= CAST('1995-01-01' AS DATE) - ) - AND ( - `L_SHIPDATE` <= CAST('1996-12-31' AS DATE) - ) -), `bfcte_3` AS ( - SELECT - `O_ORDERKEY` AS `bfcol_9`, - `O_CUSTKEY` AS `bfcol_10` - FROM `bigframes-dev-perf`.`tpch_0001t`.`ORDERS` AS `bft_1` -), `bfcte_4` AS ( - SELECT - `C_CUSTKEY` AS `bfcol_11`, - `C_NATIONKEY` AS `bfcol_12` - FROM `bigframes-dev-perf`.`tpch_0001t`.`CUSTOMER` AS `bft_0` -), `bfcte_5` AS ( - SELECT - `bfcol_22` AS `bfcol_36`, - `bfcol_23` AS `bfcol_37` - FROM `bfcte_0` -), `bfcte_6` AS ( - SELECT - `bfcol_22` AS `bfcol_38`, - `bfcol_23` AS `bfcol_39` - FROM `bfcte_0` -), `bfcte_7` AS ( - SELECT - `bfcol_11` AS `bfcol_40`, - `bfcol_39` AS `bfcol_41` - FROM `bfcte_4` - INNER JOIN `bfcte_6` - ON `bfcol_12` = `bfcol_38` -), `bfcte_8` AS ( - SELECT - `bfcol_41` AS `bfcol_42`, - `bfcol_9` AS `bfcol_43` - FROM `bfcte_7` - INNER JOIN `bfcte_3` - ON `bfcol_40` = `bfcol_10` -), `bfcte_9` AS ( - SELECT - `bfcol_42` AS `bfcol_44`, - `bfcol_32` AS `bfcol_45`, - `bfcol_33` AS `bfcol_46`, - `bfcol_34` AS `bfcol_47`, - `bfcol_35` AS `bfcol_48` - FROM `bfcte_8` - INNER JOIN `bfcte_2` - ON `bfcol_43` = `bfcol_31` -), `bfcte_10` AS ( - SELECT - `bfcol_44` AS `bfcol_49`, - `bfcol_46` AS `bfcol_50`, - `bfcol_47` AS `bfcol_51`, - `bfcol_48` AS `bfcol_52`, - `bfcol_3` AS `bfcol_53` - FROM `bfcte_9` - INNER JOIN `bfcte_1` - ON `bfcol_45` = `bfcol_2` -), `bfcte_11` AS ( - SELECT - `bfcol_49`, - `bfcol_50`, - `bfcol_51`, - `bfcol_52`, - `bfcol_53`, - `bfcol_36`, - `bfcol_37`, - `bfcol_49` AS `bfcol_59`, - `bfcol_50` AS `bfcol_60`, - `bfcol_51` AS `bfcol_61`, - `bfcol_52` AS `bfcol_62`, - `bfcol_37` AS `bfcol_63`, - `bfcol_49` <> `bfcol_37` AS `bfcol_64`, - `bfcol_49` AS `bfcol_76`, - `bfcol_52` AS `bfcol_77`, - `bfcol_37` AS `bfcol_78`, - `bfcol_50` * ( - 1.0 - `bfcol_51` - ) AS `bfcol_79`, - `bfcol_49` AS `bfcol_84`, - `bfcol_37` AS `bfcol_85`, - `bfcol_50` * ( - 1.0 - `bfcol_51` - ) AS `bfcol_86`, - EXTRACT(YEAR FROM `bfcol_52`) AS `bfcol_87` - FROM `bfcte_10` - INNER JOIN `bfcte_5` - ON `bfcol_53` = `bfcol_36` - WHERE - `bfcol_49` <> `bfcol_37` -), `bfcte_12` AS ( - SELECT - `bfcol_85`, - `bfcol_84`, - `bfcol_87`, - COALESCE(SUM(`bfcol_86`), 0) AS `bfcol_92` - FROM `bfcte_11` - WHERE - NOT `bfcol_87` IS NULL - GROUP BY - `bfcol_85`, - `bfcol_84`, - `bfcol_87` -) -SELECT - `bfcol_85` AS `SUPP_NATION`, - `bfcol_84` AS `CUST_NATION`, - `bfcol_87` AS `L_YEAR`, - `bfcol_92` AS `REVENUE` -FROM `bfcte_12` -ORDER BY - `bfcol_85` ASC NULLS LAST, - `bfcol_84` ASC NULLS LAST, - `bfcol_87` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/8/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/8/out.sql deleted file mode 100644 index edaf51f18b8..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/8/out.sql +++ /dev/null @@ -1,186 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `N_NATIONKEY` AS `bfcol_0`, - `N_NAME` AS `bfcol_1` - FROM `bigframes-dev-perf`.`tpch_0001t`.`NATION` AS `bft_6` -), `bfcte_1` AS ( - SELECT - `N_NATIONKEY` AS `bfcol_4`, - `N_REGIONKEY` AS `bfcol_5` - FROM `bigframes-dev-perf`.`tpch_0001t`.`NATION` AS `bft_6` -), `bfcte_2` AS ( - SELECT - `R_REGIONKEY` AS `bfcol_2`, - `R_NAME` AS `bfcol_3` - FROM `bigframes-dev-perf`.`tpch_0001t`.`REGION` AS `bft_5` -), `bfcte_3` AS ( - SELECT - `C_CUSTKEY` AS `bfcol_6`, - `C_NATIONKEY` AS `bfcol_7` - FROM `bigframes-dev-perf`.`tpch_0001t`.`CUSTOMER` AS `bft_4` -), `bfcte_4` AS ( - SELECT - `O_ORDERKEY` AS `bfcol_8`, - `O_CUSTKEY` AS `bfcol_9`, - `O_ORDERDATE` AS `bfcol_10` - FROM `bigframes-dev-perf`.`tpch_0001t`.`ORDERS` AS `bft_3` -), `bfcte_5` AS ( - SELECT - `S_SUPPKEY` AS `bfcol_11`, - `S_NATIONKEY` AS `bfcol_12` - FROM `bigframes-dev-perf`.`tpch_0001t`.`SUPPLIER` AS `bft_2` -), `bfcte_6` AS ( - SELECT - `L_ORDERKEY` AS `bfcol_13`, - `L_PARTKEY` AS `bfcol_14`, - `L_SUPPKEY` AS `bfcol_15`, - `L_EXTENDEDPRICE` AS `bfcol_16`, - `L_DISCOUNT` AS `bfcol_17` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_1` -), `bfcte_7` AS ( - SELECT - `P_PARTKEY` AS `bfcol_18`, - `P_TYPE` AS `bfcol_19` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PART` AS `bft_0` -), `bfcte_8` AS ( - SELECT - `bfcol_19` AS `bfcol_20`, - `bfcol_13` AS `bfcol_21`, - `bfcol_15` AS `bfcol_22`, - `bfcol_16` AS `bfcol_23`, - `bfcol_17` AS `bfcol_24` - FROM `bfcte_7` - INNER JOIN `bfcte_6` - ON `bfcol_18` = `bfcol_14` -), `bfcte_9` AS ( - SELECT - `bfcol_20` AS `bfcol_25`, - `bfcol_21` AS `bfcol_26`, - `bfcol_23` AS `bfcol_27`, - `bfcol_24` AS `bfcol_28`, - `bfcol_12` AS `bfcol_29` - FROM `bfcte_8` - INNER JOIN `bfcte_5` - ON `bfcol_22` = `bfcol_11` -), `bfcte_10` AS ( - SELECT - `bfcol_25` AS `bfcol_30`, - `bfcol_27` AS `bfcol_31`, - `bfcol_28` AS `bfcol_32`, - `bfcol_29` AS `bfcol_33`, - `bfcol_9` AS `bfcol_34`, - `bfcol_10` AS `bfcol_35` - FROM `bfcte_9` - INNER JOIN `bfcte_4` - ON `bfcol_26` = `bfcol_8` -), `bfcte_11` AS ( - SELECT - `bfcol_30` AS `bfcol_36`, - `bfcol_31` AS `bfcol_37`, - `bfcol_32` AS `bfcol_38`, - `bfcol_33` AS `bfcol_39`, - `bfcol_35` AS `bfcol_40`, - `bfcol_7` AS `bfcol_41` - FROM `bfcte_10` - INNER JOIN `bfcte_3` - ON `bfcol_34` = `bfcol_6` -), `bfcte_12` AS ( - SELECT - `bfcol_36` AS `bfcol_42`, - `bfcol_37` AS `bfcol_43`, - `bfcol_38` AS `bfcol_44`, - `bfcol_39` AS `bfcol_45`, - `bfcol_40` AS `bfcol_46`, - `bfcol_5` AS `bfcol_47` - FROM `bfcte_11` - INNER JOIN `bfcte_1` - ON `bfcol_41` = `bfcol_4` -), `bfcte_13` AS ( - SELECT - `bfcol_42` AS `bfcol_66`, - `bfcol_43` AS `bfcol_67`, - `bfcol_44` AS `bfcol_68`, - `bfcol_45` AS `bfcol_69`, - `bfcol_46` AS `bfcol_70` - FROM `bfcte_12` - INNER JOIN `bfcte_2` - ON `bfcol_47` = `bfcol_2` - WHERE - `bfcol_3` = 'AMERICA' -), `bfcte_14` AS ( - SELECT - `bfcol_66`, - `bfcol_67`, - `bfcol_68`, - `bfcol_69`, - `bfcol_70`, - `bfcol_0`, - `bfcol_1`, - `bfcol_66` AS `bfcol_76`, - `bfcol_67` AS `bfcol_77`, - `bfcol_68` AS `bfcol_78`, - `bfcol_70` AS `bfcol_79`, - `bfcol_1` AS `bfcol_80`, - ( - `bfcol_70` >= CAST('1995-01-01' AS DATE) - ) - AND ( - `bfcol_70` <= CAST('1996-12-31' AS DATE) - ) AS `bfcol_81`, - `bfcol_67` AS `bfcol_93`, - `bfcol_68` AS `bfcol_94`, - `bfcol_70` AS `bfcol_95`, - `bfcol_1` AS `bfcol_96`, - `bfcol_66` = 'ECONOMY ANODIZED STEEL' AS `bfcol_97`, - `bfcol_67` AS `bfcol_107`, - `bfcol_68` AS `bfcol_108`, - `bfcol_1` AS `bfcol_109`, - EXTRACT(YEAR FROM `bfcol_70`) AS `bfcol_110`, - `bfcol_1` AS `bfcol_115`, - EXTRACT(YEAR FROM `bfcol_70`) AS `bfcol_116`, - `bfcol_67` * ( - 1.0 - `bfcol_68` - ) AS `bfcol_117`, - EXTRACT(YEAR FROM `bfcol_70`) AS `bfcol_121`, - `bfcol_67` * ( - 1.0 - `bfcol_68` - ) AS `bfcol_122`, - IF(`bfcol_1` = 'BRAZIL', `bfcol_67` * ( - 1.0 - `bfcol_68` - ), 0) AS `bfcol_123`, - EXTRACT(YEAR FROM `bfcol_70`) AS `bfcol_127`, - IF(`bfcol_1` = 'BRAZIL', `bfcol_67` * ( - 1.0 - `bfcol_68` - ), 0) AS `bfcol_128`, - `bfcol_67` * ( - 1.0 - `bfcol_68` - ) AS `bfcol_129` - FROM `bfcte_13` - INNER JOIN `bfcte_0` - ON `bfcol_69` = `bfcol_0` - WHERE - ( - `bfcol_70` >= CAST('1995-01-01' AS DATE) - ) - AND ( - `bfcol_70` <= CAST('1996-12-31' AS DATE) - ) - AND `bfcol_66` = 'ECONOMY ANODIZED STEEL' -), `bfcte_15` AS ( - SELECT - `bfcol_127`, - COALESCE(SUM(`bfcol_128`), 0) AS `bfcol_133`, - COALESCE(SUM(`bfcol_129`), 0) AS `bfcol_134` - FROM `bfcte_14` - WHERE - NOT `bfcol_127` IS NULL - GROUP BY - `bfcol_127` -) -SELECT - `bfcol_127` AS `O_YEAR`, - ROUND(IEEE_DIVIDE(`bfcol_133`, `bfcol_134`), 2) AS `MKT_SHARE` -FROM `bfcte_15` -ORDER BY - `bfcol_127` ASC NULLS LAST, - `bfcol_127` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/9/out.sql b/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/9/out.sql deleted file mode 100644 index 949d45e8dae..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/snapshots/test_tpch/test_tpch_query/9/out.sql +++ /dev/null @@ -1,143 +0,0 @@ -WITH `bfcte_0` AS ( - SELECT - `N_NATIONKEY` AS `bfcol_0`, - `N_NAME` AS `bfcol_1` - FROM `bigframes-dev-perf`.`tpch_0001t`.`NATION` AS `bft_5` -), `bfcte_1` AS ( - SELECT - `O_ORDERKEY` AS `bfcol_2`, - `O_ORDERDATE` AS `bfcol_3` - FROM `bigframes-dev-perf`.`tpch_0001t`.`ORDERS` AS `bft_4` -), `bfcte_2` AS ( - SELECT - `S_SUPPKEY` AS `bfcol_4`, - `S_NATIONKEY` AS `bfcol_5` - FROM `bigframes-dev-perf`.`tpch_0001t`.`SUPPLIER` AS `bft_3` -), `bfcte_3` AS ( - SELECT - `PS_PARTKEY` AS `bfcol_6`, - `PS_SUPPKEY` AS `bfcol_7`, - `PS_SUPPLYCOST` AS `bfcol_8` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PARTSUPP` AS `bft_2` -), `bfcte_4` AS ( - SELECT - `L_ORDERKEY` AS `bfcol_9`, - `L_PARTKEY` AS `bfcol_10`, - `L_SUPPKEY` AS `bfcol_11`, - `L_QUANTITY` AS `bfcol_12`, - `L_EXTENDEDPRICE` AS `bfcol_13`, - `L_DISCOUNT` AS `bfcol_14` - FROM `bigframes-dev-perf`.`tpch_0001t`.`LINEITEM` AS `bft_1` -), `bfcte_5` AS ( - SELECT - `P_PARTKEY` AS `bfcol_15`, - `P_NAME` AS `bfcol_16` - FROM `bigframes-dev-perf`.`tpch_0001t`.`PART` AS `bft_0` -), `bfcte_6` AS ( - SELECT - `bfcol_16` AS `bfcol_17`, - `bfcol_9` AS `bfcol_18`, - `bfcol_10` AS `bfcol_19`, - `bfcol_11` AS `bfcol_20`, - `bfcol_12` AS `bfcol_21`, - `bfcol_13` AS `bfcol_22`, - `bfcol_14` AS `bfcol_23` - FROM `bfcte_5` - INNER JOIN `bfcte_4` - ON `bfcol_15` = `bfcol_10` -), `bfcte_7` AS ( - SELECT - `bfcol_17` AS `bfcol_24`, - `bfcol_18` AS `bfcol_25`, - `bfcol_20` AS `bfcol_26`, - `bfcol_21` AS `bfcol_27`, - `bfcol_22` AS `bfcol_28`, - `bfcol_23` AS `bfcol_29`, - `bfcol_8` AS `bfcol_30` - FROM `bfcte_6` - INNER JOIN `bfcte_3` - ON `bfcol_20` = `bfcol_7` AND `bfcol_19` = `bfcol_6` -), `bfcte_8` AS ( - SELECT - `bfcol_24` AS `bfcol_31`, - `bfcol_25` AS `bfcol_32`, - `bfcol_27` AS `bfcol_33`, - `bfcol_28` AS `bfcol_34`, - `bfcol_29` AS `bfcol_35`, - `bfcol_30` AS `bfcol_36`, - `bfcol_5` AS `bfcol_37` - FROM `bfcte_7` - INNER JOIN `bfcte_2` - ON `bfcol_26` = `bfcol_4` -), `bfcte_9` AS ( - SELECT - `bfcol_31` AS `bfcol_38`, - `bfcol_33` AS `bfcol_39`, - `bfcol_34` AS `bfcol_40`, - `bfcol_35` AS `bfcol_41`, - `bfcol_36` AS `bfcol_42`, - `bfcol_37` AS `bfcol_43`, - `bfcol_3` AS `bfcol_44` - FROM `bfcte_8` - INNER JOIN `bfcte_1` - ON `bfcol_32` = `bfcol_2` -), `bfcte_10` AS ( - SELECT - `bfcol_38`, - `bfcol_39`, - `bfcol_40`, - `bfcol_41`, - `bfcol_42`, - `bfcol_43`, - `bfcol_44`, - `bfcol_0`, - `bfcol_1`, - `bfcol_39` AS `bfcol_52`, - `bfcol_40` AS `bfcol_53`, - `bfcol_41` AS `bfcol_54`, - `bfcol_42` AS `bfcol_55`, - `bfcol_44` AS `bfcol_56`, - `bfcol_1` AS `bfcol_57`, - REGEXP_CONTAINS(`bfcol_38`, 'green') AS `bfcol_58`, - `bfcol_39` AS `bfcol_72`, - `bfcol_40` AS `bfcol_73`, - `bfcol_41` AS `bfcol_74`, - `bfcol_42` AS `bfcol_75`, - `bfcol_1` AS `bfcol_76`, - EXTRACT(YEAR FROM `bfcol_44`) AS `bfcol_77`, - `bfcol_1` AS `bfcol_84`, - EXTRACT(YEAR FROM `bfcol_44`) AS `bfcol_85`, - ( - `bfcol_40` * ( - 1 - `bfcol_41` - ) - ) - ( - `bfcol_42` * `bfcol_39` - ) AS `bfcol_86` - FROM `bfcte_9` - INNER JOIN `bfcte_0` - ON `bfcol_43` = `bfcol_0` - WHERE - REGEXP_CONTAINS(`bfcol_38`, 'green') -), `bfcte_11` AS ( - SELECT - `bfcol_84`, - `bfcol_85`, - COALESCE(SUM(`bfcol_86`), 0) AS `bfcol_90` - FROM `bfcte_10` - WHERE - NOT `bfcol_85` IS NULL - GROUP BY - `bfcol_84`, - `bfcol_85` -) -SELECT - `bfcol_84` AS `NATION`, - `bfcol_85` AS `O_YEAR`, - ROUND(`bfcol_90`, 2) AS `SUM_PROFIT` -FROM `bfcte_11` -ORDER BY - `bfcol_84` ASC NULLS LAST, - `bfcol_85` DESC, - `bfcol_84` ASC NULLS LAST, - `bfcol_85` ASC NULLS LAST \ No newline at end of file diff --git a/tests/unit/core/compile/sqlglot/tpch/test_tpch.py b/tests/unit/core/compile/sqlglot/tpch/test_tpch.py deleted file mode 100644 index 8988a5512f2..00000000000 --- a/tests/unit/core/compile/sqlglot/tpch/test_tpch.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -import pytest - -freezegun = pytest.importorskip("freezegun") -pytest.importorskip("pytest_snapshot") - - -@pytest.mark.parametrize("query_num", range(1, 23)) -def test_tpch_query(tpch_session, query_num, snapshot): - project_id = "bigframes-dev-perf" - dataset_id = "tpch_0001t" - - query_file_path = f"third_party/bigframes_vendored/tpch/queries/q{query_num}.py" - - with open(query_file_path, "r") as f: - query_code = f.read() - - # We want to capture the result dataframe instead of running next(result.to_pandas_batches(...)) - modified_code = re.sub( - r"next\((\w+)\.to_pandas_batches\((.*?)\)\)", - r"return \1", - query_code, - ) - - exec_globals = {} # type: ignore[var-annotated] - exec(modified_code, exec_globals) - q_func = exec_globals["q"] - - result = q_func(project_id, dataset_id, tpch_session) - - # result should be a DataFrame - snapshot.assert_match(result.sql, "out.sql") diff --git a/tests/unit/core/logging/test_log_adapter.py b/tests/unit/core/logging/test_log_adapter.py index 0722ef62a29..ecef966afca 100644 --- a/tests/unit/core/logging/test_log_adapter.py +++ b/tests/unit/core/logging/test_log_adapter.py @@ -14,8 +14,8 @@ from unittest import mock -import pytest from google.cloud import bigquery +import pytest from bigframes.core.logging import log_adapter diff --git a/tests/unit/core/rewrite/conftest.py b/tests/unit/core/rewrite/conftest.py index ab168427f29..8c7ee290ae6 100644 --- a/tests/unit/core/rewrite/conftest.py +++ b/tests/unit/core/rewrite/conftest.py @@ -16,9 +16,8 @@ import google.cloud.bigquery import pytest -import bigframes import bigframes.core as core -from bigframes.core import bq_data +import bigframes.core.schema TABLE_REF = google.cloud.bigquery.TableReference.from_string("project.dataset.table") SCHEMA = ( @@ -72,7 +71,7 @@ def fake_session(): def leaf(fake_session, table): return core.ArrayValue.from_table( session=fake_session, - table=bq_data.GbqNativeTable.from_table(table), + table=table, ).node @@ -80,5 +79,5 @@ def leaf(fake_session, table): def leaf_too(fake_session, table_too): return core.ArrayValue.from_table( session=fake_session, - table=bq_data.GbqNativeTable.from_table(table_too), + table=table_too, ).node diff --git a/tests/unit/core/rewrite/test_identifiers.py b/tests/unit/core/rewrite/test_identifiers.py index 4d4609bb0fa..09904ac4ba2 100644 --- a/tests/unit/core/rewrite/test_identifiers.py +++ b/tests/unit/core/rewrite/test_identifiers.py @@ -14,13 +14,10 @@ import typing import bigframes.core as core -import bigframes.core.agg_expressions as agg_ex import bigframes.core.expression as ex import bigframes.core.identifiers as identifiers import bigframes.core.nodes as nodes import bigframes.core.rewrite.identifiers as id_rewrite -import bigframes.operations.aggregations as agg_ops -from bigframes.core import bq_data def test_remap_variables_single_node(leaf): @@ -54,56 +51,11 @@ def test_remap_variables_projection(leaf): assert set(mapping.values()) == {identifiers.ColumnId(f"id_{i}") for i in range(3)} -def test_remap_variables_aggregate(leaf): - # Aggregation: sum(col_a) AS sum_a - # Group by nothing - agg_op = agg_ex.UnaryAggregation( - op=agg_ops.sum_op, - arg=ex.DerefOp(leaf.fields[0].id), - ) - node = nodes.AggregateNode( - child=leaf, - aggregations=((agg_op, identifiers.ColumnId("sum_a")),), - by_column_ids=(), - ) - - id_generator = (identifiers.ColumnId(f"id_{i}") for i in range(100)) - _, mapping = id_rewrite.remap_variables(node, id_generator) - - # leaf has 2 columns: col_a, col_b - # AggregateNode defines 1 column: sum_a - # Output of AggregateNode should only be sum_a - assert len(mapping) == 1 - assert identifiers.ColumnId("sum_a") in mapping - - -def test_remap_variables_aggregate_with_grouping(leaf): - # Aggregation: sum(col_b) AS sum_b - # Group by col_a - agg_op = agg_ex.UnaryAggregation( - op=agg_ops.sum_op, - arg=ex.DerefOp(leaf.fields[1].id), - ) - node = nodes.AggregateNode( - child=leaf, - aggregations=((agg_op, identifiers.ColumnId("sum_b")),), - by_column_ids=(ex.DerefOp(leaf.fields[0].id),), - ) - - id_generator = (identifiers.ColumnId(f"id_{i}") for i in range(100)) - _, mapping = id_rewrite.remap_variables(node, id_generator) - - # Output should have 2 columns: col_a (grouping) and sum_b (agg) - assert len(mapping) == 2 - assert leaf.fields[0].id in mapping - assert identifiers.ColumnId("sum_b") in mapping - - def test_remap_variables_nested_join_stability(leaf, fake_session, table): # Create two more distinct leaf nodes leaf2_uncached = core.ArrayValue.from_table( session=fake_session, - table=bq_data.GbqNativeTable.from_table(table), + table=table, ).node leaf2 = leaf2_uncached.remap_vars( { @@ -113,7 +65,7 @@ def test_remap_variables_nested_join_stability(leaf, fake_session, table): ) leaf3_uncached = core.ArrayValue.from_table( session=fake_session, - table=bq_data.GbqNativeTable.from_table(table), + table=table, ).node leaf3 = leaf3_uncached.remap_vars( { @@ -134,7 +86,6 @@ def test_remap_variables_nested_join_stability(leaf, fake_session, table): ), type="inner", propogate_order=False, - nulls_equal=True, ) outer_join = nodes.JoinNode( left_child=inner_join, @@ -147,7 +98,6 @@ def test_remap_variables_nested_join_stability(leaf, fake_session, table): ), type="inner", propogate_order=False, - nulls_equal=True, ) # Run remap_variables twice and assert stability diff --git a/tests/unit/core/sql/snapshots/test_ml/test_create_model_hparam_tuning/create_model_hparam_tuning.sql b/tests/unit/core/sql/snapshots/test_ml/test_create_model_hparam_tuning/create_model_hparam_tuning.sql deleted file mode 100644 index c7ed32e54fc..00000000000 --- a/tests/unit/core/sql/snapshots/test_ml/test_create_model_hparam_tuning/create_model_hparam_tuning.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE MODEL `my_model` -OPTIONS(model_type = 'LINEAR_REG', learn_rate = HPARAM_RANGE(0.0001, 1.0), optimizer = HPARAM_CANDIDATES(['ADAGRAD', 'SGD'])) -AS SELECT * FROM t diff --git a/tests/unit/core/sql/snapshots/test_ml/test_evaluate_model_with_options/evaluate_model_with_options.sql b/tests/unit/core/sql/snapshots/test_ml/test_evaluate_model_with_options/evaluate_model_with_options.sql index cdb66bbf0e1..01eb4d37819 100644 --- a/tests/unit/core/sql/snapshots/test_ml/test_evaluate_model_with_options/evaluate_model_with_options.sql +++ b/tests/unit/core/sql/snapshots/test_ml/test_evaluate_model_with_options/evaluate_model_with_options.sql @@ -1 +1 @@ -SELECT * FROM ML.EVALUATE(MODEL `my_model`, STRUCT(FALSE AS `perform_aggregation`, 10 AS `horizon`, 0.95 AS `confidence_level`)) +SELECT * FROM ML.EVALUATE(MODEL `my_model`, STRUCT(False AS perform_aggregation, 10 AS horizon, 0.95 AS confidence_level)) diff --git a/tests/unit/core/sql/snapshots/test_ml/test_explain_predict_model_with_options/explain_predict_model_with_options.sql b/tests/unit/core/sql/snapshots/test_ml/test_explain_predict_model_with_options/explain_predict_model_with_options.sql index 7569463ea2d..1214bba8706 100644 --- a/tests/unit/core/sql/snapshots/test_ml/test_explain_predict_model_with_options/explain_predict_model_with_options.sql +++ b/tests/unit/core/sql/snapshots/test_ml/test_explain_predict_model_with_options/explain_predict_model_with_options.sql @@ -1 +1 @@ -SELECT * FROM ML.EXPLAIN_PREDICT(MODEL `my_model`, (SELECT * FROM new_data), STRUCT(5 AS `top_k_features`)) +SELECT * FROM ML.EXPLAIN_PREDICT(MODEL `my_model`, (SELECT * FROM new_data), STRUCT(5 AS top_k_features)) diff --git a/tests/unit/core/sql/snapshots/test_ml/test_generate_embedding_model_basic/generate_embedding_model_basic.sql b/tests/unit/core/sql/snapshots/test_ml/test_generate_embedding_model_basic/generate_embedding_model_basic.sql deleted file mode 100644 index 7294f1655f7..00000000000 --- a/tests/unit/core/sql/snapshots/test_ml/test_generate_embedding_model_basic/generate_embedding_model_basic.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM ML.GENERATE_EMBEDDING(MODEL `my_project.my_dataset.my_model`, (SELECT * FROM new_data)) diff --git a/tests/unit/core/sql/snapshots/test_ml/test_generate_embedding_model_with_options/generate_embedding_model_with_options.sql b/tests/unit/core/sql/snapshots/test_ml/test_generate_embedding_model_with_options/generate_embedding_model_with_options.sql deleted file mode 100644 index 3be957079cf..00000000000 --- a/tests/unit/core/sql/snapshots/test_ml/test_generate_embedding_model_with_options/generate_embedding_model_with_options.sql +++ /dev/null @@ -1,5 +0,0 @@ -SELECT * FROM ML.GENERATE_EMBEDDING(MODEL `my_project.my_dataset.my_model`, (SELECT * FROM new_data), STRUCT( - TRUE AS `flatten_json_output`, - 'RETRIEVAL_DOCUMENT' AS `task_type`, - 256 AS `output_dimensionality` -)) diff --git a/tests/unit/core/sql/snapshots/test_ml/test_generate_text_model_basic/generate_text_model_basic.sql b/tests/unit/core/sql/snapshots/test_ml/test_generate_text_model_basic/generate_text_model_basic.sql deleted file mode 100644 index 9d986876448..00000000000 --- a/tests/unit/core/sql/snapshots/test_ml/test_generate_text_model_basic/generate_text_model_basic.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM ML.GENERATE_TEXT(MODEL `my_project.my_dataset.my_model`, (SELECT * FROM new_data)) diff --git a/tests/unit/core/sql/snapshots/test_ml/test_generate_text_model_with_options/generate_text_model_with_options.sql b/tests/unit/core/sql/snapshots/test_ml/test_generate_text_model_with_options/generate_text_model_with_options.sql deleted file mode 100644 index 0ea26747287..00000000000 --- a/tests/unit/core/sql/snapshots/test_ml/test_generate_text_model_with_options/generate_text_model_with_options.sql +++ /dev/null @@ -1,10 +0,0 @@ -SELECT * FROM ML.GENERATE_TEXT(MODEL `my_project.my_dataset.my_model`, (SELECT * FROM new_data), STRUCT( - 0.5 AS `temperature`, - 128 AS `max_output_tokens`, - 20 AS `top_k`, - 0.9 AS `top_p`, - TRUE AS `flatten_json_output`, - ['a', 'b'] AS `stop_sequences`, - TRUE AS `ground_with_google_search`, - 'TYPE' AS `request_type` -)) diff --git a/tests/unit/core/sql/snapshots/test_ml/test_get_insights_model_basic/get_insights_model_basic.sql b/tests/unit/core/sql/snapshots/test_ml/test_get_insights_model_basic/get_insights_model_basic.sql deleted file mode 100644 index a3f2680c179..00000000000 --- a/tests/unit/core/sql/snapshots/test_ml/test_get_insights_model_basic/get_insights_model_basic.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM ML.GET_INSIGHTS(MODEL `my_project.my_dataset.my_model`) diff --git a/tests/unit/core/sql/snapshots/test_ml/test_global_explain_model_with_options/global_explain_model_with_options.sql b/tests/unit/core/sql/snapshots/test_ml/test_global_explain_model_with_options/global_explain_model_with_options.sql index 396648aa1db..1a3baa0c13b 100644 --- a/tests/unit/core/sql/snapshots/test_ml/test_global_explain_model_with_options/global_explain_model_with_options.sql +++ b/tests/unit/core/sql/snapshots/test_ml/test_global_explain_model_with_options/global_explain_model_with_options.sql @@ -1 +1 @@ -SELECT * FROM ML.GLOBAL_EXPLAIN(MODEL `my_model`, STRUCT(TRUE AS `class_level_explain`)) +SELECT * FROM ML.GLOBAL_EXPLAIN(MODEL `my_model`, STRUCT(True AS class_level_explain)) diff --git a/tests/unit/core/sql/snapshots/test_ml/test_predict_model_with_options/predict_model_with_options.sql b/tests/unit/core/sql/snapshots/test_ml/test_predict_model_with_options/predict_model_with_options.sql index e19f39eebba..96c8074e4c1 100644 --- a/tests/unit/core/sql/snapshots/test_ml/test_predict_model_with_options/predict_model_with_options.sql +++ b/tests/unit/core/sql/snapshots/test_ml/test_predict_model_with_options/predict_model_with_options.sql @@ -1 +1 @@ -SELECT * FROM ML.PREDICT(MODEL `my_model`, (SELECT * FROM new_data), STRUCT(TRUE AS `keep_original_columns`)) +SELECT * FROM ML.PREDICT(MODEL `my_model`, (SELECT * FROM new_data), STRUCT(True AS keep_original_columns)) diff --git a/tests/unit/core/sql/snapshots/test_ml/test_transform_model_basic/transform_model_basic.sql b/tests/unit/core/sql/snapshots/test_ml/test_transform_model_basic/transform_model_basic.sql deleted file mode 100644 index e6cedc16477..00000000000 --- a/tests/unit/core/sql/snapshots/test_ml/test_transform_model_basic/transform_model_basic.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM ML.TRANSFORM(MODEL `my_project.my_dataset.my_model`, (SELECT * FROM new_data)) diff --git a/tests/unit/core/sql/test_ml.py b/tests/unit/core/sql/test_ml.py index a03d8cd805a..fe8c1a04d48 100644 --- a/tests/unit/core/sql/test_ml.py +++ b/tests/unit/core/sql/test_ml.py @@ -14,7 +14,6 @@ import pytest -import bigframes.bigquery as bbq import bigframes.core.sql.ml pytest.importorskip("pytest_snapshot") @@ -98,19 +97,6 @@ def test_create_model_list_option(snapshot): snapshot.assert_match(sql, "create_model_list_option.sql") -def test_create_model_hparam_tuning(snapshot): - sql = bigframes.core.sql.ml.create_model_ddl( - model_name="my_model", - options={ - "model_type": "LINEAR_REG", - "learn_rate": bbq.hparam_range(0.0001, 1.0), - "optimizer": bbq.hparam_candidates(["ADAGRAD", "SGD"]), - }, - training_data="SELECT * FROM t", - ) - snapshot.assert_match(sql, "create_model_hparam_tuning.sql") - - def test_evaluate_model_basic(snapshot): sql = bigframes.core.sql.ml.evaluate( model_name="my_project.my_dataset.my_model", @@ -183,61 +169,3 @@ def test_global_explain_model_with_options(snapshot): class_level_explain=True, ) snapshot.assert_match(sql, "global_explain_model_with_options.sql") - - -def test_transform_model_basic(snapshot): - sql = bigframes.core.sql.ml.transform( - model_name="my_project.my_dataset.my_model", - table="SELECT * FROM new_data", - ) - snapshot.assert_match(sql, "transform_model_basic.sql") - - -def test_generate_text_model_basic(snapshot): - sql = bigframes.core.sql.ml.generate_text( - model_name="my_project.my_dataset.my_model", - table="SELECT * FROM new_data", - ) - snapshot.assert_match(sql, "generate_text_model_basic.sql") - - -def test_generate_text_model_with_options(snapshot): - sql = bigframes.core.sql.ml.generate_text( - model_name="my_project.my_dataset.my_model", - table="SELECT * FROM new_data", - temperature=0.5, - max_output_tokens=128, - top_k=20, - top_p=0.9, - flatten_json_output=True, - stop_sequences=["a", "b"], - ground_with_google_search=True, - request_type="TYPE", - ) - snapshot.assert_match(sql, "generate_text_model_with_options.sql") - - -def test_get_insights_model_basic(snapshot): - sql = bigframes.core.sql.ml.get_insights( - model_name="my_project.my_dataset.my_model", - ) - snapshot.assert_match(sql, "get_insights_model_basic.sql") - - -def test_generate_embedding_model_basic(snapshot): - sql = bigframes.core.sql.ml.generate_embedding( - model_name="my_project.my_dataset.my_model", - table="SELECT * FROM new_data", - ) - snapshot.assert_match(sql, "generate_embedding_model_basic.sql") - - -def test_generate_embedding_model_with_options(snapshot): - sql = bigframes.core.sql.ml.generate_embedding( - model_name="my_project.my_dataset.my_model", - table="SELECT * FROM new_data", - flatten_json_output=True, - task_type="RETRIEVAL_DOCUMENT", - output_dimensionality=256, - ) - snapshot.assert_match(sql, "generate_embedding_model_with_options.sql") diff --git a/tests/unit/core/test_bytecode.py b/tests/unit/core/test_bytecode.py deleted file mode 100644 index 036e3f00e8f..00000000000 --- a/tests/unit/core/test_bytecode.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import math - -import pytest - -import bigframes.core.expression as ex -import bigframes.operations as ops -from bigframes.core.bytecode import py_to_expression - - -def test_py_to_expression_simple_arithmetic(): - func = lambda x: x + 1 - expr = py_to_expression(func) - assert expr is not None - - expected = ops.add_op.as_expr(ex.free_var("x"), ex.const(1)) - assert expr == expected - - -def test_py_to_expression_math_function(): - func = lambda x: math.sin(x) - expr = py_to_expression(func) - assert expr is not None - - expected = ops.numeric_ops.sin_op.as_expr(ex.free_var("x")) - assert expr == expected - - -def test_py_to_expression_negation(): - func = lambda x: -x - expr = py_to_expression(func) - assert expr is not None - - expected = ops.numeric_ops.neg_op.as_expr(ex.free_var("x")) - assert expr == expected - - -def test_py_to_expression_comparison(): - func = lambda x, y: x == y - expr = py_to_expression(func) - assert expr is not None - - expected = ops.comparison_ops.eq_op.as_expr(ex.free_var("x"), ex.free_var("y")) - assert expr == expected - - -def test_py_to_expression_unsupported(): - # Control flow or unsupported structures should return None - def func_with_loop(x): - res = 0 - for val in range(int(x)): - res += val - return res - - with pytest.raises(ValueError): - py_to_expression(func_with_loop) - - -global_none_val = None - - -def test_py_to_expression_global_none(): - # Test resolving a global variable explicitly set to None - func = lambda x: x == global_none_val - expr = py_to_expression(func) - assert expr is not None - - expected = ops.comparison_ops.eq_op.as_expr(ex.free_var("x"), ex.const(None)) - assert expr == expected diff --git a/tests/unit/core/test_expression.py b/tests/unit/core/test_expression.py index 68fc3a2b540..4c3d233879f 100644 --- a/tests/unit/core/test_expression.py +++ b/tests/unit/core/test_expression.py @@ -16,11 +16,11 @@ import pytest +from bigframes.core import field import bigframes.core.expression as ex import bigframes.core.identifiers as ids import bigframes.dtypes as dtypes import bigframes.operations as ops -from bigframes.core import field def test_simple_expression_dtype(): @@ -105,7 +105,7 @@ def test_nested_expression_dtypes_are_cached(): def _create_field_bindings( - col_dtypes: typing.Dict[str, dtypes.Dtype], + col_dtypes: typing.Dict[str, dtypes.Dtype] ) -> typing.Dict[ids.ColumnId, field.Field]: return { ids.ColumnId(col): field.Field(ids.ColumnId(col), dtype) diff --git a/tests/unit/core/test_googlesql.py b/tests/unit/core/test_googlesql.py deleted file mode 100644 index e83391e4b33..00000000000 --- a/tests/unit/core/test_googlesql.py +++ /dev/null @@ -1,268 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest.mock as mock - -import pandas as pd - -import bigframes.core.col as col -import bigframes.core.expression as ex -import bigframes.core.global_session -import bigframes.core.googlesql as core_googlesql -import bigframes.series as series -from bigframes.operations import googlesql -from bigframes.testing import mocks - -# Define a test op -_TEST_OP = googlesql.GoogleSqlScalarOp( - "TEST_OP", - args=(googlesql.ArgSpec(), googlesql.ArgSpec()), - signature=lambda *args: None, -) - - -def test_apply_googlesql_scalar_op_expressions(): - # Only expressions - result = core_googlesql.apply_googlesql_scalar_op( - _TEST_OP, - col.col("a"), - col.col("b"), - ) - assert isinstance(result, col.Expression) - - -def test_apply_googlesql_scalar_op_pandas_series_global_session(monkeypatch): - # Setup mock session - session = mocks.create_bigquery_session() - monkeypatch.setattr(bigframes.core.global_session, "_global_session", session) - bigframes.options.bigquery._session_started = True - - # Create a real-ish Series to return from read_pandas - df = mocks.create_dataframe(monkeypatch, session=session, data={"col": [1, 2, 3]}) - bf_series = df["col"] - - # Mock read_pandas on the session - mock_read_pandas = mock.MagicMock(return_value=bf_series) - session.read_pandas = mock_read_pandas # type: ignore - - # Mock _apply_nary_op on Series class to avoid real compilation/execution - mock_apply_nary_op = mock.MagicMock(return_value=bf_series) - monkeypatch.setattr(series.Series, "_apply_nary_op", mock_apply_nary_op) - - pd_series = pd.Series([1, 2, 3]) - - # Call the function with a pandas Series and a literal - result = core_googlesql.apply_googlesql_scalar_op(_TEST_OP, pd_series, 42) - - # Verify read_pandas was called on the global session - mock_read_pandas.assert_called_once_with(pd_series) - - # Verify _apply_nary_op was called on the converted series - mock_apply_nary_op.assert_called_once() - # First arg to _apply_nary_op is the op, second is the processed_args - assert mock_apply_nary_op.call_args[0][0] == _TEST_OP - # processed_args should contain the converted bf_series and the literal 42 - processed_args = mock_apply_nary_op.call_args[0][1] - assert processed_args[0] is bf_series - assert processed_args[1] == 42 - - # Verify result is a Series - assert isinstance(result, series.Series) - - -def test_apply_googlesql_scalar_op_pandas_series_with_bf_series(monkeypatch): - # Setup mock session 1 (global) and session 2 (associated with bf_series) - global_session = mocks.create_bigquery_session(session_id="global") - monkeypatch.setattr( - bigframes.core.global_session, "_global_session", global_session - ) - bigframes.options.bigquery._session_started = True - - bf_session = mocks.create_bigquery_session(session_id="bf_session") - - # Create a bf_series associated with bf_session - df = mocks.create_dataframe( - monkeypatch, session=bf_session, data={"col": [1, 2, 3]} - ) - bf_series = df["col"] - - assert bf_series._session == bf_session - - # Mock read_pandas on both sessions - mock_global_read_pandas = mock.MagicMock() - global_session.read_pandas = mock_global_read_pandas # type: ignore - - mock_bf_read_pandas = mock.MagicMock(return_value=bf_series) - bf_session.read_pandas = mock_bf_read_pandas # type: ignore - - # Mock _apply_nary_op - mock_apply_nary_op = mock.MagicMock(return_value=bf_series) - monkeypatch.setattr(series.Series, "_apply_nary_op", mock_apply_nary_op) - - pd_series = pd.Series([1, 2, 3]) - - # Call with both pandas Series and BigFrames Series - result = core_googlesql.apply_googlesql_scalar_op(_TEST_OP, pd_series, bf_series) - - # Verify read_pandas was called on bf_session, NOT global_session - mock_bf_read_pandas.assert_called_once_with(pd_series) - mock_global_read_pandas.assert_not_called() - - # Verify _apply_nary_op was called - mock_apply_nary_op.assert_called_once() - processed_args = mock_apply_nary_op.call_args[0][1] - # Both arguments to the op should now be BigFrames Series - assert processed_args[0] is bf_series - assert processed_args[1] is bf_series - - assert isinstance(result, series.Series) - - -def test_apply_googlesql_scalar_op_mixed_args(monkeypatch): - session = mocks.create_bigquery_session() - monkeypatch.setattr(bigframes.core.global_session, "_global_session", session) - bigframes.options.bigquery._session_started = True - - df = mocks.create_dataframe(monkeypatch, session=session, data={"col": [1, 2, 3]}) - bf_series = df["col"] - - mock_read_pandas = mock.MagicMock(return_value=bf_series) - session.read_pandas = mock_read_pandas # type: ignore - - mock_apply_nary_op = mock.MagicMock(return_value=bf_series) - monkeypatch.setattr(series.Series, "_apply_nary_op", mock_apply_nary_op) - - pd_series = pd.Series([1, 2, 3]) - expr = col.Expression(ex.const(10)) - - # Call with pandas Series, Expression, and Literal - result = core_googlesql.apply_googlesql_scalar_op(_TEST_OP, pd_series, expr, 42) - - # Verify pandas Series was converted - mock_read_pandas.assert_called_once_with(pd_series) - - # Verify _apply_nary_op was called - mock_apply_nary_op.assert_called_once() - processed_args = mock_apply_nary_op.call_args[0][1] - - # Processed args should be: - # 1. bf_series (converted from pd_series) - # 2. A new Series (projected from the expression onto bf_series' block) - # 3. Literal 42 - assert isinstance(processed_args[0], series.Series) - assert processed_args[0] is bf_series - - assert isinstance(processed_args[1], series.Series) - assert processed_args[1] is not bf_series - - assert processed_args[2] == 42 - - assert isinstance(result, series.Series) - - -def test_apply_googlesql_scalar_op_pandas_series_with_bf_dataframe(monkeypatch): - # Setup mock session 2 (associated with bf_dataframe) - bf_session = mocks.create_bigquery_session(session_id="bf_session") - - # Create a bf_dataframe associated with bf_session - bf_dataframe = mocks.create_dataframe( - monkeypatch, session=bf_session, data={"col": [1, 2, 3]} - ) - bf_series = bf_dataframe["col"] - - # Setup mock session 1 (global) AFTER creating the dataframe - global_session = mocks.create_bigquery_session(session_id="global") - monkeypatch.setattr( - bigframes.core.global_session, "_global_session", global_session - ) - bigframes.options.bigquery._session_started = True - - assert bf_dataframe._session == bf_session - - # Mock read_pandas on both sessions - mock_global_read_pandas = mock.MagicMock() - global_session.read_pandas = mock_global_read_pandas # type: ignore - - mock_bf_read_pandas = mock.MagicMock(return_value=bf_series) - bf_session.read_pandas = mock_bf_read_pandas # type: ignore - - # Mock _apply_nary_op - mock_apply_nary_op = mock.MagicMock(return_value=bf_series) - monkeypatch.setattr(series.Series, "_apply_nary_op", mock_apply_nary_op) - - pd_series = pd.Series([1, 2, 3]) - - # Call with pandas Series and BigFrames DataFrame - result = core_googlesql.apply_googlesql_scalar_op(_TEST_OP, pd_series, bf_dataframe) - - # Verify read_pandas was called on bf_session, NOT global_session - mock_bf_read_pandas.assert_called_once_with(pd_series) - mock_global_read_pandas.assert_not_called() - - # Verify _apply_nary_op was called - mock_apply_nary_op.assert_called_once() - processed_args = mock_apply_nary_op.call_args[0][1] - assert processed_args[0] is bf_series - assert processed_args[1] is bf_dataframe - - assert isinstance(result, series.Series) - - -def test_apply_googlesql_scalar_op_pandas_series_with_bf_index(monkeypatch): - # Setup mock session 2 (associated with bf_index) - bf_session = mocks.create_bigquery_session(session_id="bf_session") - - # Create a bf_dataframe associated with bf_session to get an index - bf_dataframe = mocks.create_dataframe( - monkeypatch, session=bf_session, data={"col": [1, 2, 3]} - ) - bf_index = bf_dataframe.index - bf_series = bf_dataframe["col"] - - # Setup mock session 1 (global) AFTER creating the dataframe - global_session = mocks.create_bigquery_session(session_id="global") - monkeypatch.setattr( - bigframes.core.global_session, "_global_session", global_session - ) - bigframes.options.bigquery._session_started = True - - assert bf_index._session == bf_session - - # Mock read_pandas on both sessions - mock_global_read_pandas = mock.MagicMock() - global_session.read_pandas = mock_global_read_pandas # type: ignore - - mock_bf_read_pandas = mock.MagicMock(return_value=bf_series) - bf_session.read_pandas = mock_bf_read_pandas # type: ignore - - # Mock _apply_nary_op - mock_apply_nary_op = mock.MagicMock(return_value=bf_series) - monkeypatch.setattr(series.Series, "_apply_nary_op", mock_apply_nary_op) - - pd_series = pd.Series([1, 2, 3]) - - # Call with pandas Series and BigFrames Index - result = core_googlesql.apply_googlesql_scalar_op(_TEST_OP, pd_series, bf_index) - - # Verify read_pandas was called on bf_session, NOT global_session - mock_bf_read_pandas.assert_called_once_with(pd_series) - mock_global_read_pandas.assert_not_called() - - # Verify _apply_nary_op was called - mock_apply_nary_op.assert_called_once() - processed_args = mock_apply_nary_op.call_args[0][1] - assert processed_args[0] is bf_series - assert processed_args[1] is bf_index - - assert isinstance(result, series.Series) diff --git a/tests/unit/core/test_groupby.py b/tests/unit/core/test_groupby.py index b23199da331..4bef581b2f7 100644 --- a/tests/unit/core/test_groupby.py +++ b/tests/unit/core/test_groupby.py @@ -18,7 +18,7 @@ import bigframes.core.utils as utils import bigframes.pandas as bpd -import bigframes.testing.utils +from bigframes.testing.utils import assert_series_equal pytest.importorskip("polars") pytest.importorskip("pandas", minversion="2.0.0") @@ -33,7 +33,7 @@ def test_groupby_df_iter_by_key_singular(polars_session): bf_result = bf_group_df.to_pandas() pd_key, pd_result = pd_group assert bf_key == pd_key - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -47,7 +47,7 @@ def test_groupby_df_iter_by_key_list(polars_session): bf_result = bf_group_df.to_pandas() pd_key, pd_result = pd_group assert bf_key == pd_key - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -69,7 +69,7 @@ def test_groupby_df_iter_by_key_list_multiple(polars_session): bf_result = bf_group_df.to_pandas() pd_key, pd_result = pd_group assert bf_key == pd_key - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -85,7 +85,7 @@ def test_groupby_df_iter_by_level_singular(polars_session): bf_result = bf_group_df.to_pandas() pd_key, pd_result = pd_group assert bf_key == pd_key - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -109,7 +109,7 @@ def test_groupby_df_iter_by_level_list_one_item(polars_session): assert bf_key == tuple(pd_key) else: assert bf_key == (pd_key,) - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -131,7 +131,7 @@ def test_groupby_df_iter_by_level_list_multiple(polars_session): bf_result = bf_group_df.to_pandas() pd_key, pd_result = pd_group assert bf_key == pd_key - bigframes.testing.utils.assert_frame_equal( + pandas.testing.assert_frame_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -149,7 +149,7 @@ def test_groupby_series_iter_by_level_singular(polars_session): bf_result = bf_group_series.to_pandas() pd_key, pd_result = pd_group assert bf_key == pd_key - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -175,7 +175,7 @@ def test_groupby_series_iter_by_level_list_one_item(polars_session): assert bf_key == tuple(pd_key) else: assert bf_key == (pd_key,) - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -199,7 +199,7 @@ def test_groupby_series_iter_by_level_list_multiple(polars_session): bf_result = bf_group_df.to_pandas() pd_key, pd_result = pd_group assert bf_key == pd_key - bigframes.testing.utils.assert_series_equal( + pandas.testing.assert_series_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -218,7 +218,7 @@ def test_groupby_series_iter_by_series(polars_session): bf_result = bf_group_series.to_pandas() pd_key, pd_result = pd_group assert bf_key == pd_key - bigframes.testing.utils.assert_series_equal( + assert_series_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -237,7 +237,7 @@ def test_groupby_series_iter_by_series_list_one_item(polars_session): bf_result = bf_group_series.to_pandas() pd_key, pd_result = pd_group assert bf_key == pd_key - bigframes.testing.utils.assert_series_equal( + assert_series_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) @@ -259,6 +259,6 @@ def test_groupby_series_iter_by_series_list_multiple(polars_session): bf_result = bf_group_series.to_pandas() pd_key, pd_result = pd_group assert bf_key == pd_key - bigframes.testing.utils.assert_series_equal( + assert_series_equal( bf_result, pd_result, check_dtype=False, check_index_type=False ) diff --git a/tests/unit/core/test_pyformat.py b/tests/unit/core/test_pyformat.py index 239a59237f6..db7cedba8f1 100644 --- a/tests/unit/core/test_pyformat.py +++ b/tests/unit/core/test_pyformat.py @@ -62,72 +62,6 @@ def test_parse_fields(sql_template: str, expected: List[str]): assert fields == expected -def test_get_error_context_at_pos_invalid_pos(): - assert pyformat.get_error_context_at_pos("SELECT 1", -1) == "" - assert pyformat.get_error_context_at_pos("SELECT 1", 100) == "" - - -def test_get_error_context_at_pos_single_line(): - sql = "SELECT {foo}" - # pos of '{' is 7 - context = pyformat.get_error_context_at_pos(sql, 7) - expected = " 1: SELECT {foo}\n ^" - assert context == expected - - -def test_get_error_context_at_pos_multi_line(): - sql = "SELECT 1\nFROM my_table\nWHERE col = {foo}\nAND active = True\nLIMIT 10" - # Lines: - # 1: SELECT 1 (len 9 including \n) - # 2: FROM my_table (len 14 including \n) -> total 23 - # 3: WHERE col = {foo} -> '{' is at 23 + 12 = 35 - - context = pyformat.get_error_context_at_pos(sql, 35) - expected = ( - " 1: SELECT 1\n" - " 2: FROM my_table\n" - " 3: WHERE col = {foo}\n" - " ^\n" - " 4: AND active = True\n" - " 5: LIMIT 10" - ) - assert context == expected - - -def test_get_error_context_at_pos_multi_line_limits(): - # Test that it only shows at most 2 lines before and 2 lines after - sql = ( - "LINE 1\n" - "LINE 2\n" - "LINE 3\n" - "LINE 4\n" - "LINE 5\n" - "TARGET {foo}\n" - "LINE 7\n" - "LINE 8\n" - "LINE 9\n" - "LINE 10" - ) - # Line lengths: - # LINE 1\n (7) - # LINE 2\n (7) -> 14 - # LINE 3\n (7) -> 21 - # LINE 4\n (7) -> 28 - # LINE 5\n (7) -> 35 - # TARGET {foo}\n -> '{' is at 35 + 7 = 42 - - context = pyformat.get_error_context_at_pos(sql, 42) - expected = ( - " 4: LINE 4\n" - " 5: LINE 5\n" - " 6: TARGET {foo}\n" - " ^\n" - " 7: LINE 7\n" - " 8: LINE 8" - ) - assert context == expected - - def test_pyformat_with_unsupported_type_raises_typeerror(session): pyformat_args = {"my_object": object()} sql = "SELECT {my_object}" @@ -136,75 +70,13 @@ def test_pyformat_with_unsupported_type_raises_typeerror(session): pyformat.pyformat(sql, pyformat_args=pyformat_args, session=session) -def test_pyformat_with_missing_variable_raises_valueerror(session): +def test_pyformat_with_missing_variable_raises_keyerror(session): pyformat_args: Dict[str, Any] = {} sql = "SELECT {my_object}" - with pytest.raises(ValueError) as exc_info: - pyformat.pyformat(sql, pyformat_args=pyformat_args, session=session) - - err_msg = str(exc_info.value) - assert "Undetected variable 'my_object' in SQL template" in err_msg - assert "Did you mean to escape '{' and '}'" in err_msg - assert " 1: SELECT {my_object}" in err_msg - assert " ^" in err_msg - - -def test_pyformat_with_unescaped_braces_raises_valueerror_with_context(session): - pyformat_args = {"active": True} - sql = """SELECT * FROM my_table -WHERE json_col = { "generation_config": { "temperature": 0.9 } } -AND active = {active} -""" - - with pytest.raises(ValueError) as exc_info: + with pytest.raises(KeyError, match="my_object"): pyformat.pyformat(sql, pyformat_args=pyformat_args, session=session) - err_msg = str(exc_info.value) - assert "Undetected variable ' \"generation_config\"' in SQL template" in err_msg - assert "Did you mean to escape '{' and '}'" in err_msg - # The triple quote string starts with SELECT immediately, so lines are: - # 1: SELECT * FROM my_table - # 2: WHERE json_col = { "generation_config": { "temperature": 0.9 } } - # 3: AND active = {active} - assert " 1: SELECT * FROM my_table" in err_msg - assert ( - ' 2: WHERE json_col = { "generation_config": { "temperature": 0.9 } }' - in err_msg - ) - assert " ^" in err_msg - assert " 3: AND active = {active}" in err_msg - - -@pytest.mark.parametrize( - ("sql_template", "expected_error"), - ( - pytest.param( - "SELECT {foo", - "expected '}' before end of string", - id="missing_closing_brace", - ), - pytest.param( - "SELECT foo}", - "Single '}' encountered in format string", - id="missing_opening_brace", - ), - ), -) -def test_pyformat_with_malformed_template_raises_valueerror( - session, sql_template: str, expected_error: str -): - pyformat_args: Dict[str, Any] = {} - - # Case 1: Single '{' (unmatched) - with pytest.raises(ValueError) as exc_info: - pyformat.pyformat(sql_template, pyformat_args=pyformat_args, session=session) - - error_message = str(exc_info.value) - assert "Failed to parse SQL template" in error_message - assert "Did you mean to escape '{' and '}'" in error_message - assert expected_error in error_message - def test_pyformat_with_no_variables(session): pyformat_args: Dict[str, Any] = {} @@ -628,15 +500,6 @@ def test_pyformat_with_query_string_replaces_variables(session): ), "SELECT * FROM `ListedProject`.`ListedDataset`.`ListedTable`", ), - ( - google.cloud.bigquery.TableReference( - google.cloud.bigquery.DatasetReference( - "my-project", "my-catalog.my-namespace" - ), - "my-table", - ), - "SELECT * FROM `my-project`.`my-catalog`.`my-namespace`.`my-table`", - ), ), ) def test_pyformat_with_table_replaces_variables(table, expected_sql, session=session): @@ -648,51 +511,3 @@ def test_pyformat_with_table_replaces_variables(table, expected_sql, session=ses sql = "SELECT * FROM {table}" got_sql = pyformat.pyformat(sql, pyformat_args=pyformat_args, session=session) assert got_sql == expected_sql - - -def test_pyformat_with_bigframes_dataframe_biglake_table(session): - # Create a real BigFrames DataFrame that points to a BigLake table. - import bigframes.core.array_value as array_value - import bigframes.core.blocks as blocks - import bigframes.core.bq_data as bq_data - import bigframes.dataframe - - # Define the BigLake table - project_id = "my-project" - catalog_id = "my-catalog" - namespace_id = "my-namespace" - table_id = "my-table" - schema = (google.cloud.bigquery.SchemaField("col", "INTEGER"),) - - biglake_table = bq_data.BiglakeIcebergTable( - project_id=project_id, - catalog_id=catalog_id, - namespace_id=namespace_id, - table_id=table_id, - physical_schema=schema, - cluster_cols=(), - metadata=bq_data.TableMetadata( - location=bq_data.BigQueryRegion("us-central1"), - type="TABLE", - ), - ) - - # ArrayValue.from_table is what read_gbq uses. - av = array_value.ArrayValue.from_table(biglake_table, session) - block = blocks.Block(av, index_columns=[], column_labels=["col"]) - df = bigframes.dataframe.DataFrame(block) - - pyformat_args = {"df": df} - sql = "SELECT * FROM {df}" - - got_sql = pyformat.pyformat(sql, pyformat_args=pyformat_args, session=session) - - # For BigLake, we now expect a SUBQUERY, not a view reference. - # The subquery should have correctly quoted 4-part ID. - assert "SELECT" in got_sql - assert project_id in got_sql - assert catalog_id in got_sql - assert namespace_id in got_sql - assert table_id in got_sql - assert got_sql.startswith("SELECT * FROM (SELECT") - assert got_sql.endswith(")") diff --git a/tests/unit/core/test_sql.py b/tests/unit/core/test_sql.py index 04ebb28764d..17da3008fc4 100644 --- a/tests/unit/core/test_sql.py +++ b/tests/unit/core/test_sql.py @@ -12,9 +12,128 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime +import decimal +import re + +import pytest +import shapely.geometry # type: ignore + from bigframes.core import sql +@pytest.mark.parametrize( + ("value", "expected_pattern"), + ( + # Try to have some literals for each scalar data type: + # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types + (None, "NULL"), + # TODO: support ARRAY type (possibly another method?) + (True, "True"), + (False, "False"), + ( + b"\x01\x02\x03ABC", + re.escape(r"b'\x01\x02\x03ABC'"), + ), + ( + datetime.date(2025, 1, 1), + re.escape("DATE('2025-01-01')"), + ), + ( + datetime.datetime(2025, 1, 2, 3, 45, 6, 789123), + re.escape("DATETIME('2025-01-02T03:45:06.789123')"), + ), + ( + shapely.geometry.Point(0, 1), + r"ST_GEOGFROMTEXT\('POINT \(0[.]?0* 1[.]?0*\)'\)", + ), + # TODO: INTERVAL type (e.g. from dateutil.relativedelta) + # TODO: JSON type (TBD what Python object that would correspond to) + (123, re.escape("123")), + (decimal.Decimal("123.75"), re.escape("CAST('123.75' AS NUMERIC)")), + # TODO: support BIGNUMERIC by looking at precision/scale of the DECIMAL + (123.75, re.escape("123.75")), + # TODO: support RANGE type + ("abc", re.escape("'abc'")), + # TODO: support STRUCT type (possibly another method?) + ( + datetime.time(12, 34, 56, 789123), + re.escape("TIME(DATETIME('1970-01-01 12:34:56.789123'))"), + ), + ( + datetime.datetime( + 2025, 1, 2, 3, 45, 6, 789123, tzinfo=datetime.timezone.utc + ), + re.escape("TIMESTAMP('2025-01-02T03:45:06.789123+00:00')"), + ), + ), +) +def test_simple_literal(value, expected_pattern): + got = sql.simple_literal(value) + assert re.match(expected_pattern, got) is not None + + +@pytest.mark.parametrize( + ("value", "expected_pattern"), + ( + # Try to have some list of literals for each scalar data type: + # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types + ([None, None], re.escape("[NULL, NULL]")), + ([True, False], re.escape("[True, False]")), + ( + [b"\x01\x02\x03ABC", b"\x01\x02\x03ABC"], + re.escape("[b'\\x01\\x02\\x03ABC', b'\\x01\\x02\\x03ABC']"), + ), + ( + [datetime.date(2025, 1, 1), datetime.date(2025, 1, 1)], + re.escape("[DATE('2025-01-01'), DATE('2025-01-01')]"), + ), + ( + [datetime.datetime(2025, 1, 2, 3, 45, 6, 789123)], + re.escape("[DATETIME('2025-01-02T03:45:06.789123')]"), + ), + ( + [shapely.geometry.Point(0, 1), shapely.geometry.Point(0, 2)], + r"\[ST_GEOGFROMTEXT\('POINT \(0[.]?0* 1[.]?0*\)'\), ST_GEOGFROMTEXT\('POINT \(0[.]?0* 2[.]?0*\)'\)\]", + ), + # TODO: INTERVAL type (e.g. from dateutil.relativedelta) + # TODO: JSON type (TBD what Python object that would correspond to) + ([123, 456], re.escape("[123, 456]")), + ( + [decimal.Decimal("123.75"), decimal.Decimal("456.78")], + re.escape("[CAST('123.75' AS NUMERIC), CAST('456.78' AS NUMERIC)]"), + ), + # TODO: support BIGNUMERIC by looking at precision/scale of the DECIMAL + ([123.75, 456.78], re.escape("[123.75, 456.78]")), + # TODO: support RANGE type + (["abc", "def"], re.escape("['abc', 'def']")), + # TODO: support STRUCT type (possibly another method?) + ( + [datetime.time(12, 34, 56, 789123), datetime.time(11, 25, 56, 789123)], + re.escape( + "[TIME(DATETIME('1970-01-01 12:34:56.789123')), TIME(DATETIME('1970-01-01 11:25:56.789123'))]" + ), + ), + ( + [ + datetime.datetime( + 2025, 1, 2, 3, 45, 6, 789123, tzinfo=datetime.timezone.utc + ), + datetime.datetime( + 2025, 2, 1, 4, 45, 6, 789123, tzinfo=datetime.timezone.utc + ), + ], + re.escape( + "[TIMESTAMP('2025-01-02T03:45:06.789123+00:00'), TIMESTAMP('2025-02-01T04:45:06.789123+00:00')]" + ), + ), + ), +) +def test_simple_literal_w_list(value: list, expected_pattern: str): + got = sql.simple_literal(value) + assert re.match(expected_pattern, got) is not None + + def test_create_vector_search_sql_simple(): result_query = sql.create_vector_search_sql( sql_string="SELECT embedding FROM my_embeddings_table WHERE id = 1", @@ -61,6 +180,6 @@ def test_create_vector_search_sql_all_named_parameters(): query_column_to_search => 'another_embedding_column', top_k=> 10, distance_type => 'cosine', -options => '{"fraction_lists_to_search": 0.1, "use_brute_force": false}') +options => '{\\"fraction_lists_to_search\\": 0.1, \\"use_brute_force\\": false}') """ ) diff --git a/tests/unit/core/tools/test_bigquery_schema.py b/tests/unit/core/tools/test_bigquery_schema.py index 2b6693c13e6..aed8ae03231 100644 --- a/tests/unit/core/tools/test_bigquery_schema.py +++ b/tests/unit/core/tools/test_bigquery_schema.py @@ -1,5 +1,5 @@ -import pytest from google.cloud import bigquery +import pytest from bigframes.core.tools import bigquery_schema diff --git a/tests/unit/display/test_anywidget.py b/tests/unit/display/test_anywidget.py index 25f19cf495c..252ba8100e6 100644 --- a/tests/unit/display/test_anywidget.py +++ b/tests/unit/display/test_anywidget.py @@ -16,7 +16,6 @@ import unittest.mock as mock import pandas as pd -import pyarrow as pa import pytest import bigframes @@ -25,17 +24,14 @@ pytest.importorskip("anywidget") pytest.importorskip("traitlets") -import bigframes.dataframe -import bigframes.dtypes -import bigframes.series -from bigframes.display.anywidget import TableWidget - def test_navigation_to_invalid_page_resets_to_valid_page_without_deadlock(): """ Given a widget on a page beyond available data, when navigating, then it should reset to the last valid page without deadlock. """ + from bigframes.display.anywidget import TableWidget + mock_df = mock.create_autospec(bigframes.dataframe.DataFrame, instance=True) mock_df.columns = ["col1"] mock_df.dtypes = {"col1": "object"} @@ -45,13 +41,11 @@ def test_navigation_to_invalid_page_resets_to_valid_page_without_deadlock(): mock_df._block = mock_block # We mock _initial_load to avoid complex setup - with ( - mock.patch.object(TableWidget, "_initial_load"), - bigframes.option_context( - "display.render_mode", "anywidget", "display.max_rows", 10 - ), - ): - widget = TableWidget(mock_df) + with mock.patch.object(TableWidget, "_initial_load"): + with bigframes.option_context( + "display.repr_mode", "anywidget", "display.max_rows", 10 + ): + widget = TableWidget(mock_df) # Simulate "loaded data but unknown total rows" state widget.page_size = 10 @@ -88,6 +82,8 @@ def handler(signum, frame): def test_css_contains_dark_mode_selectors(): """Test that the CSS for dark mode is loaded with all required selectors.""" + from bigframes.display.anywidget import TableWidget + mock_df = mock.create_autospec(bigframes.dataframe.DataFrame, instance=True) # mock_df.columns and mock_df.dtypes are needed for __init__ mock_df.columns = ["col1"] @@ -132,7 +128,9 @@ def mock_df(): def test_sorting_single_column(mock_df): """Test that the widget can be sorted by a single column.""" - with bigframes.option_context("display.render_mode", "anywidget"): + from bigframes.display.anywidget import TableWidget + + with bigframes.option_context("display.repr_mode", "anywidget"): widget = TableWidget(mock_df) # Verify initial state @@ -149,7 +147,9 @@ def test_sorting_single_column(mock_df): def test_sorting_multi_column(mock_df): """Test that the widget can be sorted by multiple columns.""" - with bigframes.option_context("display.render_mode", "anywidget"): + from bigframes.display.anywidget import TableWidget + + with bigframes.option_context("display.repr_mode", "anywidget"): widget = TableWidget(mock_df) # Apply multi-column sort @@ -163,7 +163,9 @@ def test_sorting_multi_column(mock_df): def test_page_size_change_resets_sort(mock_df): """Test that changing the page size resets the sorting.""" - with bigframes.option_context("display.render_mode", "anywidget"): + from bigframes.display.anywidget import TableWidget + + with bigframes.option_context("display.repr_mode", "anywidget"): widget = TableWidget(mock_df) # Set sort state @@ -177,367 +179,3 @@ def test_page_size_change_resets_sort(mock_df): # to_pandas_batches called again (reset) assert mock_df.to_pandas_batches.call_count >= 2 - - -def test_cell_execution_count_propagation(mock_df): - """Test that the captured cell_execution_count is propagated to to_pandas_batches.""" - with ( - mock.patch("bigframes.core.utils.get_ipython_execution_count", return_value=42), - bigframes.option_context("display.render_mode", "anywidget"), - ): - widget = TableWidget(mock_df) - - assert widget._cell_execution_count == 42 - - mock_df.to_pandas_batches.assert_called_with( - page_size=widget.page_size, - cell_execution_count=42, - ) - - -def test_json_column_converted_to_string_for_display(polars_session): - series = bigframes.series.Series( - ['{"a": 1}', '{"b": 2}'], - dtype=bigframes.dtypes.JSON_DTYPE, - session=polars_session, - ) - df = series.to_frame("col_json") - - result = df._prepare_display_df() - - assert result["col_json"].dtype == bigframes.dtypes.STRING_DTYPE - - -def test_struct_column_with_nested_json_converted_to_string_for_display( - polars_session, -): - if not hasattr(pa, "json_"): - pytest.skip(reason=f"pyarrow=={pa.__version__} does not support json_") - - # Arrange - json_type = pa.json_(storage_type=pa.utf8()) - json_data = pa.array(['{"a": 1}'], type=json_type) - string_data = pa.array(["hello"], type=pa.string()) - struct_data = pa.StructArray.from_arrays( - [string_data, json_data], names=["field1", "field2"] - ) - nested_data = pa.table([struct_data], names=["nested"]) - df = polars_session.read_arrow(nested_data) - exploded = df["nested"].struct.explode() - # Ensure that we are actually using the JSON dtype in this test. - assert exploded["field2"].dtype == bigframes.dtypes.JSON_DTYPE - - # Act - result = df._prepare_display_df() - - # Assert - assert result["nested"].dtype == bigframes.dtypes.STRING_DTYPE - - -@pytest.fixture -def mock_df_deferred(): - with mock.patch("bigframes.display.anywidget._ANYWIDGET_INSTALLED", True): - df = mock.Mock(spec=bigframes.dataframe.DataFrame) - df.shape = (100, 4) - df.columns = ["A", "B", "C", "D"] - df.dtypes = { - "A": bigframes.dtypes.INT_DTYPE, - "B": bigframes.dtypes.STRING_DTYPE, - "C": bigframes.dtypes.FLOAT_DTYPE, - "D": bigframes.dtypes.BOOL_DTYPE, - } - - df.to_pandas_batches.return_value = iter( - [pd.DataFrame({"A": [1], "B": ["a"], "C": [1.0], "D": [True]})] - ) - - df.sort_values.return_value = df - - df._block = mock.Mock() - df._block.has_index = False - df._prepare_display_df.return_value = df - - yield df - - -@pytest.fixture -def mock_deferred_df(): - from bigframes.session.deferred import DeferredBigQueryDataFrame - - with mock.patch("bigframes.display.anywidget._ANYWIDGET_INSTALLED", True): - # We create a mock that subclasses DeferredBigQueryDataFrame so isinstance passes - class MockDeferredBigQueryDataFrame(DeferredBigQueryDataFrame): - def __init__(self): - pass - - df = mock.MagicMock(spec=MockDeferredBigQueryDataFrame) - df.__class__ = DeferredBigQueryDataFrame # type: ignore[assignment] - yield df - - -def test_init_raises_if_anywidget_not_installed(): - with ( - mock.patch("bigframes.display.anywidget._ANYWIDGET_INSTALLED", False), - pytest.raises(ImportError), - ): - from bigframes.display.anywidget import TableWidget - - TableWidget(mock.Mock()) - - -def test_init_initializes_attributes(mock_df_deferred): - from bigframes.display.anywidget import TableWidget - - with ( - bigframes.option_context("display.render_mode", "anywidget"), - mock.patch.object(TableWidget, "_initial_load"), - ): - widget = TableWidget(mock_df_deferred) - - assert widget._dataframe is mock_df_deferred - assert widget.page == 0 - assert widget.page_size > 0 - assert widget.orderable_columns == [ - "A", - "B", - "C", - "D", - ] - - -def test_init_calls_initial_load(mock_df_deferred): - from bigframes.display.anywidget import TableWidget - - with mock.patch.object(TableWidget, "_initial_load") as mock_load: - TableWidget(mock_df_deferred) - mock_load.assert_called_once() - - -def test_validate_page_clamping(mock_df_deferred): - from bigframes.display.anywidget import TableWidget - - with mock.patch.object(TableWidget, "_initial_load"): - widget = TableWidget(mock_df_deferred) - widget.row_count = 100 - widget.page_size = 10 - - widget.page = 5 - assert widget.page == 5 - - with pytest.raises(ValueError): - widget.page = -1 - - widget.page = 100 - assert widget.page == 9 - - -def test_validate_page_size(mock_df_deferred): - from bigframes.display.anywidget import TableWidget - - with ( - bigframes.option_context("display.render_mode", "anywidget"), - mock.patch.object(TableWidget, "_initial_load"), - ): - widget = TableWidget(mock_df_deferred) - - widget.page_size = 50 - assert widget.page_size == 50 - - original_size = widget.page_size - widget.page_size = -5 - assert widget.page_size == original_size - - widget.page_size = 10000 - assert widget.page_size == 1000 - - -def test_page_size_change_resets_page_and_sort(mock_df_deferred): - from bigframes.display.anywidget import TableWidget - - with mock.patch.object(TableWidget, "_initial_load"): - widget = TableWidget(mock_df_deferred) - widget._initial_load_complete = True - widget.page = 5 - widget.sort_context = [{"column": "A", "ascending": True}] - - widget.page_size = 20 - - assert widget.page == 0 - assert widget.sort_context == [] - - -def test_page_size_change_resets_batches(mock_df_deferred): - from bigframes.display.anywidget import TableWidget - - with mock.patch.object(TableWidget, "_initial_load"): - widget = TableWidget(mock_df_deferred) - widget._initial_load_complete = True - - widget.page_size = 50 - - mock_df_deferred.to_pandas_batches.assert_called() - - -def test_sort_change_resets_batches(mock_df_deferred): - from bigframes.display.anywidget import TableWidget - - with ( - bigframes.option_context("display.render_mode", "anywidget"), - mock.patch.object(TableWidget, "_initial_load"), - ): - widget = TableWidget(mock_df_deferred) - widget._initial_load_complete = True - - mock_df_deferred.to_pandas_batches.reset_mock() - - widget.sort_context = [{"column": "B", "ascending": False}] - - assert mock_df_deferred.to_pandas_batches.call_count >= 1 - - -def test_deferred_mode_initialization(mock_deferred_df): - from bigframes.display.anywidget import TableWidget - - with mock.patch.object(TableWidget, "_initial_load") as mock_load: - widget = TableWidget(mock_deferred_df) - - assert widget.is_deferred_mode is True - mock_load.assert_not_called() - - -def test_deferred_mode_execution(mock_deferred_df, mock_df_deferred): - from bigframes.display.anywidget import TableWidget - - mock_deferred_df.execute.return_value = mock_df_deferred - - widget = TableWidget(mock_deferred_df) - - assert widget.is_deferred_mode is True - - import bigframes - - with bigframes.option_context( - "display.render_mode", bigframes.options.display.render_mode - ): - widget.start_execution = True - - thread = getattr(widget, "_execution_thread", None) - if thread is not None: - thread.join(timeout=5) - - mock_deferred_df.execute.assert_called_once() - mock_df_deferred.to_pandas_batches.assert_called_once() - assert widget.is_deferred_mode is False - - -def test_deferred_mode_execution_updates_table_html(mock_deferred_df, mock_df_deferred): - from bigframes.display.anywidget import TableWidget - - mock_deferred_df.execute.return_value = mock_df_deferred - - batches = mock.MagicMock() - batch_df = pd.DataFrame({"A": [1], "B": ["a"], "C": [1.0], "D": [True]}) - batches.__iter__.return_value = iter([batch_df]) - batches.total_rows = 1 - mock_df_deferred.to_pandas_batches.return_value = batches - - with bigframes.option_context("display.render_mode", "anywidget"): - widget = TableWidget(mock_deferred_df) - widget.is_deferred_mode = True - widget._deferred_dataframe = mock_deferred_df - assert widget.table_html == "" - - widget.start_execution = True - thread = getattr(widget, "_execution_thread", None) - if thread is not None: - thread.join(timeout=5) - - assert widget.is_deferred_mode is False - assert widget.table_html != "" - assert "table" in widget.table_html - - -def test_deferred_mode_execution_error(mock_deferred_df): - from bigframes.display.anywidget import TableWidget - - mock_deferred_df.execute.side_effect = RuntimeError("Query Failed") - - with mock.patch.object(TableWidget, "_initial_load"): - widget = TableWidget(mock_deferred_df) - - import bigframes - - with bigframes.option_context( - "display.render_mode", bigframes.options.display.render_mode - ): - widget.start_execution = True - - thread = getattr(widget, "_execution_thread", None) - if thread is not None: - thread.join(timeout=5) - - assert widget.is_deferred_mode is True - assert widget._error_message == "Query Failed" - - -def test_deferred_mode_execution_does_not_reset_page_on_navigation( - mock_deferred_df, mock_df_deferred -): - from bigframes.display.anywidget import TableWidget - - mock_deferred_df.execute.return_value = mock_df_deferred - - batches = mock.MagicMock() - batch_df = pd.DataFrame({"A": [1], "B": ["a"], "C": [1.0], "D": [True]}) - batches.__iter__.return_value = iter([batch_df]) - batches.total_rows = 50 - mock_df_deferred.to_pandas_batches.return_value = batches - - with bigframes.option_context("display.render_mode", "anywidget"): - widget = TableWidget(mock_deferred_df) - widget.page_size = 10 - widget.start_execution = True - - thread = getattr(widget, "_execution_thread", None) - if thread is not None: - thread.join(timeout=5) - - assert widget.page == 0 - widget.page = 1 - assert widget.page == 1 - - -def test_deferred_mode_execution_in_colab(mock_deferred_df, mock_df_deferred): - import sys - - from bigframes.display.anywidget import TableWidget - - mock_deferred_df.execute.return_value = mock_df_deferred - - batches = mock.MagicMock() - batch_df = pd.DataFrame({"A": [1], "B": ["a"], "C": [1.0], "D": [True]}) - batches.__iter__.return_value = iter([batch_df]) - batches.total_rows = 1 - mock_df_deferred.to_pandas_batches.return_value = batches - - with ( - mock.patch.dict(sys.modules, {"google.colab": mock.MagicMock()}), - bigframes.option_context("display.render_mode", "anywidget"), - ): - widget = TableWidget(mock_deferred_df) - widget.is_deferred_mode = True - - widget.start_execution = True - - thread = getattr(widget, "_execution_thread", None) - if thread is not None: - thread.join(timeout=5) - - assert widget.is_deferred_mode is True - assert widget.table_html == "" - - # Simulate frontend ping callback - widget.ping = 1 - - assert widget.is_deferred_mode is False - assert widget.table_html != "" diff --git a/tests/unit/display/test_html.py b/tests/unit/display/test_html.py index 239033861a4..35a74d098ae 100644 --- a/tests/unit/display/test_html.py +++ b/tests/unit/display/test_html.py @@ -13,7 +13,6 @@ # limitations under the License. import datetime -from unittest.mock import Mock, patch import pandas as pd import pyarrow as pa @@ -186,37 +185,3 @@ def test_render_html_max_columns_truncation(): assert "col_8" not in html assert "col_9" in html assert "..." in html - - -def test_repr_mimebundle_head(): - mock_df = Mock() - mock_df.columns = ["col1"] - - mock_df._prepare_display_df.return_value = mock_df - - # Mock the call to retrieve_repr_request_results - pandas_df = pd.DataFrame({"col1": [1, 2, 3]}) - mock_df._block.retrieve_repr_request_results.return_value = ( - pandas_df, - 3, - Mock(), # query_job - ) - - # Mock _get_obj_metadata - with ( - patch("bigframes.display.html._get_obj_metadata", return_value=(False, False)), - patch( - "bigframes.display.html.create_html_representation", return_value="" - ) as mock_create_html, - patch( - "bigframes.display.plaintext.create_text_representation", - return_value="text", - ) as mock_create_text, - ): - bundle = bf_html.repr_mimebundle_head(mock_df) - - assert bundle == {"text/html": "", "text/plain": "text"} - mock_df._prepare_display_df.assert_called_once() - mock_df._block.retrieve_repr_request_results.assert_called_once() - mock_create_html.assert_called_once() - mock_create_text.assert_called_once() diff --git a/tests/unit/display/test_render_mode.py b/tests/unit/display/test_render_mode.py deleted file mode 100644 index 478bfd30eaf..00000000000 --- a/tests/unit/display/test_render_mode.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest.mock as mock - -import pytest - -import bigframes.display.html as bf_html -import bigframes.pandas as bpd - - -def test_render_mode_options(): - assert bpd.options.display.render_mode == "html" - - with bpd.option_context("display.render_mode", "plaintext"): - assert bpd.options.display.render_mode == "plaintext" - - with bpd.option_context("display.render_mode", "html"): - assert bpd.options.display.render_mode == "html" - - with bpd.option_context("display.render_mode", "anywidget"): - assert bpd.options.display.render_mode == "anywidget" - - -def test_repr_mimebundle_selection_logic(): - mock_obj = mock.Mock() - - # Mocking dependencies - with ( - mock.patch("bigframes.display.html.repr_mimebundle_head") as mock_head, - mock.patch("bigframes.display.html.get_anywidget_bundle") as mock_anywidget, - mock.patch("bigframes.display.html.repr_mimebundle_deferred") as mock_deferred, - ): - mock_head.side_effect = lambda obj: {"text/plain": "plain", "text/html": "html"} - mock_anywidget.return_value = ( - { - "application/vnd.jupyter.widget-view+json": {}, - "text/plain": "plain", - "text/html": "html", - }, - {}, - ) - mock_deferred.return_value = {"text/plain": "deferred"} - - # Test deferred repr_mode when anywidget is available - with bpd.option_context("display.repr_mode", "deferred"): - bundle = bf_html.repr_mimebundle(mock_obj) - assert "application/vnd.jupyter.widget-view+json" in bundle[0] - mock_anywidget.assert_called_once() - mock_deferred.assert_not_called() - - mock_anywidget.reset_mock() - - # Test fallback to static deferred repr when anywidget fails - mock_anywidget.side_effect = Exception("Anywidget failed") - with ( - bpd.option_context("display.repr_mode", "deferred"), - pytest.warns(UserWarning, match="Anywidget mode is not available"), - ): - bundle = bf_html.repr_mimebundle(mock_obj) - assert bundle == {"text/plain": "deferred"} - mock_deferred.assert_called_once() - - mock_anywidget.side_effect = None - mock_deferred.reset_mock() - mock_anywidget.reset_mock() - - # Test plaintext render_mode - with bpd.option_context("display.render_mode", "plaintext"): - bundle = bf_html.repr_mimebundle(mock_obj) - assert "text/plain" in bundle - assert "text/html" not in bundle - mock_head.assert_called_once() - - mock_head.reset_mock() - - # Test html render_mode - with bpd.option_context("display.render_mode", "html"): - bundle = bf_html.repr_mimebundle(mock_obj) - assert "text/plain" in bundle - assert "text/html" in bundle - mock_head.assert_called_once() - - mock_head.reset_mock() - - # Test anywidget render_mode - with bpd.option_context("display.render_mode", "anywidget"): - bundle = bf_html.repr_mimebundle(mock_obj) - assert "application/vnd.jupyter.widget-view+json" in bundle[0] - mock_anywidget.assert_called_once() - mock_head.assert_not_called() - - mock_anywidget.reset_mock() - - # Test anywidget repr_mode (backward compatibility) - with bpd.option_context("display.repr_mode", "anywidget"): - bundle = bf_html.repr_mimebundle(mock_obj) - assert "application/vnd.jupyter.widget-view+json" in bundle[0] - mock_anywidget.assert_called_once() - mock_head.assert_not_called() - - mock_anywidget.reset_mock() - - # Test default render_mode (should be "html") - bundle = bf_html.repr_mimebundle(mock_obj) - assert "text/plain" in bundle - assert "text/html" in bundle - mock_head.assert_called_once() - mock_anywidget.assert_not_called() diff --git a/tests/unit/extensions/__init__.py b/tests/unit/extensions/__init__.py deleted file mode 100644 index 58d482ea386..00000000000 --- a/tests/unit/extensions/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/unit/extensions/bigframes/__init__.py b/tests/unit/extensions/bigframes/__init__.py deleted file mode 100644 index 58d482ea386..00000000000 --- a/tests/unit/extensions/bigframes/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/unit/extensions/bigframes/test_series_accessor.py b/tests/unit/extensions/bigframes/test_series_accessor.py deleted file mode 100644 index 4c74b60a1a0..00000000000 --- a/tests/unit/extensions/bigframes/test_series_accessor.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import cast -from unittest.mock import MagicMock, patch - -import pytest - -import bigframes.series as series -from bigframes.testing import mocks - - -def test_bigframes_series_has_accessor(monkeypatch: pytest.MonkeyPatch): - # Arrange - from bigframes.extensions.bigframes.series_accessor import ( - BigframesBigQuerySeriesAccessor, - ) - - bf_df = mocks.create_dataframe(monkeypatch, data={"col": [1, 2]}) - bf_series = cast(series.Series, bf_df["col"]) - - # Act - has_bq = hasattr(bf_series, "bigquery") - bq_obj = bf_series.bigquery - - # Assert - assert has_bq - assert isinstance(bq_obj, BigframesBigQuerySeriesAccessor) - - -@patch("bigframes.operations.googlesql.global_namespace.array.array_length") -def test_bigframes_series_accessor_global_routing( - mock_array_length, monkeypatch: pytest.MonkeyPatch -): - # Arrange - bf_df = mocks.create_dataframe(monkeypatch, data={"col": [[1, 2], [3, 4, 5]]}) - bf_series = cast(series.Series, bf_df["col"]) - mock_result_series = MagicMock() - mock_array_length.return_value = mock_result_series - - # Act - result = bf_series.bigquery.array_length() - - # Assert - mock_array_length.assert_called_once_with(bf_series) - assert result is mock_result_series - - -@patch("bigframes.operations.googlesql.aead.encrypt") -def test_bigframes_series_accessor_namespaced_routing( - mock_encrypt, monkeypatch: pytest.MonkeyPatch -): - # Arrange - bf_df = mocks.create_dataframe(monkeypatch, data={"keyset": [b"key1", b"key2"]}) - keyset_series = cast(series.Series, bf_df["keyset"]) - mock_result_series = MagicMock() - mock_encrypt.return_value = mock_result_series - - plaintext = "my secret" - additional_data = "context" - - # Act - result = keyset_series.bigquery.aead.encrypt(plaintext, additional_data) - - # Assert - mock_encrypt.assert_called_once_with(keyset_series, plaintext, additional_data) - assert result is mock_result_series diff --git a/tests/unit/extensions/core/__init__.py b/tests/unit/extensions/core/__init__.py deleted file mode 100644 index 58d482ea386..00000000000 --- a/tests/unit/extensions/core/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/unit/extensions/core/test_dataframe_accessor.py b/tests/unit/extensions/core/test_dataframe_accessor.py deleted file mode 100644 index c207070bb15..00000000000 --- a/tests/unit/extensions/core/test_dataframe_accessor.py +++ /dev/null @@ -1,525 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest.mock as mock - -import pandas as pd - -import bigframes.bigquery.ai -import bigframes.pandas as bpd -import bigframes.session - - -def test_ai_forecast(monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - bf_df = mock.create_autospec(bpd.DataFrame) - session.read_pandas.return_value = bf_df - - mock_forecast = mock.MagicMock() - forecast_result_df = mock.create_autospec(bpd.DataFrame) - mock_forecast.return_value = forecast_result_df - expected_result = mock.create_autospec(pd.DataFrame) - forecast_result_df.to_pandas.return_value = expected_result - - monkeypatch.setattr(bigframes.bigquery.ai, "forecast", mock_forecast) - - df = pd.DataFrame({"date": ["2020-01-01"], "value": [1.0]}) - actual_result = df.bigquery.ai.forecast( - timestamp_col="date", - data_col="value", - horizon=5, - session=session, - ) - - session.read_pandas.assert_called_once() - - mock_forecast.assert_called_once_with( - bf_df, - timestamp_col="date", - data_col="value", - model="TimesFM 2.0", - id_cols=None, - horizon=5, - confidence_level=0.95, - context_window=None, - output_historical_time_series=False, - ) - forecast_result_df.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_forecast(scalar_types_df: bpd.DataFrame, monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - forecast_result = mock.create_autospec(bpd.DataFrame) - mock_forecast = mock.MagicMock() - mock_forecast.return_value = forecast_result - - monkeypatch.setattr(bigframes.bigquery.ai, "forecast", mock_forecast) - - actual_result = scalar_types_df.bigquery.ai.forecast( - timestamp_col="date", - data_col="value", - horizon=5, - session=session, - ) - - session.read_pandas.assert_not_called() - mock_forecast.assert_called_once() - args, kwargs = mock_forecast.call_args - assert args[0] is scalar_types_df - assert kwargs == { - "timestamp_col": "date", - "data_col": "value", - "model": "TimesFM 2.0", - "id_cols": None, - "horizon": 5, - "confidence_level": 0.95, - "context_window": None, - "output_historical_time_series": False, - } - # BigFrames accessor returns the bf_df directly without calling to_pandas - forecast_result.to_pandas.assert_not_called() - assert actual_result is forecast_result - - -def test_ai_generate(monkeypatch): - mock_generate = mock.MagicMock() - result_series = mock.create_autospec(bpd.Series) - mock_generate.return_value = result_series - expected_result = mock.create_autospec(pd.Series) - result_series.to_pandas.return_value = expected_result - - monkeypatch.setattr(bigframes.bigquery.ai, "generate", mock_generate) - - prompt = mock.create_autospec(pd.Series) - df = pd.DataFrame({"text_input": ["Is this a positive review?"]}) - actual_result = df.bigquery.ai.generate( - prompt, - connection_id="conn", - endpoint="endpoint", - request_type="dedicated", - model_params={"temp": 0.5}, - output_schema={"res": "STRING"}, - ) - - mock_generate.assert_called_once_with( - prompt, - connection_id="conn", - endpoint="endpoint", - request_type="dedicated", - model_params={"temp": 0.5}, - output_schema={"res": "STRING"}, - ) - result_series.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_generate(scalar_types_df: bpd.DataFrame, monkeypatch): - bf_series = mock.create_autospec(bpd.Series) - result_series = mock.create_autospec(bpd.Series) - - mock_generate = mock.MagicMock() - mock_generate.return_value = result_series - - monkeypatch.setattr(bigframes.bigquery.ai, "generate", mock_generate) - - actual_result = scalar_types_df.bigquery.ai.generate( - bf_series, - connection_id="conn", - endpoint="endpoint", - request_type="dedicated", - model_params={"temp": 0.5}, - output_schema={"res": "STRING"}, - ) - - mock_generate.assert_called_once() - args, kwargs = mock_generate.call_args - assert args[0] is bf_series - assert kwargs == { - "connection_id": "conn", - "endpoint": "endpoint", - "request_type": "dedicated", - "model_params": {"temp": 0.5}, - "output_schema": {"res": "STRING"}, - } - result_series.to_pandas.assert_not_called() - assert actual_result is result_series - - -def test_ai_generate_bool(monkeypatch): - mock_generate_bool = mock.MagicMock() - result_series = mock.create_autospec(bpd.Series) - mock_generate_bool.return_value = result_series - expected_result = mock.create_autospec(pd.Series) - result_series.to_pandas.return_value = expected_result - - monkeypatch.setattr(bigframes.bigquery.ai, "generate_bool", mock_generate_bool) - - prompt = mock.create_autospec(pd.Series) - df = pd.DataFrame({"text_input": ["Is this a positive review?"]}) - actual_result = df.bigquery.ai.generate_bool( - prompt, - connection_id="conn", - endpoint="endpoint", - request_type="dedicated", - model_params={"temp": 0.5}, - ) - - mock_generate_bool.assert_called_once_with( - prompt, - connection_id="conn", - endpoint="endpoint", - request_type="dedicated", - model_params={"temp": 0.5}, - ) - result_series.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_generate_bool(scalar_types_df: bpd.DataFrame, monkeypatch): - bf_series = mock.create_autospec(bpd.Series) - result_series = mock.create_autospec(bpd.Series) - - mock_generate_bool = mock.MagicMock() - mock_generate_bool.return_value = result_series - - monkeypatch.setattr(bigframes.bigquery.ai, "generate_bool", mock_generate_bool) - - actual_result = scalar_types_df.bigquery.ai.generate_bool( - bf_series, - connection_id="conn", - endpoint="endpoint", - request_type="dedicated", - model_params={"temp": 0.5}, - ) - - mock_generate_bool.assert_called_once() - args, kwargs = mock_generate_bool.call_args - assert args[0] is bf_series - assert kwargs == { - "connection_id": "conn", - "endpoint": "endpoint", - "request_type": "dedicated", - "model_params": {"temp": 0.5}, - } - result_series.to_pandas.assert_not_called() - assert actual_result is result_series - - -def test_ai_generate_int(monkeypatch): - mock_generate_int = mock.MagicMock() - result_series = mock.create_autospec(bpd.Series) - mock_generate_int.return_value = result_series - expected_result = mock.create_autospec(pd.Series) - result_series.to_pandas.return_value = expected_result - - monkeypatch.setattr(bigframes.bigquery.ai, "generate_int", mock_generate_int) - - prompt = mock.create_autospec(pd.Series) - df = pd.DataFrame({"text_input": ["How many legs?"]}) - actual_result = df.bigquery.ai.generate_int( - prompt, - connection_id="conn", - endpoint="endpoint", - request_type="dedicated", - model_params={"temp": 0.5}, - ) - - mock_generate_int.assert_called_once_with( - prompt, - connection_id="conn", - endpoint="endpoint", - request_type="dedicated", - model_params={"temp": 0.5}, - ) - result_series.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_generate_int(scalar_types_df: bpd.DataFrame, monkeypatch): - bf_series = mock.create_autospec(bpd.Series) - result_series = mock.create_autospec(bpd.Series) - - mock_generate_int = mock.MagicMock() - mock_generate_int.return_value = result_series - - monkeypatch.setattr(bigframes.bigquery.ai, "generate_int", mock_generate_int) - - actual_result = scalar_types_df.bigquery.ai.generate_int( - bf_series, - connection_id="conn", - endpoint="endpoint", - request_type="dedicated", - model_params={"temp": 0.5}, - ) - - mock_generate_int.assert_called_once() - args, kwargs = mock_generate_int.call_args - assert args[0] is bf_series - assert kwargs == { - "connection_id": "conn", - "endpoint": "endpoint", - "request_type": "dedicated", - "model_params": {"temp": 0.5}, - } - result_series.to_pandas.assert_not_called() - assert actual_result is result_series - - -def test_ai_generate_double(monkeypatch): - mock_generate_double = mock.MagicMock() - result_series = mock.create_autospec(bpd.Series) - mock_generate_double.return_value = result_series - expected_result = mock.create_autospec(pd.Series) - result_series.to_pandas.return_value = expected_result - - monkeypatch.setattr(bigframes.bigquery.ai, "generate_double", mock_generate_double) - - prompt = mock.create_autospec(pd.Series) - df = pd.DataFrame({"text_input": ["How tall?"]}) - actual_result = df.bigquery.ai.generate_double( - prompt, - connection_id="conn", - endpoint="endpoint", - request_type="dedicated", - model_params={"temp": 0.5}, - ) - - mock_generate_double.assert_called_once_with( - prompt, - connection_id="conn", - endpoint="endpoint", - request_type="dedicated", - model_params={"temp": 0.5}, - ) - result_series.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_generate_double(scalar_types_df: bpd.DataFrame, monkeypatch): - bf_series = mock.create_autospec(bpd.Series) - result_series = mock.create_autospec(bpd.Series) - - mock_generate_double = mock.MagicMock() - mock_generate_double.return_value = result_series - - monkeypatch.setattr(bigframes.bigquery.ai, "generate_double", mock_generate_double) - - actual_result = scalar_types_df.bigquery.ai.generate_double( - bf_series, - connection_id="conn", - endpoint="endpoint", - request_type="dedicated", - model_params={"temp": 0.5}, - ) - - mock_generate_double.assert_called_once() - args, kwargs = mock_generate_double.call_args - assert args[0] is bf_series - assert kwargs == { - "connection_id": "conn", - "endpoint": "endpoint", - "request_type": "dedicated", - "model_params": {"temp": 0.5}, - } - result_series.to_pandas.assert_not_called() - assert actual_result is result_series - - -def test_ai_classify(monkeypatch): - mock_classify = mock.MagicMock() - result_series = mock.create_autospec(bpd.Series) - mock_classify.return_value = result_series - expected_result = mock.create_autospec(pd.Series) - result_series.to_pandas.return_value = expected_result - - monkeypatch.setattr(bigframes.bigquery.ai, "classify", mock_classify) - - input_prompt = mock.create_autospec(pd.Series) - df = pd.DataFrame({"text_input": ["Is this a positive review?"]}) - actual_result = df.bigquery.ai.classify( - input_prompt, - categories=["Mammal", "Fish"], - examples=[("Cat", "Mammal")], - connection_id="conn", - endpoint="endpoint", - output_mode="single", - optimization_mode="minimize_cost", - max_error_ratio=0.1, - ) - - mock_classify.assert_called_once_with( - input_prompt, - ["Mammal", "Fish"], - examples=[("Cat", "Mammal")], - connection_id="conn", - endpoint="endpoint", - output_mode="single", - optimization_mode="minimize_cost", - max_error_ratio=0.1, - ) - result_series.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_classify(scalar_types_df: bpd.DataFrame, monkeypatch): - bf_series = mock.create_autospec(bpd.Series) - result_series = mock.create_autospec(bpd.Series) - - mock_classify = mock.MagicMock() - mock_classify.return_value = result_series - - monkeypatch.setattr(bigframes.bigquery.ai, "classify", mock_classify) - - actual_result = scalar_types_df.bigquery.ai.classify( - bf_series, - categories=["Mammal", "Fish"], - examples=[("Cat", "Mammal")], - connection_id="conn", - endpoint="endpoint", - output_mode="single", - optimization_mode="minimize_cost", - max_error_ratio=0.1, - ) - - mock_classify.assert_called_once() - args, kwargs = mock_classify.call_args - assert args[0] is bf_series - assert args[1] == ["Mammal", "Fish"] - assert kwargs == { - "examples": [("Cat", "Mammal")], - "connection_id": "conn", - "endpoint": "endpoint", - "output_mode": "single", - "optimization_mode": "minimize_cost", - "max_error_ratio": 0.1, - } - result_series.to_pandas.assert_not_called() - assert actual_result is result_series - - -def test_ai_if(monkeypatch): - mock_if = mock.MagicMock() - result_series = mock.create_autospec(bpd.Series) - mock_if.return_value = result_series - expected_result = mock.create_autospec(pd.Series) - result_series.to_pandas.return_value = expected_result - - monkeypatch.setattr(bigframes.bigquery.ai, "if_", mock_if) - - prompt = mock.create_autospec(pd.Series) - df = pd.DataFrame({"text_input": ["Is this a positive review?"]}) - actual_result = df.bigquery.ai.if_( - prompt, - connection_id="conn", - endpoint="endpoint", - optimization_mode="minimize_cost", - max_error_ratio=0.1, - ) - - mock_if.assert_called_once_with( - prompt, - connection_id="conn", - endpoint="endpoint", - optimization_mode="minimize_cost", - max_error_ratio=0.1, - ) - result_series.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_if(scalar_types_df: bpd.DataFrame, monkeypatch): - bf_series = mock.create_autospec(bpd.Series) - result_series = mock.create_autospec(bpd.Series) - - mock_if = mock.MagicMock() - mock_if.return_value = result_series - - monkeypatch.setattr(bigframes.bigquery.ai, "if_", mock_if) - - actual_result = scalar_types_df.bigquery.ai.if_( - bf_series, - connection_id="conn", - endpoint="endpoint", - optimization_mode="minimize_cost", - max_error_ratio=0.1, - ) - - mock_if.assert_called_once() - args, kwargs = mock_if.call_args - assert args[0] is bf_series - assert kwargs == { - "connection_id": "conn", - "endpoint": "endpoint", - "optimization_mode": "minimize_cost", - "max_error_ratio": 0.1, - } - result_series.to_pandas.assert_not_called() - assert actual_result is result_series - - -def test_ai_score(monkeypatch): - mock_score = mock.MagicMock() - result_series = mock.create_autospec(bpd.Series) - mock_score.return_value = result_series - expected_result = mock.create_autospec(pd.Series) - result_series.to_pandas.return_value = expected_result - - monkeypatch.setattr(bigframes.bigquery.ai, "score", mock_score) - - prompt = mock.create_autospec(pd.Series) - df = pd.DataFrame({"text_input": ["Is this a positive review?"]}) - actual_result = df.bigquery.ai.score( - prompt, - connection_id="conn", - endpoint="endpoint", - max_error_ratio=0.1, - ) - - mock_score.assert_called_once_with( - prompt, - connection_id="conn", - endpoint="endpoint", - max_error_ratio=0.1, - ) - result_series.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_score(scalar_types_df: bpd.DataFrame, monkeypatch): - bf_series = mock.create_autospec(bpd.Series) - result_series = mock.create_autospec(bpd.Series) - - mock_score = mock.MagicMock() - mock_score.return_value = result_series - - monkeypatch.setattr(bigframes.bigquery.ai, "score", mock_score) - - actual_result = scalar_types_df.bigquery.ai.score( - bf_series, - connection_id="conn", - endpoint="endpoint", - max_error_ratio=0.1, - ) - - mock_score.assert_called_once() - args, kwargs = mock_score.call_args - assert args[0] is bf_series - assert kwargs == { - "connection_id": "conn", - "endpoint": "endpoint", - "max_error_ratio": 0.1, - } - result_series.to_pandas.assert_not_called() - assert actual_result is result_series diff --git a/tests/unit/extensions/core/test_series_mixins.py b/tests/unit/extensions/core/test_series_mixins.py deleted file mode 100644 index c6e7e4078b1..00000000000 --- a/tests/unit/extensions/core/test_series_mixins.py +++ /dev/null @@ -1,384 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest.mock as mock - -import pandas as pd - -import bigframes.bigquery.ai -import bigframes.pandas as bpd -import bigframes.session - - -def test_ai_generate_embedding(monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - bf_series = mock.create_autospec(bpd.Series) - session.read_pandas.return_value = bf_series - - mock_generate_embedding = mock.MagicMock() - result_df = mock.create_autospec(bpd.DataFrame) - mock_generate_embedding.return_value = result_df - expected_result = mock.create_autospec(pd.DataFrame) - result_df.to_pandas.return_value = expected_result - - monkeypatch.setattr( - bigframes.bigquery.ai, "generate_embedding", mock_generate_embedding - ) - - series = pd.Series(["apple"], name="content") - actual_result = series.bigquery.ai.generate_embedding( # type: ignore - model="my_model", - output_dimensionality=256, - task_type="retrieval_document", - start_second=1.0, - end_second=2.0, - interval_seconds=3.0, - trial_id=4, - session=session, - ) - - session.read_pandas.assert_called_once() - mock_generate_embedding.assert_called_once_with( - "my_model", - bf_series, - output_dimensionality=256, - task_type="retrieval_document", - start_second=1.0, - end_second=2.0, - interval_seconds=3.0, - trial_id=4, - ) - result_df.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_generate_embedding(scalar_types_df: bpd.DataFrame, monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - result_df = mock.create_autospec(bpd.DataFrame) - - mock_generate_embedding = mock.MagicMock() - mock_generate_embedding.return_value = result_df - - monkeypatch.setattr( - bigframes.bigquery.ai, "generate_embedding", mock_generate_embedding - ) - - scalar_types_series = scalar_types_df["string_col"] - actual_result = scalar_types_series.bigquery.ai.generate_embedding( - model="my_model", - output_dimensionality=256, - session=session, - ) - - session.read_pandas.assert_not_called() - mock_generate_embedding.assert_called_once() - args, kwargs = mock_generate_embedding.call_args - assert args[0] == "my_model" - assert args[1] is scalar_types_series - assert kwargs == { - "output_dimensionality": 256, - "task_type": None, - "start_second": None, - "end_second": None, - "interval_seconds": None, - "trial_id": None, - } - result_df.to_pandas.assert_not_called() - assert actual_result is result_df - - -def test_ai_generate_text(monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - bf_series = mock.create_autospec(bpd.Series) - session.read_pandas.return_value = bf_series - - mock_generate_text = mock.MagicMock() - result_df = mock.create_autospec(bpd.DataFrame) - mock_generate_text.return_value = result_df - expected_result = mock.create_autospec(pd.DataFrame) - result_df.to_pandas.return_value = expected_result - - monkeypatch.setattr(bigframes.bigquery.ai, "generate_text", mock_generate_text) - - series = pd.Series(["write a poem"], name="prompt") - actual_result = series.bigquery.ai.generate_text( # type: ignore - model="my_model", - temperature=0.7, - max_output_tokens=100, - top_k=50, - top_p=0.9, - stop_sequences=["\n"], - ground_with_google_search=True, - request_type="dedicated", - session=session, - ) - - session.read_pandas.assert_called_once() - mock_generate_text.assert_called_once_with( - "my_model", - bf_series, - temperature=0.7, - max_output_tokens=100, - top_k=50, - top_p=0.9, - stop_sequences=["\n"], - ground_with_google_search=True, - request_type="dedicated", - ) - result_df.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_generate_text(scalar_types_df: bpd.DataFrame, monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - result_df = mock.create_autospec(bpd.DataFrame) - - mock_generate_text = mock.MagicMock() - mock_generate_text.return_value = result_df - - monkeypatch.setattr(bigframes.bigquery.ai, "generate_text", mock_generate_text) - - scalar_types_series = scalar_types_df["string_col"] - actual_result = scalar_types_series.bigquery.ai.generate_text( - model="my_model", - temperature=0.7, - session=session, - ) - - session.read_pandas.assert_not_called() - mock_generate_text.assert_called_once() - args, kwargs = mock_generate_text.call_args - assert args[0] == "my_model" - assert args[1] is scalar_types_series - assert kwargs == { - "temperature": 0.7, - "max_output_tokens": None, - "top_k": None, - "top_p": None, - "stop_sequences": None, - "ground_with_google_search": None, - "request_type": None, - } - result_df.to_pandas.assert_not_called() - assert actual_result is result_df - - -def test_ai_generate_table(monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - bf_series = mock.create_autospec(bpd.Series) - session.read_pandas.return_value = bf_series - - mock_generate_table = mock.MagicMock() - result_df = mock.create_autospec(bpd.DataFrame) - mock_generate_table.return_value = result_df - expected_result = mock.create_autospec(pd.DataFrame) - result_df.to_pandas.return_value = expected_result - - monkeypatch.setattr(bigframes.bigquery.ai, "generate_table", mock_generate_table) - - series = pd.Series(["generate something"], name="prompt") - actual_result = series.bigquery.ai.generate_table( # type: ignore - model="my_model", - output_schema="category STRING", - temperature=0.7, - top_p=0.9, - max_output_tokens=100, - stop_sequences=["\n"], - request_type="dedicated", - session=session, - ) - - session.read_pandas.assert_called_once() - mock_generate_table.assert_called_once_with( - "my_model", - bf_series, - output_schema="category STRING", - temperature=0.7, - top_p=0.9, - max_output_tokens=100, - stop_sequences=["\n"], - request_type="dedicated", - ) - result_df.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_generate_table(scalar_types_df: bpd.DataFrame, monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - result_df = mock.create_autospec(bpd.DataFrame) - - mock_generate_table = mock.MagicMock() - mock_generate_table.return_value = result_df - - monkeypatch.setattr(bigframes.bigquery.ai, "generate_table", mock_generate_table) - - scalar_types_series = scalar_types_df["string_col"] - actual_result = scalar_types_series.bigquery.ai.generate_table( - model="my_model", - output_schema="category STRING", - temperature=0.7, - session=session, - ) - - session.read_pandas.assert_not_called() - mock_generate_table.assert_called_once() - args, kwargs = mock_generate_table.call_args - assert args[0] == "my_model" - assert args[1] is scalar_types_series - assert kwargs == { - "output_schema": "category STRING", - "temperature": 0.7, - "top_p": None, - "max_output_tokens": None, - "stop_sequences": None, - "request_type": None, - } - result_df.to_pandas.assert_not_called() - assert actual_result is result_df - - -def test_ai_embed(monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - bf_series = mock.create_autospec(bpd.Series) - session.read_pandas.return_value = bf_series - - mock_embed = mock.MagicMock() - result_series = mock.create_autospec(bpd.Series) - mock_embed.return_value = result_series - expected_result = mock.create_autospec(pd.Series) - result_series.to_pandas.return_value = expected_result - - monkeypatch.setattr(bigframes.bigquery.ai, "embed", mock_embed) - - series = pd.Series(["hello world"], name="content") - actual_result = series.bigquery.ai.embed( # type: ignore - endpoint="my_endpoint", - model="my_model", - task_type="retrieval_query", - title="my_title", - model_params={"key": "val"}, - connection_id="my_connection", - session=session, - ) - - session.read_pandas.assert_called_once() - mock_embed.assert_called_once_with( - bf_series, - endpoint="my_endpoint", - model="my_model", - task_type="retrieval_query", - title="my_title", - model_params={"key": "val"}, - connection_id="my_connection", - ) - result_series.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_embed(scalar_types_df: bpd.DataFrame, monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - result_series = mock.create_autospec(bpd.Series) - - mock_embed = mock.MagicMock() - mock_embed.return_value = result_series - - monkeypatch.setattr(bigframes.bigquery.ai, "embed", mock_embed) - - scalar_types_series = scalar_types_df["string_col"] - actual_result = scalar_types_series.bigquery.ai.embed( - endpoint="my_endpoint", - session=session, - ) - - session.read_pandas.assert_not_called() - mock_embed.assert_called_once() - args, kwargs = mock_embed.call_args - assert args[0] is scalar_types_series - assert kwargs == { - "endpoint": "my_endpoint", - "model": None, - "task_type": None, - "title": None, - "model_params": None, - "connection_id": None, - } - result_series.to_pandas.assert_not_called() - assert actual_result is result_series - - -def test_ai_similarity(monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - bf_series = mock.create_autospec(bpd.Series) - session.read_pandas.return_value = bf_series - - mock_similarity = mock.MagicMock() - result_series = mock.create_autospec(bpd.Series) - mock_similarity.return_value = result_series - expected_result = mock.create_autospec(pd.Series) - result_series.to_pandas.return_value = expected_result - - monkeypatch.setattr(bigframes.bigquery.ai, "similarity", mock_similarity) - - series = pd.Series(["apple"], name="content") - actual_result = series.bigquery.ai.similarity( # type: ignore - "banana", - endpoint="my_endpoint", - model="my_model", - model_params={"key": "val"}, - connection_id="my_connection", - session=session, - ) - - session.read_pandas.assert_called_once() - mock_similarity.assert_called_once_with( - bf_series, - "banana", - endpoint="my_endpoint", - model="my_model", - model_params={"key": "val"}, - connection_id="my_connection", - ) - result_series.to_pandas.assert_called_once() - assert actual_result is expected_result - - -def test_bigframes_ai_similarity(scalar_types_df: bpd.DataFrame, monkeypatch): - session = mock.create_autospec(bigframes.session.Session) - result_series = mock.create_autospec(bpd.Series) - - mock_similarity = mock.MagicMock() - mock_similarity.return_value = result_series - - monkeypatch.setattr(bigframes.bigquery.ai, "similarity", mock_similarity) - - scalar_types_series = scalar_types_df["string_col"] - actual_result = scalar_types_series.bigquery.ai.similarity( - "other_text", - endpoint="my_endpoint", - session=session, - ) - - session.read_pandas.assert_not_called() - mock_similarity.assert_called_once() - args, kwargs = mock_similarity.call_args - assert args[0] is scalar_types_series - assert args[1] == "other_text" - assert kwargs == { - "endpoint": "my_endpoint", - "model": None, - "model_params": None, - "connection_id": None, - } - result_series.to_pandas.assert_not_called() - assert actual_result is result_series diff --git a/tests/unit/extensions/pandas/__init__.py b/tests/unit/extensions/pandas/__init__.py deleted file mode 100644 index 58d482ea386..00000000000 --- a/tests/unit/extensions/pandas/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/unit/extensions/pandas/test_registration.py b/tests/unit/extensions/pandas/test_registration.py deleted file mode 100644 index 7007d6f9f2f..00000000000 --- a/tests/unit/extensions/pandas/test_registration.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pandas as pd - -# Importing bigframes registers the accessor. -import bigframes # noqa: F401 - - -def test_bigframes_import_registers_accessor(): - df = pd.DataFrame({"a": [1]}) - # If bigframes was imported, df.bigquery should exist - assert hasattr(df, "bigquery") - from bigframes.extensions.pandas.dataframe_accessor import ( - PandasBigQueryDataFrameAccessor, - ) - - assert isinstance(df.bigquery, PandasBigQueryDataFrameAccessor) diff --git a/tests/unit/extensions/pandas/test_series_accessor.py b/tests/unit/extensions/pandas/test_series_accessor.py deleted file mode 100644 index bfb68323f6d..00000000000 --- a/tests/unit/extensions/pandas/test_series_accessor.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from unittest.mock import MagicMock, patch - -import pandas as pd - -import bigframes # noqa: F401 registers pandas extensions -import bigframes.series as series - - -def test_pandas_series_registers_accessor(): - # Arrange - from bigframes.extensions.pandas.series_accessor import ( - PandasBigQuerySeriesAccessor, - ) - - s = pd.Series([1, 2]) - - # Act - has_bq = hasattr(s, "bigquery") - bq_obj = s.bigquery - - # Assert - assert has_bq - assert isinstance(bq_obj, PandasBigQuerySeriesAccessor) - - -@patch("bigframes.operations.googlesql.global_namespace.array.array_length") -def test_pandas_series_accessor_global_routing(mock_array_length): - # Arrange - mock_bf_series = MagicMock() - mock_bf_series.to_pandas.return_value = pd.Series([2, 3]) - mock_array_length.return_value = mock_bf_series - mock_session = MagicMock() - mock_bf_self = MagicMock() - mock_session.read_pandas.return_value = mock_bf_self - - s = pd.Series([[1, 2], [3, 4, 5]]) - - # Act - result = s.bigquery.array_length(session=mock_session) - - # Assert - mock_session.read_pandas.assert_called_once_with(s) - mock_array_length.assert_called_once_with(mock_bf_self) - mock_bf_series.to_pandas.assert_called_once_with(ordered=True) - pd.testing.assert_series_equal(result, pd.Series([2, 3])) - - -@patch("bigframes.operations.googlesql.aead.encrypt") -def test_pandas_series_accessor_namespaced_routing(mock_encrypt): - # Arrange - mock_bf_series = MagicMock() - mock_bf_series.to_pandas.return_value = pd.Series([b"encrypted1", b"encrypted2"]) - mock_encrypt.return_value = mock_bf_series - mock_session = MagicMock() - mock_bf_self = MagicMock() - mock_session.read_pandas.return_value = mock_bf_self - - keyset_series = pd.Series([b"key1", b"key2"]) - plaintext = "my secret" - additional_data = "context" - - # Act - result = keyset_series.bigquery.aead.encrypt( # type: ignore - plaintext, additional_data, session=mock_session - ) - - # Assert - mock_session.read_pandas.assert_called_once_with(keyset_series) - mock_encrypt.assert_called_once_with(mock_bf_self, plaintext, additional_data) - mock_bf_series.to_pandas.assert_called_once_with(ordered=True) - pd.testing.assert_series_equal(result, pd.Series([b"encrypted1", b"encrypted2"])) - - -@patch("bigframes.operations.googlesql.global_namespace.array.array_concat") -def test_pandas_series_accessor_global_routing_uses_series_session(mock_array_concat): - # Arrange - mock_bf_series = MagicMock() - mock_bf_series.to_pandas.return_value = pd.Series([[1, 2, 3, 4]]) - mock_array_concat.return_value = mock_bf_series - mock_session = MagicMock() - mock_bf_other = MagicMock(spec=series.Series) - mock_bf_other._session = mock_session - mock_bf_self = MagicMock() - mock_session.read_pandas.return_value = mock_bf_self - s = pd.Series([[1, 2]]) - - # Act - result = s.bigquery.array_concat(mock_bf_other) - - # Assert - assert result is not None - mock_session.read_pandas.assert_called_once_with(s) - mock_array_concat.assert_called_once_with(mock_bf_self, mock_bf_other) - - -@patch("bigframes.operations.googlesql.aead.encrypt") -def test_pandas_series_accessor_namespaced_routing_uses_series_session( - mock_encrypt, -): - # Arrange - mock_bf_series = MagicMock() - mock_bf_series.to_pandas.return_value = pd.Series([b"encrypted1", b"encrypted2"]) - mock_encrypt.return_value = mock_bf_series - mock_session = MagicMock() - mock_bf_plaintext = MagicMock(spec=series.Series) - mock_bf_plaintext._session = mock_session - mock_bf_self = MagicMock() - mock_session.read_pandas.return_value = mock_bf_self - keyset_series = pd.Series([b"key1", b"key2"]) - additional_data = "context" - - # Act - result = keyset_series.bigquery.aead.encrypt( # type: ignore - mock_bf_plaintext, additional_data - ) - - # Assert - assert result is not None - mock_session.read_pandas.assert_called_once_with(keyset_series) - mock_encrypt.assert_called_once_with( - mock_bf_self, mock_bf_plaintext, additional_data - ) diff --git a/tests/unit/functions/test_remote_function.py b/tests/unit/functions/test_remote_function.py index 19de301790d..e9e0d0df677 100644 --- a/tests/unit/functions/test_remote_function.py +++ b/tests/unit/functions/test_remote_function.py @@ -12,15 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re + +import pandas import pytest +import bigframes.exceptions +import bigframes.functions.function as bff from bigframes.testing import mocks +def test_series_input_types_to_str(): + """Check that is_row_processor=True uses str as the input type to serialize a row.""" + session = mocks.create_bigquery_session() + remote_function_decorator = bff.remote_function( + session=session, cloud_function_service_account="default" + ) + + with pytest.warns( + bigframes.exceptions.PreviewWarning, + match=re.escape("input_types=Series is in preview."), + ): + + @remote_function_decorator + def axis_1_function(myparam: pandas.Series) -> str: # type: ignore + return "Hello, " + myparam["str_col"] + "!" # type: ignore + + # Still works as a normal function. + assert axis_1_function(pandas.Series({"str_col": "World"})) == "Hello, World!" + + def test_missing_input_types(): session = mocks.create_bigquery_session() - remote_function_decorator = session._function_session.remote_function( - cloud_function_service_account="default" + remote_function_decorator = bff.remote_function( + session=session, cloud_function_service_account="default" ) def function_without_parameter_annotations(myparam) -> str: @@ -37,8 +62,8 @@ def function_without_parameter_annotations(myparam) -> str: def test_missing_output_type(): session = mocks.create_bigquery_session() - remote_function_decorator = session._function_session.remote_function( - cloud_function_service_account="default" + remote_function_decorator = bff.remote_function( + session=session, cloud_function_service_account="default" ) def function_without_return_annotation(myparam: int): @@ -53,6 +78,36 @@ def function_without_return_annotation(myparam: int): remote_function_decorator(function_without_return_annotation) +def test_deploy_remote_function(): + session = mocks.create_bigquery_session() + + def my_remote_func(x: int) -> int: + return x * 2 + + deployed = session.deploy_remote_function( + my_remote_func, cloud_function_service_account="test_sa@example.com" + ) + + # Test that the function would have been deployed somewhere. + assert deployed.bigframes_bigquery_function + + +def test_deploy_remote_function_with_name(): + session = mocks.create_bigquery_session() + + def my_remote_func(x: int) -> int: + return x * 2 + + deployed = session.deploy_remote_function( + my_remote_func, + name="my_custom_name", + cloud_function_service_account="test_sa@example.com", + ) + + # Test that the function would have been deployed somewhere. + assert "my_custom_name" in deployed.bigframes_bigquery_function + + def test_deploy_udf(): session = mocks.create_bigquery_session() @@ -61,7 +116,8 @@ def my_remote_func(x: int) -> int: deployed = session.deploy_udf(my_remote_func) - assert deployed.udf_def is not None + # Test that the function would have been deployed somewhere. + assert deployed.bigframes_bigquery_function def test_deploy_udf_with_name(): diff --git a/tests/unit/functions/test_remote_function_utils.py b/tests/unit/functions/test_remote_function_utils.py index cbdb289e265..812d65bbad2 100644 --- a/tests/unit/functions/test_remote_function_utils.py +++ b/tests/unit/functions/test_remote_function_utils.py @@ -23,18 +23,98 @@ @pytest.mark.parametrize( - ("input_location", "expected_cf_region"), + ("input_location", "expected_bq_location", "expected_cf_region"), [ - ("us", "us-central1"), - ("eu", "europe-west1"), - ("US-east4", "us-east4"), + (None, "us", "us-central1"), + ("us", "us", "us-central1"), + ("eu", "eu", "europe-west1"), + ("US-east4", "us-east4", "us-east4"), ], ) -def test_gcf_location_from_bq_location(input_location, expected_cf_region): - """Tests getting cloud function locations for various BigQuery locations.""" - gcf_location = _utils.gcf_location_from_bq_location(input_location) +def test_get_remote_function_locations( + input_location, expected_bq_location, expected_cf_region +): + """Tests getting remote function locations for various locations.""" + bq_location, cf_region = _utils.get_remote_function_locations(input_location) + + assert bq_location == expected_bq_location + assert cf_region == expected_cf_region + + +@pytest.mark.parametrize( + "func_hash, session_id, uniq_suffix, expected_name", + [ + ( + "hash123", + None, + None, + "bigframes-hash123", + ), + ( + "hash456", + "session789", + None, + "bigframes-session789-hash456", + ), + ( + "hash123", + None, + "suffixABC", + "bigframes-hash123-suffixABC", + ), + ( + "hash456", + "session789", + "suffixDEF", + "bigframes-session789-hash456-suffixDEF", + ), + ], +) +def test_get_cloud_function_name(func_hash, session_id, uniq_suffix, expected_name): + """Tests the construction of the cloud function name from its parts.""" + result = _utils.get_cloud_function_name(func_hash, session_id, uniq_suffix) - assert gcf_location == expected_cf_region + assert result == expected_name + + +@pytest.mark.parametrize( + "function_hash, session_id, uniq_suffix, expected_name", + [ + ( + "hash123", + "session456", + None, + "bigframes_session456_hash123", + ), + ( + "hash789", + "sessionABC", + "suffixDEF", + "bigframes_sessionABC_hash789_suffixDEF", + ), + ], +) +def test_get_bigframes_function_name( + function_hash, session_id, uniq_suffix, expected_name +): + """Tests the construction of the BigQuery function name from its parts.""" + result = _utils.get_bigframes_function_name(function_hash, session_id, uniq_suffix) + + assert result == expected_name + + +def test_get_updated_package_requirements_no_extra_package(): + """Tests with no extra package.""" + result = _utils.get_updated_package_requirements(capture_references=False) + + assert result is None + + initial_packages = ["xgboost"] + result = _utils.get_updated_package_requirements( + initial_packages, capture_references=False + ) + + assert result == initial_packages @patch("bigframes.functions._utils.numpy.__version__", "1.24.4") @@ -82,7 +162,7 @@ def test_get_updated_package_requirements_capture_references_false(): # Case 1: Only capture_references=False. result_1 = _utils.get_updated_package_requirements(capture_references=False) - assert len(result_1) == 0 + assert result_1 is None # Case 2: capture_references=False but is_row_processor=True. expected_2 = ["numpy==1.24.4", "pandas==2.0.3", "pyarrow==14.0.1"] @@ -184,6 +264,78 @@ def test_package_existed_helper(): assert not _utils._package_existed([], "pandas") +def _function_add_one(x): + return x + 1 + + +def _function_add_two(x): + return x + 2 + + +@pytest.mark.parametrize( + "func1, func2, should_be_equal, description", + [ + ( + _function_add_one, + _function_add_one, + True, + "Identical functions should have the same hash.", + ), + ( + _function_add_one, + _function_add_two, + False, + "Different functions should have different hashes.", + ), + ], +) +def test_get_hash_without_package_requirements( + func1, func2, should_be_equal, description +): + """Tests function hashes without any requirements.""" + hash1 = _utils.get_hash(func1) + hash2 = _utils.get_hash(func2) + + if should_be_equal: + assert hash1 == hash2, f"FAILED: {description}" + else: + assert hash1 != hash2, f"FAILED: {description}" + + +@pytest.mark.parametrize( + "reqs1, reqs2, should_be_equal, description", + [ + ( + None, + ["pandas>=1.0"], + False, + "Hash with or without requirements should differ from hash.", + ), + ( + ["pandas", "numpy", "scikit-learn"], + ["numpy", "scikit-learn", "pandas"], + True, + "Same requirements should produce the same hash.", + ), + ( + ["pandas==1.0"], + ["pandas==2.0"], + False, + "Different requirement versions should produce different hashes.", + ), + ], +) +def test_get_hash_with_package_requirements(reqs1, reqs2, should_be_equal, description): + """Tests how package requirements affect the final hash.""" + hash1 = _utils.get_hash(_function_add_one, package_requirements=reqs1) + hash2 = _utils.get_hash(_function_add_one, package_requirements=reqs2) + + if should_be_equal: + assert hash1 == hash2, f"FAILED: {description}" + else: + assert hash1 != hash2, f"FAILED: {description}" + + # Helper functions for signature inspection tests def _func_one_arg_annotated(x: int) -> int: """A function with one annotated arg and an annotated return type.""" @@ -289,6 +441,7 @@ def test_has_conflict_output_type_no_annotation(): ), ) def test_get_bigframes_metadata(metadata_options, metadata_string): + assert _utils.get_bigframes_metadata(**metadata_options) == metadata_string @@ -361,6 +514,7 @@ def test_get_bigframes_metadata_array_type_not_serializable(output_type): def test_get_python_output_type_from_bigframes_metadata( metadata_string, python_output_type ): + assert ( _utils.get_python_output_type_from_bigframes_metadata(metadata_string) == python_output_type diff --git a/tests/unit/ml/test_compose.py b/tests/unit/ml/test_compose.py index 7779bafadfa..86cbb111f4d 100644 --- a/tests/unit/ml/test_compose.py +++ b/tests/unit/ml/test_compose.py @@ -13,13 +13,13 @@ # limitations under the License. from unittest import mock -import pytest from google.cloud import bigquery +import pytest -import bigframes.pandas as bpd from bigframes.ml import compose, preprocessing from bigframes.ml.compose import ColumnTransformer, SQLScalarColumnTransformer from bigframes.ml.core import BqmlModel +import bigframes.pandas as bpd def test_columntransformer_init_expectedtransforms(): diff --git a/tests/unit/ml/test_golden_sql.py b/tests/unit/ml/test_golden_sql.py index 7babf476117..7f6843aacf6 100644 --- a/tests/unit/ml/test_golden_sql.py +++ b/tests/unit/ml/test_golden_sql.py @@ -14,14 +14,14 @@ from unittest import mock +from google.cloud import bigquery import pandas as pd import pytest -from google.cloud import bigquery import bigframes +from bigframes.ml import core, decomposition, linear_model import bigframes.ml.core import bigframes.pandas as bpd -from bigframes.ml import core, decomposition, linear_model TEMP_MODEL_ID = bigquery.ModelReference.from_string( "test-project._anon123.temp_model_id" @@ -124,7 +124,7 @@ def test_linear_regression_default_fit( model.fit(mock_X, mock_y) mock_session._start_query_ml_ddl.assert_called_once_with( - "CREATE OR REPLACE MODEL `test-project`.`_anon123`.`temp_model_id`\nOPTIONS(\n model_type='LINEAR_REG',\n data_split_method='NO_SPLIT',\n optimize_strategy='auto_strategy',\n fit_intercept=TRUE,\n l2_reg=0.0,\n max_iterations=20,\n learn_rate_strategy='line_search',\n min_rel_progress=0.01,\n calculate_p_values=FALSE,\n enable_global_explain=FALSE,\n INPUT_LABEL_COLS=['input_column_label'])\nAS input_X_y_no_index_sql" + "CREATE OR REPLACE MODEL `test-project`.`_anon123`.`temp_model_id`\nOPTIONS(\n model_type='LINEAR_REG',\n data_split_method='NO_SPLIT',\n optimize_strategy='auto_strategy',\n fit_intercept=True,\n l2_reg=0.0,\n max_iterations=20,\n learn_rate_strategy='line_search',\n min_rel_progress=0.01,\n calculate_p_values=False,\n enable_global_explain=False,\n INPUT_LABEL_COLS=['input_column_label'])\nAS input_X_y_no_index_sql" ) @@ -134,7 +134,7 @@ def test_linear_regression_params_fit(bqml_model_factory, mock_session, mock_X, model.fit(mock_X, mock_y) mock_session._start_query_ml_ddl.assert_called_once_with( - "CREATE OR REPLACE MODEL `test-project`.`_anon123`.`temp_model_id`\nOPTIONS(\n model_type='LINEAR_REG',\n data_split_method='NO_SPLIT',\n optimize_strategy='auto_strategy',\n fit_intercept=FALSE,\n l2_reg=0.0,\n max_iterations=20,\n learn_rate_strategy='line_search',\n min_rel_progress=0.01,\n calculate_p_values=FALSE,\n enable_global_explain=FALSE,\n INPUT_LABEL_COLS=['input_column_label'])\nAS input_X_y_no_index_sql" + "CREATE OR REPLACE MODEL `test-project`.`_anon123`.`temp_model_id`\nOPTIONS(\n model_type='LINEAR_REG',\n data_split_method='NO_SPLIT',\n optimize_strategy='auto_strategy',\n fit_intercept=False,\n l2_reg=0.0,\n max_iterations=20,\n learn_rate_strategy='line_search',\n min_rel_progress=0.01,\n calculate_p_values=False,\n enable_global_explain=False,\n INPUT_LABEL_COLS=['input_column_label'])\nAS input_X_y_no_index_sql" ) @@ -169,7 +169,7 @@ def test_logistic_regression_default_fit( model.fit(mock_X, mock_y) mock_session._start_query_ml_ddl.assert_called_once_with( - "CREATE OR REPLACE MODEL `test-project`.`_anon123`.`temp_model_id`\nOPTIONS(\n model_type='LOGISTIC_REG',\n data_split_method='NO_SPLIT',\n fit_intercept=TRUE,\n auto_class_weights=FALSE,\n optimize_strategy='auto_strategy',\n l2_reg=0.0,\n max_iterations=20,\n learn_rate_strategy='line_search',\n min_rel_progress=0.01,\n calculate_p_values=FALSE,\n enable_global_explain=FALSE,\n INPUT_LABEL_COLS=['input_column_label'])\nAS input_X_y_no_index_sql", + "CREATE OR REPLACE MODEL `test-project`.`_anon123`.`temp_model_id`\nOPTIONS(\n model_type='LOGISTIC_REG',\n data_split_method='NO_SPLIT',\n fit_intercept=True,\n auto_class_weights=False,\n optimize_strategy='auto_strategy',\n l2_reg=0.0,\n max_iterations=20,\n learn_rate_strategy='line_search',\n min_rel_progress=0.01,\n calculate_p_values=False,\n enable_global_explain=False,\n INPUT_LABEL_COLS=['input_column_label'])\nAS input_X_y_no_index_sql", ) @@ -191,7 +191,7 @@ def test_logistic_regression_params_fit( model.fit(mock_X, mock_y) mock_session._start_query_ml_ddl.assert_called_once_with( - "CREATE OR REPLACE MODEL `test-project`.`_anon123`.`temp_model_id`\nOPTIONS(\n model_type='LOGISTIC_REG',\n data_split_method='NO_SPLIT',\n fit_intercept=FALSE,\n auto_class_weights=TRUE,\n optimize_strategy='batch_gradient_descent',\n l2_reg=0.2,\n max_iterations=30,\n learn_rate_strategy='constant',\n min_rel_progress=0.02,\n calculate_p_values=FALSE,\n enable_global_explain=FALSE,\n l1_reg=0.2,\n learn_rate=0.2,\n INPUT_LABEL_COLS=['input_column_label'])\nAS input_X_y_no_index_sql" + "CREATE OR REPLACE MODEL `test-project`.`_anon123`.`temp_model_id`\nOPTIONS(\n model_type='LOGISTIC_REG',\n data_split_method='NO_SPLIT',\n fit_intercept=False,\n auto_class_weights=True,\n optimize_strategy='batch_gradient_descent',\n l2_reg=0.2,\n max_iterations=30,\n learn_rate_strategy='constant',\n min_rel_progress=0.02,\n calculate_p_values=False,\n enable_global_explain=False,\n l1_reg=0.2,\n learn_rate=0.2,\n INPUT_LABEL_COLS=['input_column_label'])\nAS input_X_y_no_index_sql" ) diff --git a/tests/unit/ml/test_llm.py b/tests/unit/ml/test_llm.py deleted file mode 100644 index df17bbc6e7b..00000000000 --- a/tests/unit/ml/test_llm.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from unittest import mock - -import pytest -from google.cloud import bigquery - -import bigframes.session -from bigframes.ml import llm - - -def test_gemini_text_generator_default_model(): - mock_session = mock.create_autospec(spec=bigframes.session.Session) - mock_session._create_bq_connection.return_value = ( - "projects/test-project/locations/us-central1/connections/test-conn" - ) - mock_session._anonymous_dataset = bigquery.DatasetReference( - "test-project", "test_dataset" - ) - mock_job = mock.MagicMock() - mock_job.destination.project = "test-project" - mock_job.destination.dataset_id = "test_dataset" - mock_job.destination.table_id = "test_model" - mock_session._start_query_ml_ddl.return_value = (None, mock_job) - mock_session.bqclient.get_model.return_value = mock.MagicMock(spec=bigquery.Model) - - with pytest.warns( - FutureWarning, match="default model will be removed in BigFrames 3.0" - ): - model = llm.GeminiTextGenerator( - session=mock_session, - connection_name="test-conn", - ) - - assert model.model_name == "gemini-2.5-flash" - mock_session._start_query_ml_ddl.assert_called_once() - generated_sql = mock_session._start_query_ml_ddl.call_args[0][0] - assert "gemini-2.5-flash" in generated_sql diff --git a/tests/unit/operations/test_output_schemas.py b/tests/unit/operations/test_output_schemas.py index 204078a5ceb..c609098c98b 100644 --- a/tests/unit/operations/test_output_schemas.py +++ b/tests/unit/operations/test_output_schemas.py @@ -70,7 +70,7 @@ def test_parse_sql_to_pyarrow_dtype(sql, expected): "a INT64", "ARRAY<>", "ARRAYARRAYSTRUCT<>", + "ARRAY" "ARRAY" "STRUCT<>", "DATE", "STRUCT", "ARRAY>", diff --git a/tests/unit/session/test_clients.py b/tests/unit/session/test_clients.py index 0de6c75e01b..5304c99466b 100644 --- a/tests/unit/session/test_clients.py +++ b/tests/unit/session/test_clients.py @@ -15,8 +15,8 @@ import os import pathlib import tempfile +from typing import cast, Optional import unittest.mock as mock -from typing import Optional, cast import google.auth.credentials import google.cloud.bigquery @@ -182,18 +182,12 @@ def test_user_agent_not_in_vscode(monkeypatch): @mock.patch.dict(os.environ, {"VSCODE_PID": "12345"}, clear=True) def test_user_agent_in_vscode(monkeypatch): monkeypatch_client_constructors(monkeypatch) + provider = create_clients_provider() + assert_clients_w_user_agent(provider, "vscode") + assert_clients_wo_user_agent(provider, "googlecloudtools.cloudcode") - with tempfile.TemporaryDirectory() as tmpdir: - user_home = pathlib.Path(tmpdir) - with mock.patch("pathlib.Path.home", return_value=user_home): - provider = create_clients_provider() - assert_clients_w_user_agent(provider, "vscode") - assert_clients_wo_user_agent(provider, "googlecloudtools.cloudcode") - - # We still need to include attribution to bigframes - assert_clients_w_user_agent( - provider, f"bigframes/{bigframes.version.__version__}" - ) + # We still need to include attribution to bigframes + assert_clients_w_user_agent(provider, f"bigframes/{bigframes.version.__version__}") @mock.patch.dict(os.environ, {"VSCODE_PID": "12345"}, clear=True) diff --git a/tests/unit/session/test_io_bigquery.py b/tests/unit/session/test_io_bigquery.py index e6fa7a901ec..eb58c6bb52d 100644 --- a/tests/unit/session/test_io_bigquery.py +++ b/tests/unit/session/test_io_bigquery.py @@ -24,10 +24,10 @@ import bigframes import bigframes.core.events +from bigframes.core.logging import log_adapter import bigframes.pandas as bpd import bigframes.session._io.bigquery import bigframes.session._io.bigquery as io_bq -from bigframes.core.logging import log_adapter from bigframes.testing import mocks @@ -66,6 +66,18 @@ def test_create_job_configs_labels_always_includes_bigframes_api(): } +def test_create_job_configs_labels_includes_extra_query_labels(): + user_labels = {"my-label-1": "my-value-1", "my-label-2": "my-value-2"} + + with bigframes.option_context("compute.extra_query_labels", user_labels): + labels = io_bq.create_job_configs_labels(None, []) + assert labels == { + "my-label-1": "my-value-1", + "my-label-2": "my-value-2", + "bigframes-api": "unknown", + } + + def test_create_job_configs_labels_length_limit_not_met(): cur_labels = { "source": "bigquery-dataframes-temp", @@ -172,7 +184,7 @@ def test_add_and_trim_labels_length_limit_met(): for _ in range(52): df.head() - io_bq.add_and_trim_labels(job_config=job_config, session=df._session) + io_bq.add_and_trim_labels(job_config=job_config) assert job_config.labels is not None assert len(job_config.labels) == 56 assert "dataframe-max" not in job_config.labels.values() @@ -185,7 +197,7 @@ def test_add_and_trim_labels_length_limit_met(): ("timeout", "api_name"), [(None, None), (30.0, "test_api")], ) -def test_start_query_with_job_labels_length_limit_met( +def test_start_query_with_client_labels_length_limit_met( mock_bq_client: bigquery.Client, timeout: Optional[float], api_name ): sql = "select * from abc" @@ -209,7 +221,7 @@ def test_start_query_with_job_labels_length_limit_met( for _ in range(52): df.head() - io_bq.start_query_with_job( + io_bq.start_query_with_client( mock_bq_client, sql, job_config=job_config, @@ -217,8 +229,8 @@ def test_start_query_with_job_labels_length_limit_met( project=None, timeout=timeout, metrics=None, + query_with_job=True, publisher=bigframes.core.events.Publisher(), - session=df._session, ) assert job_config.labels is not None @@ -332,8 +344,8 @@ def test_bq_schema_to_sql(schema: Iterable[bigquery.SchemaField], expected: str) 2024, 5, 14, 12, 42, 36, 125125, tzinfo=datetime.timezone.utc ), ( - "SELECT `_bf_source`.`row_index`, `_bf_source`.`string_col` FROM `test_table` AS _bf_source " - "FOR SYSTEM_TIME AS OF CAST('2024-05-14T12:42:36.125125+00:00' AS TIMESTAMP) " + "SELECT `row_index`, `string_col` FROM `test_table` " + "FOR SYSTEM_TIME AS OF TIMESTAMP('2024-05-14T12:42:36.125125+00:00') " "WHERE `rowindex` NOT IN (0, 6) OR `string_col` IN ('Hello, World!', " "'こんにちは') LIMIT 123" ), @@ -357,12 +369,12 @@ def test_bq_schema_to_sql(schema: Iterable[bigquery.SchemaField], expected: str) 2024, 5, 14, 12, 42, 36, 125125, tzinfo=datetime.timezone.utc ), ( - """SELECT `_bf_source`.`rowindex`, `_bf_source`.`string_col` FROM (SELECT + """SELECT `rowindex`, `string_col` FROM (SELECT rowindex, string_col, FROM `test_table` AS t - ) AS _bf_source """ - "FOR SYSTEM_TIME AS OF CAST('2024-05-14T12:42:36.125125+00:00' AS TIMESTAMP) " + ) """ + "FOR SYSTEM_TIME AS OF TIMESTAMP('2024-05-14T12:42:36.125125+00:00') " "WHERE `rowindex` < 4 AND `string_col` = 'Hello, World!' " "LIMIT 123" ), @@ -374,7 +386,7 @@ def test_bq_schema_to_sql(schema: Iterable[bigquery.SchemaField], expected: str) [], None, # max_results None, # time_travel_timestampe - "SELECT `_bf_source`.`col_a`, `_bf_source`.`col_b` FROM `test_table` AS _bf_source", + "SELECT `col_a`, `col_b` FROM `test_table`", id="table-columns", ), pytest.param( @@ -383,7 +395,7 @@ def test_bq_schema_to_sql(schema: Iterable[bigquery.SchemaField], expected: str) [("date_col", ">", "2022-10-20")], None, # max_results None, # time_travel_timestampe - "SELECT * FROM `test_table` AS _bf_source WHERE `date_col` > '2022-10-20'", + "SELECT * FROM `test_table` WHERE `date_col` > '2022-10-20'", id="table-filter", ), pytest.param( @@ -392,7 +404,7 @@ def test_bq_schema_to_sql(schema: Iterable[bigquery.SchemaField], expected: str) [], None, # max_results None, # time_travel_timestampe - "SELECT * FROM `test_table*` AS _bf_source", + "SELECT * FROM `test_table*`", id="wildcard-no_params", ), pytest.param( @@ -401,7 +413,7 @@ def test_bq_schema_to_sql(schema: Iterable[bigquery.SchemaField], expected: str) [("_TABLE_SUFFIX", ">", "2022-10-20")], None, # max_results None, # time_travel_timestampe - "SELECT * FROM `test_table*` AS _bf_source WHERE `_TABLE_SUFFIX` > '2022-10-20'", + "SELECT * FROM `test_table*` WHERE `_TABLE_SUFFIX` > '2022-10-20'", id="wildcard-filter", ), ], diff --git a/tests/unit/session/test_io_pandas.py b/tests/unit/session/test_io_pandas.py index f4141ec8a23..224f343c7e1 100644 --- a/tests/unit/session/test_io_pandas.py +++ b/tests/unit/session/test_io_pandas.py @@ -14,8 +14,8 @@ import datetime import re -import unittest.mock as mock from typing import Dict, Union +import unittest.mock as mock import geopandas # type: ignore import numpy diff --git a/tests/unit/session/test_local_scan_executor.py b/tests/unit/session/test_local_scan_executor.py index 66dcdf590ce..fc59253153b 100644 --- a/tests/unit/session/test_local_scan_executor.py +++ b/tests/unit/session/test_local_scan_executor.py @@ -13,19 +13,13 @@ # limitations under the License. from __future__ import annotations -import asyncio - import pyarrow import pytest from bigframes.core import identifiers, local_data, nodes -from bigframes.session import execution_spec, local_scan_executor +from bigframes.session import local_scan_executor from bigframes.testing import mocks -SPEC = execution_spec.ExecutionSpec( - ordered=True, -) - @pytest.fixture def object_under_test(): @@ -78,7 +72,7 @@ def test_local_scan_executor_with_slice(start, stop, expected_rows, object_under stop=stop, ) - result = asyncio.run(object_under_test.execute(plan, SPEC)) + result = object_under_test.execute(plan, ordered=True) result_table = pyarrow.Table.from_batches(result.batches().arrow_batches) assert result_table.num_rows == expected_rows @@ -104,4 +98,4 @@ def test_local_scan_executor_with_slice_unsupported_inputs( stop=stop, step=step, ) - assert asyncio.run(object_under_test.execute(plan, SPEC)) is None + assert object_under_test.execute(plan, ordered=True) is None diff --git a/tests/unit/session/test_metrics.py b/tests/unit/session/test_metrics.py index 4e550b1c77a..7c2f01c5b98 100644 --- a/tests/unit/session/test_metrics.py +++ b/tests/unit/session/test_metrics.py @@ -245,58 +245,3 @@ def test_write_stats_to_disk_no_env_var(tmp_path, monkeypatch): exec_seconds=1.23, ) assert len(list(tmp_path.iterdir())) == 0 - - -def test_on_event_with_local_execute_result(): - import bigframes.core.events - from bigframes.session.executor import LocalExecuteResult - - # fmt: off - local_result = unittest.mock.create_autospec( - LocalExecuteResult, instance=True - ) - # fmt: on - local_result.total_bytes_processed = 1024 - - event = bigframes.core.events.ExecutionFinished(result=local_result) - envelope = bigframes.core.events.EventEnvelope(event) - execution_metrics = metrics.ExecutionMetrics() - execution_metrics.on_event(envelope) - - assert execution_metrics.execution_count == 1 - assert len(execution_metrics.jobs) == 1 - assert execution_metrics.jobs[0].job_type == "polars" - assert execution_metrics.jobs[0].status == "DONE" - assert execution_metrics.jobs[0].total_bytes_processed == 1024 - - -def test_count_job_stats_with_explicit_cell_execution_count(): - row_iterator = unittest.mock.create_autospec( - bigquery.table.RowIterator, instance=True - ) - row_iterator.total_bytes_processed = 1024 - row_iterator.query = "SELECT * FROM table" - row_iterator.slot_millis = 1234 - execution_metrics = metrics.ExecutionMetrics() - execution_metrics.count_job_stats( - row_iterator=row_iterator, cell_execution_count=42 - ) - - assert len(execution_metrics.jobs) == 1 - assert execution_metrics.jobs[0].cell_execution_count == 42 - - -def test_on_event_with_explicit_cell_execution_count(): - import bigframes.core.events - from bigframes.session.executor import LocalExecuteResult - - local_result = unittest.mock.create_autospec(LocalExecuteResult, instance=True) - local_result.total_bytes_processed = 1024 - - event = bigframes.core.events.ExecutionFinished(result=local_result) - envelope = bigframes.core.events.EventEnvelope(event=event, cell_execution_count=42) - execution_metrics = metrics.ExecutionMetrics() - execution_metrics.on_event(envelope) - - assert len(execution_metrics.jobs) == 1 - assert execution_metrics.jobs[0].cell_execution_count == 42 diff --git a/tests/unit/session/test_proxy_executor.py b/tests/unit/session/test_proxy_executor.py deleted file mode 100644 index c20fd57236b..00000000000 --- a/tests/unit/session/test_proxy_executor.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from unittest import mock - -import google.cloud.bigquery as bigquery -import google.cloud.exceptions -import pytest - -import bigframes -from bigframes.session.proxy_executor import DualCompilerProxyExecutor - - -@pytest.fixture -def mock_executor(): - bqclient = mock.create_autospec(bigquery.Client) - bqclient.project = "test-project" - storage_manager = mock.Mock() - bqstoragereadclient = mock.Mock() - loader = mock.Mock() - publisher = mock.Mock() - function_manager = mock.Mock() - return DualCompilerProxyExecutor( - bqclient, - storage_manager, - bqstoragereadclient, - loader, - publisher=publisher, - function_manager=function_manager, - ) - - -def test_execute_legacy_routes_to_ibis(mock_executor, monkeypatch): - array_value = mock.Mock(spec=bigframes.core.ArrayValue) - execution_spec = mock.Mock(spec=bigframes.session.execution_spec.ExecutionSpec) - execution_spec.with_bq_labels.return_value = execution_spec - - mock_executor._ibis_executor = mock.Mock() - mock_executor._sqlglot_executor = mock.Mock() - - monkeypatch.setattr(bigframes.options.experiments, "sql_compiler", "legacy") - mock_executor.execute(array_value, execution_spec) - - execution_spec.with_bq_labels.assert_called_once_with( - {"bigframes-compiler": "ibis"} - ) - mock_executor._ibis_executor.execute.assert_called_once_with( - array_value, execution_spec - ) - mock_executor._sqlglot_executor.execute.assert_not_called() - - -def test_execute_experimental_routes_to_sqlglot(mock_executor, monkeypatch): - array_value = mock.Mock(spec=bigframes.core.ArrayValue) - execution_spec = mock.Mock(spec=bigframes.session.execution_spec.ExecutionSpec) - execution_spec.with_bq_labels.return_value = execution_spec - - mock_executor._ibis_executor = mock.Mock() - mock_executor._sqlglot_executor = mock.Mock() - - monkeypatch.setattr(bigframes.options.experiments, "sql_compiler", "experimental") - mock_executor.execute(array_value, execution_spec) - - execution_spec.with_bq_labels.assert_called_once_with( - {"bigframes-compiler": "sqlglot"} - ) - mock_executor._sqlglot_executor.execute.assert_called_once_with( - array_value, execution_spec - ) - mock_executor._ibis_executor.execute.assert_not_called() - - -def test_execute_stable_routes_to_sqlglot_success(mock_executor, monkeypatch): - array_value = mock.Mock(spec=bigframes.core.ArrayValue) - execution_spec = mock.Mock(spec=bigframes.session.execution_spec.ExecutionSpec) - execution_spec.with_bq_labels.return_value = execution_spec - - mock_executor._ibis_executor = mock.Mock() - mock_executor._sqlglot_executor = mock.Mock() - - monkeypatch.setattr(bigframes.options.experiments, "sql_compiler", "stable") - with mock.patch("uuid.uuid1") as mock_uuid: - mock_uuid.return_value.hex = "1234567890123456" - mock_executor.execute(array_value, execution_spec) - - execution_spec.with_bq_labels.assert_called_once_with( - {"bigframes-compiler": "sqlglot-123456789012"} - ) - mock_executor._sqlglot_executor.execute.assert_called_once_with( - array_value, execution_spec - ) - mock_executor._ibis_executor.execute.assert_not_called() - - -def test_execute_stable_routes_to_sqlglot_fallback_to_ibis(mock_executor, monkeypatch): - array_value = mock.Mock(spec=bigframes.core.ArrayValue) - execution_spec = mock.Mock(spec=bigframes.session.execution_spec.ExecutionSpec) - - spec_sqlglot = mock.Mock(spec=bigframes.session.execution_spec.ExecutionSpec) - spec_ibis = mock.Mock(spec=bigframes.session.execution_spec.ExecutionSpec) - execution_spec.with_bq_labels.side_effect = [spec_sqlglot, spec_ibis] - - mock_executor._ibis_executor = mock.Mock() - mock_executor._sqlglot_executor = mock.Mock() - - mock_executor._sqlglot_executor.execute.side_effect = ( - google.cloud.exceptions.BadRequest("test error") - ) - - monkeypatch.setattr(bigframes.options.experiments, "sql_compiler", "stable") - with mock.patch("uuid.uuid1") as mock_uuid: - mock_uuid.return_value.hex = "1234567890123456" - with pytest.warns( - UserWarning, match="Compiler ID 123456789012: Exception on sqlglot" - ): - mock_executor.execute(array_value, execution_spec) - - execution_spec.with_bq_labels.assert_has_calls( - [ - mock.call({"bigframes-compiler": "sqlglot-123456789012"}), - mock.call({"bigframes-compiler": "ibis-123456789012"}), - ] - ) - - mock_executor._sqlglot_executor.execute.assert_called_once_with( - array_value, spec_sqlglot - ) - mock_executor._ibis_executor.execute.assert_called_once_with(array_value, spec_ibis) - - -def test_cached_legacy_routes_to_ibis(mock_executor, monkeypatch): - array_value = mock.Mock(spec=bigframes.core.ArrayValue) - config = mock.Mock() - - mock_executor._ibis_executor = mock.Mock() - mock_executor._sqlglot_executor = mock.Mock() - - monkeypatch.setattr(bigframes.options.experiments, "sql_compiler", "legacy") - mock_executor.cached(array_value, config=config) - - mock_executor._ibis_executor.cached.assert_called_once_with( - array_value, config=config - ) - mock_executor._sqlglot_executor.cached.assert_not_called() - - -def test_cached_experimental_routes_to_sqlglot(mock_executor, monkeypatch): - array_value = mock.Mock(spec=bigframes.core.ArrayValue) - config = mock.Mock() - - mock_executor._ibis_executor = mock.Mock() - mock_executor._sqlglot_executor = mock.Mock() - - monkeypatch.setattr(bigframes.options.experiments, "sql_compiler", "experimental") - mock_executor.cached(array_value, config=config) - - mock_executor._sqlglot_executor.cached.assert_called_once_with( - array_value, config=config - ) - mock_executor._ibis_executor.cached.assert_not_called() - - -def test_cached_stable_routes_to_sqlglot_fallback_to_ibis(mock_executor, monkeypatch): - array_value = mock.Mock(spec=bigframes.core.ArrayValue) - config = mock.Mock() - - mock_executor._ibis_executor = mock.Mock() - mock_executor._sqlglot_executor = mock.Mock() - - mock_executor._sqlglot_executor.cached.side_effect = ( - google.cloud.exceptions.BadRequest("test error") - ) - - monkeypatch.setattr(bigframes.options.experiments, "sql_compiler", "stable") - with mock.patch("uuid.uuid1") as mock_uuid: - mock_uuid.return_value.hex = "1234567890123456" - with pytest.warns( - UserWarning, match="Compiler ID 123456789012: Exception on sqlglot" - ): - mock_executor.cached(array_value, config=config) - - mock_executor._sqlglot_executor.cached.assert_called_once_with( - array_value, config=config - ) - mock_executor._ibis_executor.cached.assert_called_once_with( - array_value, config=config - ) diff --git a/tests/unit/session/test_read_gbq_colab.py b/tests/unit/session/test_read_gbq_colab.py index fc4181b6a2b..cc0508b75a1 100644 --- a/tests/unit/session/test_read_gbq_colab.py +++ b/tests/unit/session/test_read_gbq_colab.py @@ -18,10 +18,10 @@ import textwrap from unittest import mock +from google.cloud import bigquery import numpy import pandas import pytest -from google.cloud import bigquery from bigframes.testing import mocks @@ -47,36 +47,6 @@ def test_read_gbq_colab_includes_label(): assert "session-read_gbq_colab" in label_values -def test_read_gbq_colab_includes_label_in_anywidget_mode(): - """Make sure read_gbq_colab label is preserved in recent-bigframes-api labels in anywidget mode.""" - pytest.importorskip("anywidget") - pytest.importorskip("traitlets") - - import bigframes - import bigframes.display.html as bf_html - - bqclient = mock.create_autospec(bigquery.Client, instance=True) - bqclient.project = "proj" - session = mocks.create_bigquery_session(bqclient=bqclient) - df = session._read_gbq_colab("SELECT 'read-gbq-colab-test'") - - with bigframes.option_context("display.render_mode", "anywidget"): - _ = bf_html.get_anywidget_bundle(df) - - label_values = [] - for kall in itertools.chain( - bqclient.query_and_wait.call_args_list, - bqclient._query_and_wait_bigframes.call_args_list, - bqclient.query.call_args_list, - ): - job_config = kall.kwargs.get("job_config") - if job_config is None: - continue - label_values.extend(job_config.labels.values()) - - assert "session-read_gbq_colab" in label_values - - @pytest.mark.parametrize("dry_run", [True, False]) def test_read_gbq_colab_includes_formatted_values_in_dry_run(monkeypatch, dry_run): bqclient = mock.create_autospec(bigquery.Client, instance=True) @@ -156,91 +126,3 @@ def test_read_gbq_colab_doesnt_set_destination_table(): assert query == "SELECT 'my-test-query';" assert config.destination is None - - -def test_read_gbq_colab_with_callback(): - """Make sure callback receives events during execution.""" - session = mocks.create_bigquery_session() - callback = mock.Mock() - - _ = session._read_gbq_colab("SELECT 'my-test-query';", callback=callback) - - assert callback.call_count > 0 - - -def test_read_gbq_colab_filters_by_cell(): - """Verify that callbacks are scoped to individual executions.""" - session = mocks.create_bigquery_session() - callback1 = mock.Mock() - callback2 = mock.Mock() - - _ = session._read_gbq_colab("SELECT 'cell_1_query';", callback=callback1) - callback1_initial_count = callback1.call_count - - _ = session._read_gbq_colab("SELECT 'cell_2_query';", callback=callback2) - - # Verify callback1 was automatically unsubscribed upon completion - # of the first query. - assert callback1.call_count == callback1_initial_count - assert callback2.call_count > 0 - - -def test_execution_history_filtering(): - """Verify that execution_history can be filtered by job_ids or events.""" - from bigframes.session import metrics - - session = mocks.create_bigquery_session() - - job1 = metrics.JobMetadata(job_id="job_1", job_type="query", query="SELECT 1") - job2 = metrics.JobMetadata(job_id="job_2", job_type="query", query="SELECT 2") - session._metrics.jobs.extend([job1, job2]) - - history_job1 = session.execution_history(job_ids=["job_1"]).to_dataframe() - assert len(history_job1) == 1 - assert history_job1.iloc[0]["job_id"] == "job_1" - - event2 = mock.Mock() - event2.job_id = "job_2" - history_job2 = session.execution_history(events=[event2]).to_dataframe() - assert len(history_job2) == 1 - assert history_job2.iloc[0]["job_id"] == "job_2" - - -def test_execution_history_returns_all_executions_by_default(): - """Verify that execution_history returns all executions by default.""" - from bigframes.session import metrics - - session = mocks.create_bigquery_session() - job1 = metrics.JobMetadata( - job_id="job_1", job_type="query", query="SELECT 1", cell_execution_count=10 - ) - job2 = metrics.JobMetadata( - job_id="job_2", job_type="query", query="SELECT 2", cell_execution_count=20 - ) - session._metrics.jobs.extend([job1, job2]) - - history = session.execution_history().to_dataframe() - - assert len(history) == 2 - - -def test_execution_history_filters_by_notebook_cell_when_all_cells_is_false(): - """Verify that execution_history filters to the current cell when all_cells is False.""" - from bigframes.session import metrics - - session = mocks.create_bigquery_session() - job1 = metrics.JobMetadata( - job_id="job_1", job_type="query", query="SELECT 1", cell_execution_count=10 - ) - job2 = metrics.JobMetadata( - job_id="job_2", job_type="query", query="SELECT 2", cell_execution_count=20 - ) - session._metrics.jobs.extend([job1, job2]) - - with mock.patch( - "bigframes.core.utils.get_ipython_execution_count", return_value=20 - ): - history = session.execution_history(all_cells=False).to_dataframe() - - assert len(history) == 1 - assert history.iloc[0]["job_id"] == "job_2" diff --git a/tests/unit/session/test_read_gbq_table.py b/tests/unit/session/test_read_gbq_table.py index 97ac0efb753..ce9b587d6bd 100644 --- a/tests/unit/session/test_read_gbq_table.py +++ b/tests/unit/session/test_read_gbq_table.py @@ -23,7 +23,6 @@ import bigframes.enums import bigframes.exceptions import bigframes.session._io.bigquery.read_gbq_table as bf_read_gbq_table -from bigframes.core import bq_data from bigframes.testing import mocks @@ -82,9 +81,7 @@ def test_infer_unique_columns(index_cols, primary_keys, expected): }, } - result = bf_read_gbq_table.infer_unique_columns( - bq_data.GbqNativeTable.from_table(table), index_cols - ) + result = bf_read_gbq_table.infer_unique_columns(table, index_cols) assert result == expected @@ -143,7 +140,7 @@ def test_check_if_index_columns_are_unique(index_cols, values_distinct, expected result = bf_read_gbq_table.check_if_index_columns_are_unique( bqclient=bqclient, - table=bq_data.GbqNativeTable.from_table(table), + table=table, index_cols=index_cols, publisher=session._publisher, ) @@ -173,7 +170,7 @@ def test_get_index_cols_warns_if_clustered_but_sequential_index(): with pytest.warns(bigframes.exceptions.DefaultIndexWarning, match="is clustered"): bf_read_gbq_table.get_index_cols( - bq_data.GbqNativeTable.from_table(table), + table, index_col=(), default_index_type=bigframes.enums.DefaultIndexKind.SEQUENTIAL_INT64, ) @@ -185,7 +182,7 @@ def test_get_index_cols_warns_if_clustered_but_sequential_index(): "error", category=bigframes.exceptions.DefaultIndexWarning ) bf_read_gbq_table.get_index_cols( - bq_data.GbqNativeTable.from_table(table), + table, index_col=(), default_index_type=bigframes.enums.DefaultIndexKind.NULL, ) diff --git a/tests/unit/session/test_session.py b/tests/unit/session/test_session.py index a6c8446967e..fe73643b0c8 100644 --- a/tests/unit/session/test_session.py +++ b/tests/unit/session/test_session.py @@ -16,8 +16,8 @@ import datetime import os import re -import warnings from unittest import mock +import warnings import google.api_core.exceptions import google.cloud.bigquery @@ -25,10 +25,9 @@ import pytest import bigframes +from bigframes import version import bigframes.enums import bigframes.exceptions -from bigframes import version -from bigframes.core import bq_data from bigframes.testing import mocks TABLE_REFERENCE = { @@ -244,7 +243,7 @@ def test_read_gbq_cached_table(): table._properties["type"] = "TABLE" session._loader._df_snapshot[str(table_ref)] = ( datetime.datetime(1999, 1, 2, 3, 4, 5, 678901, tzinfo=datetime.timezone.utc), - bq_data.GbqNativeTable.from_table(table), + table, ) session.bqclient._query_and_wait_bigframes = mock.MagicMock( @@ -275,7 +274,7 @@ def test_read_gbq_cached_table_doesnt_warn_for_anonymous_tables_and_doesnt_inclu table._properties["type"] = "TABLE" session._loader._df_snapshot[str(table_ref)] = ( datetime.datetime(1999, 1, 2, 3, 4, 5, 678901, tzinfo=datetime.timezone.utc), - bq_data.GbqNativeTable.from_table(table), + table, ) session.bqclient._query_and_wait_bigframes = mock.MagicMock( diff --git a/tests/unit/test_clients.py b/tests/unit/test_clients.py index 08d111b8662..9daa7598382 100644 --- a/tests/unit/test_clients.py +++ b/tests/unit/test_clients.py @@ -14,9 +14,9 @@ from unittest import mock -import pytest from google.cloud import bigquery_connection_v1, resourcemanager_v3 from google.iam.v1 import policy_pb2 +import pytest from bigframes import clients diff --git a/tests/unit/test_col.py b/tests/unit/test_col.py deleted file mode 100644 index c8caf9136c0..00000000000 --- a/tests/unit/test_col.py +++ /dev/null @@ -1,269 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import operator -import pathlib -from typing import Generator - -import numpy as np -import pandas as pd -import pytest - -import bigframes -import bigframes.pandas as bpd -from bigframes.testing.utils import assert_frame_equal, convert_pandas_dtypes - -pytest.importorskip("polars") -pytest.importorskip("pandas", minversion="3.0.0") - - -CURRENT_DIR = pathlib.Path(__file__).parent -DATA_DIR = CURRENT_DIR.parent / "data" - - -@pytest.fixture(scope="module", autouse=True) -def session() -> Generator[bigframes.Session, None, None]: - import bigframes.core.global_session - from bigframes.testing import polars_session - - session = polars_session.TestSession() - with bigframes.core.global_session._GlobalSessionContext(session): - yield session - - -@pytest.fixture(scope="module") -def scalars_pandas_df_index() -> pd.DataFrame: - """pd.DataFrame pointing at test data.""" - - df = pd.read_json( - DATA_DIR / "scalars.jsonl", - lines=True, - ) - convert_pandas_dtypes(df, bytes_col=True) - - df = df.set_index("rowindex", drop=False) - df.index.name = None - return df.set_index("rowindex").sort_index() - - -@pytest.fixture(scope="module") -def scalars_df_index( - session: bigframes.Session, scalars_pandas_df_index -) -> bpd.DataFrame: - return session.read_pandas(scalars_pandas_df_index) - - -@pytest.fixture(scope="module") -def scalars_df_2_index( - session: bigframes.Session, scalars_pandas_df_index -) -> bpd.DataFrame: - return session.read_pandas(scalars_pandas_df_index) - - -@pytest.fixture(scope="module") -def scalars_dfs( - scalars_df_index, - scalars_pandas_df_index, -): - return scalars_df_index, scalars_pandas_df_index - - -@pytest.mark.parametrize( - ("op",), - [ - (operator.invert,), - ], -) -def test_pd_col_unary_operators(scalars_dfs, op): - scalars_df, scalars_pandas_df = scalars_dfs - bf_kwargs = { - "result": op(bpd.col("bool_col")), - } - pd_kwargs = { - "result": op(pd.col("bool_col")), # type: ignore - } - df = scalars_df.assign(**bf_kwargs) - - bf_result = df.to_pandas() - pd_result = scalars_pandas_df.assign(**pd_kwargs) - - assert_frame_equal(bf_result, pd_result) - - -@pytest.mark.parametrize( - ("op"), - [ - (lambda x: x.sum()), - (lambda x: x.mean()), - (lambda x: x.min()), - (lambda x: x.max()), - (lambda x: x.std()), - (lambda x: x.var()), - ], - ids=[ - "sum", - "mean", - "min", - "max", - "std", - "var", - ], -) -def test_pd_col_aggregate_op(scalars_dfs, op): - scalars_df, scalars_pandas_df = scalars_dfs - bf_kwargs = { - "result": op(bpd.col("float64_col")), - } - pd_kwargs = { - "result": op(pd.col("float64_col")), # type: ignore - } - df = scalars_df.assign(**bf_kwargs) - - bf_result = df.to_pandas() - pd_result = scalars_pandas_df.assign(**pd_kwargs) - - assert_frame_equal(bf_result, pd_result) - - -def test_pd_col_aggregate_of_aggregate(scalars_dfs): - scalars_df, scalars_pandas_df = scalars_dfs - bf_kwargs = { - "result": (bpd.col("int64_col") - bpd.col("int64_col").mean()).mean(), - } - pd_kwargs = { - "result": (pd.col("int64_col") - pd.col("int64_col").mean()).mean(), # type: ignore - } - df = scalars_df.assign(**bf_kwargs) - - bf_result = df.to_pandas() - pd_result = scalars_pandas_df.assign(**pd_kwargs) - - assert_frame_equal(bf_result, pd_result) - - -@pytest.mark.parametrize( - ("op",), - [ - (operator.add,), - (operator.sub,), - (operator.mul,), - (operator.truediv,), - (operator.floordiv,), - (operator.gt,), - (operator.lt,), - (operator.ge,), - (operator.le,), - (operator.eq,), - (operator.mod,), - ], -) -def test_pd_col_binary_operators(scalars_dfs, op): - scalars_df, scalars_pandas_df = scalars_dfs - bf_kwargs = { - "result": op(bpd.col("float64_col"), 2.4), - "reverse_result": op(2.4, bpd.col("float64_col")), - } - pd_kwargs = { - "result": op(pd.col("float64_col"), 2.4), # type: ignore - "reverse_result": op(2.4, pd.col("float64_col")), # type: ignore - } - df = scalars_df.assign(**bf_kwargs) - - bf_result = df.to_pandas() - pd_result = scalars_pandas_df.assign(**pd_kwargs) - - assert_frame_equal(bf_result, pd_result) - - -@pytest.mark.parametrize( - ("op",), - [ - (operator.and_,), - (operator.or_,), - (operator.xor,), - ], -) -def test_pd_col_binary_bool_operators(scalars_dfs, op): - scalars_df, scalars_pandas_df = scalars_dfs - bf_kwargs = { - "result": op(bpd.col("bool_col"), True), - "reverse_result": op(False, bpd.col("bool_col")), - } - pd_kwargs = { - "result": op(pd.col("bool_col"), True), # type: ignore - "reverse_result": op(False, pd.col("bool_col")), # type: ignore - } - df = scalars_df.assign(**bf_kwargs) - - bf_result = df.to_pandas() - pd_result = scalars_pandas_df.assign(**pd_kwargs) - - assert_frame_equal(bf_result, pd_result) - - -def test_loc_with_pd_col(scalars_dfs): - scalars_df, scalars_pandas_df = scalars_dfs - - bf_result = scalars_df.loc[bpd.col("float64_col") > 4].to_pandas() - pd_result = scalars_pandas_df.loc[pd.col("float64_col") > 4] # type: ignore - - assert_frame_equal(bf_result, pd_result) - - -def test_getitem_with_pd_col(scalars_dfs): - scalars_df, scalars_pandas_df = scalars_dfs - - bf_result = scalars_df[bpd.col("float64_col") > 4].to_pandas() - pd_result = scalars_pandas_df[pd.col("float64_col") > 4] # type: ignore - - assert_frame_equal(bf_result, pd_result) - - -def test_col_str_accessor(scalars_dfs): - scalars_df, scalars_pandas_df = scalars_dfs - - bf_result = scalars_df.assign(result=bpd.col("string_col").str.lower()).to_pandas() - pd_result = scalars_pandas_df.assign(result=pd.col("string_col").str.lower()) # type: ignore - - assert_frame_equal(bf_result, pd_result) - - -def test_col_dt_accessor(scalars_dfs): - scalars_df, scalars_pandas_df = scalars_dfs - - bf_result = scalars_df.assign(result=bpd.col("date_col").dt.year).to_pandas() - pd_result = scalars_pandas_df.assign(result=pd.col("date_col").dt.year) # type: ignore - - # int64[pyarrow] vs Int64 - assert_frame_equal(bf_result, pd_result, check_dtype=False) - - -def test_col_numpy_ufunc(scalars_dfs): - scalars_df, scalars_pandas_df = scalars_dfs - - bf_result = scalars_df.assign( - sqrt=np.sqrt(bpd.col("float64_col")), # type: ignore - add_const=np.add(bpd.col("float64_col"), 2.4), # type: ignore - radd_const=np.add(2.4, bpd.col("float64_col")), # type: ignore - add_cols=np.add(bpd.col("float64_col"), bpd.col("int64_col")), # type: ignore - ).to_pandas() - pd_result = scalars_pandas_df.assign( - sqrt=np.sqrt(pd.col("float64_col")), # type: ignore - add_const=np.add(pd.col("float64_col"), 2.4), # type: ignore - radd_const=np.add(2.4, pd.col("float64_col")), # type: ignore - add_cols=np.add(pd.col("float64_col"), pd.col("int64_col")), # type: ignore - ) - - # int64[pyarrow] vs Int64 - assert_frame_equal(bf_result, pd_result, check_dtype=False) diff --git a/tests/unit/test_dataframe.py b/tests/unit/test_dataframe.py index d045bf7c3fc..015dbd030e3 100644 --- a/tests/unit/test_dataframe.py +++ b/tests/unit/test_dataframe.py @@ -223,3 +223,23 @@ def test_dataframe_drop_columns_returns_new_dataframe(monkeypatch: pytest.Monkey new_dataframe = dataframe.drop(columns=["col1", "col3"]) assert dataframe.columns.to_list() == ["col1", "col2", "col3"] assert new_dataframe.columns.to_list() == ["col2"] + + +def test_dataframe_semantics_property_future_warning( + monkeypatch: pytest.MonkeyPatch, +): + dataframe = mocks.create_dataframe(monkeypatch) + + with bigframes.option_context("experiments.semantic_operators", True), pytest.warns( + FutureWarning + ): + dataframe.semantics + + +def test_dataframe_ai_property_future_warning( + monkeypatch: pytest.MonkeyPatch, +): + dataframe = mocks.create_dataframe(monkeypatch) + + with pytest.warns(FutureWarning): + dataframe.ai diff --git a/tests/unit/test_dataframe_polars.py b/tests/unit/test_dataframe_polars.py index c2dc979b71e..1c73d9dc6b0 100644 --- a/tests/unit/test_dataframe_polars.py +++ b/tests/unit/test_dataframe_polars.py @@ -828,26 +828,6 @@ def test_assign_new_column(scalars_dfs): assert_frame_equal(bf_result, pd_result) -def test_assign_using_pd_col(scalars_dfs): - if pd.__version__.startswith("1.") or pd.__version__.startswith("2."): - pytest.skip("col expression interface only supported for pandas 3+") - scalars_df, scalars_pandas_df = scalars_dfs - bf_kwargs = { - "new_col_1": 4 - bpd.col("int64_col"), - "new_col_2": bpd.col("int64_col") / (bpd.col("float64_col") * 0.5), - } - pd_kwargs = { - "new_col_1": 4 - pd.col("int64_col"), # type: ignore - "new_col_2": pd.col("int64_col") / (pd.col("float64_col") * 0.5), # type: ignore - } - - df = scalars_df.assign(**bf_kwargs) - bf_result = df.to_pandas() - pd_result = scalars_pandas_df.assign(**pd_kwargs) - - assert_frame_equal(bf_result, pd_result) - - def test_assign_new_column_w_loc(scalars_dfs): scalars_df, scalars_pandas_df = scalars_dfs bf_df = scalars_df.copy() @@ -1287,49 +1267,6 @@ def test_apply_series_scalar_callable( pandas.testing.assert_series_equal(bf_result, pd_result) -def test_df_map_with_udf(session): - df = bpd.DataFrame({"x": [1, 2, None, 4], "y": [5, None, 7, 8]}, dtype="Int64") - - @session.udf() - def foo(row: pd.Series) -> int: - if pd.isna(row["x"]) or pd.isna(row["y"]): - return -1 - return int(row["x"] * row["y"]) - - bf_result = df.apply(foo, axis=1).to_pandas() - pd_result = pd.Series([5, -1, -1, 32]) - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_df_apply_complex_udf(session): - df = bpd.DataFrame( - {"x": [1, 2, 3], "y": ["a", "b", "c"]}, - index=["row0", "row1", "row2"], - ) - - @session.udf() - def foo(row: pd.Series) -> str: - idx = str(row.name) - items_str = ";".join(f"{k}={v}" for k, v in row.items()) - return f"({idx}) -> {items_str}" - - bf_result = df.apply(foo, axis=1).to_pandas() - - pd_df = pd.DataFrame( - {"x": [1, 2, 3], "y": ["a", "b", "c"]}, - index=["row0", "row1", "row2"], - ) - - def pd_foo(row): - idx = str(row.name) - items_str = ";".join(f"{k}={v}" for k, v in row.items()) - return f"({idx}) -> {items_str}" - - pd_result = pd_df.apply(pd_foo, axis=1) - - assert_series_equal(bf_result, pd_result, check_dtype=False, check_index_type=False) - - def test_df_pipe( scalars_df_index, scalars_pandas_df_index, @@ -2042,6 +1979,7 @@ def test_df_idxmax(): ], ) def test_df_align(join, axis): + index1: pandas.Index = pandas.Index([1, 2, 3, 4], dtype="Int64") index2: pandas.Index = pandas.Index([1, 2, 4, 5], dtype="Int64") @@ -3968,6 +3906,7 @@ def test_iloc_list_multiindex(scalars_dfs): def test_iloc_empty_list(scalars_df_index, scalars_pandas_df_index): + index_list: List[int] = [] bf_result = scalars_df_index.iloc[index_list] @@ -4157,12 +4096,9 @@ def test_df_to_dict(scalars_df_index, scalars_pandas_df_index): def test_df_to_json_local_str(scalars_df_index, scalars_pandas_df_index): - # pandas 3.0 bugged for serializing date col - bf_result = scalars_df_index.drop(columns="date_col").to_json() + bf_result = scalars_df_index.to_json() # default_handler for arrow types that have no default conversion - pd_result = scalars_pandas_df_index.drop(columns="date_col").to_json( - default_handler=str - ) + pd_result = scalars_pandas_df_index.to_json(default_handler=str) assert bf_result == pd_result @@ -4173,10 +4109,7 @@ def test_df_to_json_local_file(scalars_df_index, scalars_pandas_df_index): # duration not fully supported at pandas level scalars_df_index = scalars_df_index.drop(columns="duration_col") scalars_pandas_df_index = scalars_pandas_df_index.drop(columns="duration_col") - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.to_json(bf_result_file, orient="table") # default_handler for arrow types that have no default conversion scalars_pandas_df_index.to_json( @@ -4198,10 +4131,7 @@ def test_df_to_csv_local_str(scalars_df_index, scalars_pandas_df_index): def test_df_to_csv_local_file(scalars_df_index, scalars_pandas_df_index): - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.to_csv(bf_result_file) scalars_pandas_df_index.to_csv(pd_result_file) @@ -4225,10 +4155,7 @@ def test_df_to_parquet_local_bytes(scalars_df_index, scalars_pandas_df_index): def test_df_to_parquet_local_file(scalars_df_index, scalars_pandas_df_index): # GEOGRAPHY not supported in parquet export. unsupported = ["geography_col"] - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.drop(columns=unsupported).to_parquet(bf_result_file) scalars_pandas_df_index.drop(columns=unsupported).to_parquet(pd_result_file) @@ -4275,10 +4202,7 @@ def test_df_to_markdown(scalars_df_index, scalars_pandas_df_index): def test_df_to_pickle(scalars_df_index, scalars_pandas_df_index): - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.to_pickle(bf_result_file) scalars_pandas_df_index.to_pickle(pd_result_file) bf_result = bf_result_file.read() @@ -4526,10 +4450,3 @@ def test_dataframe_explode_reserve_order(session, ignore_index, ordered): def test_dataframe_explode_xfail(col_names): df = bpd.DataFrame({"A": [[0, 1, 2], [], [3, 4]]}) df.explode(col_names) - - -def test_recursion_limit_unit(scalars_df_index): - scalars_df_index = scalars_df_index[["int64_too", "int64_col", "float64_col"]] - for i in range(250): - scalars_df_index = scalars_df_index + 4 - scalars_df_index.to_pandas() diff --git a/tests/unit/test_dtypes.py b/tests/unit/test_dtypes.py index bb2b57d4090..0e600de9645 100644 --- a/tests/unit/test_dtypes.py +++ b/tests/unit/test_dtypes.py @@ -71,11 +71,3 @@ def test_infer_literal_type_arrow_scalar(scalar, expected_dtype): ) def test_contains_db_dtypes_json_arrow_type(type_, expected): assert bigframes.dtypes.contains_db_dtypes_json_arrow_type(type_) == expected - - -def test_convert_to_schema_field_list_description(): - bf_dtype = bigframes.dtypes.OBJ_REF_DTYPE - list_bf_dtype = bigframes.dtypes.list_type(bf_dtype) - field = bigframes.dtypes.convert_to_schema_field("my_list", list_bf_dtype) - assert field.description == "bigframes_dtype: OBJ_REF_DTYPE" - assert field.mode == "REPEATED" diff --git a/tests/unit/test_formatting_helpers.py b/tests/unit/test_formatting_helpers.py index 8917f540501..7a1cf1ab13a 100644 --- a/tests/unit/test_formatting_helpers.py +++ b/tests/unit/test_formatting_helpers.py @@ -197,44 +197,3 @@ def test_render_bqquery_finished_event_plaintext(): assert "finished" in text assert "1.0 kB processed" in text assert "Slot time: 2 seconds" in text - - -def test_get_job_url(): - job_id = "my-job-id" - location = "us-central1" - project_id = "my-project" - expected_url = ( - f"https://console.cloud.google.com/bigquery?project={project_id}" - f"&j=bq:{location}:{job_id}&page=queryresults" - ) - - actual_url = formatting_helpers.get_job_url( - job_id=job_id, location=location, project_id=project_id - ) - assert actual_url == expected_url - - -def test_progress_callback_falls_back_to_global(): - event = bfevents.BigQuerySentEvent( - query="SELECT * FROM my_table", - ) - envelope = bfevents.EventEnvelope(event=event, progress_bar=bfevents._DEFAULT) - - with mock.patch("bigframes._config.options.display.progress_bar", "terminal"): - with mock.patch("bigframes.formatting_helpers.in_ipython", return_value=False): - with mock.patch("builtins.print") as mock_print: - formatting_helpers.create_progress_callback()(envelope) - mock_print.assert_called_once() - - -def test_progress_callback_respects_envelope_progress_bar(): - event = bfevents.BigQuerySentEvent( - query="SELECT * FROM my_table", - ) - envelope = bfevents.EventEnvelope(event=event, progress_bar=None) - - with mock.patch("bigframes._config.options.display.progress_bar", "terminal"): - with mock.patch("bigframes.formatting_helpers.in_ipython", return_value=False): - with mock.patch("builtins.print") as mock_print: - formatting_helpers.create_progress_callback()(envelope) - mock_print.assert_not_called() diff --git a/tests/unit/test_groupby_transpile.py b/tests/unit/test_groupby_transpile.py deleted file mode 100644 index 4f841bc1b2d..00000000000 --- a/tests/unit/test_groupby_transpile.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pathlib import Path - -import pandas as pd -import pytest -from pandas.testing import assert_frame_equal, assert_series_equal - -import bigframes -import bigframes.core.global_session -import bigframes.pandas as bpd -from bigframes.testing.utils import convert_pandas_dtypes - -pytest.importorskip("polars") -pytest.importorskip("pandas", minversion="2.0.0") - -CURRENT_DIR = Path(__file__).parent -DATA_DIR = CURRENT_DIR.parent / "data" - - -@pytest.fixture(scope="module") -def scalars_pandas_df_index(): - df = pd.read_json( - DATA_DIR / "scalars.jsonl", - lines=True, - ) - convert_pandas_dtypes(df, bytes_col=True) - - df = df.set_index("rowindex", drop=False) - df.index.name = None - return df.set_index("rowindex").sort_index() - - -@pytest.fixture(scope="module", autouse=True) -def session(): - # import inline to allow polars importorskip to happen first - from bigframes.testing import polars_session - - with bpd.option_context("experiments.enable_python_transpiler", True): - session = polars_session.TestSession() - with bigframes.core.global_session._GlobalSessionContext(session): - yield session - - -@pytest.fixture(scope="module") -def scalars_df_index( - session: bigframes.Session, scalars_pandas_df_index -) -> bpd.DataFrame: - return session.read_pandas(scalars_pandas_df_index) - - -# Tests for groupby.agg custom lambdas - - -def test_series_groupby_agg_transpile(scalars_df_index, scalars_pandas_df_index): - def custom_agg(s): - return s.sum() - s.mean() - - bf_df = scalars_df_index.dropna(subset=["int64_col", "bool_col"]) - pd_df = scalars_pandas_df_index.dropna(subset=["int64_col", "bool_col"]) - - bf_result = bf_df.groupby("bool_col")["int64_col"].agg(custom_agg).to_pandas() - pd_result = pd_df.groupby("bool_col")["int64_col"].agg(custom_agg) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_dataframe_groupby_agg_func_transpile( - scalars_df_index, scalars_pandas_df_index -): - def custom_agg(s): - return (s.max() - s.min()) / s.count() - - bf_df = scalars_df_index.dropna(subset=["int64_col", "int64_too", "bool_col"]) - pd_df = scalars_pandas_df_index.dropna( - subset=["int64_col", "int64_too", "bool_col"] - ) - - bf_result = ( - bf_df[["int64_col", "int64_too", "bool_col"]] - .groupby("bool_col") - .agg(custom_agg) - .to_pandas() - ) - pd_result = ( - pd_df[["int64_col", "int64_too", "bool_col"]] - .groupby("bool_col") - .agg(custom_agg) - ) - - assert_frame_equal(bf_result, pd_result, check_dtype=False) - - -def test_dataframe_groupby_agg_dict_transpile( - scalars_df_index, scalars_pandas_df_index -): - def custom_agg1(s): - return s.sum() - s.mean() - - def custom_agg2(s): - return s.max() - s.min() - - bf_df = scalars_df_index.dropna(subset=["int64_col", "int64_too", "bool_col"]) - pd_df = scalars_pandas_df_index.dropna( - subset=["int64_col", "int64_too", "bool_col"] - ) - - bf_result = ( - bf_df.groupby("bool_col") - .agg({"int64_col": custom_agg1, "int64_too": custom_agg2}) - .to_pandas() - ) - pd_result = pd_df.groupby("bool_col").agg( - {"int64_col": custom_agg1, "int64_too": custom_agg2} - ) - - assert_frame_equal(bf_result, pd_result, check_dtype=False) - - -def test_dataframe_groupby_agg_list_transpile( - scalars_df_index, scalars_pandas_df_index -): - def custom_agg1(s): - return s.sum() - s.mean() - - def custom_agg2(s): - return s.max() - s.min() - - bf_df = scalars_df_index.dropna(subset=["int64_col", "bool_col"]) - pd_df = scalars_pandas_df_index.dropna(subset=["int64_col", "bool_col"]) - - bf_result = ( - bf_df[["int64_col", "bool_col"]] - .groupby("bool_col") - .agg([custom_agg1, custom_agg2]) - .to_pandas() - ) - pd_result = ( - pd_df[["int64_col", "bool_col"]] - .groupby("bool_col") - .agg([custom_agg1, custom_agg2]) - ) - - assert_frame_equal(bf_result, pd_result, check_dtype=False) - - -# Tests for groupby.transform broadcasting lambdas - - -def test_series_groupby_transform_transpile(scalars_df_index, scalars_pandas_df_index): - def custom_transform(s): - return s - s.mean() - - bf_df = scalars_df_index.dropna(subset=["int64_col", "bool_col"]) - pd_df = scalars_pandas_df_index.dropna(subset=["int64_col", "bool_col"]) - - bf_result = ( - bf_df.groupby("bool_col")["int64_col"].transform(custom_transform).to_pandas() - ) - pd_result = pd_df.groupby("bool_col")["int64_col"].transform(custom_transform) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_dataframe_groupby_transform_transpile( - scalars_df_index, scalars_pandas_df_index -): - def custom_transform(s): - return (s - s.min()) / (s.max() - s.min()) - - bf_df = scalars_df_index.dropna(subset=["int64_col", "int64_too", "bool_col"]) - pd_df = scalars_pandas_df_index.dropna( - subset=["int64_col", "int64_too", "bool_col"] - ) - - bf_result = ( - bf_df[["int64_col", "int64_too", "bool_col"]] - .groupby("bool_col") - .transform(custom_transform) - .to_pandas() - ) - pd_result = ( - pd_df[["int64_col", "int64_too", "bool_col"]] - .groupby("bool_col") - .transform(custom_transform) - ) - - assert_frame_equal(bf_result, pd_result, check_dtype=False) diff --git a/tests/unit/test_iloc_getitem.py b/tests/unit/test_iloc_getitem.py deleted file mode 100644 index 7f030a16c92..00000000000 --- a/tests/unit/test_iloc_getitem.py +++ /dev/null @@ -1,297 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Generator - -import numpy as np -import pandas as pd -import pyarrow as pa -import pytest - -import bigframes -import bigframes.pandas as bpd -from bigframes.testing.utils import assert_frame_equal, assert_series_equal - -pytest.importorskip("polars") - - -@pytest.fixture(scope="module", autouse=True) -def session() -> Generator[bigframes.Session, None, None]: - import bigframes.core.global_session - from bigframes.testing import polars_session - - session = polars_session.TestSession() - with bigframes.core.global_session._GlobalSessionContext(session): - yield session - - -@pytest.fixture -def sample_df() -> bpd.DataFrame: - pd_df = pd.DataFrame( - { - "A": [1, 2, 3], - "B": [4, 5, 6], - "C": [7, 8, 9], - } - ) - return bpd.read_pandas(pd_df) - - -@pytest.fixture -def unordered_sample_df( - sample_df: bpd.DataFrame, -) -> Generator[bpd.DataFrame, None, None]: - session = sample_df._session - original_strictly_ordered = session._strictly_ordered - original_allow_ambiguity = session._allow_ambiguity - - try: - session._strictly_ordered = False - session._allow_ambiguity = True - - import unittest.mock as mock - - with ( - mock.patch.object( - type(sample_df._block.expr), - "order_ambiguous", - new_callable=mock.PropertyMock, - ) as mock_ambiguous, - mock.patch.object( - type(sample_df._block), - "explicitly_ordered", - new_callable=mock.PropertyMock, - ) as mock_explicit, - ): - mock_ambiguous.return_value = True - mock_explicit.return_value = False - yield sample_df - finally: - session._strictly_ordered = original_strictly_ordered - session._allow_ambiguity = original_allow_ambiguity - - -@pytest.fixture -def duplicate_columns_df() -> bpd.DataFrame: - pd_df = pd.DataFrame( - [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - columns=["A", "B", "A"], - ) - return bpd.read_pandas(pd_df) - - -def test_iloc_getitem_column_single_integer(sample_df): - bf_df = sample_df - pd_df = sample_df.to_pandas() - - bf_result = bf_df.iloc[:, 1].to_pandas() - pd_result = pd_df.iloc[:, 1] - - assert_series_equal(bf_result, pd_result) - - -def test_iloc_getitem_column_numpy_scalar(sample_df): - bf_df = sample_df - pd_df = sample_df.to_pandas() - - bf_result = bf_df.iloc[:, np.int64(1)].to_pandas() - pd_result = pd_df.iloc[:, np.int64(1)] - - assert_series_equal(bf_result, pd_result) - - -def test_iloc_getitem_columns_numpy_array(sample_df): - bf_df = sample_df - pd_df = sample_df.to_pandas() - - bf_result = bf_df.iloc[:, np.array([0, 2], dtype=np.int64)].to_pandas() - pd_result = pd_df.iloc[:, np.array([0, 2], dtype=np.int64)] - - assert_frame_equal(bf_result, pd_result) - - -def test_iloc_getitem_column_pyarrow_scalar(sample_df): - bf_df = sample_df - pd_df = sample_df.to_pandas() - - bf_result = bf_df.iloc[:, pa.scalar(1, type=pa.int64())].to_pandas() - pd_result = pd_df.iloc[:, 1] - - assert_series_equal(bf_result, pd_result) - - -def test_iloc_getitem_columns_pyarrow_array(sample_df): - bf_df = sample_df - pd_df = sample_df.to_pandas() - - bf_result = bf_df.iloc[:, pa.array([0, 2], type=pa.int64())].to_pandas() - pd_result = pd_df.iloc[:, pa.array([0, 2], type=pa.int64())] - - assert_frame_equal(bf_result, pd_result) - - -def test_iloc_getitem_row_numpy_scalar(sample_df): - bf_df = sample_df - pd_df = sample_df.to_pandas() - - bf_result = bf_df.iloc[np.int64(1)] - pd_result = pd_df.iloc[np.int64(1)] - - assert_series_equal(bf_result, pd_result) - - -def test_iloc_getitem_rows_numpy_array(sample_df): - bf_df = sample_df - pd_df = sample_df.to_pandas() - - bf_result = bf_df.iloc[np.array([0, 2], dtype=np.int64)].to_pandas() - pd_result = pd_df.iloc[np.array([0, 2], dtype=np.int64)] - - assert_frame_equal(bf_result, pd_result) - - -def test_iloc_getitem_row_pyarrow_scalar(sample_df): - bf_df = sample_df - pd_df = sample_df.to_pandas() - - bf_result = bf_df.iloc[pa.scalar(1, type=pa.int64())] - pd_result = pd_df.iloc[1] - - assert_series_equal(bf_result, pd_result) - - -def test_iloc_getitem_rows_pyarrow_array(sample_df): - bf_df = sample_df - pd_df = sample_df.to_pandas() - - bf_result = bf_df.iloc[pa.array([0, 2], type=pa.int64())].to_pandas() - pd_result = pd_df.iloc[pa.array([0, 2], type=pa.int64())] - - assert_frame_equal(bf_result, pd_result) - - -@pytest.mark.parametrize( - ["key", "value", "expected_error"], - [ - pytest.param((slice(None), 1), None, None, id="col_index"), - pytest.param((slice(0, None), 1), None, None, id="col_index_slice_0_none"), - pytest.param( - (slice(None, None, 1), 1), None, None, id="col_index_slice_none_none_1" - ), - pytest.param( - (slice(1, None), 1), - None, - bigframes.exceptions.OrderRequiredError, - id="col_index_slice_1_none", - ), - pytest.param( - (slice(None, 2), 1), - None, - bigframes.exceptions.OrderRequiredError, - id="col_index_slice_none_2", - ), - pytest.param((slice(None), 1), 99, None, id="col_setitem"), - pytest.param( - (1, slice(None)), - None, - bigframes.exceptions.OrderRequiredError, - id="row_index_slice", - ), - pytest.param( - 1, - None, - bigframes.exceptions.OrderRequiredError, - id="single_row_index", - ), - ], -) -def test_iloc_getitem_unordered(unordered_sample_df, key, value, expected_error): - if value is not None: - bf_df = unordered_sample_df.copy() - bf_df.iloc[key] = value - elif expected_error is not None: - with pytest.raises(expected_error): - unordered_sample_df.iloc[key] - else: - unordered_sample_df.iloc[key] - - -def test_iloc_getitem_duplicate_columns_single_integer(duplicate_columns_df): - bf_df = duplicate_columns_df - pd_df = duplicate_columns_df.to_pandas() - - bf_result = bf_df.iloc[:, 2].to_pandas() - pd_result = pd_df.iloc[:, 2] - - assert_series_equal(bf_result, pd_result) - - -def test_iloc_getitem_duplicate_columns_list_integer(duplicate_columns_df): - bf_df = duplicate_columns_df - pd_df = duplicate_columns_df.to_pandas() - - bf_result = bf_df.iloc[:, [0, 2]].to_pandas() - pd_result = pd_df.iloc[:, [0, 2]] - - assert_frame_equal(bf_result, pd_result) - - -def test_iloc_getitem_duplicate_columns_slice(duplicate_columns_df): - bf_df = duplicate_columns_df - pd_df = duplicate_columns_df.to_pandas() - - bf_result = bf_df.iloc[:, 1:3].to_pandas() - pd_result = pd_df.iloc[:, 1:3] - - assert_frame_equal(bf_result, pd_result) - - -def test_iloc_getitem_duplicate_columns_numpy_scalar(duplicate_columns_df): - bf_df = duplicate_columns_df - pd_df = duplicate_columns_df.to_pandas() - - bf_result = bf_df.iloc[:, np.int64(2)].to_pandas() - pd_result = pd_df.iloc[:, np.int64(2)] - - assert_series_equal(bf_result, pd_result) - - -def test_iloc_getitem_duplicate_columns_numpy_array(duplicate_columns_df): - bf_df = duplicate_columns_df - pd_df = duplicate_columns_df.to_pandas() - - bf_result = bf_df.iloc[:, np.array([0, 2], dtype=np.int64)].to_pandas() - pd_result = pd_df.iloc[:, np.array([0, 2], dtype=np.int64)] - - assert_frame_equal(bf_result, pd_result) - - -def test_iloc_getitem_duplicate_columns_pyarrow_scalar(duplicate_columns_df): - bf_df = duplicate_columns_df - pd_df = duplicate_columns_df.to_pandas() - - bf_result = bf_df.iloc[:, pa.scalar(2, type=pa.int64())].to_pandas() - pd_result = pd_df.iloc[:, 2] - - assert_series_equal(bf_result, pd_result) - - -def test_iloc_getitem_duplicate_columns_pyarrow_array(duplicate_columns_df): - bf_df = duplicate_columns_df - pd_df = duplicate_columns_df.to_pandas() - - bf_result = bf_df.iloc[:, pa.array([0, 2], type=pa.int64())].to_pandas() - pd_result = pd_df.iloc[:, pa.array([0, 2], type=pa.int64())] - - assert_frame_equal(bf_result, pd_result) diff --git a/tests/unit/test_iloc_setitem.py b/tests/unit/test_iloc_setitem.py deleted file mode 100644 index 98c515fd4ee..00000000000 --- a/tests/unit/test_iloc_setitem.py +++ /dev/null @@ -1,245 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Generator - -import numpy as np -import pandas as pd -import pyarrow as pa -import pytest - -import bigframes -import bigframes.pandas as bpd -from bigframes.testing.utils import assert_frame_equal - -pytest.importorskip("polars") - - -@pytest.fixture(scope="module", autouse=True) -def session() -> Generator[bigframes.Session, None, None]: - import bigframes.core.global_session - from bigframes.testing import polars_session - - session = polars_session.TestSession() - with bigframes.core.global_session._GlobalSessionContext(session): - yield session - - -@pytest.fixture -def sample_df() -> bpd.DataFrame: - pd_df = pd.DataFrame( - { - "A": [1, 2, 3], - "B": [4, 5, 6], - "C": [7, 8, 9], - } - ) - return bpd.read_pandas(pd_df) - - -def test_iloc_setitem_column_single_integer(sample_df): - bf_df = sample_df.copy() - pd_df = sample_df.to_pandas() - - bf_df.iloc[:, 1] = 99 - pd_df.iloc[:, 1] = 99 - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_column_single_integer_negative(sample_df): - bf_df = sample_df.copy() - pd_df = sample_df.to_pandas() - - bf_df.iloc[:, -1] = 99 - pd_df.iloc[:, -1] = 99 - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_columns_list_integer(sample_df): - bf_df = sample_df.copy() - pd_df = sample_df.to_pandas() - - bf_df.iloc[:, [0, 2]] = [99, 88] - pd_df.iloc[:, [0, 2]] = [99, 88] - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_columns_slice(sample_df): - bf_df = sample_df.copy() - pd_df = sample_df.to_pandas() - - bf_df.iloc[:, 0:2] = 99 - pd_df.iloc[:, 0:2] = 99 - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_columns_boolean_mask(sample_df): - bf_df = sample_df.copy() - pd_df = sample_df.to_pandas() - - mask = [True, False, True] - bf_df.iloc[:, mask] = 99 - pd_df.iloc[:, np.array(mask)] = 99 - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_columns_dataframe(sample_df): - bf_df = sample_df.copy() - pd_df = sample_df.to_pandas() - - value_df = bpd.DataFrame({"B": [99, 88, 77], "C": [66, 55, 44]}) - bf_df.iloc[:, 1:3] = value_df - pd_df.iloc[:, 1:3] = value_df.to_pandas() - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_column_numpy_scalar(sample_df): - bf_df = sample_df.copy() - pd_df = sample_df.to_pandas() - - bf_df.iloc[:, np.int64(1)] = 99 - pd_df.iloc[:, np.int64(1)] = 99 - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_columns_numpy_array(sample_df): - bf_df = sample_df.copy() - pd_df = sample_df.to_pandas() - - bf_df.iloc[:, np.array([0, 2], dtype=np.int64)] = [99, 88] - pd_df.iloc[:, np.array([0, 2], dtype=np.int64)] = [99, 88] - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_column_pyarrow_scalar(sample_df): - bf_df = sample_df.copy() - pd_df = sample_df.to_pandas() - - bf_df.iloc[:, pa.scalar(1, type=pa.int64())] = 99 - pd_df.iloc[:, 1] = 99 - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_columns_pyarrow_array(sample_df): - bf_df = sample_df.copy() - pd_df = sample_df.to_pandas() - - bf_df.iloc[:, pa.array([0, 2], type=pa.int64())] = [99, 88] - pd_df.iloc[:, pa.array([0, 2], type=pa.int64())] = [99, 88] - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -@pytest.mark.parametrize( - ["key", "expected_error"], - [ - pytest.param((slice(None), 3), IndexError, id="out_of_bounds_positive"), - pytest.param((slice(None), -4), IndexError, id="out_of_bounds_negative"), - pytest.param((0, 1), NotImplementedError, id="invalid_row_indexer"), - pytest.param((slice(None), "B"), TypeError, id="invalid_col_indexer_type"), - ], -) -def test_iloc_setitem_column_errors(sample_df, key, expected_error): - bf_df = sample_df.copy() - - with pytest.raises(expected_error): - bf_df.iloc[key] = 99 - - -@pytest.fixture -def duplicate_columns_df() -> bpd.DataFrame: - pd_df = pd.DataFrame( - [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - columns=["A", "B", "A"], - ) - return bpd.read_pandas(pd_df) - - -def test_iloc_setitem_duplicate_columns_single_integer(duplicate_columns_df): - bf_df = duplicate_columns_df.copy() - pd_df = duplicate_columns_df.to_pandas() - - bf_df.iloc[:, 2] = 99 - pd_df.iloc[:, 2] = 99 - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_duplicate_columns_list_integer(duplicate_columns_df): - bf_df = duplicate_columns_df.copy() - pd_df = duplicate_columns_df.to_pandas() - - bf_df.iloc[:, [0, 2]] = [99, 88] - pd_df.iloc[:, [0, 2]] = [99, 88] - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_duplicate_columns_slice(duplicate_columns_df): - bf_df = duplicate_columns_df.copy() - pd_df = duplicate_columns_df.to_pandas() - - bf_df.iloc[:, 1:3] = 99 - pd_df.iloc[:, 1:3] = 99 - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_duplicate_columns_numpy_scalar(duplicate_columns_df): - bf_df = duplicate_columns_df.copy() - pd_df = duplicate_columns_df.to_pandas() - - bf_df.iloc[:, np.int64(2)] = 99 - pd_df.iloc[:, np.int64(2)] = 99 - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_duplicate_columns_numpy_array(duplicate_columns_df): - bf_df = duplicate_columns_df.copy() - pd_df = duplicate_columns_df.to_pandas() - - bf_df.iloc[:, np.array([0, 2], dtype=np.int64)] = [99, 88] - pd_df.iloc[:, np.array([0, 2], dtype=np.int64)] = [99, 88] - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_duplicate_columns_pyarrow_scalar(duplicate_columns_df): - bf_df = duplicate_columns_df.copy() - pd_df = duplicate_columns_df.to_pandas() - - bf_df.iloc[:, pa.scalar(2, type=pa.int64())] = 99 - pd_df.iloc[:, 2] = 99 - - assert_frame_equal(bf_df.to_pandas(), pd_df) - - -def test_iloc_setitem_duplicate_columns_pyarrow_array(duplicate_columns_df): - bf_df = duplicate_columns_df.copy() - pd_df = duplicate_columns_df.to_pandas() - - bf_df.iloc[:, pa.array([0, 2], type=pa.int64())] = [99, 88] - pd_df.iloc[:, pa.array([0, 2], type=pa.int64())] = [99, 88] - - assert_frame_equal(bf_df.to_pandas(), pd_df) diff --git a/tests/unit/test_local_engine.py b/tests/unit/test_local_engine.py index fe5052771f2..5f80e4928c0 100644 --- a/tests/unit/test_local_engine.py +++ b/tests/unit/test_local_engine.py @@ -170,12 +170,7 @@ def test_polars_local_engine_agg(polars_session): bf_result = bf_df.agg(["sum", "count"]).to_pandas() pd_result = pd_df.agg(["sum", "count"]) # local engine appears to produce uint32 - pandas.testing.assert_frame_equal( - bf_result, # type: ignore[arg-type] - pd_result, - check_dtype=False, - check_index_type=False, - ) + pandas.testing.assert_frame_equal(bf_result, pd_result, check_dtype=False, check_index_type=False) # type: ignore def test_polars_local_engine_groupby_sum(polars_session): diff --git a/tests/unit/test_pandas.py b/tests/unit/test_pandas.py index c85d92e024d..e1e713697db 100644 --- a/tests/unit/test_pandas.py +++ b/tests/unit/test_pandas.py @@ -14,6 +14,7 @@ import inspect import re +import sys import unittest.mock as mock import pandas as pd @@ -36,8 +37,6 @@ def all_session_methods(): session_attributes.remove("close") # streaming isn't in pandas session_attributes.remove("read_gbq_table_streaming") - # execution_history is in base namespace, not pandas - session_attributes.remove("execution_history") for attribute in sorted(session_attributes): session_method = getattr(bigframes.session.Session, attribute) @@ -53,6 +52,11 @@ def all_session_methods(): [(method_name,) for method_name in all_session_methods()], ) def test_method_matches_session(method_name: str): + if sys.version_info < (3, 10): + pytest.skip( + "Need Python 3.10 to reconcile deferred annotations." + ) # pragma: no cover + session_method = getattr(bigframes.session.Session, method_name) session_doc = inspect.getdoc(session_method) assert session_doc is not None, "docstrings are required" diff --git a/tests/unit/test_planner.py b/tests/unit/test_planner.py index 36a568a4165..66d83f362dd 100644 --- a/tests/unit/test_planner.py +++ b/tests/unit/test_planner.py @@ -19,9 +19,9 @@ import pandas as pd import bigframes.core as core -import bigframes.core.bq_data import bigframes.core.expression as ex import bigframes.core.identifiers as ids +import bigframes.core.schema import bigframes.operations as ops import bigframes.session.planner as planner @@ -38,7 +38,7 @@ type(FAKE_SESSION)._strictly_ordered = mock.PropertyMock(return_value=True) LEAF: core.ArrayValue = core.ArrayValue.from_table( session=FAKE_SESSION, - table=bigframes.core.bq_data.GbqNativeTable.from_table(TABLE), + table=TABLE, ) diff --git a/tests/unit/test_py_udf.py b/tests/unit/test_py_udf.py deleted file mode 100644 index dcd6fa28658..00000000000 --- a/tests/unit/test_py_udf.py +++ /dev/null @@ -1,761 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pathlib -from typing import Generator - -import numpy as np -import pandas as pd -import pandas.testing -import pyarrow as pa -import pytest - -import bigframes -import bigframes.core.global_session -import bigframes.pandas as bpd -from bigframes.core.bytecode import py_to_expression -from bigframes.testing.utils import ( - assert_frame_equal, - assert_series_equal, - convert_pandas_dtypes, -) - -pytest.importorskip("polars") -pytest.importorskip("pandas", minversion="2.0.0") - -CURRENT_DIR = pathlib.Path(__file__).parent -DATA_DIR = CURRENT_DIR.parent / "data" - - -@pytest.fixture(scope="module", autouse=True) -def session() -> Generator[bigframes.Session, None, None]: - # import inline to allow polars importorskip to happen first - from bigframes.testing import polars_session - - with bpd.option_context("experiments.enable_python_transpiler", True): - session = polars_session.TestSession() - with bigframes.core.global_session._GlobalSessionContext(session): - yield session - - -@pytest.fixture(scope="module") -def scalars_pandas_df_index() -> pd.DataFrame: - """pd.DataFrame pointing at test data.""" - - df = pd.read_json( - DATA_DIR / "scalars.jsonl", - lines=True, - ) - convert_pandas_dtypes(df, bytes_col=True) - - df = df.set_index("rowindex", drop=False) - df.index.name = None - return df.set_index("rowindex").sort_index() - - -@pytest.fixture(scope="module") -def scalars_df_index( - session: bigframes.Session, scalars_pandas_df_index -) -> bpd.DataFrame: - return session.read_pandas(scalars_pandas_df_index) - - -@pytest.fixture(scope="module") -def scalars_dfs( - scalars_df_index, - scalars_pandas_df_index, -): - return scalars_df_index, scalars_pandas_df_index - - -def test_dataframe_map_transpile( - scalars_df_index, - scalars_pandas_df_index, -): - columns = ["int64_too", "int64_col"] - - def foo(input): - return input * 3 + 12 - - bf_result = scalars_df_index[columns].map(foo, na_action="ignore").to_pandas() - - pd_result = ( - scalars_pandas_df_index[columns].map(foo, na_action="ignore").astype("Int64") - ) - - assert_frame_equal(bf_result, pd_result) - - -def test_dataframe_apply_axis_1_transpile( - scalars_df_index, - scalars_pandas_df_index, -): - columns = ["int64_too", "int64_col"] - - def foo(input): - return input.int64_too + input.int64_col - - bf_result = scalars_df_index[columns].apply(foo, axis=1).to_pandas() - - pd_result = scalars_pandas_df_index[columns].apply(foo, axis=1).astype("Int64") - - assert_series_equal(bf_result, pd_result) - - -def test_series_combine_transpile( - scalars_df_index, - scalars_pandas_df_index, -): - def which_smaller(left, right): - return (left * right) + 3 - - bf_result = ( - scalars_df_index["int64_too"] - .combine(scalars_df_index["int64_col"], which_smaller) - .to_pandas() - ) - - pd_result = scalars_pandas_df_index["int64_too"].combine( - scalars_pandas_df_index["int64_col"], which_smaller - ) - - assert_series_equal(bf_result, pd_result) - - -def test_dataframe_apply_axis_1_transpile_with_defaults( - scalars_df_index, - scalars_pandas_df_index, -): - columns = ["int64_too", "int64_col"] - - def foo(input, x=10, y=5): - return input.int64_too + input.int64_col + x + y - - bf_result = scalars_df_index[columns].apply(foo, axis=1).to_pandas() - pd_result = scalars_pandas_df_index[columns].apply(foo, axis=1).astype("Int64") - - assert_series_equal(bf_result, pd_result) - - -def test_dataframe_apply_axis_1_transpile_with_args( - scalars_df_index, - scalars_pandas_df_index, -): - columns = ["int64_too", "int64_col"] - - def foo(input, x, y=5): - return input.int64_too + input.int64_col + x + y - - bf_result = ( - scalars_df_index[columns].apply(foo, axis=1, args=(12,), y=20).to_pandas() - ) - pd_result = ( - scalars_pandas_df_index[columns] - .apply(foo, axis=1, args=(12,), y=20) - .astype("Int64") - ) - - assert_series_equal(bf_result, pd_result) - - -def test_dataframe_apply_axis_1_transpile_invalid_bindings( - scalars_df_index, -): - columns = ["int64_too", "int64_col"] - - def foo(input, x, y=5): - return input.int64_too + input.int64_col + x + y - - # 1. Unexpected keyword argument - with pytest.raises(TypeError, match="unexpected keyword argument 'z'"): - scalars_df_index[columns].apply(foo, axis=1, args=(10,), z=20) - - # 2. Multiple values for keyword argument 'x' - with pytest.raises(TypeError, match="multiple values for argument 'x'"): - scalars_df_index[columns].apply(foo, axis=1, args=(10,), x=20) - - # 3. Too many positional arguments - with pytest.raises(TypeError, match="too many positional arguments"): - scalars_df_index[columns].apply(foo, axis=1, args=(10, 20, 30)) - - # 4. Missing required argument 'x' - with pytest.raises(TypeError, match="missing a required argument: 'x'"): - scalars_df_index[columns].apply(foo, axis=1) - - -def test_series_apply_transpile( - scalars_df_index, - scalars_pandas_df_index, -): - def foo(x, y=10): - return x * 2 + y - - bf_result = scalars_df_index["int64_col"].apply(foo, args=(5,)).to_pandas() - pd_result = ( - scalars_pandas_df_index["int64_col"].apply(foo, args=(5,)).astype("Int64") - ) - - assert_series_equal(bf_result, pd_result) - - -def test_series_apply_transpile_invalid_bindings( - scalars_df_index, -): - def foo(x, y): - return x + y - - # Too many positional args: foo takes 2 args (x, y), we pass self and 2 more args (total 3 positional) - with pytest.raises( - TypeError, match="too many positional arguments: expected 2, got 3" - ): - scalars_df_index["int64_col"].apply(foo, args=(10, 20)) - - # Missing required argument: foo takes 2 args, we only pass self (so y is missing) - with pytest.raises(TypeError, match="missing required argument: 'y'"): - scalars_df_index["int64_col"].apply(foo) - - -def test_transpilation_unsupported_ops_raise( - scalars_df_index, -): - def foo_with_loop(x): - total = 0 - for i in range(x): - total += i - return total - - with pytest.raises(ValueError): - scalars_df_index["int64_col"].apply(foo_with_loop) - - -def my_foo(x: int): - return x + 1 - - -def test_local_series_apply_simple(scalars_df_index, scalars_pandas_df_index): - bf_result = scalars_df_index["int64_col"].apply(my_foo).to_pandas() - pd_result = scalars_pandas_df_index["int64_col"].apply(my_foo) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def my_numpy_foo(x: int): - return np.add(x, x) * (np.cos(x) - np.sin(3)) - - -def test_local_series_apply_w_numpy(scalars_df_index, scalars_pandas_df_index): - bf_result = scalars_df_index["int64_col"].apply(my_numpy_foo).to_pandas() - pd_result = scalars_pandas_df_index["int64_col"].apply(my_numpy_foo) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_series_apply_w_simple_lamdba(scalars_df_index, scalars_pandas_df_index): - bf_result = scalars_df_index["int64_col"].apply(lambda x: x + 3).to_pandas() - pd_result = scalars_pandas_df_index["int64_col"].apply(lambda x: x + 3) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_series_apply_w_ternary_lamdba(scalars_df_index, scalars_pandas_df_index): - bf_result = ( - scalars_df_index["int64_col"] - .apply(lambda x: "positive" if x > 0 else "negative") - .to_pandas() - ) - pd_result = scalars_pandas_df_index["int64_col"].apply( - lambda x: "positive" if x > 0 else "negative" - ) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_series_apply_w_nested_fizzbuzz(session): - # challenging: closure, multiple exits, mutating variables - foo_div = 3 - buzz_div = 5 - pd_series = pd.Series( - range(20), - dtype="Int64", - index=pd.Index(range(20), dtype="Int64"), - name="integers", - ) - bf_series = bpd.Series(pd_series, session=session) - - def fizzbuzz(x): - if (x % 3) and (x % 5): - return str(x) - val = "" - if (x % foo_div) == 0: - val += "fizz" - if (x % buzz_div) == 0: - val += "buzz" - return val - - bf_result = bf_series.apply(fizzbuzz).to_pandas() - pd_result = pd_series.apply(fizzbuzz).astype(pd.StringDtype(storage="pyarrow")) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_dataframe_apply_w_ternary_lamdba( - scalars_df_index, scalars_pandas_df_index -): - bf_result = scalars_df_index.apply( - lambda x: x.int64_col if x.rowindex_2 > 5 else x.float64_col, axis=1 - ).to_pandas() - pd_result = scalars_pandas_df_index.apply( - lambda x: x.int64_col if x.rowindex_2 > 5 else x.float64_col, axis=1 - ).astype("Float64") - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_series_apply_w_nested_ifs(scalars_df_index, scalars_pandas_df_index): - def nested_ifs(x): - if x > 0: - if x > 100: - return x * 10 - else: - return x * 2 - else: - if x < -100: - return x * 20 - return x * -1 - - bf_result = scalars_df_index["int64_col"].apply(nested_ifs).to_pandas() - pd_result = scalars_pandas_df_index["int64_col"].apply(nested_ifs) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_series_apply_w_elif(scalars_df_index, scalars_pandas_df_index): - def elif_fn(x): - if x > 100: - return 1 - elif x > 50: - return 2 - elif x > 0: - return 3 - else: - return 4 - - bf_result = scalars_df_index["int64_col"].apply(elif_fn).to_pandas() - pd_result = scalars_pandas_df_index["int64_col"].apply(elif_fn) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_series_apply_w_logical_not(scalars_df_index, scalars_pandas_df_index): - def logical_not_fn(x): - if not (x > 0): - return -x - return x - - bf_result = scalars_df_index["int64_col"].apply(logical_not_fn).to_pandas() - pd_result = scalars_pandas_df_index["int64_col"].apply(logical_not_fn) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_series_apply_w_short_circuit(scalars_df_index, scalars_pandas_df_index): - def short_circuit(x): - if (x > 0 and x < 100) or x == 55555: - return 1 - return 0 - - bf_result = scalars_df_index["int64_col"].apply(short_circuit).to_pandas() - pd_result = scalars_pandas_df_index["int64_col"].apply(short_circuit) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_series_apply_w_var_assignments( - scalars_df_index, scalars_pandas_df_index -): - def var_assign(x): - val = x - if x > 0: - val = val + 10 - if val > 100: - val = val * 2 - else: - val = val - 10 - return val - - bf_result = scalars_df_index["int64_col"].apply(var_assign).to_pandas() - pd_result = scalars_pandas_df_index["int64_col"].apply(var_assign) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_series_apply_w_logical_and_val( - scalars_df_index, scalars_pandas_df_index -): - def logical_and_val(x): - return (x % 3) and 100 - - bf_result = ( - scalars_df_index["int64_col"].dropna().apply(logical_and_val).to_pandas() - ) - pd_result = scalars_pandas_df_index["int64_col"].dropna().apply(logical_and_val) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_series_apply_w_logical_or_val(scalars_df_index, scalars_pandas_df_index): - def logical_or_val(x): - return (x % 3) or 200 - - bf_result = scalars_df_index["int64_col"].dropna().apply(logical_or_val).to_pandas() - pd_result = scalars_pandas_df_index["int64_col"].dropna().apply(logical_or_val) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_series_apply_w_logical_and_mixed( - scalars_df_index, -): - def logical_and_mixed(x): - return (x % 3) and "hello" - - with pytest.raises(TypeError, match="Cannot coerce"): - scalars_df_index["int64_col"].apply(logical_and_mixed) - - -def test_local_series_apply_w_logical_not_val( - scalars_df_index, scalars_pandas_df_index -): - def logical_not_val(x): - return not x - - bf_result = scalars_df_index["bool_col"].dropna().apply(logical_not_val).to_pandas() - pd_result = scalars_pandas_df_index["bool_col"].dropna().apply(logical_not_val) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_local_series_apply_w_compare_chain(scalars_df_index, scalars_pandas_df_index): - def compare_chain(x): - return 0 < x < 1000 - - bf_result = scalars_df_index["int64_col"].dropna().apply(compare_chain).to_pandas() - pd_result = scalars_pandas_df_index["int64_col"].dropna().apply(compare_chain) - - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_dataframe_apply_axis_1_with_integer_subscript( - scalars_df_index, scalars_pandas_df_index -): - columns = ["int64_too", "int64_col"] - bf_df = scalars_df_index[columns].rename(columns={"int64_too": 0, "int64_col": 1}) - pd_df = scalars_pandas_df_index[columns].rename( - columns={"int64_too": 0, "int64_col": 1} - ) - - def foo(input): - return input[0] + input[1] - - bf_result = bf_df.apply(foo, axis=1).to_pandas() - pd_result = pd_df.apply(foo, axis=1).astype("Int64") - - assert_series_equal(bf_result, pd_result) - - -def test_dataframe_apply_axis_1_with_invalid_subscript_raises( - scalars_df_index, -): - columns = ["int64_too", "int64_col"] - - def foo_invalid_label(input): - return input["non_existent_column"] - - with pytest.raises(KeyError, match="non_existent_column"): - scalars_df_index[columns].apply(foo_invalid_label, axis=1) - - -def test_series_map_with_struct_subscript(session): - # Struct setup - struct_pa_type = pa.struct([("str_field", pa.string()), ("int_field", pa.int64())]) - pd_struct_series = pd.Series( - pa.array([{"str_field": "hello", "int_field": 1}], struct_pa_type), - dtype=pd.ArrowDtype(struct_pa_type), - ) - bf_struct_series = bpd.Series(pd_struct_series, session=session) - - # Struct subscripting in UDF - def get_struct_val(x): - return x["str_field"] - - bf_struct_res = bf_struct_series.map(get_struct_val).to_pandas() - pd_struct_res: pd.Series = pd_struct_series.map(get_struct_val) - assert_series_equal(bf_struct_res, pd_struct_res, check_dtype=False) - - -def test_series_map_with_array_subscript(session): - # Array setup - array_pa_type = pa.list_(pa.int64()) - pd_array_series = pd.Series( - pa.array([[10, 20]], array_pa_type), - dtype=pd.ArrowDtype(array_pa_type), - ) - bf_array_series = bpd.Series(pd_array_series, session=session) - - # Array subscripting in UDF - def get_array_val(x): - return x[1] - - bf_array_res = bf_array_series.map(get_array_val).to_pandas() - pd_array_res: pd.Series = pd_array_series.map(get_array_val) - assert_series_equal(bf_array_res, pd_array_res, check_dtype=False) - - -def test_series_map_with_string_subscript(session): - # String setup - pd_string_series = pd.Series(["hello", "world"]) - bf_string_series = bpd.Series(pd_string_series, session=session) - - # String subscripting in UDF - def get_string_val(x): - return x[1] - - bf_string_res = bf_string_series.map(get_string_val).to_pandas() - pd_string_res = pd_string_series.map(get_string_val) # type: ignore - assert_series_equal(bf_string_res, pd_string_res, check_dtype=False) - - -def test_dataframe_apply_axis_1_with_dynamic_subscript_raises( - scalars_df_index, -): - columns = ["int64_too", "int64_col"] - - def foo_dynamic(input): - return input[input[0]] - - with pytest.raises( - NotImplementedError, match="Dynamic column lookup is not supported" - ): - scalars_df_index[columns].apply(foo_dynamic, axis=1) - - -def test_dataframe_apply_axis_1_with_dynamic_array_subscript(session): - array_pa_type = pa.list_(pa.int64()) - pd_df = pd.DataFrame( - { - "array_col": pd.Series( - pa.array([[10, 20], [30, 40, 50], [60]], array_pa_type), - dtype=pd.ArrowDtype(array_pa_type), - ), - "index_col": pd.Series([1, 2, 0], dtype="Int64"), - } - ) - bf_df = bpd.DataFrame(pd_df, session=session) - - def foo(row): - return row["array_col"][row["index_col"]] - - bf_result = bf_df.apply(foo, axis=1).to_pandas() - pd_result = pd_df.apply(foo, axis=1).astype("Int64") - - assert_series_equal(bf_result, pd_result) - - -def test_series_apply_fstrings(session): - pd_series = pd.Series(["apple", "banana", None], dtype="string") - bf_series = bpd.Series(pd_series, session=session) - - def format_udf(x): - if x is None: - return "Null value" - return f"Fruit: {x}!" - - bf_result = bf_series.apply(format_udf).to_pandas() - pd_result = pd.Series( - ["Fruit: apple!", "Fruit: banana!", "Null value"], dtype="string" - ) - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_series_apply_nullity_jumps(session): - pd_series = pd.Series([10, None, 20], dtype="Int64") - bf_series = bpd.Series(pd_series, session=session) - - def nullity_udf(x): - if x is None: - return "Absent" - if x is not None: - return "Present" - return "Unknown" - - bf_result = bf_series.apply(nullity_udf).to_pandas() - pd_result = pd.Series(["Present", "Absent", "Present"], dtype="string") - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_series_apply_string_ops(session): - pd_series = pd.Series(["hello world", "BigFrames", "123a"], dtype="string") - bf_series = bpd.Series(pd_series, session=session) - - def str_udf(x): - if x is None: - return None - return x.upper() + " " + x.lower() + " " + str.upper(x) + " " + x.capitalize() - - bf_result = bf_series.apply(str_udf).to_pandas() - pd_result = pd.Series( - [ - "HELLO WORLD hello world HELLO WORLD Hello world", - "BIGFRAMES bigframes BIGFRAMES Bigframes", - "123A 123a 123A 123a", - ], - dtype="string", - ) - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_series_apply_string_predicates(session): - pd_series = pd.Series( - ["hello world", "abc123", "123", "HELLO!", None], dtype="string" - ) - bf_series = bpd.Series(pd_series, session=session) - - def predicates_udf(x): - if x is None: - return None - return f"{x.islower()}_{x.isupper()}" - - bf_result = bf_series.apply(predicates_udf).to_pandas() - pd_result = pd.Series( - ["True_False", "True_False", "False_False", "False_True", None], dtype="string" - ) - assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_fstring_multiple_placeholders(session): - pd_series = pd.Series(["apple", "banana"], dtype="string") - bf_series = bpd.Series(pd_series, session=session) - - def multiple_placeholders(x): - return f"{x} and {x.upper()}!" - - bf_res = bf_series.apply(multiple_placeholders).to_pandas() - pd_res = pd.Series(["apple and APPLE!", "banana and BANANA!"], dtype="string") - assert_series_equal(bf_res, pd_res, check_dtype=False) - - -def test_fstring_empty(session): - pd_series = pd.Series(["apple", "banana"], dtype="string") - bf_series = bpd.Series(pd_series, session=session) - - def empty_fstring(x): - return "" - - bf_res = bf_series.apply(empty_fstring).to_pandas() - pd_res = pd.Series(["", ""], dtype="string") - assert_series_equal(bf_res, pd_res, check_dtype=False) - - -def test_fstring_consecutive_placeholders(session): - pd_series = pd.Series(["apple", "banana"], dtype="string") - bf_series = bpd.Series(pd_series, session=session) - - def consecutive_placeholders(x): - return f"{x}{x.upper()}" - - bf_res = bf_series.apply(consecutive_placeholders).to_pandas() - pd_res = pd.Series(["appleAPPLE", "bananaBANANA"], dtype="string") - assert_series_equal(bf_res, pd_res, check_dtype=False) - - -def test_fstring_with_specifier_raises(): - def format_with_spec(x): - return f"{x:2d}" - - with pytest.raises( - NotImplementedError, match="Formatting with specifier is not supported" - ): - py_to_expression(format_with_spec) - - -def test_fstring_with_repr_raises(): - def format_with_repr(x): - return f"{x!r}" - - with pytest.raises( - NotImplementedError, - match="repr\\(\\) and ascii\\(\\) conversions are not supported", - ): - py_to_expression(format_with_repr) - - -def test_fstring_with_ascii_raises(): - def format_with_ascii(x): - return f"{x!a}" - - with pytest.raises( - NotImplementedError, - match="repr\\(\\) and ascii\\(\\) conversions are not supported", - ): - py_to_expression(format_with_ascii) - - -def test_identity_unsupported_raises(): - def is_true_udf(x): - return x is True - - with pytest.raises( - NotImplementedError, - match="Identity comparison \\(is/is not\\) is only supported for None", - ): - py_to_expression(is_true_udf) - - -def test_fstring_int_input(session): - pd_int_series = pd.Series([10, 20, None], dtype="Int64") - bf_int_series = bpd.Series(pd_int_series, session=session) - bf_res_int = bf_int_series.apply(lambda x: f"val: {x}").to_pandas() - pd_res_int = pd.Series(["val: 10", "val: 20", None], dtype="string") - assert_series_equal(bf_res_int, pd_res_int, check_dtype=False) - - -def test_fstring_float_input(session): - pd_float_series = pd.Series([1.5, 2.75], dtype="Float64") - bf_float_series = bpd.Series(pd_float_series, session=session) - bf_res_float = bf_float_series.apply(lambda x: f"val: {x}").to_pandas() - pd_res_float = pd.Series(["val: 1.5", "val: 2.75"], dtype="string") - assert_series_equal(bf_res_float, pd_res_float, check_dtype=False) - - -def test_fstring_bool_input(session): - pd_bool_series = pd.Series([True, False], dtype="boolean") - bf_bool_series = bpd.Series(pd_bool_series, session=session) - bf_res_bool = bf_bool_series.apply(lambda x: f"val: {x}").to_pandas() - pd_res_bool = pd.Series(["val: True", "val: False"], dtype="string") - assert_series_equal(bf_res_bool, pd_res_bool, check_dtype=False) - - -def test_fstring_list_input_raises(session): - array_pa_type = pa.list_(pa.int64()) - pd_series = pd.Series( - pa.array([[10, 20]], array_pa_type), - dtype=pd.ArrowDtype(array_pa_type), - ) - bf_series = bpd.Series(pd_series, session=session) - - def udf_with_list(x): - return f"list: {x}" - - with pytest.raises((TypeError, ValueError)): - bf_series.apply(udf_with_list) diff --git a/tests/unit/test_series_polars.py b/tests/unit/test_series_polars.py index 8b6d97d8b4b..516a46d4dd1 100644 --- a/tests/unit/test_series_polars.py +++ b/tests/unit/test_series_polars.py @@ -25,11 +25,11 @@ import geopandas as gpd # type: ignore import google.api_core.exceptions import numpy +from packaging.version import Version import pandas as pd import pyarrow as pa # type: ignore import pytest import shapely.geometry # type: ignore -from packaging.version import Version import bigframes import bigframes.dtypes as dtypes @@ -1571,11 +1571,7 @@ def test_isin_bigframes_index(scalars_dfs, session): scalars_df, scalars_pandas_df = scalars_dfs bf_result = ( scalars_df["string_col"] - .isin( - bigframes.pandas.Index( - ["Hello, World!", "Hi", "こんにちは"], session=session - ) - ) + .isin(bigframes.pandas.Index(["Hello, World!", "Hi", "こんにちは"], session=session)) .to_pandas() ) pd_result = ( @@ -3029,6 +3025,7 @@ def test_value_counts_w_cut(scalars_dfs): def test_iloc_nested(scalars_df_index, scalars_pandas_df_index): + bf_result = scalars_df_index["string_col"].iloc[1:].iloc[1:].to_pandas() pd_result = scalars_pandas_df_index["string_col"].iloc[1:].iloc[1:] @@ -3521,10 +3518,7 @@ def test_series_to_json_local_str(scalars_df_index, scalars_pandas_df_index): def test_series_to_json_local_file(scalars_df_index, scalars_pandas_df_index): # TODO: supply a reason why this isn't compatible with pandas 1.x pytest.importorskip("pandas", minversion="2.0.0") - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.int64_col.to_json(bf_result_file) scalars_pandas_df_index.int64_col.to_json(pd_result_file) @@ -3543,10 +3537,7 @@ def test_series_to_csv_local_str(scalars_df_index, scalars_pandas_df_index): def test_series_to_csv_local_file(scalars_df_index, scalars_pandas_df_index): - with ( - tempfile.TemporaryFile() as bf_result_file, - tempfile.TemporaryFile() as pd_result_file, - ): + with tempfile.TemporaryFile() as bf_result_file, tempfile.TemporaryFile() as pd_result_file: scalars_df_index.int64_col.to_csv(bf_result_file) scalars_pandas_df_index.int64_col.to_csv(pd_result_file) @@ -4561,20 +4552,6 @@ def test_map_series_input_duplicates_error(scalars_dfs): scalars_df.int64_too.map(bf_map_series, verify_integrity=True) -def test_series_map_with_udf(session): - series = bpd.Series([1, 2, None, 4], dtype="Int64") - - @session.udf(input_types=[int], output_type=int) - def foo(x): - if x is None: - return -1 - return x * 2 - - bf_result = series.map(foo).to_pandas() - pd_result = pd.Series([2, 4, -1, 8]) - assert_series_equal(bf_result, pd_result, check_dtype=False) - - @pytest.mark.skip( reason="NotImplementedError: Polars compiler hasn't implemented hash()" ) @@ -4885,7 +4862,9 @@ def foo(x: int, y: int, df): ) pd_result = ( - scalars_pandas_df_index[column].pipe((foo, "df"), x=7, y=9).pipe(lambda x: x**2) + scalars_pandas_df_index[column] + .pipe((foo, "df"), x=7, y=9) + .pipe(lambda x: x**2) ) assert_series_equal(bf_result, pd_result) @@ -5160,42 +5139,3 @@ def test_series_dt_total_seconds(scalars_df_index, scalars_pandas_df_index): # bigframes uses Float64, newer pandas may use double[pyarrow] check_dtype=False, ) - - -def test_series_where_with_expression(scalars_dfs): - scalars_df, scalars_pandas_df = scalars_dfs - s1 = scalars_df["float64_col"] - s2 = scalars_df["bool_col"] - - bf_result = s1.where(s2, bpd.col("bool_col")).to_pandas() - - s1_pd = scalars_pandas_df["float64_col"] - s2_pd = scalars_pandas_df["bool_col"] - - pd_result = s1_pd.where(s2_pd, s2_pd) - - pd.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) - - -def test_series_expression_unbound_fails(scalars_dfs): - scalars_df, _ = scalars_dfs - s1 = scalars_df["float64_col"] - s2 = scalars_df["bool_col"] - - with pytest.raises(ValueError, match="remains unbound"): - s1.where(s2, bpd.col("non_existent_column")) - - -def test_series_where_with_expression_resolving_to_self(scalars_dfs): - scalars_df, scalars_pandas_df = scalars_dfs - s1 = scalars_df["float64_col"] - s2 = scalars_df["bool_col"] - - bf_result = s1.where(s2, bpd.col("float64_col")).to_pandas() - - s1_pd = scalars_pandas_df["float64_col"] - s2_pd = scalars_pandas_df["bool_col"] - - pd_result = s1_pd.where(s2_pd, s1_pd) - - pd.testing.assert_series_equal(bf_result, pd_result, check_dtype=False) diff --git a/tests/unit/test_series_struct.py b/tests/unit/test_series_struct.py index f99d5859a56..c92b87cf481 100644 --- a/tests/unit/test_series_struct.py +++ b/tests/unit/test_series_struct.py @@ -15,7 +15,7 @@ from __future__ import annotations import pathlib -from typing import TYPE_CHECKING, Generator +from typing import Generator, TYPE_CHECKING import pandas as pd import pandas.testing diff --git a/third_party/bigframes_vendored/constants.py b/third_party/bigframes_vendored/constants.py index aa331483a9c..9705b19c904 100644 --- a/third_party/bigframes_vendored/constants.py +++ b/third_party/bigframes_vendored/constants.py @@ -55,6 +55,3 @@ "_deferred", ] VALID_WRITE_ENGINES = typing.get_args(WriteEngineType) - -DEFAULT_SORT_KIND = "stable" -STABLE_SORT_KINDS = ("stable", "mergesort") diff --git a/third_party/bigframes_vendored/cpython/_pprint.py b/third_party/bigframes_vendored/cpython/_pprint.py index 62450985816..9b586c939bd 100644 --- a/third_party/bigframes_vendored/cpython/_pprint.py +++ b/third_party/bigframes_vendored/cpython/_pprint.py @@ -70,11 +70,11 @@ # - removed global get_config, set _changed_only=True # - replace is_scalar_nan with isinstance(x, numbers.Real) and math.isnan +from collections import OrderedDict import inspect import math import numbers import pprint -from collections import OrderedDict from bigframes.ml.base import BaseEstimator diff --git a/third_party/bigframes_vendored/geopandas/geoseries.py b/third_party/bigframes_vendored/geopandas/geoseries.py index 6b56a318011..642cf2fc900 100644 --- a/third_party/bigframes_vendored/geopandas/geoseries.py +++ b/third_party/bigframes_vendored/geopandas/geoseries.py @@ -497,9 +497,7 @@ def is_closed(self: GeoSeries) -> bigframes.series.Series: """ raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) - def simplify( - self, tolerance: float, preserve_topology: bool = True - ) -> bigframes.series.Series: # type: ignore + def simplify(self, tolerance: float, preserve_topology: bool = True) -> bigframes.series.Series: # type: ignore """[Not Implemented] Use ``bigframes.bigquery.st_simplify(series, tolerance_meters)``, instead to set the tolerance in meters. diff --git a/third_party/bigframes_vendored/google_cloud_bigquery/retry.py b/third_party/bigframes_vendored/google_cloud_bigquery/retry.py index 9117d7aa569..15ecda4fbc4 100644 --- a/third_party/bigframes_vendored/google_cloud_bigquery/retry.py +++ b/third_party/bigframes_vendored/google_cloud_bigquery/retry.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import google.api_core.future.polling -import requests.exceptions from google.api_core import exceptions, retry +import google.api_core.future.polling from google.auth import exceptions as auth_exceptions # type: ignore +import requests.exceptions _RETRYABLE_REASONS = frozenset( ["rateLimitExceeded", "backendError", "internalError", "badGateway"] diff --git a/third_party/bigframes_vendored/google_cloud_bigquery/tests/unit/test_pandas_helpers.py b/third_party/bigframes_vendored/google_cloud_bigquery/tests/unit/test_pandas_helpers.py index c87444f41e2..c798b0d1695 100644 --- a/third_party/bigframes_vendored/google_cloud_bigquery/tests/unit/test_pandas_helpers.py +++ b/third_party/bigframes_vendored/google_cloud_bigquery/tests/unit/test_pandas_helpers.py @@ -16,11 +16,11 @@ import functools import warnings +from google.cloud.bigquery import schema import pyarrow import pyarrow.parquet import pyarrow.types import pytest -from google.cloud.bigquery import schema @pytest.fixture diff --git a/third_party/bigframes_vendored/ibis/__init__.py b/third_party/bigframes_vendored/ibis/__init__.py index 54a896da889..236f471cd36 100644 --- a/third_party/bigframes_vendored/ibis/__init__.py +++ b/third_party/bigframes_vendored/ibis/__init__.py @@ -6,12 +6,12 @@ __version__ = "9.2.0" -import warnings from typing import Any +import warnings -import bigframes_vendored.ibis.backends.bigquery as bigquery from bigframes_vendored.ibis import util from bigframes_vendored.ibis.backends import BaseBackend +import bigframes_vendored.ibis.backends.bigquery as bigquery from bigframes_vendored.ibis.common.exceptions import IbisError from bigframes_vendored.ibis.config import options from bigframes_vendored.ibis.expr import api diff --git a/third_party/bigframes_vendored/ibis/backends/__init__.py b/third_party/bigframes_vendored/ibis/backends/__init__.py index 0d0feca9d38..23e3f03f4d2 100644 --- a/third_party/bigframes_vendored/ibis/backends/__init__.py +++ b/third_party/bigframes_vendored/ibis/backends/__init__.py @@ -7,18 +7,18 @@ import functools import importlib.metadata import keyword +from pathlib import Path import re +from typing import Any, ClassVar, TYPE_CHECKING import urllib.parse -from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar import bigframes_vendored.ibis +from bigframes_vendored.ibis import util +from bigframes_vendored.ibis.common.caching import RefCountedCache import bigframes_vendored.ibis.common.exceptions as exc import bigframes_vendored.ibis.config import bigframes_vendored.ibis.expr.operations as ops import bigframes_vendored.ibis.expr.types as ir -from bigframes_vendored.ibis import util -from bigframes_vendored.ibis.common.caching import RefCountedCache if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping, MutableMapping diff --git a/third_party/bigframes_vendored/ibis/backends/bigquery/__init__.py b/third_party/bigframes_vendored/ibis/backends/bigquery/__init__.py index 5a84a6a80fd..b342c7e4a99 100644 --- a/third_party/bigframes_vendored/ibis/backends/bigquery/__init__.py +++ b/third_party/bigframes_vendored/ibis/backends/bigquery/__init__.py @@ -9,21 +9,9 @@ import glob import os import re -from typing import TYPE_CHECKING, Any, Optional +from typing import Any, Optional, TYPE_CHECKING import bigframes_vendored.ibis -import bigframes_vendored.ibis.common.exceptions as com -import bigframes_vendored.ibis.expr.datatypes as ibis_dtypes -import bigframes_vendored.ibis.expr.operations as ops -import bigframes_vendored.ibis.expr.schema as sch -import bigframes_vendored.ibis.expr.types as ir -import bigframes_vendored.sqlglot as sg -import bigframes_vendored.sqlglot.expressions as sge -import google.api_core.exceptions -import google.auth.credentials -import google.cloud.bigquery as bq -import google.cloud.bigquery_storage_v1 as bqstorage -import pydata_google_auth from bigframes_vendored.ibis import util from bigframes_vendored.ibis.backends import CanCreateDatabase, CanCreateSchema from bigframes_vendored.ibis.backends.bigquery.client import ( @@ -33,9 +21,24 @@ schema_from_bigquery_table, ) from bigframes_vendored.ibis.backends.bigquery.datatypes import BigQuerySchema +from bigframes_vendored.ibis.backends.bigquery.udf.core import ( + PythonToJavaScriptTranslator, +) from bigframes_vendored.ibis.backends.sql import SQLBackend from bigframes_vendored.ibis.backends.sql.compilers import BigQueryCompiler from bigframes_vendored.ibis.backends.sql.datatypes import BigQueryType +import bigframes_vendored.ibis.common.exceptions as com +import bigframes_vendored.ibis.expr.datatypes as ibis_dtypes +import bigframes_vendored.ibis.expr.operations as ops +import bigframes_vendored.ibis.expr.schema as sch +import bigframes_vendored.ibis.expr.types as ir +import bigframes_vendored.sqlglot as sg +import bigframes_vendored.sqlglot.expressions as sge +import google.api_core.exceptions +import google.auth.credentials +import google.cloud.bigquery as bq +import google.cloud.bigquery_storage_v1 as bqstorage +import pydata_google_auth from pydata_google_auth import cache if TYPE_CHECKING: @@ -728,7 +731,15 @@ def compile( ): """Compile an Ibis expression to a SQL string.""" query = self._to_sqlglot(expr, limit=limit, params=params, **kwargs) - sql = query.sql(dialect=self.name, pretty=True) + udf_sources = [] + for udf_node in expr.op().find(ops.ScalarUDF): + compile_func = getattr( + self, f"_compile_{udf_node.__input_type__.name.lower()}_udf" + ) + if sql := compile_func(udf_node): + udf_sources.append(sql.sql(self.name, pretty=True)) + + sql = ";\n".join([*udf_sources, query.sql(dialect=self.name, pretty=True)]) self._log(sql) return sql @@ -1175,6 +1186,68 @@ def _clean_up_cached_table(self, name): force=True, ) + def _get_udf_source(self, udf_node: ops.ScalarUDF): + name = type(udf_node).__name__ + type_mapper = self.compiler.udf_type_mapper + + body = PythonToJavaScriptTranslator(udf_node.__func__).compile() + config = udf_node.__config__ + libraries = config.get("libraries", []) + + signature = [ + sge.ColumnDef( + this=sg.to_identifier(name, quoted=self.compiler.quoted), + kind=type_mapper.from_ibis(param.annotation.pattern.dtype), + ) + for name, param in udf_node.__signature__.parameters.items() + ] + + lines = ['"""'] + + if config.get("strict", True): + lines.append('"use strict";') + + lines += [ + body, + "", + f"return {udf_node.__func_name__}({', '.join(udf_node.argnames)});", + '"""', + ] + + func = sge.Create( + kind="FUNCTION", + this=sge.UserDefinedFunction( + this=sg.to_identifier(name), expressions=signature, wrapped=True + ), + # not exactly what I had in mind, but it works + # + # quoting is too simplistic to handle multiline strings + expression=sge.Var(this="\n".join(lines)), + exists=False, + properties=sge.Properties( + expressions=[ + sge.TemporaryProperty(), + sge.ReturnsProperty(this=type_mapper.from_ibis(udf_node.dtype)), + sge.StabilityProperty( + this="IMMUTABLE" if config.get("determinism") else "VOLATILE" + ), + sge.LanguageProperty(this=sg.to_identifier("js")), + ] + + [ + sge.Property( + this=sg.to_identifier("library"), + value=self.compiler.f.array(*libraries), + ) + ] + * bool(libraries) + ), + ) + + return func + + def _compile_python_udf(self, udf_node: ops.ScalarUDF) -> None: + return self._get_udf_source(udf_node) + def _register_udfs(self, expr: ir.Expr) -> None: """No op because UDFs made with CREATE TEMPORARY FUNCTION must be followed by a query.""" diff --git a/third_party/bigframes_vendored/ibis/backends/bigquery/backend.py b/third_party/bigframes_vendored/ibis/backends/bigquery/backend.py index 312b284bbb8..bac508dc7ab 100644 --- a/third_party/bigframes_vendored/ibis/backends/bigquery/backend.py +++ b/third_party/bigframes_vendored/ibis/backends/bigquery/backend.py @@ -9,21 +9,9 @@ import contextlib import glob import os -from typing import TYPE_CHECKING, Any, Optional +from typing import Any, Optional, TYPE_CHECKING import bigframes_vendored.ibis -import bigframes_vendored.ibis.backends.sql.compilers as sc -import bigframes_vendored.ibis.common.exceptions as com -import bigframes_vendored.ibis.expr.operations as ops -import bigframes_vendored.ibis.expr.schema as sch -import bigframes_vendored.ibis.expr.types as ir -import bigframes_vendored.sqlglot as sg -import bigframes_vendored.sqlglot.expressions as sge -import google.api_core.exceptions -import google.auth.credentials -import google.cloud.bigquery as bq -import google.cloud.bigquery_storage_v1 as bqstorage -import pydata_google_auth from bigframes_vendored.ibis import util from bigframes_vendored.ibis.backends import CanCreateDatabase, CanCreateSchema from bigframes_vendored.ibis.backends.bigquery.client import ( @@ -37,6 +25,18 @@ BigQueryType, ) from bigframes_vendored.ibis.backends.sql import SQLBackend +import bigframes_vendored.ibis.backends.sql.compilers as sc +import bigframes_vendored.ibis.common.exceptions as com +import bigframes_vendored.ibis.expr.operations as ops +import bigframes_vendored.ibis.expr.schema as sch +import bigframes_vendored.ibis.expr.types as ir +import bigframes_vendored.sqlglot as sg +import bigframes_vendored.sqlglot.expressions as sge +import google.api_core.exceptions +import google.auth.credentials +import google.cloud.bigquery as bq +import google.cloud.bigquery_storage_v1 as bqstorage +import pydata_google_auth from pydata_google_auth import cache if TYPE_CHECKING: diff --git a/third_party/bigframes_vendored/ibis/backends/bigquery/client.py b/third_party/bigframes_vendored/ibis/backends/bigquery/client.py index 0af2f924cfb..2ff8d5a5f56 100644 --- a/third_party/bigframes_vendored/ibis/backends/bigquery/client.py +++ b/third_party/bigframes_vendored/ibis/backends/bigquery/client.py @@ -6,14 +6,14 @@ import functools -import bigframes_vendored.ibis.common.exceptions as com -import bigframes_vendored.ibis.expr.datatypes as dt -import google.cloud.bigquery as bq -import pandas as pd from bigframes_vendored.ibis.backends.bigquery.datatypes import ( BigQuerySchema, BigQueryType, ) +import bigframes_vendored.ibis.common.exceptions as com +import bigframes_vendored.ibis.expr.datatypes as dt +import google.cloud.bigquery as bq +import pandas as pd NATIVE_PARTITION_COL = "_PARTITIONTIME" diff --git a/third_party/bigframes_vendored/ibis/backends/bigquery/converter.py b/third_party/bigframes_vendored/ibis/backends/bigquery/converter.py index 2afccd454af..c2db774b2bb 100644 --- a/third_party/bigframes_vendored/ibis/backends/bigquery/converter.py +++ b/third_party/bigframes_vendored/ibis/backends/bigquery/converter.py @@ -13,6 +13,10 @@ def convert_GeoSpatial(cls, s, dtype, pandas_type): return gpd.GeoSeries(shp.from_wkt(s)) - convert_Point = convert_LineString = convert_Polygon = convert_MultiLineString = ( - convert_MultiPoint - ) = convert_MultiPolygon = convert_GeoSpatial + convert_Point = ( + convert_LineString + ) = ( + convert_Polygon + ) = ( + convert_MultiLineString + ) = convert_MultiPoint = convert_MultiPolygon = convert_GeoSpatial diff --git a/third_party/bigframes_vendored/ibis/backends/bigquery/datatypes.py b/third_party/bigframes_vendored/ibis/backends/bigquery/datatypes.py index aa9ac062a17..6039ecdf1bc 100644 --- a/third_party/bigframes_vendored/ibis/backends/bigquery/datatypes.py +++ b/third_party/bigframes_vendored/ibis/backends/bigquery/datatypes.py @@ -5,9 +5,9 @@ import bigframes_vendored.ibis import bigframes_vendored.ibis.expr.datatypes as dt import bigframes_vendored.ibis.expr.schema as sch +from bigframes_vendored.ibis.formats import SchemaMapper, TypeMapper import bigframes_vendored.sqlglot as sg import google.cloud.bigquery as bq -from bigframes_vendored.ibis.formats import SchemaMapper, TypeMapper _from_bigquery_types = { "INT64": dt.Int64, diff --git a/third_party/bigframes_vendored/ibis/backends/bigquery/udf/__init__.py b/third_party/bigframes_vendored/ibis/backends/bigquery/udf/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/third_party/bigframes_vendored/ibis/backends/bigquery/udf/core.py b/third_party/bigframes_vendored/ibis/backends/bigquery/udf/core.py new file mode 100644 index 00000000000..6f59a2becd7 --- /dev/null +++ b/third_party/bigframes_vendored/ibis/backends/bigquery/udf/core.py @@ -0,0 +1,604 @@ +# Contains code from https://github.com/ibis-project/ibis/blob/9.2.0/ibis/backends/bigquery/udf/core.py + +"""Translate a Python AST to JavaScript.""" + +from __future__ import annotations + +import ast +from collections import ChainMap +import contextlib +import functools +import inspect +import textwrap +from typing import TYPE_CHECKING + +from bigframes_vendored.ibis.backends.bigquery.udf.find import find_names +from bigframes_vendored.ibis.backends.bigquery.udf.rewrite import rewrite + +if TYPE_CHECKING: + from collections.abc import Callable + + +class SymbolTable(ChainMap): + """ChainMap subclass implementing scope for the translator. + + Notes + ----- + JavaScript requires declarations in strict mode, so to implement this we + shove a "let" at the beginning of every variable name if it doesn't already + exist in the current scope. + + """ + + def __getitem__(self, key): + if key not in self: + self[key] = key + return f"let {key}" + return key + + +def indent(lines, spaces=4): + """Indent `lines` by `spaces` spaces. + + Parameters + ---------- + lines : Union[str, List[str]] + A string or list of strings to indent + spaces : int + The number of spaces to indent `lines` + + Returns + ------- + indented_lines : str + + """ + if isinstance(lines, str): + text = [lines] + text = "\n".join(lines) + return textwrap.indent(text, " " * spaces) + + +def semicolon(f: Callable) -> Callable: + """Add a semicolon to the result of a `visit_*` call.""" + + @functools.wraps(f) + def wrapper(*args, **kwargs): + return f(*args, **kwargs) + ";" + + return wrapper + + +@rewrite.register(ast.Call(func=ast.Name(id="print"))) +def rewrite_print(node): + return ast.Call( + func=ast.Attribute( + value=ast.Name(id="console", ctx=ast.Load()), + attr="log", + ctx=ast.Load(), + ), + args=node.args, + keywords=node.keywords, + ) + + +@rewrite.register(ast.Call(func=ast.Name(id="len"))) +def rewrite_len(node): + assert len(node.args) == 1 + return ast.Attribute(value=node.args[0], attr="length", ctx=ast.Load()) + + +@rewrite.register(ast.Call(func=ast.Attribute(attr="append"))) +def rewrite_append(node): + return ast.Call( + func=ast.Attribute(value=node.func.value, attr="push", ctx=ast.Load()), + args=node.args, + keywords=node.keywords, + ) + + +@rewrite.register( + ast.Call(func=ast.Attribute(value=ast.Name(id="Array"), attr="from_")) +) +def rewrite_array_from(node): + return ast.Call( + func=ast.Attribute(value=node.func.value, attr="from"), + args=node.args, + keywords=node.keywords, + ) + + +class PythonToJavaScriptTranslator: + constructor_map = { + "list": "Array", + "Array": "Array", + "Date": "Date", + "dict": "Object", + "Map": "Map", + "WeakMap": "WeakMap", + "str": "String", + "String": "String", + "set": "Set", + "Set": "Set", + "WeakSet": "WeakSet", + } + + def __init__(self, function): + self.function = function + self.source = textwrap.dedent(inspect.getsource(function)) + self.ast = ast.parse(self.source) + self.scope = SymbolTable() + self.current_function = None + self.current_class = None + self.is_generator = False + self.is_nested_definition = False + + def compile(self): + return self.visit(self.ast) + + def visit(self, node): + node = rewrite(node) + typename = node.__class__.__name__ + method_name = f"visit_{typename}" + method = getattr(self, method_name, None) + if method is None: + raise NotImplementedError(f"{method_name!r} nodes not yet implemented") + assert callable(method) + + result = method(node) + return result + + def visit_Name(self, node): + if self.current_class is not None and node.id == "self": + return "this" + return node.id + + def visit_Yield(self, node): + self.is_generator = True + return f"yield {self.visit(node.value)}" + + def visit_YieldFrom(self, node): + self.is_generator = True + return f"yield* {self.visit(node.value)}" + + @semicolon + def visit_Assign(self, node): + try: + (target,) = node.targets + except ValueError: + raise NotImplementedError("Only single assignment supported for now") + + if not isinstance(target, (ast.Name, ast.Subscript, ast.Attribute)): + raise NotImplementedError( + "Only index, attribute, and variable name assignment " + f"supported, got {type(target).__name__}" + ) + + is_name = isinstance(target, ast.Name) + compiled_target = self.visit(target) + if not is_name or ( + self.current_class is not None and compiled_target.startswith("this.") + ): + self.scope[compiled_target] = compiled_target + return f"{self.scope[compiled_target]} = {self.visit(node.value)}" + + def translate_special_method(self, name): + return {"__init__": "constructor"}.get(name, name) + + def visit_FunctionDef(self, node): + self.current_function = node + + is_property_getter = any( + getattr(dec, "id", None) == "property" for dec in node.decorator_list + ) + + if self.current_class is None: # not a method + if is_property_getter: + raise TypeError("Functions cannot be properties, only methods can") + prefix = "function" + else: + if is_property_getter and self.is_generator: + raise TypeError("generator methods cannot be properties") + prefix = "get " * is_property_getter + + with self.local_scope(): + body = indent(map(self.visit, node.body)) + + if self.is_generator: + prefix += "* " + else: + prefix += " " * (self.current_class is None) + + lines = [ + prefix + + self.translate_special_method(node.name) + + f"({self.visit(node.args)}) {{", + body, + "}", + ] + + self.current_function = None + self.is_generator = False + return "\n".join(lines) + + @semicolon + def visit_Return(self, node): + return f"return {self.visit(node.value)}" + + def visit_Add(self, node): + return "+" + + def visit_Sub(self, node): + return "-" + + def visit_Mult(self, node): + return "*" + + def visit_Div(self, node): + return "/" + + def visit_FloorDiv(self, node): + raise AssertionError("should never reach FloorDiv") + + def visit_Pow(self, node): + raise AssertionError("should never reach Pow") + + def visit_UnaryOp(self, node): + return f"({self.visit(node.op)}{self.visit(node.operand)})" + + def visit_USub(self, node): + return "-" + + def visit_UAdd(self, node): + return "+" + + def visit_BinOp(self, node): + left, op, right = node.left, node.op, node.right + + if isinstance(op, ast.Pow): + return f"Math.pow({self.visit(left)}, {self.visit(right)})" + elif isinstance(op, ast.FloorDiv): + return f"Math.floor({self.visit(left)} / {self.visit(right)})" + return f"({self.visit(left)} {self.visit(op)} {self.visit(right)})" + + def visit_Constant(self, node): + value = node.value + if value is None: + return "null" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float, str)): + return repr(value) + raise NotImplementedError( + f"{value.__class__.__name__!r} constants not yet implemented" + ) + + def visit_NameConstant(self, node): + value = node.value + if value is True: + return "true" + elif value is False: + return "false" + assert ( + value is None + ), f"value is not True and is not False, must be None, got {value}" + return "null" + + def visit_Str(self, node): + return repr(node.s) + + def visit_Num(self, node): + return repr(node.n) + + def visit_List(self, node): + return "[{}]".format(", ".join(map(self.visit, node.elts))) + + def visit_Tuple(self, node): + # tuples becomes lists in javascript + return "[{}]".format(", ".join(map(self.visit, node.elts))) + + def visit_Dict(self, node): + return "{{{}}}".format( + ", ".join( + f"[{self.visit(key)}]: {self.visit(value)}" + for key, value in zip(node.keys, node.values) + ) + ) + + @semicolon + def visit_Expr(self, node): + return self.visit(node.value) + + def visit_Starred(self, node): + return f"...{self.visit(node.value)}" + + def visit_Call(self, node): + thing_to_call = self.visit(node.func) + constructors = self.__class__.constructor_map + args = ", ".join(map(self.visit, node.args)) + try: + thing_to_call = constructors[thing_to_call] + except KeyError: + format_string = "{}({})" + else: + format_string = "(new {}({}))" + return format_string.format(thing_to_call, args) + + def visit_Attribute(self, node): + return f"{self.visit(node.value)}.{node.attr}" + + def visit_For(self, node): + lines = [f"for (let {self.visit(node.target)} of {self.visit(node.iter)}) {{"] + with self.local_scope(): + lines.append(indent(map(self.visit, node.body))) + lines.append("}") + return "\n".join(lines) + + def visit_While(self, node): + lines = [f"while ({self.visit(node.test)}) {{"] + with self.local_scope(): + lines.append(indent(map(self.visit, node.body))) + lines.append("}") + return "\n".join(lines) + + @semicolon + def visit_Break(self, node): + return "break" + + @semicolon + def visit_Continue(self, node): + return "continue" + + def visit_Eq(self, node): + return "===" + + def visit_NotEq(self, node): + return "!==" + + def visit_Or(self, node): + return "||" + + def visit_And(self, node): + return "&&" + + def visit_BoolOp(self, node): + return "({})".format( + f" {self.visit(node.op)} ".join(map(self.visit, node.values)) + ) + + def visit_Lt(self, node): + return "<" + + def visit_LtE(self, node): + return "<=" + + def visit_Gt(self, node): + return ">" + + def visit_GtE(self, node): + return ">=" + + def visit_Compare(self, node): + rights = node.comparators + ops = node.ops + + left = node.left + comparisons = [] + for op, right in zip(ops, rights): + comparisons.append( + f"({self.visit(left)} {self.visit(op)} {self.visit(right)})" + ) + left = right + return " && ".join(comparisons) + + @semicolon + def visit_AugAssign(self, node): + target = self.visit(node.target) + op = self.visit(node.op) + value = self.visit(node.value) + return f"{target} {op}= {value}" + + def visit_Module(self, node): + return "\n\n".join(map(self.visit, node.body)) + + def visit_arg(self, node): + if self.current_class is not None and node.arg == "self": + return "" + return node.arg + + def visit_arguments(self, node): + args = list(filter(None, map(self.visit, node.args[:]))) + vararg = node.vararg + if vararg is not None: + args.append(f"...{vararg.arg}") + return ", ".join(args) + + def visit_Lambda(self, node): + args = node.args + generated_args = self.visit(args) + return f"(({generated_args}) => {self.visit(node.body)})" + + @contextlib.contextmanager + def local_scope(self): + """Assign symbols to local variables.""" + self.scope = self.scope.new_child() + try: + yield self.scope + finally: + self.scope = self.scope.parents + + def visit_If(self, node): + lines = [f"if ({self.visit(node.test)}) {{"] + + with self.local_scope(): + lines.append(indent(map(self.visit, node.body))) + lines.append("}") + + if node.orelse: + lines[-1] += " else {" + with self.local_scope(): + lines.append(indent(map(self.visit, node.orelse))) + lines.append("}") + return "\n".join(lines) + + def visit_IfExp(self, node): + test = self.visit(node.test) + body = self.visit(node.body) + orelse = self.visit(node.orelse) + return f"({test} ? {body} : {orelse})" + + def visit_Index(self, node): + return self.visit(node.value) + + def visit_Subscript(self, node): + return f"{self.visit(node.value)}[{self.visit(node.slice)}]" + + def visit_ClassDef(self, node): + self.current_class = node + bases = node.bases + + lines = [f"class {node.name}"] + if bases: + lines[-1] += " extends {}".format(", ".join(map(self.visit, bases))) + lines[-1] += " {" + lines.append(indent(map(self.visit, node.body))) + lines.append("}") + self.current_class = None + self.__class__.constructor_map[node.name] = node.name + return "\n".join(lines) + + def visit_Not(self, node): + return "!" + + def visit_ListComp(self, node): + """Generate a curried lambda function. + + [x + y for x, y in [[1, 4], [2, 5], [3, 6]]] + + becomes + + [[1, 4], [2, 5], [3, 6]]].map(([x, y]) => x + y) + """ + try: + (generator,) = node.generators + except ValueError: + raise NotImplementedError("Only single loop comprehensions are allowed") + + names = find_names(generator.target) + argslist = [ast.arg(arg=name.id, annotation=None) for name in names] + if len(names) <= 1: + signature = ast.arguments( + args=argslist, + vararg=None, + kwonlyargs=[], + kw_defaults=[], + kwarg=None, + defaults=[], + ) + else: + signature = ast.List(elts=argslist, ctx=ast.Load()) + + array = generator.iter + lam_sig = functools.partial(ast.Lambda, args=signature) + + filters = generator.ifs + if filters: + filt = ast.BoolOp(op=ast.And(), values=filters) + # array.filter + method = ast.Attribute(value=array, attr="filter", ctx=ast.Load()) + # array.filter(func) + array = ast.Call(func=method, args=[lam_sig(body=filt)], keywords=[]) + + method = ast.Attribute(value=array, attr="map", ctx=ast.Load()) + mapped = ast.Call(func=method, args=[lam_sig(body=node.elt)], keywords=[]) + result = self.visit(mapped) + return result + + def visit_Delete(self, node): + return "\n".join(f"delete {self.visit(target)};" for target in node.targets) + + +if __name__ == "__main__": + import bigframes_vendored.ibis + from bigframes_vendored.ibis import udf + + @udf.scalar.python(strict=False) + def my_func(a: float, b: float, n: float) -> list[float]: + class Rectangle: + def __init__(self, width, height): + self.width = width + self.height = height + + @property + def area(self): + return self.width * self.height + + @property + def perimeter(self): + return self.width * 2 + self.height * 2 + + def foobar(self, n): + yield from range(n) + + def sum(values): + result = 0 + for value in values: + result += value + console.log(result) # noqa: F821 + return values.reduce(lambda a, b: a + b, 0) + + def range(n): + i = 0 + while i < n: + yield i + i += 1 + + some_stuff = [x + y for x, y in [[1, 4], [2, 5], [3, 6]] if 2 < x < 3] + some_stuff1 = [range(x) for x in [1, 2, 3]] + some_stuff2 = [x + y for x, y in [(1, 4), (2, 5), (3, 6)]] + print(some_stuff) # noqa: T201 + print(some_stuff1) # noqa: T201 + print(some_stuff2) # noqa: T201 + + x = 1 + y = 2 + x = 3 + values = [] + for i in range(10): + values.append(i) + + i = 0 + foo = 2 + bar = lambda x: x # noqa: E731 + bazel = lambda x: y # noqa: E731 + while i < n: + foo = bar(bazel(10)) + i += 1 + console.log(i) # noqa: F821 + + foo = 2 + + if i == 10 and (y < 2 or i != 42): + y += 2 + else: + y -= 2 + + z = 42.0 + w = 3 + w = not False + yyz = None + print(yyz) # noqa: T201 + foobar = x < y < z < w # x < y and y < z and z < w + foobar = 1 + baz = foobar // 3 + console.log(baz) # noqa: F821 + + my_obj = {"a": 1, "b": 2} # noqa: F841 + + z = (x if y else b) + 2 + foobar + foo = Rectangle(1, 2) + nnn = len(values) + return [sum(values) - a + b * y**-x, z, foo.width, nnn] + + print( + bigframes_vendored.ibis.bigquery.compile(my_func(42.7, 13.2, 1)) + ) # noqa: T201 diff --git a/third_party/bigframes_vendored/ibis/backends/bigquery/udf/find.py b/third_party/bigframes_vendored/ibis/backends/bigquery/udf/find.py new file mode 100644 index 00000000000..b1f353ae4f1 --- /dev/null +++ b/third_party/bigframes_vendored/ibis/backends/bigquery/udf/find.py @@ -0,0 +1,64 @@ +# Contains code from https://github.com/ibis-project/ibis/blob/9.2.0/ibis/backends/bigquery/udf/find.py + +from __future__ import annotations + +import ast + +import toolz + + +class NameFinder: + """Helper class to find the unique names in an AST.""" + + __slots__ = () + + def find(self, node): + typename = type(node).__name__ + method = getattr(self, f"find_{typename}", None) + if method is None: + fields = getattr(node, "_fields", None) + if fields is None: + return + for field in fields: + value = getattr(node, field) + yield from self.find(value) + else: + yield from method(node) + + def find_Name(self, node): + # TODO not sure if this is robust to scope changes + yield node + + def find_list(self, node): + return list(toolz.concat(map(self.find, node))) + + def find_Call(self, node): + if not isinstance(node.func, ast.Name): + fields = node._fields + else: + fields = [field for field in node._fields if field != "func"] + return toolz.concat(map(self.find, (getattr(node, field) for field in fields))) + + +def find_names(node: ast.AST) -> list[ast.Name]: + """Return the unique `ast.Name` instances in an AST. + + Examples + -------- + >>> import ast + >>> node = ast.parse("a + b") + >>> names = find_names(node) + >>> names + [<....Name object at 0x...>, <....Name object at 0x...>] + >>> names[0].id + 'a' + >>> names[1].id + 'b' + + """ + return list( + toolz.unique( + filter(None, NameFinder().find(node)), + key=lambda node: (node.id, type(node.ctx)), + ) + ) diff --git a/third_party/bigframes_vendored/ibis/backends/bigquery/udf/rewrite.py b/third_party/bigframes_vendored/ibis/backends/bigquery/udf/rewrite.py new file mode 100644 index 00000000000..6d2b0df7cdf --- /dev/null +++ b/third_party/bigframes_vendored/ibis/backends/bigquery/udf/rewrite.py @@ -0,0 +1,54 @@ +# Contains code from https://github.com/ibis-project/ibis/blob/9.2.0/ibis/backends/bigquery/udf/rewrite.py + +from __future__ import annotations + +import ast +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + + +def matches(value: ast.AST, pattern: ast.AST) -> bool: + """Check whether `value` matches `pattern`.""" + # types must match exactly + if type(value) is not type(pattern): + return False + + # primitive value, such as None, True, False etc + if not isinstance(value, ast.AST) and not isinstance(pattern, ast.AST): + return value == pattern + + fields = [ + (field, getattr(pattern, field)) + for field in pattern._fields + if hasattr(pattern, field) + ] + return all( + matches(getattr(value, field_name), field_value) + for field_name, field_value in fields + ) + + +class Rewriter: + """AST pattern matcher to enable rewrite rules.""" + + def __init__(self): + self.funcs: list[tuple[ast.AST, Callable[[ast.expr], ast.expr]]] = [] + + def register(self, pattern): + def wrapper(f): + self.funcs.append((pattern, f)) + return f + + return wrapper + + def __call__(self, node): + # TODO: more efficient way of doing this? + for pattern, func in self.funcs: + if matches(node, pattern): + return func(node) + return node + + +rewrite = Rewriter() diff --git a/third_party/bigframes_vendored/ibis/backends/sql/__init__.py b/third_party/bigframes_vendored/ibis/backends/sql/__init__.py index 9035bb0755a..0e7b31527a0 100644 --- a/third_party/bigframes_vendored/ibis/backends/sql/__init__.py +++ b/third_party/bigframes_vendored/ibis/backends/sql/__init__.py @@ -4,26 +4,26 @@ import abc from functools import partial -from typing import TYPE_CHECKING, Any, ClassVar +from typing import Any, ClassVar, TYPE_CHECKING import bigframes_vendored.ibis +from bigframes_vendored.ibis import util +from bigframes_vendored.ibis.backends import BaseBackend +from bigframes_vendored.ibis.backends.sql.compilers.base import STAR import bigframes_vendored.ibis.common.exceptions as exc import bigframes_vendored.ibis.expr.operations as ops import bigframes_vendored.ibis.expr.schema as sch import bigframes_vendored.ibis.expr.types as ir import bigframes_vendored.sqlglot as sg import bigframes_vendored.sqlglot.expressions as sge -from bigframes_vendored.ibis import util -from bigframes_vendored.ibis.backends import BaseBackend -from bigframes_vendored.ibis.backends.sql.compilers.base import STAR if TYPE_CHECKING: from collections.abc import Iterable, Mapping - import pandas as pd - import pyarrow as pa from bigframes_vendored.ibis.backends.sql.compilers.base import SQLGlotCompiler from bigframes_vendored.ibis.expr.schema import SchemaLike + import pandas as pd + import pyarrow as pa class _DatabaseSchemaHandler: @@ -89,8 +89,8 @@ def has_operation(cls, operation: type[ops.Value]) -> bool: ) def _fetch_from_cursor(self, cursor, schema: sch.Schema) -> pd.DataFrame: - import pandas as pd from bigframes_vendored.ibis.formats.pandas import PandasData + import pandas as pd try: df = pd.DataFrame.from_records( diff --git a/third_party/bigframes_vendored/ibis/backends/sql/compilers/base.py b/third_party/bigframes_vendored/ibis/backends/sql/compilers/base.py index e6ab427be5e..b95e4280538 100644 --- a/third_party/bigframes_vendored/ibis/backends/sql/compilers/base.py +++ b/third_party/bigframes_vendored/ibis/backends/sql/compilers/base.py @@ -4,34 +4,34 @@ import abc import calendar +from functools import partial, reduce import itertools import math import operator import string -from functools import partial, reduce -from typing import TYPE_CHECKING, Any, ClassVar +from typing import Any, ClassVar, TYPE_CHECKING -import bigframes_vendored.ibis.common.exceptions as ibis_exceptions -import bigframes_vendored.ibis.common.patterns as pats -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.operations as ops -import bigframes_vendored.sqlglot as sg -import bigframes_vendored.sqlglot.expressions as sge from bigframes_vendored.ibis.backends.sql.rewrites import ( - FirstValue, - LastValue, add_one_to_nth_value_input, add_order_by_to_empty_ranking_window_functions, empty_in_values_right_side, + FirstValue, + LastValue, lower_bucket, lower_capitalize, lower_sample, one_to_zero_index, sqlize, ) +import bigframes_vendored.ibis.common.exceptions as ibis_exceptions +import bigframes_vendored.ibis.common.patterns as pats from bigframes_vendored.ibis.config import options +import bigframes_vendored.ibis.expr.datatypes as dt +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.expr.operations.udf import InputType from bigframes_vendored.ibis.expr.rewrites import lower_stringslice +import bigframes_vendored.sqlglot as sg +import bigframes_vendored.sqlglot.expressions as sge from public import public try: @@ -47,9 +47,9 @@ def AlterTable(*args, kind="TABLE", **kwargs): if TYPE_CHECKING: from collections.abc import Callable, Iterable, Mapping + from bigframes_vendored.ibis.backends.bigquery.datatypes import SqlglotType import bigframes_vendored.ibis.expr.schema as sch import bigframes_vendored.ibis.expr.types as ir - from bigframes_vendored.ibis.backends.bigquery.datatypes import SqlglotType def get_leaf_classes(op): @@ -200,9 +200,9 @@ def array(self, *args: Any) -> sge.Array: first, *rest = args if isinstance(first, sge.Select): - assert not rest, ( - "only one argument allowed when `first` is a select statement" - ) + assert ( + not rest + ), "only one argument allowed when `first` is a select statement" return sge.Array(expressions=list(map(sge.convert, (first, *rest)))) @@ -1084,9 +1084,9 @@ def visit_VarianceStandardDevCovariance(self, op, *, how, where, **kw): funcname = f"{funcs[type(op)]}_{hows[how]}" return self.agg[funcname](*args, where=where) - visit_Variance = visit_StandardDev = visit_Covariance = ( - visit_VarianceStandardDevCovariance - ) + visit_Variance = ( + visit_StandardDev + ) = visit_Covariance = visit_VarianceStandardDevCovariance def visit_SimpleCase(self, op, *, base=None, cases, results, default): return sge.Case( @@ -1394,17 +1394,9 @@ def _generate_groups(groups): return map(sge.convert, range(1, len(groups) + 1)) def visit_Aggregate(self, op, *, parent, groups, metrics): - exprs = [] - if groups: - exprs.extend(self._cleanup_names(groups)) - if metrics: - exprs.extend(self._cleanup_names(metrics)) - - if not exprs: - # Empty aggregated projections are invalid in BigQuery - exprs = [sge.Literal.number(1)] - - sel = sg.select(*exprs, copy=False).from_(parent, copy=False) + sel = sg.select( + *self._cleanup_names(groups), *self._cleanup_names(metrics), copy=False + ).from_(parent, copy=False) if groups: sel = sel.group_by(*self._generate_groups(groups.values()), copy=False) @@ -1545,9 +1537,11 @@ def visit_Add(self, op, *, left, right): def visit_Subtract(self, op, *, left, right): return sge.Sub(this=left, expression=right) - visit_DateSub = visit_DateDiff = visit_TimestampSub = visit_TimestampDiff = ( - visit_IntervalSubtract - ) = visit_Subtract + visit_DateSub = ( + visit_DateDiff + ) = ( + visit_TimestampSub + ) = visit_TimestampDiff = visit_IntervalSubtract = visit_Subtract @parenthesize_inputs def visit_Multiply(self, op, *, left, right): diff --git a/third_party/bigframes_vendored/ibis/backends/sql/compilers/bigquery/__init__.py b/third_party/bigframes_vendored/ibis/backends/sql/compilers/bigquery/__init__.py index e47164f6c46..1fa5432a166 100644 --- a/third_party/bigframes_vendored/ibis/backends/sql/compilers/bigquery/__init__.py +++ b/third_party/bigframes_vendored/ibis/backends/sql/compilers/bigquery/__init__.py @@ -7,21 +7,15 @@ import decimal import math import re -from typing import TYPE_CHECKING, Any +from typing import Any, TYPE_CHECKING -import bigframes_vendored.ibis.backends.bigquery.datatypes as bq_datatypes -import bigframes_vendored.ibis.common.exceptions as ibis_exceptions -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.operations as ops -import bigframes_vendored.sqlglot as sg -import bigframes_vendored.sqlglot.expressions as sge -import numpy as np from bigframes_vendored.ibis import util +import bigframes_vendored.ibis.backends.bigquery.datatypes as bq_datatypes from bigframes_vendored.ibis.backends.sql.compilers.base import ( - NULL, - STAR, AggGen, + NULL, SQLGlotCompiler, + STAR, ) from bigframes_vendored.ibis.backends.sql.datatypes import BigQueryType, BigQueryUDFType from bigframes_vendored.ibis.backends.sql.rewrites import ( @@ -29,13 +23,19 @@ exclude_unsupported_window_frame_from_rank, exclude_unsupported_window_frame_from_row_number, ) +import bigframes_vendored.ibis.common.exceptions as ibis_exceptions from bigframes_vendored.ibis.common.temporal import ( DateUnit, IntervalUnit, TimestampUnit, TimeUnit, ) +import bigframes_vendored.ibis.expr.datatypes as dt +import bigframes_vendored.ibis.expr.operations as ops +import bigframes_vendored.sqlglot as sg from bigframes_vendored.sqlglot.dialects import BigQuery +import bigframes_vendored.sqlglot.expressions as sge +import numpy as np if TYPE_CHECKING: from collections.abc import Mapping @@ -405,7 +405,8 @@ def visit_StringToTimestamp(self, op, *, arg, format_str): def visit_ArrayCollect(self, op, *, arg, where, order_by, include_null): if where is not None and include_null: raise ibis_exceptions.UnsupportedOperationError( - "Combining `include_null=True` and `where` is not supported by bigquery" + "Combining `include_null=True` and `where` is not supported " + "by bigquery" ) out = self.agg.array_agg(arg, where=where, order_by=order_by) if not include_null: @@ -539,15 +540,6 @@ def visit_TimestampFromUNIX(self, op, *, arg, unit): def visit_Cast(self, op, *, arg, to): from_ = op.arg.dtype - if to.is_null(): - return sge.Null() - if arg is NULL or ( - isinstance(arg, sge.Cast) - and getattr(arg, "to", None) is not None - and str(arg.to).upper() == "NULL" - ): - if to.is_struct() or to.is_array(): - return sge.Cast(this=NULL, to=self.type_mapper.from_ibis(to)) if from_.is_timestamp() and to.is_integer(): return self.f.unix_micros(arg) elif from_.is_integer() and to.is_timestamp(): @@ -1134,9 +1126,6 @@ def visit_AIGenerateInt(self, op, **kwargs): def visit_AIGenerateDouble(self, op, **kwargs): return sge.func("AI.GENERATE_DOUBLE", *self._compile_ai_args(**kwargs)) - def visit_AIEmbed(self, op, **kwargs): - return sge.func("AI.EMBED", *self._compile_ai_args(**kwargs)) - def visit_AIIf(self, op, **kwargs): return sge.func("AI.IF", *self._compile_ai_args(**kwargs)) @@ -1146,9 +1135,6 @@ def visit_AIClassify(self, op, **kwargs): def visit_AIScore(self, op, **kwargs): return sge.func("AI.SCORE", *self._compile_ai_args(**kwargs)) - def visit_AISimilarity(self, op, **kwargs): - return sge.func("AI.SIMILARITY", *self._compile_ai_args(**kwargs)) - def _compile_ai_args(self, **kwargs): args = [] diff --git a/third_party/bigframes_vendored/ibis/backends/sql/datatypes.py b/third_party/bigframes_vendored/ibis/backends/sql/datatypes.py index 7a71ecf5efb..169871000a8 100644 --- a/third_party/bigframes_vendored/ibis/backends/sql/datatypes.py +++ b/third_party/bigframes_vendored/ibis/backends/sql/datatypes.py @@ -7,9 +7,9 @@ import bigframes_vendored.ibis.common.exceptions as com import bigframes_vendored.ibis.expr.datatypes as dt +from bigframes_vendored.ibis.formats import TypeMapper import bigframes_vendored.sqlglot as sg import bigframes_vendored.sqlglot.expressions as sge -from bigframes_vendored.ibis.formats import TypeMapper typecode = sge.DataType.Type @@ -414,7 +414,11 @@ def _from_ibis_SpecificGeometry(cls, dtype: dt.GeoSpatial): this = getattr(typecode, dtype.geotype.upper()) return sge.DataType(this=this, expressions=expressions) - _from_ibis_Point = _from_ibis_LineString = _from_ibis_Polygon = ( + _from_ibis_Point = ( + _from_ibis_LineString + ) = ( + _from_ibis_Polygon + ) = ( _from_ibis_MultiLineString ) = _from_ibis_MultiPoint = _from_ibis_MultiPolygon = _from_ibis_SpecificGeometry @@ -461,9 +465,11 @@ def _from_sqlglot_GEOGRAPHY( def _from_sqlglot_TINYINT(cls) -> dt.Int64: return dt.Int64(nullable=cls.default_nullable) - _from_sqlglot_UINT = _from_sqlglot_USMALLINT = _from_sqlglot_UTINYINT = ( - _from_sqlglot_INT - ) = _from_sqlglot_SMALLINT = _from_sqlglot_TINYINT + _from_sqlglot_UINT = ( + _from_sqlglot_USMALLINT + ) = ( + _from_sqlglot_UTINYINT + ) = _from_sqlglot_INT = _from_sqlglot_SMALLINT = _from_sqlglot_TINYINT @classmethod def _from_sqlglot_UBIGINT(cls) -> NoReturn: diff --git a/third_party/bigframes_vendored/ibis/backends/sql/rewrites.py b/third_party/bigframes_vendored/ibis/backends/sql/rewrites.py index dbdce90517c..a252f116ddb 100644 --- a/third_party/bigframes_vendored/ibis/backends/sql/rewrites.py +++ b/third_party/bigframes_vendored/ibis/backends/sql/rewrites.py @@ -4,24 +4,24 @@ from __future__ import annotations -import operator from collections.abc import Mapping from functools import reduce -from typing import TYPE_CHECKING, Any +import operator +from typing import Any, TYPE_CHECKING -import bigframes_vendored.ibis.common.exceptions as ibis_exceptions -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.operations as ops -import toolz from bigframes_vendored.ibis.common.annotations import attribute from bigframes_vendored.ibis.common.collections import FrozenDict # noqa: TCH001 from bigframes_vendored.ibis.common.deferred import var +import bigframes_vendored.ibis.common.exceptions as ibis_exceptions from bigframes_vendored.ibis.common.graph import Graph from bigframes_vendored.ibis.common.patterns import InstanceOf, Object, Pattern, replace from bigframes_vendored.ibis.common.typing import VarTuple # noqa: TCH001 +import bigframes_vendored.ibis.expr.datatypes as dt +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.expr.rewrites import d, p, replace_parameter from bigframes_vendored.ibis.expr.schema import Schema from public import public +import toolz if TYPE_CHECKING: from collections.abc import Sequence diff --git a/third_party/bigframes_vendored/ibis/common/annotations.py b/third_party/bigframes_vendored/ibis/common/annotations.py index 9eb0de4ee24..9365c968707 100644 --- a/third_party/bigframes_vendored/ibis/common/annotations.py +++ b/third_party/bigframes_vendored/ibis/common/annotations.py @@ -5,8 +5,8 @@ import functools import inspect import types -from typing import TYPE_CHECKING from typing import Any as AnyType +from typing import TYPE_CHECKING from bigframes_vendored.ibis.common.bases import Immutable, Slotted from bigframes_vendored.ibis.common.patterns import ( @@ -15,9 +15,9 @@ NoMatch, Option, Pattern, - TupleOf, ) from bigframes_vendored.ibis.common.patterns import pattern as ensure_pattern +from bigframes_vendored.ibis.common.patterns import TupleOf from bigframes_vendored.ibis.common.typing import format_typehint, get_type_hints if TYPE_CHECKING: diff --git a/third_party/bigframes_vendored/ibis/common/bases.py b/third_party/bigframes_vendored/ibis/common/bases.py index 2d5d798c318..c9389dececf 100644 --- a/third_party/bigframes_vendored/ibis/common/bases.py +++ b/third_party/bigframes_vendored/ibis/common/bases.py @@ -2,9 +2,9 @@ from __future__ import annotations -import collections.abc from abc import abstractmethod -from typing import TYPE_CHECKING, Any +import collections.abc +from typing import Any, TYPE_CHECKING from weakref import WeakValueDictionary if TYPE_CHECKING: @@ -128,7 +128,8 @@ def __prohibit_inheritance__(cls, **kwargs): @collections.abc.Hashable.register class Hashable(Abstract): @abstractmethod - def __hash__(self) -> int: ... + def __hash__(self) -> int: + ... class Comparable(Abstract): @@ -146,7 +147,8 @@ class Comparable(Abstract): __cache__ = {} @abstractmethod - def __equals__(self, other) -> bool: ... + def __equals__(self, other) -> bool: + ... def __eq__(self, other) -> bool: if self is other: diff --git a/third_party/bigframes_vendored/ibis/common/caching.py b/third_party/bigframes_vendored/ibis/common/caching.py index b4257410e1e..66723cec61b 100644 --- a/third_party/bigframes_vendored/ibis/common/caching.py +++ b/third_party/bigframes_vendored/ibis/common/caching.py @@ -2,10 +2,10 @@ from __future__ import annotations +from collections import namedtuple import functools import sys -from collections import namedtuple -from typing import TYPE_CHECKING, Any +from typing import Any, TYPE_CHECKING from weakref import finalize, ref if TYPE_CHECKING: diff --git a/third_party/bigframes_vendored/ibis/common/collections.py b/third_party/bigframes_vendored/ibis/common/collections.py index 718b94235dd..363e51c5341 100644 --- a/third_party/bigframes_vendored/ibis/common/collections.py +++ b/third_party/bigframes_vendored/ibis/common/collections.py @@ -2,10 +2,10 @@ from __future__ import annotations -import collections.abc from abc import abstractmethod +import collections.abc from itertools import tee -from typing import TYPE_CHECKING, Any, Generic, TypeVar +from typing import Any, Generic, TYPE_CHECKING, TypeVar from bigframes_vendored.ibis.common.bases import Abstract, Hashable from bigframes_vendored.ibis.common.exceptions import ConflictingValuesError @@ -29,7 +29,8 @@ class Iterable(Abstract, Generic[V]): """Iterable abstract base class for quicker isinstance checks.""" @abstractmethod - def __iter__(self): ... + def __iter__(self): + ... @collections.abc.Reversible.register @@ -37,7 +38,8 @@ class Reversible(Iterable[V]): """Reverse iterable abstract base class for quicker isinstance checks.""" @abstractmethod - def __reversed__(self): ... + def __reversed__(self): + ... @collections.abc.Iterator.register @@ -45,7 +47,8 @@ class Iterator(Iterable[V]): """Iterator abstract base class for quicker isinstance checks.""" @abstractmethod - def __next__(self): ... + def __next__(self): + ... def __iter__(self): return self @@ -56,7 +59,8 @@ class Sized(Abstract): """Sized abstract base class for quicker isinstance checks.""" @abstractmethod - def __len__(self): ... + def __len__(self): + ... @collections.abc.Container.register @@ -64,7 +68,8 @@ class Container(Abstract, Generic[V]): """Container abstract base class for quicker isinstance checks.""" @abstractmethod - def __contains__(self, x): ... + def __contains__(self, x): + ... @collections.abc.Collection.register @@ -77,7 +82,8 @@ class Sequence(Reversible[V], Collection[V]): """Sequence abstract base class for quicker isinstance checks.""" @abstractmethod - def __getitem__(self, index): ... + def __getitem__(self, index): + ... def __iter__(self): i = 0 @@ -121,7 +127,8 @@ class Mapping(Collection[K], Generic[K, V]): """Mapping abstract base class for quicker isinstance checks.""" @abstractmethod - def __getitem__(self, key): ... + def __getitem__(self, key): + ... def get(self, key, default=None): try: diff --git a/third_party/bigframes_vendored/ibis/common/deferred.py b/third_party/bigframes_vendored/ibis/common/deferred.py index 70e54be1503..1c6f06e223d 100644 --- a/third_party/bigframes_vendored/ibis/common/deferred.py +++ b/third_party/bigframes_vendored/ibis/common/deferred.py @@ -2,13 +2,13 @@ from __future__ import annotations +from abc import abstractmethod import collections.abc +from collections.abc import Callable import functools import inspect import operator -from abc import abstractmethod -from collections.abc import Callable -from typing import Any, TypeVar, overload +from typing import Any, overload, TypeVar from bigframes_vendored.ibis.common.bases import ( Final, @@ -51,7 +51,8 @@ def resolve(self, context: dict): """ @abstractmethod - def __eq__(self, other: Resolver) -> bool: ... + def __eq__(self, other: Resolver) -> bool: + ... @classmethod def __coerce__(cls, value): @@ -578,11 +579,13 @@ def _contains_deferred(obj: Any) -> bool: @overload -def deferrable(*, repr: str | None = None) -> Callable[[F], F]: ... +def deferrable(*, repr: str | None = None) -> Callable[[F], F]: + ... @overload -def deferrable(func: F) -> F: ... +def deferrable(func: F) -> F: + ... def deferrable(func=None, *, repr=None): diff --git a/third_party/bigframes_vendored/ibis/common/dispatch.py b/third_party/bigframes_vendored/ibis/common/dispatch.py index 9808d1fdb2b..d2920defad8 100644 --- a/third_party/bigframes_vendored/ibis/common/dispatch.py +++ b/third_party/bigframes_vendored/ibis/common/dispatch.py @@ -3,11 +3,11 @@ from __future__ import annotations import abc +from collections import defaultdict import functools import inspect import re import sys -from collections import defaultdict from typing import Union from bigframes_vendored.ibis.common.typing import ( diff --git a/third_party/bigframes_vendored/ibis/common/egraph.py b/third_party/bigframes_vendored/ibis/common/egraph.py index 437e85eda63..5497a980a44 100644 --- a/third_party/bigframes_vendored/ibis/common/egraph.py +++ b/third_party/bigframes_vendored/ibis/common/egraph.py @@ -3,9 +3,9 @@ from __future__ import annotations import collections +from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping import itertools import math -from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping from typing import Any, TypeVar from bigframes_vendored.ibis.common.bases import FrozenSlotted as Slotted diff --git a/third_party/bigframes_vendored/ibis/common/exceptions.py b/third_party/bigframes_vendored/ibis/common/exceptions.py index 4c6392cfc6a..cd46ef6bd04 100644 --- a/third_party/bigframes_vendored/ibis/common/exceptions.py +++ b/third_party/bigframes_vendored/ibis/common/exceptions.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import Any, TYPE_CHECKING if TYPE_CHECKING: from collections.abc import Callable diff --git a/third_party/bigframes_vendored/ibis/common/graph.py b/third_party/bigframes_vendored/ibis/common/graph.py index 9bf13e93ecf..6e7995ec030 100644 --- a/third_party/bigframes_vendored/ibis/common/graph.py +++ b/third_party/bigframes_vendored/ibis/common/graph.py @@ -4,11 +4,11 @@ from __future__ import annotations -import itertools from abc import abstractmethod from collections import deque from collections.abc import Callable, Iterable, Iterator, KeysView, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union +import itertools +from typing import Any, Optional, TYPE_CHECKING, TypeVar, Union from bigframes_vendored.ibis.common.bases import Hashable from bigframes_vendored.ibis.common.patterns import NoMatch, Pattern diff --git a/third_party/bigframes_vendored/ibis/common/grounds.py b/third_party/bigframes_vendored/ibis/common/grounds.py index 874e18c4057..fd639ce2e82 100644 --- a/third_party/bigframes_vendored/ibis/common/grounds.py +++ b/third_party/bigframes_vendored/ibis/common/grounds.py @@ -3,7 +3,7 @@ import contextlib from copy import copy -from typing import Any, ClassVar, Union, get_origin +from typing import Any, ClassVar, get_origin, Union from bigframes_vendored.ibis.common.annotations import ( Annotation, @@ -23,7 +23,7 @@ from bigframes_vendored.ibis.common.collections import FrozenDict # noqa: TCH001 from bigframes_vendored.ibis.common.patterns import Pattern from bigframes_vendored.ibis.common.typing import evaluate_annotations -from typing_extensions import Self, dataclass_transform +from typing_extensions import dataclass_transform, Self class AnnotableMeta(AbstractMeta): diff --git a/third_party/bigframes_vendored/ibis/common/patterns.py b/third_party/bigframes_vendored/ibis/common/patterns.py index 68861aa1908..5d4b5ba35d5 100644 --- a/third_party/bigframes_vendored/ibis/common/patterns.py +++ b/third_party/bigframes_vendored/ibis/common/patterns.py @@ -2,50 +2,50 @@ from __future__ import annotations -import math -import numbers from abc import abstractmethod from collections.abc import Callable, Mapping, Sequence from enum import Enum from inspect import Parameter +import math +import numbers +from typing import Annotated +from typing import Any as AnyType from typing import ( - Annotated, ForwardRef, Generic, + get_args, + get_origin, Literal, Optional, TypeVar, Union, - get_args, - get_origin, ) -from typing import Any as AnyType -import toolz from bigframes_vendored.ibis.common.bases import FrozenSlotted as Slotted from bigframes_vendored.ibis.common.bases import Hashable, Singleton from bigframes_vendored.ibis.common.collections import ( FrozenDict, - RewindableIterator, frozendict, + RewindableIterator, ) +from bigframes_vendored.ibis.common.deferred import _ # noqa: F401 from bigframes_vendored.ibis.common.deferred import ( Deferred, Factory, Resolver, - Variable, - _, # noqa: F401 resolver, + Variable, ) from bigframes_vendored.ibis.common.typing import ( Coercible, CoercionError, - Sentinel, format_typehint, get_bound_typevars, get_type_params, + Sentinel, ) from bigframes_vendored.ibis.util import import_object, is_iterable, unalias_package +import toolz from typing_extensions import GenericMeta T_co = TypeVar("T_co", covariant=True) @@ -225,7 +225,8 @@ def describe(self, plural=False): return f"matching {self!r}" @abstractmethod - def __eq__(self, other: Pattern) -> bool: ... + def __eq__(self, other: Pattern) -> bool: + ... def __invert__(self) -> Not: """Syntax sugar for matching the inverse of the pattern.""" @@ -1379,7 +1380,7 @@ def __init__(self, args, return_=_any): super().__init__(args=tuple(args), return_=return_) def match(self, value, context): - from bigframes_vendored.ibis.common.annotations import EMPTY, annotated + from bigframes_vendored.ibis.common.annotations import annotated, EMPTY if not callable(value): return NoMatch diff --git a/third_party/bigframes_vendored/ibis/common/temporal.py b/third_party/bigframes_vendored/ibis/common/temporal.py index 68042ad51a9..8d84caf5a1e 100644 --- a/third_party/bigframes_vendored/ibis/common/temporal.py +++ b/third_party/bigframes_vendored/ibis/common/temporal.py @@ -3,18 +3,18 @@ from __future__ import annotations import datetime -import numbers from decimal import Decimal from enum import Enum, EnumMeta +import numbers -import dateutil.parser -import dateutil.tz -import pytz from bigframes_vendored.ibis import util from bigframes_vendored.ibis.common.bases import AbstractMeta from bigframes_vendored.ibis.common.dispatch import lazy_singledispatch from bigframes_vendored.ibis.common.patterns import Coercible, CoercionError +import dateutil.parser +import dateutil.tz from public import public +import pytz class AbstractEnumMeta(EnumMeta, AbstractMeta): diff --git a/third_party/bigframes_vendored/ibis/common/typing.py b/third_party/bigframes_vendored/ibis/common/typing.py index c0c8ff3928c..a464054fdb5 100644 --- a/third_party/bigframes_vendored/ibis/common/typing.py +++ b/third_party/bigframes_vendored/ibis/common/typing.py @@ -2,13 +2,14 @@ from __future__ import annotations +from abc import abstractmethod import inspect +from itertools import zip_longest import re import sys -from abc import abstractmethod -from itertools import zip_longest -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, get_args, get_origin +from typing import Any, get_args, get_origin from typing import get_type_hints as _get_type_hints +from typing import Optional, TYPE_CHECKING, TypeVar, Union from bigframes_vendored.ibis.common.bases import Abstract from bigframes_vendored.ibis.common.caching import memoize @@ -247,7 +248,8 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any: raise TypeError("Sentinels are not constructible") -class CoercionError(Exception): ... +class CoercionError(Exception): + ... class Coercible(Abstract): @@ -260,7 +262,8 @@ class Coercible(Abstract): @classmethod @abstractmethod - def __coerce__(cls, value: Any, **kwargs: Any) -> Self: ... + def __coerce__(cls, value: Any, **kwargs: Any) -> Self: + ... def get_defining_frame(obj): diff --git a/third_party/bigframes_vendored/ibis/config.py b/third_party/bigframes_vendored/ibis/config.py index 8c2b0b1c718..39e5d94e0b3 100644 --- a/third_party/bigframes_vendored/ibis/config.py +++ b/third_party/bigframes_vendored/ibis/config.py @@ -2,8 +2,8 @@ from __future__ import annotations -import contextlib from collections.abc import Callable # noqa: TCH003 +import contextlib from typing import Annotated, Any, Optional import bigframes_vendored.ibis.common.exceptions as com diff --git a/third_party/bigframes_vendored/ibis/expr/api.py b/third_party/bigframes_vendored/ibis/expr/api.py index 953ecb2979f..fa09e23b75b 100644 --- a/third_party/bigframes_vendored/ibis/expr/api.py +++ b/third_party/bigframes_vendored/ibis/expr/api.py @@ -5,22 +5,17 @@ from __future__ import annotations import builtins +from collections import Counter import datetime import functools import itertools import numbers import operator -from collections import Counter -from typing import TYPE_CHECKING, Any, overload +from typing import Any, overload, TYPE_CHECKING -import bigframes_vendored.ibis.expr.builders as bl -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.operations as ops -import bigframes_vendored.ibis.expr.schema as sch -import bigframes_vendored.ibis.expr.types as ir from bigframes_vendored.ibis import selectors, util from bigframes_vendored.ibis.backends import BaseBackend, connect -from bigframes_vendored.ibis.common.deferred import Deferred, _, deferrable +from bigframes_vendored.ibis.common.deferred import _, deferrable, Deferred from bigframes_vendored.ibis.common.dispatch import lazy_singledispatch from bigframes_vendored.ibis.common.exceptions import IbisInputError from bigframes_vendored.ibis.common.grounds import Concrete @@ -28,34 +23,39 @@ normalize_datetime, normalize_timezone, ) +import bigframes_vendored.ibis.expr.builders as bl +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.decompile import decompile +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.expr.schema import Schema +import bigframes_vendored.ibis.expr.schema as sch from bigframes_vendored.ibis.expr.sql import parse_sql, to_sql from bigframes_vendored.ibis.expr.types import ( + array, Column, DateValue, Expr, + literal, + map, + null, Scalar, + struct, Table, TimestampValue, TimeValue, Value, - array, - literal, - map, - null, - struct, ) +import bigframes_vendored.ibis.expr.types as ir from bigframes_vendored.ibis.util import experimental if TYPE_CHECKING: from collections.abc import Iterable, Sequence from pathlib import Path + from bigframes_vendored.ibis.expr.schema import SchemaLike import pandas as pd import polars as pl import pyarrow as pa - from bigframes_vendored.ibis.expr.schema import SchemaLike __all__ = ( "Column", @@ -474,8 +474,8 @@ def _memtable( schema: SchemaLike | None = None, name: str | None = None, ) -> Table: - import pandas as pd from bigframes_vendored.ibis.formats.pandas import PandasDataFrameProxy + import pandas as pd if not isinstance(data, pd.DataFrame): df = pd.DataFrame(data, columns=columns) @@ -777,11 +777,13 @@ def timestamp( second: int | ir.IntegerValue | Deferred, /, timezone: str | None = None, -) -> TimestampValue: ... +) -> TimestampValue: + ... @overload -def timestamp(value_or_year: Any, /, timezone: str | None = None) -> TimestampValue: ... +def timestamp(value_or_year: Any, /, timezone: str | None = None) -> TimestampValue: + ... @deferrable @@ -879,11 +881,13 @@ def date( month: int | ir.IntegerValue | Deferred, day: int | ir.IntegerValue | Deferred, /, -) -> DateValue: ... +) -> DateValue: + ... @overload -def date(value_or_year: Any, /) -> DateValue: ... +def date(value_or_year: Any, /) -> DateValue: + ... @deferrable @@ -952,11 +956,13 @@ def time( minute: int | ir.IntegerValue | Deferred, second: int | ir.IntegerValue | Deferred, /, -) -> TimeValue: ... +) -> TimeValue: + ... @overload -def time(value_or_hour: Any, /) -> TimeValue: ... +def time(value_or_hour: Any, /) -> TimeValue: + ... @deferrable @@ -2467,7 +2473,3 @@ def least(*args: Any) -> ir.Value: └────────────┘ """ return ops.Least(args).to_expr() - - -def omitted() -> ir.Value: - return ops.Omitted().to_expr() diff --git a/third_party/bigframes_vendored/ibis/expr/builders.py b/third_party/bigframes_vendored/ibis/expr/builders.py index c7b6e538ff5..a6530629c9b 100644 --- a/third_party/bigframes_vendored/ibis/expr/builders.py +++ b/third_party/bigframes_vendored/ibis/expr/builders.py @@ -3,20 +3,20 @@ from __future__ import annotations import math -from typing import TYPE_CHECKING, Any, Literal, Optional, Union +from typing import Any, Literal, Optional, TYPE_CHECKING, Union import bigframes_vendored.ibis -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.operations as ops -import bigframes_vendored.ibis.expr.rules as rlz -import bigframes_vendored.ibis.expr.types as ir from bigframes_vendored.ibis import util from bigframes_vendored.ibis.common.annotations import annotated, attribute -from bigframes_vendored.ibis.common.deferred import Deferred, Resolver, deferrable +from bigframes_vendored.ibis.common.deferred import deferrable, Deferred, Resolver from bigframes_vendored.ibis.common.exceptions import IbisInputError from bigframes_vendored.ibis.common.grounds import Concrete from bigframes_vendored.ibis.common.selectors import Selector # noqa: TCH001 from bigframes_vendored.ibis.common.typing import VarTuple # noqa: TCH001 +import bigframes_vendored.ibis.expr.datatypes as dt +import bigframes_vendored.ibis.expr.operations as ops +import bigframes_vendored.ibis.expr.rules as rlz +import bigframes_vendored.ibis.expr.types as ir if TYPE_CHECKING: from typing_extensions import Self diff --git a/third_party/bigframes_vendored/ibis/expr/datatypes/cast.py b/third_party/bigframes_vendored/ibis/expr/datatypes/cast.py index e7e18419441..af8ddcf1b5f 100644 --- a/third_party/bigframes_vendored/ibis/expr/datatypes/cast.py +++ b/third_party/bigframes_vendored/ibis/expr/datatypes/cast.py @@ -3,10 +3,10 @@ from __future__ import annotations import functools -from typing import TYPE_CHECKING, Any +from typing import Any, TYPE_CHECKING -import bigframes_vendored.ibis.expr.datatypes.core as dt from bigframes_vendored.ibis.common.exceptions import IbisTypeError +import bigframes_vendored.ibis.expr.datatypes.core as dt from public import public if TYPE_CHECKING: diff --git a/third_party/bigframes_vendored/ibis/expr/datatypes/core.py b/third_party/bigframes_vendored/ibis/expr/datatypes/core.py index 75bff716626..4bacebd6d75 100644 --- a/third_party/bigframes_vendored/ibis/expr/datatypes/core.py +++ b/third_party/bigframes_vendored/ibis/expr/datatypes/core.py @@ -2,26 +2,25 @@ from __future__ import annotations +from abc import abstractmethod +from collections.abc import Iterable, Iterator, Mapping, Sequence import datetime as pydatetime import decimal as pydecimal import numbers -import uuid as pyuuid -from abc import abstractmethod -from collections.abc import Iterable, Iterator, Mapping, Sequence from numbers import Integral, Real from typing import ( Any, Generic, + get_args, + get_origin, + get_type_hints, Literal, NamedTuple, Optional, TypeVar, - get_args, - get_origin, - get_type_hints, ) +import uuid as pyuuid -import toolz from bigframes_vendored.ibis.common.annotations import attribute from bigframes_vendored.ibis.common.collections import FrozenOrderedDict, MapSet from bigframes_vendored.ibis.common.dispatch import lazy_singledispatch @@ -29,6 +28,7 @@ from bigframes_vendored.ibis.common.patterns import Coercible, CoercionError from bigframes_vendored.ibis.common.temporal import IntervalUnit, TimestampUnit from public import public +import toolz from typing_extensions import Self @@ -112,11 +112,13 @@ class DataType(Concrete, Coercible): @property @abstractmethod - def scalar(self): ... + def scalar(self): + ... @property @abstractmethod - def column(self): ... + def column(self): + ... # TODO(kszucs): remove it, prefer to use Annotable.__repr__ instead @property @@ -772,7 +774,8 @@ def __init__( if precision is not None: if not isinstance(precision, numbers.Integral): raise TypeError( - f"Decimal type precision must be an integer; got {type(precision)}" + "Decimal type precision must be an integer; " + f"got {type(precision)}" ) if precision < 0: raise ValueError("Decimal type precision cannot be negative") diff --git a/third_party/bigframes_vendored/ibis/expr/datatypes/value.py b/third_party/bigframes_vendored/ibis/expr/datatypes/value.py index 5856cb8cf94..85be0ac7497 100644 --- a/third_party/bigframes_vendored/ibis/expr/datatypes/value.py +++ b/third_party/bigframes_vendored/ibis/expr/datatypes/value.py @@ -3,20 +3,17 @@ from __future__ import annotations import collections +from collections.abc import Mapping, Sequence import datetime import decimal import enum +from functools import partial import ipaddress import json -import uuid -from collections.abc import Mapping, Sequence -from functools import partial from operator import attrgetter from typing import Any +import uuid -import bigframes_vendored.ibis.expr.datatypes as dt -import pyarrow as pa -import toolz from bigframes_vendored.ibis.common.collections import frozendict from bigframes_vendored.ibis.common.dispatch import lazy_singledispatch from bigframes_vendored.ibis.common.exceptions import IbisTypeError, InputTypeError @@ -27,8 +24,11 @@ normalize_timedelta, normalize_timezone, ) +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.datatypes.cast import highest_precedence from public import public +import pyarrow as pa +import toolz @lazy_singledispatch diff --git a/third_party/bigframes_vendored/ibis/expr/decompile.py b/third_party/bigframes_vendored/ibis/expr/decompile.py index 62913c9fa90..e32a7298f93 100644 --- a/third_party/bigframes_vendored/ibis/expr/decompile.py +++ b/third_party/bigframes_vendored/ibis/expr/decompile.py @@ -7,12 +7,12 @@ import io import itertools +from bigframes_vendored.ibis.common.graph import Graph import bigframes_vendored.ibis.expr.datatypes as dt import bigframes_vendored.ibis.expr.operations as ops +from bigframes_vendored.ibis.expr.rewrites import simplify import bigframes_vendored.ibis.expr.types as ibis_types import bigframes_vendored.ibis.expr.types as ir -from bigframes_vendored.ibis.common.graph import Graph -from bigframes_vendored.ibis.expr.rewrites import simplify from bigframes_vendored.ibis.util import experimental _method_overrides = { diff --git a/third_party/bigframes_vendored/ibis/expr/format.py b/third_party/bigframes_vendored/ibis/expr/format.py index 27eac21ddeb..39530823bc6 100644 --- a/third_party/bigframes_vendored/ibis/expr/format.py +++ b/third_party/bigframes_vendored/ibis/expr/format.py @@ -2,18 +2,18 @@ from __future__ import annotations +from collections.abc import Mapping, Sequence import functools import itertools import textwrap import types -from collections.abc import Mapping, Sequence from typing import Optional import bigframes_vendored.ibis -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis import util from bigframes_vendored.ibis.common.graph import Node +import bigframes_vendored.ibis.expr.datatypes as dt +import bigframes_vendored.ibis.expr.operations as ops from public import public _infix_ops = { diff --git a/third_party/bigframes_vendored/ibis/expr/operations/ai_ops.py b/third_party/bigframes_vendored/ibis/expr/operations/ai_ops.py index 9fa043d0bab..ef387d33792 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/ai_ops.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/ai_ops.py @@ -6,12 +6,12 @@ from typing import Optional -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz -import pyarrow as pa from bigframes_vendored.ibis.common.annotations import attribute +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.core import Value +import bigframes_vendored.ibis.expr.rules as rlz from public import public +import pyarrow as pa from bigframes.operations import output_schemas @@ -39,7 +39,7 @@ def dtype(self) -> dt.Struct: pyarrow_output_type = pa.struct( ( *output_pa_fields, - pa.field("full_response", pa.string()), + pa.field("full_resposne", pa.string()), pa.field("status", pa.string()), ) ) @@ -62,7 +62,7 @@ class AIGenerateBool(Value): @attribute def dtype(self) -> dt.Struct: return dt.Struct.from_tuples( - (("result", dt.bool), ("full_response", dt.string), ("status", dt.string)) + (("result", dt.bool), ("full_resposne", dt.string), ("status", dt.string)) ) @@ -81,7 +81,7 @@ class AIGenerateInt(Value): @attribute def dtype(self) -> dt.Struct: return dt.Struct.from_tuples( - (("result", dt.int64), ("full_response", dt.string), ("status", dt.string)) + (("result", dt.int64), ("full_resposne", dt.string), ("status", dt.string)) ) @@ -102,31 +102,7 @@ def dtype(self) -> dt.Struct: return dt.Struct.from_tuples( ( ("result", dt.float64), - ("full_response", dt.string), - ("status", dt.string), - ) - ) - - -@public -class AIEmbed(Value): - """Create embeddings from text or image data.""" - - content: Value - connection_id: Optional[Value[dt.String]] - endpoint: Optional[Value[dt.String]] - model: Optional[Value[dt.String]] - task_type: Optional[Value[dt.String]] - title: Optional[Value[dt.String]] - model_params: Optional[Value[dt.String]] - - shape = rlz.shape_like("content") - - @attribute - def dtype(self) -> dt.Struct: - return dt.Struct.from_tuples( - ( - ("result", dt.Array(dt.float64)), + ("full_resposne", dt.string), ("status", dt.string), ) ) @@ -137,10 +113,7 @@ class AIIf(Value): """Generate True/False based on the prompt""" prompt: Value - connection_id: Optional[Value[dt.String]] - endpoint: Optional[Value[dt.String]] - optimization_mode: Optional[Value[dt.String]] - max_error_ratio: Optional[Value[dt.Float64]] + connection_id: Value[dt.String] shape = rlz.shape_like("prompt") @@ -151,55 +124,28 @@ def dtype(self) -> dt.Struct: @public class AIClassify(Value): - """Generate categories based on the prompt""" + """Generate True/False based on the prompt""" input: Value categories: Value[dt.Array[dt.String]] - examples: Optional[Value] - connection_id: Optional[Value[dt.String]] - endpoint: Optional[Value[dt.String]] - output_mode: Optional[Value[dt.String]] - optimization_mode: Optional[Value[dt.String]] - max_error_ratio: Optional[Value[dt.Float64]] + connection_id: Value[dt.String] shape = rlz.shape_like("input") @attribute - def dtype(self) -> dt.DataType: - if self.output_mode is not None: - return dt.Array(dt.string) + def dtype(self) -> dt.Struct: return dt.string @public class AIScore(Value): - """Generate scores based on the prompt""" + """Generate doubles based on the prompt""" prompt: Value - connection_id: Optional[Value[dt.String]] - endpoint: Optional[Value[dt.String]] - max_error_ratio: Optional[Value[dt.Float64]] + connection_id: Value[dt.String] shape = rlz.shape_like("prompt") - @attribute - def dtype(self) -> dt.DataType: - return dt.float64 - - -@public -class AISimilarity(Value): - """Calculate the similarity between two contents""" - - content1: Value - content2: Value - endpoint: Optional[Value[dt.String]] - model: Optional[Value[dt.String]] - model_params: Optional[Value[dt.String]] - connection_id: Optional[Value[dt.String]] - - shape = rlz.shape_like("content1") - @attribute def dtype(self) -> dt.Struct: return dt.float64 diff --git a/third_party/bigframes_vendored/ibis/expr/operations/analytic.py b/third_party/bigframes_vendored/ibis/expr/operations/analytic.py index 584fba23f66..c394fb1fc9d 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/analytic.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/analytic.py @@ -8,9 +8,9 @@ import bigframes_vendored.ibis.expr.datashape as ds import bigframes_vendored.ibis.expr.datatypes as dt +from bigframes_vendored.ibis.expr.operations.core import Column, Scalar, Value import bigframes_vendored.ibis.expr.operations.udf as ibis_udf import bigframes_vendored.ibis.expr.rules as rlz -from bigframes_vendored.ibis.expr.operations.core import Column, Scalar, Value from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/operations/arrays.py b/third_party/bigframes_vendored/ibis/expr/operations/arrays.py index 7e10a3e26f9..8134506255f 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/arrays.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/arrays.py @@ -6,12 +6,12 @@ from typing import Optional -import bigframes_vendored.ibis.expr.datashape as ds -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis.common.annotations import attribute from bigframes_vendored.ibis.common.typing import VarTuple # noqa: TCH001 +import bigframes_vendored.ibis.expr.datashape as ds +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.core import Unary, Value +import bigframes_vendored.ibis.expr.rules as rlz from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/operations/core.py b/third_party/bigframes_vendored/ibis/expr/operations/core.py index ad0bd095b6c..1b8fb684fe0 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/core.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/core.py @@ -5,14 +5,14 @@ from abc import abstractmethod from typing import Generic, Optional -import bigframes_vendored.ibis.expr.datashape as ds -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis.common.annotations import attribute from bigframes_vendored.ibis.common.graph import Node as Traversable from bigframes_vendored.ibis.common.grounds import Concrete from bigframes_vendored.ibis.common.patterns import Coercible, CoercionError from bigframes_vendored.ibis.common.typing import DefaultTypeVars +import bigframes_vendored.ibis.expr.datashape as ds +import bigframes_vendored.ibis.expr.datatypes as dt +import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis.util import is_iterable from public import public from typing_extensions import Any, Self, TypeVar @@ -48,7 +48,7 @@ def __coerce__( ) -> Self: # note that S=Shape is unused here since the pattern will check the # shape of the value expression after executing Value.__coerce__() - from bigframes_vendored.ibis.expr.operations.generic import NULL, Literal + from bigframes_vendored.ibis.expr.operations.generic import Literal, NULL from bigframes_vendored.ibis.expr.types import Expr if isinstance(value, Expr): @@ -136,10 +136,6 @@ def to_expr(self): return getattr(ir, typename)(self) - @property - def omitted(self) -> bool: - return False - # convenience aliases Scalar = Value[T, ds.Scalar] diff --git a/third_party/bigframes_vendored/ibis/expr/operations/generic.py b/third_party/bigframes_vendored/ibis/expr/operations/generic.py index cc0caf21b2e..c77ecc3e718 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/generic.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/generic.py @@ -4,19 +4,20 @@ from __future__ import annotations import itertools -from typing import Annotated, Any, Optional +from typing import Annotated, Any from typing import Literal as LiteralType +from typing import Optional -import bigframes_vendored.ibis.expr.datashape as ds -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis.common.annotations import attribute from bigframes_vendored.ibis.common.deferred import Deferred # noqa: TCH001 from bigframes_vendored.ibis.common.grounds import Singleton from bigframes_vendored.ibis.common.patterns import InstanceOf, Length # noqa: TCH001 from bigframes_vendored.ibis.common.typing import VarTuple # noqa: TCH001 +import bigframes_vendored.ibis.expr.datashape as ds +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.core import Scalar, Unary, Value from bigframes_vendored.ibis.expr.operations.relations import Relation # noqa: TCH001 +import bigframes_vendored.ibis.expr.rules as rlz from public import public from typing_extensions import TypeVar @@ -188,11 +189,6 @@ class Impure(Value): pass -@public -class OmittedArg(Value): - pass - - @public class TimestampNow(Constant): """Return the current timestamp.""" diff --git a/third_party/bigframes_vendored/ibis/expr/operations/histograms.py b/third_party/bigframes_vendored/ibis/expr/operations/histograms.py index e7487887761..b3b6ad48bba 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/histograms.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/histograms.py @@ -7,10 +7,10 @@ import numbers # noqa: TCH003 from typing import Literal +from bigframes_vendored.ibis.common.annotations import attribute, ValidationError +from bigframes_vendored.ibis.common.typing import VarTuple # noqa: TCH001 import bigframes_vendored.ibis.expr.datashape as ds import bigframes_vendored.ibis.expr.datatypes as dt -from bigframes_vendored.ibis.common.annotations import ValidationError, attribute -from bigframes_vendored.ibis.common.typing import VarTuple # noqa: TCH001 from bigframes_vendored.ibis.expr.operations.core import Column, Value from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/operations/json.py b/third_party/bigframes_vendored/ibis/expr/operations/json.py index 6b03cb36672..ea4845dae25 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/json.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/json.py @@ -3,10 +3,10 @@ from __future__ import annotations -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis.common.annotations import attribute +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations import Unary, Value +import bigframes_vendored.ibis.expr.rules as rlz from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/operations/logical.py b/third_party/bigframes_vendored/ibis/expr/operations/logical.py index 74ac495642a..737db560faf 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/logical.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/logical.py @@ -3,12 +3,12 @@ from __future__ import annotations -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz -from bigframes_vendored.ibis.common.annotations import ValidationError, attribute +from bigframes_vendored.ibis.common.annotations import attribute, ValidationError from bigframes_vendored.ibis.common.exceptions import IbisTypeError from bigframes_vendored.ibis.common.typing import VarTuple # noqa: TCH001 +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.core import Binary, Unary, Value +import bigframes_vendored.ibis.expr.rules as rlz from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/operations/maps.py b/third_party/bigframes_vendored/ibis/expr/operations/maps.py index 1111e1e6898..10b44e3f38e 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/maps.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/maps.py @@ -4,10 +4,10 @@ from __future__ import annotations -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis.common.annotations import attribute +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.core import Unary, Value +import bigframes_vendored.ibis.expr.rules as rlz from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/operations/numeric.py b/third_party/bigframes_vendored/ibis/expr/operations/numeric.py index f4ba57e9d70..384323c5965 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/numeric.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/numeric.py @@ -7,11 +7,11 @@ import operator from typing import Optional -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis import util from bigframes_vendored.ibis.common.annotations import attribute +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.core import Binary, Unary, Value +import bigframes_vendored.ibis.expr.rules as rlz from public import public Integer = Value[dt.Integer] @@ -158,6 +158,8 @@ class Round(Value): def dtype(self): if self.arg.dtype.is_decimal(): return self.arg.dtype + elif self.digits is None: + return dt.int64 else: return dt.double diff --git a/third_party/bigframes_vendored/ibis/expr/operations/reductions.py b/third_party/bigframes_vendored/ibis/expr/operations/reductions.py index b739c7048fd..c3f2a032231 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/reductions.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/reductions.py @@ -6,13 +6,13 @@ from typing import Literal, Optional -import bigframes_vendored.ibis.expr.datashape as ds -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis.common.annotations import attribute from bigframes_vendored.ibis.common.typing import VarTuple +import bigframes_vendored.ibis.expr.datashape as ds +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.core import Column, Value from bigframes_vendored.ibis.expr.operations.relations import Relation # noqa: TCH001 +import bigframes_vendored.ibis.expr.rules as rlz from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/operations/relations.py b/third_party/bigframes_vendored/ibis/expr/operations/relations.py index c230cbe20b5..ef45fdfc0de 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/relations.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/relations.py @@ -4,13 +4,11 @@ from __future__ import annotations +from abc import abstractmethod import itertools import typing -from abc import abstractmethod from typing import Annotated, Any, Literal, Optional, TypeVar -import bigframes_vendored.ibis.expr.datashape as ds -import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.common.annotations import attribute from bigframes_vendored.ibis.common.collections import FrozenDict, FrozenOrderedDict from bigframes_vendored.ibis.common.exceptions import ( @@ -21,6 +19,8 @@ from bigframes_vendored.ibis.common.grounds import Concrete from bigframes_vendored.ibis.common.patterns import Between, InstanceOf from bigframes_vendored.ibis.common.typing import Coercible, VarTuple +import bigframes_vendored.ibis.expr.datashape as ds +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.core import ( Alias, Column, diff --git a/third_party/bigframes_vendored/ibis/expr/operations/sortkeys.py b/third_party/bigframes_vendored/ibis/expr/operations/sortkeys.py index f1c5b9820ac..728deba035a 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/sortkeys.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/sortkeys.py @@ -4,8 +4,8 @@ from __future__ import annotations -import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis.expr.operations.core import Value +import bigframes_vendored.ibis.expr.rules as rlz from public import public # TODO(kszucs): move the content of this file to generic.py diff --git a/third_party/bigframes_vendored/ibis/expr/operations/strings.py b/third_party/bigframes_vendored/ibis/expr/operations/strings.py index c2dc151ae07..050c079b6b3 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/strings.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/strings.py @@ -6,11 +6,11 @@ from typing import Optional -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis.common.annotations import attribute from bigframes_vendored.ibis.common.typing import VarTuple # noqa: TCH001 +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.core import Unary, Value +import bigframes_vendored.ibis.expr.rules as rlz from public import public @@ -361,10 +361,9 @@ class ExtractFragment(ExtractURLField): @public -class StringLength(Unary): - """Compute the length of a string or binary value.""" +class StringLength(StringUnary): + """Compute the length of a string.""" - arg: Value[dt.String | dt.Binary] dtype = dt.int64 diff --git a/third_party/bigframes_vendored/ibis/expr/operations/structs.py b/third_party/bigframes_vendored/ibis/expr/operations/structs.py index aa26841d9ab..6f083bed0db 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/structs.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/structs.py @@ -4,11 +4,11 @@ from __future__ import annotations -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz -from bigframes_vendored.ibis.common.annotations import ValidationError, attribute +from bigframes_vendored.ibis.common.annotations import attribute, ValidationError from bigframes_vendored.ibis.common.typing import VarTuple # noqa: TCH001 +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.core import Value +import bigframes_vendored.ibis.expr.rules as rlz from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/operations/subqueries.py b/third_party/bigframes_vendored/ibis/expr/operations/subqueries.py index c0b95a5d36c..0fbb2278969 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/subqueries.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/subqueries.py @@ -3,13 +3,13 @@ from __future__ import annotations -import bigframes_vendored.ibis.expr.datashape as ds -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis.common.annotations import attribute from bigframes_vendored.ibis.common.exceptions import IntegrityError +import bigframes_vendored.ibis.expr.datashape as ds +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.core import Value from bigframes_vendored.ibis.expr.operations.relations import Relation # noqa: TCH001 +import bigframes_vendored.ibis.expr.rules as rlz from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/operations/temporal.py b/third_party/bigframes_vendored/ibis/expr/operations/temporal.py index 729b7f14b91..b527e14f042 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/temporal.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/temporal.py @@ -7,8 +7,6 @@ import operator from typing import Annotated, Optional -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis.common.annotations import attribute from bigframes_vendored.ibis.common.patterns import As, Attrs from bigframes_vendored.ibis.common.temporal import ( @@ -17,8 +15,10 @@ TimestampUnit, TimeUnit, ) +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.core import Binary, Scalar, Unary, Value from bigframes_vendored.ibis.expr.operations.logical import Between +import bigframes_vendored.ibis.expr.rules as rlz from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/operations/udf.py b/third_party/bigframes_vendored/ibis/expr/operations/udf.py index e3e528ee90d..91366cace80 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/udf.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/udf.py @@ -11,8 +11,12 @@ import inspect import itertools import typing -from typing import TYPE_CHECKING, Any, Optional, TypeVar, overload +from typing import Any, Optional, overload, TYPE_CHECKING, TypeVar +from bigframes_vendored.ibis import util +from bigframes_vendored.ibis.common.annotations import Argument, attribute +from bigframes_vendored.ibis.common.collections import FrozenDict +from bigframes_vendored.ibis.common.deferred import deferrable import bigframes_vendored.ibis.common.exceptions as exc import bigframes_vendored.ibis.expr.datashape as ds import bigframes_vendored.ibis.expr.datatypes as dt @@ -20,10 +24,6 @@ import bigframes_vendored.ibis.expr.operations.reductions as reductions import bigframes_vendored.ibis.expr.operations.relations as relations import bigframes_vendored.ibis.expr.rules as rlz -from bigframes_vendored.ibis import util -from bigframes_vendored.ibis.common.annotations import Argument, attribute -from bigframes_vendored.ibis.common.collections import FrozenDict -from bigframes_vendored.ibis.common.deferred import deferrable from public import public if TYPE_CHECKING: @@ -35,9 +35,9 @@ EMPTY = inspect.Parameter.empty -_udf_name_cache: MutableMapping[type[core.Node], Iterable[int]] = ( - collections.defaultdict(itertools.count) -) +_udf_name_cache: MutableMapping[ + type[core.Node], Iterable[int] +] = collections.defaultdict(itertools.count) def _make_udf_name(name: str) -> str: @@ -186,7 +186,8 @@ class scalar(_UDF): @overload @classmethod - def builtin(cls, fn: Callable) -> Callable[..., ir.Value]: ... + def builtin(cls, fn: Callable) -> Callable[..., ir.Value]: + ... @overload @classmethod @@ -198,7 +199,8 @@ def builtin( catalog: str | None = None, signature: tuple[tuple[Any, ...], Any] | None = None, **kwargs: Any, - ) -> Callable[[Callable], Callable[..., ir.Value]]: ... + ) -> Callable[[Callable], Callable[..., ir.Value]]: + ... @util.experimental @classmethod @@ -261,7 +263,8 @@ def builtin( @overload @classmethod - def python(cls, fn: Callable) -> Callable[..., ir.Value]: ... + def python(cls, fn: Callable) -> Callable[..., ir.Value]: + ... @overload @classmethod @@ -273,7 +276,8 @@ def python( catalog: str | None = None, signature: tuple[tuple[Any, ...], Any] | None = None, **kwargs: Any, - ) -> Callable[[Callable], Callable[..., ir.Value]]: ... + ) -> Callable[[Callable], Callable[..., ir.Value]]: + ... @util.experimental @classmethod @@ -381,7 +385,8 @@ def python( @overload @classmethod - def pandas(cls, fn: Callable) -> Callable[..., ir.Value]: ... + def pandas(cls, fn: Callable) -> Callable[..., ir.Value]: + ... @overload @classmethod @@ -393,7 +398,8 @@ def pandas( catalog: str | None = None, signature: tuple[tuple[Any, ...], Any] | None = None, **kwargs: Any, - ) -> Callable[[Callable], Callable[..., ir.Value]]: ... + ) -> Callable[[Callable], Callable[..., ir.Value]]: + ... @util.experimental @classmethod @@ -478,7 +484,8 @@ def pandas( @overload @classmethod - def pyarrow(cls, fn: Callable) -> Callable[..., ir.Value]: ... + def pyarrow(cls, fn: Callable) -> Callable[..., ir.Value]: + ... @overload @classmethod @@ -490,7 +497,8 @@ def pyarrow( catalog: str | None = None, signature: tuple[tuple[Any, ...], Any] | None = None, **kwargs: Any, - ) -> Callable[[Callable], Callable[..., ir.Value]]: ... + ) -> Callable[[Callable], Callable[..., ir.Value]]: + ... @util.experimental @classmethod @@ -578,7 +586,8 @@ class agg(_UDF): @overload @classmethod - def builtin(cls, fn: Callable) -> Callable[..., ir.Value]: ... + def builtin(cls, fn: Callable) -> Callable[..., ir.Value]: + ... @overload @classmethod @@ -590,7 +599,8 @@ def builtin( catalog: str | None = None, signature: tuple[tuple[Any, ...], Any] | None = None, **kwargs: Any, - ) -> Callable[[Callable], Callable[..., ir.Value]]: ... + ) -> Callable[[Callable], Callable[..., ir.Value]]: + ... @util.experimental @classmethod diff --git a/third_party/bigframes_vendored/ibis/expr/operations/window.py b/third_party/bigframes_vendored/ibis/expr/operations/window.py index c40c9db2f0b..0fcecb41093 100644 --- a/third_party/bigframes_vendored/ibis/expr/operations/window.py +++ b/third_party/bigframes_vendored/ibis/expr/operations/window.py @@ -8,17 +8,17 @@ from typing import Optional import bigframes_vendored.ibis.common.exceptions as com -import bigframes_vendored.ibis.expr.datashape as ds -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.rules as rlz from bigframes_vendored.ibis.common.patterns import CoercionError from bigframes_vendored.ibis.common.typing import VarTuple # noqa: TCH001 +import bigframes_vendored.ibis.expr.datashape as ds +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.operations.analytic import Analytic # noqa: TCH001 from bigframes_vendored.ibis.expr.operations.core import Column, Value from bigframes_vendored.ibis.expr.operations.generic import Literal from bigframes_vendored.ibis.expr.operations.numeric import Negate from bigframes_vendored.ibis.expr.operations.reductions import Reduction # noqa: TCH001 from bigframes_vendored.ibis.expr.operations.sortkeys import SortKey # noqa: TCH001 +import bigframes_vendored.ibis.expr.rules as rlz from public import public from typing_extensions import TypeVar diff --git a/third_party/bigframes_vendored/ibis/expr/rewrites.py b/third_party/bigframes_vendored/ibis/expr/rewrites.py index 3ec5ea12714..779a5081caf 100644 --- a/third_party/bigframes_vendored/ibis/expr/rewrites.py +++ b/third_party/bigframes_vendored/ibis/expr/rewrites.py @@ -6,17 +6,17 @@ from collections import defaultdict -import bigframes_vendored.ibis.expr.operations as ops -import toolz from bigframes_vendored.ibis.common.collections import FrozenDict # noqa: TCH001 -from bigframes_vendored.ibis.common.deferred import Item, _, deferred, var +from bigframes_vendored.ibis.common.deferred import _, deferred, Item, var from bigframes_vendored.ibis.common.exceptions import ExpressionError, IbisInputError from bigframes_vendored.ibis.common.graph import Node as Traversable from bigframes_vendored.ibis.common.graph import traverse from bigframes_vendored.ibis.common.grounds import Concrete from bigframes_vendored.ibis.common.patterns import Check, pattern, replace from bigframes_vendored.ibis.common.typing import VarTuple # noqa: TCH001 +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.util import Namespace, promote_list +import toolz p = Namespace(pattern, module=ops) d = Namespace(deferred, module=ops) diff --git a/third_party/bigframes_vendored/ibis/expr/rules.py b/third_party/bigframes_vendored/ibis/expr/rules.py index 95050a6a5bc..a1980291825 100644 --- a/third_party/bigframes_vendored/ibis/expr/rules.py +++ b/third_party/bigframes_vendored/ibis/expr/rules.py @@ -5,13 +5,13 @@ from itertools import product, starmap from typing import Optional -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis import util from bigframes_vendored.ibis.common.annotations import attribute from bigframes_vendored.ibis.common.grounds import Concrete from bigframes_vendored.ibis.common.patterns import CoercionError, NoMatch, Pattern from bigframes_vendored.ibis.common.temporal import IntervalUnit +import bigframes_vendored.ibis.expr.datatypes as dt +import bigframes_vendored.ibis.expr.operations as ops from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/schema.py b/third_party/bigframes_vendored/ibis/expr/schema.py index edc704c3664..acfa3824fd5 100644 --- a/third_party/bigframes_vendored/ibis/expr/schema.py +++ b/third_party/bigframes_vendored/ibis/expr/schema.py @@ -3,15 +3,15 @@ from __future__ import annotations from collections.abc import Iterable, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Union +from typing import Any, TYPE_CHECKING, Union -import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.common.annotations import attribute from bigframes_vendored.ibis.common.collections import FrozenOrderedDict, MapSet from bigframes_vendored.ibis.common.dispatch import lazy_singledispatch from bigframes_vendored.ibis.common.exceptions import InputTypeError, IntegrityError from bigframes_vendored.ibis.common.grounds import Concrete from bigframes_vendored.ibis.common.patterns import Coercible +import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.util import indent if TYPE_CHECKING: diff --git a/third_party/bigframes_vendored/ibis/expr/sql.py b/third_party/bigframes_vendored/ibis/expr/sql.py index f375a7351f1..0d6df4684a4 100644 --- a/third_party/bigframes_vendored/ibis/expr/sql.py +++ b/third_party/bigframes_vendored/ibis/expr/sql.py @@ -3,8 +3,8 @@ from __future__ import annotations import contextlib -import operator from functools import singledispatch +import operator import bigframes_vendored.ibis import bigframes_vendored.ibis.expr.api as api @@ -12,11 +12,11 @@ import bigframes_vendored.ibis.expr.schema as sch import bigframes_vendored.ibis.expr.types as ibis_types import bigframes_vendored.ibis.expr.types as ir +from bigframes_vendored.ibis.util import experimental import bigframes_vendored.sqlglot as sg import bigframes_vendored.sqlglot.expressions as sge import bigframes_vendored.sqlglot.optimizer as sgo import bigframes_vendored.sqlglot.planner as sgp -from bigframes_vendored.ibis.util import experimental from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/types/arrays.py b/third_party/bigframes_vendored/ibis/expr/types/arrays.py index ee11acd56b5..47ae997738a 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/arrays.py +++ b/third_party/bigframes_vendored/ibis/expr/types/arrays.py @@ -5,8 +5,8 @@ import inspect from typing import TYPE_CHECKING +from bigframes_vendored.ibis.common.deferred import deferrable, Deferred import bigframes_vendored.ibis.expr.operations as ops -from bigframes_vendored.ibis.common.deferred import Deferred, deferrable from bigframes_vendored.ibis.expr.types.generic import Column, Scalar, Value from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/types/binary.py b/third_party/bigframes_vendored/ibis/expr/types/binary.py index b89eb6c1f1a..08fea31a1ce 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/binary.py +++ b/third_party/bigframes_vendored/ibis/expr/types/binary.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Literal +from typing import Literal, TYPE_CHECKING if TYPE_CHECKING: from bigframes_vendored.ibis.expr import types as ir @@ -35,16 +35,6 @@ def hashbytes( def __invert__(self) -> BinaryValue: return ops.BitwiseNot(self).to_expr() - def length(self) -> ir.IntegerValue: - """Compute the length of a binary value. - - Returns - ------- - IntegerValue - The length of each binary value in the expression - """ - return ops.StringLength(self).to_expr() - @public class BinaryScalar(Scalar, BinaryValue): diff --git a/third_party/bigframes_vendored/ibis/expr/types/core.py b/third_party/bigframes_vendored/ibis/expr/types/core.py index 7a527bbda28..5704dc993ae 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/core.py +++ b/third_party/bigframes_vendored/ibis/expr/types/core.py @@ -4,12 +4,10 @@ import contextlib import os +from typing import Any, NoReturn, TYPE_CHECKING import webbrowser -from typing import TYPE_CHECKING, Any, NoReturn import bigframes_vendored.ibis -import bigframes_vendored.ibis.expr.operations as ops -import pandas as pd from bigframes_vendored.ibis.common.annotations import ValidationError from bigframes_vendored.ibis.common.exceptions import IbisError, TranslationError from bigframes_vendored.ibis.common.grounds import Immutable @@ -18,8 +16,10 @@ from bigframes_vendored.ibis.config import _default_backend from bigframes_vendored.ibis.config import options as opts from bigframes_vendored.ibis.expr.format import pretty +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.expr.types.pretty import to_rich from bigframes_vendored.ibis.util import experimental +import pandas as pd from public import public from rich.console import Console from rich.jupyter import JupyterMixin @@ -29,15 +29,15 @@ from collections.abc import Iterator, Mapping from pathlib import Path - import bigframes_vendored.ibis.expr.types as ir - import polars as pl - import pyarrow as pa - import torch from bigframes_vendored.ibis.backends import BaseBackend + import bigframes_vendored.ibis.expr.types as ir from bigframes_vendored.ibis.expr.visualize import ( EdgeAttributeGetter, NodeAttributeGetter, ) + import polars as pl + import pyarrow as pa + import torch class _FixedTextJupyterMixin(JupyterMixin): diff --git a/third_party/bigframes_vendored/ibis/expr/types/generic.py b/third_party/bigframes_vendored/ibis/expr/types/generic.py index 52d07183f66..596d3134f67 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/generic.py +++ b/third_party/bigframes_vendored/ibis/expr/types/generic.py @@ -3,21 +3,21 @@ from __future__ import annotations from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING, Any +from typing import Any, TYPE_CHECKING import bigframes_vendored.ibis +from bigframes_vendored.ibis.common.deferred import _, deferrable, Deferred import bigframes_vendored.ibis.common.exceptions as com +from bigframes_vendored.ibis.common.grounds import Singleton import bigframes_vendored.ibis.expr.builders as bl import bigframes_vendored.ibis.expr.datatypes as dt import bigframes_vendored.ibis.expr.operations as ops -from bigframes_vendored.ibis.common.deferred import Deferred, _, deferrable -from bigframes_vendored.ibis.common.grounds import Singleton from bigframes_vendored.ibis.expr.rewrites import rewrite_window_input from bigframes_vendored.ibis.expr.types.core import ( - Expr, _binop, _FixedTextJupyterMixin, _is_null_literal, + Expr, ) from bigframes_vendored.ibis.expr.types.pretty import to_rich from bigframes_vendored.ibis.util import deprecated, warn_deprecated @@ -26,11 +26,11 @@ if TYPE_CHECKING: import bigframes_vendored.ibis.expr.schema as sch import bigframes_vendored.ibis.expr.types as ir + from bigframes_vendored.ibis.formats.pyarrow import PyArrowData import pandas as pd import polars as pl import pyarrow as pa import rich.table - from bigframes_vendored.ibis.formats.pyarrow import PyArrowData @public @@ -1411,9 +1411,9 @@ def __pandas_result__( ) -> pd.Series: from bigframes_vendored.ibis.formats.pandas import PandasData - assert len(df.columns) == 1, ( - "more than one column when converting columnar result DataFrame to Series" - ) + assert ( + len(df.columns) == 1 + ), "more than one column when converting columnar result DataFrame to Series" # in theory we could use df.iloc[:, 0], but there seems to be a bug in # older geopandas where df.iloc[:, 0] doesn't return the same kind of # object as df.loc[:, column_name] when df is a GeoDataFrame diff --git a/third_party/bigframes_vendored/ibis/expr/types/groupby.py b/third_party/bigframes_vendored/ibis/expr/types/groupby.py index 369eb8a0a82..102800065fa 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/groupby.py +++ b/third_party/bigframes_vendored/ibis/expr/types/groupby.py @@ -18,16 +18,16 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import Annotated, TYPE_CHECKING import bigframes_vendored.ibis -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.operations as ops -import bigframes_vendored.ibis.expr.types as ir from bigframes_vendored.ibis.common.grounds import Concrete from bigframes_vendored.ibis.common.patterns import Length # noqa: TCH001 from bigframes_vendored.ibis.common.typing import VarTuple # noqa: TCH001 +import bigframes_vendored.ibis.expr.datatypes as dt +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.expr.rewrites import rewrite_window_input +import bigframes_vendored.ibis.expr.types as ir from public import public if TYPE_CHECKING: diff --git a/third_party/bigframes_vendored/ibis/expr/types/joins.py b/third_party/bigframes_vendored/ibis/expr/types/joins.py index 62c4a334fb5..90aeff655ba 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/joins.py +++ b/third_party/bigframes_vendored/ibis/expr/types/joins.py @@ -3,10 +3,9 @@ from __future__ import annotations import functools -from typing import TYPE_CHECKING, Any +from typing import Any, TYPE_CHECKING import bigframes_vendored.ibis -import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis import util from bigframes_vendored.ibis.common.deferred import Deferred from bigframes_vendored.ibis.common.egraph import DisjointSet @@ -16,12 +15,13 @@ InputTypeError, IntegrityError, ) +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.expr.rewrites import flatten_predicates, peel_join_field from bigframes_vendored.ibis.expr.types.generic import Value from bigframes_vendored.ibis.expr.types.relations import ( + bind, DerefMap, Table, - bind, unwrap_aliases, ) from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/types/logical.py b/third_party/bigframes_vendored/ibis/expr/types/logical.py index 68ad6feae10..cc86c747f6f 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/logical.py +++ b/third_party/bigframes_vendored/ibis/expr/types/logical.py @@ -339,7 +339,7 @@ def any(self, where: BooleanValue | None = None) -> BooleanValue: │ np.False_ │ └───────────┘ """ - from bigframes_vendored.ibis.common.deferred import Call, Deferred, _ + from bigframes_vendored.ibis.common.deferred import _, Call, Deferred parents = self.op().relations diff --git a/third_party/bigframes_vendored/ibis/expr/types/maps.py b/third_party/bigframes_vendored/ibis/expr/types/maps.py index 0be7241b241..65237decc7b 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/maps.py +++ b/third_party/bigframes_vendored/ibis/expr/types/maps.py @@ -2,10 +2,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import Any, TYPE_CHECKING -import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.common.deferred import deferrable +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.expr.types.generic import Column, Scalar, Value from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/types/numeric.py b/third_party/bigframes_vendored/ibis/expr/types/numeric.py index 84ef30b9f80..c8f0d3eb347 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/numeric.py +++ b/third_party/bigframes_vendored/ibis/expr/types/numeric.py @@ -3,11 +3,11 @@ from __future__ import annotations import functools -from typing import TYPE_CHECKING, Literal +from typing import Literal, TYPE_CHECKING import bigframes_vendored.ibis -import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.common.exceptions import IbisTypeError +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.expr.types.core import _binop from bigframes_vendored.ibis.expr.types.generic import Column, Scalar, Value from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/types/pretty.py b/third_party/bigframes_vendored/ibis/expr/types/pretty.py index 22617d84615..d5796484def 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/pretty.py +++ b/third_party/bigframes_vendored/ibis/expr/types/pretty.py @@ -3,8 +3,8 @@ from __future__ import annotations import datetime -import json from functools import singledispatch +import json from math import isfinite from typing import TYPE_CHECKING from urllib.parse import urlparse @@ -12,11 +12,11 @@ import bigframes_vendored.ibis import bigframes_vendored.ibis.expr.datatypes as dt import rich -import rich.table from rich import box from rich.align import Align from rich.panel import Panel from rich.pretty import Pretty +import rich.table from rich.text import Text if TYPE_CHECKING: diff --git a/third_party/bigframes_vendored/ibis/expr/types/relations.py b/third_party/bigframes_vendored/ibis/expr/types/relations.py index 956bb95dfae..d3d66b1512f 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/relations.py +++ b/third_party/bigframes_vendored/ibis/expr/types/relations.py @@ -2,44 +2,44 @@ from __future__ import annotations -import itertools -import operator -import re from collections import deque from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +import itertools from keyword import iskeyword -from typing import TYPE_CHECKING, Any, Literal +import operator +import re +from typing import Any, Literal, TYPE_CHECKING import bigframes_vendored.ibis -import bigframes_vendored.ibis.common.exceptions as com -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.operations as ops -import bigframes_vendored.ibis.expr.schema as sch -import toolz from bigframes_vendored.ibis import util from bigframes_vendored.ibis.common.deferred import Deferred, Resolver +import bigframes_vendored.ibis.common.exceptions as com from bigframes_vendored.ibis.common.selectors import Selector +import bigframes_vendored.ibis.expr.datatypes as dt +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.expr.rewrites import DerefMap -from bigframes_vendored.ibis.expr.types.core import Expr, _FixedTextJupyterMixin -from bigframes_vendored.ibis.expr.types.generic import Value, literal +import bigframes_vendored.ibis.expr.schema as sch +from bigframes_vendored.ibis.expr.types.core import _FixedTextJupyterMixin, Expr +from bigframes_vendored.ibis.expr.types.generic import literal, Value from bigframes_vendored.ibis.expr.types.pretty import to_rich from bigframes_vendored.ibis.expr.types.temporal import TimestampColumn from bigframes_vendored.ibis.util import deprecated from public import public +import toolz if TYPE_CHECKING: - import bigframes_vendored.ibis.expr.types as ir - import bigframes_vendored.ibis.selectors as s - import pandas as pd - import polars as pl - import pyarrow as pa from bigframes_vendored.ibis.expr.operations.relations import JoinKind, Set from bigframes_vendored.ibis.expr.schema import SchemaLike from bigframes_vendored.ibis.expr.types import Table + import bigframes_vendored.ibis.expr.types as ir from bigframes_vendored.ibis.expr.types.groupby import GroupedTable from bigframes_vendored.ibis.expr.types.temporal_windows import WindowedTable from bigframes_vendored.ibis.formats.pyarrow import PyArrowData from bigframes_vendored.ibis.selectors import IfAnyAll + import bigframes_vendored.ibis.selectors as s + import pandas as pd + import polars as pl + import pyarrow as pa from rich.table import Table as RichTable @@ -3032,8 +3032,8 @@ def describe( │ island │ 1 │ string │ 344 │ 0 │ 3 │ Biscoe │ └─────────┴───────┴────────┴───────┴───────┴────────┴────────┘ """ - import bigframes_vendored.ibis.selectors as s from bigframes_vendored.ibis.expr.types.generic import literal as lit + import bigframes_vendored.ibis.selectors as s quantile = sorted(quantile) aggs = [] @@ -3050,7 +3050,7 @@ def describe( col_max = lit(None).cast(float) col_mode = lit(None).cast(str) quantile_values = { - f"p{100 * q:.6f}".rstrip("0").rstrip("."): lit(None).cast(float) + f"p{100*q:.6f}".rstrip("0").rstrip("."): lit(None).cast(float) for q in quantile } @@ -3061,9 +3061,7 @@ def describe( col_min = col.min().cast(float) col_max = col.max().cast(float) quantile_values = { - f"p{100 * q:.6f}".rstrip("0").rstrip("."): col.quantile(q).cast( - float - ) + f"p{100*q:.6f}".rstrip("0").rstrip("."): col.quantile(q).cast(float) for q in quantile } elif typ.is_string(): @@ -4356,9 +4354,9 @@ def pivot_wider( │ … │ … │ … │ … │ └───────┴──────────┴──────────┴──────────┘ """ + from bigframes_vendored.ibis.expr.rewrites import _, p, x import bigframes_vendored.ibis.selectors as s import pandas as pd - from bigframes_vendored.ibis.expr.rewrites import _, p, x orig_names_from = util.promote_list(names_from) diff --git a/third_party/bigframes_vendored/ibis/expr/types/strings.py b/third_party/bigframes_vendored/ibis/expr/types/strings.py index 29502740082..f63cf96e72c 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/strings.py +++ b/third_party/bigframes_vendored/ibis/expr/types/strings.py @@ -4,10 +4,10 @@ import functools import operator -from typing import TYPE_CHECKING, Any, Literal +from typing import Any, Literal, TYPE_CHECKING -import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis import util +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.expr.types.core import _binop from bigframes_vendored.ibis.expr.types.generic import Column, Scalar, Value from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/types/structs.py b/third_party/bigframes_vendored/ibis/expr/types/structs.py index eb5b5595a23..1f87e1bb473 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/structs.py +++ b/third_party/bigframes_vendored/ibis/expr/types/structs.py @@ -6,10 +6,10 @@ from keyword import iskeyword from typing import TYPE_CHECKING -import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.common.deferred import deferrable from bigframes_vendored.ibis.common.exceptions import IbisError -from bigframes_vendored.ibis.expr.types.generic import Column, Scalar, Value, literal +import bigframes_vendored.ibis.expr.operations as ops +from bigframes_vendored.ibis.expr.types.generic import Column, literal, Scalar, Value from public import public if TYPE_CHECKING: diff --git a/third_party/bigframes_vendored/ibis/expr/types/temporal.py b/third_party/bigframes_vendored/ibis/expr/types/temporal.py index 91e978d5402..72e41dd942c 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/temporal.py +++ b/third_party/bigframes_vendored/ibis/expr/types/temporal.py @@ -2,15 +2,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Literal +from typing import Any, Literal, TYPE_CHECKING -import bigframes_vendored.ibis.expr.datashape as ds -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.operations as ops -from bigframes_vendored import ibis from bigframes_vendored.ibis import util from bigframes_vendored.ibis.common.annotations import annotated from bigframes_vendored.ibis.common.temporal import IntervalUnit +import bigframes_vendored.ibis.expr.datashape as ds +import bigframes_vendored.ibis.expr.datatypes as dt +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.expr.types.core import _binop from bigframes_vendored.ibis.expr.types.generic import Column, Scalar, Value from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/types/temporal_windows.py b/third_party/bigframes_vendored/ibis/expr/types/temporal_windows.py index 13e917c744c..93faf9c7d41 100644 --- a/third_party/bigframes_vendored/ibis/expr/types/temporal_windows.py +++ b/third_party/bigframes_vendored/ibis/expr/types/temporal_windows.py @@ -2,14 +2,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Literal +from typing import Literal, TYPE_CHECKING -import bigframes_vendored.ibis.common.exceptions as com -import bigframes_vendored.ibis.expr.operations as ops -import bigframes_vendored.ibis.expr.types as ir from bigframes_vendored.ibis.common.collections import FrozenOrderedDict # noqa: TCH001 +import bigframes_vendored.ibis.common.exceptions as com from bigframes_vendored.ibis.common.grounds import Concrete +import bigframes_vendored.ibis.expr.operations as ops from bigframes_vendored.ibis.expr.operations.relations import Unaliased # noqa: TCH001 +import bigframes_vendored.ibis.expr.types as ir from bigframes_vendored.ibis.expr.types.relations import unwrap_aliases from public import public diff --git a/third_party/bigframes_vendored/ibis/expr/visualize.py b/third_party/bigframes_vendored/ibis/expr/visualize.py index 390c9c98282..3b5bb70ccc9 100644 --- a/third_party/bigframes_vendored/ibis/expr/visualize.py +++ b/third_party/bigframes_vendored/ibis/expr/visualize.py @@ -2,18 +2,18 @@ from __future__ import annotations +from collections.abc import Callable import contextlib +from html import escape import sys import tempfile -from collections.abc import Callable -from html import escape from typing import Optional import bigframes_vendored.ibis import bigframes_vendored.ibis.common.exceptions as com +from bigframes_vendored.ibis.common.graph import Graph import bigframes_vendored.ibis.expr.operations as ops import graphviz as gv -from bigframes_vendored.ibis.common.graph import Graph def get_type(node): @@ -187,8 +187,8 @@ def draw(graph, path=None, format="png", verbose: bool = False): if __name__ == "__main__": - import json from argparse import ArgumentParser + import json from bigframes_vendored.ibis import _ diff --git a/third_party/bigframes_vendored/ibis/formats/__init__.py b/third_party/bigframes_vendored/ibis/formats/__init__.py index 627299a0da1..96b5b86b7e1 100644 --- a/third_party/bigframes_vendored/ibis/formats/__init__.py +++ b/third_party/bigframes_vendored/ibis/formats/__init__.py @@ -3,16 +3,16 @@ from __future__ import annotations from abc import abstractmethod -from typing import TYPE_CHECKING, Generic, TypeVar +from typing import Generic, TYPE_CHECKING, TypeVar -from bigframes_vendored.ibis.util import PseudoHashable, indent +from bigframes_vendored.ibis.util import indent, PseudoHashable if TYPE_CHECKING: + from bigframes_vendored.ibis.expr.datatypes import DataType + from bigframes_vendored.ibis.expr.schema import Schema import pandas as pd import polars as pl import pyarrow as pa - from bigframes_vendored.ibis.expr.datatypes import DataType - from bigframes_vendored.ibis.expr.schema import Schema C = TypeVar("C") T = TypeVar("T") diff --git a/third_party/bigframes_vendored/ibis/formats/numpy.py b/third_party/bigframes_vendored/ibis/formats/numpy.py index 76cab2888b3..a1e53b76b70 100644 --- a/third_party/bigframes_vendored/ibis/formats/numpy.py +++ b/third_party/bigframes_vendored/ibis/formats/numpy.py @@ -4,9 +4,9 @@ import bigframes_vendored.ibis.expr.datatypes as dt import bigframes_vendored.ibis.expr.schema as sch +from bigframes_vendored.ibis.formats import SchemaMapper, TypeMapper import numpy as np import toolz -from bigframes_vendored.ibis.formats import SchemaMapper, TypeMapper _from_numpy_types = toolz.keymap( np.dtype, diff --git a/third_party/bigframes_vendored/ibis/formats/pandas.py b/third_party/bigframes_vendored/ibis/formats/pandas.py index a24c170ac50..f47f94853c4 100644 --- a/third_party/bigframes_vendored/ibis/formats/pandas.py +++ b/third_party/bigframes_vendored/ibis/formats/pandas.py @@ -4,19 +4,16 @@ import contextlib import datetime -import warnings -from functools import partial from importlib.util import find_spec as _find_spec +from functools import partial from typing import TYPE_CHECKING +import warnings -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.schema as sch -import numpy as np -import pandas as pd -import pandas.api.types as pdt from bigframes_vendored.ibis import util from bigframes_vendored.ibis.common.numeric import normalize_decimal from bigframes_vendored.ibis.common.temporal import normalize_timezone +import bigframes_vendored.ibis.expr.datatypes as dt +import bigframes_vendored.ibis.expr.schema as sch from bigframes_vendored.ibis.formats import DataMapper, SchemaMapper, TableProxy from bigframes_vendored.ibis.formats.numpy import NumpyType from bigframes_vendored.ibis.formats.pyarrow import ( @@ -24,6 +21,9 @@ PyArrowSchema, PyArrowType, ) +import numpy as np +import pandas as pd +import pandas.api.types as pdt if TYPE_CHECKING: import polars as pl @@ -179,9 +179,13 @@ def convert_GeoSpatial(cls, s, dtype, pandas_type): return gpd.GeoSeries(s) return gpd.GeoSeries.from_wkb(s) - convert_Point = convert_LineString = convert_Polygon = convert_MultiLineString = ( - convert_MultiPoint - ) = convert_MultiPolygon = convert_GeoSpatial + convert_Point = ( + convert_LineString + ) = ( + convert_Polygon + ) = ( + convert_MultiLineString + ) = convert_MultiPoint = convert_MultiPolygon = convert_GeoSpatial @classmethod def convert_default(cls, s, dtype, pandas_type): @@ -420,8 +424,8 @@ def to_pyarrow(self, schema: sch.Schema) -> pa.Table: return pa.Table.from_pandas(self.obj, schema=pyarrow_schema) def to_polars(self, schema: sch.Schema) -> pl.DataFrame: - import polars as pl from bigframes_vendored.ibis.formats.polars import PolarsSchema + import polars as pl pl_schema = PolarsSchema.from_ibis(schema) return pl.from_pandas(self.obj, schema_overrides=pl_schema) diff --git a/third_party/bigframes_vendored/ibis/formats/polars.py b/third_party/bigframes_vendored/ibis/formats/polars.py index 1c0b38ee804..4825672cfda 100644 --- a/third_party/bigframes_vendored/ibis/formats/polars.py +++ b/third_party/bigframes_vendored/ibis/formats/polars.py @@ -2,10 +2,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import Any, TYPE_CHECKING import bigframes_vendored.ibis.expr.datatypes as dt -import polars as pl from bigframes_vendored.ibis.expr.schema import Schema from bigframes_vendored.ibis.formats import ( DataMapper, @@ -13,6 +12,7 @@ TableProxy, TypeMapper, ) +import polars as pl if TYPE_CHECKING: from collections.abc import Sequence diff --git a/third_party/bigframes_vendored/ibis/formats/pyarrow.py b/third_party/bigframes_vendored/ibis/formats/pyarrow.py index 5428264ee7c..491e551ec1e 100644 --- a/third_party/bigframes_vendored/ibis/formats/pyarrow.py +++ b/third_party/bigframes_vendored/ibis/formats/pyarrow.py @@ -3,7 +3,7 @@ from __future__ import annotations import functools -from typing import TYPE_CHECKING, Any +from typing import Any, TYPE_CHECKING import bigframes_vendored.ibis.expr.datatypes as dt from bigframes_vendored.ibis.expr.schema import Schema @@ -372,8 +372,8 @@ def to_pyarrow(self, schema: Schema) -> pa.Table: return self.obj def to_polars(self, schema: Schema) -> pl.DataFrame: - import polars as pl from bigframes_vendored.ibis.formats.polars import PolarsData + import polars as pl df = pl.from_arrow(self.obj) return PolarsData.convert_table(df, schema) diff --git a/third_party/bigframes_vendored/ibis/selectors.py b/third_party/bigframes_vendored/ibis/selectors.py index 3b9f0107728..401aba253f7 100644 --- a/third_party/bigframes_vendored/ibis/selectors.py +++ b/third_party/bigframes_vendored/ibis/selectors.py @@ -52,22 +52,22 @@ from __future__ import annotations +from collections.abc import Callable, Iterable, Mapping, Sequence import functools import inspect import operator import re -from collections.abc import Callable, Iterable, Mapping, Sequence from typing import Optional, Union -import bigframes_vendored.ibis.common.exceptions as exc -import bigframes_vendored.ibis.expr.datatypes as dt -import bigframes_vendored.ibis.expr.types as ir from bigframes_vendored.ibis import util from bigframes_vendored.ibis.common.collections import frozendict # noqa: TCH001 from bigframes_vendored.ibis.common.deferred import Deferred, Resolver from bigframes_vendored.ibis.common.exceptions import IbisError +import bigframes_vendored.ibis.common.exceptions as exc from bigframes_vendored.ibis.common.grounds import Singleton from bigframes_vendored.ibis.common.selectors import Selector +import bigframes_vendored.ibis.expr.datatypes as dt +import bigframes_vendored.ibis.expr.types as ir from public import public diff --git a/third_party/bigframes_vendored/ibis/util.py b/third_party/bigframes_vendored/ibis/util.py index 7da2a7afff8..319f03aeaf2 100644 --- a/third_party/bigframes_vendored/ibis/util.py +++ b/third_party/bigframes_vendored/ibis/util.py @@ -15,14 +15,14 @@ import sys import textwrap import types -import uuid -import warnings from types import ModuleType -from typing import TYPE_CHECKING, Any, Generic, TypeVar +from typing import Any, Generic, TYPE_CHECKING, TypeVar +import uuid from uuid import uuid4 +import warnings -import toolz from bigframes_vendored.ibis.common.typing import Coercible +import toolz if TYPE_CHECKING: from collections.abc import Callable, Iterator, Sequence diff --git a/third_party/bigframes_vendored/pandas/core/col.py b/third_party/bigframes_vendored/pandas/core/col.py deleted file mode 100644 index 9b71293a7e3..00000000000 --- a/third_party/bigframes_vendored/pandas/core/col.py +++ /dev/null @@ -1,36 +0,0 @@ -# Contains code from https://github.com/pandas-dev/pandas/blob/main/pandas/core/col.py -from __future__ import annotations - -from collections.abc import Hashable - -from bigframes import constants - - -class Expression: - """ - Class representing a deferred column. - - This is not meant to be instantiated directly. Instead, use :meth:`pandas.col`. - """ - - -def col(col_name: Hashable) -> Expression: - """ - Generate deferred object representing a column of a DataFrame. - - Any place which accepts ``lambda df: df[col_name]``, such as - :meth:`DataFrame.assign` or :meth:`DataFrame.loc`, can also accept - ``pd.col(col_name)``. - - Args: - col_name (Hashable): - Column name. - - Returns: - Expression: - A deferred object representing a column of a DataFrame. - """ - raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) - - -__all__ = ["Expression", "col"] diff --git a/third_party/bigframes_vendored/pandas/core/common.py b/third_party/bigframes_vendored/pandas/core/common.py index 970ba92a91c..872a64db6c3 100644 --- a/third_party/bigframes_vendored/pandas/core/common.py +++ b/third_party/bigframes_vendored/pandas/core/common.py @@ -1,7 +1,7 @@ # Contains code from https://github.com/pandas-dev/pandas/blob/main/pandas/core/common.py from __future__ import annotations -from typing import TYPE_CHECKING, Callable +from typing import Callable, TYPE_CHECKING from bigframes_vendored.pandas.core.dtypes.inference import iterable_not_string diff --git a/third_party/bigframes_vendored/pandas/core/computation/align.py b/third_party/bigframes_vendored/pandas/core/computation/align.py index fbc53a094df..2608dabe7ac 100644 --- a/third_party/bigframes_vendored/pandas/core/computation/align.py +++ b/third_party/bigframes_vendored/pandas/core/computation/align.py @@ -2,34 +2,30 @@ """ Core eval alignment algorithms. """ - from __future__ import annotations -import warnings from functools import partial, wraps -from typing import TYPE_CHECKING, Callable, Union +from typing import Callable, TYPE_CHECKING +import warnings import bigframes_vendored.pandas.core.common as com -import numpy as np from bigframes_vendored.pandas.core.computation.common import result_type_many from bigframes_vendored.pandas.util._exceptions import find_stack_level +import numpy as np from pandas.errors import PerformanceWarning if TYPE_CHECKING: from collections.abc import Sequence + from bigframes_vendored.pandas.core.generic import NDFrame from bigframes_vendored.pandas.core.indexes.base import Index from pandas._typing import F - from bigframes.pandas import DataFrame, Series - - FrameT = Union[Series, DataFrame] - def _align_core_single_unary_op( term, -) -> tuple[partial | FrameT, dict[str, Index] | None]: - typ: partial | FrameT +) -> tuple[partial | type[NDFrame], dict[str, Index] | None]: + typ: partial | type[NDFrame] axes: dict[str, Index] | None = None if isinstance(term.value, np.ndarray): @@ -42,7 +38,9 @@ def _align_core_single_unary_op( return typ, axes -def _zip_axes_from_type(typ: FrameT, new_axes: Sequence[Index]) -> dict[str, Index]: +def _zip_axes_from_type( + typ: type[NDFrame], new_axes: Sequence[Index] +) -> dict[str, Index]: return {name: new_axes[i] for i, name in enumerate(typ._AXIS_ORDERS)} @@ -209,18 +207,20 @@ def is_series(obj) -> bool: def is_series_or_dataframe(obj) -> bool: - from bigframes.pandas import DataFrame, Series + from bigframes_vendored.pandas.core.frame import NDFrame - return isinstance(obj, Series | DataFrame) + return isinstance(obj, NDFrame) def is_pandas_object(obj) -> bool: - from bigframes.pandas import DataFrame, Index, Series + from bigframes_vendored.pandas.core.frame import NDFrame + from bigframes_vendored.pandas.core.indexes.base import Index - return isinstance(obj, Series | DataFrame | Index) + return isinstance(obj, NDFrame) or isinstance(obj, Index) def is_pandas_type(type) -> bool: - from bigframes.pandas import DataFrame, Index, Series + from bigframes_vendored.pandas.core.frame import NDFrame + from bigframes_vendored.pandas.core.indexes.base import Index - return issubclass(type, Series | DataFrame | Index) + return issubclass(type, NDFrame) or issubclass(type, Index) diff --git a/third_party/bigframes_vendored/pandas/core/computation/engines.py b/third_party/bigframes_vendored/pandas/core/computation/engines.py index 8902bb08adb..15fd48b2376 100644 --- a/third_party/bigframes_vendored/pandas/core/computation/engines.py +++ b/third_party/bigframes_vendored/pandas/core/computation/engines.py @@ -2,7 +2,6 @@ """ Engine classes for :func:`~pandas.eval` """ - from __future__ import annotations import abc diff --git a/third_party/bigframes_vendored/pandas/core/computation/eval.py b/third_party/bigframes_vendored/pandas/core/computation/eval.py index bf7e1de3bf1..a1809f6cb3a 100644 --- a/third_party/bigframes_vendored/pandas/core/computation/eval.py +++ b/third_party/bigframes_vendored/pandas/core/computation/eval.py @@ -2,17 +2,17 @@ """ Top level ``eval`` module. """ - from __future__ import annotations import tokenize -import warnings from typing import TYPE_CHECKING +import warnings from bigframes_vendored.pandas.core.computation.engines import ENGINES -from bigframes_vendored.pandas.core.computation.expr import PARSERS, Expr +from bigframes_vendored.pandas.core.computation.expr import Expr, PARSERS from bigframes_vendored.pandas.core.computation.parsing import tokenize_string from bigframes_vendored.pandas.core.computation.scope import ensure_scope +from bigframes_vendored.pandas.core.generic import NDFrame from bigframes_vendored.pandas.util._validators import validate_bool_kwarg from pandas.io.formats.printing import pprint_thing @@ -317,8 +317,6 @@ def eval( # assign if needed assigner = parsed_expr.assigner - from bigframes.pandas import DataFrame, Series - if env.target is not None and assigner is not None: target_modified = True @@ -326,7 +324,7 @@ def eval( if not inplace and first_expr: try: target = env.target - if isinstance(target, Series | DataFrame): + if isinstance(target, NDFrame): target = target.copy() except AttributeError as err: raise ValueError("Cannot return a copy of the target") from err @@ -340,7 +338,7 @@ def eval( try: with warnings.catch_warnings(record=True): # TODO: Filter the warnings we actually care about here. - if inplace and isinstance(target, Series | DataFrame): + if inplace and isinstance(target, NDFrame): target.loc[:, assigner] = ret else: target[ # pyright: ignore[reportGeneralTypeIssues] diff --git a/third_party/bigframes_vendored/pandas/core/computation/expr.py b/third_party/bigframes_vendored/pandas/core/computation/expr.py index e8def559a88..44f649e59dc 100644 --- a/third_party/bigframes_vendored/pandas/core/computation/expr.py +++ b/third_party/bigframes_vendored/pandas/core/computation/expr.py @@ -2,37 +2,36 @@ """ :func:`~pandas.eval` parsers. """ - from __future__ import annotations import ast -import tokenize from functools import partial, reduce from keyword import iskeyword +import tokenize from typing import Callable, TypeVar import bigframes_vendored.pandas.core.common as com -import numpy as np from bigframes_vendored.pandas.core.computation.ops import ( ARITH_OPS_SYMS, + BinOp, BOOL_OPS_SYMS, CMP_OPS_SYMS, - LOCAL_TAG, - UNARY_OPS_SYMS, - BinOp, Constant, Div, FuncNode, + is_term, + LOCAL_TAG, Op, Term, + UNARY_OPS_SYMS, UnaryOp, - is_term, ) from bigframes_vendored.pandas.core.computation.parsing import ( clean_backtick_quoted_toks, tokenize_string, ) from bigframes_vendored.pandas.core.computation.scope import Scope +import numpy as np from pandas.errors import UndefinedVariableError from pandas.io.formats import printing diff --git a/third_party/bigframes_vendored/pandas/core/computation/ops.py b/third_party/bigframes_vendored/pandas/core/computation/ops.py index 0dfd77daf36..75b914c876b 100644 --- a/third_party/bigframes_vendored/pandas/core/computation/ops.py +++ b/third_party/bigframes_vendored/pandas/core/computation/ops.py @@ -5,18 +5,18 @@ from __future__ import annotations -import operator from datetime import datetime from functools import partial -from typing import TYPE_CHECKING, Callable, Literal +import operator +from typing import Callable, Literal, TYPE_CHECKING import bigframes_vendored.pandas.core.common as com -import numpy as np from bigframes_vendored.pandas.core.computation.common import ( ensure_decoded, result_type_many, ) from bigframes_vendored.pandas.core.computation.scope import DEFAULT_GLOBALS +import numpy as np from pandas._libs.tslibs import Timestamp from pandas.core.dtypes.common import is_list_like, is_scalar from pandas.io.formats.printing import pprint_thing, pprint_thing_encoded diff --git a/third_party/bigframes_vendored/pandas/core/computation/parsing.py b/third_party/bigframes_vendored/pandas/core/computation/parsing.py index 569c3c50330..e54f4597357 100644 --- a/third_party/bigframes_vendored/pandas/core/computation/parsing.py +++ b/third_party/bigframes_vendored/pandas/core/computation/parsing.py @@ -2,13 +2,12 @@ """ :func:`~pandas.eval` source string parsing functions """ - from __future__ import annotations -import token -import tokenize from io import StringIO from keyword import iskeyword +import token +import tokenize from typing import TYPE_CHECKING if TYPE_CHECKING: diff --git a/third_party/bigframes_vendored/pandas/core/computation/scope.py b/third_party/bigframes_vendored/pandas/core/computation/scope.py index 51b15e74a27..bfd7eb1d12b 100644 --- a/third_party/bigframes_vendored/pandas/core/computation/scope.py +++ b/third_party/bigframes_vendored/pandas/core/computation/scope.py @@ -2,17 +2,16 @@ """ Module for scope operations """ - from __future__ import annotations +from collections import ChainMap import datetime import inspect +from io import StringIO import itertools import pprint import struct import sys -from collections import ChainMap -from io import StringIO from typing import TypeVar import numpy as np diff --git a/third_party/bigframes_vendored/pandas/core/config_init.py b/third_party/bigframes_vendored/pandas/core/config_init.py index bd40d05154b..194ec4a8a71 100644 --- a/third_party/bigframes_vendored/pandas/core/config_init.py +++ b/third_party/bigframes_vendored/pandas/core/config_init.py @@ -1,5 +1,4 @@ -# Contains code from -# https://github.com/pandas-dev/pandas/blob/main/pandas/core/config_init.py +# Contains code from https://github.com/pandas-dev/pandas/blob/main/pandas/core/config_init.py """ This module is imported from the pandas package __init__.py file in order to ensure that the core.config options registered here will @@ -30,15 +29,13 @@ class DisplayOptions: >>> import bigframes.pandas as bpd >>> df = bpd.read_gbq("bigquery-public-data.ml_datasets.penguins") - >>> bpd.options.display.repr_mode = "deferred" # doctest: +SKIP - >>> df.head(20) # will no longer run the job # doctest: +SKIP + >>> bpd.options.display.repr_mode = "deferred" + >>> df.head(20) # will no longer run the job Computation deferred. Computation will process 28.9 kB - Users can also get a dry run of the job by accessing the query_job - property before they've run the job. This will return a dry run - instance of the job they can inspect. + Users can also get a dry run of the job by accessing the query_job property before they've run the job. This will return a dry run instance of the job they can inspect. - >>> df.query_job.total_bytes_processed # doctest: +SKIP + >>> df.query_job.total_bytes_processed 28947 User can execute the job by calling .to_pandas() @@ -47,22 +44,21 @@ class DisplayOptions: Reset repr_mode option - >>> bpd.options.display.repr_mode = "head" # doctest: +SKIP + >>> bpd.options.display.repr_mode = "head" Can also set the progress_bar option to see the progress bar in terminal, - >>> bpd.options.display.progress_bar = "terminal" # doctest: +SKIP + >>> bpd.options.display.progress_bar = "terminal" notebook, - >>> bpd.options.display.progress_bar = "notebook" # doctest: +SKIP + >>> bpd.options.display.progress_bar = "notebook" or just remove it. - Setting to default value "auto" will detect and show progress bar - automatically. + Setting to default value "auto" will detect and show progress bar automatically. - >>> bpd.options.display.progress_bar = "auto" # doctest: +SKIP + >>> bpd.options.display.progress_bar = "auto" """ # Options borrowed from pandas. @@ -71,11 +67,6 @@ class DisplayOptions: Maximum number of columns to display. Default 20. If `max_columns` is exceeded, switch to truncate view. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.display.max_columns = 50 # doctest: +SKIP """ max_rows: int = 10 @@ -83,11 +74,6 @@ class DisplayOptions: Maximum number of rows to display. Default 10. If `max_rows` is exceeded, switch to truncate view. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.display.max_rows = 50 # doctest: +SKIP """ precision: int = 6 @@ -95,25 +81,15 @@ class DisplayOptions: Controls the floating point output precision. Defaults to 6. See :attr:`pandas.options.display.precision`. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.display.precision = 2 # doctest: +SKIP """ # Options unique to BigQuery DataFrames. - progress_bar: Optional[Literal["auto", "notebook", "terminal"]] = "auto" + progress_bar: Optional[str] = "auto" """ Determines if progress bars are shown during job runs. Default "auto". Valid values are `auto`, `notebook`, and `terminal`. Set to `None` to remove progress bars. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.display.progress_bar = "terminal" # doctest: +SKIP """ repr_mode: Literal["head", "deferred", "anywidget"] = "head" @@ -125,33 +101,10 @@ class DisplayOptions: Dataframe and Series objects during repr. `deferred` - Prevent executions from repr statements in DataFrame and - Series objects. + Prevent executions from repr statements in DataFrame and Series objects. Instead, estimated bytes processed will be shown. DataFrame and Series objects can still be computed with methods that explicitly execute and download results. - - `anywidget` - Display as interactive widget using `anywidget` library. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.display.repr_mode = "deferred" # doctest: +SKIP - """ - - render_mode: Literal["plaintext", "html", "anywidget"] = "html" - """ - Determines how to visualize a DataFrame or Series. Default "html". - - `plaintext` - Display as plain text. - - `html` - Display as HTML table. - - `anywidget` - Display as interactive widget using `anywidget` library. """ max_colwidth: Optional[int] = 50 @@ -160,39 +113,23 @@ class DisplayOptions: When the column overflows, a "..." placeholder is embedded in the output. A 'None' value means unlimited. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.display.max_colwidth = 20 # doctest: +SKIP """ max_info_columns: int = 100 """ Used in DataFrame.info method to decide if information in each column will be printed. Default 100. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.display.max_info_columns = 50 # doctest: +SKIP """ max_info_rows: Optional[int] = 200_000 """ - Limit null check in ``df.info()`` only to frames with smaller - dimensions than + Limit null check in ``df.info()`` only to frames with smaller dimensions than max_info_rows. Default 200,000. df.info() will usually show null-counts for each column. For large frames, this can be quite slow. max_info_rows and max_info_cols limit this null check only to frames with smaller dimensions than specified. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.display.max_info_rows = 100 # doctest: +SKIP """ memory_usage: bool = True @@ -201,39 +138,19 @@ class DisplayOptions: df.info() is called. Default True. Valid values True, False. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.display.memory_usage = False # doctest: +SKIP """ blob_display: bool = True """ If True, display the blob content in notebook DataFrame preview. Default True. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.display.blob_display = True # doctest: +SKIP """ blob_display_width: Optional[int] = None """ Width in pixels that the blob constrained to. Default None.. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.display.blob_display_width = 100 # doctest: +SKIP """ blob_display_height: Optional[int] = None """ Height in pixels that the blob constrained to. Default None.. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> bpd.options.display.blob_display_height = 100 # doctest: +SKIP """ diff --git a/third_party/bigframes_vendored/pandas/core/dtypes/inference.py b/third_party/bigframes_vendored/pandas/core/dtypes/inference.py index 7875c297bc2..fcbb4c242f4 100644 --- a/third_party/bigframes_vendored/pandas/core/dtypes/inference.py +++ b/third_party/bigframes_vendored/pandas/core/dtypes/inference.py @@ -1,5 +1,5 @@ # Contains code from https://github.com/pandas-dev/pandas/blob/main/pandas/core/dtypes/inference.py -"""basic inference routines""" +""" basic inference routines """ from __future__ import annotations diff --git a/third_party/bigframes_vendored/pandas/core/frame.py b/third_party/bigframes_vendored/pandas/core/frame.py index e84f46861d9..dc1bcca213e 100644 --- a/third_party/bigframes_vendored/pandas/core/frame.py +++ b/third_party/bigframes_vendored/pandas/core/frame.py @@ -9,16 +9,15 @@ alignment and a host of useful data manipulation methods having to do with the labeling information """ - from __future__ import annotations import datetime from typing import Hashable, Iterable, Literal, Optional, Sequence, Union +from bigframes_vendored import constants import bigframes_vendored.pandas.core.generic as generic import numpy as np import pandas as pd -from bigframes_vendored import constants from pandas.api import extensions as pd_ext # ----------------------------------------------------------------------- @@ -66,9 +65,9 @@ def axes(self) -> list: >>> df = bpd.DataFrame({'col1': [1, 2], 'col2': [3, 4]}) >>> df.axes[1:] - [Index(['col1', 'col2'], dtype='str')] + [Index(['col1', 'col2'], dtype='object')] """ - raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) + return [self.index, self.columns] @property def values(self) -> np.ndarray: @@ -553,49 +552,6 @@ def to_parquet( """ raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) - def to_csv( - self, - path_or_buf=None, - sep=",", - *, - header: bool = True, - index: bool = True, - allow_large_results: Optional[bool] = None, - ) -> Optional[str]: - """ - Write object to a comma-separated values (csv) file. - - **Examples:** - - >>> import bigframes.pandas as bpd - - >>> df = bpd.DataFrame({'col1': [1, 2], 'col2': [3, 4]}) - >>> df.to_csv() - \',col1,col2\\n0,1,3\\n1,2,4\\n\' - - Args: - path_or_buf (str, path object, file-like object, or None, default None): - String, path object (implementing os.PathLike[str]), or file-like object - implementing a write() function. If None, the result is returned as a string. - If a non-binary file object is passed, it should be opened with newline='', - disabling universal newlines. If a binary file object is passed, - mode might need to contain a 'b'. - Must contain a wildcard character '*' if this is a GCS path. - sep (str, default ','): - String of length 1. Field delimiter for the output file. - header (bool, default True): - Write out the column names. - index (bool, default True): - Write row names (index). - allow_large_results (bool, default None): - If not None, overrides the global setting to allow or disallow large - query results over the default size limit of 10 GB. - - Returns: - If path_or_buf is None, returns the resulting csv format as a string. Otherwise returns None. - """ - raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) - def to_dict( self, orient: Literal[ @@ -1963,7 +1919,7 @@ def keys(self): ... 'B': [4, 5, 6], ... }) >>> df.keys() - Index(['A', 'B'], dtype='str') + Index(['A', 'B'], dtype='object') Returns: pandas.Index: Info axis. @@ -2070,6 +2026,7 @@ def where(self, cond, other): **Examples:** + >>> df = bpd.DataFrame({'a': [20, 10, 0], 'b': [0, 10, 20]}) >>> df a b @@ -2140,14 +2097,14 @@ def where(self, cond, other): with corresponding value from other. If cond is callable, it is computed on the Series/DataFrame and returns boolean Series/DataFrame or array. The callable must not change input - Series/DataFrame. + Series/DataFrame (though pandas doesn’t check it). other (scalar, DataFrame, or callable): Entries where cond is False are replaced with corresponding value from other. If other is callable, it is computed on the DataFrame and returns scalar or DataFrame. The callable must not - change input DataFrame. If not specified, entries will be filled - with the corresponding NULL value (np.nan for numpy dtypes, - pd.NA for extension dtypes). + change input DataFrame (though pandas doesn’t check it). If not + specified, entries will be filled with the corresponding NULL + value (np.nan for numpy dtypes, pd.NA for extension dtypes). Returns: DataFrame: DataFrame after the replacement. @@ -2253,7 +2210,7 @@ def sort_values( *, inplace: bool = False, ascending: bool | Sequence[bool] = True, - kind: str | None = None, + kind: str = "quicksort", na_position: Literal["first", "last"] = "last", ): """Sort by the values along row axis. @@ -2339,7 +2296,7 @@ def sort_values( the by. inplace (bool, default False): If True, perform operation in-place. - kind (str, default None): + kind (str, default 'quicksort'): Choice of sorting algorithm. Accepts 'quicksort', 'mergesort', 'heapsort', 'stable'. Ignored except when determining whether to sort stably. 'mergesort' or 'stable' will result in stable reorder. @@ -2363,7 +2320,6 @@ def sort_index( axis: str | int = 0, ascending: bool = True, inplace: bool = False, - kind: str | None = None, na_position: Literal["first", "last"] = "last", ): """Sort object by labels (along an axis). @@ -2376,10 +2332,6 @@ def sort_index( Sort ascending vs. descending. inplace (bool, default False): Whether to modify the DataFrame rather than creating a new one. - kind (str, default None): - Choice of sorting algorithm. Accepts 'quicksort', 'mergesort', - 'heapsort', 'stable'. Ignored except when determining whether to - sort stably. 'mergesort' or 'stable' will result in stable reorder. na_position ({'first', 'last'}, default 'last'): Puts NaNs at the beginning if `first`; `last` puts NaNs at the end. Not implemented for MultiIndex. @@ -4470,22 +4422,6 @@ def map(self, func, na_action: Optional[str] = None) -> DataFrame: [7 rows x 2 columns] - With experimental Python Transpiler enabled, you can use some lambda functions without - deploying them as remote functions. - - >>> bpd.options.experiments.enable_python_transpiler = True - >>> df_minutes.map(lambda hours: hours / 60) - system_minutes user_minutes - 0 0.0 0.0 - 1 0.5 0.25 - 2 1.0 1.25 - 3 1.5 - 4 1.5 0.1 - 5 2.0 - 6 - - [7 rows x 2 columns] - Args: func (function): Python function wrapped by ``remote_function`` decorator, @@ -4835,8 +4771,7 @@ def resample( >>> df = bpd.DataFrame(data).set_index("timestamp_col") >>> df.resample(rule="7s").min() - int64_col int64_too - timestamp_col + int64_col int64_too 2021-01-01 12:59:55 0 10 2021-01-01 13:00:02 2 12 2021-01-01 13:00:09 9 19 @@ -4849,8 +4784,7 @@ def resample( >>> df = bpd.DataFrame(data) >>> df.resample(rule="7s", on = "timestamp_col", origin="start").min() - int64_col int64_too - timestamp_col + int64_col int64_too 2021-01-01 13:00:00 0 10 2021-01-01 13:00:07 7 17 2021-01-01 13:00:14 14 24 @@ -5069,15 +5003,6 @@ def apply(self, func, *, axis=0, args=(), **kwargs): 1 3.8 dtype: Float64 - With experimental Python Transpiler enabled, you can use some lambda functions without - deploying them as remote functions: - - >>> bpd.options.experiments.enable_python_transpiler = True - >>> df.apply(lambda row: 1 + row.col1 + row.col2/row.col3, axis=1) - 0 2.6 - 1 3.8 - dtype: Float64 - Args: func (function): Function to apply to each column or row. To apply to each row @@ -6660,7 +6585,7 @@ def columns(self): [3 rows x 3 columns] >>> df.columns - Index(['Name', 'Age', 'Location'], dtype='str') + Index(['Name', 'Age', 'Location'], dtype='object') You can also set new labels for columns. @@ -6673,7 +6598,7 @@ def columns(self): [3 rows x 3 columns] >>> df.columns - Index(['NewName', 'NewAge', 'NewLocation'], dtype='str') + Index(['NewName', 'NewAge', 'NewLocation'], dtype='object') """ raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) @@ -7360,7 +7285,7 @@ def plot(self): Make plots of Dataframes. Returns: - bigframes.pandas.api.typing.PlotAccessor: + bigframes.operations.plotting.PlotAccessor: An accessor making plots. """ raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) diff --git a/third_party/bigframes_vendored/pandas/core/generic.py b/third_party/bigframes_vendored/pandas/core/generic.py index 0e4ac335c8a..63b9f8199b6 100644 --- a/third_party/bigframes_vendored/pandas/core/generic.py +++ b/third_party/bigframes_vendored/pandas/core/generic.py @@ -1,10 +1,11 @@ # Contains code from https://github.com/pandas-dev/pandas/blob/main/pandas/core/generic.py from __future__ import annotations -from typing import TYPE_CHECKING, Callable, Iterator, Literal, Optional +from typing import Callable, Iterator, Literal, Optional, TYPE_CHECKING import bigframes_vendored.constants as constants from bigframes_vendored.pandas.core import indexing +import bigframes_vendored.pandas.core.common as common if TYPE_CHECKING: from bigframes_vendored.pandas.pandas._typing import T @@ -394,7 +395,10 @@ def get(self, key, default=None): Any: same type as items contained in object """ - raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) + try: + return self[key] + except (KeyError, ValueError, IndexError): + return default def add_prefix(self, prefix: str, axis: int | str | None = None): """Prefix labels with string `prefix`. @@ -629,9 +633,9 @@ def dtypes(self): >>> df = bpd.DataFrame({'float': [1.0], 'int': [1], 'string': ['foo']}) >>> df.dtypes - float Float64 - int Int64 - string string + float Float64 + int Int64 + string string[pyarrow] dtype: object Returns: @@ -1223,7 +1227,16 @@ def pipe( bigframes.pandas.DataFrame or bigframes.pandas.Series: Object of same type as caller """ - raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) + return common.pipe(self, func, *args, **kwargs) + + def __nonzero__(self): + """Returns the truth value of the object.""" + raise ValueError( + f"The truth value of a {type(self).__name__} is ambiguous. " + "Use a.empty, a.bool(), a.item(), a.any() or a.all()." + ) + + __bool__ = __nonzero__ def __getattr__(self, name: str): """ diff --git a/third_party/bigframes_vendored/pandas/core/groupby/__init__.py b/third_party/bigframes_vendored/pandas/core/groupby/__init__.py index 579765fad5f..01852beb9c1 100644 --- a/third_party/bigframes_vendored/pandas/core/groupby/__init__.py +++ b/third_party/bigframes_vendored/pandas/core/groupby/__init__.py @@ -7,7 +7,6 @@ class providing the base-class of operations. (defined in pandas.core.groupby.generic) expose these user-facing objects to provide specific functionality. """ - from __future__ import annotations from typing import Literal diff --git a/third_party/bigframes_vendored/pandas/core/indexes/accessor.py b/third_party/bigframes_vendored/pandas/core/indexes/accessor.py index da5f9e3b88a..a0388317be8 100644 --- a/third_party/bigframes_vendored/pandas/core/indexes/accessor.py +++ b/third_party/bigframes_vendored/pandas/core/indexes/accessor.py @@ -1,5 +1,3 @@ -from typing import Literal - from bigframes import constants @@ -281,7 +279,7 @@ def month(self): **Examples:** >>> s = bpd.Series( - ... pd.date_range("2000-01-01", periods=3, freq="ME") + ... pd.date_range("2000-01-01", periods=3, freq="M") ... ) >>> s 0 2000-01-31 00:00:00 @@ -404,7 +402,7 @@ def year(self): **Examples:** >>> s = bpd.Series( - ... pd.date_range("2000-01-01", periods=3, freq="YE") + ... pd.date_range("2000-01-01", periods=3, freq="Y") ... ) >>> s 0 2000-12-31 00:00:00 @@ -501,34 +499,6 @@ def tz(self): raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) - @property - def tz_localize(self, tz: Literal["UTC"] | None): - """Localize tz-naive Datetime Array/Index to tz-aware Datetime Array/Index. - - This method takes a time zone (tz) naive Datetime Array/Index object and makes - this time zone aware. It does not move the time to another time zone. Only "UTC" - timezone is supported. - - This method can also be used to do the inverse - to create a time zone unaware - object from an aware object. To that end, pass tz=None. - - **Examples:** - - >>> import bigframes.pandas as bpd - >>> s = bpd.Series([pd.Timestamp(year = 2026, month=1, day=1)]) - >>> s - 0 2026-01-01 00:00:00 - dtype: timestamp[us][pyarrow] - >>> s.dt.tz_localize('UTC') - 0 2026-01-01 00:00:00+00:00 - dtype: timestamp[us, tz=UTC][pyarrow] - - Returns: - A BigFrames series with the updated timezone. - """ - - raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) - @property def unit(self) -> str: """Returns the unit of time precision. diff --git a/third_party/bigframes_vendored/pandas/core/indexes/base.py b/third_party/bigframes_vendored/pandas/core/indexes/base.py index 632026a3311..d21056a8cf3 100644 --- a/third_party/bigframes_vendored/pandas/core/indexes/base.py +++ b/third_party/bigframes_vendored/pandas/core/indexes/base.py @@ -1,8 +1,8 @@ # Contains code from https://github.com/pandas-dev/pandas/blob/main/pandas/core/indexes/base.py from __future__ import annotations -import typing from collections.abc import Hashable +import typing import bigframes from bigframes import constants @@ -828,11 +828,7 @@ def nunique(self) -> int: raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) def sort_values( - self, - *, - ascending: bool = True, - kind: str | None = None, - na_position: str = "last", + self, *, ascending: bool = True, na_position: str = "last" ) -> Index: """ Return a sorted copy of the index. @@ -855,10 +851,6 @@ def sort_values( Args: ascending (bool, default True): Should the index values be sorted in an ascending order. - kind (str, default None): - Choice of sorting algorithm. Accepts 'quicksort', 'mergesort', - 'heapsort', 'stable'. Ignored except when determining whether to - sort stably. 'mergesort' or 'stable' will result in stable reorder. na_position ({'first' or 'last'}, default 'last'): Argument 'first' puts NaNs at the beginning, 'last' puts NaNs at the end. diff --git a/third_party/bigframes_vendored/pandas/core/reshape/concat.py b/third_party/bigframes_vendored/pandas/core/reshape/concat.py index 0a6c4153bd7..685a73abc20 100644 --- a/third_party/bigframes_vendored/pandas/core/reshape/concat.py +++ b/third_party/bigframes_vendored/pandas/core/reshape/concat.py @@ -2,7 +2,6 @@ """ Concat routines """ - from __future__ import annotations from bigframes import constants diff --git a/third_party/bigframes_vendored/pandas/core/reshape/encoding.py b/third_party/bigframes_vendored/pandas/core/reshape/encoding.py index 8d3b26a2a2b..31b2ba4a59b 100644 --- a/third_party/bigframes_vendored/pandas/core/reshape/encoding.py +++ b/third_party/bigframes_vendored/pandas/core/reshape/encoding.py @@ -2,7 +2,6 @@ """ Encoding routines """ - from __future__ import annotations from bigframes import constants diff --git a/third_party/bigframes_vendored/pandas/core/reshape/merge.py b/third_party/bigframes_vendored/pandas/core/reshape/merge.py index 448b21819c0..49ff409c9ae 100644 --- a/third_party/bigframes_vendored/pandas/core/reshape/merge.py +++ b/third_party/bigframes_vendored/pandas/core/reshape/merge.py @@ -2,7 +2,6 @@ """ SQL-style merge routines """ - from __future__ import annotations diff --git a/third_party/bigframes_vendored/pandas/core/reshape/tile.py b/third_party/bigframes_vendored/pandas/core/reshape/tile.py index 546a65b73c2..0f424333847 100644 --- a/third_party/bigframes_vendored/pandas/core/reshape/tile.py +++ b/third_party/bigframes_vendored/pandas/core/reshape/tile.py @@ -2,7 +2,6 @@ """ Quantilization functions and related routines """ - from __future__ import annotations import typing diff --git a/third_party/bigframes_vendored/pandas/core/series.py b/third_party/bigframes_vendored/pandas/core/series.py index 183f36ef5a4..2c0f493d81e 100644 --- a/third_party/bigframes_vendored/pandas/core/series.py +++ b/third_party/bigframes_vendored/pandas/core/series.py @@ -1,27 +1,26 @@ """ Data structure for 1-dimensional cross-sectional and time series data """ - from __future__ import annotations import datetime from typing import ( - IO, - TYPE_CHECKING, Hashable, + IO, List, Literal, Mapping, Optional, Sequence, Tuple, + TYPE_CHECKING, Union, ) +from bigframes_vendored.pandas.core.generic import NDFrame import numpy import numpy as np import pandas as pd -from bigframes_vendored.pandas.core.generic import NDFrame from pandas._typing import Axis, FilePath, NaPosition, WriteBuffer from pandas.api import extensions as pd_ext @@ -33,10 +32,6 @@ class Series(NDFrame): # type: ignore[misc] - """ - One-dimensional ndarray with axis labels (including time series). - """ - @property def dt(self): """ @@ -549,49 +544,6 @@ def to_markdown( """ raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) - def to_csv( - self, - path_or_buf=None, - sep=",", - *, - header: bool = True, - index: bool = True, - allow_large_results: Optional[bool] = None, - ) -> Optional[str]: - """ - Write object to a comma-separated values (csv) file. - - **Examples:** - - >>> import bigframes.pandas as bpd - - >>> s = bpd.Series([1,2,3], name='my_series') - >>> s.to_csv() - \',my_series\\n0,1\\n1,2\\n2,3\\n\' - - Args: - path_or_buf (str, path object, file-like object, or None, default None): - String, path object (implementing os.PathLike[str]), or file-like object - implementing a write() function. If None, the result is returned as a string. - If a non-binary file object is passed, it should be opened with newline='', - disabling universal newlines. If a binary file object is passed, - mode might need to contain a 'b'. - Must contain a wildcard character '*' if this is a GCS path. - sep (str, default ','): - String of length 1. Field delimiter for the output file. - header (bool, default True): - Write out the column names. - index (bool, default True): - Write row names (index). - allow_large_results (bool, default None): - If not None, overrides the global setting to allow or disallow large - query results over the default size limit of 10 GB. - - Returns: - If path_or_buf is None, returns the resulting csv format as a string. Otherwise returns None. - """ - raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) - def to_dict( self, into: type[dict] = dict, @@ -1502,7 +1454,7 @@ def sort_values( axis: Axis = 0, inplace: bool = False, ascending: bool | int | Sequence[bool] | Sequence[int] = True, - kind: str | None = None, + kind: str = "quicksort", na_position: str = "last", ): """ @@ -1579,7 +1531,7 @@ def sort_values( Whether to modify the Series rather than creating a new one. ascending (bool or list of bools, default True): If True, sort values in ascending order, otherwise descending. - kind (str, default to None): + kind (str, default to 'quicksort'): Choice of sorting algorithm. Accepts quicksort', 'mergesort', 'heapsort', 'stable'. Ignored except when determining whether to sort stably. 'mergesort' or 'stable' will result in stable reorder @@ -1599,7 +1551,6 @@ def sort_index( axis: Axis = 0, inplace: bool = False, ascending: bool | Sequence[bool] = True, - kind: str | None = None, na_position: NaPosition = "last", ): """ @@ -1647,10 +1598,6 @@ def sort_index( ascending (bool or list-like of bools, default True): Sort ascending vs. descending. When the index is a MultiIndex the sort direction can be controlled for each level individually. - kind (str, default None): - Choice of sorting algorithm. Accepts 'quicksort', 'mergesort', - 'heapsort', 'stable'. Ignored except when determining whether to - sort stably. 'mergesort' or 'stable' will result in stable reorder. na_position ({'first', 'last'}, default 'last'): If 'first' puts NaNs at the beginning, 'last' puts NaNs at the end. Not implemented for MultiIndex. @@ -2582,8 +2529,7 @@ def resample( ... } >>> s = bpd.DataFrame(data).set_index("timestamp_col") >>> s.resample(rule="7s", origin="epoch").min() - int64_col - timestamp_col + int64_col 2021-01-01 12:59:56 0 2021-01-01 13:00:03 3 2021-01-01 13:00:10 10 @@ -5450,7 +5396,7 @@ def plot(self): Returns: - bigframes.pandas.api.typing.PlotAccessor: + bigframes.operations.plotting.PlotAccessor: An accessor making plots. """ raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) @@ -5631,17 +5577,6 @@ def map( 3 rAbbIt dtype: string - With experimental Python Transpiler enabled, you can use some lambda functions without - deploying them as remote functions: - - >>> bpd.options.experiments.enable_python_transpiler = True - >>> s.map(lambda val: val + "fish") - 0 catfish - 1 dogfish - 2 - 3 rabbitfish - dtype: string - Args: arg (function, Mapping, Series): remote function, collections.abc.Mapping subclass or Series @@ -5686,8 +5621,8 @@ def iloc(self): With a scalar integer. - >>> type(df.iloc[0]) # doctest: +ELLIPSIS - + >>> type(df.iloc[0]) + >>> df.iloc[0] a 1 diff --git a/third_party/bigframes_vendored/pandas/core/tools/datetimes.py b/third_party/bigframes_vendored/pandas/core/tools/datetimes.py index c5f9f8330f6..655f801b3df 100644 --- a/third_party/bigframes_vendored/pandas/core/tools/datetimes.py +++ b/third_party/bigframes_vendored/pandas/core/tools/datetimes.py @@ -1,7 +1,5 @@ # Contains code from https://github.com/pandas-dev/pandas/blob/main/pandas/core/tools/datetimes.py -from __future__ import annotations - from datetime import date, datetime from typing import List, Mapping, Tuple, Union diff --git a/third_party/bigframes_vendored/pandas/core/tools/timedeltas.py b/third_party/bigframes_vendored/pandas/core/tools/timedeltas.py index 92cac856a59..4e418af4061 100644 --- a/third_party/bigframes_vendored/pandas/core/tools/timedeltas.py +++ b/third_party/bigframes_vendored/pandas/core/tools/timedeltas.py @@ -2,8 +2,8 @@ import typing -import pandas as pd from bigframes_vendored import constants +import pandas as pd from bigframes import series diff --git a/third_party/bigframes_vendored/pandas/io/common.py b/third_party/bigframes_vendored/pandas/io/common.py index cab3c36cc9f..e186f02b5bd 100644 --- a/third_party/bigframes_vendored/pandas/io/common.py +++ b/third_party/bigframes_vendored/pandas/io/common.py @@ -1,6 +1,5 @@ # Contains code from https://github.com/pandas-dev/pandas/blob/main/pandas/io/common.py """Common IO api utilities""" - from __future__ import annotations from collections import defaultdict diff --git a/third_party/bigframes_vendored/pandas/io/gbq.py b/third_party/bigframes_vendored/pandas/io/gbq.py index 242d2c50c8d..3190c92b921 100644 --- a/third_party/bigframes_vendored/pandas/io/gbq.py +++ b/third_party/bigframes_vendored/pandas/io/gbq.py @@ -1,12 +1,12 @@ # Contains code from https://github.com/pandas-dev/pandas/blob/main/pandas/io/gbq.py -"""Google BigQuery support""" +""" Google BigQuery support """ from __future__ import annotations from typing import Any, Dict, Iterable, Literal, Optional, Tuple, Union -import bigframes.enums from bigframes import constants +import bigframes.enums FilterOps = Literal["in", "not in", "<", "<=", "==", "!=", ">=", ">", "LIKE"] FilterType = Tuple[str, FilterOps, Any] @@ -86,11 +86,9 @@ def read_gbq( ... WHERE year = 2016 ... GROUP BY pitcherFirstName, pitcherLastName ... ''', index_col="rowindex") - >>> print("START_OF_OUTPUT"); df.head(2) # doctest: +ELLIPSIS,+NORMALIZE_WHITESPACE - START_OF_OUTPUT - ... + >>> df.head(2) pitcherFirstName pitcherLastName averagePitchSpeed - ... + rowindex 1 Albertin Chapman 96.514113 2 Zachary Britton 94.591039 diff --git a/third_party/bigframes_vendored/pandas/io/parquet.py b/third_party/bigframes_vendored/pandas/io/parquet.py index cfb653481bf..c02c5e52c5f 100644 --- a/third_party/bigframes_vendored/pandas/io/parquet.py +++ b/third_party/bigframes_vendored/pandas/io/parquet.py @@ -1,6 +1,5 @@ # Contains code from https://github.com/pandas-dev/pandas/blob/main/pandas/io/parquet.py -"""parquet compat""" - +""" parquet compat """ from __future__ import annotations from bigframes import constants diff --git a/third_party/bigframes_vendored/pandas/io/parsers/readers.py b/third_party/bigframes_vendored/pandas/io/parsers/readers.py index 537974b5f32..5a505c2859d 100644 --- a/third_party/bigframes_vendored/pandas/io/parsers/readers.py +++ b/third_party/bigframes_vendored/pandas/io/parsers/readers.py @@ -4,13 +4,12 @@ GH#48849 provides a convenient way of deprecating keyword arguments """ - from __future__ import annotations from typing import ( - IO, Any, Dict, + IO, Literal, MutableSequence, Optional, @@ -21,8 +20,8 @@ import numpy as np -import bigframes.enums from bigframes import constants +import bigframes.enums class ReaderIOMixin: diff --git a/third_party/bigframes_vendored/pandas/io/pickle.py b/third_party/bigframes_vendored/pandas/io/pickle.py index 10ceab3c2fa..03f1afe35e3 100644 --- a/third_party/bigframes_vendored/pandas/io/pickle.py +++ b/third_party/bigframes_vendored/pandas/io/pickle.py @@ -1,6 +1,5 @@ # Contains code from https://github.com/pandas-dev/pandas/blob/main/pandas/io/pickle.py -"""pickle compat""" - +""" pickle compat """ from __future__ import annotations from pandas._typing import ( diff --git a/third_party/bigframes_vendored/pandas/pandas/_typing.py b/third_party/bigframes_vendored/pandas/pandas/_typing.py index 3640ba25163..e665339fc83 100644 --- a/third_party/bigframes_vendored/pandas/pandas/_typing.py +++ b/third_party/bigframes_vendored/pandas/pandas/_typing.py @@ -1,11 +1,10 @@ # Copied from https://github.com/pandas-dev/pandas/blob/main/pandas/_typing.py from __future__ import annotations -import sys from datetime import datetime, timedelta, tzinfo from os import PathLike +import sys from typing import ( - TYPE_CHECKING, Any, Callable, Dict, @@ -18,10 +17,9 @@ Protocol, Sequence, Tuple, - TypeVar, - Union, ) from typing import Type as type_t +from typing import TYPE_CHECKING, TypeVar, Union import numpy as np @@ -236,11 +234,13 @@ def flush(self) -> Any: class ReadPickleBuffer(ReadBuffer[bytes], Protocol): - def readline(self) -> bytes: ... + def readline(self) -> bytes: + ... class WriteExcelBuffer(WriteBuffer[bytes], Protocol): - def truncate(self, size: int | None = ...) -> int: ... + def truncate(self, size: int | None = ...) -> int: + ... class ReadCsvBuffer(ReadBuffer[AnyStr_co], Protocol): diff --git a/third_party/bigframes_vendored/pandas/util/_validators.py b/third_party/bigframes_vendored/pandas/util/_validators.py index fe8c9b5d9c6..1f36e0d528c 100644 --- a/third_party/bigframes_vendored/pandas/util/_validators.py +++ b/third_party/bigframes_vendored/pandas/util/_validators.py @@ -3,7 +3,6 @@ Module that contains many useful utilities for validating data or function arguments """ - from __future__ import annotations from typing import TypeVar diff --git a/third_party/bigframes_vendored/sklearn/compose/_column_transformer.py b/third_party/bigframes_vendored/sklearn/compose/_column_transformer.py index e0c03536e09..e4e71c1ff9f 100644 --- a/third_party/bigframes_vendored/sklearn/compose/_column_transformer.py +++ b/third_party/bigframes_vendored/sklearn/compose/_column_transformer.py @@ -3,6 +3,7 @@ # Andreas Mueller # License: BSD + from abc import ABCMeta from bigframes_vendored.sklearn.base import BaseEstimator diff --git a/third_party/bigframes_vendored/sklearn/decomposition/_mf.py b/third_party/bigframes_vendored/sklearn/decomposition/_mf.py index 0ce79995d0c..7dad1962377 100644 --- a/third_party/bigframes_vendored/sklearn/decomposition/_mf.py +++ b/third_party/bigframes_vendored/sklearn/decomposition/_mf.py @@ -1,4 +1,5 @@ -"""Matrix Factorization.""" +""" Matrix Factorization. +""" # Author: Alexandre Gramfort # Olivier Grisel @@ -29,7 +30,7 @@ class MatrixFactorization(BaseEstimator, metaclass=ABCMeta): ... "value": [1, 1, 2, 1, 3, 1.2, 4, 1, 5, 0.8, 6, 1, 2, 3], ... }) >>> model = MatrixFactorization(feedback_type='explicit', num_factors=6, user_col='row', item_col='column', rating_col='value', l2_reg=2.06) - >>> W = model.fit(X) # doctest: +SKIP + >>> W = model.fit(X) Args: feedback_type ('explicit' | 'implicit'): diff --git a/third_party/bigframes_vendored/sklearn/decomposition/_pca.py b/third_party/bigframes_vendored/sklearn/decomposition/_pca.py index 138b7772b24..f90e193064f 100644 --- a/third_party/bigframes_vendored/sklearn/decomposition/_pca.py +++ b/third_party/bigframes_vendored/sklearn/decomposition/_pca.py @@ -1,4 +1,5 @@ -"""Principal Component Analysis.""" +""" Principal Component Analysis. +""" # Author: Alexandre Gramfort # Olivier Grisel diff --git a/third_party/bigframes_vendored/sklearn/linear_model/_logistic.py b/third_party/bigframes_vendored/sklearn/linear_model/_logistic.py index a309166c90c..1efece251f9 100644 --- a/third_party/bigframes_vendored/sklearn/linear_model/_logistic.py +++ b/third_party/bigframes_vendored/sklearn/linear_model/_logistic.py @@ -11,6 +11,7 @@ # Arthur Mensch >> y_pred = bpd.DataFrame([0, 2, 1, 0, 0, 1]) >>> recall_score = bigframes.ml.metrics.recall_score(y_true, y_pred, average=None) >>> recall_score - 0 1.0 - 1 0.0 - 2 0.0 - dtype: float64 + 0 1 + 1 0 + 2 0 + dtype: int64 Args: diff --git a/third_party/bigframes_vendored/sklearn/metrics/pairwise.py b/third_party/bigframes_vendored/sklearn/metrics/pairwise.py index 37cbf23b29c..7584230be6b 100644 --- a/third_party/bigframes_vendored/sklearn/metrics/pairwise.py +++ b/third_party/bigframes_vendored/sklearn/metrics/pairwise.py @@ -7,8 +7,8 @@ # Joel Nothman # License: BSD 3 clause -import bigframes.pandas as bpd from bigframes import constants +import bigframes.pandas as bpd def paired_cosine_distances(X, Y) -> bpd.DataFrame: diff --git a/third_party/bigframes_vendored/sklearn/model_selection/_split.py b/third_party/bigframes_vendored/sklearn/model_selection/_split.py index 2398cbe77ca..326589be7da 100644 --- a/third_party/bigframes_vendored/sklearn/model_selection/_split.py +++ b/third_party/bigframes_vendored/sklearn/model_selection/_split.py @@ -11,6 +11,7 @@ # Rodion Martynov # License: BSD 3 clause + from abc import ABCMeta from bigframes import constants diff --git a/third_party/bigframes_vendored/sklearn/preprocessing/_encoder.py b/third_party/bigframes_vendored/sklearn/preprocessing/_encoder.py index 1301ef329ab..64a5786f17d 100644 --- a/third_party/bigframes_vendored/sklearn/preprocessing/_encoder.py +++ b/third_party/bigframes_vendored/sklearn/preprocessing/_encoder.py @@ -84,6 +84,5 @@ def transform(self, X): Returns: bigframes.dataframe.DataFrame: The result is categorized as index: number, value: number, - where index is the position of the dict seeing the category, and value is 0 or 1. - """ + where index is the position of the dict seeing the category, and value is 0 or 1.""" raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE) diff --git a/third_party/bigframes_vendored/sqlglot/__init__.py b/third_party/bigframes_vendored/sqlglot/__init__.py index 7369b9b444b..f3679caf8d6 100644 --- a/third_party/bigframes_vendored/sqlglot/__init__.py +++ b/third_party/bigframes_vendored/sqlglot/__init__.py @@ -24,9 +24,6 @@ from bigframes_vendored.sqlglot.errors import ( # noqa: F401 UnsupportedError as UnsupportedError, ) -from bigframes_vendored.sqlglot.expressions import ( # noqa: F401 - Expression as Expression, -) from bigframes_vendored.sqlglot.expressions import alias_ as alias # noqa: F401 from bigframes_vendored.sqlglot.expressions import and_ as and_ # noqa: F401 from bigframes_vendored.sqlglot.expressions import case as case # noqa: F401 @@ -35,6 +32,9 @@ from bigframes_vendored.sqlglot.expressions import condition as condition # noqa: F401 from bigframes_vendored.sqlglot.expressions import delete as delete # noqa: F401 from bigframes_vendored.sqlglot.expressions import except_ as except_ # noqa: F401 +from bigframes_vendored.sqlglot.expressions import ( # noqa: F401 + Expression as Expression, +) from bigframes_vendored.sqlglot.expressions import ( # noqa: F401 find_tables as find_tables, ) @@ -114,11 +114,13 @@ def parse( @t.overload -def parse_one(sql: str, *, into: t.Type[E], **opts) -> E: ... +def parse_one(sql: str, *, into: t.Type[E], **opts) -> E: + ... @t.overload -def parse_one(sql: str, **opts) -> Expression: ... +def parse_one(sql: str, **opts) -> Expression: + ... def parse_one( diff --git a/third_party/bigframes_vendored/sqlglot/dialects/bigquery.py b/third_party/bigframes_vendored/sqlglot/dialects/bigquery.py index 7da30231ff4..4a7e748de07 100644 --- a/third_party/bigframes_vendored/sqlglot/dialects/bigquery.py +++ b/third_party/bigframes_vendored/sqlglot/dialects/bigquery.py @@ -15,14 +15,13 @@ transforms, ) from bigframes_vendored.sqlglot.dialects.dialect import ( - Dialect, - NormalizationStrategy, arg_max_or_min_no_count, binary_from_function, build_date_delta_with_interval, build_formatted_time, date_add_interval_sql, datestrtodate_sql, + Dialect, filter_array_using_unnest, groupconcat_sql, if_sql, @@ -30,6 +29,7 @@ max_or_greatest, min_or_least, no_ilike_sql, + NormalizationStrategy, regexp_replace_sql, rename_func, sha2_digest_sql, @@ -969,10 +969,12 @@ def _parse_column(self) -> t.Optional[exp.Expression]: return column @t.overload - def _parse_json_object(self, agg: Lit[False]) -> exp.JSONObject: ... + def _parse_json_object(self, agg: Lit[False]) -> exp.JSONObject: + ... @t.overload - def _parse_json_object(self, agg: Lit[True]) -> exp.JSONObjectAgg: ... + def _parse_json_object(self, agg: Lit[True]) -> exp.JSONObjectAgg: + ... def _parse_json_object(self, agg=False): json_object = super()._parse_json_object() @@ -1287,8 +1289,7 @@ class Generator(generator.Generator): e.this, e.args.get("form"), ), - exp.PartitionedByProperty: lambda self, - e: f"PARTITION BY {self.sql(e, 'this')}", + exp.PartitionedByProperty: lambda self, e: f"PARTITION BY {self.sql(e, 'this')}", exp.RegexpExtract: lambda self, e: self.func( "REGEXP_EXTRACT", e.this, diff --git a/third_party/bigframes_vendored/sqlglot/dialects/dialect.py b/third_party/bigframes_vendored/sqlglot/dialects/dialect.py index 8e26b777abd..8dbb5c3f1c2 100644 --- a/third_party/bigframes_vendored/sqlglot/dialects/dialect.py +++ b/third_party/bigframes_vendored/sqlglot/dialects/dialect.py @@ -2,12 +2,12 @@ from __future__ import annotations +from enum import auto, Enum +from functools import reduce import importlib import logging import sys import typing as t -from enum import Enum, auto -from functools import reduce from bigframes_vendored.sqlglot import exp from bigframes_vendored.sqlglot.dialects import DIALECT_MODULE_NAMES @@ -24,7 +24,7 @@ from bigframes_vendored.sqlglot.jsonpath import JSONPathTokenizer from bigframes_vendored.sqlglot.jsonpath import parse as parse_json_path from bigframes_vendored.sqlglot.parser import Parser -from bigframes_vendored.sqlglot.time import TIMEZONES, format_time, subsecond_precision +from bigframes_vendored.sqlglot.time import format_time, subsecond_precision, TIMEZONES from bigframes_vendored.sqlglot.tokens import Token, Tokenizer, TokenType from bigframes_vendored.sqlglot.trie import new_trie from bigframes_vendored.sqlglot.typing import EXPRESSION_METADATA @@ -166,7 +166,7 @@ def _try_load(cls, key: str | Dialects) -> None: # files. Custom user dialects need to be imported at the top-level package, in # order for them to be registered as soon as possible. if key in DIALECT_MODULE_NAMES: - importlib.import_module(f"bigframes_vendored.sqlglot.dialects.{key}") + importlib.import_module(f"sqlglot.dialects.{key}") @classmethod def __getitem__(cls, key: str) -> t.Type[Dialect]: @@ -1236,14 +1236,7 @@ def inline_array_sql(self: Generator, expression: exp.Expression) -> str: def inline_array_unless_query(self: Generator, expression: exp.Expression) -> str: elem = seq_get(expression.expressions, 0) - if ( - len(expression.expressions) == 1 - and isinstance(elem, exp.Expression) - and ( - isinstance(elem, exp.Query) - or (isinstance(elem, exp.Subquery) and isinstance(elem.this, exp.Query)) - ) - ): + if isinstance(elem, exp.Expression) and elem.find(exp.Query): return self.func("ARRAY", elem) return inline_array_sql(self, expression) diff --git a/third_party/bigframes_vendored/sqlglot/diff.py b/third_party/bigframes_vendored/sqlglot/diff.py index e0e4eb4b0be..1d33fe6b0dc 100644 --- a/third_party/bigframes_vendored/sqlglot/diff.py +++ b/third_party/bigframes_vendored/sqlglot/diff.py @@ -8,11 +8,11 @@ from __future__ import annotations -import typing as t from collections import defaultdict from dataclasses import dataclass from heapq import heappop, heappush from itertools import chain +import typing as t from bigframes_vendored.sqlglot import Dialect from bigframes_vendored.sqlglot import expressions as exp diff --git a/third_party/bigframes_vendored/sqlglot/errors.py b/third_party/bigframes_vendored/sqlglot/errors.py index fe8e31d1960..b40146f91b2 100644 --- a/third_party/bigframes_vendored/sqlglot/errors.py +++ b/third_party/bigframes_vendored/sqlglot/errors.py @@ -2,8 +2,8 @@ from __future__ import annotations -import typing as t from enum import auto +import typing as t from bigframes_vendored.sqlglot.helper import AutoName diff --git a/third_party/bigframes_vendored/sqlglot/expressions.py b/third_party/bigframes_vendored/sqlglot/expressions.py index e8e4cc8e10d..996df3a6424 100644 --- a/third_party/bigframes_vendored/sqlglot/expressions.py +++ b/third_party/bigframes_vendored/sqlglot/expressions.py @@ -14,18 +14,18 @@ from __future__ import annotations +from collections import deque +from copy import deepcopy import datetime +from decimal import Decimal +from enum import auto +from functools import reduce import math import numbers import re import sys import textwrap import typing as t -from collections import deque -from copy import deepcopy -from decimal import Decimal -from enum import auto -from functools import reduce from bigframes_vendored.sqlglot.errors import ErrorLevel, ParseError from bigframes_vendored.sqlglot.helper import ( @@ -648,9 +648,9 @@ def flatten(self, unnest=True): prune=lambda n: n.parent and type(n) is not self.__class__ ): if type(node) is not self.__class__: - yield ( - node.unnest() if unnest and not isinstance(node, Subquery) else node - ) + yield node.unnest() if unnest and not isinstance( + node, Subquery + ) else node def __str__(self) -> str: return self.sql() @@ -715,10 +715,12 @@ def transform( return root.assert_is(Expression) @t.overload - def replace(self, expression: E) -> E: ... + def replace(self, expression: E) -> E: + ... @t.overload - def replace(self, expression: None) -> None: ... + def replace(self, expression: None) -> None: + ... def replace(self, expression): """ @@ -5906,7 +5908,7 @@ class FarmFingerprint(Func): class Flatten(Func): - arg_types = {"this": True, "depth": False} + pass class Float64(Func): @@ -8539,7 +8541,8 @@ def maybe_parse( prefix: t.Optional[str] = None, copy: bool = False, **opts, -) -> E: ... +) -> E: + ... @t.overload @@ -8551,7 +8554,8 @@ def maybe_parse( prefix: t.Optional[str] = None, copy: bool = False, **opts, -) -> E: ... +) -> E: + ... def maybe_parse( @@ -8603,11 +8607,13 @@ def maybe_parse( @t.overload -def maybe_copy(instance: None, copy: bool = True) -> None: ... +def maybe_copy(instance: None, copy: bool = True) -> None: + ... @t.overload -def maybe_copy(instance: E, copy: bool = True) -> E: ... +def maybe_copy(instance: E, copy: bool = True) -> E: + ... def maybe_copy(instance, copy=True): @@ -8855,11 +8861,13 @@ def _combine( @t.overload -def _wrap(expression: None, kind: t.Type[Expression]) -> None: ... +def _wrap(expression: None, kind: t.Type[Expression]) -> None: + ... @t.overload -def _wrap(expression: E, kind: t.Type[Expression]) -> E | Paren: ... +def _wrap(expression: E, kind: t.Type[Expression]) -> E | Paren: + ... def _wrap(expression: t.Optional[E], kind: t.Type[Expression]) -> t.Optional[E] | Paren: @@ -8941,9 +8949,9 @@ def intersect( Returns: The new Intersect instance. """ - assert len(expressions) >= 2, ( - "At least two expressions are required by `intersect`." - ) + assert ( + len(expressions) >= 2 + ), "At least two expressions are required by `intersect`." return _apply_set_operation( *expressions, set_operation=Intersect, @@ -9425,13 +9433,15 @@ def paren(expression: ExpOrStr, copy: bool = True) -> Paren: @t.overload def to_identifier( name: None, quoted: t.Optional[bool] = None, copy: bool = True -) -> None: ... +) -> None: + ... @t.overload def to_identifier( name: str | Identifier, quoted: t.Optional[bool] = None, copy: bool = True -) -> Identifier: ... +) -> Identifier: + ... def to_identifier(name, quoted=None, copy=True): diff --git a/third_party/bigframes_vendored/sqlglot/generator.py b/third_party/bigframes_vendored/sqlglot/generator.py index 80546fadc44..1084d5de899 100644 --- a/third_party/bigframes_vendored/sqlglot/generator.py +++ b/third_party/bigframes_vendored/sqlglot/generator.py @@ -2,17 +2,17 @@ from __future__ import annotations +from collections import defaultdict +from functools import reduce, wraps import logging import re import typing as t -from collections import defaultdict -from functools import reduce, wraps from bigframes_vendored.sqlglot import exp from bigframes_vendored.sqlglot.errors import ( + concat_messages, ErrorLevel, UnsupportedError, - concat_messages, ) from bigframes_vendored.sqlglot.helper import ( apply_index_offset, @@ -129,23 +129,18 @@ class Generator(metaclass=_Generator): TRANSFORMS: t.Dict[t.Type[exp.Expression], t.Callable[..., str]] = { **JSON_PATH_PART_TRANSFORMS, exp.Adjacent: lambda self, e: self.binary(e, "-|-"), - exp.AllowedValuesProperty: lambda self, - e: f"ALLOWED_VALUES {self.expressions(e, flat=True)}", + exp.AllowedValuesProperty: lambda self, e: f"ALLOWED_VALUES {self.expressions(e, flat=True)}", exp.AnalyzeColumns: lambda self, e: self.sql(e, "this"), exp.AnalyzeWith: lambda self, e: self.expressions(e, prefix="WITH ", sep=" "), exp.ArrayContainsAll: lambda self, e: self.binary(e, "@>"), exp.ArrayOverlaps: lambda self, e: self.binary(e, "&&"), exp.AutoRefreshProperty: lambda self, e: f"AUTO REFRESH {self.sql(e, 'this')}", exp.BackupProperty: lambda self, e: f"BACKUP {self.sql(e, 'this')}", - exp.CaseSpecificColumnConstraint: lambda _, - e: f"{'NOT ' if e.args.get('not_') else ''}CASESPECIFIC", + exp.CaseSpecificColumnConstraint: lambda _, e: f"{'NOT ' if e.args.get('not_') else ''}CASESPECIFIC", exp.Ceil: lambda self, e: self.ceil_floor(e), - exp.CharacterSetColumnConstraint: lambda self, - e: f"CHARACTER SET {self.sql(e, 'this')}", - exp.CharacterSetProperty: lambda self, - e: f"{'DEFAULT ' if e.args.get('default') else ''}CHARACTER SET={self.sql(e, 'this')}", - exp.ClusteredColumnConstraint: lambda self, - e: f"CLUSTERED ({self.expressions(e, 'this', indent=False)})", + exp.CharacterSetColumnConstraint: lambda self, e: f"CHARACTER SET {self.sql(e, 'this')}", + exp.CharacterSetProperty: lambda self, e: f"{'DEFAULT ' if e.args.get('default') else ''}CHARACTER SET={self.sql(e, 'this')}", + exp.ClusteredColumnConstraint: lambda self, e: f"CLUSTERED ({self.expressions(e, 'this', indent=False)})", exp.CollateColumnConstraint: lambda self, e: f"COLLATE {self.sql(e, 'this')}", exp.CommentColumnConstraint: lambda self, e: f"COMMENT {self.sql(e, 'this')}", exp.ConnectByRoot: lambda self, e: f"CONNECT_BY_ROOT {self.sql(e, 'this')}", @@ -153,8 +148,7 @@ class Generator(metaclass=_Generator): "CONVERT", e.this, e.args["dest"], e.args.get("source") ), exp.CopyGrantsProperty: lambda *_: "COPY GRANTS", - exp.CredentialsProperty: lambda self, - e: f"CREDENTIALS=({self.expressions(e, 'expressions', sep=' ')})", + exp.CredentialsProperty: lambda self, e: f"CREDENTIALS=({self.expressions(e, 'expressions', sep=' ')})", exp.CurrentCatalog: lambda *_: "CURRENT_CATALOG", exp.SessionUser: lambda *_: "SESSION_USER", exp.DateFormatColumnConstraint: lambda self, e: f"FORMAT {self.sql(e, 'this')}", @@ -162,12 +156,9 @@ class Generator(metaclass=_Generator): exp.DynamicProperty: lambda *_: "DYNAMIC", exp.EmptyProperty: lambda *_: "EMPTY", exp.EncodeColumnConstraint: lambda self, e: f"ENCODE {self.sql(e, 'this')}", - exp.EnviromentProperty: lambda self, - e: f"ENVIRONMENT ({self.expressions(e, flat=True)})", - exp.EphemeralColumnConstraint: lambda self, - e: f"EPHEMERAL{(' ' + self.sql(e, 'this')) if e.this else ''}", - exp.ExcludeColumnConstraint: lambda self, - e: f"EXCLUDE {self.sql(e, 'this').lstrip()}", + exp.EnviromentProperty: lambda self, e: f"ENVIRONMENT ({self.expressions(e, flat=True)})", + exp.EphemeralColumnConstraint: lambda self, e: f"EPHEMERAL{(' ' + self.sql(e, 'this')) if e.this else ''}", + exp.ExcludeColumnConstraint: lambda self, e: f"EXCLUDE {self.sql(e, 'this').lstrip()}", exp.ExecuteAsProperty: lambda self, e: self.naked_property(e), exp.Except: lambda self, e: self.set_operations(e), exp.ExternalProperty: lambda *_: "EXTERNAL", @@ -176,14 +167,11 @@ class Generator(metaclass=_Generator): exp.GlobalProperty: lambda *_: "GLOBAL", exp.HeapProperty: lambda *_: "HEAP", exp.IcebergProperty: lambda *_: "ICEBERG", - exp.InheritsProperty: lambda self, - e: f"INHERITS ({self.expressions(e, flat=True)})", - exp.InlineLengthColumnConstraint: lambda self, - e: f"INLINE LENGTH {self.sql(e, 'this')}", + exp.InheritsProperty: lambda self, e: f"INHERITS ({self.expressions(e, flat=True)})", + exp.InlineLengthColumnConstraint: lambda self, e: f"INLINE LENGTH {self.sql(e, 'this')}", exp.InputModelProperty: lambda self, e: f"INPUT{self.sql(e, 'this')}", exp.Intersect: lambda self, e: self.set_operations(e), - exp.IntervalSpan: lambda self, - e: f"{self.sql(e, 'this')} TO {self.sql(e, 'expression')}", + exp.IntervalSpan: lambda self, e: f"{self.sql(e, 'this')} TO {self.sql(e, 'expression')}", exp.Int64: lambda self, e: self.sql(exp.cast(e.this, exp.DataType.Type.BIGINT)), exp.JSONBContainsAnyTopKeys: lambda self, e: self.binary(e, "?|"), exp.JSONBContainsAllTopKeys: lambda self, e: self.binary(e, "?&"), @@ -192,15 +180,12 @@ class Generator(metaclass=_Generator): exp.LocationProperty: lambda self, e: self.naked_property(e), exp.LogProperty: lambda _, e: f"{'NO ' if e.args.get('no') else ''}LOG", exp.MaterializedProperty: lambda *_: "MATERIALIZED", - exp.NonClusteredColumnConstraint: lambda self, - e: f"NONCLUSTERED ({self.expressions(e, 'this', indent=False)})", + exp.NonClusteredColumnConstraint: lambda self, e: f"NONCLUSTERED ({self.expressions(e, 'this', indent=False)})", exp.NoPrimaryIndexProperty: lambda *_: "NO PRIMARY INDEX", exp.NotForReplicationColumnConstraint: lambda *_: "NOT FOR REPLICATION", - exp.OnCommitProperty: lambda _, - e: f"ON COMMIT {'DELETE' if e.args.get('delete') else 'PRESERVE'} ROWS", + exp.OnCommitProperty: lambda _, e: f"ON COMMIT {'DELETE' if e.args.get('delete') else 'PRESERVE'} ROWS", exp.OnProperty: lambda self, e: f"ON {self.sql(e, 'this')}", - exp.OnUpdateColumnConstraint: lambda self, - e: f"ON UPDATE {self.sql(e, 'this')}", + exp.OnUpdateColumnConstraint: lambda self, e: f"ON UPDATE {self.sql(e, 'this')}", exp.Operator: lambda self, e: self.binary( e, "" ), # The operator is produced in `binary` @@ -216,12 +201,10 @@ class Generator(metaclass=_Generator): ), exp.PivotAny: lambda self, e: f"ANY{self.sql(e, 'this')}", exp.PositionalColumn: lambda self, e: f"#{self.sql(e, 'this')}", - exp.ProjectionPolicyColumnConstraint: lambda self, - e: f"PROJECTION POLICY {self.sql(e, 'this')}", + exp.ProjectionPolicyColumnConstraint: lambda self, e: f"PROJECTION POLICY {self.sql(e, 'this')}", exp.ZeroFillColumnConstraint: lambda self, e: "ZEROFILL", exp.Put: lambda self, e: self.get_put_sql(e), - exp.RemoteWithConnectionModelProperty: lambda self, - e: f"REMOTE WITH CONNECTION {self.sql(e, 'this')}", + exp.RemoteWithConnectionModelProperty: lambda self, e: f"REMOTE WITH CONNECTION {self.sql(e, 'this')}", exp.ReturnsProperty: lambda self, e: ( "RETURNS NULL ON NULL INPUT" if e.args.get("null") @@ -232,8 +215,7 @@ class Generator(metaclass=_Generator): exp.SecurityProperty: lambda self, e: f"SECURITY {self.sql(e, 'this')}", exp.SetConfigProperty: lambda self, e: self.sql(e, "this"), exp.SetProperty: lambda _, e: f"{'MULTI' if e.args.get('multi') else ''}SET", - exp.SettingsProperty: lambda self, - e: f"SETTINGS{self.seg('')}{(self.expressions(e))}", + exp.SettingsProperty: lambda self, e: f"SETTINGS{self.seg('')}{(self.expressions(e))}", exp.SharingProperty: lambda self, e: f"SHARING={self.sql(e, 'this')}", exp.SqlReadWriteProperty: lambda _, e: e.name, exp.SqlSecurityProperty: lambda self, e: f"SQL SECURITY {self.sql(e, 'this')}", @@ -254,8 +236,7 @@ class Generator(metaclass=_Generator): exp.TransientProperty: lambda *_: "TRANSIENT", exp.Union: lambda self, e: self.set_operations(e), exp.UnloggedProperty: lambda *_: "UNLOGGED", - exp.UsingTemplateProperty: lambda self, - e: f"USING TEMPLATE {self.sql(e, 'this')}", + exp.UsingTemplateProperty: lambda self, e: f"USING TEMPLATE {self.sql(e, 'this')}", exp.UsingData: lambda self, e: f"USING DATA {self.sql(e, 'this')}", exp.UppercaseColumnConstraint: lambda *_: "UPPERCASE", exp.UtcDate: lambda self, e: self.sql( @@ -270,14 +251,10 @@ class Generator(metaclass=_Generator): exp.VarMap: lambda self, e: self.func("MAP", e.args["keys"], e.args["values"]), exp.ViewAttributeProperty: lambda self, e: f"WITH {self.sql(e, 'this')}", exp.VolatileProperty: lambda *_: "VOLATILE", - exp.WithJournalTableProperty: lambda self, - e: f"WITH JOURNAL TABLE={self.sql(e, 'this')}", - exp.WithProcedureOptions: lambda self, - e: f"WITH {self.expressions(e, flat=True)}", - exp.WithSchemaBindingProperty: lambda self, - e: f"WITH SCHEMA {self.sql(e, 'this')}", - exp.WithOperator: lambda self, - e: f"{self.sql(e, 'this')} WITH {self.sql(e, 'op')}", + exp.WithJournalTableProperty: lambda self, e: f"WITH JOURNAL TABLE={self.sql(e, 'this')}", + exp.WithProcedureOptions: lambda self, e: f"WITH {self.expressions(e, flat=True)}", + exp.WithSchemaBindingProperty: lambda self, e: f"WITH SCHEMA {self.sql(e, 'this')}", + exp.WithOperator: lambda self, e: f"{self.sql(e, 'this')} WITH {self.sql(e, 'op')}", exp.ForceProperty: lambda *_: "FORCE", } @@ -2277,8 +2254,7 @@ def table_sql(self, expression: exp.Table, sep: str = " AS ") -> str: else: indexed = "" - # Workaround https://github.com/tobymao/sqlglot/issues/7073 - return f"{only}{table}{changes}{alias}{partition}{version}{file_format}{sample_pre_alias}{indexed}{hints}{pivots}{sample_post_alias}{joins}{laterals}{ordinality}" + return f"{only}{table}{changes}{partition}{version}{file_format}{sample_pre_alias}{alias}{indexed}{hints}{pivots}{sample_post_alias}{joins}{laterals}{ordinality}" def tablefromrows_sql(self, expression: exp.TableFromRows) -> str: table = self.func("TABLE", expression.this) @@ -2750,9 +2726,7 @@ def order_sql(self, expression: exp.Order, flat: bool = False) -> str: this = self.sql(expression, "this") this = f"{this} " if this else this siblings = "SIBLINGS " if expression.args.get("siblings") else "" - return self.op_expressions( - f"{this}ORDER {siblings}BY", expression, flat=this or flat - ) # type: ignore + return self.op_expressions(f"{this}ORDER {siblings}BY", expression, flat=this or flat) # type: ignore def withfill_sql(self, expression: exp.WithFill) -> str: from_sql = self.sql(expression, "from_") diff --git a/third_party/bigframes_vendored/sqlglot/helper.py b/third_party/bigframes_vendored/sqlglot/helper.py index 5cd16e2c3cd..da47f3c7b99 100644 --- a/third_party/bigframes_vendored/sqlglot/helper.py +++ b/third_party/bigframes_vendored/sqlglot/helper.py @@ -2,17 +2,17 @@ from __future__ import annotations +from collections.abc import Collection, Set +from copy import copy import datetime +from difflib import get_close_matches +from enum import Enum import inspect +from itertools import count import logging import re import sys import typing as t -from collections.abc import Collection, Set -from copy import copy -from difflib import get_close_matches -from enum import Enum -from itertools import count if t.TYPE_CHECKING: from bigframes_vendored.sqlglot import exp @@ -70,15 +70,18 @@ def seq_get(seq: t.Sequence[T], index: int) -> t.Optional[T]: @t.overload -def ensure_list(value: t.Collection[T]) -> t.List[T]: ... +def ensure_list(value: t.Collection[T]) -> t.List[T]: + ... @t.overload -def ensure_list(value: None) -> t.List: ... +def ensure_list(value: None) -> t.List: + ... @t.overload -def ensure_list(value: T) -> t.List[T]: ... +def ensure_list(value: T) -> t.List[T]: + ... def ensure_list(value): @@ -100,11 +103,13 @@ def ensure_list(value): @t.overload -def ensure_collection(value: t.Collection[T]) -> t.Collection[T]: ... +def ensure_collection(value: t.Collection[T]) -> t.Collection[T]: + ... @t.overload -def ensure_collection(value: T) -> t.Collection[T]: ... +def ensure_collection(value: T) -> t.Collection[T]: + ... def ensure_collection(value): diff --git a/third_party/bigframes_vendored/sqlglot/jsonpath.py b/third_party/bigframes_vendored/sqlglot/jsonpath.py index cc5ac7edd3d..08f0f0dfd02 100644 --- a/third_party/bigframes_vendored/sqlglot/jsonpath.py +++ b/third_party/bigframes_vendored/sqlglot/jsonpath.py @@ -4,8 +4,8 @@ import typing as t -import bigframes_vendored.sqlglot.expressions as exp from bigframes_vendored.sqlglot.errors import ParseError +import bigframes_vendored.sqlglot.expressions as exp from bigframes_vendored.sqlglot.tokens import Token, Tokenizer, TokenType if t.TYPE_CHECKING: @@ -230,8 +230,7 @@ def _parse_var_text() -> str: if p is not None ), exp.JSONPathSubscript: lambda self, e: self._jsonpathsubscript_sql(e), - exp.JSONPathUnion: lambda self, - e: f"[{','.join(self.json_path_part(p) for p in e.expressions)}]", + exp.JSONPathUnion: lambda self, e: f"[{','.join(self.json_path_part(p) for p in e.expressions)}]", exp.JSONPathWildcard: lambda *_: "*", } diff --git a/third_party/bigframes_vendored/sqlglot/lineage.py b/third_party/bigframes_vendored/sqlglot/lineage.py index 826e64bfdc8..8cdb862a0d0 100644 --- a/third_party/bigframes_vendored/sqlglot/lineage.py +++ b/third_party/bigframes_vendored/sqlglot/lineage.py @@ -2,19 +2,19 @@ from __future__ import annotations +from dataclasses import dataclass, field import json import logging import typing as t -from dataclasses import dataclass, field -from bigframes_vendored.sqlglot import Schema, exp, maybe_parse +from bigframes_vendored.sqlglot import exp, maybe_parse, Schema from bigframes_vendored.sqlglot.errors import SqlglotError from bigframes_vendored.sqlglot.optimizer import ( - Scope, build_scope, find_all_in_scope, normalize_identifiers, qualify, + Scope, ) from bigframes_vendored.sqlglot.optimizer.scope import ScopeType diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/__init__.py b/third_party/bigframes_vendored/sqlglot/optimizer/__init__.py index 9cc759fbe23..5de0f3bc78b 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/__init__.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/__init__.py @@ -2,11 +2,10 @@ # ruff: noqa: F401 -from bigframes_vendored.sqlglot.optimizer.optimizer import RULES as RULES # noqa: F401 from bigframes_vendored.sqlglot.optimizer.optimizer import ( # noqa: F401 optimize as optimize, ) -from bigframes_vendored.sqlglot.optimizer.scope import Scope as Scope # noqa: F401 +from bigframes_vendored.sqlglot.optimizer.optimizer import RULES as RULES # noqa: F401 from bigframes_vendored.sqlglot.optimizer.scope import ( # noqa: F401 build_scope as build_scope, ) @@ -16,6 +15,7 @@ from bigframes_vendored.sqlglot.optimizer.scope import ( # noqa: F401 find_in_scope as find_in_scope, ) +from bigframes_vendored.sqlglot.optimizer.scope import Scope as Scope # noqa: F401 from bigframes_vendored.sqlglot.optimizer.scope import ( # noqa: F401 traverse_scope as traverse_scope, ) diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/annotate_types.py b/third_party/bigframes_vendored/sqlglot/optimizer/annotate_types.py index cca95feee82..a1e5413e31f 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/annotate_types.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/annotate_types.py @@ -16,7 +16,7 @@ seq_get, ) from bigframes_vendored.sqlglot.optimizer.scope import Scope, traverse_scope -from bigframes_vendored.sqlglot.schema import MappingSchema, Schema, ensure_schema +from bigframes_vendored.sqlglot.schema import ensure_schema, MappingSchema, Schema if t.TYPE_CHECKING: from bigframes_vendored.sqlglot._typing import B, E @@ -830,10 +830,12 @@ def _annotate_struct(self, expression: exp.Struct) -> exp.Struct: return expression @t.overload - def _annotate_map(self, expression: exp.Map) -> exp.Map: ... + def _annotate_map(self, expression: exp.Map) -> exp.Map: + ... @t.overload - def _annotate_map(self, expression: exp.VarMap) -> exp.VarMap: ... + def _annotate_map(self, expression: exp.VarMap) -> exp.VarMap: + ... def _annotate_map(self, expression): keys = expression.args.get("keys") diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/eliminate_ctes.py b/third_party/bigframes_vendored/sqlglot/optimizer/eliminate_ctes.py index 8714c6bfa1b..ce1c3975a7e 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/eliminate_ctes.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/eliminate_ctes.py @@ -1,6 +1,6 @@ # Contains code from https://github.com/tobymao/sqlglot/blob/v28.5.0/sqlglot/optimizer/eliminate_ctes.py -from bigframes_vendored.sqlglot.optimizer.scope import Scope, build_scope +from bigframes_vendored.sqlglot.optimizer.scope import build_scope, Scope def eliminate_ctes(expression): diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/eliminate_subqueries.py b/third_party/bigframes_vendored/sqlglot/optimizer/eliminate_subqueries.py index 9deb0f65dc5..58a2e5fa888 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/eliminate_subqueries.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/eliminate_subqueries.py @@ -7,7 +7,7 @@ from bigframes_vendored.sqlglot import expressions as exp from bigframes_vendored.sqlglot.helper import find_new_name -from bigframes_vendored.sqlglot.optimizer.scope import Scope, build_scope +from bigframes_vendored.sqlglot.optimizer.scope import build_scope, Scope if t.TYPE_CHECKING: ExistingCTEsMapping = t.Dict[exp.Expression, str] diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/merge_subqueries.py b/third_party/bigframes_vendored/sqlglot/optimizer/merge_subqueries.py index 81e213ee814..33c9c143064 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/merge_subqueries.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/merge_subqueries.py @@ -2,8 +2,8 @@ from __future__ import annotations -import typing as t from collections import defaultdict +import typing as t from bigframes_vendored.sqlglot import expressions as exp from bigframes_vendored.sqlglot.helper import find_new_name, seq_get diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/normalize.py b/third_party/bigframes_vendored/sqlglot/optimizer/normalize.py index daa4bfb84d0..09b54fa13a8 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/normalize.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/normalize.py @@ -8,7 +8,7 @@ from bigframes_vendored.sqlglot.errors import OptimizeError from bigframes_vendored.sqlglot.helper import while_changing from bigframes_vendored.sqlglot.optimizer.scope import find_all_in_scope -from bigframes_vendored.sqlglot.optimizer.simplify import Simplifier, flatten +from bigframes_vendored.sqlglot.optimizer.simplify import flatten, Simplifier logger = logging.getLogger("sqlglot") diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/normalize_identifiers.py b/third_party/bigframes_vendored/sqlglot/optimizer/normalize_identifiers.py index eacf4305c49..9db0e729aba 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/normalize_identifiers.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/normalize_identifiers.py @@ -16,7 +16,8 @@ def normalize_identifiers( expression: E, dialect: DialectType = None, store_original_column_identifiers: bool = False, -) -> E: ... +) -> E: + ... @t.overload @@ -24,7 +25,8 @@ def normalize_identifiers( expression: str, dialect: DialectType = None, store_original_column_identifiers: bool = False, -) -> exp.Identifier: ... +) -> exp.Identifier: + ... def normalize_identifiers( diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/optimizer.py b/third_party/bigframes_vendored/sqlglot/optimizer/optimizer.py index ba13d17383e..93944747b03 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/optimizer.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/optimizer.py @@ -5,7 +5,7 @@ import inspect import typing as t -from bigframes_vendored.sqlglot import Schema, exp +from bigframes_vendored.sqlglot import exp, Schema from bigframes_vendored.sqlglot.dialects.dialect import DialectType from bigframes_vendored.sqlglot.optimizer.annotate_types import annotate_types from bigframes_vendored.sqlglot.optimizer.canonicalize import canonicalize diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/pushdown_projections.py b/third_party/bigframes_vendored/sqlglot/optimizer/pushdown_projections.py index b83dcb2c563..a7489b3f2f1 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/pushdown_projections.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/pushdown_projections.py @@ -2,8 +2,8 @@ from __future__ import annotations -import typing as t from collections import defaultdict +import typing as t from bigframes_vendored.sqlglot import alias, exp from bigframes_vendored.sqlglot.errors import OptimizeError diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/qualify.py b/third_party/bigframes_vendored/sqlglot/optimizer/qualify.py index cf518b06015..eb2ab1d5177 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/qualify.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/qualify.py @@ -22,7 +22,7 @@ validate_qualify_columns as validate_qualify_columns_func, ) from bigframes_vendored.sqlglot.optimizer.qualify_tables import qualify_tables -from bigframes_vendored.sqlglot.schema import Schema, ensure_schema +from bigframes_vendored.sqlglot.schema import ensure_schema, Schema def qualify( diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/qualify_columns.py b/third_party/bigframes_vendored/sqlglot/optimizer/qualify_columns.py index 51a0a6d4dc0..bc3d7dd55d8 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/qualify_columns.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/qualify_columns.py @@ -7,18 +7,18 @@ from bigframes_vendored.sqlglot import alias, exp from bigframes_vendored.sqlglot.dialects.dialect import Dialect, DialectType -from bigframes_vendored.sqlglot.errors import OptimizeError, highlight_sql +from bigframes_vendored.sqlglot.errors import highlight_sql, OptimizeError from bigframes_vendored.sqlglot.helper import seq_get from bigframes_vendored.sqlglot.optimizer.annotate_types import TypeAnnotator from bigframes_vendored.sqlglot.optimizer.resolver import Resolver from bigframes_vendored.sqlglot.optimizer.scope import ( - Scope, build_scope, + Scope, traverse_scope, walk_in_scope, ) from bigframes_vendored.sqlglot.optimizer.simplify import simplify_parens -from bigframes_vendored.sqlglot.schema import Schema, ensure_schema +from bigframes_vendored.sqlglot.schema import ensure_schema, Schema if t.TYPE_CHECKING: from bigframes_vendored.sqlglot._typing import E diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/resolver.py b/third_party/bigframes_vendored/sqlglot/optimizer/resolver.py index 02b216ff0e6..2f5098e4656 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/resolver.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/resolver.py @@ -8,7 +8,7 @@ from bigframes_vendored.sqlglot import exp from bigframes_vendored.sqlglot.dialects.dialect import Dialect from bigframes_vendored.sqlglot.errors import OptimizeError -from bigframes_vendored.sqlglot.helper import SingleValuedMapping, seq_get +from bigframes_vendored.sqlglot.helper import seq_get, SingleValuedMapping from bigframes_vendored.sqlglot.optimizer.scope import Scope if t.TYPE_CHECKING: diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/scope.py b/third_party/bigframes_vendored/sqlglot/optimizer/scope.py index 4256abc6173..b99d09d37dd 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/scope.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/scope.py @@ -2,11 +2,11 @@ from __future__ import annotations +from collections import defaultdict +from enum import auto, Enum import itertools import logging import typing as t -from collections import defaultdict -from enum import Enum, auto from bigframes_vendored.sqlglot import exp from bigframes_vendored.sqlglot.errors import OptimizeError diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/simplify.py b/third_party/bigframes_vendored/sqlglot/optimizer/simplify.py index 573dc9e67d3..1053b8ff343 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/simplify.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/simplify.py @@ -2,13 +2,13 @@ from __future__ import annotations +from collections import defaultdict, deque import datetime import functools +from functools import reduce, wraps import itertools import logging import typing as t -from collections import defaultdict, deque -from functools import reduce, wraps import bigframes_vendored.sqlglot from bigframes_vendored.sqlglot import Dialect, exp diff --git a/third_party/bigframes_vendored/sqlglot/optimizer/unnest_subqueries.py b/third_party/bigframes_vendored/sqlglot/optimizer/unnest_subqueries.py index 0e3431bfa9b..f57c569d6c3 100644 --- a/third_party/bigframes_vendored/sqlglot/optimizer/unnest_subqueries.py +++ b/third_party/bigframes_vendored/sqlglot/optimizer/unnest_subqueries.py @@ -3,8 +3,8 @@ from bigframes_vendored.sqlglot import exp from bigframes_vendored.sqlglot.helper import name_sequence from bigframes_vendored.sqlglot.optimizer.scope import ( - ScopeType, find_in_scope, + ScopeType, traverse_scope, ) diff --git a/third_party/bigframes_vendored/sqlglot/parser.py b/third_party/bigframes_vendored/sqlglot/parser.py index 706649f43fb..11d552117b2 100644 --- a/third_party/bigframes_vendored/sqlglot/parser.py +++ b/third_party/bigframes_vendored/sqlglot/parser.py @@ -2,25 +2,25 @@ from __future__ import annotations +from collections import defaultdict import itertools import logging import re import typing as t -from collections import defaultdict from bigframes_vendored.sqlglot import exp from bigframes_vendored.sqlglot.errors import ( - ErrorLevel, - ParseError, - TokenError, concat_messages, + ErrorLevel, highlight_sql, merge_errors, + ParseError, + TokenError, ) from bigframes_vendored.sqlglot.helper import apply_index_offset, ensure_list, seq_get from bigframes_vendored.sqlglot.time import format_time from bigframes_vendored.sqlglot.tokens import Token, Tokenizer, TokenType -from bigframes_vendored.sqlglot.trie import TrieResult, in_trie, new_trie +from bigframes_vendored.sqlglot.trie import in_trie, new_trie, TrieResult if t.TYPE_CHECKING: from bigframes_vendored.sqlglot._typing import E, Lit @@ -290,11 +290,11 @@ class Parser(metaclass=_Parser): "RIGHTPAD": lambda args: build_pad(args, is_left=False), "RPAD": lambda args: build_pad(args, is_left=False), "RTRIM": lambda args: build_trim(args, is_left=False), - "SCOPE_RESOLUTION": lambda args: ( - exp.ScopeResolution(expression=seq_get(args, 0)) - if len(args) != 2 - else exp.ScopeResolution(this=seq_get(args, 0), expression=seq_get(args, 1)) - ), + "SCOPE_RESOLUTION": lambda args: exp.ScopeResolution( + expression=seq_get(args, 0) + ) + if len(args) != 2 + else exp.ScopeResolution(this=seq_get(args, 0), expression=seq_get(args, 1)), "STRPOS": exp.StrPosition.from_arg_list, "CHARINDEX": lambda args: build_locate_strposition(args), "INSTR": exp.StrPosition.from_arg_list, @@ -943,9 +943,7 @@ class Parser(metaclass=_Parser): } UNARY_PARSERS = { - TokenType.PLUS: lambda self: ( - self._parse_unary() - ), # Unary + is handled as a no-op + TokenType.PLUS: lambda self: self._parse_unary(), # Unary + is handled as a no-op TokenType.NOT: lambda self: self.expression( exp.Not, this=self._parse_equality() ), @@ -1248,14 +1246,12 @@ class Parser(metaclass=_Parser): exp.NotNullColumnConstraint, allow_null=True ), "ON": lambda self: ( - ( - self._match(TokenType.UPDATE) - and self.expression( - exp.OnUpdateColumnConstraint, this=self._parse_function() - ) + self._match(TokenType.UPDATE) + and self.expression( + exp.OnUpdateColumnConstraint, this=self._parse_function() ) - or self.expression(exp.OnProperty, this=self._parse_id_var()) - ), + ) + or self.expression(exp.OnProperty, this=self._parse_id_var()), "PATH": lambda self: self.expression( exp.PathColumnConstraint, this=self._parse_string() ), @@ -3825,10 +3821,12 @@ def _implicit_unnests_to_explicit(self, this: E) -> E: return this @t.overload - def _parse_query_modifiers(self, this: E) -> E: ... + def _parse_query_modifiers(self, this: E) -> E: + ... @t.overload - def _parse_query_modifiers(self, this: None) -> None: ... + def _parse_query_modifiers(self, this: None) -> None: + ... def _parse_query_modifiers(self, this): if isinstance(this, self.MODIFIABLES): @@ -3889,9 +3887,8 @@ def _parse_hint_body(self) -> t.Optional[exp.Hint]: try: for hint in iter( lambda: self._parse_csv( - lambda: ( - self._parse_hint_function_call() or self._parse_var(upper=True) - ), + lambda: self._parse_hint_function_call() + or self._parse_var(upper=True), ), [], ): @@ -4310,9 +4307,8 @@ def _parse_table_hints(self) -> t.Optional[t.List[exp.Expression]]: self.expression( exp.WithTableHint, expressions=self._parse_csv( - lambda: ( - self._parse_function() or self._parse_var(any_token=True) - ) + lambda: self._parse_function() + or self._parse_var(any_token=True) ), ) ) @@ -4475,14 +4471,6 @@ def _parse_table( if schema: return self._parse_schema(this=this) - # see: https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from_clause - # from_item, then alias, then time travel, then sample. - alias = self._parse_table_alias( - alias_tokens=alias_tokens or self.TABLE_ALIAS_TOKENS - ) - if alias: - this.set("alias", alias) - version = self._parse_version() if version: @@ -4491,6 +4479,12 @@ def _parse_table( if self.dialect.ALIAS_POST_TABLESAMPLE: this.set("sample", self._parse_table_sample()) + alias = self._parse_table_alias( + alias_tokens=alias_tokens or self.TABLE_ALIAS_TOKENS + ) + if alias: + this.set("alias", alias) + if self._match(TokenType.INDEXED_BY): this.set("indexed", self._parse_table_parts()) elif self._match_text_seq("NOT", "INDEXED"): @@ -4626,7 +4620,8 @@ def _parse_derived_table_values(self) -> t.Optional[exp.Values]: is_derived = self._match_pair(TokenType.L_PAREN, TokenType.VALUES) if not is_derived and not ( # ClickHouse's `FORMAT Values` is equivalent to `VALUES` - self._match_text_seq("VALUES") or self._match_text_seq("FORMAT", "VALUES") + self._match_text_seq("VALUES") + or self._match_text_seq("FORMAT", "VALUES") ): return None @@ -4943,13 +4938,11 @@ def _parse_group(self, skip_group_by_token: bool = False) -> t.Optional[exp.Grou elements["expressions"].extend( self._parse_csv( - lambda: ( - None - if self._match_set( - (TokenType.CUBE, TokenType.ROLLUP), advance=False - ) - else self._parse_disjunction() + lambda: None + if self._match_set( + (TokenType.CUBE, TokenType.ROLLUP), advance=False ) + else self._parse_disjunction() ) ) @@ -6235,9 +6228,9 @@ def _parse_column_ops( # https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-reference#function_call_rules if isinstance(field, (exp.Func, exp.Window)) and this: this = this.transform( - lambda n: ( - n.to_dot(include_dots=False) if isinstance(n, exp.Column) else n - ) + lambda n: n.to_dot(include_dots=False) + if isinstance(n, exp.Column) + else n ) if op: @@ -7459,10 +7452,12 @@ def _parse_on_handling( return None @t.overload - def _parse_json_object(self, agg: Lit[False]) -> exp.JSONObject: ... + def _parse_json_object(self, agg: Lit[False]) -> exp.JSONObject: + ... @t.overload - def _parse_json_object(self, agg: Lit[True]) -> exp.JSONObjectAgg: ... + def _parse_json_object(self, agg: Lit[True]) -> exp.JSONObjectAgg: + ... def _parse_json_object(self, agg=False): star = self._parse_star() diff --git a/third_party/bigframes_vendored/sqlglot/schema.py b/third_party/bigframes_vendored/sqlglot/schema.py index 87928f2fc6f..748fd1fd658 100644 --- a/third_party/bigframes_vendored/sqlglot/schema.py +++ b/third_party/bigframes_vendored/sqlglot/schema.py @@ -9,7 +9,7 @@ from bigframes_vendored.sqlglot.dialects.dialect import Dialect from bigframes_vendored.sqlglot.errors import SchemaError from bigframes_vendored.sqlglot.helper import dict_depth, first -from bigframes_vendored.sqlglot.trie import TrieResult, in_trie, new_trie +from bigframes_vendored.sqlglot.trie import in_trie, new_trie, TrieResult if t.TYPE_CHECKING: from bigframes_vendored.sqlglot.dialects.dialect import DialectType diff --git a/third_party/bigframes_vendored/sqlglot/serde.py b/third_party/bigframes_vendored/sqlglot/serde.py index dc21407e5f3..65c8e05a653 100644 --- a/third_party/bigframes_vendored/sqlglot/serde.py +++ b/third_party/bigframes_vendored/sqlglot/serde.py @@ -74,11 +74,13 @@ def dump(expression: exp.Expression) -> t.List[t.Dict[str, t.Any]]: @t.overload -def load(payloads: None) -> None: ... +def load(payloads: None) -> None: + ... @t.overload -def load(payloads: t.List[t.Dict[str, t.Any]]) -> exp.Expression: ... +def load(payloads: t.List[t.Dict[str, t.Any]]) -> exp.Expression: + ... def load(payloads): diff --git a/third_party/bigframes_vendored/sqlglot/time.py b/third_party/bigframes_vendored/sqlglot/time.py index 05873b187a9..1c8f34a59d5 100644 --- a/third_party/bigframes_vendored/sqlglot/time.py +++ b/third_party/bigframes_vendored/sqlglot/time.py @@ -5,7 +5,7 @@ # The generic time format is based on python time.strftime. # https://docs.python.org/3/library/time.html#time.strftime -from bigframes_vendored.sqlglot.trie import TrieResult, in_trie, new_trie +from bigframes_vendored.sqlglot.trie import in_trie, new_trie, TrieResult def format_time( diff --git a/third_party/bigframes_vendored/sqlglot/tokens.py b/third_party/bigframes_vendored/sqlglot/tokens.py index 9a2c4f7a650..b21f0e31738 100644 --- a/third_party/bigframes_vendored/sqlglot/tokens.py +++ b/third_party/bigframes_vendored/sqlglot/tokens.py @@ -2,13 +2,13 @@ from __future__ import annotations +from enum import auto import os import typing as t -from enum import auto from bigframes_vendored.sqlglot.errors import SqlglotError, TokenError from bigframes_vendored.sqlglot.helper import AutoName -from bigframes_vendored.sqlglot.trie import TrieResult, in_trie, new_trie +from bigframes_vendored.sqlglot.trie import in_trie, new_trie, TrieResult if t.TYPE_CHECKING: from bigframes_vendored.sqlglot.dialects.dialect import DialectType diff --git a/third_party/bigframes_vendored/sqlglot/transforms.py b/third_party/bigframes_vendored/sqlglot/transforms.py index edb1a21d6cb..3c769a77cea 100644 --- a/third_party/bigframes_vendored/sqlglot/transforms.py +++ b/third_party/bigframes_vendored/sqlglot/transforms.py @@ -961,9 +961,9 @@ def eliminate_join_marks(expression: exp.Expression) -> exp.Expression: if not left_join_table: continue - assert not (len(left_join_table) > 1), ( - "Cannot combine JOIN predicates from different tables" - ) + assert not ( + len(left_join_table) > 1 + ), "Cannot combine JOIN predicates from different tables" for col in join_cols: col.set("join_mark", False) @@ -993,9 +993,9 @@ def eliminate_join_marks(expression: exp.Expression) -> exp.Expression: if query_from.alias_or_name in new_joins: only_old_joins = old_joins.keys() - new_joins.keys() - assert len(only_old_joins) >= 1, ( - "Cannot determine which table to use in the new FROM clause" - ) + assert ( + len(only_old_joins) >= 1 + ), "Cannot determine which table to use in the new FROM clause" new_from_name = list(only_old_joins)[0] query.set("from_", exp.From(this=old_joins[new_from_name].this)) diff --git a/third_party/bigframes_vendored/sqlglot/trie.py b/third_party/bigframes_vendored/sqlglot/trie.py index 1475ea58774..16c23337a25 100644 --- a/third_party/bigframes_vendored/sqlglot/trie.py +++ b/third_party/bigframes_vendored/sqlglot/trie.py @@ -1,7 +1,7 @@ # Contains code from https://github.com/tobymao/sqlglot/blob/v28.5.0/sqlglot/trie.py +from enum import auto, Enum import typing as t -from enum import Enum, auto key = t.Sequence[t.Hashable] diff --git a/third_party/bigframes_vendored/tpch/queries/q1.py b/third_party/bigframes_vendored/tpch/queries/q1.py index aa6289866c5..f533776e858 100644 --- a/third_party/bigframes_vendored/tpch/queries/q1.py +++ b/third_party/bigframes_vendored/tpch/queries/q1.py @@ -1,7 +1,7 @@ # Contains code from https://github.com/pola-rs/tpch/blob/main/queries/pandas/q1.py -import typing from datetime import datetime +import typing import bigframes import bigframes.pandas as bpd diff --git a/third_party/bigframes_vendored/tpch/queries/q10.py b/third_party/bigframes_vendored/tpch/queries/q10.py index 19a33b07ec2..8c0d93dc261 100644 --- a/third_party/bigframes_vendored/tpch/queries/q10.py +++ b/third_party/bigframes_vendored/tpch/queries/q10.py @@ -1,7 +1,7 @@ # Contains code from https://github.com/pola-rs/tpch/blob/main/queries/polars/q10.py -import typing from datetime import date +import typing import bigframes import bigframes.pandas as bpd diff --git a/third_party/bigframes_vendored/tpch/queries/q12.py b/third_party/bigframes_vendored/tpch/queries/q12.py index 20097f72ca1..1bc22f1167e 100644 --- a/third_party/bigframes_vendored/tpch/queries/q12.py +++ b/third_party/bigframes_vendored/tpch/queries/q12.py @@ -1,7 +1,7 @@ # Contains code from https://github.com/pola-rs/tpch/blob/main/queries/polars/q12.py -import typing from datetime import date +import typing import bigframes import bigframes.pandas as bpd diff --git a/third_party/bigframes_vendored/tpch/queries/q19.py b/third_party/bigframes_vendored/tpch/queries/q19.py index a217db3dc32..1371af53fc0 100644 --- a/third_party/bigframes_vendored/tpch/queries/q19.py +++ b/third_party/bigframes_vendored/tpch/queries/q19.py @@ -53,11 +53,5 @@ def q(project_id: str, dataset_id: str, session: bigframes.Session): ) ] - result_df = ( - (filtered["L_EXTENDEDPRICE"] * (1 - filtered["L_DISCOUNT"])) - .agg(["sum"]) - .rename("REVENUE") - .to_frame() - ) - - next(result_df.to_pandas_batches(max_results=1500)) + revenue = (filtered["L_EXTENDEDPRICE"] * (1 - filtered["L_DISCOUNT"])).sum() + _ = round(revenue, 2) diff --git a/third_party/bigframes_vendored/tpch/queries/q4.py b/third_party/bigframes_vendored/tpch/queries/q4.py index 9c855704ab9..3782a7273fc 100644 --- a/third_party/bigframes_vendored/tpch/queries/q4.py +++ b/third_party/bigframes_vendored/tpch/queries/q4.py @@ -1,8 +1,8 @@ # Contains code from https://github.com/pola-rs/tpch/blob/main/queries/pandas/q4.py -import typing from datetime import date +import typing import bigframes import bigframes.pandas as bpd diff --git a/third_party/bigframes_vendored/tpch/queries/q7.py b/third_party/bigframes_vendored/tpch/queries/q7.py index 0756bcc6566..81cdda87886 100644 --- a/third_party/bigframes_vendored/tpch/queries/q7.py +++ b/third_party/bigframes_vendored/tpch/queries/q7.py @@ -1,7 +1,7 @@ # Contains code from https://github.com/pola-rs/tpch/blob/main/queries/pandas/q7.py -import typing from datetime import date +import typing import bigframes import bigframes.dataframe diff --git a/third_party/bigframes_vendored/version.py b/third_party/bigframes_vendored/version.py index 3eecebee5a1..f36c6789c1a 100644 --- a/third_party/bigframes_vendored/version.py +++ b/third_party/bigframes_vendored/version.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.48.0" +__version__ = "2.32.0" # {x-release-please-start-date} -__release_date__ = "2026-06-12" +__release_date__ = "2026-01-05" # {x-release-please-end} diff --git a/third_party/sphinx/ext/autosummary/templates/autosummary/class.rst b/third_party/sphinx/ext/autosummary/templates/autosummary/class.rst index 6651591be64..89550cb3864 100644 --- a/third_party/sphinx/ext/autosummary/templates/autosummary/class.rst +++ b/third_party/sphinx/ext/autosummary/templates/autosummary/class.rst @@ -2,13 +2,11 @@ .. currentmodule:: {{ module }} -{% set is_pandas = module.startswith("bigframes.pandas") or module.startswith("bigframes.geopandas") %} -{% set skip_inherited = is_pandas and not module.startswith("bigframes.pandas.typing.api") %} - -{% if is_pandas %} .. autoclass:: {{ objname }} :no-members: + {% block methods %} + {% block attributes %} {% if attributes %} .. rubric:: {{ _('Attributes') }} @@ -16,27 +14,18 @@ .. autosummary:: :toctree: {% for item in attributes %} - {%- if not skip_inherited or not item in inherited_members%} ~{{ name }}.{{ item }} - {%- endif %} {%- endfor %} {% endif %} {% endblock %} - {% block methods %} {% if methods %} .. rubric:: {{ _('Methods') }} .. autosummary:: :toctree: - {% for item in methods %} - {%- if not skip_inherited or not item in inherited_members%} ~{{ name }}.{{ item }} - {%- endif %} {%- endfor %} {% endif %} {% endblock %} -{% else %} -.. autoclass:: {{ objname }} -{% endif %}