diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 9dd4b72..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Please see the documentation for all configuration options: -# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file - -version: 2 -updates: - - package-ecosystem: "uv" - directory: "/" - schedule: - interval: "weekly" - reviewers: - - "larsewi" - - "olehermanse" - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - reviewers: - - "larsewi" - - "olehermanse" - prefix: "GitHub Actions" diff --git a/.github/renovate-config.js b/.github/renovate-config.js new file mode 100644 index 0000000..49e6e44 --- /dev/null +++ b/.github/renovate-config.js @@ -0,0 +1,10 @@ +// Global (self-hosted) Renovate configuration, used by the +// .github/workflows/renovate.yml GitHub Action. +// Repository-level configuration is in renovate.json. +module.exports = { + platform: "github", + onboarding: false, + requireConfig: "optional", + branchPrefix: "renovate/", + repositories: ["cfengine/cfengine-cli"], +}; diff --git a/.github/workflows/lint-policy-in-other-repos.yml b/.github/workflows/lint-policy-in-other-repos.yml index 2c5c71e..b9d976c 100644 --- a/.github/workflows/lint-policy-in-other-repos.yml +++ b/.github/workflows/lint-policy-in-other-repos.yml @@ -16,28 +16,28 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout cfengine-cli - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: path: cfengine-cli - name: Checkout masterfiles - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: repository: cfengine/masterfiles path: masterfiles - name: Checkout documentation - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: repository: cfengine/documentation path: documentation - name: Checkout modules - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: repository: cfengine/modules path: modules - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: - python-version: "3.13" + python-version: "3.14" - name: Install dependencies working-directory: cfengine-cli run: | diff --git a/.github/workflows/make-check.yml b/.github/workflows/make-check.yml index b59382e..db06581 100644 --- a/.github/workflows/make-check.yml +++ b/.github/workflows/make-check.yml @@ -25,9 +25,9 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 403c4bc..8925f44 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -9,8 +9,8 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.x" - name: Build release distributions @@ -21,7 +21,7 @@ jobs: uv lock --check uv build - name: Upload release-dists as artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: release-dists path: dist/ @@ -30,7 +30,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - run: | git fetch --all --tags pip install uv @@ -46,9 +46,9 @@ jobs: id-token: write steps: - name: Retrieve release distributions - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: release-dists path: dist/ - name: Publish release distributions to PyPI - uses: pypa/gh-action-pypi-publish@v1.13.0 + uses: pypa/gh-action-pypi-publish@v1.14.1 diff --git a/.github/workflows/renovate.yml b/.github/workflows/renovate.yml new file mode 100644 index 0000000..fd8cb56 --- /dev/null +++ b/.github/workflows/renovate.yml @@ -0,0 +1,18 @@ +name: Renovate + +on: + schedule: + - cron: "0 6 * * *" # Run every day at 6am UTC + workflow_dispatch: # Enables manual trigger + +jobs: + renovate: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7.0.1 + - name: Self-hosted Renovate + uses: renovatebot/github-action@v46.1.20 + with: + configurationFile: .github/renovate-config.js + token: ${{ secrets.RENOVATE_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d0b2e5d..0f8800c 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -15,7 +15,7 @@ jobs: issues: read steps: - - uses: actions/stale@v5 + - uses: actions/stale@v10 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: "Marking this PR as stale due to inactivity; it will be closed in 7 days." diff --git a/.github/workflows/update-syntax-description.yml b/.github/workflows/update-syntax-description.yml index e1f2dde..806d536 100644 --- a/.github/workflows/update-syntax-description.yml +++ b/.github/workflows/update-syntax-description.yml @@ -21,13 +21,13 @@ jobs: pull-requests: write steps: - name: Checks-out repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: ref: "main" - name: Set up Python 3.12 - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: - python-version: "3.12" + python-version: "3.14" - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..2f329a5 --- /dev/null +++ b/renovate.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["config:recommended"], + "reviewers": ["larsewi", "olehermanse"] +} diff --git a/src/cfengine_cli/cfengine_wrapper/arg_parse.py b/src/cfengine_cli/cfengine_wrapper/arg_parse.py new file mode 100644 index 0000000..a7b6871 --- /dev/null +++ b/src/cfengine_cli/cfengine_wrapper/arg_parse.py @@ -0,0 +1,261 @@ +import argparse + + +def parse_wrapper_args(subp: argparse._SubParsersAction): + + sp = subp.add_parser( + "save", help="Save host(s) with a group name to use in other commands" + ) + sp.add_argument( + "--role", + help="Role of the hosts", + choices=["hub", "hubs", "client", "clients"], + required=True, + ) + sp.add_argument( + "--name", + help="Name of the group of hosts (can be used in other commands)", + required=True, + ) + sp.add_argument( + "--hosts", + "-H", + help="SSH usernames and IPs for SSH and CFEngine in the form of user@ip", + required=True, + ) + + sp = subp.add_parser( + "setup-code", help="Fetches a new setup-code for mission-portal login" + ) + sp.add_argument( + "--hub", + "-H", + help="Hub from which to fetch new setup-code", + type=str, + default=None, + ) + + subp.add_parser("build", help="Build a policy set from a CFEngine Build project\n\ +A wrapper arount the cfbs `build`-function.") + sp = subp.add_parser("deploy", help="Deploy policy-set (masterfiles) to hub\n\ +A wrapper around the cf-remote `deploy`-function with some added niceties.") + sp.add_argument("--hub", help="Hub(s) to deploy to", type=str) + sp.add_argument( + "masterfiles", + help="Policy-set location (tarball URL or local path to tarball / directory)", + type=str, + nargs="?", + ) + + install_parser = subp.add_parser( + "install", + help="Install CFEngine on the given hosts", + description="A wrapper around the cf-remote `install` function", + ) + install_parser.add_argument( + "--version", + "-V", + help="Specify version", + type=str, + ) + # install_parser._option_string_actions.get("--version").help = "absdfsf" + # TODO: Update cf-remote/cfbs to have more modular arg-parsing, then we can import + # and override any differences? technically illegal since _option_string_actions, + # but will save ~ 200-1000 loc depending on how much we import into cfengine-cli + + install_parser.add_argument( + "--edition", + "-E", + choices=["community", "enterprise"], + help="Enterprise or community packages", + type=str, + ) + install_parser.add_argument( + "--package", help="Local path to package or URL to download", type=str + ) + install_parser.add_argument( + "--hub-package", + help="Local path to package or URL to download for --hub", + type=str, + ) + install_parser.add_argument( + "--client-package", + help="Local path to package or URL to download for --clients", + type=str, + ) + install_parser.add_argument( + "--bootstrap", "-B", help="cf-agent --bootstrap argument", type=str + ) + install_parser.add_argument( + "--clients", "-c", help="Where to install client package", type=str + ) + install_parser.add_argument("--hub", help="Where to install hub package", type=str) + install_parser.add_argument( + "--demo", + help="Use defaults to make demos smoother (NOT secure)", + action="store_true", + ) + install_parser.add_argument( + "--call-collect", + help="Enable call collect in --demo def.json", + action="store_true", + ) + install_parser.add_argument( + "--remote-download", + help="Package will be downloaded directly to the target machine", + action="store_true", + ) + install_parser.add_argument( + "--trust-keys", + help="Comma-separated list of paths to keys hosts should trust" + + " (implies '--trust-server no' when boostraping)", + type=str, + ) + install_parser.add_argument( + "--insecure", + help="Ignore mismatching checksums when downloading urls", + action="store_true", + ) + + uninstall_parser = subp.add_parser( + "uninstall", + help="Uninstall CFEngine on the given hosts", + description="A wrapper around the cf-remote `uninstall` function", + ) + uninstall_parser.add_argument( + "--purge", help="Complete uninstallation", action="store_true" + ) + uninstall_parser.add_argument( + "--clients", "-c", help="Where to uninstall", type=str + ) + uninstall_parser.add_argument("--hub", help="Where to uninstall", type=str) + uninstall_parser.add_argument("--hosts", "-H", help="Where to uninstall", type=str) + + report_parser = subp.add_parser( + "report", + help="Refresh reporting data", + ) + report_parser.add_argument( + "--run-agent", + action="store_true", + help="Runs the agent on the chosen host(s) before collecting report data.", + ) + report_parser.add_argument( + "--hub", + "-H", + type=str, + default=None, + help="Only refresh one hub specified by name/IP (e.g. 'local' or '192.168.56.90') and accompanying clients", + ) + + run_parser = subp.add_parser( + "run", + description="Run the CFEngine agent, fetching, evaluating, and enforcing policy.\n\ +A wrapper around the cf-remote `run`-function with some added niceties", + epilog="""Examples: + `cfengine run` defaults to use `cf-agent -KIf update.cf && cf-agent -KI` + + Run can also be used directly on a specific file, e.g. + 'cfengine run /tmp/some_policy.cf' or 'cfengine run "-KIf /tmp/some_policy.cf"' + If no flags are present in the command, then -KIf will be automatically prepended. + + Multiple commands can also be run in sequence, such as: + 'cfengine run /tmp/some_policy.cf /tmp/some_other_policy.cf /tmp/and_another.cf' + Where all three files will be run in sequence, exiting on first fail + """, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + run_parser.add_argument( + "run_args", + nargs="*", + help="Command(s) to run with cf-agent", + ) + run_parser.add_argument( + "--host", + type=str, + default=None, + help="Select which installation of cf-agent to use by name/IP (e.g. 'local' or '192.168.56.90'). " + "If omitted and multiple installations are found, you'll be prompted.", + ) + + sp = subp.add_parser( + "spawn", + help="Spawn hosts in the clouds", + description="A wrapper around the cf-remote `spawn`-function", + ) + sp.add_argument( + "--list-platforms", help="List supported platforms", action="store_true" + ) + sp.add_argument( + "--list-boxes", help="List installed vagrant boxes", action="store_true" + ) + sp.add_argument( + "--init-config", + help="Initialize configuration file for spawn functionality", + action="store_true", + ) + sp.add_argument("--platform", help="Platform or vagrant box to use", type=str) + sp.add_argument("--count", default=1, help="How many hosts to spawn", type=int) + sp.add_argument( + "--role", help="Role of the hosts", choices=["hub", "hubs", "client", "clients"] + ) + sp.add_argument( + "--name", help="Name of the group of hosts (can be used in other commands)" + ) + sp.add_argument( + "--append", + help="Append the new VMs to a pre-existing group", + action="store_true", + ) + sp.add_argument( + "--provider", + help="VM provider", + type=str, + default="aws", + choices=["aws", "gcp", "vagrant"], + ) + sp.add_argument("--cpus", help="Number of CPUs of the vagrant instances", type=int) + sp.add_argument( + "--sync-folder", + help="Root folder of synchronized folders of vagrant instance", + type=str, + ) + sp.add_argument( + "--provision", + help="full path to provision shell script for Vagrant VM", + type=str, + ) + sp.add_argument("--size", help="Size/type of the instances", type=str) + sp.add_argument( + "--network", help="network/subnet to assign the VMs to (GCP only)", type=str + ) + sp.add_argument( + "--no-public-ip", + help="No public IP needed (GCP only; WARNING: The VMs will only be accessible" + + " from some other VM in the same cloud/network!)", + action="store_true", + ) + + dp = subp.add_parser( + "destroy", + help="Destroy hosts spawned in the clouds", + description="A wrapper around the cf-remote `destroy`-function", + ) + dp.add_argument( + "--all", help="Destroy all hosts spawned in the clouds", action="store_true" + ) + dp.add_argument("name", help="Name of the group of hosts to destroy", nargs="?") + + profile_parser = subp.add_parser( + "profile", help="Parse CFEngine profiling output (cf-agent -Kp)" + ) + profile_parser.add_argument( + "profiling_input", help="Path to the profiling input file" + ) + profile_parser.add_argument("--top", type=int, default=10) + profile_parser.add_argument("--bundles", action="store_true") + profile_parser.add_argument("--promises", action="store_true") + profile_parser.add_argument("--functions", action="store_true") + profile_parser.add_argument( + "--flamegraph", type=str, help="Generate input file for ./flamegraph.pl" + ) diff --git a/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py b/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py index 7d4b429..ba4bba4 100644 --- a/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py +++ b/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py @@ -3,20 +3,21 @@ from cfbs.commands import build_command from cf_remote.commands import deploy as deploy_command -from cf_remote.commands import install as install_command from cf_remote.commands import destroy as destroy_command +from cf_remote.commands import save as save_command from cf_remote.remote import run_command, transfer_file from cfengine_cli.utils import UserError from cfengine_cli.cfengine_wrapper.cfengine_objects import ( Executable, - _ensure_default_agent_flags, + ensure_default_agent_flags, ) from cfengine_cli.cfengine_wrapper.cfengine_utils import ( extract_agent_file, prompt_two_options, + prompt_yes_no, require_executable, - require_installation, + select_report_targets, ) _DEFAULT_CFENGINE_INPUTS_DIR = "/var/cfengine/inputs" @@ -113,7 +114,7 @@ def _resolve_command_for_agent(agent: Executable, command: str) -> str: if agent.name != "cf-agent": return command - command = _ensure_default_agent_flags(command) + command = ensure_default_agent_flags(command) file_arg = extract_agent_file(command) if not file_arg: return command @@ -131,14 +132,74 @@ def _resolve_command_for_agent(agent: Executable, command: str) -> str: # --------------------------------------------------------------------------- -def report(target: str | None = None) -> int: # TODO? ENT-14122 - installation = require_installation(target) - rc = installation.agent.run("-KIf update.cf", "-KI") - if rc != 0: - return rc - return installation.hub.run( - "--query rebase -H 127.0.0.1", "--query delta -H 127.0.0.1" - ) +def save(hosts: str, role: str, name: str) -> int: # TODO: Add to existing group + return save_command(hosts=hosts, role=role, name=name) + + +def _refresh_agent(agent: Executable) -> int: + try: + return agent.run("-KIf update.cf", "-KI") + except (Exception, SystemExit) as e: + logging.error(f"Skipping {agent.label}: {e}") + return 1 + + +def _query_hub_delta(hub: Executable, client_ips: list[str]) -> int: + """ + Ask a hub to recompute delta report data for itself and + for every client bootstrapped to it. + """ + try: + queries = ["--query delta -H 127.0.0.1"] + [ + f"--query delta -H {ip}" for ip in client_ips + ] + return hub.run(*queries) + except (Exception, SystemExit) as e: + logging.error(f"Skipping hub {hub.label}: {e}") + return 1 + + +def report( + target: str | None = None, + run_agent: bool = False, +) -> int: + errors = 0 + hubs, clients = select_report_targets(target) + + hub_agent_failed = {} + if run_agent: + for hub in hubs: + rc = _refresh_agent(hub.agent) + hub_agent_failed[hub.location] = rc != 0 + if rc != 0: + logging.error(f"Agent run failed on {hub.agent.label}") + errors += 1 + + for agent in clients: + rc = _refresh_agent(agent) + if rc != 0: + logging.error(f"Refresh failed on {agent.label})") + errors += 1 + + for hub in hubs: + if run_agent and hub_agent_failed[hub.location]: + logging.warning( + f"Agent run failed for {hub.location}, some data may be stale." + ) + client_ips = [client.location.split("@", 1)[1] for client in clients] + rc = _query_hub_delta(hub.hub, client_ips) + if rc != 0: + logging.error(f"Hub refresh failed on {hub.label})") + errors += 1 + + if errors > 0: + logging.error(f"Encountered {errors}.") + return errors + + +def setup_code(target: str | None = None) -> int: + hub = require_executable("cf-hub", target) + return hub.run("--new-setup-code") def run(*args, target: str | None = None) -> int: @@ -149,19 +210,23 @@ def run(*args, target: str | None = None) -> int: return agent.run(*resolved) -def install() -> int: # TODO ENT-14117 - return install_command(None, None) - - def destroy(groupname, del_all=False) -> int: if del_all: return destroy_command(None) return destroy_command(groupname) -def build() -> int: # TODO ENT-14119 - return build_command() +def build() -> int: + rc = build_command() + if rc != 0: + return rc + if prompt_yes_no("Deploy the built policy set now?", default=True): + return deploy(None, None) + return 0 -def deploy() -> int: # TODO ENT-14119 - return deploy_command(None, None) +def deploy(target: str | list[str] | None, masterfiles: str | None = None) -> int: + if isinstance(target, str): + target = [target] + hubs = [require_executable("cf-agent", h).location for h in (target or [])] or None + return deploy_command(hubs, masterfiles) diff --git a/src/cfengine_cli/cfengine_wrapper/cfengine_objects.py b/src/cfengine_cli/cfengine_wrapper/cfengine_objects.py index 854674c..5ac5b9e 100644 --- a/src/cfengine_cli/cfengine_wrapper/cfengine_objects.py +++ b/src/cfengine_cli/cfengine_wrapper/cfengine_objects.py @@ -5,7 +5,7 @@ import os -def _ensure_default_agent_flags(command: str) -> str: +def ensure_default_agent_flags(command: str) -> str: """ cf-agent needs -K (no-lock), -I (inform), and -f (specify file) to actually run against a policy file the way people expect. @@ -52,17 +52,14 @@ def label(self) -> str: return self.location def run(self, *commands) -> int: - rc = 0 + errors = 0 for command in commands: rc = self._run_one(command) if rc != 0: - return rc - return rc + errors += 1 + return errors def _run_one(self, command: str) -> int: - if self.name == "cf-agent": - command = _ensure_default_agent_flags(command) - if self.is_local: args = [self.path] + command.split() # cf-agent picks its workdir based on privilege: as root it uses diff --git a/src/cfengine_cli/cfengine_wrapper/cfengine_utils.py b/src/cfengine_cli/cfengine_wrapper/cfengine_utils.py index c223d7c..510e6aa 100644 --- a/src/cfengine_cli/cfengine_wrapper/cfengine_utils.py +++ b/src/cfengine_cli/cfengine_wrapper/cfengine_utils.py @@ -1,16 +1,28 @@ import os import shutil import logging -import sys +import random from collections.abc import Iterator +from functools import lru_cache + +from cf_remote.remote import get_info from cfengine_cli.paths import bin from cfengine_cli.utils import UserError from cfengine_cli.cfengine_wrapper.cfengine_objects import Executable, Installation -from cf_remote.remote import get_info from cf_remote.paths import CLOUD_STATE_FPATH from cf_remote.utils import read_json +DEFAULT_MAX_REPORT_HOSTS = 25 + + +def prompt_yes_no(prompt: str, default: bool = True) -> bool: + suffix = "[Y/n]" if default else "[y/N]" + answer = input(f"{prompt} {suffix} ").strip().lower() + if not answer: + return default + return answer in ("y", "yes") + def prompt_two_options(header: str, option_a: str, option_b: str) -> str: print(header) @@ -75,6 +87,22 @@ def _known_hosts(role_filter=None) -> Iterator[tuple[str, list[str]]]: yield host_id, aliases +@lru_cache(maxsize=None) +def _host_info(host: str): + try: + return get_info(host) or None + except (Exception, SystemExit) as e: + logging.warning(f"Skipping {host}: {e}") + return None + + +def _hosts_with_info(role_filter=None): + for host, aliases in _known_hosts(role_filter): + data = _host_info(host) + if data: + yield host, aliases, data + + def _find_all(binary_name: str) -> list[Executable]: """Every location -- local, plus every matching remote host -- with `binary_name` installed.""" executables = [] @@ -83,73 +111,26 @@ def _find_all(binary_name: str) -> list[Executable]: if local_path: executables.append(Executable(binary_name, "local", local_path)) - role_filter = "hub" if binary_name == "cf-hub" else None - key = "agent" if binary_name == "cf-agent" else "hub" - for host, aliases in _known_hosts(role_filter=role_filter): - try: - data = get_info(host) - except (Exception, SystemExit) as e: - """Need to catch SystemExit as cf-remote's get_info() will SystemExit if - any ssh-connections does not work, for our case we still want to fetch - the ones that are up in case the user wants to use a different host""" - logging.warning(f"Skipping {host}: {e}") - continue - if not data: - continue - binary_path = data.get(key) - if binary_path: - executables.append( - Executable(binary_name, host, binary_path, aliases=aliases) - ) - + is_agent = binary_name == "cf-agent" + for host, aliases, data in _hosts_with_info(None if is_agent else "hub"): + # band-aid: hostinfo has no path for cf-hub, so assume it's on PATH + path = data.get("agent") if is_agent else "cf-hub" + if path: + executables.append(Executable(binary_name, host, path, aliases)) return executables def _find_all_paired() -> list[Installation]: """Every location -- local or remote -- that has BOTH cf-agent and cf-hub.""" - installations = [] - - local_agent_path = _find_local_path("cf-agent") - local_hub_path = _find_local_path("cf-hub") - if local_agent_path and local_hub_path: - installations.append( - Installation( - location="local", - agent=Executable("cf-agent", "local", local_agent_path), - hub=Executable("cf-hub", "local", local_hub_path), - ) - ) - - for host, aliases in _known_hosts(role_filter="hub"): - try: - data = get_info(host) - except (Exception, SystemExit) as e: - # Same reasoning as _find_all() - logging.warning(f"Skipping {host}: {e}") - continue - if not data: - continue - agent_path = data.get("agent") - hub_path = data.get("hub") - if agent_path and hub_path: - installations.append( - Installation( - location=host, - agent=Executable("cf-agent", host, agent_path, aliases=aliases), - hub=Executable("cf-hub", host, hub_path, aliases=aliases), - ) - ) - - return installations + hubs = {e.location: e for e in _find_all("cf-hub")} + return [ + Installation(agent.location, agent, hubs[agent.location]) + for agent in _find_all("cf-agent") + if agent.location in hubs + ] def _prompt_choice(candidates, description): - if not sys.stdin.isatty(): - labels = ", ".join(c.label for c in candidates) - raise UserError( - f"Multiple installations of {description} found ({labels}) " - f"and no terminal to prompt on. Specify one with --host." - ) print(f"Multiple installations of {description} found:") for i, c in enumerate(candidates, 1): print(f" {i}) {c.label}") @@ -186,6 +167,14 @@ def _select(candidates, description, target: str | None = None): f"Could not find {description} locally or on any configured remote host." ) + if isinstance(target, list): + if len(target) > 1: + raise UserError( + f"Expected a single {description}, but got {len(target)}: " + f"{', '.join(target)}." + ) + target = target[0] if target else None + if target: matches = [c for c in candidates if _exact_match(c, target)] if not matches: @@ -213,9 +202,69 @@ def require_executable(name: str, target: str | None = None) -> Executable: return chosen -def require_installation(target: str | None = None) -> Installation: - chosen = _select(_find_all_paired(), "cf-agent + cf-hub", target) +def select_report_targets( + target: str | None = None, +) -> tuple[list[Installation], list[Executable]]: + """ + Decide which hosts `report()` should refresh new reporting data on. + + - `target` given -> just that one hub + - `target` omitted -> every known hub, and a random + sample of the remaining (non-hub) clients, capped so the + total (hubs + sampled hosts) doesn't exceed 25. + """ + installations = _find_all_paired() + if not installations: + raise UserError( + "Could not find any installation of cf-agent + cf-hub locally " + "or on any configured remote host." + ) + + if target: + hub_client_map = clients_by_hub_ip() + chosen_hub = _select(installations, "cf-agent + cf-hub", target) + hub_ip = None if chosen_hub.is_local else chosen_hub.location.split("@", 1)[1] + installations = [chosen_hub] + other_agents = [ + agent + for agent in _find_all("cf-agent") + if hub_ip + and any( + client_ip in agent.location + for client_ip in hub_client_map.get(hub_ip, []) + ) + ] + + else: + hub_locations = {installation.location for installation in installations} + other_agents = [ + agent + for agent in _find_all("cf-agent") + if agent.location not in hub_locations + ] + + cap = DEFAULT_MAX_REPORT_HOSTS + budget = max(0, cap - len(installations)) + if len(other_agents) <= budget: + return installations, other_agents + + sampled_agents = random.sample(other_agents, budget) logging.warning( - f"Using {'local' if chosen.is_local else 'remote'} installation of cf-agent and cf-hub ({chosen.label})" + f"{len(other_agents)} additional host(s) found; refreshing a random " + f"{budget} of them (plus {len(installations)} hub(s)) to keep this fast. " ) - return chosen + return installations, sampled_agents + + +def clients_by_hub_ip() -> dict[str, list[str]]: + """ + Maps each hub's IP (as every host reports it via its 'policy_server' + field) to the list of client IPs bootstrapped to it. + """ + mapping: dict[str, list[str]] = {} + for host, _, data in _hosts_with_info(): + policy_server = data.get("policy_server") + client_ip = data.get("ssh_host") or host.split("@", 1)[1] + if policy_server and policy_server != client_ip: + mapping.setdefault(policy_server, []).append(client_ip) + return mapping diff --git a/src/cfengine_cli/main.py b/src/cfengine_cli/main.py index e5db58d..00823b9 100644 --- a/src/cfengine_cli/main.py +++ b/src/cfengine_cli/main.py @@ -6,11 +6,21 @@ import subprocess from cf_remote import log +from cf_remote.main import resolve_hosts +from cf_remote.utils import is_package_url, strip_user from cfengine_cli.cfengine_wrapper import cfengine_commands +from cfengine_cli.cfengine_wrapper.arg_parse import parse_wrapper_args from cfengine_cli.version import cfengine_cli_version_string from cfengine_cli import commands from cfengine_cli.utils import UserError -from cf_remote.commands import spawn, list_boxes, list_platforms, init_cloud_config +from cf_remote.commands import ( + spawn, + list_boxes, + list_platforms, + init_cloud_config, + install, + uninstall, +) from cf_remote.spawn import CFRUserError, Providers from cfbs.utils import CFBSProgrammerError @@ -40,14 +50,15 @@ def _get_arg_parser(): % os.path.basename(sys.argv[0]) ) subp = ap.add_subparsers(dest="command", title=command_help_hint) + parse_wrapper_args( + subp + ) # The flags for run/report/spawn/destroy/...all wrapper functions subp.add_parser("help", help="Print help information") subp.add_parser( "version", help="Print the version string", ) - subp.add_parser("build", help="Build a policy set from a CFEngine Build project") - subp.add_parser("deploy", help="Deploy a built policy set") fmt = subp.add_parser("format", help="Autoformat .json and .cf files") fmt.add_argument("files", nargs="*", help="Files to format") fmt.add_argument("--line-length", default=80, type=int, help="Maximum line length") @@ -68,106 +79,6 @@ def _get_arg_parser(): help="Lint based on a user given syntax description", ) lnt.add_argument("files", nargs="*", help="Files to lint") - report_parser = subp.add_parser( - "report", - help="Run the agent and hub commands necessary to get new reporting data", - ) - report_parser.add_argument( - "--host", - type=str, - default=None, - help="Select which installation to use by name/IP (e.g. 'local' or '192.168.56.90'). " - "If omitted and multiple installations of cf-agent+cf-hub are found, you'll be prompted.", - ) - run_parser = subp.add_parser( - "run", help="Run the CFEngine agent, fetching, evaluating, and enforcing policy" - ) - run_parser.add_argument( - "run_args", - nargs="*", - help="Command(s) to run with cf-agent", - ) - run_parser.add_argument( - "--host", - type=str, - default=None, - help="Select which installation of cf-agent to use by name/IP (e.g. 'local' or '192.168.56.90'). " - "If omitted and multiple installations are found, you'll be prompted.", - ) - - sp = subp.add_parser("spawn", help="Spawn hosts in the clouds") - sp.add_argument( - "--list-platforms", help="List supported platforms", action="store_true" - ) - sp.add_argument( - "--list-boxes", help="List installed vagrant boxes", action="store_true" - ) - sp.add_argument( - "--init-config", - help="Initialize configuration file for spawn functionality", - action="store_true", - ) - sp.add_argument("--platform", help="Platform or vagrant box to use", type=str) - sp.add_argument("--count", default=1, help="How many hosts to spawn", type=int) - sp.add_argument( - "--role", help="Role of the hosts", choices=["hub", "hubs", "client", "clients"] - ) - sp.add_argument( - "--name", help="Name of the group of hosts (can be used in other commands)" - ) - sp.add_argument( - "--append", - help="Append the new VMs to a pre-existing group", - action="store_true", - ) - sp.add_argument( - "--provider", - help="VM provider", - type=str, - default="aws", - choices=["aws", "gcp", "vagrant"], - ) - sp.add_argument("--cpus", help="Number of CPUs of the vagrant instances", type=int) - sp.add_argument( - "--sync-folder", - help="Root folder of synchronized folders of vagrant instance", - type=str, - ) - sp.add_argument( - "--provision", - help="full path to provision shell script for Vagrant VM", - type=str, - ) - sp.add_argument("--size", help="Size/type of the instances", type=str) - sp.add_argument( - "--network", help="network/subnet to assign the VMs to (GCP only)", type=str - ) - sp.add_argument( - "--no-public-ip", - help="No public IP needed (GCP only; WARNING: The VMs will only be accessible" - + " from some other VM in the same cloud/network!)", - action="store_true", - ) - - dp = subp.add_parser("destroy", help="Destroy hosts spawned in the clouds") - dp.add_argument( - "--all", help="Destroy all hosts spawned in the clouds", action="store_true" - ) - dp.add_argument("name", help="Name of the group of hosts to destroy", nargs="?") - - profile_parser = subp.add_parser( - "profile", help="Parse CFEngine profiling output (cf-agent -Kp)" - ) - profile_parser.add_argument( - "profiling_input", help="Path to the profiling input file" - ) - profile_parser.add_argument("--top", type=int, default=10) - profile_parser.add_argument("--bundles", action="store_true") - profile_parser.add_argument("--promises", action="store_true") - profile_parser.add_argument("--functions", action="store_true") - profile_parser.add_argument( - "--flamegraph", type=str, help="Generate input file for ./flamegraph.pl" - ) dev_parser = subp.add_parser( "dev", help="Utilities intended for developers / maintainers of CFEngine" @@ -268,10 +179,12 @@ def run_command_with_args(args) -> int: if args.command == "version": return commands.version() # The real commands: + if args.command == "save": + return cfengine_commands.save(hosts=args.hosts, role=args.role, name=args.name) if args.command == "build": return cfengine_commands.build() if args.command == "deploy": - return cfengine_commands.deploy() + return cfengine_commands.deploy(args.hub, args.masterfiles) if args.command == "format": return commands.format(args.files, args.line_length, args.check) if args.command == "lint": @@ -281,7 +194,36 @@ def run_command_with_args(args) -> int: args.syntax_description, ) if args.command == "report": - return cfengine_commands.report(target=args.host) + return cfengine_commands.report( + target=args.hub, + run_agent=args.run_agent, + ) + if args.command == "setup-code": + return cfengine_commands.setup_code(target=args.hub) + if args.command == "install": + if args.trust_keys: + trust_keys = args.trust_keys.split(",") + else: + trust_keys = None + + return install( + args.hub, + args.clients, + package=args.package, + bootstrap=args.bootstrap, + hub_package=args.hub_package, + client_package=args.client_package, + version=args.version, + demo=args.demo, + call_collect=args.call_collect, + edition=args.edition, + remote_download=args.remote_download, + trust_keys=trust_keys, + insecure=args.insecure, + ) + elif args.command == "uninstall": + all_hosts = (args.hosts or []) + (args.hub or []) + (args.clients or []) + return uninstall(all_hosts, purge=args.purge) if args.command == "run": return cfengine_commands.run(*args.run_args, target=args.host) if args.command == "spawn": @@ -373,6 +315,57 @@ def validate_args(args): raise UserError( "Only one of '--all' or 'NAME' may be specified for destruction" ) + if args.command in ["install"]: # , "packages", "list", "download"]: + if args.edition: + args.edition = args.edition.lower() + if args.edition == "core": + args.edition = "community" + if args.edition not in ["enterprise", "community"]: + raise UserError("--edition must be either community or enterprise") + else: + args.edition = "enterprise" + + if "hosts" in args and args.hosts: + log.debug(f"validate_args, hosts in args, args.hosts='{args.hosts}'") + args.hosts = resolve_hosts(args.hosts) + if "clients" in args and args.clients: + args.clients = resolve_hosts(args.clients) + if "bootstrap" in args and args.bootstrap: + args.bootstrap = [ + strip_user(host_info) + for host_info in resolve_hosts(args.bootstrap, bootstrap_ips=True) + ] + if "hub" in args and args.hub: + log.debug(f"validate_args, hubs in args, args.hub='{args.hub}'") + args.hub = resolve_hosts(args.hub) + + if args.command in ["uninstall"] and not (args.hosts or args.hub or args.clients): + raise UserError("Use --hosts, --hub or --clients to specify remote hosts") + + if args.command == "install": + if args.call_collect and not args.demo: + raise UserError("--call-collect must be used with --demo") + if not args.clients and not args.hub: + raise UserError("Specify hosts using --hub and --clients") + if args.hub and args.clients and args.package: + raise UserError( + "Use --hub-package / --client-package instead to distinguish between hosts" + ) + if args.package and (args.hub_package or args.client_package): + raise UserError( + "--package cannot be used in combination with --hub-package / --client-package" + ) + if args.package and not is_package_url(args.package): + if not os.path.exists(os.path.expanduser(args.package)): + raise UserError("Package/directory '%s' does not exist" % args.package) + if args.hub_package and not is_package_url(args.hub_package): + if not os.path.isfile(args.hub_package): + raise UserError("Hub package '%s' does not exist" % args.hub_package) + if args.client_package and not is_package_url(args.client_package): + if not os.path.isfile(args.client_package): + raise UserError( + "Client package '%s' does not exist" % args.client_package + ) def _main():