This policy file handles Federated reporting setup and ongoing operations.

file bodies

control

Prototype: control

Implementation:

code
body file control
{
        namespace => "cfengine_enterprise_federation";
}

contain bodies

cfengine_enterprise_federation:cfpostgres_user

Prototype: cfengine_enterprise_federation:cfpostgres_user

Implementation:

code
body contain cfpostgres_user
{
  useshell   => "useshell";
  exec_owner => "cfpostgres";
  exec_group => "cfpostgres";
  chdir      => "/tmp";
  no_output  => "false";
}

cfengine_enterprise_federation:contain_transport_user

Prototype: cfengine_enterprise_federation:contain_transport_user

Implementation:

code
body contain contain_transport_user
{
    exec_owner => "$(cfengine_enterprise_federation:config.transport_user)";
    exec_group => "$(cfengine_enterprise_federation:config.transport_user)";
    chdir => "$(cfengine_enterprise_federation:config.transport_home)";
    useshell => "true";
}

classes bodies

cfengine_enterprise_federation:psql_wrapper_exit_codes

Prototype: cfengine_enterprise_federation:psql_wrapper_exit_codes

Implementation:

code
body classes psql_wrapper_exit_codes
{
  kept_returncodes => { "0" };
  repaired_returncodes => { "1" };
  failed_returncodes => { "2" };
}

file bodies

cfengine_enterprise_federation:control

Prototype: cfengine_enterprise_federation:control

Implementation:

code
body file control
{
        namespace => "default";
}

agent bundles

cfengine_enterprise_federation:config

Prototype: cfengine_enterprise_federation:config

Description: Read/parse config JSON, define variables and classes for use later

Implementation:

code
bundle agent config
{
  vars:
    enterprise_edition.(policy_server|am_policy_hub)::
      "federation_dir" string => "/opt/cfengine/federation";
      "bin_dir" string => "$(federation_dir)/bin";
      "path" string => "$(federation_dir)/cfapache/federation-config.json";
      "path_setup_status" string => "$(federation_dir)/cfapache/setup-status.json";

      "dump_interval" -> { "ENT-4806", "ENT-10900" }
        int => "20",
        if => not( isvariable( "cfengine_enterprise_federation:config.dump_interval" ) ),
        comment => "Dump data on the feeders every 20 minutes";

      # TODO: don't hard-code cftransport user
      "transport_user" -> { "ENT-4610" } string => "cftransport";
      "transport_home"
        string => "$(cfengine_enterprise_federation:config.federation_dir)/cftransport";

    config_present::
      "data" data => readjson( $(path) );

  classes:
    enterprise_edition.(policy_server|am_policy_hub)::
      "config_present"
        expression => fileexists( $(path) );
    config_present::
      "enabled" expression => or(strcmp("on", "$(data[target_state])"),
                                 strcmp("paused", "$(data[target_state])")),
        scope => "namespace";
      "am_off" expression => strcmp("off", "$(data[target_state])"),
        scope => "namespace";
      "am_on" expression => strcmp("on", "$(data[target_state])"),
        scope => "namespace";

      # _stdlib_path_exists_getenforce and paths.getenforce are defined by masterfiles/lib/paths.cf
      default:_stdlib_path_exists_getenforce::
        "selinux_enabled"
          expression => strcmp("Enforcing", execresult("$(default:paths.getenforce)", useshell)),
          scope => "namespace";


  vars:
    enabled::
      "role" string => "$(data[role])";
      "remotes" slist => getindices( @(data[remote_hubs]) );
      "login" string => ""; # default
      "login"
        string => "$(data[remote_hubs][$(remotes)][transport][ssh_user])@$(data[remote_hubs][$(remotes)][transport][ssh_host])",
            if => and(
                       # To ensure we are using a remote hub that's actually enabled
                       strcmp( "on", "$(data[remote_hubs][$(remotes)][target_state])" ),
                       # To ensure the remote we are pushing to actually needs the data (is a superhub)
                       strcmp( "superhub", "$(data[remote_hubs][$(remotes)][role])" ));

    am_superhub::
      # Public keys of enabled pushing feeders need to be trusted (on a superhub)
      "pubkey[$(remotes)]" string => "$(data[remote_hubs][$(remotes)][transport][ssh_pubkey])",
            if => and( strcmp( "on", "$(data[remote_hubs][$(remotes)][target_state])" ),
                       strcmp( "feeder", "$(data[remote_hubs][$(remotes)][role])" ),
                       strcmp( "push_over_rsync", "$(data[remote_hubs][$(remotes)][transport][mode])"));

    am_feeder::
      # List of superhub hostkeys for use with dump process
      "superhubs[$(remotes)]" string => "$(data[remote_hubs][$(remotes)][hostkey])",
          if => and( strcmp( "on", "$(data[remote_hubs][$(remotes)][target_state])" ),
                     strcmp( "superhub", "$(data[remote_hubs][$(remotes)][role])" ),
                     strcmp( "pull_over_rsync", "$(data[remote_hubs][$(remotes)][transport][mode])"));
      "superhub_hostkeys" string => join(" ", getvalues(superhubs));

      # Public key(s) of enabled pulling superhub(s) need(s) to be trusted (on a feeder)
      "pubkey[$(remotes)]" string => "$(data[remote_hubs][$(remotes)][transport][ssh_pubkey])",
            if => and( strcmp( "on", "$(data[remote_hubs][$(remotes)][target_state])" ),
                       strcmp( "superhub", "$(data[remote_hubs][$(remotes)][role])" ),
                       strcmp( "pull_over_rsync", "$(data[remote_hubs][$(remotes)][transport][mode])"));

    am_superhub|am_feeder::
      "pubkeys" slist => getvalues( pubkey );

      "fingerprint[$(data[remote_hubs][$(remotes)][transport][ssh_host])]"
        slist => string_split("$(data[remote_hubs][$(remotes)][transport][ssh_fingerprint])", "$(const.n)", "inf"),
        # To ensure we are using a remote hub that's enabled
        if => strcmp( "on", "$(data[remote_hubs][$(remotes)][target_state])" );
      "fingerprints" slist => maparray("$(this.k) $(this.v)", fingerprint);
      "feeder[$(remotes)]" string => "$(data[remote_hubs][$(remotes)][hostkey])",
        if => strcmp( "feeder", "$(data[remote_hubs][$(remotes)][role])" );

  classes:
    enabled::

      # Knowing if feeder or superhub is based on explicit setting of role in
      # path (federation-config.json)
      "am_feeder"
        expression => strcmp("feeder", "$(data[role])"),
        scope => "namespace";

      "am_superhub"
        expression => strcmp("superhub", "$(data[role])"),
        scope => "namespace";

      "am_pusher"
        and => {strcmp("superhub", "$(data[remote_hubs][$(remotes)][role])"),
                strcmp("on", "$(data[remote_hubs][$(remotes)][target_state])"),
                strcmp("push_over_rsync", "$(data[remote_hubs][$(remotes)][transport][mode])")},
        comment => "Has an enabled remote superhub with push as transport method, should run push transport",
        scope => "namespace";

      "am_puller"
        and => {"am_superhub",
                strcmp("on", "$(data[remote_hubs][$(remotes)][target_state])"),
                strcmp("pull_over_rsync", "$(data[remote_hubs][$(remotes)][transport][mode])")},
        comment => "Superhub with some enabled remote hub with pull as transport method, should run pull transport",
        scope => "namespace";

      "am_transporter"
        or => {"am_pusher", "am_puller"},
        scope => "namespace";

      "am_paused"
        expression => strcmp("paused", "$(data[target_state])"),
        scope => "namespace";

  # Note: in order to see these debugs you must either define the default DEBUG class
  # or the namespace prefixed class like:
  # cf-agent -KI -DDEBUG
  # or
  # cf-agent -KI -Dcfengine_enterprise_federation:DEBUG_config

  reports:
    enabled.(default:DEBUG|DEBUG_config)::
      "Federation enabled!";
    am_superhub.(default:DEBUG|DEBUG_config)::
      "I'm a superhub!";
    am_feeder.(default:DEBUG|DEBUG_config)::
      "I'm a feeder!";
    am_pusher.(default:DEBUG|DEBUG_config)::
      "I'm pushing dumps!";
    am_puller.(default:DEBUG|DEBUG_config)::
      "I'm pulling dumps!";
    am_transporter.(default:DEBUG|DEBUG_config)::
      "I'm a transporter!";
    am_paused.(default:DEBUG|DEBUG_config)::
      "I'm paused so won't do any import/dump";
}

cfengine_enterprise_federation:distributed_cleanup_dependencies

Prototype: cfengine_enterprise_federation:distributed_cleanup_dependencies

Description: warn if python3 and urllib3 required dependencies are not installed if cfengine_mp_fr_enable_distributed_cleanup class is defined Note: these requirements are only needed on superhub to run the distributed cleanup python script. on feeders only the shell script is run so no python dependencies needed there.

Implementation:

code
bundle agent distributed_cleanup_dependencies
{
  vars:
    debian|ubuntu|redhat_9|rocky_9::
      "packages" slist => { "python3", "python3-urllib3" };
    redhat_8|centos_8::
        "packages" slist => { "python36", "python3-urllib3" };

    redhat_7|centos_7::
      "packages" slist => { "python3" };

  classes:
    (redhat_6|centos_6)::
      "cfengine_mp_fr_distributed_cleanup_python3_installed"
        expression => returnszero(
          "$(sys.bindir)/cfengine-selected-python --version | grep -q ' 3.'",
          "useshell"
        );

    (redhat_6|centos_6|redhat_7|centos_7)::
      "cfengine_mp_fr_distributed_cleanup_urllib3_installed"
        expression => returnszero(
          "echo 'import urllib3' | $(sys.bindir)/cfengine-selected-python 2>/dev/null",
          "useshell"
        );

  packages:
    debian|ubuntu::
      "$(packages)"
        policy => "present",
        classes => default:results("bundle", "cfengine_mp_fr_distributed_cleanup_packages"),
        action => default:policy ( "warn" ),
        package_module => default:apt_get;

    redhat.!(centos_6|redhat_6)::
      "$(packages)"
        policy => "present",
        classes => default:results("bundle", "cfengine_mp_fr_distributed_cleanup_packages"),
        package_module => default:yum;

  reports:
    (redhat_6|centos_6).!cfengine_mp_fr_distributed_cleanup_python3_installed::
      "error: python3 is required for distributed cleanup utility. On this platform it is recommened you install python3 from source (https://docs.python.org/3.10/using/unix.html#building-python)";
    (redhat_6|centos_6).!cfengine_mp_fr_distributed_cleanup_urllib3_installed::
      "error: python3 module urllib3 is required for distributed cleanup utility. On this platform it is recommend you install via pip3 after installing python3 from source";
    (redhat_7|centos_7).!cfengine_mp_fr_distributed_cleanup_urllib3_installed::
      "error: python3 module urllib3 is required for distributed cleanup utility. On this platform please install with the command `pip3 install urllib3`";
}

cfengine_enterprise_federation:semanage_installed

Prototype: cfengine_enterprise_federation:semanage_installed

Description: Install semanage utility if selinux enabled and cfengine_mp_fr_dependencies_auto_install class is defined if not defined then only warn

Implementation:

code
bundle agent semanage_installed
{
  vars:
    "semanage_action"
      string => ifelse( "default:_stdlib_path_exists_semanage", "fix", "default:cfengine_mp_fr_dependencies_auto_install", "fix", "warn" ),
      comment => "We only want to use semanage if it's available, or if we have
                  indicated it's ok to install it automatically. This variable
                  is subsequently used by a commands and packages promises to
                  warn or fix based.";
    debian_6|debian_7|debian_8|ubuntu_12|ubuntu_14|ubuntu_16|rhel_5::
      "semanage_package" string => "policycoreutils";
    debian_9|debian_10|ubuntu_18|redhat_8|centos_8|redhat_9|rocky_9::
      "semanage_package" string => "policycoreutils-python-utils";
    redhat_6|centos_6|redhat_7|centos_7::
      "semanage_package" string => "policycoreutils-python";

  packages:
    debian|ubuntu::
      "$(semanage_package)"
        policy => "present",
        package_module => default:apt_get,
        action => default:policy ( $(semanage_action) );
    redhat::
      "$(semanage_package)"
        policy => "present",
        package_module => default:yum,
        action => default:policy ( $(semanage_action) );

  reports:
    default:DEBUG|DEBUG_semanage_installed::
      "paths.semanage = $(default:paths.semanage)";
    !default:_stdlib_path_exists_semanage.!default:cfengine_mp_fr_dependencies_auto_install::
      "semanage command is not available at $(default:paths.semanage). Will only install needed package if cfengine_mp_fr_dependencies_auto_install class is defined in augments(def.json) or with --define cf-agent option.";
}

cfengine_enterprise_federation:ssh_keygen

Prototype: cfengine_enterprise_federation:ssh_keygen(key_path)

Arguments:

  • key_path: string, used in the value of attribute args of commands promiser /usr/bin/ssh-keygen

Implementation:

code
bundle agent ssh_keygen(key_path)
{
  commands:
      "/usr/bin/ssh-keygen"
        handle => "ssh_keys_configured",
        args => "-N '' -f $(key_path)",
        if => not( fileexists( "$(key_path)" ));
}

cfengine_enterprise_federation:ssh_selinux_context

Prototype: cfengine_enterprise_federation:ssh_selinux_context(home, ssh_paths)

Arguments:

  • home: string, used to set promise attribute if of classes promiser cftransport_fcontext_missing of classes promiser incorrect_ssh_context, used as promiser of type commands , used as promiser of type commands of reports promiser need to fix incorrect ssh context for transport user but semanage path in $(sys.libdir)/paths.cf $(default:paths.semanage) does not resolve of reports promiser need to fix incorrect ssh context for transport user but restorecon path in $(sys.libdir)/paths.cf $(default:paths.restorecon) does not resolve
  • ssh_paths of classes promiser cftransport_fcontext_missing of classes promiser incorrect_ssh_context of commands promiser $(default:paths.semanage) fcontext -a -t ssh_home_t '$(home)/.ssh(/.)?'* of commands promiser $(default:paths.restorecon) -R -F $(home)/.ssh/ of reports promiser need to fix incorrect ssh context for transport user but semanage path in $(sys.libdir)/paths.cf $(default:paths.semanage) does not resolve of reports promiser need to fix incorrect ssh context for transport user but restorecon path in $(sys.libdir)/paths.cf $(default:paths.restorecon) does not resolve

Implementation:

code
bundle agent ssh_selinux_context(home, ssh_paths)
{
  classes:
    default:_stdlib_path_exists_semanage::
      "cftransport_fcontext_missing"
        expression => not(returnszero("$(default:paths.semanage) fcontext -l | grep '$(home)/.ssh(/.*)?'", "useshell")),
        if => fileexists("$(home)");

    any::
      # For all the files below it must be true that if they exist they need
      # to have the right context.
      # IOW, the following implication: if fileexists() then correct_context.
      # IOW, the following OR: not(filexists()) or correct_context.
      # not( and()) means that if for one of the files the implication is false, we get a true.
      "incorrect_ssh_context"
        expression => not( and(
                                or(
                                    not(fileexists("$(home)")),
                                    regcmp(".*[\s:]ssh_home_t[\s:].*",
                                           execresult("$(default:paths.ls) -dZ $(home)/.ssh", noshell))),
                                or(
                                    not(fileexists("$(ssh_paths[auth_keys])")),
                                    regcmp(".*[\s:]ssh_home_t[\s:].*",
                                           execresult("$(default:paths.ls) -Z $(ssh_paths[auth_keys])", noshell))),
                                or(
                                    not(fileexists("$(ssh_paths[priv_key])")),
                                    regcmp(".*[\s:]ssh_home_t[\s:].*",
                                           execresult("$(default:paths.ls) -Z $(ssh_paths[priv_key])", noshell))),
                                or(
                                    not(fileexists("$(ssh_paths[pub_key])")),
                                    regcmp(".*[\s:]ssh_home_t[\s:].*",
                                           execresult("$(default:paths.ls) -Z $(ssh_paths[pub_key])", noshell))),
                                or(
                                    not(fileexists("$(ssh_paths[config])")),
                                    regcmp(".*[\s:]ssh_home_t[\s:].*",
                                           execresult("$(default:paths.ls) -Z $(ssh_paths[config])", noshell)))
        ));
  commands:
    # _stdlib_path_exists_<command> and paths.<command> are defined is masterfiles/lib/paths.cf
    cftransport_fcontext_missing.default:_stdlib_path_exists_semanage::
      "$(default:paths.semanage) fcontext -a -t ssh_home_t '$(home)/.ssh(/.*)?'";
    incorrect_ssh_context.default:_stdlib_path_exists_restorecon::
      "$(default:paths.restorecon) -R -F $(home)/.ssh/";

  reports:
    incorrect_ssh_context.!default:_stdlib_path_exists_semanage::
      "need to fix incorrect ssh context for transport user but semanage path in $(sys.libdir)/paths.cf $(default:paths.semanage) does not resolve";
    incorrect_ssh_context.!default:_stdlib_path_exists_restorecon)::
      "need to fix incorrect ssh context for transport user but restorecon path in $(sys.libdir)/paths.cf $(default:paths.restorecon) does not resolve";
}

cfengine_enterprise_federation:transport_user

Prototype: cfengine_enterprise_federation:transport_user

Description: Manage transport user and permissions for remote SSH access

Implementation:

code
bundle agent transport_user
{
  vars:
      "user"
        string => "$(cfengine_enterprise_federation:config.transport_user)";
      "home"
        string => "$(cfengine_enterprise_federation:config.transport_home)";
      "ssh_key_name" string => "id_FR";
      "ssh_priv_key" string => "$(home)/.ssh/$(ssh_key_name)";
      "ssh_pub_key" string => "$(ssh_priv_key).pub";
      "ssh_auth_keys" string => "$(home)/.ssh/authorized_keys";
      "ssh_known_hosts" string => "$(home)/.ssh/known_hosts";
      "ssh_config" string => "$(home)/.ssh/config";
      "create_files"
        slist => {
          "$(home)/.",
          "$(home)/.ssh/.",
          "$(home)/source/.",      # Dumps from feeders are taken from here
          "$(home)/destination/.", # And dropped here on superhub
          "$(ssh_auth_keys)",
          "$(ssh_known_hosts)",
          "$(ssh_config)"
        };

      "ssh_paths" data => parsejson('{
        "key_name": "id_FR",
        "priv_key": "$(home)/.ssh/$(ssh_key_name)",
        "pub_key": "$(ssh_priv_key).pub",
        "auth_keys": "$(home)/.ssh/authorized_keys",
        "known_hosts": "$(home)/.ssh/known_hosts",
        "config": "$(home)/.ssh/config"
      }');

  users:
    "$(user)"
      policy => "present",
      home_dir => "$(home)";
  files:
    "$(create_files)"
      create => "true";

    "$(home)/." -> { "CFE-951" }
      depth_search => default:recurse_with_base("inf"),
      file_select => default:dirs,
      perms => default:mog( "700", $(user), "root" ),
      comment => "The transport users home directory and children should be accessible only by the transport user itself.";

    "$(home)/." -> { "CFE-951" }
      depth_search => default:recurse_with_base("inf"),
      file_select => default:not_dir,
      perms => default:mog( "600", $(user), "root" ),
      comment => "The files within the transport users home directory should be readable and writable by the transport user";

    "$(ssh_auth_keys)"
      create => "true",
      handle => "ssh_auth_keys_configured",
      edit_template_string => "}$(const.n)",
      template_data => @(cfengine_enterprise_federation:config.pubkeys),
      template_method => "inline_mustache";
    "$(ssh_known_hosts)"
      create => "true",
      handle => "ssh_known_hosts_configured",
      edit_template_string => "}$(const.n)",
      template_data => @(cfengine_enterprise_federation:config.fingerprints),
      template_method => "inline_mustache",
      if => isvariable("cfengine_enterprise_federation:config.fingerprints");
    "$(ssh_config)"
      create => "true",
      handle => "ssh_config_configured",
      edit_line => default:insert_lines("IdentityFile $(ssh_priv_key)");

  methods:
    selinux_enabled::
      "semanage_installed" usebundle => semanage_installed;
    enabled.selinux_enabled::
      # Ensure correct SElinux context
      "ssh_selinux_context" usebundle => ssh_selinux_context("$(home)", @(ssh_paths));
    enabled::
      # Generate ssh keypair
      "ssh_keygen" usebundle => ssh_keygen("$(ssh_priv_key)");
}

cfengine_enterprise_federation:clean_when_off

Prototype: cfengine_enterprise_federation:clean_when_off

Description: cleanup changes made for federated reporting on a feeder NOTE: a superhub turned off by removing federation-config.json or setting federation_manage_files will always run regardless of off or not so as to be prepared for enablement via Mission Portal UI

Implementation:

code
bundle agent clean_when_off
{
  vars:
    "user" string => "$(cfengine_enterprise_federation:transport_user.user)";
    "home" string => "$(cfengine_enterprise_federation:transport_user.home)";
@if minimum_version(3.15)
      "remote_hubs_table_row_count"
        string => execresult(`$(sys.bindir)/psql cfsettings --quiet --tuples-only --command "SELECT COUNT(*) FROM remote_hubs" 2>/dev/null`, useshell);
      "federated_reporting_settings_table_row_count"
        string => execresult(`$(sys.bindir)/psql cfsettings --quiet --tuples-only --command "SELECT COUNT(*) FROM federated_reporting_settings" 2>/dev/null`, useshell);
@endif

  users:
    "$(user)"
      policy => "absent";

  files:
      "$(cfengine_enterprise_federation:config.path_setup_status)" -> { "ENT-7233" }
        comment => "We must remove this file for Mission Portal to understand that the federation is not configured",
        delete => default:tidy;
      "$(cfengine_enterprise_federation:config.path)" -> { "ENT-7969" }
         comment => "We must remove this file for Mission Portal to understand that the federation is not configured",
         delete => default:tidy;

  methods:
    "rm_rf_cftransport_home_dir" usebundle => default:rm_rf("$(home)");

  classes:
    selinux_enabled.default:_stdlib_path_exists_semanage::
      "has_cftransport_fcontext" expression => returnszero("$(default:paths.semanage) fcontext -l | grep $(home)", "useshell");
    "remote_hubs_table_empty" expression => returnszero(`[ $(const.dollar)($(sys.bindir)/psql cfsettings --quiet --tuples-only --command "SELECT COUNT(*) FROM remote_hubs") -eq "0"]`, "useshell");
    "federated_reporting_settings_table_empty" expression => returnszero(`[ $(const.dollar)($(sys.bindir)/psql cfsettings --quiet --tuples-only --command "SELECT COUNT(*) FROM federated_reporting_settings") -eq "0"]`, "useshell");


  commands:
      # Oh, the humanity! Where for art thou databases: promises for psql!
      `$(sys.bindir)/psql cfsettings --quiet --command "TRUNCATE TABLE remote_hubs"` -> { "ENT-7233" }
        if => isgreaterthan( "$(remote_hubs_table_row_count)", "0" );

      `$(sys.bindir)/psql cfsettings --quiet --command "TRUNCATE TABLE federated_reporting_settings"` -> { "ENT-7233" }
        if => isgreaterthan( "$(federated_reporting_settings_table_row_count)", "0" );

      # _stdlib_path_exists_<command> and paths.<command> are defined in masterfiles/lib/paths.cf
    selinux_enabled.default:_stdlib_path_exists_semanage.has_cftransport_fcontext::
      "$(default:paths.semanage) fcontext -d '$(home)/.ssh(/.*)?'";

}

cfengine_enterprise_federation:federation_manage_files

Prototype: cfengine_enterprise_federation:federation_manage_files

Description: Manage files, directories and permissions in $(cfengine_enterprise_federation:config.federation_dir)

By default the import process will not prohibit the inclusion of duplicate hostkey data from feeders. By defining the cfengine_mp_fr_handle_duplicate_hostkeys class in augments a step will be performed during import which will find the which feeder's data is most recent for each duplicate hostkey and use that data. Duplicate hostkey data will be moved to a dup schema for analysis.

This class only applies to superhubs.

code
{
  "classes": {
    "cfengine_mp_fr_handle_duplicate_hostkeys": [ "any::" ]
  }
}

Implementation:

code
bundle agent federation_manage_files
{

  vars:
    "transport_user"
      string => "$(cfengine_enterprise_federation:config.transport_user)";
    "login" data => parsejson('{"login":"$(cfengine_enterprise_federation:config.login)"}');
    "feeder_username" data => parsejson('{"feeder_username":"$(cfengine_enterprise_federation:config.transport_user)"}');
    "superhub_hostkeys" string => ifelse( isvariable("cfengine_enterprise_federation:config.superhub_hostkeys"),
                                        "$(cfengine_enterprise_federation:config.superhub_hostkeys)",
                                        "" );
    "this_hostkey" data => parsejson('{"this_hostkey":"$(default:sys.key_digest)"}');
    "feeder" data => parsejson('{"feeder": "$(sys.key_digest)"}');
    "cf_version" data => parsejson('{"cf_version":"$(sys.cf_version)"}');
    "workdir" data => parsejson('{"workdir":"$(sys.workdir)"}');
    "handle_duplicates_value" string => ifelse("default:cfengine_mp_fr_handle_duplicate_hostkeys", "yes", "no");
    "handle_duplicates" data => parsejson('{"handle_duplicates":"$(handle_duplicates_value)"}');
    "debug_import_value" string => ifelse("default:cfengine_mp_fr_debug_import", "yes", "no");
    "debug_import" data => parsejson('{"debug_import":"$(debug_import_value)"}');

  files:
    enterprise_edition.(policy_server|am_policy_hub)::
      # Both cfpache and $(transport_user) need permission so adding o+x here
      "$(cfengine_enterprise_federation:config.federation_dir)/." -> { "CFE-951" }
        create => "true",
        perms => default:mog( "755", "root", "cfapache" );

      "$(cfengine_enterprise_federation:config.federation_dir)/cfapache/."
        create => "true",
        perms => default:mog( "700", "cfapache", "root" );

      "$(cfengine_enterprise_federation:config.federation_dir)/cfapache/." -> { "CFE-951" }
        depth_search => default:recurse_with_base("inf"),
        file_select => default:dirs,
        perms => default:mog( "700", "cfapache", "root" ),
        comment => "The cfapache home directory and children need to be accessible by the web-server";

      "$(cfengine_enterprise_federation:config.federation_dir)/cfapache/." -> { "CFE-951" }
        depth_search => default:recurse_with_base("inf"),
        file_select => default:not_dir,
        perms => default:mog( "600", "cfapache", "root" ),
        comment => "Files within the cfapache home directory need to be readable and writable by the web server.";

    enabled::
      "$(cfengine_enterprise_federation:config.bin_dir)/." -> { "CFE-951" }
        create => "true",
        perms => default:mog( "0770", "root", "$(transport_user)" );

    am_superhub::
      "$(cfengine_enterprise_federation:config.federation_dir)/superhub/."
        create => "true",
        perms => default:mog( "770", "root", "$(transport_user)" );
      "$(cfengine_enterprise_federation:config.federation_dir)/superhub/import/."
        create => "true",
        perms => default:mog( "600", "root", "root" );
      "$(cfengine_enterprise_federation:config.federation_dir)/superhub/import/filters/."
        create => "true",
        perms => default:mog( "600", "root", "root" );
    am_feeder::
      "$(cfengine_enterprise_federation:config.federation_dir)/fedhub/."
        create => "true",
        perms => default:mog( "660", "root", "$(transport_user)" );
      "$(cfengine_enterprise_federation:config.federation_dir)/fedhub/dump/."
        create => "true",
        perms => default:mog( "660", "root", "$(transport_user)" );
      "$(cfengine_enterprise_federation:config.federation_dir)/fedhub/transport/."
        create => "true",
        perms => default:mog( "660", "root", "$(transport_user)" );
      "$(cfengine_enterprise_federation:config.federation_dir)/fedhub/dump/filters/."
        create => "true",
        perms => default:mog( "600", "root", "root" );

    am_feeder|am_transporter|am_superhub::
      # TODO: Instrument augments
      "$(cfengine_enterprise_federation:config.bin_dir)/config.sh"
        create => "true",
        template_method => "mustache",
        edit_template => "$(this.promise_dirname)/../../../templates/federated_reporting/config.sh.mustache",
        template_data => mergedata(@(login),
                                   @(feeder_username),
                                   @(feeder),
                                   parsejson('{"superhub_hostkeys": "$(superhub_hostkeys)"}'),
                                   @(debug_import),
                                   @(this_hostkey),
                                   @(cf_version),
                                   @(handle_duplicates),
                                   parsejson('{"inventory_refresh_cmd": ""}')),
        perms => default:mog( "640", "root", "$(transport_user)" );

      # TODO: Instrument augments
      "$(cfengine_enterprise_federation:config.bin_dir)/log.sh"
        create => "true",
        template_method => "mustache",
        edit_template => "$(this.promise_dirname)/../../../templates/federated_reporting/log.sh.mustache",
        perms => default:mog( "640", "root", "$(transport_user)" );

      "$(cfengine_enterprise_federation:config.bin_dir)/parallel.sh"
        copy_from => default:local_dcp( "$(this.promise_dirname)/../../../templates/federated_reporting/parallel.sh" ),
        perms => default:mog( "640", "root", "$(transport_user)" );

      "$(cfengine_enterprise_federation:config.bin_dir)/psql_wrapper.sh" -> { "ENT-4792"}
        create => "true",
        edit_template => "$(this.promise_dirname)/../../../templates/federated_reporting/psql_wrapper.sh.mustache",
        template_method => "mustache",
        perms => default:mog( "700", "root", "root" );

    am_feeder::
      "$(cfengine_enterprise_federation:config.bin_dir)/dump.sh"
        copy_from => default:local_dcp( "$(this.promise_dirname)/../../../templates/federated_reporting/dump.sh" ),
        perms => default:mog( "700", "root", "root" );

      "$(cfengine_enterprise_federation:config.federation_dir)/fedhub/dump/filters/50-merge_inserts.awk"
        copy_from => default:local_dcp( "$(this.promise_dirname)/../../../templates/federated_reporting/50-merge_inserts.awk" ),
        perms => default:mog( "600", "root", "root" );

    am_transporter::
      "$(cfengine_enterprise_federation:config.bin_dir)/transport.sh" -> { "CFE-951" }
        copy_from => default:local_dcp( "$(this.promise_dirname)/../../../templates/federated_reporting/transport.sh" ),
        perms => default:mog( "500", "$(transport_user)", "root" );

    am_puller::
      "$(cfengine_enterprise_federation:config.bin_dir)/pull_dumps_from.sh"
        copy_from => default:local_dcp( "$(this.promise_dirname)/../../../templates/federated_reporting/pull_dumps_from.sh" ),
        perms => default:mog( "500", "$(transport_user)", "root" );

    am_superhub::
      "$(cfengine_enterprise_federation:config.bin_dir)/import.sh"
        copy_from => default:local_dcp( "$(this.promise_dirname)/../../../templates/federated_reporting/import.sh" ),
        perms => default:mog( "700", "root", "root" );

      "$(cfengine_enterprise_federation:config.bin_dir)/import_file.sh"
        copy_from => default:local_dcp( "$(this.promise_dirname)/../../../templates/federated_reporting/import_file.sh" ),
        perms => default:mog( "700", "root", "root" );

      "$(cfengine_enterprise_federation:config.federation_dir)/superhub/import/filters/10-base_filter.sed"
        copy_from => default:local_dcp( "$(this.promise_dirname)/../../../templates/federated_reporting/10-base_filter.sed" ),
        perms => default:mog( "600", "root", "root" );

    am_superhub.default:cfengine_mp_fr_enable_distributed_cleanup::
      "$(cfengine_enterprise_federation:config.bin_dir)/transfer_distributed_cleanup_items.sh"
        copy_from => default:local_dcp( "$(this.promise_dirname)/../../../templates/federated_reporting/transfer_distributed_cleanup_items.sh" ),
        perms => default:mog( "500", "$(transport_user)", "root" );
      "$(cfengine_enterprise_federation:config.bin_dir)/distributed_cleanup.py"
        copy_from => default:local_dcp( "$(this.promise_dirname)/../../../templates/federated_reporting/distributed_cleanup.py" ),
        perms => default:mog( "500", "root", "root" );
      "$(cfengine_enterprise_federation:config.bin_dir)/nova_api.py"
        copy_from => default:local_dcp( "$(this.promise_dirname)/../../../templates/federated_reporting/nova_api.py" ),
        perms => default:mog( "500", "root", "root" );
      "$(cfengine_enterprise_federation:config.bin_dir)/cfsecret.py"
        copy_from => default:local_dcp( "$(this.promise_dirname)/../../../templates/federated_reporting/cfsecret.py" ),
        perms => default:mog( "500", "root", "root" );
}

cfengine_enterprise_federation:postgres_config

Prototype: cfengine_enterprise_federation:postgres_config

Description: Customize postgres config for superhub

Implementation:

code
bundle agent postgres_config
{
  vars:
    am_superhub::
      "c[shared_buffers]" -> { "ENT-8617" }
        string => ifelse( isvariable( "cfengine_enterprise_federation:postgres_config.shared_buffers"),
                          $(cfengine_enterprise_federation:postgres_config.shared_buffers),
                          "1GB"),
        comment => "Changing this setting requires restarting the database.";
      "c[max_locks_per_transaction]" -> { "ENT-8617" }
        string => ifelse( isvariable( "cfengine_enterprise_federation:postgres_config.max_locks_per_transaction"),
                          $(cfengine_enterprise_federation:postgres_config.max_locks_per_transaction),
                          "4000"),
        comment => "Changing this setting requires restarting the database.";
      "c[log_lock_waits]" -> { "ENT-8617" }
        string => ifelse( isvariable( "cfengine_enterprise_federation:postgres_config.log_lock_waits"),
                          $(cfengine_enterprise_federation:postgres_config.log_lock_waits),
                          "on"),
        comment => "Changing this setting requires restarting the database.";

      "c[max_wal_size]" -> { "ENT-8617" }
        string => ifelse( isvariable( "cfengine_enterprise_federation:postgres_config.max_wal_size"),
                          $(cfengine_enterprise_federation:postgres_config.max_wal_size),
                          "1GB");

      "c[checkpoint_timeout]" -> { "ENT-8617" }
        string => ifelse( isvariable( "cfengine_enterprise_federation:postgres_config.checkpoint_timeout"),
                          $(cfengine_enterprise_federation:postgres_config.checkpoint_timeout),
                          "5min");

  files:
    am_superhub::
      "$(sys.statedir)/pg/data/postgresql.conf"
        edit_line => default:set_line_based( "$(this.namespace):$(this.bundle).c",
                                     "=",
                                     "\s*=\s*",
                                     ".*",
                                     ""),
        classes => default:results( "bundle", "postgresql_conf" ),
        if => fileexists( "$(sys.statedir)/pg/data/postgresql.conf" );

  commands:
    am_superhub.postgresql_conf_repaired.!systemd::
      # smart mode tries to wait for operations to finish and clients to
      # disconnect, fast mode terminates open connections gracefully
      "$(sys.bindir)/pg_ctl --pgdata $(sys.statedir)/pg/data --log /var/log/postgresql.log --wait --mode smart restart ||
       $(sys.bindir)/pg_ctl --pgdata $(sys.statedir)/pg/data --log /var/log/postgresql.log --wait --mode fast  restart"
        contain => cfpostgres_user;

  services:
    am_superhub.postgresql_conf_repaired.systemd::
      "cf-postgres"
        service_method => default:standard_services,
        service_policy => "restart";
}

cfengine_enterprise_federation:exported_data

Prototype: cfengine_enterprise_federation:exported_data

Description: Run script to dump pg data on feeder hub

Implementation:

code
bundle agent exported_data
{
  methods:
    am_feeder.!am_paused::
      "Refresh Inventory"
        usebundle => "default:cfe_internal_refresh_inventory_view",
        handle => "fr_inventory_refresh",
        comment => "Use standard inventory refresh so that we don't run it twice";

  commands:
    am_feeder.!am_paused::
      "/bin/bash"
        arglist => {"$(cfengine_enterprise_federation:config.bin_dir)/dump.sh"},
        contain => default:in_shell,
        depends_on => { "fr_inventory_refresh" },
        comment => "Refresh Inventory must be completed before dumping data";
}

cfengine_enterprise_federation:data_transport

Prototype: cfengine_enterprise_federation:data_transport

Description: Run script to transport data from feeder to superhub

Implementation:

code
bundle agent data_transport
{
  vars:
    am_puller.!am_paused::
      # local copies of the variables to make using them below sane
      "remotes" slist => {@(cfengine_enterprise_federation:config.remotes)};
      "data"    data  => @(cfengine_enterprise_federation:config.data);

      "enabled_pull_hosts[$(remotes)]"
        string => "$(data[remote_hubs][$(remotes)][transport][ssh_host])",
        if => and(strcmp("on", "$(data[remote_hubs][$(remotes)][target_state])"),
                  strcmp("pull_over_rsync", "$(data[remote_hubs][$(remotes)][transport][mode])"));

      "pull_args" -> {"ENT-4499"}
        string => join(" ", getvalues(@(enabled_pull_hosts)));

  commands:
    am_pusher.!am_paused::
      "/bin/bash"
        arglist => {"$(cfengine_enterprise_federation:config.bin_dir)/transport.sh push"},
        contain => contain_transport_user;

    am_puller.!am_paused::
      "/bin/bash"
        arglist => {"$(cfengine_enterprise_federation:config.bin_dir)/transport.sh pull $(pull_args)"},
        contain => contain_transport_user;

    am_puller.!am_paused.default:cfengine_mp_fr_enable_distributed_cleanup::
      "/bin/bash"
        arglist => {"$(cfengine_enterprise_federation:config.bin_dir)/transfer_distributed_cleanup_items.sh $(pull_args)"},
        contain => contain_transport_user;
}

cfengine_enterprise_federation:imported_data

Prototype: cfengine_enterprise_federation:imported_data

Description: Run script to import dumps on superhub

Implementation:

code
bundle agent imported_data
{
  commands:
      "/bin/bash"
        arglist => {"$(cfengine_enterprise_federation:config.bin_dir)/import.sh"},
        contain => default:in_shell;
}

cfengine_enterprise_federation:superhub_schema

Prototype: cfengine_enterprise_federation:superhub_schema

Description: Run SQL script to ensure schema is migrated to superhub partitioned tables architecture

Implementation:

code
bundle agent superhub_schema
{
  commands:
    am_superhub::
      "$(cfengine_enterprise_federation:config.bin_dir)/psql_wrapper.sh"
        arglist => {
                     "cfdb",
                     `"select superhub_schema('$(sys.key_digest)');"`,
                   },
        classes => psql_wrapper_exit_codes;
}

cfengine_enterprise_federation:ensure_feeders

Prototype: cfengine_enterprise_federation:ensure_feeders

Description: Run SQL function to ensure that all configured feeder hubs are in __hubs table

Implementation:

code
bundle agent ensure_feeders
{
  vars:
    am_superhub::
      "feeders" slist => getvalues( "cfengine_enterprise_federation:config.feeder");
      "feeders_arg" string => concat( "ARRAY['", join( "', '", feeders ), "']");

  commands:
    am_superhub::
      "$(cfengine_enterprise_federation:config.bin_dir)/psql_wrapper.sh"
        arglist => {
                     "cfdb",
                     `"select ensure_feeders($(feeders_arg));"`
                   },
        classes => psql_wrapper_exit_codes,
        if => isgreaterthan(length(feeders), 0);
}

cfengine_enterprise_federation:entry

Prototype: cfengine_enterprise_federation:entry

Description: Conditionally runs all federated reporting bundles

Implementation:

code
bundle agent entry
{
  meta:
    (policy_server|am_policy_hub).enterprise_edition::
      "tags" -> { "ENT-4383" }
        slist => { "enterprise_maintenance" };
  classes:
    enterprise_edition.(policy_server|am_policy_hub)::
      "config_exists"
        expression => fileexists("$(cfengine_enterprise_federation:config.federation_dir)/cfapache/federation-config.json");
    enterprise_edition.(policy_server|am_policy_hub)::
      "config_not_exists"
        expression => not(fileexists("$(cfengine_enterprise_federation:config.federation_dir)/cfapache/federation-config.json"));

  methods:
    config_exists::
      "CFEngine Enterprise Federation Configuration"
        handle => "config",
        usebundle => config;
    am_policy_hub.(am_off|config_not_exists)::
      "CFEngine Enterprise Federation Transport Off"
        handle => "clean_when_off",
        usebundle => clean_when_off,
        if => cf_version_minimum("3.15");
    enabled.am_on::
      "CFEngine Enterprise Federation Transport User"
        handle => "transport_user",
        usebundle => transport_user;
    enterprise_edition.(policy_server|am_policy_hub)::
      "federation_manage_files"
        handle => "federation_manage_files",
        usebundle => federation_manage_files;
    enabled.am_on::
      "CFEngine Enterprise Federation Postgres Configuration"
        handle => "postgres_config",
        usebundle => postgres_config;
      "CFEngine Enterprise Federation Schema Migration"
        handle => "superhub_schema",
        depends_on => { "postgres_config" },
        usebundle => superhub_schema;
      "CFEngine Enterprise Federation Ensure Feeder Hubs in Database"
        handle => "ensure_feeders",
        depends_on => { "superhub_schema" },
        usebundle => ensure_feeders;
      "CFEngine Enterprise Federation Feeder Data Transport"
        handle => "data_transport",
        depends_on => { "transport_user" },
        usebundle => data_transport;
      "CFEngine Enterprise Federation Feeder Data Export"
        usebundle => exported_data,
        action => default:if_elapsed($(cfengine_enterprise_federation:config.dump_interval));
      "Configuration Status"
        usebundle => setup_status;

    enabled.am_on.am_superhub.!am_paused::
      "CFEngine Enterprise Federation Feeder Data Import"
        handle => "imported_data",
        depends_on => { "transport_user", "ensure_feeders" },
        usebundle => imported_data;

    am_policy_hub.default:cfengine_mp_fr_enable_distributed_cleanup::
      "Distributed Cleanup Dependencies"
        handle => "distributed_cleanup_dependencies",
        if => "enabled.am_on.am_superhub.!am_paused",
        usebundle => "distributed_cleanup_dependencies";
      "Distributed Cleanup Setup"
        handle => "distributed_cleanup_setup",
        depends_on => { "transport_user", "data_transport" },
        usebundle => "distributed_cleanup_setup";
      "Distributed Federated Host Cleanup"
        handle => "distributed_cleanup",
        if => "enabled.am_on.am_superhub.!am_paused",
        depends_on => { "imported_data", "distributed_cleanup_setup", "distributed_cleanup_dependencies" },
        usebundle => distributed_cleanup_run;

  reports:
    !enterprise_edition::
      "Federated reporting is only available in CFEngine Enterprise.";
    enterprise_edition.!(policy_server|am_policy_hub)::
      "Federated reporting is only available on the policy server / hub.";
}

cfengine_enterprise_federation:setup_status

Prototype: cfengine_enterprise_federation:setup_status

Implementation:

code
bundle agent setup_status
{
  vars:
    "role" string => "$(cfengine_enterprise_federation:config.role)";
    "ssh_pub_key"
      string => readfile( "$(cfengine_enterprise_federation:transport_user.ssh_pub_key)" ),
      if => fileexists( "$(cfengine_enterprise_federation:transport_user.ssh_pub_key)" );
    "ssh_server_fingerprint"
      # ssh-keyscan is used because it's more reliable/easy than trying to
      # parse sshd config to find the file and then readfile():
      string => execresult("ssh-keyscan localhost 2>/dev/null | sed 's/localhost //g' | sort", useshell);
  classes:
    "superhub_setup_status_complete"
      expression => "any",
      depends_on => {
                      "config",
                      "transport_user",
                      "postgres_config", # We are depending on a deep guard within this bundle
                      "federation_manage_files",
      };

  files:
    superhub_setup_status_complete::
      "$(cfengine_enterprise_federation:config.path_setup_status)"
        create => "true",
        perms => default:mog( "600", "cfapache", "root" ),
        template_method => "inline_mustache",
        edit_template_string => "$(const.n)",
        template_data => '{
          "configured": true,
          "role": "$(role)",
          "hostkey": "$(sys.key_digest)",
          "transport_ssh_public_key": "$(ssh_pub_key)",
          "transport_ssh_server_fingerprint": "$(ssh_server_fingerprint)",
        }',
        if => isvariable( ssh_pub_key );
}

cfengine_enterprise_federation:distributed_cleanup_setup

Prototype: cfengine_enterprise_federation:distributed_cleanup_setup

Implementation:

code
bundle agent distributed_cleanup_setup
{
  vars:
    "distributed_cleanup_dir" string => "/opt/cfengine/federation/cftransport/distributed_cleanup";

  files:
    am_superhub|am_feeder::
      "${distributed_cleanup_dir}/."
        perms => default:mog( "700", "$(cfengine_enterprise_federation:config.transport_user)", "root" ),
        create => "true";
      "${distributed_cleanup_dir}/${sys.fqhost}.pub"
        perms => default:mog( "600", "$(cfengine_enterprise_federation:config.transport_user)", "root" ),
        copy_from => default:local_cp("${sys.workdir}/ppkeys/localhost.pub");
      "${distributed_cleanup_dir}/${sys.fqhost}.cert"
        perms => default:mog( "600", "$(cfengine_enterprise_federation:config.transport_user)", "root" ),
        copy_from => default:local_cp("${sys.workdir}/httpd/ssl/certs/${sys.fqhost}.cert");
}

cfengine_enterprise_federation:distributed_cleanup_run

Prototype: cfengine_enterprise_federation:distributed_cleanup_run

Implementation:

code
bundle agent distributed_cleanup_run
{
  vars:
    "_arglist" slist => { ifelse("debug_mode", "--debug",
                                 "inform_mode",  "--inform",
                                 "") };
  commands:
    am_superhub.!am_paused.default:cfengine_mp_fr_enable_distributed_cleanup::
      "$(sys.bindir)/cfengine-selected-python"
        args => "$(cfengine_enterprise_federation:config.bin_dir)/distributed_cleanup.py",
        arglist => { @(_arglist) },
        unless => isvariable( "default:def.DISTRIBUTED_CLEANUP_SSL_CERT_DIR" );

      "SSL_CERT_DIR=$(default:def.DISTRIBUTED_CLEANUP_SSL_CERT_DIR) $(sys.bindir)/cfengine-selected-python" -> { "ENT-8477", "ENT-8464" }
        args => "$(cfengine_enterprise_federation:config.bin_dir)/distributed_cleanup.py",
        arglist => { @(_arglist) },
        if => isvariable( "default:def.DISTRIBUTED_CLEANUP_SSL_CERT_DIR"),
        contain => default:in_shell,
        comment => "When custom SSL certificates are used from the non default location we need to let the script know where to find them.";

}

main

Prototype: __main__

Description: You can run this policy file from shell without specifying bundle

Implementation:

code
bundle agent __main__
{
  methods:
    "entry" usebundle => cfengine_enterprise_federation:entry;
}