cfe_internal/update/update_policy.cf

bundle agent cfe_internal_update_policy
{
  classes:

      # Define classes if we see a user is requesting a custom policy update bundle

      "have_user_specified_update_bundle"
        expression => isvariable( "def.mpf_update_policy_bundle" );

      # Define classes if we are able to find the specific bundle they requested
      # (otherwise we may get an error about undefined bundle)

      "have_found_user_specified_update_bundle"
        expression  => some(".*", "found_matching_user_specified_bundle");

      "missing_user_specified_update_bundle"
       not  => some(".*", "found_matching_user_specified_bundle");



  vars:
      "default_policy_update_bundle" string => "cfe_internal_update_policy_cpv";

      # Look for a bundle that matches what the user wants
      "found_matching_user_specified_bundle"
        slist => bundlesmatching( "$(def.mpf_update_policy_bundle)" );

  methods:

    # Use the user specified bundle when it's found
    have_found_user_specified_update_bundle::

        "User specified policy update bundle"
          usebundle => $(found_matching_user_specified_bundle);


    # Fall back to stock policy update bundle if we have not found one
    # specified by user

    !have_found_user_specified_update_bundle::

      "Stock policy update"
        usebundle => cfe_internal_update_policy_cpv;

    any::
      "CMDB data update" -> { "ENT-6788" }
        usebundle => cfe_internal_update_cmdb;

  reports:

    inform_mode|verbose_mode|DEBUG|DEBUG_cfe_internal_update_policy::
      # Report a human readable way to understand the policy behavior

      "Found user specified update bundle."
        if => "have_user_specified_update_bundle";

      "User specified update bundle: $(def.mpf_update_policy_bundle)"
        if => "have_user_specified_update_bundle";

      "User specified update bundle MISSING! Falling back to $(default_policy_update_bundle)."
        if => and( "have_user_specified_update_bundle",
                   "missing_user_specified_update_bundle"
                  );


}

bundle agent cfe_internal_setup_python_symlink(symlink_path)
{
  vars:
      "path" string => getenv("PATH", 1024);
      "path_folders" slist => filter("$(sys.bindir)",
                                     splitstring("$(path)", ":", 128),
                                     false, true, 128);

    windows::
      "abs_path_folders" -> {"CFE-2309"}
        slist => filter("([A-Z]|//):.*", path_folders, "true", "false", 128),
        comment => "findfiles() complains about relative directories";

    !windows::
      "abs_path_folders" -> {"CFE-2309"}
        slist => filter("/.*", path_folders, "true", "false", 128),
        comment => "findfiles() complains about relative directories";

    any::
      "exact_version_globs" slist => maplist("$(this)/python[23]", @(abs_path_folders)),
        comment => "Looking for Python 2 and/or Python 3 in the $PATH folders";

      "generic_python_globs" slist => maplist("$(this)/python", @(abs_path_folders)),
        comment => "Looking for the 'python' symlink/executable which can be any
                    version of Python (usually Python 2 for backwards compatibility)";

      "python_exact[$(exact_version_globs)]" slist => findfiles("$(exact_version_globs)");
      "python_generic[$(generic_python_globs)]" slist => findfiles("$(generic_python_globs)");
      "python_platform_fallback[/usr/libexec/platform-python]" -> { "CFE-3291" }
        slist => { "/usr/libexec/platform-python" };

      "python_exact_sorted" slist => reverse(sort(getvalues(@(python_exact)), "lex")),
        comment => "Prefer higher major versions of Python";

      "pythons" slist => getvalues(mergedata(@(python_exact_sorted),
                                             getvalues(@(python_generic)),
                                             getvalues(@(python_platform_fallback)))),
        comment => "Prefer exact versions over unknown";

      "python" string => nth(@(pythons), 0),
        if => isgreaterthan(length(@(pythons)), 0),
        comment => "Taking the first item from the list (sorted by preference)";

  files:
      "$(symlink_path)"
        delete => u_tidy,
        if => not(isvariable("python"));

      "$(symlink_path)"
        link_from => u_ln_s("$(python)"),
        move_obstructions => "true",
        if => isvariable("python");

      "$(sys.bindir)/python" -> { "CFE-3512" }
        delete => u_tidy,
        if => islink( "$(sys.bindir)/python" ),
        comment => concat( "We don't want to leave a python that is potentially in $PATH ",
                           "after having re-named our python symlink that is used for various ",
                           "modules");

}

bundle agent cfe_internal_update_policy_cpv
{
  vars:
      "inputs_dir"         string => translatepath("$(sys.inputdir)"),
      comment => "Directory containing CFEngine policies",
      handle => "cfe_internal_update_policy_vars_inputs_dir";

      "master_location" -> { "ENT-3692" }
        string => "$(update_def.mpf_update_policy_master_location)",
        comment => "The path to request updates from the policy server.",
        handle => "cfe_internal_update_policy_vars_master_location";

    windows::

      "modules_dir_source"        string => "/var/cfengine/masterfiles/modules",
      comment => "Directory containing CFEngine modules",
      handle => "cfe_internal_update_policy_vars_modules_dir_windows";

    !windows::

      "modules_dir_source"        string => translatepath("$(master_location)/modules"),
      comment => "Directory containing CFEngine modules",
      handle => "cfe_internal_update_policy_vars_modules_dir";

    any::

      "file_check"         string => translatepath("$(inputs_dir)/promises.cf"),
      comment => "Path to a policy file",
      handle => "cfe_internal_update_vars_file_check";

      "ppkeys_file"        string => translatepath("$(sys.workdir)/ppkeys/localhost.pub"),
      comment => "Path to public key file",
      handle => "cfe_internal_update_policy_vars_ppkeys_file";

      "postgresdb_dir"        string => "$(sys.workdir)/state/pg/data",
      comment => "Directory where Postgres database files will be stored on hub -",
      handle => "cfe_internal_update_policy_postgresdb_dir";

      "postgresdb_log"        string => "/var/log/postgresql.log",
      comment => "File where Postgres database files will be logging -",
        handle => "cfe_internal_update_policy_postgresdb_log_file";

      "python_symlink" -> { "CFE-2602", "CFE-3512" }
        string => "$(sys.bindir)/cfengine-selected-python",
        comment => "Symlink to Python we found (if any)",
        handle => "cfe_internal_update_policy_python_symlink";

    cfredis_in_enterprise::

      # TODO Remove from MPF after 3.12 EOL

      "redis_conf_file" -> { "ENT-2797" }
        string => translatepath("$(sys.workdir)/config/redis.conf"),
        comment => "Path to Redis configuration file",
        handle => "cfe_internal_update_policy_redis_conf_file";

  classes:

      "validated_updates_ready"
        expression => "cfengine_internal_disable_cf_promises_validated",
        comment => "If cf_promises_validated is disabled, then updates are
                    always considered validated.";

    any::

      "local_files_ok" expression => fileexists("$(file_check)"),
      comment => "Check for $(sys.masterdir)/promises.cf",
      handle => "cfe_internal_update_classes_files_ok";

      # create a global files_ok class
      "cfe_internal_trigger" expression => "local_files_ok",
      classes => u_if_else("files_ok", "files_ok");

  files:

    !am_policy_hub::  # policy hub should not alter inputs/ uneccessary

      "$(inputs_dir)/cf_promises_validated"
      comment => "Check whether a validation stamp is available for a new policy update to reduce the distributed load",
      handle => "cfe_internal_update_policy_check_valid_update",
      copy_from => u_rcp("$(master_location)/cf_promises_validated", @(update_def.policy_servers)),
      action => u_immediate,
      classes => u_if_repaired("validated_updates_ready");

    am_policy_hub|validated_updates_ready::  # policy hub should always put masterfiles in inputs in order to check new policy

      "$(inputs_dir)"
      comment => "Copy policy updates from master source on policy server if a new validation was acquired",
      handle => "cfe_internal_update_policy_files_inputs_dir",
      copy_from => u_rcp("$(master_location)", @(update_def.policy_servers)),
      depth_search => u_recurse("inf"),
      file_select  => u_input_files,
      action => u_immediate,
      classes => u_results("bundle", "update_inputs"),
      move_obstructions => "true";

      # Note that here we do not filter with `update_def.input_name_patterns` so
      # that we copy any and all modules scripts.
      "$(inputs_dir)/modules"
      comment => "Copy any files in modules from master source on policy server if a new validation was acquired",
      handle => "cfe_internal_update_policy_files_modules_dir",
      copy_from => u_rcp("$(modules_dir_source)", @(update_def.policy_servers)),
      depth_search => u_recurse("inf"),
      action => u_immediate;

    update_inputs_not_kept::

      "$(inputs_dir)/cf_promises_validated" -> { "CFE-2587" }
        delete => u_tidy,
        comment => "If there is any problem copying to $(inputs_dir) then purge
                    the cf_promises_validated file must be purged so that
                    subsequent agent runs will perform a full scan.";

    !policy_server.enable_cfengine_enterprise_hub_ha::
      "$(sys.workdir)/policy_server.dat"
      comment => "Copy policy_server.dat file from server",
      handle => "cfe_internal_update_ha_policy_server",
      copy_from => u_rcp("$(sys.workdir)/state/master_hub.dat", @(update_def.policy_servers)),
      action => u_immediate,
      classes => u_if_repaired("replica_failover");  # not needed ?

    am_policy_hub::

      "$(master_location)/."
      comment => "Make sure masterfiles folder has right file permissions",
      handle => "cfe_internal_update_policy_files_sys_workdir_masterfiles",
      perms => u_m($(update_def.masterfiles_perms_mode)),
      depth_search => u_recurse_basedir("inf"),
      action => u_immediate;

  methods:
    debian|redhat|amazon_linux|suse|sles|opensuse::
      # Only needed on distros with Python-based package modules
      "setup_python_symlink" -> { "CFE-2602" }
        usebundle => cfe_internal_setup_python_symlink("$(python_symlink)");

    any::
      # Install vendored and user provided modules to $(sys.workdir) from $(sys.inputdir)
      "modules_presence";
}

bundle agent cfe_internal_update_cmdb
{
  methods:

    policy_server.enterprise_edition::
      "cfe_internal_update_cmdb_data_distribution";

@if feature(host_specific_data_load)
      # Only hosts with this feature, introduced in 3.18.0 can use the data.

      # Don't pull CMDB data on policy_hub self bootstrap because
      # there will be no cf-serverd listening to serve files yet.
    enterprise_edition.!(bootstrap_mode)::  # ENT-6840
      "cfe_internal_update_cmdb_data_consumption";
@endif
}

bundle agent cfe_internal_update_cmdb_data_distribution
{
  vars:
    !bootstrap_mode.(policy_server.enterprise_edition)::

      # The API response for host specific data from cmdb tells us the timestamp of the last data change
      # We store this timestamp and use it for the next request.
      "_cmdb_next_request_state_file"
        string => "$(sys.statedir)/cmdb_next_request_from.dat";

      # If we have the timestamp from a previous response we use it, else we start from 0
      "_cmdb_previous_next_request_from"
        string => readfile( $(_cmdb_next_request_state_file), inf ),
        if => regline( "^\d+$", $(_cmdb_next_request_state_file) );

      "_cmdb_previous_next_request_from"
        string => "0",
        unless => regline( "^\d+$", $(_cmdb_next_request_state_file) );

      # We need a script to call that should return the API response
      "_get_cmdb_data_bin" string => "$(sys.workdir)/httpd/htdocs/scripts/get_cmdb.php";
      "_get_cmdb_data_cmd" string => "/var/cfengine/httpd/php/bin/php $(_get_cmdb_data_bin) $(_cmdb_previous_next_request_from)";

      # We call the script and we pass it the timestamp from the prior call
      "_get_cmdb_data_response"
        string => execresult( $(_get_cmdb_data_cmd), useshell ),
        if => fileexists( $(_get_cmdb_data_bin) );

      "_get_cmdb_data_response_d"
        data => parsejson('$(_get_cmdb_data_response)'),
        if => validjson( '$(_get_cmdb_data_response)' );

      # So that we can write a JSON file for each host we get the indicies of data in the response
      "_i" slist => getindices( "_get_cmdb_data_response_d[data]");

      # We need to store the timestamp from the most recent change so that we can use that as a starting point for future requests.
      "_next_request_from"
        string => "$(_get_cmdb_data_response_d[meta][cmdb_epoch])";

  files:
      # "$(_get_cmdb_data_cmd)" perms => m( 700 );

@if minimum_version(3.18)
    !bootstrap_mode.(policy_server.enterprise_edition)::
      # This functionality is only present on 3.18.0+ Enterprise hubs, and this
      # promise uses the /content/ attribute which was first introduced in
      # 3.16.0.

      # If the next request state file doesn't exist, we seed one with 0, the
      # lowest epoch value possible. because we populate variables from this
      # file content.
      "$(_cmdb_next_request_state_file)"
        content => "0$(const.n)",
        handle => "cmdb_data_change_next_seed",
        if => and( not(fileexists("$(_cmdb_next_request_state_file)" )),
                   isvariable( "_cmdb_next_request_state_file" ));
@endif

      # Write out the data for each host that had a data change
      "$(sys.workdir)/cmdb/$(_i)/host_specific.json"
        create => "true", # CFE-2329, ENT-4792
        template_data => mergedata("_get_cmdb_data_response_d[data][$(_i)]" ), # mergedata() is necessary in order to pick out a substructure, parsejson() is insufficient because expanding a key results in iteration of /values/ under that key
        template_method => "inline_mustache",
        edit_template_string => string_mustache( "", "_get_cmdb_data_response_d[data][$(_i)]" ),
        if => isgreaterthan( $(_next_request_from), $(_cmdb_previous_next_request_from) );

@if minimum_version(3.18)
      # This functionality is only present on 3.18.0+ Enterprise hubs, and this
      # promise uses the /content/ attribute which was first introduced in
      # 3.16.0.

      # Write out the last data change timestamp so we can use it as a startring point
      "$(_cmdb_next_request_state_file)"
        handle => "cmdb_data_change_next_update",
        content => "$(_next_request_from)$(const.n)",
        unless => strcmp( $(_next_request_from), $(_cmdb_previous_next_request_from) );
@endif

  reports:
    DEBUG|DEBUG_cfe_internal_update_cmdb_data_distribution::
      "'$(_get_cmdb_data_cmd)' response indicates '$(sys.workdir)/cmdb/$(_i)/host_specific.json' needs refreshed"
        if => and( isvariable( "_i" ),
                   isgreaterthan( $(_next_request_from), $(_cmdb_previous_next_request_from) ));

}

bundle agent cfe_internal_update_cmdb_data_consumption
{
  files:
      "$(sys.workdir)/data/."
        create => "true",
        comment => "If a host is to load data from the CMDB, it needs to have a directory where said data is cached.";

      "$(sys.workdir)/data/." -> { "ENT-6788" }
        depth_search => u_recurse( inf ),
        file_select => u_all,
        copy_from => u_cmdb_data,
        comment => "So that hosts have access to the most recent CMDB data, we make sure that it's up to date.";

}

bundle agent modules_presence
{
  vars:
    "_vendored_dir" string => "$(this.promise_dirname)/../../modules/packages/vendored/";
    "_override_dir" string => "$(this.promise_dirname)/../../modules/packages/";
    "_vendored_paths" slist => findfiles("$(_vendored_dir)*.mustache");
    "_package_paths" slist => filter("$(_override_dir)vendored", _package_paths_tmp, "false", "true", 999);
    windows::
      "_package_paths_tmp" slist => findfiles("\Q$(_override_dir)\E*");
      "_vendored_modules" slist => maplist(regex_replace("$(this)", "\Q$(_vendored_dir)\E(.*).mustache", "$1", "g"), @(_vendored_paths));
      "_override_modules" slist => maplist(regex_replace("$(this)", "\Q$(_override_dir)\E(.*)", "$1", "g"), @(_package_paths));
      # replace single backslashes in a windows path with double-backslashes
      # to avoid problems with things like `C:\Program Files` and `\promises`
      # causing PCRE to try and interpret special escape sequences.
      "_not_vendored_modules_pathname_regex" string => regex_replace("$(sys.inputdir)$(const.dirsep)modules$(const.dirsep)(?!packages$(const.dirsep)vendored).*","\\\\","\\\\\\\\","g");
    !windows::
      "_package_paths_tmp" slist => findfiles("$(_override_dir)*");
      "_vendored_modules" slist => maplist(regex_replace("$(this)", "$(_vendored_dir)(.*).mustache", "$1", "g"), @(_vendored_paths));
      "_override_modules" slist => maplist(regex_replace("$(this)", "$(_override_dir)(.*)", "$1", "g"), @(_package_paths));
      "_not_vendored_modules_pathname_regex" string => "$(sys.inputdir)/modules/(?!packages/vendored).*";

  classes:
    "override_vendored_module_$(_vendored_modules)" expression => fileexists("$(_override_dir)$(_vendored_modules)");
    "override_module_$(_override_modules)" expression => fileexists("$(_override_dir)$(_override_modules)");

# NOTE: here we are using the .mustache extension only to
# ensure that the modules scripts are copied as part of
# update (see controls/update_def.cf input_name_patterns var.

  files:
    "$(sys.workdir)/modules/packages/$(_vendored_modules)"
      create => "true",
      perms => u_mo("755", "root"),
      unless => canonify("override_vendored_module_$(_vendored_modules)"),
      edit_template => "$(_vendored_dir)$(_vendored_modules).mustache",
      template_method => "mustache";

    "$(sys.workdir)/modules/packages/$(_override_modules)"
      copy_from => u_cp_missing_ok("$(_override_dir)$(_override_modules)"),
      perms => u_mo("755", "root"),
      if => or (
        canonify("override_vendored_module_$(_override_modules)"),
        canonify("override_module_$(_override_modules)"));

    "$(sys.workdir)/modules"
      comment => "Copy any non-packages modules",
      handle => "cfe_internal_update_policy_files_nonpackages_modules",
      copy_from => u_cp("$(sys.inputdir)$(const.dirsep)modules"),
      if => fileexists("$(sys.inputdir)$(const.dirsep)modules"),
      depth_search => u_recurse("inf"),
      perms => u_mo("755", "root"),
      action => u_immediate,
      file_select => not_vendored_modules("$(_not_vendored_modules_pathname_regex)");


  reports:
    DEBUG::
      "_not_vendored_modules_pathname_regex: $(_not_vendored_modules_pathname_regex)";
      "_vendored_modules: $(_vendored_modules)";
      "_override_modules: $(_override_modules)";
      "_vendored_dir: $(_vendored_dir)";
      "_vendored_paths: $(_vendored_paths)";
      "_override_dir: $(_override_dir)";
      "_package_paths: $(_package_paths)";
      "override_vendored_module_$(_vendored_modules)"
        if => "override_vendored_module_$(_vendored_modules)";
      "override_module_$(_override_modules)"
        if => "override_module_$(_override_modules)";
      "canonified: $(with)" with => canonify("override_vendored_module_$(_vendored_modules)");
}

body copy_from u_cmdb_data
{
        copy_backup => "false";
        trustkey => "false";
        compare => "digest";
        source  => "hub_cmdb";
        servers => { "$(sys.policy_hub)" };
        purge => "true";
@if minimum_version(3.12)
        missing_ok => "true";
@endif
}

body perms u_m(p)
{
      mode  => "$(p)";
}

body perms u_mo(p,o)
{
      mode   => "$(p)";
      !(windows|termux)::
        owners => {"$(o)"};
}

body perms u_shared_lib_perms
{
    !hpux::
      mode => "0644";
    hpux::
      mode => "0755"; # Mantis 1114, Redmine 1179
}

body file_select u_all
{
        leaf_name => { ".*" };
        file_result => "leaf_name";
}

body file_select u_cf3_files
{
      leaf_name => { "cf-.*" };
      file_result => "leaf_name";
}

body file_select u_input_files
{
      leaf_name => { @(update_def.input_name_patterns) };
      file_result => "leaf_name";
}

body copy_from u_rcp(from,server)
{
      source      => "$(from)";
      compare     => "digest";
      trustkey    => "false";
      purge => "true"; # CFE-3662

      # CFE-2932 For testing, we want to be able to avoid this local copy optimiztion
    !am_policy_hub|mpf_skip_local_copy_optimizaton::
      servers => { "$(server)" };

    !am_policy_hub.(sys_policy_hub_port_exists|mpf_skip_local_copy_optimization)::
      portnumber => "$(sys.policy_hub_port)";

    cfengine_internal_encrypt_transfers::
      encrypt => "true";

    cfengine_internal_purge_policies_disabled::
      purge => "false";

    cfengine_internal_preserve_permissions::
      preserve => "true";

    cfengine_internal_verify_update_transfers::
      verify      => "true";
}

body copy_from u_cp(from)
{
      source  => "$(from)";
      compare => "digest";
}

body copy_from u_cp_nobck(from)
{
      source      => "$(from)";
      compare     => "digest";
      copy_backup => "false";
}

body copy_from u_cp_missing_ok(from)
{
      inherit_from => u_cp($(from));
      missing_ok => "true";
}

body action u_immediate
{
      ifelapsed => "0";
}

body depth_search u_recurse(d)
{
      depth => "$(d)";
      exclude_dirs => { "\.svn", "\.git", "git-core" };
}

body depth_search u_recurse_basedir(d)
{
      include_basedir => "true";
      depth => "$(d)";
      exclude_dirs => { "\.svn", "\.git", "git-core" };
}

body classes u_if_repaired(x)
{
      promise_repaired => { "$(x)" };
}

body classes u_if_repaired_then_cancel(y)
{
      cancel_repaired => { "$(y)" };
}

body classes u_if_else(yes,no)
{
#      promise_kept     => { "$(yes)" };
      promise_repaired => { "$(yes)" };
      repair_failed    => { "$(no)" };
      repair_denied    => { "$(no)" };
      repair_timeout   => { "$(no)" };
}

body contain u_in_shell
{
      useshell => "true";
}

body contain u_in_shell_and_silent
{
      useshell => "true";
      no_output => "true";
}

body contain u_postgres
{
  useshell   => "useshell";
  exec_owner => "cfpostgres";
  exec_group => "cfpostgres";
  chdir      => "/tmp";
  no_output  => "true";
}

body action u_ifwin_bg
{
    windows::
      background => "true";
}

body service_method u_bootstart
{
      service_autostart_policy => "boot_time";
}

body contain u_in_dir(s)
{
      chdir => "$(s)";
}

body contain u_silent_in_dir(s)
{
      chdir => "$(s)";
      no_output => "true";
}

body link_from u_ln_s(x)
{
      link_type => "symlink";
      source => "$(x)";
      when_no_source => "force";
}

body delete u_tidy
{
      dirlinks => "delete";
      rmdirs   => "true";
}

body file_select not_vendored_modules(pathname)
{
  path_name => { "$(pathname)" };
  file_result => "path_name";
}