puppet: Pull in pinned puppet modules.

Using puppet modules from the puppet forge judiciously will allow us
to simplify the configuration somewhat; this specifically pulls in the
stdlib module, which we were already using parts of.

docs/THIRDPARTY

@@ -28,11 +28,6 @@ Files: docs/code-of-conduct.md
 Copyright: 2017, Kandra Labs Inc.
 License: CC-BY-SA-4.0
 
-Files: puppet/zulip/lib/puppet/parser/functions/join.rb puppet/zulip/lib/puppet/parser/functions/range.rb
-Copyright: 2011, Krzysztof Wilczynski
- 2011, Puppet Labs Inc
-License: Apache-2.0
-
 Files: puppet/zulip/files/nagios_plugins/zulip_base/check_debian_packages
 Copyright: 2005 Francesc Guasch
 License: GPL-2.0

docs/subsystems/dependencies.md

@@ -256,6 +256,14 @@ wrapper to access the desired version conveniently and efficiently
 `/srv/zulip-yarn`.  We don't do anything special to try to manage
 multiple versions of `yarn`.
 
+## Puppet packages
+
+Third-party puppet modules are downloaded from the Puppet Forge into
+subdirectories under `/srv/zulip-puppet-cache`, hashed based on their
+versions; the latest is always symlinked as
+`/srv/zulip-puppet-cache/current`.  `zulip-puppet-apply` installs
+these dependencies immediately before they are needed.
+
 ## Other third-party and generated files
 
 In this section, we discuss the other third-party dependencies,

puppet/deps.yaml

@@ -0,0 +1 @@
+puppetlabs-stdlib: 7.1.0

puppet/zulip/lib/puppet/parser/functions/join.rb

@@ -1,43 +0,0 @@
-# Taken from https://github.com/puppetlabs/puppetlabs-stdlib/blob/19cdf29f27c3e5005ee441d1ec46d7da27a0f777/lib/puppet/parser/functions/join.rb
-#
-# join.rb
-#
-module Puppet::Parser::Functions
-  newfunction(:join, :type => :rvalue, :doc => <<-DOC
-    This function joins an array into a string using a separator.
-
-    *Examples:*
-
-        join(['a','b','c'], ",")
-
-    Would result in: "a,b,c"
-
-    Note: from Puppet 5.4.0, the compatible function with the same name in Puppet core
-    will be used instead of this function.
-    DOC
-             ) do |arguments|
-
-    # Technically we support two arguments but only first is mandatory ...
-    raise(Puppet::ParseError, "join(): Wrong number of arguments given (#{arguments.size} for 1)") if arguments.empty?
-
-    array = arguments[0]
-
-    unless array.is_a?(Array)
-      raise(Puppet::ParseError, 'join(): Requires array to work with')
-    end
-
-    suffix = arguments[1] if arguments[1]
-
-    if suffix
-      unless suffix.is_a?(String)
-        raise(Puppet::ParseError, 'join(): Requires string to work with')
-      end
-    end
-
-    result = suffix ? array.join(suffix) : array.join
-
-    return result
-  end
-end
-
-# vim: set ts=2 sw=2 et :

puppet/zulip/lib/puppet/parser/functions/keys.rb

@@ -1,27 +0,0 @@
-# Taken from https://github.com/puppetlabs/puppetlabs-stdlib/blob/19cdf29f27c3e5005ee441d1ec46d7da27a0f777/lib/puppet/parser/functions/keys.rb
-#
-# keys.rb
-#
-module Puppet::Parser::Functions
-  newfunction(:keys, :type => :rvalue, :doc => <<-DOC
-    Returns the keys of a hash as an array.
-    Note: from Puppet 5.5.0, the compatible function with the same name in Puppet core
-    will be used instead of this function.
-    DOC
-             ) do |arguments|
-
-    raise(Puppet::ParseError, "keys(): Wrong number of arguments given (#{arguments.size} for 1)") if arguments.empty?
-
-    hash = arguments[0]
-
-    unless hash.is_a?(Hash)
-      raise(Puppet::ParseError, 'keys(): Requires hash to work with')
-    end
-
-    result = hash.keys
-
-    return result
-  end
-end
-
-# vim: set ts=2 sw=2 et :

puppet/zulip/lib/puppet/parser/functions/range.rb

@@ -1,90 +0,0 @@
-# Taken from https://github.com/puppetlabs/puppetlabs-stdlib/blob/19cdf29f27c3e5005ee441d1ec46d7da27a0f777/lib/puppet/parser/functions/range.rb
-#
-# range.rb
-#
-# TODO(Krzysztof Wilczynski): We probably need to approach numeric values differently ...
-module Puppet::Parser::Functions
-  newfunction(:range, :type => :rvalue, :doc => <<-DOC
-    When given range in the form of (start, stop) it will extrapolate a range as
-    an array.
-
-    *Examples:*
-
-        range("0", "9")
-
-    Will return: [0,1,2,3,4,5,6,7,8,9]
-
-        range("00", "09")
-
-    Will return: [0,1,2,3,4,5,6,7,8,9] (Zero padded strings are converted to
-    integers automatically)
-
-        range("a", "c")
-
-    Will return: ["a","b","c"]
-
-        range("host01", "host10")
-    Will return: ["host01", "host02", ..., "host09", "host10"]
-    NB Be explicit in including trailing zeros. Otherwise the underlying ruby function will fail.
-
-    Passing a third argument will cause the generated range to step by that
-    interval, e.g.
-
-        range("0", "9", "2")
-
-    Will return: [0,2,4,6,8]
-
-    The Puppet Language support Integer and Float ranges by using the type system. Those are suitable for
-    iterating a given number of times. Also see the step() function in Puppet for skipping values.
-
-        Integer[0, 9].each |$x| { notice($x) } # notices 0, 1, 2, ... 9
-    DOC
-             ) do |arguments|
-
-    raise(Puppet::ParseError, 'range(): Wrong number of arguments given (0 for 1)') if arguments.empty?
-
-    if arguments.size > 1
-      start = arguments[0]
-      stop  = arguments[1]
-      step  = arguments[2].nil? ? 1 : arguments[2].to_i.abs
-
-      type = '..' # Use the simplest type of Range available in Ruby
-
-    else # arguments.size == 1
-      value = arguments[0]
-
-      m = value.match(%r{^(\w+)(\.\.\.?|\-)(\w+)$})
-      if m
-        start = m[1]
-        stop  = m[3]
-
-        type = m[2]
-        step = 1
-      elsif value =~ %r{^.+$}
-        raise(Puppet::ParseError, "range(): Unable to compute range from the value: #{value}")
-      else
-        raise(Puppet::ParseError, "range(): Unknown range format: #{value}")
-      end
-    end
-
-    # If we were given an integer, ensure we work with one
-    if start.to_s =~ %r{^\d+$}
-      start = start.to_i
-      stop  = stop.to_i
-    else
-      start = start.to_s
-      stop  = stop.to_s
-    end
-
-    range = case type
-            when %r{^(..|-)$} then (start..stop)
-            when '...' then (start...stop) # Exclusive of last element
-            end
-
-    result = range.step(step).to_a
-
-    return result
-  end
-end
-
-# vim: set ts=2 sw=2 et :

scripts/lib/puppet_cache.py

@@ -0,0 +1,77 @@
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+
+import yaml
+
+from .zulip_tools import parse_os_release, run
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+ZULIP_SRV_PATH = "/srv"
+
+PUPPET_MODULES_CACHE_PATH = os.path.join(ZULIP_SRV_PATH, "zulip-puppet-cache")
+PUPPET_DEPS_FILE_PATH = os.path.join(ZULIP_PATH, "puppet/deps.yaml")
+PUPPET_THIRDPARTY = os.path.join(PUPPET_MODULES_CACHE_PATH, "current")
+
+
+def generate_sha1sum_puppet_modules() -> str:
+    data = {}
+    with open(PUPPET_DEPS_FILE_PATH, "r") as fb:
+        data["deps.yaml"] = fb.read().strip()
+    data["puppet-version"] = subprocess.check_output(
+        # This is 10x faster than `puppet --version`
+        ["ruby", "-r", "puppet/version", "-e", "puts Puppet.version"],
+        universal_newlines=True,
+    ).strip()
+
+    sha1sum = hashlib.sha1()
+    sha1sum.update(json.dumps(data, sort_keys=True).encode("utf-8"))
+    return sha1sum.hexdigest()
+
+
+def setup_puppet_modules() -> None:
+    sha1sum = generate_sha1sum_puppet_modules()
+    target_path = os.path.join(PUPPET_MODULES_CACHE_PATH, sha1sum)
+    success_stamp = os.path.join(target_path, ".success-stamp")
+    # Check if a cached version already exists
+    if not os.path.exists(success_stamp):
+        do_puppet_module_install(target_path, success_stamp)
+
+    if os.path.islink(PUPPET_THIRDPARTY):
+        os.remove(PUPPET_THIRDPARTY)
+    elif os.path.isdir(PUPPET_THIRDPARTY):
+        shutil.rmtree(PUPPET_THIRDPARTY)
+    os.symlink(target_path, PUPPET_THIRDPARTY)
+
+
+def do_puppet_module_install(
+    target_path: str,
+    success_stamp: str,
+) -> None:
+    # This is to suppress Puppet warnings with ruby 2.7.
+    distro_info = parse_os_release()
+    puppet_env = os.environ.copy()
+    if (distro_info["ID"], distro_info["VERSION_ID"]) in [("ubuntu", "20.04")]:
+        puppet_env["RUBYOPT"] = "-W0"
+
+    os.makedirs(target_path, exist_ok=True)
+    with open(PUPPET_DEPS_FILE_PATH, "r") as yaml_file:
+        deps = yaml.safe_load(yaml_file)
+        for module, version in deps.items():
+            run(
+                [
+                    "puppet",
+                    "module",
+                    "--modulepath",
+                    target_path,
+                    "install",
+                    module,
+                    "--version",
+                    version,
+                ],
+                env=puppet_env,
+            )
+    with open(success_stamp, "w"):
+        pass

scripts/zulip-puppet-apply

@@ -9,6 +9,7 @@ import tempfile
 
 import yaml
 
+from lib.puppet_cache import setup_puppet_modules
 from lib.zulip_tools import assert_running_as_root, parse_os_release
 
 assert_running_as_root()
@@ -25,6 +26,8 @@ args, extra_args = parser.parse_known_args()
 config = configparser.RawConfigParser()
 config.read(args.config)
 
+setup_puppet_modules()
+
 distro_info = parse_os_release()
 puppet_config = """
 Exec { path => "/usr/sbin:/usr/bin:/sbin:/bin" }
@@ -36,7 +39,13 @@ for pclass in re.split(r"\s*,\s*", config.get("machine", "puppet_classes")):
 # We use the Puppet configuration from the same Zulip checkout as this script
 scripts_path = os.path.join(BASE_DIR, "scripts")
 puppet_module_path = os.path.join(BASE_DIR, "puppet")
-puppet_cmd = ["puppet", "apply", f"--modulepath={puppet_module_path}", "-e", puppet_config]
+puppet_cmd = [
+    "puppet",
+    "apply",
+    f"--modulepath={puppet_module_path}:/srv/zulip-puppet-cache/current",
+    "-e",
+    puppet_config,
+]
 if args.noop:
     puppet_cmd += ["--noop"]
 puppet_cmd += extra_args

tools/linter_lib/exclude.py

@@ -1,8 +1,6 @@
 # Exclude some directories and files from lint checking
 EXCLUDED_FILES = [
     # Third-party code that doesn't match our style
-    "puppet/zulip/lib/puppet/parser/functions/join.rb",
-    "puppet/zulip/lib/puppet/parser/functions/range.rb",
     "puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_website_response.sh",
     "scripts/lib/third",
     "static/third",