paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-10-30 19:43:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

puppet: Pull in pinned puppet modules.

Browse Source 
Using puppet modules from the puppet forge judiciously will allow us
to simplify the configuration somewhat; this specifically pulls in the
stdlib module, which we were already using parts of.
This commit is contained in:
Alex Vandiver
2021-05-25 01:05:40 +00:00
committed by Alex Vandiver
parent 6107679f61
commit 87a109e3e0
9 changed files with 96 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/THIRDPARTY
View File

@@ -28,11 +28,6 @@ Files: docs/code-of-conduct.md
Copyright: 2017, Kandra Labs Inc.
License: CC-BY-SA-4.0
Files: puppet/zulip/lib/puppet/parser/functions/join.rb puppet/zulip/lib/puppet/parser/functions/range.rb
Copyright: 2011, Krzysztof Wilczynski
2011, Puppet Labs Inc
License: Apache-2.0
Files: puppet/zulip/files/nagios_plugins/zulip_base/check_debian_packages
Copyright: 2005 Francesc Guasch
License: GPL-2.0

8
docs/subsystems/dependencies.md
View File

@@ -256,6 +256,14 @@ wrapper to access the desired version conveniently and efficiently
`/srv/zulip-yarn`. We don't do anything special to try to manage
multiple versions of `yarn`.
## Puppet packages
Third-party puppet modules are downloaded from the Puppet Forge into
subdirectories under `/srv/zulip-puppet-cache`, hashed based on their
versions; the latest is always symlinked as
`/srv/zulip-puppet-cache/current`. `zulip-puppet-apply` installs
these dependencies immediately before they are needed.
## Other third-party and generated files
In this section, we discuss the other third-party dependencies,

1
puppet/deps.yaml Normal file
View File

@@ -0,0 +1 @@
puppetlabs-stdlib: 7.1.0

43
puppet/zulip/lib/puppet/parser/functions/join.rb
View File

@@ -1,43 +0,0 @@
# Taken from https://github.com/puppetlabs/puppetlabs-stdlib/blob/19cdf29f27c3e5005ee441d1ec46d7da27a0f777/lib/puppet/parser/functions/join.rb
#
# join.rb
#
module Puppet::Parser::Functions
newfunction(:join, :type => :rvalue, :doc => <<-DOC
This function joins an array into a string using a separator.
*Examples:*
join(['a','b','c'], ",")
Would result in: "a,b,c"
Note: from Puppet 5.4.0, the compatible function with the same name in Puppet core
will be used instead of this function.
DOC
) do |arguments|
# Technically we support two arguments but only first is mandatory ...
raise(Puppet::ParseError, "join(): Wrong number of arguments given (#{arguments.size} for 1)") if arguments.empty?
array = arguments[0]
unless array.is_a?(Array)
raise(Puppet::ParseError, 'join(): Requires array to work with')
end
suffix = arguments[1] if arguments[1]
if suffix
unless suffix.is_a?(String)
raise(Puppet::ParseError, 'join(): Requires string to work with')
end
end
result = suffix ? array.join(suffix) : array.join
return result
end
end
# vim: set ts=2 sw=2 et :

27
puppet/zulip/lib/puppet/parser/functions/keys.rb
View File

@@ -1,27 +0,0 @@
# Taken from https://github.com/puppetlabs/puppetlabs-stdlib/blob/19cdf29f27c3e5005ee441d1ec46d7da27a0f777/lib/puppet/parser/functions/keys.rb
#
# keys.rb
#
module Puppet::Parser::Functions
newfunction(:keys, :type => :rvalue, :doc => <<-DOC
Returns the keys of a hash as an array.
Note: from Puppet 5.5.0, the compatible function with the same name in Puppet core
will be used instead of this function.
DOC
) do |arguments|
raise(Puppet::ParseError, "keys(): Wrong number of arguments given (#{arguments.size} for 1)") if arguments.empty?
hash = arguments[0]
unless hash.is_a?(Hash)
raise(Puppet::ParseError, 'keys(): Requires hash to work with')
end
result = hash.keys
return result
end
end
# vim: set ts=2 sw=2 et :

90
puppet/zulip/lib/puppet/parser/functions/range.rb
View File

@@ -1,90 +0,0 @@
# Taken from https://github.com/puppetlabs/puppetlabs-stdlib/blob/19cdf29f27c3e5005ee441d1ec46d7da27a0f777/lib/puppet/parser/functions/range.rb
#
# range.rb
#
# TODO(Krzysztof Wilczynski): We probably need to approach numeric values differently ...
module Puppet::Parser::Functions
newfunction(:range, :type => :rvalue, :doc => <<-DOC
When given range in the form of (start, stop) it will extrapolate a range as
an array.
*Examples:*
range("0", "9")
Will return: [0,1,2,3,4,5,6,7,8,9]
range("00", "09")
Will return: [0,1,2,3,4,5,6,7,8,9] (Zero padded strings are converted to
integers automatically)
range("a", "c")
Will return: ["a","b","c"]
range("host01", "host10")
Will return: ["host01", "host02", ..., "host09", "host10"]
NB Be explicit in including trailing zeros. Otherwise the underlying ruby function will fail.
Passing a third argument will cause the generated range to step by that
interval, e.g.
range("0", "9", "2")
Will return: [0,2,4,6,8]
The Puppet Language support Integer and Float ranges by using the type system. Those are suitable for
iterating a given number of times. Also see the step() function in Puppet for skipping values.
Integer[0, 9].each |$x| { notice($x) } # notices 0, 1, 2, ... 9
DOC
) do |arguments|
raise(Puppet::ParseError, 'range(): Wrong number of arguments given (0 for 1)') if arguments.empty?
if arguments.size > 1
start = arguments[0]
stop = arguments[1]
step = arguments[2].nil? ? 1 : arguments[2].to_i.abs
type = '..' # Use the simplest type of Range available in Ruby
else # arguments.size == 1
value = arguments[0]
m = value.match(%r{^(\w+)(\.\.\.?|\-)(\w+)$})
if m
start = m[1]
stop = m[3]
type = m[2]
step = 1
elsif value =~ %r{^.+$}
raise(Puppet::ParseError, "range(): Unable to compute range from the value: #{value}")
else
raise(Puppet::ParseError, "range(): Unknown range format: #{value}")
end
end
# If we were given an integer, ensure we work with one
if start.to_s =~ %r{^\d+$}
start = start.to_i
stop = stop.to_i
else
start = start.to_s
stop = stop.to_s
end
range = case type
when %r{^(..|-)$} then (start..stop)
when '...' then (start...stop) # Exclusive of last element
end
result = range.step(step).to_a
return result
end
end
# vim: set ts=2 sw=2 et :

77
scripts/lib/puppet_cache.py Normal file
View File

@@ -0,0 +1,77 @@
import hashlib
import json
import os
import shutil
import subprocess
import yaml
from .zulip_tools import parse_os_release, run
ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
ZULIP_SRV_PATH = "/srv"
PUPPET_MODULES_CACHE_PATH = os.path.join(ZULIP_SRV_PATH, "zulip-puppet-cache")
PUPPET_DEPS_FILE_PATH = os.path.join(ZULIP_PATH, "puppet/deps.yaml")
PUPPET_THIRDPARTY = os.path.join(PUPPET_MODULES_CACHE_PATH, "current")
def generate_sha1sum_puppet_modules() -> str:
data = {}
with open(PUPPET_DEPS_FILE_PATH, "r") as fb:
data["deps.yaml"] = fb.read().strip()
data["puppet-version"] = subprocess.check_output(
# This is 10x faster than `puppet --version`
["ruby", "-r", "puppet/version", "-e", "puts Puppet.version"],
universal_newlines=True,
).strip()
sha1sum = hashlib.sha1()
sha1sum.update(json.dumps(data, sort_keys=True).encode("utf-8"))
return sha1sum.hexdigest()
def setup_puppet_modules() -> None:
sha1sum = generate_sha1sum_puppet_modules()
target_path = os.path.join(PUPPET_MODULES_CACHE_PATH, sha1sum)
success_stamp = os.path.join(target_path, ".success-stamp")
# Check if a cached version already exists
if not os.path.exists(success_stamp):
do_puppet_module_install(target_path, success_stamp)
if os.path.islink(PUPPET_THIRDPARTY):
os.remove(PUPPET_THIRDPARTY)
elif os.path.isdir(PUPPET_THIRDPARTY):
shutil.rmtree(PUPPET_THIRDPARTY)
os.symlink(target_path, PUPPET_THIRDPARTY)
def do_puppet_module_install(
target_path: str,
success_stamp: str,
) -> None:
# This is to suppress Puppet warnings with ruby 2.7.
distro_info = parse_os_release()
puppet_env = os.environ.copy()
if (distro_info["ID"], distro_info["VERSION_ID"]) in [("ubuntu", "20.04")]:
puppet_env["RUBYOPT"] = "-W0"
os.makedirs(target_path, exist_ok=True)
with open(PUPPET_DEPS_FILE_PATH, "r") as yaml_file:
deps = yaml.safe_load(yaml_file)
for module, version in deps.items():
run(
[
"puppet",
"module",
"--modulepath",
target_path,
"install",
module,
"--version",
version,
],
env=puppet_env,
)
with open(success_stamp, "w"):
pass

11
scripts/zulip-puppet-apply
View File

@@ -9,6 +9,7 @@ import tempfile
import yaml
from lib.puppet_cache import setup_puppet_modules
from lib.zulip_tools import assert_running_as_root, parse_os_release
assert_running_as_root()
@@ -25,6 +26,8 @@ args, extra_args = parser.parse_known_args()
config = configparser.RawConfigParser()
config.read(args.config)
setup_puppet_modules()
distro_info = parse_os_release()
puppet_config = """
Exec { path => "/usr/sbin:/usr/bin:/sbin:/bin" }
@@ -36,7 +39,13 @@ for pclass in re.split(r"\s*,\s*", config.get("machine", "puppet_classes")):
# We use the Puppet configuration from the same Zulip checkout as this script
scripts_path = os.path.join(BASE_DIR, "scripts")
puppet_module_path = os.path.join(BASE_DIR, "puppet")
puppet_cmd = ["puppet", "apply", f"--modulepath={puppet_module_path}", "-e", puppet_config]
puppet_cmd = [
"puppet",
"apply",
f"--modulepath={puppet_module_path}:/srv/zulip-puppet-cache/current",
"-e",
puppet_config,
]
if args.noop:
puppet_cmd += ["--noop"]
puppet_cmd += extra_args

2
tools/linter_lib/exclude.py
View File

@@ -1,8 +1,6 @@
# Exclude some directories and files from lint checking
EXCLUDED_FILES = [
# Third-party code that doesn't match our style
"puppet/zulip/lib/puppet/parser/functions/join.rb",
"puppet/zulip/lib/puppet/parser/functions/range.rb",
"puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_website_response.sh",
"scripts/lib/third",
"static/third",