Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions confluent_osdeploy/common/profile/scripts/functions
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,12 @@ sys.stdout.write(value + '\n')
EOF
}

# Print deployment.installedargs from a deploycfg, by default
# /etc/confluent/confluent.deploycfg, undoing its yaml single quoting.
function confluent_installedargs() {
sed -n "s/^installedargs: //p" "${1:-/etc/confluent/confluent.deploycfg}" | sed -e "s/^'\(.*\)'\$/\1/" -e "s/''/'/g"
}

function set_confluent_vars() {
if [ -z "$nodename" ]; then
nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
Expand Down
2 changes: 2 additions & 0 deletions confluent_osdeploy/debian/profiles/default/scripts/post.sh
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
fi
fi
kargs=$(python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/profile.yaml | grep ^installedargs: | sed -e 's/^installedargs:[[:space:]]*//' -e 's/#.*//')
nodeargs=$(confluent_installedargs)
if [ -n "$nodeargs" ]; then kargs="${kargs:+$kargs }$nodeargs"; fi
if [ ! -z "$kargs" ]; then
kargs=$(printf '%s' "$kargs" | sed -e 's/[\/&]/\\&/g')
sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /etc/default/grub
Expand Down
2 changes: 2 additions & 0 deletions confluent_osdeploy/el7/profiles/default/scripts/pre.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ else
fi
curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/profile.yaml > /tmp/instprofile.yaml
blargs=$(grep ^installedargs: /tmp/instprofile.yaml | sed -e 's/#.*//' -e 's/^installedargs: //')
nodeargs=$(sed -n "s/^installedargs: //p" /etc/confluent/confluent.deploycfg | sed -e "s/^'\(.*\)'\$/\1/" -e "s/''/'/g")
if [ -n "$nodeargs" ]; then blargs="${blargs:+$blargs }$nodeargs"; fi
if [ ! -z "$blargs" ]; then
blargs=' --append="'$blargs'"'
fi
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,11 @@ def set_machine_id(rootdir):

swapdev = '/dev/mapper/localstorage-swap'

def append_args(args, newargs):
# newargs go last, and an argument given twice keeps only its last place
merged = args + newargs
return [arg for idx, arg in enumerate(merged) if arg not in merged[idx + 1:]]

def fixup(rootdir, vols):
devbymount = {}
for vol in vols:
Expand Down Expand Up @@ -196,6 +201,31 @@ def fixup(rootdir, vols):
if arg == 'quiet' or arg.startswith(('console=', 'net.ifnames=', 'biosdevname=')):
currcmdline.append(arg)
currcmdlinestr = ' '.join(currcmdline)
# installedargs of the profile followed by deployment.installedargs,
# which the OS installers add to the kernel arguments of the installed
# system
profile = None
installedargs = []
with open('/etc/confluent/confluent.deploycfg') as dcfgin:
for line in dcfgin:
if line.startswith('profile: '):
profile = line[len('profile: '):].strip()
elif line.startswith('installedargs: '):
argstr = line[len('installedargs: '):].strip()
if argstr[:1] == "'" and argstr[-1:] == "'":
argstr = argstr[1:-1].replace("''", "'")
installedargs = argstr.split()
if profile:
try:
profileyaml = subprocess.check_output([
sys.executable, '/opt/confluent/bin/apiclient',
'/confluent-public/os/{}/profile.yaml'.format(profile)]).decode('utf8')
except (subprocess.CalledProcessError, OSError) as e:
sys.stderr.write('Unable to read the installedargs of profile {}: {}\n'.format(profile, e))
profileyaml = ''
for line in profileyaml.split('\n'):
if line.startswith('installedargs:'):
installedargs = line.split('#', 1)[0][len('installedargs:'):].split() + installedargs
if os.path.exists(grubsyscfg):
with open(grubsyscfg) as defgrubin:
defgrub = defgrubin.read().split('\n')
Expand All @@ -213,7 +243,7 @@ def fixup(rootdir, vols):
]
if not os.path.exists(os.path.join(rootdir, "etc/kernel/cmdline")):
with open(os.path.join(rootdir, "etc/kernel/cmdline"), "w") as cmdlineout:
cmdlineout.write("{} root=/dev/mapper/localstorage-root rd.lvm.lv=localstorage/root".format(currcmdlinestr))
cmdlineout.write("{} root=/dev/mapper/localstorage-root rd.lvm.lv=localstorage/root".format(' '.join(append_args(currcmdline, installedargs))))
with open(grubsyscfg, 'w') as defgrubout:
for gline in defgrub:
gline = gline.split()
Expand All @@ -224,6 +254,10 @@ def fixup(rootdir, vols):
newline.append('"')
continue
newline.append(ent)
if installedargs and newline and newline[0].startswith('GRUB_CMDLINE_LINUX=') and newline[-1].endswith('"'):
# an image captured from an installed node may carry them already
present = ' '.join(newline)[len('GRUB_CMDLINE_LINUX="'):-1].split()
newline = ['GRUB_CMDLINE_LINUX="{}"'.format(' '.join(append_args(present, installedargs)))]
defgrubout.write(' '.join(newline) + '\n')
grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
grubcfg = grubcfg.split('\n')
Expand Down
2 changes: 2 additions & 0 deletions confluent_osdeploy/el8/profiles/default/scripts/pre.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ else
fi
curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/profile.yaml > /tmp/instprofile.yaml
blargs=$(grep ^installedargs: /tmp/instprofile.yaml | sed -e 's/#.*//' -e 's/^installedargs: //')
nodeargs=$(sed -n "s/^installedargs: //p" /etc/confluent/confluent.deploycfg | sed -e "s/^'\(.*\)'\$/\1/" -e "s/''/'/g")
if [ -n "$nodeargs" ]; then blargs="${blargs:+$blargs }$nodeargs"; fi
if [ ! -z "$blargs" ]; then
blargs=' --append="'$blargs'"'
fi
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,11 @@ def set_machine_id(rootdir):
else:
os.rename(loadent, newent)

def append_args(args, newargs):
# newargs go last, and an argument given twice keeps only its last place
merged = args + newargs
return [arg for idx, arg in enumerate(merged) if arg not in merged[idx + 1:]]

def fixup(rootdir, vols):
devbymount = {}
for vol in vols:
Expand Down Expand Up @@ -216,6 +221,31 @@ def fixup(rootdir, vols):
if arg == 'quiet' or arg.startswith(('console=', 'net.ifnames=', 'biosdevname=')):
currcmdline.append(arg)
currcmdlinestr = ' '.join(currcmdline)
# installedargs of the profile followed by deployment.installedargs,
# which the OS installers add to the kernel arguments of the installed
# system
profile = None
installedargs = []
with open('/etc/confluent/confluent.deploycfg') as dcfgin:
for line in dcfgin:
if line.startswith('profile: '):
profile = line[len('profile: '):].strip()
elif line.startswith('installedargs: '):
argstr = line[len('installedargs: '):].strip()
if argstr[:1] == "'" and argstr[-1:] == "'":
argstr = argstr[1:-1].replace("''", "'")
installedargs = argstr.split()
if profile:
try:
profileyaml = subprocess.check_output([
sys.executable, '/opt/confluent/bin/apiclient',
'/confluent-public/os/{}/profile.yaml'.format(profile)]).decode('utf8')
except (subprocess.CalledProcessError, OSError) as e:
sys.stderr.write('Unable to read the installedargs of profile {}: {}\n'.format(profile, e))
profileyaml = ''
for line in profileyaml.split('\n'):
if line.startswith('installedargs:'):
installedargs = line.split('#', 1)[0][len('installedargs:'):].split() + installedargs
kcmdline = os.path.join(rootdir, 'etc/kernel/cmdline')
if os.path.exists(kcmdline):
with open(kcmdline) as kcmdlinein:
Expand All @@ -240,7 +270,7 @@ def fixup(rootdir, vols):
if arg.startswith(('net.ifnames=', 'biosdevname=')) and arg not in newkcmdlineent:
newkcmdlineent.append(arg)
with open(kcmdline, 'w') as kcmdlineout:
kcmdlineout.write(' '.join(newkcmdlineent) + '\n')
kcmdlineout.write(' '.join(append_args(newkcmdlineent, installedargs)) + '\n')
for loadent in glob.glob(os.path.join(rootdir, 'boot/loader/entries/*.conf')):
with open(loadent) as loadentin:
currentry = loadentin.read().split('\n')
Expand Down Expand Up @@ -277,7 +307,7 @@ def fixup(rootdir, vols):
newcfgparts.append(cfgpart)
else:
newcfgparts.append(cfgpart)
loadentout.write(' '.join(newcfgparts) + '\n')
loadentout.write(' '.join(append_args(newcfgparts, installedargs)) + '\n')
# the logical volumes this install created, for the defaults below
lvmargs = ''
for lvdev in (newrootdev, newswapdev):
Expand All @@ -300,7 +330,7 @@ def fixup(rootdir, vols):
]
if not os.path.exists(os.path.join(rootdir, "etc/kernel/cmdline")):
with open(os.path.join(rootdir, "etc/kernel/cmdline"), "w") as cmdlineout:
cmdlineout.write("{} root={}{}".format(currcmdlinestr, newrootdev, lvmargs))
cmdlineout.write("{} root={}{}".format(' '.join(append_args(currcmdline, installedargs)), newrootdev, lvmargs))
with open(grubsyscfg, 'w') as defgrubout:
for gline in defgrub:
gline = gline.split()
Expand All @@ -318,6 +348,10 @@ def fixup(rootdir, vols):
newline.append(ent)
else:
newline.append(ent)
if installedargs and newline and newline[0].startswith('GRUB_CMDLINE_LINUX=') and newline[-1].endswith('"'):
# an image captured from an installed node may carry them already
present = ' '.join(newline)[len('GRUB_CMDLINE_LINUX="'):-1].split()
newline = ['GRUB_CMDLINE_LINUX="{}"'.format(' '.join(append_args(present, installedargs)))]
defgrubout.write(' '.join(newline) + '\n')
grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
grubcfg = grubcfg.split('\n')
Expand Down
2 changes: 2 additions & 0 deletions confluent_osdeploy/rhvh4/profiles/default/scripts/pre.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ else
fi
curl -f https://$mgr/confluent-public/os/$profile/profile.yaml > /tmp/instprofile.yaml
blargs=$(grep ^installedargs: /tmp/instprofile.yaml | sed -e 's/#.*//' -e 's/^installedargs: //')
nodeargs=$(sed -n "s/^installedargs: //p" /etc/confluent/confluent.deploycfg | sed -e "s/^'\(.*\)'\$/\1/" -e "s/''/'/g")
if [ -n "$nodeargs" ]; then blargs="${blargs:+$blargs }$nodeargs"; fi
if [ ! -z "$blargs" ]; then
blargs=' --append="'$blargs'"'
fi
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,8 @@ proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg)
proto=${proto#protocol: }

append=$(grep ^installedargs: /tmp/profile.yaml | sed -e 's/^installedargs: //' -e 's/#.*//')
nodeargs=$(sed -n "s/^installedargs: //p" /etc/confluent/confluent.deploycfg | sed -e "s/^'\(.*\)'\$/\1/" -e "s/''/'/g")
if [ -n "$nodeargs" ]; then append="${append:+$append }$nodeargs"; fi
if grep console= /etc/fakecmdline >& /dev/null && [[ "$append" != *console=* ]]; then
append="$append console=${autocons#*/dev/}"
fi
Expand Down
2 changes: 2 additions & 0 deletions confluent_osdeploy/suse16/profiles/server/scripts/pre.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ sed -i 's@%%ROOTPASSWORD%%@'"$rootpw"'@g' /tmp/autoinstall.json
sed -i s!%%NODENAME%%!$(hostname)!g /tmp/autoinstall.json
python3 /opt/confluent/bin/apiclient /confluent-public/os/$profile/profile.yaml > /tmp/instprofile.yaml
blargs=$(grep ^installedargs: /tmp/instprofile.yaml | sed -e 's/#.*//' -e 's/^installedargs: //')
nodeargs=$(confluent_installedargs)
if [ -n "$nodeargs" ]; then blargs="${blargs:+$blargs }$nodeargs"; fi
blargs=$(printf '%s' "$blargs" | sed -e 's/[\\"]/\\&/g' -e 's/[\\&!]/\\&/g')
sed -i 's!%%INSTALLEDARGS%%!'"$blargs"'!g' /tmp/autoinstall.json
python3 /opt/confluent/bin/apiclient /confluent-public/os/$profile/scripts/getinstalldisk > /tmp/getinstalldisk
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
fi
fi
kargs=$(python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/profile.yaml | grep ^installedargs: | sed -e 's/^installedargs:[[:space:]]*//' -e 's/#.*//')
nodeargs=$(confluent_installedargs)
if [ -n "$nodeargs" ]; then kargs="${kargs:+$kargs }$nodeargs"; fi
if [ ! -z "$kargs" ]; then
kargs=$(printf '%s' "$kargs" | sed -e 's/[\/&]/\\&/g')
sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /etc/default/grub
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,11 @@ def set_machine_id(rootdir):

swapdev = '/dev/mapper/localstorage-swap'

def append_args(args, newargs):
# newargs go last, and an argument given twice keeps only its last place
merged = args + newargs
return [arg for idx, arg in enumerate(merged) if arg not in merged[idx + 1:]]

def fixup(rootdir, vols):
devbymount = {}
for vol in vols:
Expand Down Expand Up @@ -169,6 +174,31 @@ def fixup(rootdir, vols):
grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
if not os.path.exists(grubsyscfg):
grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
# installedargs of the profile followed by deployment.installedargs,
# which the OS installers add to the kernel arguments of the installed
# system
profile = None
installedargs = []
with open('/etc/confluent/confluent.deploycfg') as dcfgin:
for line in dcfgin:
if line.startswith('profile: '):
profile = line[len('profile: '):].strip()
elif line.startswith('installedargs: '):
argstr = line[len('installedargs: '):].strip()
if argstr[:1] == "'" and argstr[-1:] == "'":
argstr = argstr[1:-1].replace("''", "'")
installedargs = argstr.split()
if profile:
try:
profileyaml = subprocess.check_output([
sys.executable, '/opt/confluent/bin/apiclient',
'/confluent-public/os/{}/profile.yaml'.format(profile)]).decode('utf8')
except (subprocess.CalledProcessError, OSError) as e:
sys.stderr.write('Unable to read the installedargs of profile {}: {}\n'.format(profile, e))
profileyaml = ''
for line in profileyaml.split('\n'):
if line.startswith('installedargs:'):
installedargs = line.split('#', 1)[0][len('installedargs:'):].split() + installedargs
with open(grubsyscfg) as defgrubin:
defgrub = defgrubin.read().split('\n')
with open(grubsyscfg, 'w') as defgrubout:
Expand All @@ -179,6 +209,10 @@ def fixup(rootdir, vols):
if ent.startswith('resume=') or ent.startswith('rd.lvm.lv'):
continue
newline.append(ent)
if installedargs and newline and newline[0].startswith('GRUB_CMDLINE_LINUX=') and newline[-1].endswith('"'):
# an image captured from an installed node may carry them already
present = ' '.join(newline)[len('GRUB_CMDLINE_LINUX="'):-1].split()
newline = ['GRUB_CMDLINE_LINUX="{}"'.format(' '.join(append_args(present, installedargs)))]
defgrubout.write(' '.join(newline) + '\n')
grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
grubcfg = grubcfg.split('\n')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
fi
fi
kargs=$(curl https://$confluent_mgr/confluent-public/os/$confluent_profile/profile.yaml | grep ^installedargs: | sed -e 's/^installedargs:[[:space:]]*//' -e 's/#.*//')
nodeargs=$(confluent_installedargs /target/etc/confluent/confluent.deploycfg)
if [ -n "$nodeargs" ]; then kargs="${kargs:+$kargs }$nodeargs"; fi
if [ ! -z "$kargs" ]; then
kargs=$(printf '%s' "$kargs" | sed -e 's/[\/&]/\\&/g')
sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /target/etc/default/grub
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
fi
fi
kargs=$(curl https://$confluent_mgr/confluent-public/os/$confluent_profile/profile.yaml | grep ^installedargs: | sed -e 's/^installedargs:[[:space:]]*//' -e 's/#.*//')
nodeargs=$(confluent_installedargs /target/etc/confluent/confluent.deploycfg)
if [ -n "$nodeargs" ]; then kargs="${kargs:+$kargs }$nodeargs"; fi
if [ ! -z "$kargs" ]; then
kargs=$(printf '%s' "$kargs" | sed -e 's/[\/&]/\\&/g')
sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /target/etc/default/grub
Expand Down
9 changes: 9 additions & 0 deletions confluent_server/confluent/config/attributes.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,15 @@
'deployment.client_ip': {
'description': ('Client IP used when most recently reporting state.')
},
'deployment.installedargs': {
'description': ('Additional kernel command line arguments for the '
'installed OS, appended after the installedargs of '
'the profile. They are written to the bootloader '
'configuration at install time, and are not applied '
'to systems that are already installed. Characters '
'that GRUB treats specially, such as &, ;, |, <, >, $ '
'and quotes, are not supported.'),
},
'deployment.kernelargs': {
'description': ('Additional kernel command line arguments to pass at '
'network boot, appended after the kernelargs of the '
Expand Down
9 changes: 8 additions & 1 deletion confluent_server/confluent/selfservice.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@ def cleanup_inflight_sshcerts():


def yamldump(input):
return yaml.dump_all([input], Dumper=SafeDumper, default_flow_style=False).encode()
# no line wrapping, the deployment scripts read values with grep. This is
# the largest width the libyaml emitter accepts
return yaml.dump_all([input], Dumper=SafeDumper, default_flow_style=False, width=2**31 - 1).encode()

def jsondump(input):
return json.dumps(input).encode()
Expand Down Expand Up @@ -319,6 +321,11 @@ def verify_cert(certificate):
'value', None)
if ncfg['encryptboot'] in ('', 'none'):
ncfg['encryptboot'] = None
# one line of arguments, the installers read it with grep
installedargs = ' '.join(str(deployinfo.get(
'deployment.installedargs', {}).get('value', None) or '').split())
if installedargs:
ncfg['installedargs'] = installedargs
ncfg['profile'] = profile
protocol = deployinfo.get('deployment.useinsecureprotocols', {}).get(
'value', 'never')
Expand Down
Loading