Jenkins
Jenkins script console
print decrypted pass from config or credentials
# from jenkins script console,secret text
println(hudson.util.Secret.decrypt("{XXX}"))
# secretBytes
println(new String(com.cloudbees.plugins.credentials.SecretBytes.fromString("{YYY}").getPlainData(), "ASCII"))
# print all credentials
com.cloudbees.plugins.credentials.SystemCredentialsProvider.getInstance().getCredentials().forEach{
it.properties.each { prop, val ->
if (prop == "secretBytes") {
println(prop + "=>\n" + new String(com.cloudbees.plugins.credentials.SecretBytes.fromString("${val}").getPlainData()) + "\n")
} else {
println(prop + ' = "' + val + '"')
}
}
println("-----------------------")
}
or
def creds = com.cloudbees.plugins.credentials.CredentialsProvider.lookupCredentials(
com.cloudbees.plugins.credentials.common.StandardUsernameCredentials.class,
Jenkins.instance,
null,
null
)
for(c in creds) {
if(c instanceof com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey){
println(String.format("id=%s desc=%s key=%s\n", c.id, c.description, c.privateKeySource.getPrivateKeys()))
}
if (c instanceof com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl){
println(c.properties)
println(String.format("id=%s desc=%s user=%s pass=%s scope=%s\n", c.id, c.description, c.username, c.password, c.scope))
}
}
execute cmds from jenkins box
execute cmds from jenkins box
println "ifconfig".execute().text
move jobs to folder
def FOLDER_NAME = '<An existing destination folder>'
def JOB_REGEX = '<A regex to find your jobs>'
import jenkins.*
import jenkins.model.*
import hudson.*
import hudson.model.*
jenkins = Jenkins.instance
def folder = jenkins.getItemByFullName(FOLDER_NAME)
if (folder == null) {
println "ERROR: Folder '$FOLDER_NAME' not found"
return
}
// Find jobs in main folder
def found = jenkins.items.grep { it.name =~ "${JOB_REGEX}" }
println "Searching main folder : $found"
// Find jobs in other subfolders
jenkins.items.grep { it instanceof com.cloudbees.hudson.plugins.folder.Folder }.each { subfolder ->
if(!subfolder.getName().equals(FOLDER_NAME))
{
println "Searching folder '$subfolder.name'"
subfolder.getItems().grep { it.name =~ "${JOB_REGEX}" }.each { job ->
println "Found $job.name"
found.add(job);
}
}
}
// Move them
found.each { job ->
println "Moving '$job.name' to '$folder.name'"
Items.move(job, folder)
}
reload configuration-as-code
import io.jenkins.plugins.casc.ConfigurationAsCode;
ConfigurationAsCode.get().configure()
Github integration
GitHubHostKeyVerification
Go to ${JENKINS}/manage/configureSecurity/ and under Git Host Key Verification Configuration set:
Host Key Verification Strategy: manual
Approved Host Keys:
# github.com:22 SSH-2.0-babeld-f1c576c9
github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
# github.com:22 SSH-2.0-babeld-f1c576c9
github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
# github.com:22 SSH-2.0-babeld-f1c576c9
github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
# github.com:22 SSH-2.0-babeld-f1c576c9
# github.com:22 SSH-2.0-babeld-f1c576c9
Slack integration
https://plugins.jenkins.io/slack/
Keycloak integration
Configure Global Security
Goto ${JENKINS}/manage/configureSecurity/ and under Security Realm use:
Login wigh Openid Connect:
Client id: copy from keycloak
Client secret: copy from keycloak
Manual configuration:
Token server url: https://${KEYCLOAK}/realms/${REALM}/protocol/openid-connect/token
Token Authentication Method: Post
Authorization server url: https://${KEYCLOAK}/realms/${REALM}/protocol/openid-connect/auth
UserInfo server url: https://${KEYCLOAK}/realms/${REALM}/protocol/openid-connect/userinfo
Scopes: openid profile email roles
Logout from OpenID Provider: checked
End session URL for OpenID Provider: https://${KEYCLOAK}/realms/${REALM}/protocol/openid-connect/logout
Advanced:
User name field name: preferred_username
Full name field name: name
Email field name: email
# jenkins group has format with `/` as `/admins`
Groups field name: group-membership
For Authorization use Role-Based Strategy.
Manage and Assign Roles
Goto https://${JENKINS}/manage/role-strategy/
Manage roles as defined in following picture:
Assign roles as defined in following picture:
Add your user as admin so you can login.
Configuration as code snippet
securityRealm:
oic:
authorizationServerUrl: "https://${KEYCLOAK}/realms/${REALM}/protocol/openid-connect/auth"
automanualconfigure: "manual"
clientId: "jenkins"
clientSecret: "{AQAAABAAAAAw48TvUF+cGmNFoFypmJOCdClmRLmh08JZrEqgfa0ffC41OVvosAuG9MzuxO/zMbysgIIC96vPSZeY0cFyNzSxfw==}"
disableSslVerification: false
emailFieldName: "email"
endSessionEndpoint: "https://${KEYCLOAK}/realms/${REALM}/protocol/openid-connect/logout"
escapeHatchSecret: "{AQAAABAAAAAQVtOg0rUmr6K9Zn9+ziOctLqHnjceOYs9i/FfNpn1nSA=}"
fullNameFieldName: "name"
groupsFieldName: "group-membership"
scopes: "openid profile roles email"
tokenAuthMethod: "client_secret_post"
tokenServerUrl: "https://${KEYCLOAK}/realms/${REALM}/protocol/openid-connect/token"
userInfoServerUrl: "https://${KEYCLOAK}/realms/${REALM}/protocol/openid-connect/userinfo"
userNameField: "preferred_username"
Jenkins credentials
- https://docs.cloudbees.com/docs/admin-resources/latest/
- https://emilwypych.com/2019/06/15/how-to-pass-credentials-to-jenkins-pipeline/
- https://tomgregory.com/jenkins-assume-role-in-another-aws-account/
- http://tdongsi.github.io/blog/2017/07/18/basic-jenkinsfile-cookbook/
- http://tdongsi.github.io/blog/2017/12/30/groovy-hook-script-and-jenkins-configuration-as-code/
- https://www.cyberark.com/resources/threat-research-blog/configuring-and-securing-credentials-in-jenkins
Diagnostic
- https://support.cloudbees.com/hc/en-us/articles/222446987-Prepare-Jenkins-for-Support
- https://techbeacon.com/app-dev-testing/jenkins-performance-how-avoid-pitfalls-diagnose-issues-scale
- https://docs.cloudbees.com/docs/admin-resources/latest/jvm-troubleshooting/
Jenkins-as-code
- https://fishi.devtail.io/weblog/2019/01/06/jenkins-as-code-part-1/
- https://fishi.devtail.io/weblog/2019/01/12/jenkins-as-code-part-2/
- https://fishi.devtail.io/weblog/2019/02/09/jenkins-as-code-part-3/
- https://nickcharlton.net/posts/setting-jenkins-credentials-with-groovy.html
- http://tdongsi.github.io/blog/2017/12/30/groovy-hook-script-and-jenkins-configuration-as-code/
EC2 and Jenkins as code
Configure cloud nodes as
- amazonEC2:
cloudName: "jenkinsAgent"
region: "eu-west-1"
sshKeysCredentialsId: "JenkinsMasterPrivateKey"
templates:
- amiFilters:
- name: "tag:Profile"
values: "JenkinsAgent"
amiOwners: "${AWS_ACCOUNT_ID}"
amiType:
unixData:
rootCommandPrefix: "sudo"
sshPort: "22"
associatePublicIp: false
connectBySSHProcess: false
connectionStrategy: PRIVATE_IP
deleteRootOnTermination: true
description: "JenkinsAgent"
ebsEncryptRootVolume: DEFAULT
ebsOptimized: false
hostKeyVerificationStrategy: CHECK_NEW_HARD
iamInstanceProfile: "arn:aws:iam::${AWS_ACCOUNT_ID}:instance-profile/JenkinsAgentRole"
idleTerminationMinutes: "30"
javaPath: "java"
labelString: "ec2"
maxTotalUses: -1
metadataEndpointEnabled: true
metadataHopsLimit: 1
metadataTokensRequired: false
minimumNumberOfInstances: 0
minimumNumberOfSpareInstances: 0
mode: EXCLUSIVE
monitoring: false
numExecutors: 5
remoteAdmin: "ubuntu"
remoteFS: "/home/ubuntu"
securityGroups: "jenkinsAgent"
stopOnTerminate: false
t2Unlimited: false
tags:
- name: "ENV"
value: "CI"
- name: "Name"
value: "JenkinsAgent 20230124133950"
tenancy: Default
type: T3Medium
useEphemeralDevices: false
userData: |-
#!/bin/bash
sudo apt install unzip -y
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
useInstanceProfileForCredentials: true
API
~$ echo 'java -jar ~/jenkins-cli.jar -s http://localhost:8080 quiet-down' | sudo tee /usr/bin/jenkins-pause
~$ echo 'java -jar ~/jenkins-cli.jar -s http://localhost:8080 cancel-quiet-down' | sudo tee /usr/bin/jenkins-resume
~$ sudo chmod +x /usr/bin/jenkins-pause
~$ sudo chmod +x /usr/bin/jenkins-resume
Plugin mgmt
https://github.com/jenkinsci/plugin-installation-manager-tool
# installing jenkins-plugin-manager
wget https://github.com/jenkinsci/plugin-installation-manager-tool/releases/download/2.12.3/jenkins-plugin-manager-2.12.3.jar
# listing installed plugins
java -jar jenkins-plugin-manager-2.12.3.jar -l -d /var/lib/jenkins/plugins/ --war /usr/lib/jenkins/jenkins.war
# it will just download plugin in /var/lib/jenkins/plugins.
# next you need to restart jenkins or to unzip /var/lib/jenkins/plugins/subversion.hpi -d /var/lib/jenkins/plugins/subversion and reload-jenkins-from-disk
java -jar jenkins-plugin-manager-2.12.3.jar --plugins subversion:2.15.2 -d /var/lib/jenkins/plugins/ --war /usr/lib/jenkins/jenkins.war
disabling plugins
touch $JENKINS_HOME/plugins/greenballs.jpi.disabled
Jenkins Pipeline
create parametrized jenkins file
node {
def run
stage('Clean workspace'){
sh "rm -rf ${WORKSPACE}/*"
}
properties([
buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '5')),
parameters([
credentials(credentialType: 'com.cloudbees.jenkins.plugins.awscredentials.AWSCredentialsImpl', defaultValue: '', description: 'Define which AWS Credentials will be used', name: 'CREDENTIALS', required: false),
string(defaultValue: 'uat', description: 'Define environment to use: prod,uat,snapshot....', name: 'ENV', trim: false),
string(defaultValue: 'ENV', description: '''Define key which is used for tags. Keep in mind about following settings:
MToD is using \'tag:Environemnt\'=\'snapshot\'
Legacy clients are using \'tag:ENV\'=\'snapshot\'''', name: 'ENVTAGNAME', trim: false),
string(defaultValue: '-vv', description: 'optional ansible flags', name: 'FLAGS', trim: false),
[$class: 'WHideParameterDefinition', defaultValue: '2.7.5', description: '', name: 'ANSIBLE_VERSION'],
[$class: 'WHideParameterDefinition', defaultValue: 'master', description: 'Define repositoryBranch ( repositoryTags later) of repo \'operations-jenkins-ansible\' or other repos which are used in pipeline', name: 'repositoryBranch'],
string(defaultValue: 'eu-west-1', description: 'Default AWS region to use', name: 'AWS_REGION', trim: false)])
])
stage('Checkout operations-jenkins-ansible'){
checkout scm: [$class: 'GitSCM', userRemoteConfigs: [[url: '[email protected]:milekitic/operations-jenkins-ansible.git']],
branches: [[name: 'refs/heads/${repositoryBranch}']]],
changelog: false,
poll: false,
credentialsId: 'ac0c5a90-8d88-46f0-951d-e4463eea89b7'
}
stage('Load external job configuration'){
fileLoader.withGit('[email protected]:milekitic/operations-jenkins-common.git', 'master', 'ac0c5a90-8d88-46f0-951d-e4463eea89b7', '') {
run = fileLoader.load('configuration/jobscommon.groovy');
}
}
stage('Run ansible playbook'){
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: "${CREDENTIALS}", secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
run.ansible("AWS_REGION=$AWS_REGION AWS_SECURITY_TOKEN=$AWS_SESSION_TOKEN ansible-playbook -i inventory/localhost playbooks/ops/system-update-all.yml -e 'env=${ENV}' -e 'EnvTagName=${ENVTAGNAME}' ${FLAGS}")
}
}
}
Declarative pipeline
Polling changes from SCM
pipeline {
agent any
triggers {
pollSCM('') //Empty quotes tells it to build on a push
}
}
github push alternative
- generic webhook plugin
https://github.com/jenkinsci/generic-webhook-trigger-plugin
https://github.com/jenkinsci/generic-webhook-trigger-plugin/issues/14
- github actions
https://stackoverflow.com/questions/64050510/trigger-jenkins-build-when-pull-request-is-merged-in-github
Backup jenkins
.gitconfig content
# Git untracked files to ignore.
# Cache.
.cache/
# Fingerprint records.
fingerprints/
# Working directories.
workspace/
# Secret files.
secrets/
secret.*
*.enc
*.key
users/
id_rsa
# Plugins.
plugins/
# State files.
*.state
# Job state files.
builds/
lastStable
lastSuccessful
nextBuildNumber
# Updates.
updates/
# Hidden files.
.*
# Except git config files.
!.git*
!.ssh/
# User content.
userContent/
# Log files.
logs/
*.log
# Miscellaneous litter
*.tmp
*.old
*.bak
*.jar
*.json
*.lastExecVersion
git init
git add .gitconfig
git add config.xml jobs/ .gitconfig
git commit -am 'Adds Jenkins config files'
Use git variables as envs
pipeline {
options {
skipDefaultCheckout true
}
stages {
stage('Checkout source code') {
steps {
script {
Map scmVars = checkout([/* Some checkout paramaters */])
// Setting up GIT_* env variables manually
scmVars.each { k, v ->
env[k] = v
}
}
}
}
stage('Print GIT env variables'){
steps {
sh 'env | grep GIT_'
// GIT_COMMIT=f16d1ef66ac5575c4e7cc588356c630aeb4efd69
// GIT_BRANCH=origin/feature/add-spring-security
// [email protected]:XXXX/XXXX.git
// GIT_PREVIOUS_COMMIT=f16d1ef66ac5575c4e7cc588356c630aeb4efd69
}
}
}
}
Install on ec2
#!/bin/bash
yum update –y
wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo
rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key
yum upgrade
amazon-linux-extras install java-openjdk11 -y
yum install jenkins -y
systemctl enable --now jenkins
Jenkins on k8s
helm install jenkins jenkinsci/jenkins -f helm/jenkins/values.yaml
sts initcontainer for jenkins must be updated as (it won’t work with emptyDir{})
env:
- name: CACHE_DIR
value: /tmp/cache
Jenkins master on ubuntu 22.04
#!/bin/bash
sudo apt update && \
sudo apt install openjdk-11-jdk -y && \
curl -fsSL https://pkg.jenkins.io/debian-stable/jenkins.io.key | sudo tee /usr/share/keyrings/jenkins-keyring.asc > /dev/null && \
echo deb [signed-by=/usr/share/keyrings/jenkins-keyring.asc] https://pkg.jenkins.io/debian-stable binary/ | sudo tee /etc/apt/sources.list.d/jenkins.list > /dev/null && \
sudo apt update && \
sudo apt install jenkins -y && \
sudo systemctl enable --now jenkins && \
sudo iptables -I INPUT -p tcp --dport 8080 -j ACCEPT
Install caddy for jenkins
sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https && \
curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg && \
curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list && \
sudo apt update && \
sudo apt install caddy
sudo iptables -I INPUT -p tcp --dport 80 -j ACCEPT
sudo iptables -I INPUT -p tcp --dport 443 -j ACCEPT
Jenkins agent,ubuntu
sudo apt update -y
sudo apt install openjdk-11-jdk -y
sudo apt install apt-transport-https ca-certificates curl software-properties-common -y
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt update
sudo apt install docker-ce -y
sudo systemctl enable --now docker
sudo usermod -aG docker ubuntu
REFERENCES
- https://www.cyberark.com/resources/threat-research-blog/configuring-and-securing-credentials-in-jenkins
- https://www.baeldung.com/ops/jenkins-slack-integration
- https://www.jenkins.io/blog/2017/02/15/declarative-notifications/
- https://www.jenkins.io/blog/2016/07/18/pipeline-notifications/
- https://plugins.jenkins.io/slack/
- https://devopscube.com/jenkins-multibranch-pipeline-tutorial/
- https://sysadmins.co.za/install-jenkins-on-ubuntu-22-04/
- https://sysadmins.co.za/setup-ssl-for-jenkins-with-caddy-on-ubuntu-22-04/
- https://github.com/igorstojanovski/jenkins-pipeline-as-code/blob/sonar-test/Jenkinsfile
- https://igorski.co/sonarqube-scans-using-jenkins-declarative-pipelines/