Discover svn-branches and auto-create Jenkins-jobs

Kap Arkona (Rügen), own photo (by Christoph Burmeister)

Kap Arkona (Rügen), own photo (by Christoph Burmeister)


When working with branches, sometimes it’s really annoying to create Jenkins-jobs manually, because whenever a new branch was created by dev-teams, the devops-team is asked to create the corresponding Jenkins-job as soon as possible to enable a a working ci-infrastructure. So why don’t use Jenkins itself to look for new branches?

In this post, I show a simple script that uses svn-list-command in branches-directory to get the existing branches. Then the script makes a lookup into Jenkins‘ jobs-directory. If a directory with the job-name (composed by ) doesn’t exist the mechanism from my last post is used to create a job via Jenkins‘ remote API.

Please note, this is not the best code, but it works and Groovy has the charme to get well modified 🙂


Environment:

  • Groovy: 2.1.5
  • JRE: Hotspot 1.7.0_17
  • Jenkins: 1.522
  • SVN: 1.7

The SVN-structure is the following:

svn/testrepo
. testcomponent_1
… branches
….. release_1.0
….. release_1.1
….. release_1.2
….. release_2.0
….. release_2.1
. testcomponent_2
… branches
….. release_1.0
….. release_1.1
….. release_2.0
. testcomponent_3
… branches
….. release_1.0
….. release_2.0
….. release_2.1

import org.apache.commons.httpclient.*
import org.apache.commons.httpclient.auth.*
import org.apache.commons.httpclient.methods.*

/**
 * needs 
 * commons-codec-1.8.jar
 * commons-httpclient-3.1.jar
 * commons-logging-1.1.1.jar * 
 * in classpath
 */

/**
 * pattern for jenkins-job = <component>-<branchname>
 * pattern for scm = <base>/<component>/branches/<branchname>
 */

// svn-data
def svnBaseUrl = "http://localhost:9000/svn/testrepo"
def svnUsername = "christoph"
def svnPassword = "topfsecret"

// jenkins-data
def jenkinsUrl = "http://localhost:10000/"
def jenkinsUsername = "admin"
def jenkinsApiToken = "93edde39fkfffjc752e87dfcc37662"
def jenkinsSecurityRealm = "Benutzer Jenkins"

// other-data
def listOfComponents = [
	"testcomponent_1",
	"testcomponent_2",
	"testcomponent_3"
];
def localTmpDir = "./tmp"
def jobConfigTemplate = "config.xml.template"


main()


/**
 * - iterate over all components, given in listOfComponents-array
 * - executing "svn list --xml" over branches-directory of each component
 * - fetching the output and iterate over the branches that were found in svn-list-xml
 * - check, if a job for the branch/component-combination exists
 * - create a job, if nothing exists yet
 */
def main(){
	listOfComponents.each(){
		def component = it
		println "found component ${component}"
		svnCommand = "svn list --xml ${svnBaseUrl}/${component}/branches"
		def proc = svnCommand.execute()
		proc.waitFor()
		def xmlOutput = proc.in.text

		def tmpDir = new File(localTmpDir)
		if (tmpDir.exists()){
			tmpDir.delete()
		}
		tmpDir.mkdirs()
		def svnInfoFile = new File("${localTmpDir}/${component}_svnInfo.xml")
		svnInfoFile.write(xmlOutput)

		def lists = new XmlSlurper().parse(svnInfoFile)

		def listOfBranches = lists.list.entry.name

		// iterate through branches
		listOfBranches.each(){
			def branchName = it.text()
			println "- found branch '${branchName}'"
			println "--- checking job for '${component}' with branch '${branchName}'"
			if (!jobForBranchExists(component, branchName)){
				createJobForBranch(jenkinsUrl, jenkinsUsername, jenkinsApiToken, jenkinsSecurityRealm, jobConfigTemplate, svnBaseUrl, component, branchName, localTmpDir)
			}
		}
	}
}

/**
 * - create a job with several params via Jenkins' remote API
 * - uses a modified version of the given config.xml-template
 * 
 * @param jenkinsUrl
 * @param jenkinsUsername
 * @param jenkinsApiToken
 * @param jenkinsSecurityRealm
 * @param jobConfigTemplate
 * @param svnBaseUrl
 * @param component
 * @param branchName
 * @param localTmpDir
 * @return
 */
def createJobForBranch(String jenkinsUrl, String jenkinsUsername, String jenkinsApiToken, String jenkinsSecurityRealm, String jobConfigTemplate, String svnBaseUrl, String component, String branchName, String localTmpDir){
	def projectName = "${component}-${branchName}"
	def url = new URL(jenkinsUrl)
	def server = url.getHost()
	def port = url.getPort()

	def client = new HttpClient()
	client.state.setCredentials(
			new AuthScope( server, port, jenkinsSecurityRealm),
			new UsernamePasswordCredentials( jenkinsUsername, jenkinsApiToken )
			)

	client.params.authenticationPreemptive = true

	def post = new PostMethod( "${jenkinsUrl}/createItem?name=${projectName}" )
	post.doAuthentication = true

	File configXml = createJobConfigXml(jobConfigTemplate, component, branchName, svnBaseUrl, localTmpDir)
	RequestEntity entity = new FileRequestEntity(configXml, "text/xml; charset=UTF-8");
	post.setRequestEntity(entity);
	try {
		int result = client.executeMethod(post)
		if (result != 200) {
			// not nice, but the easiest way
			throw new Exception("http-result-code:" + result);
		}
		println "Return code: ${result}"
		post.responseHeaders.each{ println it.toString().trim() }
		new File("${localTmpDir}/response.html").withWriter{ it << post.getResponseBodyAsString() }
	} finally {
		post.releaseConnection()
	}
	println "--- created Jenkins job '${component}-${branchName}', pointing to url ${svnBaseUrl}/${component}/branches/${branchName}"
}

/**
 * create a job-specific config.xml by replacing the content of config-xml-template with correct values for scm-url
 * 
 * @param jobConfigTemplate
 * @param component
 * @param branchName
 * @param svnBaseUrl
 * @param localTmpDir
 * @return the correct config.xml-file
 */
def createJobConfigXml(String jobConfigTemplate, String component, String branchName, String svnBaseUrl, String localTmpDir){
	def jobConfigFile = new File("${localTmpDir}/${component}-${branchName}_config.xml")
	def jobConfigTemplateFile = new File("${jobConfigTemplate}")

	def svnUrl = "${svnBaseUrl}/${component}/branches/${branchName}"

	def jobConfigTemplateText = jobConfigTemplateFile.text
	def jobConfigText = jobConfigTemplateText.replace("@@@scm-url@@@", svnUrl)
	jobConfigFile.write(jobConfigText)
	return jobConfigFile
}

/**
 * looks into Jenkins' jobs-directory, if there is already a job for that branch of this component 
 * @param component
 * @param branchName
 * @return true, if a job exists and false if not
 */
def jobForBranchExists(String component, String branchName){
	def env = System.getenv()
	def jobDirectory = new File(env['JENKINS_HOME'] + "/jobs/${component}-${branchName}")
	if (jobDirectory.exists() ){
		println "--- /jobs/${component}-${branchName} already exists, skipping creation"
		return true
	}
	println "--- /jobs/${component}-${branchName} doesn't exist, starting creation"
	return false
}

The config-template:

<?xml version='1.0' encoding='UTF-8'?>
<project>
  <actions/>
  <description></description>
  <keepDependencies>false</keepDependencies>
  <properties/>
  <scm class="hudson.scm.SubversionSCM" plugin="subversion@1.45">
    <locations>
      <hudson.scm.SubversionSCM_-ModuleLocation>
        <remote>@@@scm-url@@@</remote>
        <local>.</local>
        <depthOption>infinity</depthOption>
        <ignoreExternalsOption>false</ignoreExternalsOption>
      </hudson.scm.SubversionSCM_-ModuleLocation>
    </locations>
    <excludedRegions></excludedRegions>
    <includedRegions></includedRegions>
    <excludedUsers></excludedUsers>
    <excludedRevprop></excludedRevprop>
    <excludedCommitMessages></excludedCommitMessages>
    <workspaceUpdater class="hudson.scm.subversion.UpdateUpdater"/>
    <ignoreDirPropChanges>false</ignoreDirPropChanges>
    <filterChangelog>false</filterChangelog>
  </scm>
  <canRoam>true</canRoam>
  <disabled>false</disabled>
  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
  <triggers class="vector"/>
  <concurrentBuild>false</concurrentBuild>
  <builders>
    <hudson.tasks.BatchFile>
      <command>echo &quot;jenkins rocks&quot;</command>
    </hudson.tasks.BatchFile>
  </builders>
  <publishers/>
  <buildWrappers/>
</project>

By putting the script into a Groovy-Jenkins-job, you can enable a continous check of the svn, for example every 30min. If a new branch was found, a new job will be created on the fly. That’s it.