Quagga/infra/buildbot/master/master.cfg

# -*- python -*-
# ex: set syntax=python:

import json

from buildbot.plugins import *
from buildbot.plugins import buildslave, util

# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

quaggagit = 'git://git.sv.gnu.org/quagga.git'

# password defs
execfile("pass.cfg")

# filter a given 'workers' entry into a property list
# suitable for public display
def workers2publicprops (worker):
	publicprops = [ "os", "version", "vm", "pkg", "texi", "cc",
			"latent", "env" ]
	return  { k:worker[k] for k in worker if k in publicprops }

# get compiler names
def compilers(workers):
	names = set ()
	for l in [ w["cc"] for w in workers.values ()]:
		for e in l:
			names |= { e["n"] }
	return names

# bots with given compiler tag
def ccbots (cc):
	return [ workers[kw]["bot"] for kw in workers 
			if len([c["n"] for c in workers[kw]["cc"] 
				if c["n"] is cc]) > 0]

# vm: non-VM are assumed faster and used for initial build
# pkg: rpm, sysv, dpkg - only do test rpm builds at moment
# texi: True or "true" if we can use for doc building
# cc: List of dicts of installed compilers, with:
#     { "n": <tag>, "v": <version>, "bin": <command>) }
#     tags: gcc, clang, sunpro, tcc
# env: JSON dictionary of environment variables to be passed to shell
#      commands as a dictionary via json.load.
# latent: VM spun up on demand via LatentSlave, uses "session" for
#         the libvirt URI.
#
# When latent is true:
# session: libvirt URI to use for latent workers. Default will be set on
#          latent VMs if not specified.
# hd_image: libvirt image to use
workers = {
	"fedora-24": { 
	  "os": "Fedora",
	  "version": "24",
	  "vm": False,
	  "pkg": "rpm",
	  "texi": True,
	  "cc": [ { "n": "gcc",   "v": "6.3.1"},
	  	  { "n": "clang", "v": "3.8.1"},
	  	  { "n": "gcc",   "v": "3.4.6", "bin": "gcc34"},
	  	],
	}, 
	"fedora-26": { 
	  "os": "Fedora",
	  "version": "26",
	  "vm": False,
	  "pkg": "rpm",
	  "cc": [ { "n": "gcc",   "v": "7.0.1" },
	  	  { "n": "clang", "v": "3.9.0" },
	  	  { "n": "gcc",   "v": "3.4.6", "bin": "gcc34" },
	  	],
	}, 
	"centos-7": {
	  "os": "CentOS",
	  "version": "7",
	  "vm": False,
	  "pkg": "rpm",
	  "cc": [ { "n": "gcc", "v": "4.8.5"} ],
	},
	"ubuntu-16.10": {
	  "os": "Ubuntu",
	  "version": "16.10",
	  "vm": False,
	  "pkg": "dpkg",
	  "cc": [
	  	{ "n": "gcc", "v": "6.2.0"},
	  	{ "n": "tcc", "v": "0.9.26"},
	  ],
	},
	"debian-8": {
	  "os": "Debian",
	  "version": "8",
	  "vm": True,	
  	  "pkg": "dpkg",
  	  "latent": True,
  	  "cc": [ { "n": "gcc", "v": "4.9.2"} ],
  	  "hd_image": "/var/lib/libvirt/images/debian8.qcow2",
  	},
  	"debian-9": { 
	  "os": "Debian",
	  "version": "9",
	  "vm": True,	
  	  "pkg": "dpkg",
  	  "cc": [ { "n": "gcc", "v": "6.3.0"} ],
  	  "latent": True,
  	  "hd_image": "/var/lib/libvirt/images/debian9.qcow2",
  	},
  	"freebsd-10": { 
	  "os": "FreeBSD",
	  "version": "10",
	  "vm": True,	
  	  "pkg": "",
  	  "latent": True,
  	  "cc": [ { "n": "clang", "v": "3.4.1"} ],
  	  "hd_image": "/var/lib/libvirt/images/freebsd103.qcow2",
  	}, 
  	"freebsd-11": { 
	  "os": "FreeBSD",
	  "version": "11",
	  "vm": True,	
  	  "pkg": "",
  	  "cc": [ {"n": "gcc",    "v": "4.9.4"},
  	  	  { "n": "clang", "v": "3.8.0"},
  	  ],
  	  "latent": True,
  	  "hd_image": "/var/lib/libvirt/images/freebsd110.qcow2",
  	},
  	"openbsd-6": { 
	  "os": "OpenBSD",
	  "version": "6.0",
	  "vm": True,	
  	  "pkg": "",
  	  "cc": [ { "n": "gcc", "v": "4.2.1"} ],
  	  "latent": True,
  	  "hd_image": "/var/lib/libvirt/images/openbsd-6.qcow2",
	  "env": ' { "AUTOMAKE_VERSION": "1.15", "AUTOCONF_VERSION": "2.69" } ',
  	},
  	"oi-hipster": {
  	  "os": "OpenIndiana",
  	  "version": "hipster",
  	  "vm": True,
  	  "pkg": "sysv",
	  "latent": True,
	  "cc": [ { "n": "gcc",    "v": "6.3.0"},
	  	  { "n": "sunpro", "v": "12.0" }, 
	  	  { "n": "gcc",    "v": "4.4.4"}
	  	],
	  "hd_image": "/var/lib/libvirt/images/buildbot-oi-hipster.qcow2",
  	},
}

# workers config preparation:
# - add defaults for important fields not set
# - add passwords
def workers_prep (workers):
	for kw in workers:
		w = workers[kw]
		w["bot"] = "buildbot-" + kw
		# 'latent' must be set.
		if "latent" not in w:
			w["latent"] = False
		# set default libvirt session for latent bots
		if w["latent"] and ("session" not in w):
			w["session"] = 'qemu+ssh://buildbot@sagan.jakma.org/system'
		w["pass"] = workers_pass[kw]

workers_prep (workers)

analyses_builders = [ "clang-analyzer" ]

osbuilders = ["build-" + kw for kw in workers]
osfastbuilders = ["build-" + kw for kw in workers if workers[kw]["vm"] == False]
osslowbuilders = ["build-" + kw for kw in workers if workers[kw]["vm"] == True]

rpmbuilders = ["rpm-" + kw for kw in workers if workers[kw]["pkg"] == "rpm"]

allbuilders =  []
allbuilders += osbuilders
allbuilders += rpmbuilders
allbuilders += analyses_builders
allbuilders += ["commit-builder"]
allbuilders += ["build-distcheck"]
allbuilders += ["build-docs" ]

# Force merging of requests.
# c['mergeRequests'] = lambda *args, **kwargs: True

####### BUILDSLAVES
c['slaves'] = []

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.


for w in (w for w in workers.values() if ("latent" not in w)
				      or (w["latent"] == False)):
	c['slaves'].append(buildslave.BuildSlave(w["bot"], w["pass"],
				properties=workers2publicprops (w),
	))

for w in (w for w in workers.values()
		  if ("latent" in w) 
		     and w["latent"]
		     and "hd_image" in w):
	c['slaves'].append(buildslave.LibVirtSlave(
					w["bot"],
					w["pass"],
					util.Connection(w["session"]),
					w["hd_image"],
					properties=workers2publicprops (w),
	))

# 'protocols' contains information about protocols which master will use for
# communicating with slaves.
# You must define at least 'port' option that slaves could connect to your master
# with this protocol.
# 'port' must match the value configured into the buildslaves (with their
# --master option)
c['protocols'] = {'pb': {'port': 9989}}

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.  Here we point to the buildbot clone of pyflakes.

c['change_source'] = []
c['change_source'].append(changes.GitPoller(
	quaggagit,
        workdir='gitpoller-workdir', 
	branches=True,
#	branches=['master','volatile/next'],
        pollinterval=300))

####### REVISION LINKS
# associate changesouce repositories to URI templates for code view
#
c['revlink'] = util.RevlinkMatch([quaggagit + r"(.*)"],
	r"http://git.savannah.gnu.org/cgit/quagga.git/commit/?id=%s")

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes. 

# We want a first line of 'quick' builds, which then trigger further builds.
#
# A control-flow builder, "commit-builder", used to sequence the 'real'
# sets of builders, via Triggers.

c['schedulers'] = []
c['schedulers'].append(schedulers.SingleBranchScheduler(
                            name="master-change",
                            change_filter=util.ChangeFilter(branch='master'),
                            treeStableTimer=10,
                            builderNames=[ "commit-builder" ]))

c['schedulers'].append(schedulers.SingleBranchScheduler(
                            name="next-change",
                            change_filter=util.ChangeFilter(
                            	branch='volatile/next'),
                            treeStableTimer=10,
                            builderNames=[ "commit-builder" ] ))

# Initial build checks on faster, non-VM
c['schedulers'].append(schedulers.Triggerable(
	name="trigger-build-first",
	builderNames=osfastbuilders))

# Build using remaining builders, after firstbuilders.
c['schedulers'].append(schedulers.Triggerable(
	name="trigger-build-rest",
	builderNames=osslowbuilders))

# Analyses tools, e.g. CLang Analyzer scan-build
c['schedulers'].append(schedulers.Triggerable(
		       name="trigger-build-analyses",	
		       builderNames=analyses_builders))
# Dist check
c['schedulers'].append(schedulers.Triggerable(
		       name="trigger-distcheck",	
		       builderNames=["build-distcheck"]))
# RPM check and build
c['schedulers'].append(schedulers.Triggerable(
		       name="trigger-rpm",	
		       builderNames=rpmbuilders))

# Doc build check (non-nightly, so no upload)
c['schedulers'].append(schedulers.Triggerable(
		       name="trigger-build-docs",	
		       builderNames=["build-docs"]))

# Try and force schedulers
c['schedulers'].append(schedulers.ForceScheduler(
                       name="force",
                       builderNames=allbuilders))

c['schedulers'].append(schedulers.Try_Userpass(
		       name="try",
		       builderNames=osbuilders
		       		    + rpmbuilders
				    + ["build-distcheck", 
				       "clang-analyzer",
				       "build-docs" ],
			userpass=users,
			port=8031))

## nightly docs build
c['schedulers'].append(schedulers.Nightly(
	name="nightly-docs",
	branch="master",
	builderNames=[ "build-docs" ],
	hour=3,
	minute=0,
	onlyIfChanged=True,
	properties = { "nightly": True },
))


####### BUILDERS
c['builders'] = []

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them.  Note that any particular build will
# only take place on one slave.

# Renderer to extract simple JSON dictionary object from the configuration
# via the 'env' property, and return as a python dictionary, so it can
# be passed to the 'env=' argument that some Steps allow.
@util.renderer
def get_config_env (props):
	jsonenv = props.getProperty('env')
	env = None
	if jsonenv:
		env = json.loads (jsonenv)
	return env

def makecmd (target):
	if isinstance(target,basestring):
		return [ "make", "-j", "2", target ]
	return [ "make", "-j", "2" ] + target

step_git = steps.Git(repourl=quaggagit, mode='incremental')
step_autoconf = steps.ShellCommand(command=["./update-autotools"],
			   description="generating autoconf",
			   descriptionDone="autoconf",
			   haltOnFailure=True,
			   env=get_config_env)

git_and_autoconf = [
	step_git, step_autoconf,
]
step_configure = steps.Configure(command="../build/configure")
step_clean = steps.ShellCommand(command=makecmd("clean"),
				description="cleaning",
				descriptionDone="make clean")

configure_and_clean = [ step_configure, step_clean ]
common_setup = git_and_autoconf + configure_and_clean

### Default 'check' build, builder instantiated for each OS

factory = util.BuildFactory()

factory.addSteps(common_setup)

factory.addStep(steps.Compile(command=makecmd("all")))
factory.addStep(steps.ShellCommand(command=makecmd("check"),
				   description="checking",
				   descriptionDone="make check"))

# create builder for every OS, for every buildbot
# XXX: at moment this assumes 1:1 OS<->bot
for kw in workers:
   c['builders'].append(util.BuilderConfig(
	name="build-" + kw,
	slavenames=workers[kw]["bot"],
	factory=factory))

### distcheck Builder, executed on any available bot
factory = util.BuildFactory()

factory.addSteps(common_setup)

factory.addStep(steps.ShellCommand(command=makecmd("distcheck"),
				   description="run make distcheck",
				   descriptionDone="make distcheck"))
c['builders'].append(
	util.BuilderConfig(name="build-distcheck",
	slavenames=list(w["bot"] for w in workers.values()),
	factory=factory,
))

### LLVM clang-analyzer build, executed on any available non-VM bot

f = util.BuildFactory()

f.addSteps(common_setup)

f.addStep(steps.SetProperty(property="clang-id",
	value=util.Interpolate("%(prop:commit-description)s-%(prop:buildnumber)s")))

f.addStep(steps.SetProperty(property="clang-output-dir",
	value=util.Interpolate("../CLANG-%(prop:clang-id)s")))
f.addStep(steps.SetProperty(property="clang-uri",
	value=util.Interpolate("/clang-analyzer/%(prop:clang-id)s")))
# relative to buildbot master working directory
f.addStep(steps.SetProperty(property="clang-upload-dir",
	value=util.Interpolate("public_html/clang-analyzer/%(prop:clang-id)s")))

f.addStep(steps.Compile(command=["scan-build",
			      	 "-analyze-headers",
			      	 "-o",
			      	 util.Interpolate("%(prop:clang-output-dir)s"),
				 "make", "all"]))
f.addStep(steps.DirectoryUpload(
	  slavesrc=util.Interpolate("%(prop:clang-output-dir)s"),
	  masterdest = util.Interpolate("%(prop:clang-upload-dir)s"),
	  compress = 'bz2',
	  name = "clang report",
	  url = util.Interpolate("%(prop:clang-uri)s"),
))
f.addStep(steps.RemoveDirectory(
	dir=util.Interpolate("%(prop:clang-output-dir)s")
))


c['builders'].append(
    util.BuilderConfig(name="clang-analyzer",
	slavenames=ccbots ("clang"),
        factory=f))

### RPM: check and build
f = util.BuildFactory ()

# check out the source
f.addStep(steps.Git(repourl=quaggagit, mode='full'))

f.addStep(step_autoconf)
f.addStep(step_configure)

f.addStep(steps.ShellCommand(command=makecmd("dist"),
				     description="run make dist",
				     descriptionDone="make dist"))
# not imported somehow
#f.addStep(steps.RpmLint(fileloc="redhat/quagga.spec"))
f.addStep(steps.ShellCommand(command=["rpmlint", "-i", "redhat/quagga.spec"],
				description="run rpmlint",
				descriptionDone="rpmlint"))
f.addStep(steps.RpmBuild(specfile="redhat/quagga.spec"))
#			 rpmdir=util.Interpolate("%(prop:builddir)s/rpm")))

# XXX: assuming 1:1 OS:buildbot mapping
for kw in (kw for kw in workers if workers[kw]["pkg"] == "rpm"):
	c['builders'].append(
    		util.BuilderConfig(name="rpm-" + kw,
      				   slavenames="buildbot-" + kw,
      				   factory=f
      		)
      	)

### Build documentation

def build_is_nightly (step):
    n = step.getProperty("nightly")
    if n == True or n == "True" or n == "true":
       return True
    return False

f = util.BuildFactory ()
f.addStep(steps.Git(repourl=quaggagit, mode='full'))
f.addStep(step_autoconf)
f.addStep(steps.Configure(command=["../build/configure"],
			  workdir="docs"))
f.addStep(steps.ShellCommand(command=["make", "V=99", "quagga.html"],
			     description="making split HTML doc",
			     descriptionDone="docs: split HTML",
			     workdir="docs/doc",
			     haltOnFailure=True,
))
#f.addStep(steps.FileUpload(
#	  slavesrc="build/doc/fig-normal-processing.png",
#	  masterdest = "public_html/docs/nightly/quagga/",
#	  name = "Upload Fig 1",
#	  doStepIf=build_is_nightly,
#))
#f.addStep(steps.FileUpload(
#	  slavesrc="build/doc/fig-rs-processing.png",
#	  masterdest = "public_html/docs/nightly/quagga/",
#	  name = "Upload Fig 2",
#	  doStepIf=build_is_nightly,
#))
f.addStep(steps.MultipleFileUpload(
	  slavesrcs=[ "doc/fig-rs-processing.png",
		      "doc/fig-normal-processing.png" ],
	  masterdest = "public_html/docs/nightly/quagga/",
	  name = "Upload Figures",
	  doStepIf=build_is_nightly,
))
f.addStep(steps.DirectoryUpload(
	  slavesrc="quagga.html",
	  masterdest = "public_html/docs/nightly/quagga",
	  compress = 'bz2',
	  name = "Upload split HTML",
	  url = "/docs/nightly/quagga/index.html",
	  workdir="docs/doc",
	  doStepIf=build_is_nightly,
))
f.addStep(steps.RemoveDirectory(
	  dir="docs/doc/quagga.html",
))
f.addStep(steps.ShellCommand(command=["make", "V=99",
					"MAKEINFOFLAGS=--no-split",
					"quagga.html"],
			     description="making one-page HTML doc",
			     descriptionDone="docs: one-page HTML",
			     workdir="docs/doc",
			     haltOnFailure=True
))
f.addStep(steps.FileUpload(
	  slavesrc="quagga.html",
	  masterdest = "public_html/docs/nightly/quagga/quagga.html",
	  name = "Upload single HTML",
	  url = "/docs/nightly/quagga/quagga.html",
	  workdir="docs/doc",
	  doStepIf=build_is_nightly,
))
f.addStep(steps.ShellCommand(command=["make", "V=99", "quagga.pdf"],
			     description="making PDF docs",
			     descriptionDone="docs: PDF",
			     workdir="docs/doc"
))
f.addStep(steps.FileUpload(
	  slavesrc="quagga.pdf",
	  masterdest = "public_html/docs/nightly/quagga/quagga.pdf",
	  name = "Upload PDF",
	  url = "/docs/nightly/quagga/quagga.pdf",
	  workdir="docs/doc",
	  doStepIf=build_is_nightly,
))

c['builders'].append(
    util.BuilderConfig(name="build-docs",
      slavenames=[w["bot"] for w in workers.values() 
			if "texi" in w and w["texi"] == True ],
      factory=f
))

### Co-ordination builds used to sequence parallel builds via Triggerable

# to understand this you have to read this list and the Triggered schedulers
# to see what sets of builds are being sequenced. Bit clunky, but Buildbot
# doesn't have a way to just specify a pipeline of groups of builders more
# cleanly.

f = util.BuildFactory()
f.addStep(steps.Trigger (
	schedulerNames = [ "trigger-build-first" ],
	waitForFinish=True,
	updateSourceStamp=True
))
f.addStep(steps.Trigger (
	schedulerNames = [ "trigger-build-rest" ],
	waitForFinish=True,
	updateSourceStamp=True
))
f.addStep(steps.Trigger (
	schedulerNames = [ "trigger-build-analyses", "trigger-distcheck",
			   "trigger-build-docs" ],
	waitForFinish=True,
	updateSourceStamp=True
))
f.addStep(steps.Trigger (
	schedulerNames = [ "trigger-rpm" ],
	waitForFinish=True,
	updateSourceStamp=True
))

c['builders'].append(
    util.BuilderConfig(name="commit-builder",
      slavenames=[w["bot"] for w in workers.values() if not w["vm"]],
      factory=f)
)

####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

from buildbot.status import html
from buildbot.status.web import authz, auth

authz_cfg=authz.Authz(
    # change any of these to True to enable; see the manual for more
    # options
    #auth=auth.BasicAuth([("pyflakes","pyflakes")]),
    auth=util.BasicAuth(users),
    gracefulShutdown = False,
    forceBuild = 'auth', # use this to test your slave once it is set up
    forceAllBuilds = 'auth',  # ..or this
    pingBuilder = 'auth',
    stopBuild = 'auth',
    stopAllBuilds = 'auth',
    cancelPendingBuild = 'auth',
    cancelAllPendingBuilds = 'auth',
    pauseSlave = 'auth',    
)
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))

c['status'].append(status.MailNotifier(
	fromaddr="buildbot@quagga.net",
	extraRecipients=["paul@jakma.org"],
	sendToInterestedUsers=False,
))

c['status'].append (status.IRC(
	"irc.freenode.net", "bb-quagga",
	useColors=True,
	channels=[{"channel": "#quagga"}],
	notify_events={
		'exception': 1,
		'successToFailure': 1,
		'failureToSuccess': 1,
	},
))

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = "Quagga"
c['titleURL'] = "https://www.quagga.net/"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "http://buildbot.quagga.net/"

####### DB URL

c['db'] = {
    # This specifies what database buildbot uses to store its state.  You can leave
    # this at its default for all but the largest installations.
    'db_url' : "sqlite:///state.sqlite",
}

#### debug
c['debugPassword'] = debugPassword