Browse Source

buildbot: build documentation, add nightly upload of master docs, other tweaks

* master.cfg: Add a "build-docs" builder to test generation of HTML and PDF
  docs into the commit checks.  With nightly=true property, upload generated
  docs to a static dir on the master.  Run from a NightlyScheduler.

  Add the properties from the internal worker config to the buildbot
  BuildSlave properties, so they're visible in the web UI. Filter through
  a workers2publicprops helper, to whitelist allowed props.
Paul Jakma 2 years ago
parent
commit
431900f023
1 changed files with 172 additions and 11 deletions
  1. 172 11
      infra/buildbot/master/master.cfg

+ 172 - 11
infra/buildbot/master/master.cfg

@@ -16,18 +16,52 @@ quaggagit = 'git://git.sv.gnu.org/quagga.git'
 # password defs
 execfile("pass.cfg")
 
+# filter a given 'workers' entry into a property list
+# suitable for public display
+def workers2publicprops (worker):
+	publicprops = [ "os", "version", "vm", "pkg", "texi", "cc",
+			"latent", ]
+	return  { k:worker[k] for k in worker if k in publicprops }
+
+# vm: non-VM are assumed faster and used for initial build
+# pkg: rpm, sysv, dpkg - only do test rpm builds at moment
+# texi: True or "true" if we can use for doc building
+# cc: List of tuples of installed compilers, with:
+#     (<tag>, <version>, <command>)
+#     tag: gcc, clang, sunpro
+# latent: VM spun up on demand via LatentSlave, uses "session" for
+#         the libvirt URI.
+# session: libvirt URI to use for latent workers. Default will be set on
+#          latent VMs if not specified.
+# hd_image: libvirt image to use
 workers = {
 	"fedora-24": { 
 	  "os": "Fedora",
 	  "version": "24",
 	  "vm": False,
 	  "pkg": "rpm",
+	  "texi": True,
+	  "cc": [ ("gcc", "6.3.1"),
+	  	  ("clang", "3.8.1"),
+	  	  ("gcc",  "3.4.6", "gcc34"),
+	  	],
+	}, 
+	"fedora-26": { 
+	  "os": "Fedora",
+	  "version": "26",
+	  "vm": False,
+	  "pkg": "rpm",
+	  "cc": [ ("gcc", "7.0.1"),
+	  	  ("clang", "3.9.0"),
+	  	  ("gcc",  "3.4.6", "gcc34"),
+	  	],
 	}, 
 	"centos-7": {
 	  "os": "CentOS",
 	  "version": "7",
 	  "vm": False,
 	  "pkg": "rpm",
+	  "cc": [ ("gcc", "4.8.5") ],
 	},
 	"debian-8": {
 	  "os": "Debian",
@@ -35,6 +69,7 @@ workers = {
 	  "vm": True,	
   	  "pkg": "dpkg",
   	  "latent": True,
+  	  "cc": [ ("gcc", "4.9.2") ],
   	  "hd_image": "/var/lib/libvirt/images/debian8.qcow2",
   	},
   	"debian-9": { 
@@ -42,6 +77,7 @@ workers = {
 	  "version": "9",
 	  "vm": True,	
   	  "pkg": "dpkg",
+  	  "cc": [ ("gcc", "6.3.0") ],
   	  "latent": True,
   	  "hd_image": "/var/lib/libvirt/images/debian9.qcow2",
   	},
@@ -51,6 +87,7 @@ workers = {
 	  "vm": True,	
   	  "pkg": "",
   	  "latent": True,
+  	  "cc": [ ("clang", "3.4.1") ],
   	  "hd_image": "/var/lib/libvirt/images/freebsd103.qcow2",
   	}, 
   	"freebsd-11": { 
@@ -58,6 +95,7 @@ workers = {
 	  "version": "11",
 	  "vm": True,	
   	  "pkg": "",
+  	  "cc": [ ("gcc", "4.9.4"), ("clang", "3.8.0"), ],
   	  "latent": True,
   	  "hd_image": "/var/lib/libvirt/images/freebsd110.qcow2",
   	},
@@ -67,6 +105,9 @@ workers = {
   	  "vm": True,
   	  "pkg": "sysv",
 	  "latent": True,
+	  "cc": [ ("gcc", "6.3.0"), ("sunpro", "12.0"), 
+	  	  ("gcc", "4.4.4") 
+	  	],
 	  "hd_image": "/var/lib/libvirt/images/buildbot-oi-hipster.qcow2",
   	},
 }
@@ -83,7 +124,7 @@ for kw in workers:
 analyses_builders = [ "clang-analyzer" ]
 
 # default Libvirt session
-for w in (w for w in workers.values () if ("latent" in w) 
+for w in (w for w in workers.values () if ("latent" in w and w["latent"])
 					and ("session" not in w)):
 	w["session"] = 'qemu+ssh://buildbot@sagan.jakma.org/system'
 
@@ -93,12 +134,17 @@ osslowbuilders = ["build-" + kw for kw in workers if workers[kw]["vm"] == True]
 
 rpmbuilders = ["rpm-" + kw for kw in workers if workers[kw]["pkg"] == "rpm"]
 
+# compilers
+# not using yet
+# [kw for kw in workers if len([v for (c,v) in workers[kw]["cc"] if c == "gcc"]) > 0 ]
+
 allbuilders =  []
 allbuilders += osbuilders
 allbuilders += rpmbuilders
 allbuilders += analyses_builders
 allbuilders += ["commit-builder"]
 allbuilders += ["build-distcheck"]
+allbuilders += ["build-docs" ]
 
 # Force merging of requests.
 # c['mergeRequests'] = lambda *args, **kwargs: True
@@ -112,7 +158,9 @@ c['slaves'] = []
 
 for w in (w for w in workers.values() if ("latent" not in w)
 				      or (w["latent"] == False)):
-	c['slaves'].append(buildslave.BuildSlave(w["bot"], w["pass"]))
+	c['slaves'].append(buildslave.BuildSlave(w["bot"], w["pass"],
+				properties=workers2publicprops (w),
+	))
 
 for w in (w for w in workers.values()
 		  if ("latent" in w) 
@@ -123,6 +171,7 @@ for w in (w for w in workers.values()
 					w["pass"],
 					util.Connection(w["session"]),
 					w["hd_image"],
+					properties=workers2publicprops (w),
 	))
 
 # 'protocols' contains information about protocols which master will use for
@@ -197,6 +246,11 @@ c['schedulers'].append(schedulers.Triggerable(
 		       name="trigger-rpm",	
 		       builderNames=rpmbuilders))
 
+# Doc build check (non-nightly, so no upload)
+c['schedulers'].append(schedulers.Triggerable(
+		       name="trigger-build-docs",	
+		       builderNames=["build-docs"]))
+
 # Try and force schedulers
 c['schedulers'].append(schedulers.ForceScheduler(
                        name="force",
@@ -207,10 +261,23 @@ c['schedulers'].append(schedulers.Try_Userpass(
 		       builderNames=osbuilders
 		       		    + rpmbuilders
 				    + ["build-distcheck", 
-					 "clang-analyzer" ],
+				       "clang-analyzer",
+				       "build-docs" ],
 			userpass=users,
 			port=8031))
 
+## nightly docs build
+c['schedulers'].append(schedulers.Nightly(
+	name="nightly-docs",
+	branch="master",
+	builderNames=[ "build-docs" ],
+	hour=3,
+	minute=0,
+	onlyIfChanged=True,
+	properties = { "nightly": True },
+))
+
+
 ####### BUILDERS
 c['builders'] = []
 
@@ -218,12 +285,15 @@ c['builders'] = []
 # what steps, and which slaves can execute them.  Note that any particular build will
 # only take place on one slave.
 
-common_steps = [
-steps.Git(repourl=quaggagit, mode='incremental'),
-steps.ShellCommand(command=["./update-autotools"]),
-steps.Configure(),
-steps.ShellCommand(command=["make", "clean"]),
-steps.Compile(),
+common_setup = [
+	steps.Git(repourl=quaggagit, mode='incremental'),
+	steps.ShellCommand(command=["./update-autotools"],
+				    description="generating autoconf",
+				    descriptionDone="autoconf"),
+	steps.Configure(command="../build/configure"),
+	steps.ShellCommand(command=["make", "clean"],
+				    description="cleaning",
+				    descriptionDone="make clean"),
 ]
 
 ### Default 'check' build, builder instantiated for each OS
@@ -238,6 +308,8 @@ factory.addStep(steps.Configure())
 factory.addStep(steps.ShellCommand(command=["make", "clean"],
 				   description="cleaning",
 				   descriptionDone="clean"))
+
+#factory.addSteps(common_setup)
 factory.addStep(steps.Compile(command=["make", "-j", "2", "all"]))
 factory.addStep(steps.ShellCommand(command=["make", "check"],
 				   description="checking",
@@ -348,6 +420,95 @@ for kw in (kw for kw in workers if workers[kw]["pkg"] == "rpm"):
       		)
       	)
 
+### Build documentation
+
+def build_is_nightly (step):
+    n = step.getProperty("nightly")
+    if n == True or n == "True" or n == "true":
+       return True
+    return False
+
+f = util.BuildFactory ()
+f.addStep(steps.Git(repourl=quaggagit, mode='full'))
+f.addStep(steps.ShellCommand(command=["./update-autotools"],
+				   description="run autotools",
+				   descriptionDone="autotools"))
+f.addStep(steps.Configure(command=["../build/configure"],
+			  workdir="docs"))
+f.addStep(steps.ShellCommand(command=["make", "V=99", "quagga.html"],
+			     description="making split HTML doc",
+			     descriptionDone="docs: split HTML",
+			     workdir="docs/doc",
+			     haltOnFailure=True,
+))
+#f.addStep(steps.FileUpload(
+#	  slavesrc="build/doc/fig-normal-processing.png",
+#	  masterdest = "public_html/docs/nightly/quagga/",
+#	  name = "Upload Fig 1",
+#	  doStepIf=build_is_nightly,
+#))
+#f.addStep(steps.FileUpload(
+#	  slavesrc="build/doc/fig-rs-processing.png",
+#	  masterdest = "public_html/docs/nightly/quagga/",
+#	  name = "Upload Fig 2",
+#	  doStepIf=build_is_nightly,
+#))
+f.addStep(steps.MultipleFileUpload(
+	  slavesrcs=[ "doc/fig-rs-processing.png",
+		      "doc/fig-normal-processing.png" ],
+	  masterdest = "public_html/docs/nightly/quagga/",
+	  name = "Upload Figures",
+	  doStepIf=build_is_nightly,
+))
+f.addStep(steps.DirectoryUpload(
+	  slavesrc="quagga.html",
+	  masterdest = "public_html/docs/nightly/quagga",
+	  compress = 'bz2',
+	  name = "Upload split HTML",
+	  url = "/docs/nightly/quagga/index.html",
+	  workdir="docs/doc",
+	  doStepIf=build_is_nightly,
+))
+f.addStep(steps.RemoveDirectory(
+	  dir="docs/doc/quagga.html",
+))
+f.addStep(steps.ShellCommand(command=["make", "V=99",
+					"MAKEINFOFLAGS=--no-split",
+					"quagga.html"],
+			     description="making one-page HTML doc",
+			     descriptionDone="docs: one-page HTML",
+			     workdir="docs/doc",
+			     haltOnFailure=True
+))
+f.addStep(steps.FileUpload(
+	  slavesrc="quagga.html",
+	  masterdest = "public_html/docs/nightly/quagga/quagga.html",
+	  name = "Upload single HTML",
+	  url = "/docs/nightly/quagga/quagga.html",
+	  workdir="docs/doc",
+	  doStepIf=build_is_nightly,
+))
+f.addStep(steps.ShellCommand(command=["make", "V=99", "quagga.pdf"],
+			     description="making PDF docs",
+			     descriptionDone="docs: PDF",
+			     workdir="docs/doc"
+))
+f.addStep(steps.FileUpload(
+	  slavesrc="quagga.pdf",
+	  masterdest = "public_html/docs/nightly/quagga/quagga.pdf",
+	  name = "Upload PDF",
+	  url = "/docs/nightly/quagga/quagga.pdf",
+	  workdir="docs/doc",
+	  doStepIf=build_is_nightly,
+))
+
+c['builders'].append(
+    util.BuilderConfig(name="build-docs",
+      slavenames=[w["bot"] for w in workers.values() 
+			if "texi" in w and w["texi"] == True ],
+      factory=f
+))
+
 ### Co-ordination builds used to sequence parallel builds via Triggerable
 
 # to understand this you have to read this list and the Triggered schedulers
@@ -367,7 +528,8 @@ f.addStep(steps.Trigger (
 	updateSourceStamp=True
 ))
 f.addStep(steps.Trigger (
-	schedulerNames = [ "trigger-build-analyses", "trigger-distcheck" ],
+	schedulerNames = [ "trigger-build-analyses", "trigger-distcheck",
+			   "trigger-build-docs" ],
 	waitForFinish=True,
 	updateSourceStamp=True
 ))
@@ -383,7 +545,6 @@ c['builders'].append(
       factory=f)
 )
 
-
 ####### STATUS TARGETS
 
 # 'status' is a list of Status Targets. The results of each build will be