bob managed layers

contrib/bash-completion/bob

@@ -30,7 +30,7 @@ __bob_complete_dir()
 }
 
 __bob_commands="build dev clean graph help init jenkins ls project status \
-                query-scm query-recipe query-path query-meta show"
+                query-scm query-recipe query-path query-meta show layers"
 
 # Complete a Bob path
 #
@@ -386,6 +386,21 @@ __bob_status()
   __bob_complete_path "--attic --develop --recursive --no-sandbox --release --sandbox --show-clean --show-overrides --verbose -D -c -r -v"
 }
 
+__bob_layers_status()
+{
+   __bob_complete_path "-c -D -v"
+}
+
+__bob_layers_update()
+{
+   __bob_complete_path "-c -D -v"
+}
+
+__bob_layers()
+{
+  __bob_subcommands "status update" "layers"
+}
+
 __bob_show()
 {
     if [[ "$prev" = "--format" ]] ; then

doc/conf.py

@@ -267,6 +267,7 @@ def __getattr__(cls, name):
     ('manpages/bob-graph', 'bob-graph', 'Generate dependency graph', ['Ralf Hubert'], 1),
     ('manpages/bob-init', 'bob-init', 'Initialize out-of-source build tree', ['Jan Klötzke'], 1),
     ('manpages/bob-jenkins', 'bob-jenkins', 'Configure Jenkins server', ['Jan Klötzke'], 1),
+    ('manpages/bob-layers', 'bob-layers', 'Manage Layers', ['BobBuildTool Team'], 1),
     ('manpages/bob-ls', 'bob-ls', 'List package hierarchy', ['Jan Klötzke'], 1),
     ('manpages/bobpaths', 'bobpaths', 'Specifying paths to Bob packages', ['Jan Klötzke'], 7),
     ('manpages/bob-project', 'bob-project', 'Create IDE project files', ['Jan Klötzke'], 1),

doc/manpages/bob-layers.rst

@@ -0,0 +1,58 @@
+.. _manpage-layers:
+
+bob-layers
+==========
+
+.. only:: not man
+
+   Name
+   ----
+
+   bob-layers - Handle layers
+
+Synopsis
+--------
+
+::
+
+    bob show [-h] [-c CONFIGFILE] [-v] [-D DEFINES] {update,status}
+
+
+Description
+-----------
+
+Update layers or show their scm-status. The following sub-commands are
+available:
+
+``update``
+
+Updates the layers.
+
+``status``
+
+Show the scm-status of each layer and optionally list modifications. See
+`man bob-status` for a description of the output fields.
+
+Options
+-------
+
+``-c CONFIGFILE``
+    Use additional configuration file.
+
+    The ``.yaml`` suffix is appended automatically and the configuration file
+    is searched relative to the project root directory unless an absolute path
+    is given. Bob will parse these user configuration files after
+    *default.yaml*. They are using the same schema.
+
+    This option can be given multiple times. The files will be parsed in the
+    order as they appeared on the command line.
+
+``-D VAR=VALUE``
+    Override default or set environment variable.
+
+    Sets the variable ``VAR`` to ``VALUE``. This overrides the value possibly
+    set by ``default.yaml``, config files passed by ``-c`` or any file that was
+    included by either of these files.
+
+``-v, --verbose``
+    Increase verbosity (may be specified multiple times)

doc/manpages/index.rst

@@ -14,6 +14,7 @@ Contents:
    bob-graph
    bob-init
    bob-jenkins
+   bob-layers
    bob-ls
    bobpaths
    bob-project

doc/manual/configuration.rst

@@ -1902,7 +1902,7 @@ equal.
 layers
 ~~~~~~
 
-Type: List of strings
+Type: List of strings or SCM-Dictionary or List of SCM-Dictionaries
 
 The ``layers`` section consists of a list of layer names that are then expected
 in the ``layers`` directory relative to the ``conig.yaml`` referencing them::
@@ -1924,6 +1924,25 @@ of lower precedence.
 
 See :ref:`configuration` for more information.
 
+Typically layers have their own VCS. To provide them to the root-recipes common
+VCS-methods like git-submodules can be used. Another possibility is to provide a
+SCM-Dictionary (see :ref:`configuration-recipes-scm`)::
+
+    layers:
+        - name: myapp
+          scm: git
+          url: git@foo.bar:myapp.git
+          commit: ...
+        - bsp
+
+Only `git`,`svn`,`url` and `cvs` scm's are supported for layers.
+
+Using this Bob takes care of the layer managment and would checkout the layers
+if they do not exists, update the layers if the spec changed during regular
+build (except `build-only` is used) and allows you to use `bob layers`.
+
+See :ref:`manpage-layers`.
+
 .. _configuration-config-plugins:
 
 plugins

pym/bob/builder.py

@@ -1474,7 +1474,7 @@ async def _downloadPackage(self, packageStep, depth, packageBuildId):
           * still same build-id -> done
           * build-id changed -> prune and try download, fall back to build
         """
-        layer = "/".join(packageStep.getPackage().getRecipe().getLayer())
+        layer = packageStep.getPackage().getRecipe().getLayer()
         layerDownloadMode = None
         if layer:
             for mode in self.__downloadLayerModes:

pym/bob/cmds/build/build.py

@@ -224,6 +224,8 @@ def _downloadLayerArgument(arg):
         recipes.defineHook('developNameFormatter', LocalBuilder.developNameFormatter)
         recipes.defineHook('developNamePersister', None)
         recipes.setConfigFiles(args.configFile)
+        if args.build_mode != 'build-only':
+            recipes.updateLayers(loop, defines, args.verbose)
         recipes.parse(defines)
 
         # if arguments are not passed on cmdline use them from default.yaml or set to default yalue

pym/bob/input.py

@@ -1737,6 +1737,21 @@ def validate(self, data):
                 None)
         return data
 
+class LayerValidator:
+    def __init__(self):
+        self.__scmValidator = ScmValidator({
+            'git' : GitScm.SCHEMA,
+            'svn' : SvnScm.SCHEMA,
+            'cvs' : CvsScm.SCHEMA,
+            'url' : UrlScm.SCHEMA})
+
+    def validate(self, data):
+        if 'name' not in data:
+            raise schema.SchemaMissingKeyError("Missing 'name' key in {}".format(data), None)
+        name = data.get('name')
+        del data['name']
+        return {'scmSpec' : self.__scmValidator.validate(data)[0], 'name' : name}
+
 class VarDefineValidator:
     def __init__(self, keyword):
         self.__varName = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$')
@@ -1852,6 +1867,12 @@ def getProvideDepsResolver(pattern):
     else:
         return VerbatimProvideDepsResolver(pattern)
 
+def getLayerName(layerSpec):
+    if isinstance(layerSpec, str):
+        return layerSpec
+    else:
+        return layerSpec.get('name')
+
 class Recipe(object):
     """Representation of a single recipe
 
@@ -1966,7 +1987,7 @@ def createVirtualRoot(recipeSet, roots, properties):
             "buildScript" : "true",
             "packageScript" : "true"
         }
-        ret = Recipe(recipeSet, recipe, [], "", ".", "", "", properties)
+        ret = Recipe(recipeSet, recipe, "", "", ".", "", "", properties)
         ret.resolveClasses(Env())
         return ret
 
@@ -2020,7 +2041,7 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b
         }
         self.__corePackagesByMatch = []
         self.__corePackagesById = {}
-        self.__layer = layer
+        self.__layer = getLayerName(layer)
 
         sourceName = ("Recipe " if isRecipe else "Class  ") + packageName + (
             ", layer "+"/".join(layer) if layer else "")
@@ -2915,6 +2936,14 @@ class RecipeSet:
             schema.Optional('max_depth') : int,
         })
 
+    SCM_SCHEMA = ScmValidator({
+        'git' : GitScm.SCHEMA,
+        'svn' : SvnScm.SCHEMA,
+        'cvs' : CvsScm.SCHEMA,
+        'url' : UrlScm.SCHEMA,
+        'import' : ImportScm.SCHEMA,
+    })
+
     STATIC_CONFIG_SCHEMA = schema.Schema({
         schema.Optional('bobMinimumVersion') : str, # validated separately in preValidate
         schema.Optional('plugins') : [str],
@@ -2940,20 +2969,12 @@ class RecipeSet:
             },
             error="Invalid policy specified! Are you using an appropriate version of Bob?"
         ),
-        schema.Optional('layers') : [str],
+        schema.Optional('layers') : [schema.Or(str, LayerValidator())],
         schema.Optional('scriptLanguage',
                         default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell"),
                                                                   schema.Use(ScriptLanguage)),
     })
 
-    SCM_SCHEMA = ScmValidator({
-        'git' : GitScm.SCHEMA,
-        'svn' : SvnScm.SCHEMA,
-        'cvs' : CvsScm.SCHEMA,
-        'url' : UrlScm.SCHEMA,
-        'import' : ImportScm.SCHEMA,
-    })
-
     MIRRORS_SCHEMA = ScmValidator({
         'url' : UrlScm.MIRRORS_SCHEMA,
     })
@@ -2992,6 +3013,7 @@ def __init__(self):
         self.__commandConfig = {}
         self.__uiConfig = {}
         self.__shareConfig = {}
+        self.__layers = []
         self.__policies = {
             'noUndefinedTools' : (
                 "0.17.3.dev57",
@@ -3383,7 +3405,8 @@ def loadYaml(self, path, schema, default={}, preValidate=lambda x: None):
         else:
             return schema[0].validate(default)
 
-    def parse(self, envOverrides={}, platform=getPlatformString(), recipesRoot=""):
+    def parse(self, envOverrides={}, platform=getPlatformString(), recipesRoot="",
+              dryRun=False):
         if not recipesRoot and os.path.isfile(".bob-project"):
             try:
                 with open(".bob-project") as f:
@@ -3393,14 +3416,14 @@ def parse(self, envOverrides={}, platform=getPlatformString(), recipesRoot=""):
         recipesDir = os.path.join(recipesRoot, "recipes")
         if not os.path.isdir(recipesDir):
             raise ParseError("No recipes directory found in " + recipesDir)
+        self.__projectRoot = recipesRoot or os.getcwd()
         self.__cache.open()
         try:
-            self.__parse(envOverrides, platform, recipesRoot)
+            self.__parse(envOverrides, platform, recipesRoot, dryRun)
         finally:
             self.__cache.close()
-        self.__projectRoot = recipesRoot or os.getcwd()
 
-    def __parse(self, envOverrides, platform, recipesRoot=""):
+    def __parse(self, envOverrides, platform, recipesRoot="", dryRun=False):
         if platform not in ('cygwin', 'darwin', 'linux', 'msys', 'win32'):
             raise ParseError("Invalid platform: " + platform)
         self.__platform = platform
@@ -3431,8 +3454,17 @@ def __parse(self, envOverrides, platform, recipesRoot=""):
             self.__parseUserConfig(os.path.join(os.environ.get('XDG_CONFIG_HOME',
                 os.path.join(os.path.expanduser("~"), '.config')), 'bob', 'default.yaml'))
 
+        osEnv = Env(os.environ)
+        osEnv.setFuns(self.__stringFunctions)
+        env = Env({ k : osEnv.substitute(v, k) for (k, v) in
+            self.__defaultEnv.items() })
+        env.setFuns(self.__stringFunctions)
+        env.update(envOverrides)
+        env["BOB_HOST_PLATFORM"] = platform
+        self.__rootEnv = env
+
         # Begin with root layer
-        self.__parseLayer([], "9999", recipesRoot)
+        self.__parseLayer("", "9999", recipesRoot, dryRun)
 
         # Out-of-tree builds may have a dedicated default.yaml
         if recipesRoot:
@@ -3446,15 +3478,15 @@ def __parse(self, envOverrides, platform, recipesRoot=""):
             self.__parseUserConfig(c)
 
         # calculate start environment
-        osEnv = Env(os.environ)
-        osEnv.setFuns(self.__stringFunctions)
         env = Env({ k : osEnv.substitute(v, k) for (k, v) in
             self.__defaultEnv.items() })
         env.setFuns(self.__stringFunctions)
         env.update(envOverrides)
-        env["BOB_HOST_PLATFORM"] = platform
         self.__rootEnv = env
 
+        if dryRun:
+            return
+
         # resolve recipes and their classes
         rootRecipes = []
         for recipe in self.__recipes.values():
@@ -3475,10 +3507,25 @@ def __parse(self, envOverrides, platform, recipesRoot=""):
         self.__rootRecipe = Recipe.createVirtualRoot(self, sorted(filteredRoots), self.__properties)
         self.__addRecipe(self.__rootRecipe)
 
-    def __parseLayer(self, layer, maxVer, recipesRoot):
-        rootDir = os.path.join(recipesRoot, *(os.path.join("layers", l) for l in layer))
+    def updateLayers(self, loop, defines, verbose):
+        from .layers import fetchLayers
+        try:
+            self.parse(defines, dryRun=True)
+        except ParseError:
+            pass
+        self.__layers = []
+        fetchLayers(self, self.__cache, loop, verbose)
+
+    def __parseLayer(self, layerSpec, maxVer, recipesRoot, dryRun):
+        layer = getLayerName(layerSpec)
+
+        if layer in self.__layers:
+            return
+        self.__layers.append(layer)
+
+        rootDir = os.path.join(recipesRoot, os.path.join("layers", layer) if layer != "" else "")
         if not os.path.isdir(rootDir or "."):
-            raise ParseError("Layer '{}' does not exist!".format("/".join(layer)))
+            raise ParseError(f"Layer '{layer}' does not exist!")
 
         configYaml = os.path.join(rootDir, "config.yaml")
         def preValidate(data):
@@ -3496,12 +3543,15 @@ def preValidate(data):
             preValidate=preValidate)
         minVer = config.get("bobMinimumVersion", "0.16")
         if compareVersion(maxVer, minVer) < 0:
-            raise ParseError("Layer '{}' reqires a higher Bob version than root project!"
+            raise ParseError("Layer '{}' requires a higher Bob version than root project!"
                                 .format("/".join(layer)))
         if compareVersion(minVer, "0.16") < 0:
             raise ParseError("Projects before bobMinimumVersion 0.16 are not supported!")
         maxVer = minVer # sub-layers must not have a higher bobMinimumVersion
 
+        if dryRun:
+            return
+
         # Determine policies. The root layer determines the default settings
         # implicitly by bobMinimumVersion or explicitly via 'policies'. All
         # sub-layer policies must not contradict root layer policies
@@ -3519,7 +3569,7 @@ def preValidate(data):
         # First parse any sub-layers. Their settings have a lower precedence
         # and may be overwritten by higher layers.
         for l in config.get("layers", []):
-            self.__parseLayer(layer + [l], maxVer, recipesRoot)
+            self.__parseLayer(l, maxVer, recipesRoot, dryRun)
 
         # Load plugins and re-create schemas as new keys may have been added
         self.__loadPlugins(rootDir, layer, config.get("plugins", []))