bob managed layers

pym/bob/cmds/build/build.py

@@ -224,6 +224,8 @@ def _downloadLayerArgument(arg):
         recipes.defineHook('developNameFormatter', LocalBuilder.developNameFormatter)
         recipes.defineHook('developNamePersister', None)
         recipes.setConfigFiles(args.configFile)
+        if args.build_mode != 'build-only':
+            recipes.updateLayers(loop, defines, args.verbose)
         recipes.parse(defines)
 
         # if arguments are not passed on cmdline use them from default.yaml or set to default yalue

pym/bob/input.py

@@ -1737,6 +1737,21 @@ def validate(self, data):
                 None)
         return data
 
+class LayerValidator:
+    def __init__(self):
+        self.__scmValidator = ScmValidator({
+            'git' : GitScm.SCHEMA,
+            'svn' : SvnScm.SCHEMA,
+            'cvs' : CvsScm.SCHEMA,
+            'url' : UrlScm.SCHEMA})
+
+    def validate(self, data):
+        if 'name' not in data:
+            raise schema.SchemaMissingKeyError("Missing 'name' key in {}".format(data), None)
+        name = data.get('name')
+        del data['name']
+        return {'scmSpec' : self.__scmValidator.validate(data)[0], 'name' : name}
+
 class VarDefineValidator:
     def __init__(self, keyword):
         self.__varName = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$')
@@ -1852,6 +1867,12 @@ def getProvideDepsResolver(pattern):
     else:
         return VerbatimProvideDepsResolver(pattern)
 
+def getLayerName(layerSpec):
+    if isinstance(layerSpec, str):
+        return layerSpec
+    else:
+        return layerSpec.get('name')
+
 class Recipe(object):
     """Representation of a single recipe
 
@@ -1966,7 +1987,7 @@ def createVirtualRoot(recipeSet, roots, properties):
             "buildScript" : "true",
             "packageScript" : "true"
         }
-        ret = Recipe(recipeSet, recipe, [], "", ".", "", "", properties)
+        ret = Recipe(recipeSet, recipe, "", "", ".", "", "", properties)
         ret.resolveClasses(Env())
         return ret
 
@@ -2020,7 +2041,7 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b
         }
         self.__corePackagesByMatch = []
         self.__corePackagesById = {}
-        self.__layer = layer
+        self.__layer = getLayerName(layer)
 
         sourceName = ("Recipe " if isRecipe else "Class  ") + packageName + (
             ", layer "+"/".join(layer) if layer else "")
@@ -2915,6 +2936,14 @@ class RecipeSet:
             schema.Optional('max_depth') : int,
         })
 
+    SCM_SCHEMA = ScmValidator({
+        'git' : GitScm.SCHEMA,
+        'svn' : SvnScm.SCHEMA,
+        'cvs' : CvsScm.SCHEMA,
+        'url' : UrlScm.SCHEMA,
+        'import' : ImportScm.SCHEMA,
+    })
+
     STATIC_CONFIG_SCHEMA = schema.Schema({
         schema.Optional('bobMinimumVersion') : str, # validated separately in preValidate
         schema.Optional('plugins') : [str],
@@ -2940,20 +2969,12 @@ class RecipeSet:
             },
             error="Invalid policy specified! Are you using an appropriate version of Bob?"
         ),
-        schema.Optional('layers') : [str],
+        schema.Optional('layers') : [schema.Or(str, LayerValidator())],
         schema.Optional('scriptLanguage',
                         default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell"),
                                                                   schema.Use(ScriptLanguage)),
     })
 
-    SCM_SCHEMA = ScmValidator({
-        'git' : GitScm.SCHEMA,
-        'svn' : SvnScm.SCHEMA,
-        'cvs' : CvsScm.SCHEMA,
-        'url' : UrlScm.SCHEMA,
-        'import' : ImportScm.SCHEMA,
-    })
-
     MIRRORS_SCHEMA = ScmValidator({
         'url' : UrlScm.MIRRORS_SCHEMA,
     })
@@ -3384,7 +3405,8 @@ def loadYaml(self, path, schema, default={}, preValidate=lambda x: None):
         else:
             return schema[0].validate(default)
 
-    def parse(self, envOverrides={}, platform=getPlatformString(), recipesRoot=""):
+    def parse(self, envOverrides={}, platform=getPlatformString(), recipesRoot="",
+              dryRun=False):
         if not recipesRoot and os.path.isfile(".bob-project"):
             try:
                 with open(".bob-project") as f:
@@ -3394,14 +3416,14 @@ def parse(self, envOverrides={}, platform=getPlatformString(), recipesRoot=""):
         recipesDir = os.path.join(recipesRoot, "recipes")
         if not os.path.isdir(recipesDir):
             raise ParseError("No recipes directory found in " + recipesDir)
+        self.__projectRoot = recipesRoot or os.getcwd()
         self.__cache.open()
         try:
-            self.__parse(envOverrides, platform, recipesRoot)
+            self.__parse(envOverrides, platform, recipesRoot, dryRun)
         finally:
             self.__cache.close()
-        self.__projectRoot = recipesRoot or os.getcwd()
 
-    def __parse(self, envOverrides, platform, recipesRoot=""):
+    def __parse(self, envOverrides, platform, recipesRoot="", dryRun=False):
         if platform not in ('cygwin', 'darwin', 'linux', 'msys', 'win32'):
             raise ParseError("Invalid platform: " + platform)
         self.__platform = platform
@@ -3432,8 +3454,17 @@ def __parse(self, envOverrides, platform, recipesRoot=""):
             self.__parseUserConfig(os.path.join(os.environ.get('XDG_CONFIG_HOME',
                 os.path.join(os.path.expanduser("~"), '.config')), 'bob', 'default.yaml'))
 
+        osEnv = Env(os.environ)
+        osEnv.setFuns(self.__stringFunctions)
+        env = Env({ k : osEnv.substitute(v, k) for (k, v) in
+            self.__defaultEnv.items() })
+        env.setFuns(self.__stringFunctions)
+        env.update(envOverrides)
+        env["BOB_HOST_PLATFORM"] = platform
+        self.__rootEnv = env
+
         # Begin with root layer
-        self.__parseLayer("", "9999", recipesRoot)
+        self.__parseLayer("", "9999", recipesRoot, dryRun)
 
         # Out-of-tree builds may have a dedicated default.yaml
         if recipesRoot:
@@ -3447,15 +3478,15 @@ def __parse(self, envOverrides, platform, recipesRoot=""):
             self.__parseUserConfig(c)
 
         # calculate start environment
-        osEnv = Env(os.environ)
-        osEnv.setFuns(self.__stringFunctions)
         env = Env({ k : osEnv.substitute(v, k) for (k, v) in
             self.__defaultEnv.items() })
         env.setFuns(self.__stringFunctions)
         env.update(envOverrides)
-        env["BOB_HOST_PLATFORM"] = platform
         self.__rootEnv = env
 
+        if dryRun:
+            return
+
         # resolve recipes and their classes
         rootRecipes = []
         for recipe in self.__recipes.values():
@@ -3476,7 +3507,18 @@ def __parse(self, envOverrides, platform, recipesRoot=""):
         self.__rootRecipe = Recipe.createVirtualRoot(self, sorted(filteredRoots), self.__properties)
         self.__addRecipe(self.__rootRecipe)
 
-    def __parseLayer(self, layer, maxVer, recipesRoot):
+    def updateLayers(self, loop, defines, verbose):
+        from .layers import fetchLayers
+        try:
+            self.parse(defines, dryRun=True)
+        except ParseError:
+            pass
+        self.__layers = []
+        fetchLayers(self, self.__cache, loop, verbose)
+
+    def __parseLayer(self, layerSpec, maxVer, recipesRoot, dryRun):
+        layer = getLayerName(layerSpec)
+
         if layer in self.__layers:
             return
         self.__layers.append(layer)
@@ -3507,6 +3549,9 @@ def preValidate(data):
             raise ParseError("Projects before bobMinimumVersion 0.16 are not supported!")
         maxVer = minVer # sub-layers must not have a higher bobMinimumVersion
 
+        if dryRun:
+            return
+
         # Determine policies. The root layer determines the default settings
         # implicitly by bobMinimumVersion or explicitly via 'policies'. All
         # sub-layer policies must not contradict root layer policies
@@ -3524,7 +3569,7 @@ def preValidate(data):
         # First parse any sub-layers. Their settings have a lower precedence
         # and may be overwritten by higher layers.
         for l in config.get("layers", []):
-            self.__parseLayer(l, maxVer, recipesRoot)
+            self.__parseLayer(l, maxVer, recipesRoot, dryRun)
 
         # Load plugins and re-create schemas as new keys may have been added
         self.__loadPlugins(rootDir, layer, config.get("plugins", []))