Merge branch '2-gitlab-ci-is-broken' into 'master'

Resolve "gitlab-ci is broken"

Closes #2

See merge request qi/qibuild!6

.gitignore

@@ -19,8 +19,11 @@ debian/qibuild/
 # IDE project's files
 .idea
 
-# Transient files
+# Generic transient files
 *~
 ~*
 *.log
 
+# PyTest transient files
+.cache
+.pytest_cache

.gitlab-ci.yml

@@ -22,6 +22,7 @@ doitall:
       qt4-linguist-tools
       gettext
       zip
+      rsync
 
   before_script:
     # Handle SSH authentication to the internal gitlab server

python/.gitignore

@@ -8,5 +8,5 @@ qibuild.egg-info/
 .pytest
 
 # Coverage
-.coverage
+.coverage*
 htmlcov/

python/qibuild/test/conftest.py

@@ -15,6 +15,9 @@ class TestBuildWorkTree(qibuild.worktree.BuildWorkTree):
     can create git projects
 
     """
+
+    __test__ = False  # Tell PyTest to ignore this Test* named class: This is as test to collect
+
     def __init__(self, worktree=None):
         if not worktree:
             worktree = TestWorkTree()

python/qidoc/test/conftest.py

@@ -12,6 +12,8 @@
 class TestDocWorkTree(qidoc.worktree.DocWorkTree):
     """ A subclass of DocWorkTree that can create doc projects """
 
+    __test__ = False  # Tell PyTest to ignore this Test* named class: This is as test to collect
+
     def __init__(self, worktree=None):
         if not worktree:
             worktree = TestWorkTree()

python/qisrc/test/conftest.py

@@ -13,6 +13,9 @@ class TestGitWorkTree(qisrc.worktree.GitWorkTree):
     can create git projects
 
     """
+
+    __test__ = False  # Tell PyTest to ignore this Test* named class: This is as test to collect
+
     def __init__(self, worktree=None):
         if not worktree:
             worktree = TestWorkTree()
@@ -278,6 +281,9 @@ def delete_file(self, project, filename):
 
 class TestGit(qisrc.git.Git):
     """ the Git class with a few other helpful methods """
+
+    __test__ = False  # Tell PyTest to ignore this Test* named class: This is as test to collect
+
     def __init__(self, repo=None):
         if repo is None:
             repo = os.getcwd()

python/qisys/test/conftest.py

@@ -32,6 +32,9 @@ class TestWorkTree(qisys.worktree.WorkTree):
     can create projects
 
     """
+
+    __test__ = False  # Tell PyTest to ignore this Test* named class: This is as test to collect
+
     def __init__(self, root=None):
         if root is None:
             # TestWorkTree is often used with cd_to_tmpdir fixture,

python/qitest/test_queue.py

@@ -22,6 +22,9 @@
 
 class TestQueue(object):
     """ A class able to run tests in parallel """
+
+    __test__ = False  # Tell PyTest to ignore this Test* named class: This is as test to collect
+
     def __init__(self, tests):
         self.tests = tests
         self.test_logger = TestLogger(tests)
@@ -155,6 +158,9 @@ class TestWorker(threading.Thread):
     the test queue, running the tests and logging the results
 
     """
+
+    __test__ = False  # Tell PyTest to ignore this Test* named class: This is as test to collect
+
     def __init__(self, queue, worker_index):
         super(TestWorker, self).__init__(name="TestWorker#%i" % worker_index)
         self.index = worker_index
@@ -202,6 +208,9 @@ class TestLogger(object):
     tests, using a mutex so that outputs are not mixed up
 
     """
+
+    __test__ = False  # Tell PyTest to ignore this Test* named class: This is as test to collect
+
     def __init__(self, tests):
         self.mutex = threading.Lock()
         self.tests = tests

python/qitoolchain/feed.py

@@ -27,7 +27,7 @@ def is_url(location):
     return "://" in location
 
 def raise_parse_error(package_tree, feed, message):
-    """ Raise a nice pasing error about the given
+    """ Raise a nice parsing error about the given
     package_tree element.
 
     """
@@ -51,11 +51,11 @@ def tree_from_feed(feed_location, branch=None, name=None):
             if is_url(feed_location):
                 fp = qisys.remote.open_remote_location(feed_location)
             else:
-                raise Exception("Feed location is not an existing path nor an url")
+                raise Exception("Could not parse %s: Feed location is not an existing path nor an url" % feed_location)
         tree = ElementTree.ElementTree()
         tree.parse(fp)
-    except Exception:
-        ui.error("Could not parse", feed_location)
+    except Exception as e:
+        ui.error(e.message)
         raise
     finally:
         if fp:
@@ -147,18 +147,16 @@ def parse(self, feed, branch=None, name=None, first_pass=True):
         for feed_tree in feeds:
             feed_name = feed_tree.get("name")
             feed_url = feed_tree.get("url")
+            feed_path = feed_tree.get("path")
+            assert feed_path or feed_url, "Either 'url' or 'path' attributes must be set in a 'feed' non-root element"
             # feed_url can be relative to feed:
-            if feed_tree.get("path") and branch:
+            if feed_path and branch:
                 feed_path = os.path.join(tc_path + ".git", feed_tree.get("path"))
                 self.parse(feed_path)
-                if feed_name:
-                    self.parse(feed_path, branch=branch, name=feed_name, first_pass=False)
             elif feed_url:
                 if not is_url(feed_url):
                     feed_url = urlparse.urljoin(feed, feed_url)
                 self.parse(feed_url)
-                if feed_name:
-                    self.parse(feed, branch=branch, name=feed_name, first_pass=False)
         select_tree = tree.find("select")
         if select_tree is not None:
             blacklist_trees = select_tree.findall("blacklist")

python/qitoolchain/test/test_feed.py

@@ -8,6 +8,29 @@
 
 import pytest
 
+#
+# Commons variables
+#
+default_oss_xml = """\
+<feed>
+    <package name="boost" url="boost.zip" />
+</feed>
+"""
+
+default_third_part_xml = """\
+<feed>
+  <package name="oracle-jdk" url="jdk.zip" />
+</feed>
+"""
+
+# feed url is either absolute or relative to the parent feed url
+default_full_xml = """ \
+<feed>
+    <feed name="oss" path="feeds/oss.xml" />
+    <feed name="3rdpart" url="3rdpart.xml" />
+</feed>
+"""
+
 def test_is_url():
     assert not is_url(r"c:\foo\bar" )
     assert is_url("http://foo.com/bar.xml")
@@ -20,33 +43,54 @@ def test_parse_non_exising_path():
     assert "not an existing path" in e.value.message
     assert "nor an url" in e.value.message
 
+
+def _generic_test_git(_git_server, _feed, full_xml, oss_xml, third_part_xml):
+    _git_server.create_repo("toolchains.git")
+
+    _git_server.push_file("toolchains.git", "feeds/oss.xml", oss_xml)
+    _git_server.push_file("toolchains.git", "feeds/3rdpart.xml", third_part_xml)
+    _git_server.push_file("toolchains.git", "feeds/full.xml", full_xml)
+
+    git_url = _git_server.get_repo("toolchains.git").clone_url
+
+    parser = ToolchainFeedParser("foo")
+    parser.parse(git_url, name="full", branch="master")
+
+    names = [x.name for x in parser.packages]
+    assert names == ["boost", "oracle-jdk"]
+
+
 def test_git(git_server, feed):
-    git_server.create_repo("toolchains.git")
-    oss_xml = """\
-<feed>
-    <package name="boost" url="boost.zip" />
-</feed>
-"""
-    third_part_xml = """\
-<feed>
-  <package name="oracle-jdk" url="jdk.zip" />
-</feed>
-"""
+    _generic_test_git(git_server, feed, default_full_xml, default_oss_xml, default_third_part_xml)
+
+
+def test_git_missing_url_and_path(git_server, feed):
     full_xml = """ \
 <feed>
     <feed name="oss" />
-    <feed name="3rdpart" />
+    <feed name="3rdpart" url="3rdpart.xml" />
 </feed>
 """
+    with pytest.raises(AssertionError) as e:
+        _generic_test_git(git_server, feed, full_xml, default_oss_xml, default_third_part_xml)
+    assert "attributes must be set" in e.value.message
+    assert "url" in e.value.message
+    assert "path" in e.value.message
 
-    git_server.push_file("toolchains.git", "feeds/oss.xml", oss_xml)
-    git_server.push_file("toolchains.git", "feeds/3rdpart.xml", third_part_xml)
-    git_server.push_file("toolchains.git", "feeds/full.xml", full_xml)
-
-    git_url = git_server.get_repo("toolchains.git").clone_url
 
-    parser = ToolchainFeedParser("foo")
-    parser.parse(git_url, name="full", branch="master")
+def test_git_bad_url(git_server, feed):
+    # URL is bad because relative url are relative to the parent URL, and this one will add one more 'feeds' prefix.
+    # The resulting built URL will end with share/qi/toolchains/foo.git/feeds/feeds/3rdpart.xml
+    # and the double feeds/feeds/ makes it unknown
+    full_xml = """ \
+<feed>
+    <feed name="oss" path="feeds/oss.xml" />
+    <feed name="3rdpart" url="feeds/3rdpart.xml" />
+</feed>
+"""
 
-    names = [x.name for x in parser.packages]
-    assert names == ["boost", "oracle-jdk"]
+    with pytest.raises(Exception) as e:
+        _generic_test_git(git_server, feed, full_xml, default_oss_xml, default_third_part_xml)
+    assert "not parse" in e.value.message
+    assert "not an existing path" in e.value.message
+    assert "nor an url" in e.value.message

requirements.txt

@@ -18,11 +18,11 @@ pyenchant==1.6.6
 breathe==4.1.0
 
 # For tests
-mock==1.0.1
-pytest==3.2.3
-pytest-cov==1.6
-pytest-timeout==1.0.0
-pytest-xdist==1.13.1
+mock==2.0.0
+pytest==3.4.0
+pytest-cov==2.5.1
+pytest-timeout==1.2.1
+pytest-xdist==1.22.0
 
 # For qidoc tests
 BeautifulSoup4==4.2.1
@@ -31,12 +31,15 @@ sphinxcontrib-spelling==2.1.1
 sphinx-intl==0.9.5
 
 # For coverage
-cov-core==1.7
-coverage==3.6
-gcovr==3.2
+cov-core==1.15.0
+coverage==4.4.2
+gcovr==3.3
 
 # For more checks, thanks pyflakes
-pyflakes==0.6.1
+pyflakes==1.6.0
 
 # For ls-package
 tabulate==0.7.5
+
+# For py2 py3 compat
+six==1.11.0