dependencies/dub: First try to describe local project

The current approach of determining dub dependencies is by specifying
a name and, optionally, a version. Dub will then be called to generate
a json summary of the package and code in meson will parse that and
extract relevant information. This can be insufficient because dub
packages can provide multiple configurations for multiple use-cases,
examples include providing a configuration for an executable and a
configuration for a library. As a practical example, the dub package
itself provides an application configuration and multiple library
configurations, the json description of dub will, by default, be for
the application configuration which will make dub as a library
unusable in meson.

This can be solved without modifying the meson build interface by
having dub describe the entire local project and collecting
dependencies information from that. This way dub will generate
information based on the project's 'dub.json' file, which is free to
require dependencies in any way accepted by dub, by specifying
configurations, by modifying compilation flags etc. This is all
transparent to meson as dub's main purpose is to provide a path to the
library file generated by the dependency in addition to other
command-line arguments for the compiler.

This change will, however, require that projects that want to build
with meson also provided a 'dub.json' file in which dependency
information is recorded. Failure to do so will not break existing
projects that didn't use a 'dub.json', but they will be limited to
what the previous implementation offered. Projects that already have a
'dub.json' should be fine, so long as the file is valid and the
information in it matches the one in 'meson.build'. For example for a
'dependency()' call in 'meson.build' that dependency must exist in
'dub.json', otherwise the call will now fail when it worked
previously.

Using a 'dub.json' also has as a consequence that the version of the
dependencies that are found are the ones specified in
'dub.selections.json', which can be helpful for projects that already
provide a 'dub.json' in addition to 'meson.build' to de-duplicate code.

In terms of other code changes:
- multiple version requirements for a dub dependency now work, though
they can only be used when a 'dub.json' is present in which case the
version of dependencies is already pinned by 'dub.selections.json'
- the 'd/11 dub' test case has been changed to auto-generate the
'dub.json' config outside of the source directory, as the
auto-generated file triggers warning when parsed by dub, which upsets
the new code as the warnings interfere with the legitimate output.

Signed-off-by: Andrei Horodniceanu <[email protected]>

ci/ciimage/opensuse/install.sh

@@ -38,11 +38,13 @@ chmod +x /ci/env_vars.sh
 
 source /ci/env_vars.sh
 
-dub_fetch urld
-dub build urld --compiler=dmd
-dub_fetch dubtestproject
+dub_fetch [email protected]
 dub build dubtestproject:test1 --compiler=dmd
 dub build dubtestproject:test2 --compiler=dmd
+dub build dubtestproject:test3 --compiler=dmd
+dub_fetch [email protected]
+dub build urld --compiler=dmd
+
 
 # Cleanup
 zypper --non-interactive clean --all

ci/ciimage/ubuntu-rolling/install.sh

@@ -45,11 +45,12 @@ eatmydata apt-get -y install --no-install-recommends wine-stable  # Wine is spec
 install_python_packages hotdoc
 
 # dub stuff
-dub_fetch urld
-dub build urld --compiler=gdc
-dub_fetch dubtestproject
+dub_fetch [email protected]
 dub build dubtestproject:test1 --compiler=ldc2
 dub build dubtestproject:test2 --compiler=ldc2
+dub build dubtestproject:test3 --compiler=gdc
+dub_fetch [email protected]
+dub build urld --compiler=gdc
 
 # Remove debian version of Rust and install latest with rustup.
 # This is needed to get the cross toolchain as well.

mesonbuild/dependencies/dub.py

@@ -5,10 +5,11 @@
 
 from .base import ExternalDependency, DependencyException, DependencyTypeName
 from .pkgconfig import PkgConfigDependency
-from ..mesonlib import (Popen_safe, join_args, version_compare)
+from ..mesonlib import (Popen_safe, join_args, version_compare, version_compare_many)
 from ..options import OptionKey
 from ..programs import ExternalProgram
 from .. import mlog
+from enum import Enum
 import re
 import os
 import json
@@ -56,6 +57,10 @@ class FindTargetEntry(TypedDict):
         search: str
         artifactPath: str
 
+class DubDescriptionSource(Enum):
+    Local = 'local'
+    External = 'external'
+
 class DubDependency(ExternalDependency):
     # dub program and version
     class_dubbin: T.Optional[T.Tuple[ExternalProgram, str]] = None
@@ -87,7 +92,6 @@ def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.
         if DubDependency.class_dubbin is None:
             if self.required:
                 raise DependencyException('DUB not found.')
-            self.is_found = False
             return
 
         (self.dubbin, dubver) = DubDependency.class_dubbin  # pylint: disable=unpacking-non-sequence
@@ -108,20 +112,11 @@ def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.
                 mlog.warning(f'DUB dependency {name} not found because Dub {dubver} '
                              "is not compatible with Meson. (Can't locate artifacts in DUB's cache)."
                              ' Upgrade to Dub >= 1.35')
-            self.is_found = False
             return
 
         mlog.debug('Determining dependency {!r} with DUB executable '
                    '{!r}'.format(name, self.dubbin.get_path()))
 
-        # if an explicit version spec was stated, use this when querying Dub
-        main_pack_spec = name
-        if 'version' in kwargs:
-            version_spec = kwargs['version']
-            if isinstance(version_spec, list):
-                version_spec = " ".join(version_spec)
-            main_pack_spec = f'{name}@{version_spec}'
-
         # we need to know the target architecture
         dub_arch = self.compiler.arch
 
@@ -135,37 +130,11 @@ def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.
         elif dub_buildtype == 'minsize':
             dub_buildtype = 'release'
 
-        # A command that might be useful in case of missing DUB package
-        def dub_build_deep_command() -> str:
-            if self._dub_has_build_deep:
-                cmd = ['dub', 'build', '--deep']
-            else:
-                cmd = ['dub', 'run', '--yes', 'dub-build-deep', '--']
-
-            return join_args(cmd + [
-                main_pack_spec,
-                '--arch=' + dub_arch,
-                '--compiler=' + self.compiler.get_exelist()[-1],
-                '--build=' + dub_buildtype
-            ])
-
-        # Ask dub for the package
-        describe_cmd = [
-            'describe', main_pack_spec, '--arch=' + dub_arch,
-            '--build=' + dub_buildtype, '--compiler=' + self.compiler.get_exelist()[-1]
-        ]
-        ret, res, err = self._call_dubbin(describe_cmd)
-
-        if ret != 0:
-            mlog.debug('DUB describe failed: ' + err)
-            if 'locally' in err:
-                mlog.error(mlog.bold(main_pack_spec), 'is not present locally. You may try the following command:')
-                mlog.log(mlog.bold(dub_build_deep_command()))
-            self.is_found = False
+        result = self._get_dub_description(dub_arch, dub_buildtype)
+        if result is None:
             return
-
+        description, build_cmd, description_source = result
         dub_comp_id = self._ID_MAP[self.compiler.get_id()]
-        description: DubDescription = json.loads(res)
 
         self.compile_args = []
         self.link_args = self.raw_link_args = []
@@ -204,7 +173,7 @@ def find_package_target(pkg: DubPackDesc) -> bool:
                     mlog.error(mlog.bold(pack_id), 'not found')
 
                 mlog.log('You may try the following command to install the necessary DUB libraries:')
-                mlog.log(mlog.bold(dub_build_deep_command()))
+                mlog.log(mlog.bold(build_cmd))
 
                 return False
 
@@ -223,33 +192,45 @@ def find_package_target(pkg: DubPackDesc) -> bool:
         # 4. Add other build settings (imports, versions etc.)
 
         # 1
-        self.is_found = False
         packages: T.Dict[str, DubPackDesc] = {}
+        found_it = False
         for pkg in description['packages']:
             packages[pkg['name']] = pkg
 
             if not pkg['active']:
                 continue
 
-            if pkg['targetType'] == 'dynamicLibrary':
-                mlog.error('DUB dynamic library dependencies are not supported.')
-                self.is_found = False
-                return
-
             # check that the main dependency is indeed a library
             if pkg['name'] == name:
-                self.is_found = True
-
                 if pkg['targetType'] not in ['library', 'sourceLibrary', 'staticLibrary']:
-                    mlog.error(mlog.bold(name), "found but it isn't a library")
-                    self.is_found = False
+                    mlog.error(mlog.bold(name), "found but it isn't a static library, it is:",
+                               pkg['targetType'])
                     return
 
+                if self.version_reqs is not None:
+                    ver = pkg['version']
+                    if not version_compare_many(ver, self.version_reqs)[0]:
+                        mlog.error(mlog.bold(f'{name}@{ver}'),
+                                   'does not satisfy all version requirements of:',
+                                   ' '.join(self.version_reqs))
+                        return
+
+                found_it = True
                 self.version = pkg['version']
                 self.pkg = pkg
 
+        if not found_it:
+            mlog.error(f'Could not find {name} in DUB description.')
+            if description_source is DubDescriptionSource.Local:
+                mlog.log('Make sure that the dependency is registered for your dub project by running:')
+                mlog.log(mlog.bold(f'dub add {name}'))
+            elif description_source is DubDescriptionSource.External:
+                # It shouldn't be possible to get here
+                mlog.log('Make sure that the dependency is built:')
+                mlog.log(mlog.bold(build_cmd))
+            return
+
         if name not in targets:
-            self.is_found = False
             if self.pkg['targetType'] == 'sourceLibrary':
                 # source libraries have no associated targets,
                 # but some build settings like import folders must be found from the package object.
@@ -258,30 +239,25 @@ def find_package_target(pkg: DubPackDesc) -> bool:
                 # (See openssl DUB package for example of sourceLibrary)
                 mlog.error('DUB targets of type', mlog.bold('sourceLibrary'), 'are not supported.')
             else:
-                mlog.error('Could not find target description for', mlog.bold(main_pack_spec))
-
-        if not self.is_found:
-            mlog.error(f'Could not find {name} in DUB description')
+                mlog.error('Could not find target description for', mlog.bold(self.name))
             return
 
         # Current impl only supports static libraries
         self.static = True
 
         # 2
         if not find_package_target(self.pkg):
-            self.is_found = False
             return
 
         # 3
         for link_dep in targets[name]['linkDependencies']:
             pkg = packages[link_dep]
             if not find_package_target(pkg):
-                self.is_found = False
                 return
 
         if show_buildtype_warning:
             mlog.log('If it is not suitable, try the following command and reconfigure Meson with', mlog.bold('--clearcache'))
-            mlog.log(mlog.bold(dub_build_deep_command()))
+            mlog.log(mlog.bold(build_cmd))
 
         # 4
         bs = targets[name]['buildSettings']
@@ -345,6 +321,60 @@ def find_package_target(pkg: DubPackDesc) -> bool:
             # fallback
             self.link_args.append('-l'+lib)
 
+        self.is_found = True
+
+    # Get the dub description needed to resolve the dependency and a
+    # build command that can be used to build the dependency in case it is
+    # not present.
+    def _get_dub_description(self, dub_arch: str, dub_buildtype: str) -> T.Optional[T.Tuple[DubDescription, str, DubDescriptionSource]]:
+        def get_build_command() -> T.List[str]:
+            if self._dub_has_build_deep:
+                cmd = ['dub', 'build', '--deep']
+            else:
+                cmd = ['dub', 'run', '--yes', 'dub-build-deep', '--']
+
+            return cmd + [
+                '--arch=' + dub_arch,
+                '--compiler=' + self.compiler.get_exelist()[-1],
+                '--build=' + dub_buildtype,
+            ]
+
+        # Ask dub for the package
+        describe_cmd = [
+            'describe', '--arch=' + dub_arch,
+            '--build=' + dub_buildtype, '--compiler=' + self.compiler.get_exelist()[-1]
+        ]
+        helper_build = join_args(get_build_command())
+        source = DubDescriptionSource.Local
+        ret, res, err = self._call_dubbin(describe_cmd)
+        if ret == 0:
+            return (json.loads(res), helper_build, source)
+
+        pack_spec = self.name
+        if self.version_reqs is not None:
+            if len(self.version_reqs) > 1:
+                mlog.error('Multiple version requirements are not supported for raw dub dependencies.')
+                mlog.error("Please specify only an exact version like '1.2.3'")
+                raise DependencyException('Multiple version requirements are not solvable for raw dub depencies')
+            elif len(self.version_reqs) == 1:
+                pack_spec += '@' + self.version_reqs[0]
+
+        describe_cmd = [
+            'describe', pack_spec, '--arch=' + dub_arch,
+            '--build=' + dub_buildtype, '--compiler=' + self.compiler.get_exelist()[-1]
+        ]
+        helper_build = join_args(get_build_command() + [pack_spec])
+        source = DubDescriptionSource.External
+        ret, res, err = self._call_dubbin(describe_cmd)
+        if ret == 0:
+            return (json.loads(res), helper_build, source)
+
+        mlog.debug('DUB describe failed: ' + err)
+        if 'locally' in err:
+            mlog.error(mlog.bold(pack_spec), 'is not present locally. You may try the following command:')
+            mlog.log(mlog.bold(helper_build))
+        return None
+
     # This function finds the target of the provided JSON package, built for the right
     # compiler, architecture, configuration...
     # It returns (target|None, {compatibilities})
@@ -469,7 +499,7 @@ def _get_comp_versions_to_find(self, dub_comp_id: str) -> T.List[str]:
 
     def _call_dubbin(self, args: T.List[str], env: T.Optional[T.Dict[str, str]] = None) -> T.Tuple[int, str, str]:
         assert isinstance(self.dubbin, ExternalProgram)
-        p, out, err = Popen_safe(self.dubbin.get_command() + args, env=env)
+        p, out, err = Popen_safe(self.dubbin.get_command() + args, env=env, cwd=self.env.get_source_dir())
         return p.returncode, out.strip(), err.strip()
 
     def _call_compbin(self, args: T.List[str], env: T.Optional[T.Dict[str, str]] = None) -> T.Tuple[int, str, str]:

test cases/d/11 dub/meson.build

@@ -17,12 +17,12 @@ test('test urld', test_exe)
 
 # If you want meson to generate/update a dub.json file
 dlang = import('dlang')
-dlang.generate_dub_file(meson.project_name().to_lower(), meson.source_root(),
+dlang.generate_dub_file(meson.project_name().to_lower(), meson.build_root(),
                         authors: 'Meson Team',
                         description: 'Test executable',
                         copyright: 'Copyright © 2018, Meson Team',
                         license: 'MIT',
                         sourceFiles: 'test.d',
                         targetType: 'executable',
                         dependencies: urld_dep
-)
+)

test cases/d/17 dub and meson project/.gitignore

@@ -0,0 +1,2 @@
+17-dub-meson-project*
+lib17-dub-meson-project*

test cases/d/17 dub and meson project/dub.json

@@ -0,0 +1,11 @@
+{
+  "name": "17-dub-meson-project",
+  "dependencies": {
+    "urld": ">=3.0.0 <3.0.1",
+    "dubtestproject:test3": "1.2.0",
+    ":multi-configuration": "*"
+  },
+  "subPackages": [
+    "multi-configuration"
+  ]
+}

test cases/d/17 dub and meson project/dub.selections.json

@@ -0,0 +1,7 @@
+{
+	"fileVersion": 1,
+	"versions": {
+		"dubtestproject": "1.2.0",
+		"urld": "3.0.0"
+	}
+}

test cases/d/17 dub and meson project/meson.build

@@ -0,0 +1,32 @@
+project('Dub dependency respects dub.selections.json', 'd')
+
+dub_exe = find_program('dub', required : false)
+if not dub_exe.found()
+    error('MESON_SKIP_TEST: Dub not found')
+endif
+
+dub_ver = dub_exe.version()
+if not dub_ver.version_compare('>=1.35.0')
+  error('MESON_SKIP_TEST: test requires dub >=1.35.0')
+endif
+
+# Multiple versions supported
+urld = dependency('urld', method: 'dub', version: [ '>=3.0.0', '<3.0.1' ])
+
+# The version we got is the one in dub.selections.json
+version = urld.version()
+if version != '3.0.0'
+  error(f'Expected urld version to be the one selected in dub.selections.json but got @version@')
+endif
+
+# dependency calls from subdirectories respect meson.source_root()/dub.selections.json
+subdir('x/y/z')
+
+# dependencies respect their configuration selected in dub.json
+run_command(dub_exe, 'build', '--deep', ':multi-configuration',
+            '--compiler', meson.get_compiler('d').cmd_array()[0],
+            '--arch', host_machine.cpu_family(),
+            '--root', meson.source_root(),
+            '--config', 'lib',
+            check: true)
+found = dependency('17-dub-meson-project:multi-configuration', method: 'dub')

test cases/d/17 dub and meson project/multi-configuration/.gitignore

@@ -0,0 +1,2 @@
+libmulti-configuration*
+multi-configuration*

test cases/d/17 dub and meson project/multi-configuration/dub.json

@@ -0,0 +1,14 @@
+{
+  "name": "multi-configuration",
+  "configurations": {
+    "app": {
+      "targetType": "executable"
+    },
+    "lib": {
+      "targetType": "library",
+      "excludedSourceFiles": [
+        "source/app.d"
+      ]
+    }
+  }
+}

test cases/d/17 dub and meson project/multi-configuration/dub.selections.json

@@ -0,0 +1,5 @@
+{
+	"fileVersion": 1,
+	"versions": {
+	}
+}

test cases/d/17 dub and meson project/multi-configuration/source/app.d

@@ -0,0 +1 @@
+void main () {}

test cases/d/17 dub and meson project/source/app.d

@@ -0,0 +1 @@
+void main () {}