Load namespace modules lazily to cut import time and memory

dropbox/_lazy_namespace.py

@@ -0,0 +1,46 @@
+"""Lazy references to generated namespace modules.
+
+The generated base clients (``dropbox.base`` and ``dropbox.base_team``) have a
+method for every route in every namespace. Importing every namespace module up
+front costs about 30 MB and a noticeable amount of import time, most of it from
+``team_log``, even for apps that only ever call a handful of ``files`` routes.
+
+The base clients refer to namespaces through :class:`LazyNamespace` instead, so a
+namespace module is only imported the first time one of its routes or types is
+used.
+"""
+
+import importlib
+
+
+class LazyNamespace(object):
+    """Stands in for a namespace module and imports it on first attribute access.
+
+    Importing goes through :func:`importlib.import_module`, which takes the
+    per-module import lock, so concurrent first accesses from several threads
+    are safe and all see the same fully initialized module.
+    """
+
+    __slots__ = ("_name", "_module")
+
+    def __init__(self, name):
+        self._name = name
+        self._module = None
+
+    def _load(self):
+        module = self._module
+        if module is None:
+            module = importlib.import_module(self._name)
+            self._module = module
+        return module
+
+    def __getattr__(self, attr):
+        # Only called for attributes not found on the proxy itself.
+        return getattr(self._load(), attr)
+
+    def __dir__(self):
+        return dir(self._load())
+
+    def __repr__(self):
+        state = "loaded" if self._module is not None else "not loaded"
+        return "<LazyNamespace {!r} ({})>".format(self._name, state)

dropbox/base.py

@@ -6,27 +6,29 @@
 from abc import ABCMeta, abstractmethod
 import warnings
 
-from dropbox import account
-from dropbox import async_
-from dropbox import auth
-from dropbox import check
-from dropbox import common
-from dropbox import contacts
-from dropbox import file_properties
-from dropbox import file_requests
-from dropbox import files
-from dropbox import openid
-from dropbox import paper
-from dropbox import riviera
-from dropbox import secondary_emails
-from dropbox import seen_state
-from dropbox import sharing
-from dropbox import team
-from dropbox import team_common
-from dropbox import team_log
-from dropbox import team_policies
-from dropbox import users
-from dropbox import users_common
+from dropbox._lazy_namespace import LazyNamespace
+
+account = LazyNamespace("dropbox.account")
+async_ = LazyNamespace("dropbox.async_")
+auth = LazyNamespace("dropbox.auth")
+check = LazyNamespace("dropbox.check")
+common = LazyNamespace("dropbox.common")
+contacts = LazyNamespace("dropbox.contacts")
+file_properties = LazyNamespace("dropbox.file_properties")
+file_requests = LazyNamespace("dropbox.file_requests")
+files = LazyNamespace("dropbox.files")
+openid = LazyNamespace("dropbox.openid")
+paper = LazyNamespace("dropbox.paper")
+riviera = LazyNamespace("dropbox.riviera")
+secondary_emails = LazyNamespace("dropbox.secondary_emails")
+seen_state = LazyNamespace("dropbox.seen_state")
+sharing = LazyNamespace("dropbox.sharing")
+team = LazyNamespace("dropbox.team")
+team_common = LazyNamespace("dropbox.team_common")
+team_log = LazyNamespace("dropbox.team_log")
+team_policies = LazyNamespace("dropbox.team_policies")
+users = LazyNamespace("dropbox.users")
+users_common = LazyNamespace("dropbox.users_common")
 
 
 class DropboxBase(object):

dropbox/base_team.py

@@ -6,34 +6,38 @@
 from abc import ABCMeta, abstractmethod
 import warnings
 
-from dropbox import account
-from dropbox import async_
-from dropbox import auth
-from dropbox import check
-from dropbox import common
-from dropbox import contacts
-from dropbox import file_properties
-from dropbox import file_requests
-from dropbox import files
-from dropbox import openid
-from dropbox import paper
-from dropbox import riviera
-from dropbox import secondary_emails
-from dropbox import seen_state
-from dropbox import sharing
-from dropbox import team
-from dropbox import team_common
-from dropbox import team_log
-from dropbox import team_policies
-from dropbox import users
-from dropbox import users_common
+from dropbox._lazy_namespace import LazyNamespace
+
+account = LazyNamespace("dropbox.account")
+async_ = LazyNamespace("dropbox.async_")
+auth = LazyNamespace("dropbox.auth")
+check = LazyNamespace("dropbox.check")
+common = LazyNamespace("dropbox.common")
+contacts = LazyNamespace("dropbox.contacts")
+file_properties = LazyNamespace("dropbox.file_properties")
+file_requests = LazyNamespace("dropbox.file_requests")
+files = LazyNamespace("dropbox.files")
+openid = LazyNamespace("dropbox.openid")
+paper = LazyNamespace("dropbox.paper")
+riviera = LazyNamespace("dropbox.riviera")
+secondary_emails = LazyNamespace("dropbox.secondary_emails")
+seen_state = LazyNamespace("dropbox.seen_state")
+sharing = LazyNamespace("dropbox.sharing")
+team = LazyNamespace("dropbox.team")
+team_common = LazyNamespace("dropbox.team_common")
+team_log = LazyNamespace("dropbox.team_log")
+team_policies = LazyNamespace("dropbox.team_policies")
+users = LazyNamespace("dropbox.users")
+users_common = LazyNamespace("dropbox.users_common")
 
 
 class DropboxTeamBase(object):
     __metaclass__ = ABCMeta
 
     @abstractmethod
-    def request(self, route, namespace, request_arg, request_binary, timeout=None):
+    def request(
+        self, route, namespace, request_arg, request_binary, timeout=None, extra_headers=None
+    ):
         pass
 
     # ------------------------------------------

generate_base_client.py

@@ -100,7 +100,17 @@ def main():
 
     o = subprocess.check_output(
         (
-            ["python", "-m", "stone.cli", "python_client", dropbox_pkg_path]
+            [
+                "python",
+                "-m",
+                "stone.cli",
+                os.path.join(
+                    os.path.dirname(__file__),
+                    "generator",
+                    "dropbox_python_client.stoneg.py",
+                ),
+                dropbox_pkg_path,
+            ]
             + specs
             + ["-a", "host", "-a", "style", "-a", "auth", "-a", "scope"]
             + [

generator/dropbox_python_client.stoneg.py

@@ -1,6 +1,6 @@
 from stone.backends import python_client
 from stone.backends.helpers import fmt_underscores
-from stone.backends.python_helpers import fmt_func
+from stone.backends.python_helpers import fmt_func, fmt_namespace
 from stone.ir import (
     is_nullable_type,
     is_struct_type,
@@ -45,6 +45,20 @@ def generate(self, api):
                 self.emit()
                 self._generate_route_methods(api.namespaces.values())
 
+    def _generate_imports(self, namespaces):
+        """Refer to namespace modules lazily.
+
+        Importing every namespace up front costs about 30 MB, mostly team_log,
+        even when an app only calls a few routes. Each namespace is imported on
+        first use instead (see dropbox/_lazy_namespace.py).
+        """
+        self.emit("from {}._lazy_namespace import LazyNamespace".format(self.args.types_package))
+        self.emit()
+        for namespace in namespaces:
+            if namespace.data_types:
+                name = fmt_namespace(namespace.name)
+                self.emit('{} = LazyNamespace("{}.{}")'.format(name, self.args.types_package, name))
+
     @staticmethod
     def _supports_extra_headers(namespace, route):
         return namespace.name == "files" and route.name == "download"

scripts/benchmark_import.py

@@ -0,0 +1,62 @@
+"""Measure what `import dropbox` costs on its own.
+
+Each run uses a fresh interpreter. requests and stone are imported first, so the
+numbers only cover the SDK's own modules. Times are medians over `runs`
+interpreters; traced memory (tracemalloc) is the median of 3 separate runs.
+
+    python scripts/benchmark_import.py [runs]
+"""
+
+import json
+import statistics
+import subprocess
+import sys
+
+CODE = r"""
+import json, resource, sys, time, tracemalloc
+import requests
+import stone.backends.python_rsrc.stone_validators
+
+trace = sys.argv[1] == "trace"
+rss_before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+if trace:
+    tracemalloc.start()
+start = time.perf_counter()
+import dropbox
+elapsed = time.perf_counter() - start
+traced = tracemalloc.get_traced_memory()[0] if trace else 0
+rss_after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+# ru_maxrss is bytes on macOS and kilobytes on Linux
+scale = 1 if sys.platform == "darwin" else 1024
+print(json.dumps({
+    "ms": elapsed * 1000,
+    "traced_mb": traced / 1e6,
+    "rss_mb": (rss_after - rss_before) * scale / 1e6,
+    "modules": sorted(m for m in sys.modules if m.startswith("dropbox.")),
+}))
+"""
+
+
+def main():
+    runs = int(sys.argv[1]) if len(sys.argv) > 1 else 15
+
+    def run(mode):
+        out = subprocess.check_output([sys.executable, "-c", CODE, mode])
+        return json.loads(out)
+
+    # Timing runs don't use tracemalloc, which slows imports down a lot.
+    results = [run("time") for _ in range(runs)]
+    traced = [run("trace")["traced_mb"] for _ in range(3)]
+    summary = {
+        "runs": runs,
+        "import_ms_median": round(statistics.median(r["ms"] for r in results), 1),
+        "traced_mb_median": round(statistics.median(traced), 1),
+        "rss_mb_median": round(statistics.median(r["rss_mb"] for r in results), 1),
+        "dropbox_modules_loaded": len(results[-1]["modules"]),
+        "team_log_loaded": "dropbox.team_log" in results[-1]["modules"],
+    }
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

test/unit/test_lazy_namespace.py

@@ -0,0 +1,117 @@
+import json
+import subprocess
+import sys
+import textwrap
+from unittest import mock
+
+import pytest
+
+from dropbox import base, base_team
+from dropbox._lazy_namespace import LazyNamespace
+
+
+def _run(code):
+    """Run code in a fresh interpreter so sys.modules starts empty."""
+    out = subprocess.check_output([sys.executable, "-c", textwrap.dedent(code)])
+    return json.loads(out.decode("utf-8").strip().splitlines()[-1])
+
+
+class TestLazyNamespace:
+    def test_importing_dropbox_does_not_load_team_namespaces(self):
+        loaded = _run(
+            """
+            import json, sys
+            import dropbox
+            print(json.dumps(sorted(m for m in sys.modules if m.startswith("dropbox."))))
+            """
+        )
+        assert "dropbox.team_log" not in loaded
+        assert "dropbox.team" not in loaded
+        assert "dropbox.files" in loaded
+
+    def test_namespace_is_loaded_on_first_route_call(self):
+        result = _run(
+            """
+            import json, sys
+            from unittest import mock
+            import dropbox
+
+            dbx = dropbox.DropboxTeam("token")
+            before = "dropbox.team_log" in sys.modules
+            with mock.patch.object(dbx, "request", return_value="ok") as request:
+                dbx.team_log_get_events(limit=5)
+            route, namespace, arg, binary = request.call_args[0]
+            from dropbox import team_log
+            print(json.dumps({
+                "before": before,
+                "after": "dropbox.team_log" in sys.modules,
+                "same_route": route is team_log.get_events,
+                "namespace": namespace,
+                "arg_type": type(arg).__name__,
+                "limit": arg.limit,
+            }))
+            """
+        )
+        assert result == {
+            "before": False,
+            "after": True,
+            "same_route": True,
+            "namespace": "team_log",
+            "arg_type": "GetTeamEventsArg",
+            "limit": 5,
+        }
+
+    def test_concurrent_first_access_sees_one_module(self):
+        result = _run(
+            """
+            import json, threading
+            from dropbox._lazy_namespace import LazyNamespace
+
+            ns = LazyNamespace("dropbox.team_log")
+            barrier = threading.Barrier(16)
+            seen = []
+
+            def worker():
+                barrier.wait()
+                seen.append(ns.get_events)
+
+            threads = [threading.Thread(target=worker) for _ in range(16)]
+            for t in threads:
+                t.start()
+            for t in threads:
+                t.join()
+            print(json.dumps({"count": len(seen), "distinct": len({id(r) for r in seen})}))
+            """
+        )
+        assert result == {"count": 16, "distinct": 1}
+
+    def test_proxy_behaves_like_the_module(self):
+        from dropbox import files
+
+        ns = LazyNamespace("dropbox.files")
+        assert "not loaded" in repr(ns)
+        assert ns.DownloadArg is files.DownloadArg
+        assert "loaded" in repr(ns) and "not loaded" not in repr(ns)
+        assert "DownloadArg" in dir(ns)
+        with pytest.raises(AttributeError):
+            ns.does_not_exist
+
+    def test_missing_module_raises_import_error_on_use(self):
+        ns = LazyNamespace("dropbox.no_such_namespace")
+        with pytest.raises(ImportError):
+            ns.anything
+
+    def test_base_clients_use_lazy_namespaces(self):
+        assert isinstance(base.team_log, LazyNamespace)
+        assert isinstance(base_team.team_log, LazyNamespace)
+
+    def test_user_route_still_builds_real_types(self):
+        from dropbox import Dropbox, files
+
+        dbx = Dropbox("token")
+        with mock.patch.object(dbx, "request", return_value="ok") as request:
+            dbx.files_get_metadata("/a")
+        route, namespace, arg, binary = request.call_args[0]
+        assert route is files.get_metadata
+        assert isinstance(arg, files.GetMetadataArg)
+        assert arg.path == "/a"