revise and improve documentation

README.md

@@ -6,7 +6,7 @@
 A compiler and scene generator for the Scenic scenario description language.
 Please see the [documentation](https://scenic-lang.readthedocs.io/) for installation instructions, as well as tutorials and other information about the Scenic language, its implementation, and its interfaces to various simulators.
 
-For a description of the language and some of its applications, see [our PLDI 2019 paper](https://arxiv.org/abs/1809.09310).
+For a description of the language and some of its applications, see [our PLDI 2019 paper](https://arxiv.org/abs/1809.09310) (*note:* the syntax of Scenic has changed slightly since then).
 Scenic was designed and implemented by Daniel J. Fremont, Tommaso Dreossi, Shromona Ghosh, Edward Kim, Xiangyu Yue, Alberto L. Sangiovanni-Vincentelli, and Sanjit A. Seshia.
 
 If you have any problems using Scenic, please submit an issue to [our GitHub repository](https://github.com/BerkeleyLearnVerify/Scenic) or contact Daniel at <[email protected]>.

docs/Makefile

@@ -15,7 +15,7 @@ help:
 .PHONY: help Makefile
 
 clean:
-	rm -rf _autosummary
+	rm -rf modules
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 # Catch-all target: route all unknown targets to Sphinx using the new

docs/conf.py

@@ -52,6 +52,7 @@
 'sphinx.ext.autosummary',
 'sphinx.ext.napoleon',
 'sphinx.ext.viewcode',
+'sphinx.ext.intersphinx',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -63,6 +64,7 @@
 exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
 
 default_role = 'any'
+#nitpicky = True
 
 add_module_names = False
 autosummary_generate = True
@@ -75,10 +77,18 @@
 
 autodoc_default_options = {
     'members': None,
+
+    # include members starting with underscores by default;
+    # we'll install an extension below to skip those with :meta private:
     'private-members': None,
+
     'show-inheritance': None,
 }
 
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3', None),
+}
+
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
@@ -145,21 +155,96 @@ def parse_class_attributes_section(self, section):
     return self._format_fields('Class Attributes', self._consume_fields())
 GoogleDocstring._parse_class_attributes_section = parse_class_attributes_section
 
+def parse_properties_section(self, section):
+    return self._format_fields('Properties', self._consume_fields())
+GoogleDocstring._parse_properties_section = parse_properties_section
+
 orig_parse = GoogleDocstring._parse
 def parse(self):
     self._sections['class attributes'] = self._parse_class_attributes_section
+    self._sections['properties'] = self._parse_properties_section
     orig_parse(self)
 GoogleDocstring._parse = parse
 
-# -- Extension for correctly displaying Scenic code --------------------------
+# -- Monkeypatch to list private members after public ones -------------------
+
+from sphinx.ext.autodoc import Documenter
+
+orig_sort_members = Documenter.sort_members
+def sort_members(self, documenters, order):
+    documenters = orig_sort_members(self, documenters, order)
+    def key(entry):
+        parts = entry[0].name.split('::')
+        if len(parts) == 2:
+            name = parts[-1]
+        else:
+            assert len(parts) == 1
+            name = parts[0].split('.')[-1]
+        return name.startswith('_')
+    documenters.sort(key=key)
+    return documenters
+Documenter.sort_members = sort_members
+
+# -- Monkeypatch to hide private base classes --------------------------------
+
+from sphinx.ext.autodoc import ClassDocumenter
+
+orig_add_directive_header = ClassDocumenter.add_directive_header
+def add_directive_header(self, sig):
+    oldBases = self.object.__bases__
+    if oldBases != (object,) and self.options.show_inheritance:
+        newBases = [base for base in oldBases if not base.__name__.startswith('_')]
+        if newBases:
+            self.object.__bases__ = tuple(newBases)
+            orig_add_directive_header(self, sig)
+            self.object.__bases__ = oldBases
+            return
+    oldSI = self.options.show_inheritance
+    self.options.show_inheritance = False
+    orig_add_directive_header(self, sig)
+    self.options.show_inheritance = oldSI
+ClassDocumenter.add_directive_header = add_directive_header
+
+# -- Monkeypatch to improve formatting of certain values ---------------------
+
+import scenic
+import sphinx.ext.autodoc
+
+orig_object_description = sphinx.ext.autodoc.object_description
+def object_description(obj):
+    if obj is scenic.core.regions.nowhere or obj is scenic.core.regions.everywhere:
+        return str(obj)
+    elif isinstance(obj, (scenic.core.regions.Region,
+                          scenic.core.vectors.VectorField,
+                          scenic.domains.driving.roads.Network)):
+        raise ValueError    # prevent rendering of this value
+    else:
+        return orig_object_description(obj)
+sphinx.ext.autodoc.object_description = object_description
+
+# TODO switch to type_aliases once Sphinx 3.3 is out;
+# this hack suggested by sjjessop at https://github.com/sphinx-doc/sphinx/issues/6518
+import typing
+orig_get_type_hints = typing.get_type_hints
+def get_type_hints(obj, globals=None, locals=None):
+    if locals is None:
+        locals = {}
+    locals['Vectorlike'] = 'Vectorlike'
+    return orig_get_type_hints(obj, globals, locals)
+typing.get_type_hints = get_type_hints
+
+# -- Extension for correctly displaying Scenic code and skipping internals ---
 
 def setup(app):
     app.connect('viewcode-find-source', handle_find_source)
+    app.connect('autodoc-process-signature', handle_process_signature)
+    app.connect('autodoc-skip-member', handle_skip_member)
 
     return { 'parallel_read_safe': True }
 
 import importlib
 from sphinx.pycode import ModuleAnalyzer
+from sphinx.util.docstrings import extract_metadata
 
 def handle_find_source(app, modname):
     try:
@@ -178,3 +263,59 @@ def handle_find_source(app, modname):
 
     # Return original Scenic source, plus tags (line numbers will correspond)
     return module._source, analyzer.tags
+
+def handle_process_signature(app, what, name, obj, options, signature, ret_anno):
+    if (what == 'class'
+        and issubclass(obj, scenic.core.object_types._Constructible)
+        and obj is not scenic.core.object_types._Constructible):
+        return ('(<specifiers>)', None)
+    else:
+        return None
+
+def handle_skip_member(app, what, name, obj, skip, options):
+    if not skip:
+        doc = getattr(obj, '__doc__')
+        if doc and 'private' in extract_metadata(doc):
+            return True
+    return None
+
+# -- Big monkeypatch to fix bug in autosummary (temporarily) -----------------
+
+from sphinx.pycode import ModuleAnalyzer, PycodeError
+import sphinx.ext.autosummary.generate as as_gen
+
+orig_generate_autosummary_content = as_gen.generate_autosummary_content
+orig_scan = as_gen.ModuleScanner.scan
+
+def generate_autosummary_content(name, obj, parent,
+                                 template, template_name,
+                                 imported_members, app,
+                                 recursive, context,
+                                 modname = None, qualname = None):
+    members = dir(obj)
+    attrs = set()
+    try:
+        analyzer = ModuleAnalyzer.for_module(name)
+        attr_docs = analyzer.find_attr_docs()
+        for namespace, attr_name in attr_docs:
+            if namespace == '' and attr_name in members:
+                attrs.add(attr_name)
+    except PycodeError:
+        pass    # give up if ModuleAnalyzer fails to parse code
+
+    def scan(self, imported_members):
+        scanned = set(orig_scan(self, imported_members))
+        allMembers = []
+        for name in dir(self.object):
+            if name in scanned or name in attrs:
+                allMembers.append(name)
+        return allMembers
+    as_gen.ModuleScanner.scan = scan
+
+    return orig_generate_autosummary_content(name, obj, parent,
+                                             template, template_name,
+                                             imported_members, app,
+                                             recursive, context,
+                                             modname=modname, qualname=qualname)
+
+as_gen.generate_autosummary_content = generate_autosummary_content

docs/credits.rst

@@ -1,3 +1,5 @@
+..  _credits:
+
 Credits
 =======
 
@@ -9,20 +11,28 @@ The Scenic project was started at UC Berkeley in Sanjit Seshia's research group.
 
 The language was initially developed by Daniel J. Fremont, Tommaso Dreossi, Shromona Ghosh, Xiangyu Yue, Alberto L. Sangiovanni-Vincentelli, and Sanjit A. Seshia.
 
-Edward Kim assisted in putting together this documentation and extending the language to dynamic scenarios.
+Edward Kim assisted in developing the library for dynamic driving scenarios and putting together this documentation.
 
-The Scenic tool has benefitted from code contributions from:
+The Scenic tool and example scenarios have benefitted from code contributions from:
 
 	* Johnathan Chiu
+	* Greg Crow
 	* Francis Indaheng
+	* Ellen Kalvan
 	* Martin Jansa (LG Electronics, Inc.)
+	* Kevin Li
+	* Gaurav Rao
+	* Matthew Rhea
+	* Jay Shenoy
 	* Wilson Wu
 
 Finally, many other people provided helpful advice and discussions, including:
 
 	* Ankush Desai
 	* Alastair Donaldson
 	* Andrew Gordon
+	* Steve Lemke
+	* Shalin Mehta
 	* Jonathan Ragan-Kelley
 	* Sriram Rajamani
 	* Marcell Vazquez-Chanlatte

docs/developing.rst

@@ -0,0 +1,41 @@
+..  _developing:
+
+Developing Scenic
+=================
+
+This page covers information useful if you will be developing Scenic, either changing the
+language itself or adding new built-in libraries or simulator interfaces.
+
+Getting Started
+---------------
+
+Start by cloning our repository on GitHub and installing Poetry as described in
+:doc:`quickstart`. When using Poetry to install Scenic in your virtual environment, use
+the command :command:`poetry install -E dev` to make sure you get all the dependencies
+needed for development.
+
+Running the Test Suite
+----------------------
+
+Scenic has an extensive test suite exercising most of the features of the language. We
+use the `pytest <https://docs.pytest.org/en/latest/index.html>`_ Python testing tool. To
+run the entire test suite, run the command :command:`pytest` inside the virtual
+environment.
+
+Some of the tests are quite slow, e.g. those which test the parsing and construction of
+road networks. We add a ``--fast`` option to pytest	which skips such tests, while
+still covering all of the core features of the language. So it is convenient to often run
+:command:`pytest --fast` as a quick check, remembering to run the full :command:`pytest`
+before making any final commits. You can also run specific parts of the test suite with a
+command like :command:`pytest tests/syntax/test_specifiers.py`, or use pytest's ``-k``
+option to filter by test name, e.g. :command:`pytest -k specifiers`.
+
+Note that many of Scenic's tests are probabilistic, so in order to reproduce a test
+failure you may need to set the random seed. We use the
+`pytest-randomly <https://github.com/pytest-dev/pytest-randomly>`_ plugin to help with
+this: at the beginning of each run of ``pytest``, it prints out a line like::
+
+	Using --randomly-seed=344295085
+
+Adding this as an option, i.e. running :command:`pytest --randomly-seed=344295085`, will
+reproduce the same sequence of tests with the same Python/Scenic random seed.

docs/glossary.rst

@@ -0,0 +1,20 @@
+:orphan:
+
+Glossary
+========
+
+.. glossary::
+
+	action
+		A primitive operation executed by an agent during a single step of a dynamic
+		simulation. For example, a car might take an action which sets its throttle, or
+		turns on its headlights. Actions are defined by the simulator interfaces (or
+		abstract domains like `scenic.domains.driving`) as subclasses of
+		:obj:`~scenic.core.simulators.Action`.
+
+	dynamic properties
+		Properties of Scenic objects which are updated at each time step of a dynamic
+		simulation. The built-in properties representing positions, orientations,
+		velocities, etc. are all dynamic. See the source code of
+		:obj:`scenic.domains.driving.model.DrivingObject` for an example of defining a
+		dynamic property.

docs/index.rst

@@ -3,28 +3,60 @@ Welcome to Scenic's documentation!
 
 Scenic is a domain-specific probabilistic programming language for modeling the environments of cyber-physical systems like robots and autonomous cars.
 A Scenic program defines a distribution over *scenes*, configurations of physical objects and agents; sampling from this distribution yields concrete scenes which can be simulated to produce training or testing data.
+Scenic can also define (probabilistic) policies for dynamic agents, allowing modeling scenarios where agents take actions over time in response to the state of the world.
 
-Scenic was designed and implemented by Daniel J. Fremont, Tommaso Dreossi, Shromona Ghosh, Xiangyu Yue, Alberto L. Sangiovanni-Vincentelli, and Sanjit A. Seshia.
+Scenic was designed and implemented by Daniel J. Fremont, Tommaso Dreossi, Shromona Ghosh, Edward Kim, Xiangyu Yue, Alberto L. Sangiovanni-Vincentelli, and Sanjit A. Seshia.
 For a description of the language and some of its applications, see `our PLDI 2019 paper <https://arxiv.org/abs/1809.09310>`_; a more in-depth discussion is in Chapters 5 and 8 of `this thesis <https://people.ucsc.edu/~dfremont/papers/thesis.pdf>`_.
 Our :doc:`publications <publications>` page lists additional papers using Scenic.
 
+.. note::
+
+   The syntax of Scenic 2.x is not completely backwards-compatible with 1.x, which was used in our papers prior to late 2020. See :doc:`new` for a list of syntax changes and new features.
+   If your existing code no longer works, install the latest 1.x release from
+   `GitHub <https://github.com/BerkeleyLearnVerify/Scenic/releases>`__.
+
 If you have any problems using Scenic, please submit an issue to `our GitHub repository <https://github.com/BerkeleyLearnVerify/Scenic>`_ or contact Daniel at [email protected].
 
 Table of Contents
 =================
 
 .. toctree::
    :maxdepth: 1
+   :caption: Introduction
 
    quickstart
-   tutorial
-   tutorial_dynamic
-   library
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Tutorials
+
+   tutorials/tutorial
+   tutorials/dynamics
+   tutorials/composition
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Language and Tool Reference
+
    syntax_guide
    syntax_details
+   options
+   developing
+   internals
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Libraries and Simulators
+
+   libraries
    simulators
    new_simulator
-   internals
+
+.. toctree::
+   :maxdepth: 1
+   :caption: General Information
+
+   new
    publications
    credits
 
@@ -33,7 +65,7 @@ Indices and Tables
 
 * :ref:`genindex`
 * :ref:`modindex`
-* :ref:`search`
+* :doc:`glossary`
 
 License
 =======