Merge pull request #260 from aleroot/master

Add JobStatus REST web service to easily get the status of a JobEspec…

docs/api.rst

@@ -111,6 +111,30 @@ Example:
     When a parameter other than ``setting`` is entered multiple times with ``-d``, only the first
     value is sent to the spider.
 
+.. _status.json
+
+status.json
+-----------
+
+.. versionadded:: 1.5.0
+
+Get the status of a job.
+
+Supported request methods
+  ``GET``
+Parameters
+  ``job`` (required)
+    the job ID
+  ``project``
+    the project name
+
+Example:
+
+.. code-block:: shell-session
+
+    $ curl http://localhost:6800/status.json?job=6487ec79947edab326d6db28a2d86511e8247444
+    {"node_name": "mynodename", "status": "ok", "currstate": "running"}
+
 .. _cancel.json:
 
 cancel.json

docs/config.rst

@@ -18,6 +18,8 @@ the :ref:`example <config-example>`).
 Environment variables
 ---------------------
 
+.. versionadded:: 1.5.0
+
 The following environment variables override corresponding options:
 
 * ``SCRAPYD_BIND_ADDRESS`` (:ref:`bind_address`)

docs/news.rst

@@ -11,10 +11,15 @@ Unreleased
 Added
 ~~~~~
 
+- Add a :ref:`status.json` webservice, to get the status of a job.
 - Respond to HTTP ``OPTIONS`` method requests.
 - Add environment variables to override common options. See :doc:`config`.
-- Add documentation on how to add webservices (endpoints). See :ref:`config-services`.
-- Add documentation on how to create a Docker image. See :ref:`docker`.
+
+Documentation
+^^^^^^^^^^^^^
+
+- How to add webservices (endpoints). See :ref:`config-services`.
+- How to create Docker images. See :ref:`docker`.
 
 Changed
 ~~~~~~~

integration_tests/test_webservice.py

@@ -20,6 +20,7 @@ def assert_webservice(method, path, expected, **kwargs):
         ("addversion", "POST"),
         ("schedule", "POST"),
         ("cancel", "POST"),
+        ("status", "GET"),
         ("listprojects", "GET"),
         ("listversions", "GET"),
         ("listspiders", "GET"),
@@ -59,6 +60,15 @@ def test_schedule():
     )
 
 
+def test_status_nonexistent():
+    assert_webservice(
+        "get",
+        "/status.json",
+        {"status": "ok", "currstate": "unknown"},
+        params={"job": "sample"},
+    )
+
+
 def test_cancel_nonexistent():
     assert_webservice(
         "post",

scrapyd/default_scrapyd.conf

@@ -36,6 +36,7 @@ dbs_dir           = dbs
 [services]
 schedule.json     = scrapyd.webservice.Schedule
 cancel.json       = scrapyd.webservice.Cancel
+status.json       = scrapyd.webservice.Status
 addversion.json   = scrapyd.webservice.AddVersion
 listprojects.json = scrapyd.webservice.ListProjects
 listversions.json = scrapyd.webservice.ListVersions

scrapyd/webservice.py

@@ -165,13 +165,46 @@ def render_GET(self, txrequest):
         return {"node_name": self.root.nodename, "status": "ok", "spiders": spiders}
 
 
+class Status(WsResource):
+
+    def render_GET(self, txrequest):
+        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
+        job = _get_required_param(args, 'job')[0]
+        project = args.get('project', [None])[0]
+
+        spiders = self.root.launcher.processes.values()
+        queues = self.root.poller.queues
+
+        result = {"node_name": self.root.nodename, "status": "ok", "currstate": "unknown"}
+
+        for s in self.root.launcher.finished:
+            if (project is None or s.project == project) and s.job == job:
+                result["currstate"] = "finished"
+                return result
+
+        for s in spiders:
+            if (project is None or s.project == project) and s.job == job:
+                result["currstate"] = "running"
+                return result
+
+        for qname in (queues if project is None else [project]):
+            for x in queues[qname].list():
+                if x["_job"] == job:
+                    result["currstate"] = "pending"
+                    return result
+
+        return result
+
+
 class ListJobs(WsResource):
 
     def render_GET(self, txrequest):
         args = native_stringify_dict(copy(txrequest.args), keys_only=False)
         project = args.get('project', [None])[0]
+
         spiders = self.root.launcher.processes.values()
         queues = self.root.poller.queues
+
         pending = [
             {"project": qname, "spider": x["name"], "id": x["_job"]}
             for qname in (queues if project is None else [project])
@@ -184,7 +217,9 @@ def render_GET(self, txrequest):
                 "id": s.job,
                 "pid": s.pid,
                 "start_time": str(s.start_time),
-            } for s in spiders if project is None or s.project == project
+            }
+            for s in spiders
+            if project is None or s.project == project
         ]
         finished = [
             {
@@ -195,9 +230,11 @@ def render_GET(self, txrequest):
                 "end_time": str(s.end_time),
                 "log_url": job_log_url(s),
                 "items_url": job_items_url(s),
-            } for s in self.root.launcher.finished
+            }
+            for s in self.root.launcher.finished
             if project is None or s.project == project
         ]
+
         return {"node_name": self.root.nodename, "status": "ok",
                 "pending": pending, "running": running, "finished": finished}