diff --git a/.circleci/test-integration.sh b/.circleci/test-integration.sh index 5121d586..a13e1611 100755 --- a/.circleci/test-integration.sh +++ b/.circleci/test-integration.sh @@ -27,7 +27,7 @@ if [ "$(ls results/results*.xml | wc -l)" -lt "1" ]; then echo "Integration test results.xml does not exist, exiting abnormally" exit 1 fi -if [ "$(grep -c 'failure ' results/results*.xml)" -ge 1 ]; then +if [ "$(grep -o 'failure ' results/results*.xml | wc -l)" -ge 1 ]; then echo "Integration test results.xml has failures, exiting abnormally" exit 1 fi diff --git a/default_vocab.ttl b/default_vocab.ttl index f6cb157c..635030e3 100644 --- a/default_vocab.ttl +++ b/default_vocab.ttl @@ -407,6 +407,7 @@ owl:Ontology rdfs:label "Ontology"; whyis:hasView "ontology_view.html". np:Nanopublication a owl:Class; + whyis:hasDescribe "nanopub_describe.json"; whyis:hasView "nanopublication_view.html". # a whyis:searchView. diff --git a/docker/compose/split/db-dev/docker-compose.yml b/docker/compose/split/db-dev/docker-compose.yml new file mode 100644 index 00000000..d161131e --- /dev/null +++ b/docker/compose/split/db-dev/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3' +services: + redis: + container_name: "redis" + image: "redis:3.0.6-alpine" + ports: + - "127.0.0.1:6379:6379" + volumes: + - ./../../../../../../data/redis:/data + blazegraph: + build: + args: + WHYIS_IMAGE_TAG: ${WHYIS_IMAGE_TAG:-latest} + context: ../../../.. + dockerfile: docker/image/whyis-blazegraph/Dockerfile + container_name: "blazegraph" + image: tetherlessworld/whyis-blazegraph:${WHYIS_IMAGE_TAG:-latest} + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./../../../../../../data:/data diff --git a/docker/compose/split/db/docker-compose.yml b/docker/compose/split/db/docker-compose.yml index d161131e..5939a4a4 100644 --- a/docker/compose/split/db/docker-compose.yml +++ b/docker/compose/split/db/docker-compose.yml @@ -3,8 +3,8 @@ services: redis: container_name: "redis" image: "redis:3.0.6-alpine" - ports: - - "127.0.0.1:6379:6379" + expose: + - 6379 volumes: - ./../../../../../../data/redis:/data blazegraph: @@ -15,7 +15,5 @@ services: dockerfile: docker/image/whyis-blazegraph/Dockerfile container_name: "blazegraph" image: tetherlessworld/whyis-blazegraph:${WHYIS_IMAGE_TAG:-latest} - ports: - - "127.0.0.1:8080:8080" volumes: - ./../../../../../../data:/data diff --git a/docker/image/whyis-blazegraph/Dockerfile b/docker/image/whyis-blazegraph/Dockerfile index edd45904..2bce05d4 100644 --- a/docker/image/whyis-blazegraph/Dockerfile +++ b/docker/image/whyis-blazegraph/Dockerfile @@ -40,3 +40,5 @@ USER jetty ENV JAVA_OPTIONS="-ea -Xmx4g -server -XX:+UseParallelOldGC -Djava.awt.headless=true -Dorg.eclipse.jetty.server.Request.maxFormContentSize=200000000" WORKDIR /var/lib/jetty + +EXPOSE 8080 diff --git a/docker/image/whyis/Dockerfile b/docker/image/whyis/Dockerfile index b55500e1..1e09f23b 100644 --- a/docker/image/whyis/Dockerfile +++ b/docker/image/whyis/Dockerfile @@ -14,6 +14,8 @@ RUN npm run build # Deploy image from here FROM tetherlessworld/whyis-deps:$WHYIS_IMAGE_TAG +RUN chmod 777 /data + # Apache configuration COPY /puppet/files/etc/apache2/sites-available/000-default.conf /apache.conf RUN sed '1d' /apache.conf > /etc/apache2/sites-available/000-default.conf diff --git a/docs/_data/navigation.yml b/docs/_data/navigation.yml index 80b33196..cac52cb2 100644 --- a/docs/_data/navigation.yml +++ b/docs/_data/navigation.yml @@ -3,18 +3,31 @@ - title: Home url: / -- title: Basic Installation - url: /install +- title: Installation + sublinks: + - title: Quick Start With Docker + url: /tutorial + + - title: Basic Installation + url: /install -- title: Docker Installation - url: /docker + - title: Docker Installation + url: /docker + +- title: Customizing Whyis + sublinks: + - title: Configuring Whyis + url: /config -- title: Whyis Views - url: /views + - title: Command Line Reference + url: /commands -- title: Inference Agents - url: /inference + - title: Creating Whyis Views + url: /views + - title: Creating Inference Agents + url: /inference + - title: Use Cases url: /usecases diff --git a/docs/commands.md b/docs/commands.md new file mode 100644 index 00000000..d9faf3af --- /dev/null +++ b/docs/commands.md @@ -0,0 +1,207 @@ + +# Using the Whyis command-line interface + +To peform the following administrative tasks, if you're not running Whyis directly, you will need to connect to the device you are running it on. + +If you're using a virtual machine with Vagrant: +``` +vagrant ssh +``` + +If you're using a Docker container: +``` +docker exec -it bash +``` + +Once you are in the server, you need to change to the **whyis** user, go to the whyis app directory, and activate the python virtual environment: + +``` +sudo su - whyis +cd /apps/whyis +source venv/bin/activate +``` + +To see a brief description of the Whyis subcommands: + +``` +python manage.py -? +``` + +To see more in-depth information about a subcommand and its options: + +``` +python manage.py -? +``` + +### Configure Whyis +**If you are using an existing Whyis knowledge graph, this step is not needed. Instead, go to the install instructions for the graph you wish to install.** + +Whyis is built on the Flask web framework, and most of the Flask authentication options are available to configure in Whyis. +A configuration script will walk you through the configuration process and make a project directory for you. +Change the default values as needed. The SECRET_KEY and SECURITY_PASSWORD_SALT are randomly generated at runtime, so you shouldn't need to change those. + +Run the following command: +``` +python manage.py configure +``` + +This will run the configuration script, which will allow you to enter the desired configuration options from stdin, which will look like this: +``` +project_name [My Knowledge Graph]: +project_short_description [An example knowledge graph configuration.]: +project_slug [my_knowledge_graph]: +location [/apps/my_knowledge_graph]: +author [J. Doe]: +email [j.doe@example.com]: +linked_data_prefix [http://localhost]: +version [0.1]: +packages []: +SECRET_KEY [J00F5f80rGSbvpUo9oBFAtksmrd7ef8u]: +SECURITY_PASSWORD_SALT [JDyCyPu0KEu/fdJr4CbG65VhCtGugwCu]: +$ +``` + +This will create a project skeleton for you at `location` (here, `/apps/my_knowledge_graph`). The files are: + +* **config.py** - Main configuration file for Whyis. +* **vocab.ttl** - Vocabulary file for configuring custom Whyis views. +* **templates/** - Directory for storing Whyis view templates. +* **my_knowledge_graph/** - Project source directory. Put any python code in here. + * **agent.py** - An empty inference agent module. +* **static/** - Files that are served up at `{linked_data_prefix}/cdn/` as static files. + * **css/** - Project-specific CSS files. + * **my_knowledge_graph.css** - Default empty project-specific CSS file. + * **html/** - Project-specific static HTML files, like for Angular.js templates. + * **js/** - Project-specific javascript files. + * **my_knowledge_graph.js** - Default empty project-specific javascript file. +* **setup.py** - File for installation using pip. + +Change directories into the project dir and install it into your virtualenv as follows. Be sure your virtualenv is activated first, by running `source venv/bin/activate` if you have not already done so. + +``` +cd /apps/my_knowledge_graph +pip install -e . +``` + +Restart apache and celeryd as a privileged user (not whyis) to have the configuration take effect: + +``` +sudo service apache2 restart +sudo service celeryd restart +``` + +### Add a User + +Registration is available on the website for users, but it's easy to add a user to the knowledge graph from the command line. +Perform this task as the `whyis` user from the `/apps/whyis` directory. +Use `--roles=admin` to make the user an administrator. + +``` +python manage.py createuser -e -p -f -l -u --roles=admin +``` + +### Modify a User + +You can change the roles a user has from the command line as well, but you'll need their user handle. +For instance, you can add a user to a role like this: + +``` +python manage.py updateuser -u --add_roles=admin +``` + +You can remove them from a role like this: + +``` +python manage.py updateuser -u --remove_roles=admin +``` + +Changing a password is also simple: + +``` +python manage.py updateuser -u -p +``` + +### Run in development mode + +Whyis can be run on a different port to enable debugging. You will see output from the log in the console and will be able to examine stack traces inside the browser. + +``` +python manage.py runserver -h 0.0.0.0 +``` + +### Loading Knowledge + +Knowledge can be added to Whyis using a command as well. This can be used to inject states that trigger larger-scale knowledge ingestion using [SETLr](https://github.com/tetherless-world/setlr/wiki/SETLr-Tutorial), or can simply add manually curated knowledge. +If the RDF format supports named graphs, and the graphs are nanopublications, the nanopublications will be added as-is. +If there are no explicit nanopublications, or if the RDF format is triples-only, each graph (including the default one), is treated as a separate nanopublication assertion. +The PublicationInfo will contain some minimal provenance about the load, and each assertion will be the graphs contained in the file. + +``` +python manage.py load -i -f +``` + +### Retire a nanopublication + +To remove a nanopublication from the Whyis knowledge graph, use the following command: + +``` +python manage.py retire -n +``` + +This will also retire anything that *prov:wasDerivedFrom* the retired nanopublication, which may happen recursively. + +Note that retired nanopublications are still accessible as linked data from a file archive that stores all nanopublications that have been published in the knowledge graph. + +### Run tests on Whyis + +To run the test suite of Whyis (unit tests, integration tests, API tests): +``` +python manage.py test +``` + +This will run every test that is specified in the source code in a `.py` file whose name begins with `test_`. + +To specify an alternate pattern for filenames, use the `--test` option. This should be specified as a filename without an extension. For example, to run the test suite specified in `tests/integration/test_login.py`: + +``` +python manage.py test --test test_login +``` + +Test files can also be specified using glob patterns. For example, to run the group of tests in `tests/api/view` whose filenames all end in `_json_view.py`, the command is: +``` +python manage.py test --test test_*_json_view +``` + +### Run an inference agent + +To run an inference agent on the Whyis knowledge graph: + +``` +python manage.py testagent -a +``` + +where `agent` is specified as a dotted Python path. Currently-supported agents include `agents.nlp.EntityExtractor`, `agents.nlp.EntityResolver`, `agents.nlp.IDFCalculator`, and `agents.nlp.HTML2Text`. + + + +### Remove the currently-installed Whyis application + +This command will remove the Whyis application that is currently installed, if there is one. Currently, having multiple apps installed into the same Whyis instance simultaneously is undefined behavior, so it is important to run this command before creating a new app. + +``` +python manage.py uninstallapp +``` + +### View a list of valid routes in the API + +``` +python manage.py listroutes +``` + +### Open a Python shell inside the application context + +``` +python manage.py shell +``` + +## [Next: Creating Whyis Views](views) \ No newline at end of file diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 00000000..56772993 --- /dev/null +++ b/docs/config.md @@ -0,0 +1,28 @@ +# Configuring Whyis + +Whyis is configured within a generated app using the file `config.py`. This includes a number of options that can be added or modified. +The file contains a baseline config object with customizations for production, development, and testing. + +## Enabling Anonymous Access + +By defualt, Whyis requires users to be authenticated to access the knowledge graph. +Read-only anonymous access can be enabled by adding the following entry: + +``` + DEFAULT_ANONYMOUS_READ=True, +``` + +## Using BlazeGraph Bulk Loading + +Whyis can use the Blazegraph custom bulk loader API if configured. +To use it with the default configuration, add the following entries to the config object: + +``` + knowledge_useBlazeGraphBulkLoad = True, + knowledge_bulkLoadEndpoint = 'http://localhost:8080/blazegraph/dataloader', + knowledge_BlazeGraphProperties = '/apps/whyis/knowledge.properties', + load_dir = '/data/loaded', + knowledge_bulkLoadNamespace = 'knowledge', + +``` + diff --git a/docs/docker.md b/docs/docker.md index 8e7d3f75..ffc0293f 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -31,20 +31,46 @@ On Mac OS X you must allow Docker to mount these directories by going into Docke #### Prerequisites * Docker +* Create and share `/data` (see Common Concerns) #### Instructions Each run of the container should be considered a fresh slate for your Whyis application to be run on. In other words, *most* changes done to the container will be erased after each run. -To start the monolithic image from Dockerhub, run: +To start the monolithic image from Dockerhub in detached mode, run: - docker run -p 80:80 -it tetherlessworld/whyis + docker run -p 80:80 -p 5000:5000 -d -it tetherlessworld/whyis -This will automatically download the `latest` version of the `whyis` image from the [Docker Hub](https://hub.docker.com/r/tetherlessworld/whyis/). To just pull the image or update the image to the latest version, run: +This will automatically download the `latest` version of the `whyis` image from the [Docker Hub](https://hub.docker.com/r/tetherlessworld/whyis/). It will also print out your container ID and map ports 80 and 5000 to the host machine. + +Once the docker image is running, you will need to open a new terminal and open a shell into the docker container. + +To find the container ID, run: + +```shell +docker ps +``` + +To open a shell inside the docker container + +```shell +docker exec -it bash +``` + +This works on Linux systems and in the Windows command prompt. Some third-party shells on Windows (such as git bash), may require instead using + +```shell +winpty docker exec -it bash +``` + + +To just pull the image or update the image to the latest version, run: ```shell -$ docker pull tetherlessworld/whyis +docker pull tetherlessworld/whyis ``` +To verify that the server is working correctly, open a browser to `localhost:80`. This should display a Whyis login screen. + ### Development #### Prerequisites @@ -95,7 +121,7 @@ Similar to the monolithic image, the _whyis-server_ image is built from multiple ./build-deps.sh ./push-deps.sh -The `db/docker-compose.yml` can be used to start the databases without the server, so that the server can e.g., be run locally: +The `db/docker-compose.yml` can be used to start the databases without the server, so that (for example) the server can be run locally: cd docker-compose/split docker-compose -f db/docker-compose.yml diff --git a/docs/install.md b/docs/install.md index e3d40da2..68fcf7f7 100644 --- a/docs/install.md +++ b/docs/install.md @@ -44,122 +44,4 @@ Under Storage, select Controller IDE and specify your ISO file. Press Ok to save Now, when you start the VM, it should install a new Ubuntu desktop. You can now just follow the instructions for installing Whyis onto a Ubuntu system. -# Administrative Tasks - -To peform the following administrative tasks, you need to connect to the VM (if you're not running directly): - -``` -vagrant ssh -``` - -Once you are in the server, you need to change to the **whyis** user, go to the whyis app directory, and activate the python virtual environment: - -``` -sudo su - whyis -cd /apps/whyis -source venv/bin/activate -``` - -### Configure Whyis -**If you are using an existing whyis knowledge graph, this step is not needed. Instead, go to the install instructions for the graph you wish to install.** - -Whyis is built on the Flask web framework, and most of the Flask authentication options are available to configure in Whyis. -A configuration script will walk you through the configuration process and make a project directory for you. -Change the default values as needed. The SECRET_KEY and SECURITY_PASSWORD_SALT are randomly generated at runtime, so you shouldn't need to change those. - -``` -$ python manage.py configure -project_name [My Knowledge Graph]: -project_short_description [An example knowledge graph configuration.]: -project_slug [my_knowledge_graph]: -location [/apps/my_knowledge_graph]: -author [J. Doe]: -email [j.doe@example.com]: -linked_data_prefix [http://localhost]: -version [0.1]: -packages []: -SECRET_KEY [J00F5f80rGSbvpUo9oBFAtksmrd7ef8u]: -SECURITY_PASSWORD_SALT [JDyCyPu0KEu/fdJr4CbG65VhCtGugwCu]: -$ -``` - -This will create a project skeleton for you at `location` (here, `/apps/my_knowledge_graph`). The files are: - -* **config.py** - Main configuration file for Whyis. -* **vocab.ttl** - Vocabulary file for configuring custom Whyis views. -* **templates/** - Directory for storing Whyis view templates. -* **my_knowledge_graph/** - Project source directory. Put any python code in here. - * **agent.py** - An empty inference agent module. -* **static/** - Files that are served up at `{linked_data_prefix}/cdn/` as static files. - * **css/** - Project-specific CSS files. - * **my_knowledge_graph.css** - Default empty project-specific CSS file. - * **html/** - Project-specific static HTML files, like for Angular.js templates. - * **js/** - Project-specific javascript files. - * **my_knowledge_graph.js** - Default empty project-specific javascript file. -* **setup.py** - File for installation using pip. - -Change directories into the project dir and install it into your virtualenv. Be sure your virtualenv is activated first: - -``` -$ cd /apps/my_knowledge_graph -$ pip install -e . -``` - -Restart apache and celeryd as a privileged user (not whyis) to have the configuration take effect: - -``` -$ sudo service apache2 restart -$ sudo service celeryd restart -``` - -### Add a User - -Registration is available on the website for users, but it's easy to add a user to the knowledge graph from the command line. -Perform this task as the `whyis` user from the `/apps/whyis` directory. -Use `--roles=admin` to make the user an administrator. - -``` -$ python manage.py createuser -e -p -f -l --roles=admin -``` - -### Modify a User - -You can change the roles a user has from the command line as well, but you'll need their user handle. -For instance, you can add a user to a role like this: - -``` -$ python manage.py updateuser -u --add_roles=admin -``` - -You can remove them from a role like this: - -``` -$ python manage.py updateuser -u --remove_roles=admin -``` - -Changing a password is also simple: - -``` -$ python manage.py updateuser -u -p -``` - -### Run in development mode - -Whyis can be run on a different port to enable debugging. You will see output from the log in the console and will be able to examine stack traces inside the browser. - -``` -$ python manage.py runserver -h 0.0.0.0 -``` - -### Loading Knowledge - -Knowledge can be added to Whyis using a command as well. This can be used to inject states that trigger larger-scale knowledge ingestion using [SETLr](https://github.com/tetherless-world/setlr/wiki/SETLr-Tutorial), or can simply add manually curated knowledge. -If the RDF format supports named graphs, and the graphs are nanopublications, the nanopublications will be added as-is. -If there are no explicit nanopublications, or if the RDF format is triples-only, each graph (including the default one), is treated as a separate nanopublication assertion. -The PublicationInfo will contain some minimal provenance about the load, and each assertion will be the graphs contained in the file. - -``` -$ python manage.py load -i -f -``` - -## [Next: Creating Whyis Views](views) +## [Next: Using the Whyis command-line interface](commands) diff --git a/docs/tutorial.md b/docs/tutorial.md new file mode 100644 index 00000000..f9c12456 --- /dev/null +++ b/docs/tutorial.md @@ -0,0 +1,127 @@ +# Whyis Tutorial + +This page contains tutorials that a new user can follow to get started with using Whyis. + +## Whyis docker tutorial + +If you don't have docker installed, see the docker Whyis prerequisites: http://tetherless-world.github.io/whyis/docker . + +### Useful docker commands +A few useful docker commands are listed below. + +To run the docker instance (and set up a shared folder): +```shell +$ docker run -p 80:80 -v {absolute_url}/docker-data:/data -it tetherlessworld/whyis +``` +To find the container ID: +```shell +$ docker ps +``` +To bash into the container: +```shell +$ docker exec -it bash +``` +To kill a container: +```shell +$ docker kill +``` +To update to latest whyis version: +```shell +$ docker pull tetherlessworld/whyis +``` +Once in the docker instance, to use multiple terminals, install screen: +```shell +root$ apt install screen +``` +### Configuring Whyis +Change to the whyis user: +```shell +root$ su - whyis +``` +Change to the whyis directory: +```shell +whyis$ cd whyis +``` + +To see a list of manage.py commands: +```shell +whyis$ python manage.py +``` + +To configure to knowledge graph (creating a new knowledge graph instance): +```shell +whyis$ python manage.py configure +``` +-- Set project name + +-- Give the project a description + +-- Give the project a slug (Used for generating files) + +-- Set the location of the KG + +-- Set your name and email + +-- Set the linked data prefix (in general if you are publishing, you do not set to localhost. In this case, when working on your own computer, it is okay. But when deploying, you want to put the server address.) + +-- Version, packages, secret key, and salt can be left at default values + +Once the knowledge graph instance is created, go to that folder: +```shell +whyis$ cd ../whyis_demo/ +``` +And install the packages using pip3 +```shell +whyis$ pip3 install -e . +``` +Note that you should have one KG instance on a machine at a time. If you want to create another KG instance, create another docker instance for it) + +Install your favorite text editor (emacs, nano, or vim) as root: +```shell +root$ apt update +root$ apt install vim +``` +To open the config file (as the whyis user) +```shell +root$ su - whyis +whyis$ cd whyis_demo +whyis$ emacs config.py +``` +You can update contents of the config, such as the project description, if you wish. +### Loading knowledge +Next, we want to load an ontology into the knowledge graph + +For this tutorial, we will use a dataset ontology: +```shell +whyis$ python3 manage.py load -i http://orion.tw.rpi.edu/~jimmccusker/dataset.ttl -f turtle +``` +Set annonymous read to true: +```shell +whyis$ cd ../whyis_demo +``` +```shell +whyis$ emacs config.py +``` +Under root_path = '/apps/whyis', add: +``` +DEFAULT_ANNONYMOUS_READ = True, +``` +Restart the apache2 server as root: +```shell +root$ service apache2 restart +``` +You should now see a dataset page at localhost/about?uri=http%3A%2F%2Fschema.org%2FDataset + +You can check out dbpedia pages like localhost/dbpedia/JamesHendler or localhost/dbpedia/IBM + +From the Dataset page, you can use a template to create a new dataset reference by clicking the Add button (+) at the bottom of the page + +Next we want to load some experiment data +```shell +whyis$python3 manage.py load -i https://github.com/tetherless-world/whyis-demo/raw/master/data/ae_experiments.ttl -f turtle +``` +You should see that the data loaded into whyis + +You can click the list view to see descriptions of the content that was loaded +### Next steps +Visit http://tetherless-world.github.io/whyis/inference to learn about inference agents diff --git a/docs/usecases.md b/docs/usecases.md index 2b79ce24..0acb5ab9 100644 --- a/docs/usecases.md +++ b/docs/usecases.md @@ -20,7 +20,7 @@ SETLr itself is powerful enough to support the creation of named graphs, which l SETLr in Whyis also supports the parameterization of SETL scripts by file type. Users can upload files to nodes by HTTP POSTing a file to a node's URI. The node then represents that file. -When adding new metadata about that node, it can include \emph{rdf:type}. +When adding new metadata about that node, it can include *rdf:type*. If a file node has a type that matches one that is used in a SETL script, the file is converted using that script into RDF. This lets users (and developers) upload domain-specific file types to contribute knowledge. We have provided an example that supports the [conversion of BibTeX files into publication metadata](https://raw.githubusercontent.com/tetherless-world/whyis/master/setl_scripts/bibtex.setl.ttl) that is compatible with Digital Object Identifier (DOI) Linked Data. @@ -31,11 +31,11 @@ We have provided an example that supports the [conversion of BibTeX files into p Revisions are expressed by creating a new nanopublication and marking it as a *prov:wasRevisionOf* the original. The revision and anything that *prov:wasDerivedFrom* the prior version are "retired", or removed from the RDF database. -Retired nanopublication are still accessible as linked data from a file archive that stores all nanopublications ever published in the knowledge graph. +Retired nanopublications are still accessible as linked data from a file archive that stores all nanopublications ever published in the knowledge graph. It is therefore possible to query on current knowledge, but trace back to historical knowledge. -The use of *prov:wasDerivedFrom}* is essential to truth maintenance, in that agents (and other users of the knowledge graph) are expected to enumerate the nanopublications they use to produce additional knowledge. +The use of *prov:wasDerivedFrom* is essential to truth maintenance, in that agents (and other users of the knowledge graph) are expected to enumerate the nanopublications they use to produce additional knowledge. Whyis is fundamentally organized around the nanopublication as an atom of knowledge and provenance as the means of tracking and organizing that knowledge. -Every statement in the knowledge graph is part of a nanopublication, and meta-knowledge, like the probability of a knowledge statement, is expressed as a nanopublication that talks about other nanopublications. +Every statement in the knowledge graph is part of a nanopublication, and meta-knowledge, like the probability of a knowledge statement, is expressed as a nanopublication that talks about other nanopublications. ### On Demand Load @@ -76,7 +76,7 @@ For all nodes that are of type *sio:Protein*, when a user visits the node page, sio:Protein whyis:hasView "protein_view.html". ``` -If different views for a type are desired, developers can define those custom views. For instance, if the code below is added to the vocabulary, when the page for a given protein is given the parameter `view=structure`, the `protein\_structure\_view.html` template will be used. +If different views for a type are desired, developers can define those custom views. For instance, if the code below is added to the vocabulary, when the page for a given protein is given the parameter `view=structure`, the `protein_structure_view.html` template will be used. Other templates can be used for the same view, if the same predicate is used to link types to the desired template. In BioKG, this capability is used to provide biology-specific incoming and outgoing link results. For more details, please see the [view documentation](http://tetherless-world.github.io/whyis/views). @@ -113,22 +113,22 @@ The agent framework provides custom inference capability, and is composed of a S The agent is invoked when new nanopublications are added to the knowledge graph that match the SPARQL query defined by the agent. Developers can choose to run this query either on just the single nanopublication that has been added, or on the entire graph. Whole-graph queries will need to exclude query matches that would cause the agent to be invoked over and over. -This can take some consideration for complex cases, but excluding similar knowledge to the expected output or nodes that have already had the agent run on them will often suffice. +This can take some consideration for complex cases, but excluding similar knowledge to the expected output or nodes that have already had the agent run on them will often suffice. The function *head* is invoked on each query match. This function can produce unqualified RDF or full nanopublications. -The agent superclass will assign some basic provenance and publication information related to the given inference activity, but developers can expand on this by overriding the *explain()* function. +The agent superclass will assign some basic provenance and publication information related to the given inference activity, but developers can expand on this by overriding the *explain()* function. ### Custom Rules - + > As a knowledge graph developer, I can add custom deductive rules so that I can expand the knowledge graph using domain-specific rule expansion knowledge. -Whyis provides support for custom deductive rules using the autonomic.Deductor class. -Developers can write rules by providing a \textit{construct} clause as the head and a \textit{where} clause as the body. +Whyis provides support for custom deductive rules using the autonomic.Deductor class. +Developers can write rules by providing a _construct_ clause as the head and a _where_ clause as the body. ### Standard Inferencing - + > As a knowledge graph developer, I can add deductive inferencing support for standard entailment regimes, like RDFS, OWL 2 profiles (DL, RL, QL, and EL) so that I can query over the deductive closure of the graph as well as the explicit inferences. - + Whyis provides customized Deductor instances that are collected up into OWL 2 partial profiles (with an eye towards near-term completion of them) for OWL 2 EL, RL, and QL. ### NLP Support @@ -140,6 +140,6 @@ Default inference agent types include some NLP support, including entity detecti ### Truth Maintenance > As a knowledge graph system, I apply generalized truth maintenance to all inferred knowledge, regardless of source, so that revisions to the graph maintain consistency with itself. - + Truth maintenance is performed through derivation tracing. When a nanopublication is retired from the knowledge graph, either through revision or retirement, all nanopublications that are transitively derived from (*prov:wasDerivedFrom*) the original nanopublication are also retired. -When a revision occurs, the inclusion of a new nanopublication triggers inference agents to be run on its content, creatiing a re-calculation cascade in the case of revisions. +When a revision occurs, the inclusion of a new nanopublication triggers inference agents to be run on its content, creating a re-calculation cascade in the case of revisions. diff --git a/main.py b/main.py index 4451fc0a..1fed69c7 100644 --- a/main.py +++ b/main.py @@ -31,7 +31,9 @@ from whyis import database from whyis import filters from whyis import search +from whyis.blueprint.entity import entity_blueprint from whyis.blueprint.nanopub import nanopub_blueprint +from whyis.blueprint.tableview import tableview_blueprint from whyis.blueprint.sparql import sparql_blueprint from whyis.data_extensions import DATA_EXTENSIONS from whyis.data_formats import DATA_FORMATS @@ -186,7 +188,7 @@ def run_update(nanopub_uri): @self.celery.task(retry_backoff=True, retry_jitter=True,autoretry_for=(Exception,),max_retries=4, bind=True) def run_importer(self, entity_name): entity_name = URIRef(entity_name) - counter = app.redis.incr(("import",entity_name)) + counter = app.redis.incr("import__"+entity_name) if counter > 1: return print('importing', entity_name) @@ -201,7 +203,7 @@ def run_importer(self, entity_name): print("Remote modified:", updated, type(updated), "Local modified:", modified, type(modified)) if modified is None or (updated - modified).total_seconds() > importer.min_modified: importer.load(entity_name, app.db, app.nanopub_manager) - app.redis.set(("import",entity_name),0) + app.redis.set("import__"+entity_name,0) self.run_importer = run_importer self.template_imports = {} @@ -219,6 +221,24 @@ def run_importer(self, entity_name): self, update_listener=self.nanopub_update_listener) + _file_depot = None + @property + def file_depot(self): + if self._file_depot is None: + if DepotManager.get('files') is None: + DepotManager.configure('files', self.config['file_archive']) + self._file_depot = DepotManager.get('files') + return self._file_depot + + _nanopub_depot = None + @property + def nanopub_depot(self): + if self._nanopub_depot is None: + if DepotManager.get('nanopublications') is None: + DepotManager.configure('nanopublications', self.config['nanopub_archive']) + self._nanopub_depot = DepotManager.get('nanopublications') + return self._nanopub_depot + def configure_database(self): """ Database configuration should be set here @@ -242,13 +262,6 @@ def configure_database(self): self.security = Security(self, self.datastore, register_form=ExtendedRegisterForm) - self.file_depot = DepotManager.get('files') - if self.file_depot is None: - DepotManager.configure('files', self.config['file_archive']) - self.file_depot = DepotManager.get('files') - if DepotManager.get('nanopublications') is None: - DepotManager.configure('nanopublications', self.config['nanopub_archive']) - def __weighted_route(self, *args, **kwargs): """ Override the match_compare_key function of the Rule created by invoking Flask.route. @@ -304,7 +317,7 @@ def description(self): ?o skos:prefLabel ?prefLabel. ?o dc:title ?title. ?o foaf:name ?name. - ?o ?pattr ?oatter. + ?o ?pattr ?oattr. ?oattr rdfs:label ?oattrlabel } where { graph ?g { @@ -482,7 +495,7 @@ def camel_case_split(identifier): matches = finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)', identifier) return [m.group(0) for m in matches] - label_properties = [self.NS.skos.prefLabel, self.NS.RDFS.label, self.NS.schema.name, self.NS.dc.title, self.NS.foaf.name, self.NS.schema.name] + label_properties = [self.NS.skos.prefLabel, self.NS.RDFS.label, self.NS.schema.name, self.NS.dc.title, self.NS.foaf.name, self.NS.schema.name, self.NS.skos.notation] @lru_cache(maxsize=1000) def get_remote_label(uri): @@ -669,73 +682,6 @@ def get_summary(resource): def cdn(filename): return send_from_directory(self.config['WHYIS_CDN_DIR'], filename) - @self.route('/about.', methods=['GET','POST','DELETE']) - @self.__weighted_route('/', compare_key=bottom_compare_key, methods=['GET','POST','DELETE']) - @self.__weighted_route('/.', compare_key=bottom_compare_key, methods=['GET','POST','DELETE']) - @self.route('/', methods=['GET','POST','DELETE']) - @self.route('/home', methods=['GET','POST','DELETE']) - @self.route('/about', methods=['GET','POST','DELETE']) - @conditional_login_required - def view(name=None, format=None, view=None): - self.db.store.nsBindings = {} - content_type = None - if format is not None: - if format in DATA_EXTENSIONS: - content_type = DATA_EXTENSIONS[format] - else: - name = '.'.join([name, format]) - #argstring = '&'.join(["%s=%s"%(k,v) for k,v in request.args.iteritems(multi=True) if k != 'value']) - if name is not None: - #if len(argstring) > 0: - # name = name + "?" + argstring - entity = self.NS.local[name] - elif 'uri' in request.args: - entity = URIRef(request.args['uri']) - else: - entity = self.NS.local.Home - - #print(request.method, 'view()', entity, view) - if request.method == 'POST': - print ("uploading file",entity) - if len(request.files) == 0: - flash('No file uploaded') - return redirect(request.url) - upload_type = rdflib.URIRef(request.form['upload_type']) - self.add_files(entity, [y for x, y in request.files.items(multi=True)], - upload_type=upload_type) - url = "/about?%s" % urlencode(dict(uri=str(entity), view="view")) - print ("redirecting to",url) - return redirect(url) - elif request.method == 'DELETE': - self.delete_file(entity) - return '', 204 - elif request.method == 'GET': - resource = self.get_resource(entity) - - # 'view' is the default view - fileid = resource.value(self.NS.whyis.hasFileID) - if fileid is not None and 'view' not in request.args: - print (resource.identifier, fileid) - f = self.file_depot.get(fileid) - fsa = FileServeApp(f, self.config["file_archive"].get("cache_max_age",3600*24*7)) - return fsa - - if content_type is None: - content_type = request.headers['Accept'] if 'Accept' in request.headers else 'text/turtle' - #print entity - - fmt = sadi.mimeparse.best_match([mt for mt in list(DATA_FORMATS.keys()) if mt is not None],content_type) - if 'view' in request.args or fmt in HTML_MIME_TYPES: - return render_view(resource) - elif fmt in DATA_FORMATS: - output_graph = ConjunctiveGraph() - result, status, headers = render_view(resource, view='describe') - output_graph.parse(data=result, format="json-ld") - return output_graph.serialize(format=DATA_FORMATS[fmt]), 200, {'Content-Type':content_type} - #elif 'view' in request.args or sadi.mimeparse.best_match(htmls, content_type) in htmls: - else: - return render_view(resource) - def render_view(resource, view=None, args=None): template_args = dict() template_args.update(self.template_imports) @@ -746,6 +692,7 @@ def render_view(resource, view=None, args=None): isinstance=isinstance, args=request.args if args is None else args, url_for=url_for, + app=self, get_entity=get_entity, get_summary=get_summary, search = search, @@ -799,7 +746,24 @@ def render_view(resource, view=None, args=None): # Register blueprints self.register_blueprint(nanopub_blueprint) self.register_blueprint(sparql_blueprint) - + self.register_blueprint(entity_blueprint) + self.register_blueprint(tableview_blueprint) + + def get_entity_uri(self, name, format): + content_type = None + if format is not None: + if format in DATA_EXTENSIONS: + content_type = DATA_EXTENSIONS[format] + else: + name = '.'.join([name, format]) + if name is not None: + entity = self.NS.local[name] + elif 'uri' in request.args: + entity = URIRef(request.args['uri']) + else: + entity = self.NS.local.Home + return entity, content_type + def get_send_file_max_age(self, filename): if self.debug: return 0 diff --git a/requirements/common.txt b/requirements/common.txt index 1e80910e..adec7578 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -5,7 +5,7 @@ Babel==2.7.0 bcrypt==3.1.6 beautifulsoup4==4.7.1 bibtexparser==1.1.0 -billiard==3.5.0.5 +billiard==3.6.1.0 binaryornot==0.4.4 blinker==1.4 bsddb3==6.2.6 @@ -39,11 +39,11 @@ javabridge==1.0.18 Jinja2==2.10.1 jinja2-time==0.2.0 keepalive==0.5 -kombu==4.4.0 +kombu==4.6.4 lxml==4.3.4 Markdown==3.1.1 MarkupSafe==1.1.1 -nltk==3.4.3 +nltk==3.4.5 nose==1.3.7 numpy==1.16.4 pandas==0.24.2 @@ -66,7 +66,7 @@ pytz==2019.1 git+git://github.com/gordom6/rdflib.git#egg=rdflib git+git://github.com/RDFLib/rdflib-jsonld.git#egg=rdflib-jsonld git+git://github.com/tetherless-world/markdown-rdfa.git#egg=markdown-rdfa -redis==3.2.0 +redis==3.3.8 requests[security]==2.22.0 requests-testadapter==0.3.0 sadi==0.5.7 @@ -84,7 +84,7 @@ vine==1.3.0 virtualenv==16.6.0 webencodings==0.5.1 WebOb==1.8.5 -Werkzeug==0.14.1 +Werkzeug==0.15.3 whichcraft==0.5.2 WTForms==2.2.1 xlrd==1.2.0 diff --git a/static/.gitignore b/static/.gitignore index 886c230d..99f8c369 100644 --- a/static/.gitignore +++ b/static/.gitignore @@ -1,3 +1,4 @@ js/whyis_vue_bundle.js* node_modules/ package-lock.json +css/whyis_vue_bundle.css diff --git a/static/js/rdfviewer.js b/static/js/rdfviewer.js index 9e3d8c52..1fc0b802 100644 --- a/static/js/rdfviewer.js +++ b/static/js/rdfviewer.js @@ -124,7 +124,7 @@ extraLabels[RDFS+"subClassOf"] = "sub-class of"; extraLabels[OWL+"equivalentClass"] = "equivalent class"; extraLabels[OWL+"inverseOf"] = "inverse of"; extraLabels[OWL+"ObjectProperty"] = "Object Property"; -extraLabels[OWL+"DataProperty"] = "Data Property"; +extraLabels[OWL+"DatatypeProperty"] = "Data Property"; extraLabels[XSD+"length"] = "length"; extraLabels[XSD+"minLength"] = "minLength"; extraLabels[XSD+"maxLength"] = "maxLength"; diff --git a/static/js/whyis.js b/static/js/whyis.js index 90632391..27018fb6 100644 --- a/static/js/whyis.js +++ b/static/js/whyis.js @@ -1050,11 +1050,11 @@ function whyis() { } } nanopubs.forEach(function(nanopub) { - nanopub.np_local_url = nanopub.np.replace(LOD_PREFIX,''); + nanopub.np_local_url = ROOT_URL+"about?uri="+nanopub.np; if (nanopub.modified) nanopub.modified = new Date(nanopub.modified); if (nanopub.created) nanopub.created = new Date(nanopub.created); if (nanopub.updated) nanopub.updated = new Date(nanopub.updated); - workMap[nanopub.work] = nanopub; + workMap[nanopub.np] = nanopub; npMap[nanopub.np] = nanopub; nanopub.replies = []; nanopub.derivations = []; @@ -2615,7 +2615,7 @@ FILTER ( !strstarts(str(?id), "bnode:") )\n\ if (!d.type) d.type = "basic"; }, - 'http://www.w3.org/2002/07/owl#DataProperty' : function(d) { + 'http://www.w3.org/2002/07/owl#DatatypeProperty' : function(d) { d.name = d.label; if (!d.facetId) d.facetId = b64_sha256(d.property); diff --git a/static/js/whyis_vue/components/table-view/.gitignore b/static/js/whyis_vue/components/table-view/.gitignore new file mode 100644 index 00000000..c8544e8b --- /dev/null +++ b/static/js/whyis_vue/components/table-view/.gitignore @@ -0,0 +1,62 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ +venv/ + +# TypeScript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + +# next.js build output +.next diff --git a/static/js/whyis_vue/components/table-view/api/api.js b/static/js/whyis_vue/components/table-view/api/api.js new file mode 100644 index 00000000..7921069e --- /dev/null +++ b/static/js/whyis_vue/components/table-view/api/api.js @@ -0,0 +1,9 @@ +/* eslint-disable */ +export default class Api { + postFile(file) { + throw new EvalError("Abstract"); + } + getFile() { + throw new EvalError("Abstract"); + } +} diff --git a/static/js/whyis_vue/components/table-view/api/mockapi.js b/static/js/whyis_vue/components/table-view/api/mockapi.js new file mode 100644 index 00000000..265e6c60 --- /dev/null +++ b/static/js/whyis_vue/components/table-view/api/mockapi.js @@ -0,0 +1,36 @@ +import Api from './api.js'; + +export default class MockApi extends Api { + postFile(file) { + let x = new Promise((resolve, reject) => { + file ? + setTimeout(function() { + resolve(file); + }, 300) + : reject(); + }); + x.then((value) => { + //Set data in backend + console.log("File contents:", value); + }); + return x; + } + getFile() { + return new Promise(function(resolve, reject) { + setTimeout(() => { + resolve([ + {id:1, name:"Billy Bob", age:"12", col:"red", dob:""}, + {id:2, name:"Mary May", age:"1",col:"blue", dob:"14/05/1982"}, + {id:3, name:"Christine Lobowski", age:"42", height:0, col:"green", dob:"22/05/1982", cheese:"true"}, + {id:4, name:"Brendon Philips", age:"125", gender:"male", height:1, col:"orange", dob:"01/08/1980"}, + {id:5, name:"Margret Marmajuke", age:"16", gender:"female", height:5, col:"yellow", dob:"31/01/1999"}, + {id:6, name:"Billy Bob", age:"12", gender:"male", height:1, col:"red", dob:"", cheese:1}, + {id:7, name:"Mary May", age:"1", gender:"female", height:2, col:"blue", dob:"14/05/1982", cheese:true}, + {id:8, name:"Christine Lobowski", age:"42", height:0, col:"green", dob:"22/05/1982", cheese:"true"}, + {id:9, name:"Brendon Philips", age:"125", gender:"male", height:1, col:"orange", dob:"01/08/1980"}, + {id:10, name:"Margret Marmajuke", age:"16", gender:"female", height:5, col:"yellow", dob:"31/01/1999"}, + ]); + }, 300); + }); + } +} diff --git a/static/js/whyis_vue/components/table-view/components/columnAddition.vue b/static/js/whyis_vue/components/table-view/components/columnAddition.vue new file mode 100644 index 00000000..e80988ba --- /dev/null +++ b/static/js/whyis_vue/components/table-view/components/columnAddition.vue @@ -0,0 +1,25 @@ + + + + + diff --git a/static/js/whyis_vue/components/table-view/components/tableControl.vue b/static/js/whyis_vue/components/table-view/components/tableControl.vue new file mode 100644 index 00000000..d11358de --- /dev/null +++ b/static/js/whyis_vue/components/table-view/components/tableControl.vue @@ -0,0 +1,22 @@ + + + diff --git a/static/js/whyis_vue/components/table-view/components/tableView.vue b/static/js/whyis_vue/components/table-view/components/tableView.vue new file mode 100644 index 00000000..41eca639 --- /dev/null +++ b/static/js/whyis_vue/components/table-view/components/tableView.vue @@ -0,0 +1,83 @@ + + + + + diff --git a/static/js/whyis_vue/components/table-view/index.js b/static/js/whyis_vue/components/table-view/index.js index fe50611f..736ad458 100644 --- a/static/js/whyis_vue/components/table-view/index.js +++ b/static/js/whyis_vue/components/table-view/index.js @@ -1,2 +1,3 @@ // Matthew: import your module here // import "./your-module"; +import "./components/tableView.vue"; \ No newline at end of file diff --git a/static/js/whyis_vue/main.js b/static/js/whyis_vue/main.js index 33e7171b..7d2e4788 100644 --- a/static/js/whyis_vue/main.js +++ b/static/js/whyis_vue/main.js @@ -4,9 +4,9 @@ import "./components"; Vue.use(VueMaterial.default); -new Vue({ - el: '#page', - data: { +let data; +if(typeof(ATTRIBUTES) !== "undefined"){ + data = { attributes: ATTRIBUTES, uri: NODE_URI, description: DESCRIPTION, @@ -16,5 +16,11 @@ new Vue({ base_rate: BASE_RATE, lod_prefix: LOD_PREFIX, } +} else { + data = {}; +} +new Vue({ + el: '#page', + data, }); diff --git a/static/package.json b/static/package.json index 3bc1ea86..9a2c65b1 100644 --- a/static/package.json +++ b/static/package.json @@ -5,6 +5,7 @@ "author": "", "license": "Apache-2.0", "dependencies": { + "tabulator-tables": "^4.4.3", "vue": "^2.6.10", "vue-material": "^1.0.0-beta-11", "webpack": "^4.35.2", @@ -19,7 +20,8 @@ "raw-loader": "1.0.0", "style-loader": "0.23.1", "vue-loader": "^15.7.1", - "vue-template-compiler": "^2.6.10" + "vue-template-compiler": "^2.6.10", + "webpack-dev-server": "^3.8.2" }, "scripts": { "lint": "eslint", diff --git a/static/tabletest.html b/static/tabletest.html new file mode 100644 index 00000000..35012813 --- /dev/null +++ b/static/tabletest.html @@ -0,0 +1,14 @@ + + + + + + + + +
+ +
+ + + \ No newline at end of file diff --git a/templates/_macros.html b/templates/_macros.html index 90290987..5ec5613b 100644 --- a/templates/_macros.html +++ b/templates/_macros.html @@ -1,5 +1,5 @@ {% macro render_resource_link(resource) %} - {{get_label(resource)}} + {{get_label(resource)}} {% endmacro %} {% macro render_rdfa_resource_link(resource, predicate) %} diff --git a/templates/base.html b/templates/base.html index 912a5a82..1847162d 100644 --- a/templates/base.html +++ b/templates/base.html @@ -70,13 +70,13 @@
- Home + Home