Merge pull request #88 from gucorpling/dev

New auth, add docker, CSV batch export

.gitignore

@@ -0,0 +1 @@
+**.pyc

Dockerfile

@@ -0,0 +1,53 @@
+FROM ubuntu:18.04
+EXPOSE 80
+
+# install deps
+RUN apt update
+RUN apt -y install apache2 python python-pip redis-server gzip git curl \
+libssl-dev pkg-config build-essential npm libxml2-utils
+
+# add gitdox and give permissions
+RUN rm -rf /var/www/html
+ADD . /var/www/html
+RUN chown -R www-data:www-data /var/www/html
+RUN chmod +x /var/www/html/*.py
+RUN chmod +x /var/www/html/modules/*.py
+
+# keep these in sync with requirements.txt
+RUN pip install lxml requests github3.py==0.9.3 passlib
+
+# install ethercalc and run as a service
+RUN npm install -g ethercalc
+RUN adduser --system --no-create-home --group ethercalc
+
+# enable cgi
+RUN a2enmod cgi
+RUN echo "                       \n \
+<Directory /var/www/html>        \n \
+   Options +ExecCGI              \n \
+   AddHandler cgi-script .py     \n \
+   DirectoryIndex index.py       \n \
+</Directory>                     \n \
+" >> /etc/apache2/apache2.conf
+
+RUN a2enmod proxy_html proxy_http proxy_wstunnel
+# set up a proxy for ethercalc
+RUN echo " \n\
+<VirtualHost *:80> \n\
+	ServerName localhost \n\
+\n\
+	# Proxy pass to the node.js server (port 8000) \n\
+	ProxyPass /ethercalc/ http://127.0.0.1:8000/ \n\
+	ProxyPassReverse /ethercalc/ http://127.0.0.1:8000/ \n\
+  ProxyPreserveHost On \n\
+</VirtualHost> \n\
+" >> /etc/apache2/apache2.conf
+
+# against best practices both to (1) not use a different container for each
+# service and (2) not to use supervisord to manage the execution of these
+# processes. But (1) is too heavy a solution, and (2) seems unnecessary unless
+# one of our services leaks memory/is unstable
+RUN echo "/usr/bin/redis-server &" >> /etc/startup.sh
+RUN echo "/usr/local/bin/ethercalc &" >> /etc/startup.sh
+RUN echo "/usr/sbin/apache2ctl -D FOREGROUND" >> /etc/startup.sh
+ENTRYPOINT ["/bin/sh", "/etc/startup.sh"]

README.md

@@ -6,6 +6,86 @@ The editor interface is based on [CodeMirror](https://codemirror.net). GitHub is
 GitDox is used by [Coptic SCRIPTORIUM](http://copticscriptorium.org/) as an xml editor/transcription tool for Coptic texts. 
 
 # Installation
+You have three choices:
+
+1. Pull the latest GitDox Docker image
+2. Build and run a GitDox Docker image
+3. Manually install GitDox on your machine 
+
+Unless you have a good reason to do (2) or (3), we recommend you do (1).
+
+# Pull the latest Docker image
+**Note:** currently, only the `gucorpling/gitdox-dev` image is available. We
+hope to provide a stable release soon.
+
+First, [install Docker](https://docs.docker.com/install/). You may be able to
+install it using your platform's package manager.
+
+```bash
+docker run -dit --restart unless-stopped --name gitdox-dev -p 5000:80 gucorpling/gitdox-dev
+```
+
+GitDox should now be running the docker container you've set up, and you may
+visit `http://localhost:5000` on your machine to verify that it works. GitDox should
+now always be running on your machine, even if you reboot it. If for some reason
+you need to stop it manually, you may do so:
+
+```bash
+docker stop gitdox
+# since you stopped it manually, it will no longer start automatically, even on
+# reboot. to start again:
+docker start gitdox
+```
+
+If for whatever reason you need to manually edit GitDox files, you may start a
+bash session inside of the Docker container:
+
+```bash
+docker exec -it gitdox-instance bash
+# now you are inside--install vim so you can edit files
+apt install vim 
+cd /var/www/html
+vim user/admin.ini # or whatever you need to edit
+```
+
+# Build and run a Docker image
+First, [install Docker](https://docs.docker.com/install/). You may be able to
+install it using your platform's package manager.
+
+Run the following code:
+
+```bash
+git clone https://github.com/gucorpling/gitdox ~/gitdox
+# compile the docker image
+docker build -t gitdox .
+# launch the image on your machine, mapping the image's port 80 to your machine's 80
+docker run -dit --restart unless-stopped --name gitdox -p 5000:80 gitdox
+```
+
+GitDox should now be running the docker container you've set up, and you may
+visit `http://localhost:5000` on your machine to verify that it works. GitDox should
+now always be running on your machine, even if you reboot it. If for some reason
+you need to stop it manually, you may do so:
+
+```bash
+docker stop gitdox
+# since you stopped it manually, it will no longer start automatically, even on
+# reboot. to start again:
+docker start gitdox
+```
+
+If for whatever reason you need to manually edit GitDox files, you may start a
+bash session inside of the Docker container:
+
+```bash
+docker exec -it gitdox-instance bash
+# now you are inside--install vim so you can edit files
+apt install vim 
+cd /var/www/html
+vim user/admin.ini # or whatever you need to edit
+```
+
+# Manual Installation
 The following instructions assume you are installing on a recent (16.04-18.04) version of Ubuntu.
 
 ## Install Redis
@@ -90,8 +170,12 @@ sudo pip install -r /var/www/html/requirements.txt
    of those features.
 
 4. Modify the value of `ether_url` in `users/config.ini` so that it reflects where
-   GitDox can find your Ethercalc service over HTTP. By default, GitDox assumes it 
-   is on your local machine on port 8000.
+   GitDox can find your Ethercalc service over HTTP. It defaults to assuming
+   that you will serve Ethercalc via reverse proxy into a subdirectory of your
+   website, /ethercalc/. If you want to serve it over its original port, you
+   should change it to something like `yourdomain.com:8000`, and if you want to
+   serve it over a subdomain, change it to something like
+   `your.subdomain.yourdomain.com`.
 
 5. Navigate to `http://localhost`. The default login is `admin`, `pass1`.
 

admin.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
-# -*- coding: UTF-8 -*-
+# -*- coding: utf-8 -*-
 
-import cgi, cgitb 
+import cgi, cgitb
 import os, platform
 from os.path import isfile, join
 from os import listdir
@@ -14,6 +14,9 @@
 from paths import get_menu
 from editor import harvest_meta
 from modules.ether import make_spreadsheet, get_ether_stylesheet_select, get_corpus_select
+from passlib.apps import custom_app_context as pwd_context
+import github3
+import time
 
 # Support IIS site prefix on Windows
 if platform.system() == "Windows":
@@ -51,16 +54,25 @@ def write_user_file(username,password,admin,email,realname,git_username,git_pass
 	userdir=prefix+"users"+os.sep
 	f=open(userdir+username+'.ini',"w")
 	f.write('username='+username+'\n')
-	f.write('password='+pass_enc(password)+'\n')
+	f.write('password='+pass_enc(pwd_context.hash(password,salt=""))+'\n')
 	f.write('realname='+realname+'\n')
 	f.write('admin='+str(admin)+'\n')
 	f.write('email='+email+'\n')
 	f.write('max-age=0'+'\n')
 	f.write('editable=Yes'+'\n')
 	f.write('numlogins = 85\nnumused = 2869\n')
-	f.write('git_username='+git_username+'\n')
-	f.write('git_password='+pass_enc(git_password)+'\n')
-	f.write('git_2fa='+str(git_2fa).lower()+'\n')
+
+	# get oauth token for github. Add current date to note since they need to be unique or an error will occur
+	note = project + ", " + time.ctime()
+	try:
+		auth = github3.authorize(git_username, git_password, ['repo'], note, "")
+		f.write('git_username='+git_username+'\n')
+		f.write('git_token='+auth.token+'\n')
+		f.write('git_id='+str(auth.id)+'\n') # in case we ever need to update authorizations
+		f.write('git_2fa='+str(git_2fa).lower()+'\n')
+	except:
+		# would be ideal to show an error, but just fail silently
+		pass 
 	f.close()
 
 
@@ -86,31 +98,22 @@ def update_password(user,new_pass):
 
 
 def update_git_info(user,new_git_username,new_git_password,new_git_2fa=False):
-	f=open(prefix+'users'+os.sep+user+'.ini','r')
-	ff=f.read().split('\n')
-	f.close()
-
-	new_file=[]
-	for line in ff:
-		if line!='':
-			line_split=line.split('=')
-			if line_split[0].strip().startswith('git_password'):
-				newline='git_password = ' + pass_enc(new_git_password)
-				new_file.append(newline)
-			elif line_split[0].strip().startswith('git_username'):
-				newline='git_username = ' + new_git_username
-				new_file.append(newline)
-			elif line_split[0].strip().startswith('git_2fa'):
-				newline = 'git_2fa = ' + str(new_git_2fa).lower()
-				new_file.append(newline)
-			else:
-				new_file.append(line)
-	open(prefix + 'users'+os.sep+user+'.ini', 'w').close()
-	g = open(prefix+ 'users'+os.sep+user+'.ini','a')
-	for l in new_file:
-		g.write(l+'\n')
-	g.close()
-
+	o = ConfigObj(prefix + 'users' + os.sep + user + '.ini')
+	o['git_username'] = new_git_username
+	o['git_2fa'] = str(new_git_2fa).lower()
+
+	try: 
+		note = project + ", " + time.ctime()
+		auth = github3.authorize(new_git_username, new_git_password, ['repo'], note, "")
+		o['git_token'] = auth.token
+		o['git_id'] = auth.id
+		if 'git_password' in o:
+			del o['git_password']
+		o.write()
+	except:
+		# fail silently--would want to display an error ideally, but 
+		# users will know to try again if the credentials are wrong
+		pass
 
 def load_admin(user,admin,theform):
 	warn=""

editor.py

@@ -91,9 +91,9 @@ def load_page(user,admin,theform):
 	old_docname, old_corpus, old_repo, old_status, old_assignee, old_mode, old_schema = ["", "", "", "", "", "", ""]
 
 	if int(admin) > 0:
-		git_username, git_password, git_2fa = get_git_credentials(user, admin, code_2fa)
+		git_username, git_token, git_2fa = get_git_credentials(user, admin, code_2fa)
 	else:
-		git_username, git_password, git_2fa = (None,None,None)
+		git_username, git_token, git_2fa = (None, None, None)
 
 	if theform.getvalue('id'):
 		doc_id = theform.getvalue('id')
@@ -284,7 +284,7 @@ def load_page(user,admin,theform):
 			file_name = file_name.replace(" ","_") + "_ether.sgml"
 		saved_file = subdir + file_name
 		serialize_file(serializable_content, saved_file)
-		git_status = push_update_to_git(git_username, git_password, saved_file, git_account, git_repo, commit_message)
+		git_status = push_update_to_git(git_username, git_token, saved_file, git_account, git_repo, commit_message)
 
 		# File system cleanup
 		if subdir == "":

export.py

@@ -3,7 +3,7 @@
 
 import cgi
 from modules.gitdox_sql import *
-from modules.ether import ether_to_sgml, get_socialcalc, build_meta_tag
+from modules.ether import ether_to_sgml, get_socialcalc, build_meta_tag, ether_to_csv
 from modules.logintools import login
 import zipfile
 from StringIO import StringIO
@@ -42,13 +42,17 @@ def export_all_docs(config=None, corpus_filter=None, status=None, extension="sgm
 			filename = corpus + "_" + docname
 		else:  # Only exporting one user specified corpus, name documents without prefix
 			filename = docname
-		if mode == "xml":
+		if mode == "xml" and config!="[CSV]":
 			content = build_meta_tag(doc_id) + content.strip() + "\n</meta>\n"
 			files.append((content,filename + ".xml"))
 		elif mode == "ether":
-			ether_name = "_".join(["gd",corpus,docname])
-			sgml = ether_to_sgml(get_socialcalc(ether_url, ether_name),doc_id,config=config)
-			files.append((sgml, filename + "." + extension))
+			ether_name = "_".join(["gd", corpus, docname])
+			if config=="[CSV]":
+				csv = ether_to_csv(ether_url,ether_name)
+				files.append((csv, filename + ".csv"))
+			else:
+				sgml = ether_to_sgml(get_socialcalc(ether_url, ether_name),doc_id,config=config)
+				files.append((sgml, filename + "." + extension))
 
 	for corp in all_corpus_meta:
 		serialized_meta = ""

modules/ether.py

@@ -188,8 +188,7 @@ def get_ether_stylesheet_select():
 
 	stylesheet_list = get_file_list(stylesheet_dir,"ini",hide_extension=True)
 	select = """<select name="ether_stylesheet" id="ether_stylesheet">\n"""
-	if len(stylesheet_list) == 0:
-		select += "\t<option>--default--</option>\n"
+	select += "\t<option>[CSV]</option>\n"
 
 	for f in stylesheet_list:
 		select += '\t<option value="' + f + '">' + f + '</option>\n'
@@ -340,6 +339,13 @@ def sgml_to_ether(sgml, ignore_elements=False):
 	return output
 
 
+def ether_to_csv(ether_path, name):
+	command = "curl --netrc -X GET " + ether_path + "_/" + name + "/csv/"
+	proc = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+	(stdout, stderr) = proc.communicate()
+	return stdout.decode("utf8")
+
+
 def ether_to_sgml(ether, doc_id,config=None):
 	"""
 

modules/gitdox_git.py

@@ -1,5 +1,6 @@
 import github3, os, platform
 from modules.dataenc import pass_dec, pass_enc
+from modules.configobj import ConfigObj
 code_2fa = ""
 
 
@@ -18,9 +19,9 @@ def get_two_factor():
 		return code_2fa
 
 
-def push_update_to_git(username,password,path,account,repo,message):
+def push_update_to_git(username, token, path, account, repo, message):
 	files_to_upload = [path]
-	gh = github3.login(username=username, password=password, two_factor_callback=get_two_factor)
+	gh = github3.login(username=username, token=token, two_factor_callback=get_two_factor)
 	repository = gh.repository(account, repo)
 	for file_info in files_to_upload:
 		with open(prefix+file_info, 'rb') as fd:
@@ -36,22 +37,18 @@ def push_update_to_git(username,password,path,account,repo,message):
 			return str(push_status['commit'])
 
 
-def get_git_credentials(user,admin,code):
+def get_git_credentials(user, admin, code):
 	global code_2fa
 	code_2fa = code
 	if admin==0:
 		return
+
 	scriptpath = os.path.dirname(os.path.realpath(__file__)) + os.sep + ".." + os.sep
 	userdir = scriptpath + "users" + os.sep
-	userfile = userdir + user + '.ini'
-	f=open(userfile,'r').read().split('\n')
-	user_dict={}
-	for line in f:
-		if line!='':
-			l=line.split(' = ')
-			user_dict[l[0]]=l[1]
-	git_username=user_dict['git_username'] if "git_username" in user_dict else "_"
-	git_password=pass_dec(user_dict['git_password']) if "git_password" in user_dict else "_"
-	git_use2fa=user_dict['git_2fa'] if "git_2fa" in user_dict else "false"
-	return git_username,git_password[0],git_use2fa
+	user_dict = ConfigObj(userdir + user + '.ini')
+
+	git_username = user_dict['git_username'] if "git_username" in user_dict else "_"
+	git_token = user_dict['git_token'] if "git_token" in user_dict else "_"
+	git_use2fa = user_dict['git_2fa'] if "git_2fa" in user_dict else "false"
+	return git_username, git_token, git_use2fa