diff --git a/NEWS b/NEWS index ae34d80ac..1ead17a9e 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,8 @@ 0.20.47 UNRELEASED + * Add basic ``dulwich.aiohttp`` module that provides + server support. (Jelmer Vernooij) + * Fix Repo.reset_index. Previously, it instead took the union with the given tree. (Christian Sattler, #1072) diff --git a/dulwich/aiohttp.py b/dulwich/aiohttp.py new file mode 100644 index 000000000..9e1067864 --- /dev/null +++ b/dulwich/aiohttp.py @@ -0,0 +1,280 @@ +# aiohttp.py -- aiohttp smart client/server +# Copyright (C) 2022 Jelmer Vernooij +# +# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU +# General Public License as public by the Free Software Foundation; version 2.0 +# or (at your option) any later version. You can redistribute it and/or +# modify it under the terms of either of these two licenses. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# You should have received a copy of the licenses; if not, see +# for a copy of the GNU General Public License +# and for a copy of the Apache +# License, Version 2.0. +# + +"""aiohttp client/server support.""" + +import asyncio +from io import BytesIO +import sys + +from aiohttp import web + +from . import log_utils +from .protocol import ReceivableProtocol, HangupException +from .server import ( + DEFAULT_HANDLERS, + DictBackend, + generate_objects_info_packs, + generate_info_refs, +) +from .repo import Repo +from .web import NO_CACHE_HEADERS, cache_forever_headers + + +logger = log_utils.getLogger(__name__) + + +async def send_file(req, f, headers): + """Send a file-like object to the request output. + + Args: + req: The HTTPGitRequest object to send output to. + f: An open file-like object to send; will be closed. + headers: Headers to send + Returns: Iterator over the contents of the file, as chunks. + """ + if f is None: + raise web.HTTPNotFound(text="File not found") + response = web.StreamResponse(status=200, reason='OK', headers=headers) + await response.prepare(req) + try: + while True: + data = f.read(10240) + if not data: + break + await response.write(data) + except IOError: + raise web.HTTPInternalServerError(text="Error reading file") + finally: + f.close() + await response.write_eof() + return response + + +async def get_loose_object(request): + sha = (request.match_info['dir'] + + request.match_info['filename']).encode("ascii") + logger.info("Sending loose object %s", sha) + object_store = request.app['repo'].object_store + if not object_store.contains_loose(sha): + raise web.HTTPNotFound(text="Object not found") + try: + data = object_store[sha].as_legacy_object() + except IOError: + raise web.HTTPInternalServerError(text="Error reading object") + headers = {'Content-Type': "application/x-git-loose-object"} + headers.update(cache_forever_headers()) + return web.Response(status=200, headers=headers, body=data) + + +async def get_text_file(request): + headers = {'Content-Type': 'text/plain'} + headers.update(NO_CACHE_HEADERS) + path = request.match_info['file'] + logger.info("Sending plain text file %s", path) + repo = request.app['repo'] + return await send_file( + request, repo.get_named_file(path), headers) + + +async def refs_request(repo, request, handlers=None): + service = request.query.get("service") + if service: + handler_cls = request.get(service.encode("ascii"), None) + if handler_cls is None: + raise web.HTTPForbidden(text="Unsupported service") + headers = { + 'Content-Type': "application/x-%s-advertisement" % service} + headers.update(NO_CACHE_HEADERS) + + response = web.StreamResponse(status=200, headers=headers) + + await response.prepare(request) + + out = BytesIO() + proto = ReceivableProtocol(BytesIO().read, out.write) + handler = handler_cls( + DictBackend({".": repo}), ["."], proto, stateless_rpc=True, advertise_refs=True + ) + handler.proto.write_pkt_line(b"# service=" + service.encode("ascii") + b"\n") + handler.proto.write_pkt_line(None) + + # TODO(jelmer): Implement this with proper async code + await asyncio.to_thread(handler.handle) + + await response.write(out.getvalue()) + + await response.write_eof() + + return response + else: + # non-smart fallback + headers = {'Content-Type': 'text/plain'} + headers.update(NO_CACHE_HEADERS) + logger.info("Emulating dumb info/refs") + return web.Response(body=b''.join(generate_info_refs(repo)), headers=headers) + + +async def get_info_refs(request): + repo = request.app['repo'] + return await refs_request(repo, request, request.app['handlers']) + + +async def get_info_packs(request): + headers = {'Content-Type': 'text/plain'} + headers.update(NO_CACHE_HEADERS) + logger.info("Emulating dumb info/packs") + return web.Response( + body=b''.join(generate_objects_info_packs(request.app['repo'])), + headers=headers) + + +async def get_pack_file(request): + headers = {'Content-Type': "application/x-git-packed-objects"} + headers.update(cache_forever_headers()) + sha = request.match_info['sha'] + path = 'objects/pack/pack-%s.pack' % sha + logger.info("Sending pack file %s", path) + return await send_file( + request, + request.app['repo'].get_named_file(path), + headers=headers, + ) + + +async def get_index_file(request): + headers = { + 'Content-Type': "application/x-git-packed-objects-toc" + } + headers.update(cache_forever_headers()) + sha = request.match_info['sha'] + path = 'objects/pack/pack-%s.idx' % sha + logger.info("Sending pack file %s", path) + return await send_file( + request, + request.app['repo'].get_named_file(path), + headers=headers + ) + + +async def service_request(repo, request, handlers=None): + service = request.match_info['service'] + if handlers is None: + handlers = dict(DEFAULT_HANDLERS) + logger.info("Handling service request for %s", service) + handler_cls = handlers.get(service.encode("ascii"), None) + if handler_cls is None: + raise web.HTTPForbidden(text="Unsupported service") + headers = { + 'Content-Type': "application/x-%s-result" % service + } + headers.update(NO_CACHE_HEADERS) + + response = web.StreamResponse(status=200, headers=headers) + + await response.prepare(request) + + inf = BytesIO(await request.read()) + outf = BytesIO() + + def handle(): + proto = ReceivableProtocol(inf.read, outf.write) + handler = handler_cls(DictBackend({".": repo}), ["."], proto, stateless_rpc=True) + try: + handler.handle() + except HangupException: + response.force_close() + + # TODO(jelmer): Implement this with proper async code + await asyncio.to_thread(handle) + + await response.write(outf.getvalue()) + + await response.write_eof() + return response + + +async def handle_service_request(request): + repo = request.app['repo'] + + return await service_request(repo, request, request.app['handlers']) + + +def create_repo_app(repo, handlers=None, dumb=False): + app = web.Application() + app['repo'] = repo + if handlers is None: + handlers = dict(DEFAULT_HANDLERS) + app['handlers'] = handlers + app['dumb'] = dumb + app.router.add_get('/info/refs', get_info_refs) + app.router.add_post( + '/{service:git-upload-pack|git-receive-pack}', + handle_service_request) + if dumb: + app.router.add_get('/{file:HEAD}', get_text_file) + app.router.add_get('/{file:objects/info/alternates}', get_text_file) + app.router.add_get( + '/{file:objects/info/http-alternates}', get_text_file) + app.router.add_get('/objects/info/packs', get_info_packs) + app.router.add_get( + '/objects/{dir:[0-9a-f]{2}}/{file:[0-9a-f]{38}}', get_loose_object) + app.router.add_get( + '/objects/pack/pack-{sha:[0-9a-f]{40}}\\.pack', get_pack_file) + app.router.add_get( + '/objects/pack/pack-{sha:[0-9a-f]{40}}\\.idx', get_index_file) + return app + + +def main(argv=None): + """Entry point for starting an HTTP git server.""" + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "-l", + "--listen_address", + dest="listen_address", + default="localhost", + help="Binding IP address.", + ) + parser.add_argument( + "-p", + "--port", + dest="port", + type=int, + default=8000, + help="Port to listen on.", + ) + parser.add_argument('gitdir', type=str, default='.', nargs='?') + args = parser.parse_args(argv) + + log_utils.default_logging_config() + app = create_repo_app(Repo(args.gitdir)) + logger.info( + "Listening for HTTP connections on %s:%d", + args.listen_address, + args.port, + ) + web.run_app(app, port=args.port, host=args.listen_address) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/setup.cfg b/setup.cfg index e1c523e09..fb97604df 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,6 +35,7 @@ fastimport = fastimport https = urllib3>=1.24.1 pgp = gpg paramiko = paramiko +aiohttp = aiohttp [options.entry_points] console_scripts =